billion-context-omp 0.2.8 → 0.2.9

package/dist/index.js

@@ -395,8 +395,13 @@ function resolveBoundaries(input) {
   input.messages.forEach(
     (message, index) => indexByRawId.set(message.id, index)
   );
-  let startIndex = resolveAnchorIndex(start, input.state, indexByRawId, "start");
-  let endIndex = resolveAnchorIndex(end, input.state, indexByRawId, "end");
+  let snappedBoundaries = [];
+  const startAnchor = resolveAnchorIndex(start, input.state, indexByRawId, "start");
+  if (startAnchor.snapped) snappedBoundaries.push(startAnchor.snapped);
+  const endAnchor = resolveAnchorIndex(end, input.state, indexByRawId, "end");
+  if (endAnchor.snapped) snappedBoundaries.push(endAnchor.snapped);
+  let startIndex = startAnchor.index;
+  let endIndex = endAnchor.index;
   if (startIndex > endIndex) {
     [startIndex, endIndex] = [endIndex, startIndex];
   }
@@ -424,7 +429,8 @@ function resolveBoundaries(input) {
     messageIds,
     nestedBlockIds,
     boundaryKind,
-    protectedGaps
+    protectedGaps,
+    snappedBoundaries
   };
 }
 function resolveAnchorIndex(boundary, state, indexByRawId, endpoint) {
@@ -439,14 +445,21 @@ function resolveAnchorIndex(boundary, state, indexByRawId, endpoint) {
       );
     }
     const index = indexByRawId.get(rawId);
-    if (index === void 0) {
-      throw new BoundaryNotFoundError(
-        "consumed",
-        endpoint,
-        `${label}="${boundary.raw}" not found in visible context (likely consumed by an existing block).`
-      );
+    if (index !== void 0) {
+      return { index, snapped: null };
+    }
+    const owner2 = activeOwnerAnchor(state, [rawId], indexByRawId);
+    if (owner2 !== null) {
+      return {
+        index: owner2,
+        snapped: `${label}="${boundary.raw}" refers to a message already compressed into an active block \u2014 anchored to that block's summary instead.`
+      };
     }
-    return index;
+    throw new BoundaryNotFoundError(
+      "consumed",
+      endpoint,
+      `${label}="${boundary.raw}" not found in visible context (likely consumed by an existing block).`
+    );
   }
   const block = blockById(state, `b${boundary.numericId}`);
   if (!block) {
@@ -456,6 +469,19 @@ function resolveAnchorIndex(boundary, state, indexByRawId, endpoint) {
       `${label}="b${boundary.numericId}" does not exist in this session (typo or wrong session) \u2014 run acp_status for current refs.`
     );
   }
+  if (block.active) {
+    const anchor = earliestIndexOfIds(block.effectiveMessageIds, indexByRawId);
+    if (anchor !== null) {
+      return { index: anchor, snapped: null };
+    }
+  }
+  const owner = activeOwnerAnchor(state, block.effectiveMessageIds, indexByRawId);
+  if (owner !== null) {
+    return {
+      index: owner,
+      snapped: `${label}="b${boundary.numericId}" was consumed by a higher-tier block \u2014 anchored to the active block covering its content instead.`
+    };
+  }
   if (!block.active) {
     throw new BoundaryNotFoundError(
       "consumed",
@@ -463,15 +489,26 @@ function resolveAnchorIndex(boundary, state, indexByRawId, endpoint) {
       `${label}="b${boundary.numericId}" not found in visible context (block distilled/consumed by a higher-tier block).`
     );
   }
-  const anchor = earliestIndexOfIds(block.effectiveMessageIds, indexByRawId);
-  if (anchor === null) {
-    throw new BoundaryNotFoundError(
-      "consumed",
-      endpoint,
-      `${label}="b${boundary.numericId}" not found in visible context (block messages consumed by a higher-tier block).`
-    );
+  throw new BoundaryNotFoundError(
+    "consumed",
+    endpoint,
+    `${label}="b${boundary.numericId}" not found in visible context (block messages consumed by a higher-tier block).`
+  );
+}
+function activeOwnerAnchor(state, ownedIds, indexByRawId) {
+  if (ownedIds.length === 0) return null;
+  const owned = new Set(ownedIds);
+  let best = null;
+  for (const block of state.blocks) {
+    if (!block.active) continue;
+    const anchor = earliestIndexOfIds(block.effectiveMessageIds, indexByRawId);
+    if (anchor === null) continue;
+    const ownsContent = block.effectiveMessageIds.some((id) => owned.has(id));
+    if (ownsContent && (best === null || anchor < best)) {
+      best = anchor;
+    }
   }
-  return anchor;
+  return best;
 }
 function formatPaddedRef(index) {
   return `m${String(index).padStart(5, "0")}`;
@@ -1092,6 +1129,10 @@ function runPipeline(nodes, initial, ctx) {
 function rangeError(spec, message) {
   return `range ${spec.startRef}..${spec.endRef}: ${message}`;
 }
+function numericBlockId(id) {
+  const parsed = /^b(\d+)$/.exec(id);
+  return parsed ? Number(parsed[1]) : 0;
+}
 function createCore(ports = {}) {
   const countTokens = ports.countTokens ?? defaultCountTokens;
   function applyCompression(input) {
@@ -1137,6 +1178,12 @@ function createCore(ports = {}) {
         }
       }
     }
+    let resolvableCount = 0;
+    let unknownCount = 0;
+    for (const resolution of classifications.values()) {
+      if (resolution.status === "ok") resolvableCount++;
+      else if (resolution.status === "unknown") unknownCount++;
+    }
     const rangeIndexSets = [];
     for (const [spec, resolution] of classifications) {
       if (resolution.status !== "ok") continue;
@@ -1181,7 +1228,9 @@ function createCore(ports = {}) {
         }
       }
       if (!hasBlockBoundaryRange && totalRangeChars < input.config.compress.minCompressRange) {
-        const gateMessage = consumedRanges.length > 0 ? `Requested range(s) already compressed (e.g. ${consumedRanges[0].startRef}..${consumedRanges[0].endRef}); remaining compressible content ${totalRangeChars} chars < min ${input.config.compress.minCompressRange}. Nothing to do \u2014 run acp_status to see current compressible ranges.` : `Total compressible content too small (${totalRangeChars} chars across ${countedRanges} range(s), min ${input.config.compress.minCompressRange}). Combine more messages into your range(s) to meet the threshold.`;
+        const live = activeBlocks(state).map((b) => b.blockId).sort((x, y) => numericBlockId(x) - numericBlockId(y));
+        const liveHint = live.length > 0 ? ` Current active blocks span ${live[0]}..${live[live.length - 1]} \u2014 retry with startId/endId set to active block IDs in that span.` : "";
+        const gateMessage = resolvableCount === 0 && consumedRanges.length === 0 && unknownCount > 0 ? `None of the ${input.ranges.length} requested range(s) resolved \u2014 every ref failed with "does not exist in this session". Refs recorded before an earlier compress are stale: each successful compress renumbers the remaining refs. Run acp_status, then re-issue the compress in the same turn using only the refs it reports.` : consumedRanges.length > 0 ? `Requested range(s) already compressed (e.g. ${consumedRanges[0].startRef}..${consumedRanges[0].endRef}); remaining compressible content ${totalRangeChars} chars < min ${input.config.compress.minCompressRange}. Nothing to do.${liveHint}` : `Total compressible content too small (${totalRangeChars} chars across ${countedRanges} range(s), min ${input.config.compress.minCompressRange}). Combine more messages into your range(s) to meet the threshold.`;
         return {
           state: input.state,
           result: {
@@ -1207,6 +1256,7 @@ function createCore(ports = {}) {
         errors.push(rangeError(spec, resolution.error.message));
         continue;
       }
+      warnings.push(...resolution.resolved.snappedBoundaries);
       try {
         const outcome = applySingleRange({
           spec,
@@ -2422,24 +2472,6 @@ function renderCompressedDrilldown(blocks, state, sort, limit, countTokens) {
   }
   return lines.join("\n");
 }
-var substringAlgorithm = {
-  name: "substring",
-  description: "Exact substring counting (original baseline). Predictable, no normalization.",
-  score(docs, query) {
-    const terms = query.toLowerCase().trim().split(/\s+/).filter((t) => t.length > 0);
-    if (terms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
-    return docs.map((d) => {
-      const haystack = d.text.toLowerCase();
-      let score = 0;
-      for (const term of terms) score += countOccurrences2(haystack, term);
-      return { ref: d.ref, score };
-    });
-  }
-};
-function countOccurrences2(haystack, needle) {
-  if (!needle) return 0;
-  return haystack.split(needle).length - 1;
-}
 function stem(word) {
   let w = word;
   if (w.length <= 3) return w;
@@ -2458,8 +2490,17 @@ function stem(word) {
   return w;
 }
 var CJK = /[\u3400-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]/;
-var CJK_RUN = new RegExp(`${CJK.source}+`, "g");
 var LATIN_WORD = /[a-z][a-z0-9_]*[a-z0-9]|[a-z0-9]/g;
+var cjkSegmenter = new Intl.Segmenter("zh", { granularity: "word" });
+function cjkRunTokens(segs) {
+  const words = segs.filter((w) => w.length >= 2);
+  if (words.length > 0) return words;
+  const run = segs.join("");
+  const out = [];
+  for (let i = 0; i < run.length - 1; i++) out.push(run.slice(i, i + 2));
+  for (const ch of run) out.push(ch);
+  return out;
+}
 function tokenize(text, opts = {}) {
   const lower = text.toLowerCase();
   const tokens = [];
@@ -2470,15 +2511,23 @@ function tokenize(text, opts = {}) {
       tokens.push(w);
     }
   }
-  const cjkRuns = lower.match(CJK_RUN) ?? [];
-  for (const run of cjkRuns) {
-    if (run.length === 1) {
-      tokens.push(run);
-    } else {
-      for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
-      for (const ch of run) tokens.push(ch);
+  if (!CJK.test(lower)) return tokens;
+  const runSegs = [];
+  let cur = null;
+  for (const s of cjkSegmenter.segment(lower)) {
+    const t = s.segment;
+    if (t.length === 0) continue;
+    if (CJK.test(t)) {
+      (cur ??= []).push(t);
+    } else if (cur) {
+      runSegs.push(cur);
+      cur = null;
     }
   }
+  if (cur) runSegs.push(cur);
+  for (const segs of runSegs) {
+    tokens.push(...cjkRunTokens(segs));
+  }
   return tokens;
 }
 function charBigrams(text) {
@@ -2494,6 +2543,50 @@ function tfMap(text, stem22) {
   for (const t of tokenize(text, { stem: stem22 })) m.set(t, (m.get(t) ?? 0) + 1);
   return m;
 }
+var DEFAULT_CAP_CHARS = 8 * 1024 * 1024;
+var capChars = DEFAULT_CAP_CHARS;
+var cache = /* @__PURE__ */ new Map();
+var cachedChars = 0;
+function build(text) {
+  const tf = tfMap(text, true);
+  let len = 0;
+  for (const v of tf.values()) len += v;
+  const lower = text.toLowerCase();
+  return { tf, len, lower, grams: new Set(charBigrams(lower)) };
+}
+function docFeatures(text) {
+  const hit = cache.get(text);
+  if (hit) return hit;
+  const f = build(text);
+  if (text.length > 0 && text.length <= capChars) {
+    while (cachedChars + text.length > capChars && cache.size > 0) {
+      const k = cache.keys().next().value;
+      cachedChars -= k.length;
+      cache.delete(k);
+    }
+    cache.set(text, f);
+    cachedChars += text.length;
+  }
+  return f;
+}
+var substringAlgorithm = {
+  name: "substring",
+  description: "Exact substring counting (original baseline). Predictable, no normalization.",
+  score(docs, query) {
+    const terms = query.toLowerCase().trim().split(/\s+/).filter((t) => t.length > 0);
+    if (terms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
+    return docs.map((d) => {
+      const haystack = docFeatures(d.text).lower;
+      let score = 0;
+      for (const term of terms) score += countOccurrences2(haystack, term);
+      return { ref: d.ref, score };
+    });
+  }
+};
+function countOccurrences2(haystack, needle) {
+  if (!needle) return 0;
+  return haystack.split(needle).length - 1;
+}
 var bm25Algorithm = {
   name: "bm25",
   description: "BM25 with stemming + CJK bigram tokenization. IR-standard relevance ranking.",
@@ -2502,11 +2595,8 @@ var bm25Algorithm = {
     const k1 = 1.2;
     const b = 0.75;
     const parsed = docs.map((d) => {
-      const text = d.text;
-      const tf = tfMap(text, true);
-      let len = 0;
-      for (const v of tf.values()) len += v;
-      return { id: d.ref, tf, len };
+      const f = docFeatures(d.text);
+      return { id: d.ref, tf: f.tf, len: f.len };
     });
     const avgdl = parsed.reduce((s, d) => s + d.len, 0) / (N || 1);
     const qTerms = tokenize(query, { stem: true });
@@ -2533,14 +2623,13 @@ var fuzzyAlgorithm = {
   name: "fuzzy",
   description: "Character bigram overlap. Typo-tolerant, script-agnostic, high recall.",
   score(docs, query) {
-    const qTokens = query.toLowerCase().split(/[\s,]+/).filter((t) => t.length >= 4);
+    const qTokens = query.toLowerCase().split(/[\s,]+/).filter((t) => t.length >= 4 || t.length >= 2 && CJK.test(t));
     if (qTokens.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
     const qGrams = /* @__PURE__ */ new Set();
     for (const t of qTokens) for (const g of charBigrams(t)) qGrams.add(g);
     if (qGrams.size === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
     return docs.map((d) => {
-      const haystack = d.text.toLowerCase();
-      const docGrams = new Set(charBigrams(haystack));
+      const docGrams = docFeatures(d.text).grams;
       let hits = 0;
       for (const g of qGrams) if (docGrams.has(g)) hits++;
       return { ref: d.ref, score: hits / qGrams.size };
@@ -3099,14 +3188,17 @@ function openaiToCore(body) {
       }
       case "user": {
         const text = stringContent(m.content);
-        const img = firstImagePart(m.content);
+        const imgs = allImageParts(m.content);
+        const firstImg = imgs[0];
+        const firstUrl = firstImg ? firstImg.image_url.url : void 0;
+        const firstParsed = firstUrl ? parseDataUrl(firstUrl) : void 0;
         const base = deriveMessageId("user", "text", text);
         msgs.push({
           id: clusters.next(base),
           role: "user",
           contentType: "text",
           text,
-          ...img ? { rawOpenaiContent: img.part, imageMediaType: img.mediaType, imageBase64: img.base64 } : {}
+          ...imgs.length === 1 && firstParsed ? { rawOpenaiContent: imgs[0], imageMediaType: firstParsed.mediaType, imageBase64: firstParsed.base64 } : imgs.length > 1 ? { rawOpenaiContentParts: imgs } : {}
         });
         break;
       }
@@ -3201,10 +3293,12 @@ function coreToOpenai(messages) {
       if (m.role === "system") {
         out.push({ role: m.originalRole === "developer" ? "developer" : "system", content: m.text ?? "" });
       } else if (m.role === "user") {
-        if (m.rawOpenaiContent || m.imageBase64) {
+        if (m.rawOpenaiContent || m.imageBase64 || m.rawOpenaiContentParts) {
           const parts = [];
           if (m.text) parts.push({ type: "text", text: m.text });
-          if (m.rawOpenaiContent) {
+          if (m.rawOpenaiContentParts && m.rawOpenaiContentParts.length > 0) {
+            for (const part of m.rawOpenaiContentParts) parts.push(part);
+          } else if (m.rawOpenaiContent) {
             parts.push(m.rawOpenaiContent);
           } else if (m.imageBase64 && m.imageMediaType) {
             parts.push({ type: "image_url", image_url: { url: `data:${m.imageMediaType};base64,${m.imageBase64}` } });
@@ -3229,19 +3323,17 @@ function stringContent(content) {
   }
   return "";
 }
-function firstImagePart(content) {
-  if (!Array.isArray(content)) return void 0;
+function allImageParts(content) {
+  if (!Array.isArray(content)) return [];
+  const out = [];
   for (const p of content) {
-    if (p && typeof p === "object" && p.type === "image_url") {
-      const iu = p.image_url;
-      const url = iu?.url;
-      if (typeof url === "string") {
-        const parsed = parseDataUrl(url);
-        if (parsed) return { part: p, mediaType: parsed.mediaType, base64: parsed.base64 };
-      }
-    }
+    if (typeof p !== "object" || p === null) continue;
+    if (!("type" in p) || p.type !== "image_url" || !("image_url" in p)) continue;
+    const imagePart = p;
+    const url = imagePart.image_url.url;
+    if (typeof url === "string" && parseDataUrl(url)) out.push(p);
   }
-  return void 0;
+  return out;
 }
 function createSubagentNamespaces() {
   const anchors = /* @__PURE__ */ new Map();
@@ -3728,7 +3820,6 @@ function unrepresentableOpenaiMessage(message) {
   const content = message.content;
   if (content == null || typeof content === "string") return null;
   if (!Array.isArray(content)) return "content neither string nor part array";
-  let dataImages = 0;
   for (const part of content) {
     if (typeof part === "string") continue;
     const type5 = part?.type;
@@ -3736,7 +3827,6 @@ function unrepresentableOpenaiMessage(message) {
     if (type5 === "image_url" && role === "user") {
       const url = part?.image_url?.url;
       if (typeof url !== "string" || !url.startsWith("data:")) return "image_url without a data: URL is dropped";
-      if (++dataImages > 1) return "second image_url in one message is dropped";
       continue;
     }
     return `openai content part type ${JSON.stringify(type5) ?? "missing"}`;
@@ -5096,7 +5186,7 @@ function stem2(word) {
   return w;
 }
 var CJK2 = /[\u3400-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]/;
-var CJK_RUN2 = new RegExp(`${CJK2.source}+`, "g");
+var CJK_RUN = new RegExp(`${CJK2.source}+`, "g");
 var LATIN_WORD2 = /[a-z][a-z0-9_]*[a-z0-9]|[a-z0-9]/g;
 function tokenize2(text, opts = {}) {
   const lower = text.toLowerCase();
@@ -5108,7 +5198,7 @@ function tokenize2(text, opts = {}) {
       tokens.push(w);
     }
   }
-  const cjkRuns = lower.match(CJK_RUN2) ?? [];
+  const cjkRuns = lower.match(CJK_RUN) ?? [];
   for (const run of cjkRuns) {
     if (run.length === 1) {
       tokens.push(run);
@@ -5472,7 +5562,7 @@ async function statusReport(runtime, ctx) {
   const coveredIds = collectCoveredMessageIds(state);
   const sentTokens = estimateTokens(coreMessages, coveredIds) + systemPromptTokens;
   const turn = runtime.core.processTurn({ messages: coreMessages, state, config, tokenCount: sentTokens });
-  const versionStr = "0.2.8" ? `billion-context-omp@${"0.2.8"}` : void 0;
+  const versionStr = "0.2.9" ? `billion-context-omp@${"0.2.9"}` : void 0;
   return buildStatusPanel({
     version: versionStr,
     tokenCount: sessionTokens,
@@ -5766,7 +5856,7 @@ async function checkForUpdate(autoUpdate, notify) {
     const data = await res.json();
     const latest = data.version;
     if (!latest) return;
-    const current = runtimeVersion ?? "0.2.8";
+    const current = runtimeVersion ?? "0.2.9";
     const hasUpdate = isNewer(latest, current);
     debug.event("update-check", {
       current,
@@ -6041,9 +6131,9 @@ var index_default = createAcpExtension();
 function wireSessionLifecycle(pi, runtime) {
   pi.on("session_start", async (_event, ctx) => {
     const sid = ctx.sessionManager.getSessionId();
-    logInfo("session", { event: "start", sid, cwd: ctx.cwd, debug: runtime.adapter.debug ?? null, version: true ? "0.2.8" : null });
+    logInfo("session", { event: "start", sid, cwd: ctx.cwd, debug: runtime.adapter.debug ?? null, version: true ? "0.2.9" : null });
     const selfPath = import.meta.url;
-    const conflict = stampAndDetect(selfPath, true ? "0.2.8" : null);
+    const conflict = stampAndDetect(selfPath, true ? "0.2.9" : null);
     if (conflict) {
       logWarn("instance", { event: "dual-instance", self: selfPath, other: conflict.path, otherPid: conflict.pid, otherVersion: conflict.version });
       try {
@@ -6389,18 +6479,18 @@ function wireProviderDebug(pi) {
   pi.on("after_provider_response", (event, ctx) => {
     if (!debug.enabled) return;
     const h = event.headers;
-    const cache = {};
+    const cache2 = {};
     for (const [k, v] of Object.entries(h)) {
       const lk = k.toLowerCase();
       if (lk.includes("cache") || lk.includes("usage") || lk.includes("token") || lk.includes("rate") || lk.includes("x-")) {
-        cache[lk] = v;
+        cache2[lk] = v;
       }
     }
     debug.event("provider-response", {
       sid: ctx.sessionManager.getSessionId(),
       status: event.status,
       requestId: event.requestId ?? null,
-      cache
+      cache: cache2
     });
   });
 }