@cortexkit/pi-magic-context 0.47.0 → 0.49.0

package/dist/{index-266qrgss.js → index-r56xgha8.js}

@@ -1,25 +1,13 @@
 import {
-  log
-} from "./index-7n8gb6j3.js";
-
-// ../plugin/src/shared/internal-initiator-marker.ts
-var OMO_INTERNAL_INITIATOR_MARKER = "<!-- OMO_INTERNAL_INITIATOR -->";
-
-// ../plugin/src/shared/system-directive.ts
-var SYSTEM_DIRECTIVE_PREFIX = "[SYSTEM DIRECTIVE: MAGIC-CONTEXT";
-function isSystemDirective(text) {
-  return text.trimStart().startsWith(SYSTEM_DIRECTIVE_PREFIX);
-}
-function removeSystemReminders(text) {
-  return text.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, "").trim();
-}
+  log2
+} from "./index-x3dcnxh0.js";
 
 // ../plugin/src/hooks/magic-context/read-session-formatting.ts
 import { existsSync, readFileSync, realpathSync, statSync } from "node:fs";
 import { createRequire } from "node:module";
 import { homedir } from "node:os";
 import { dirname, join, resolve } from "node:path";
-import { pathToFileURL } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
 
 // ../plugin/src/shared/commit-detection.ts
 var HASH_HEX = "[0-9a-f]{7,12}";
@@ -32,6 +20,284 @@ function createCommitHashExtractPattern() {
   return new RegExp(`\`?\\b(${HASH_HEX})\\b\`?`, "gi");
 }
 
+// ../plugin/src/shared/internal-initiator-marker.ts
+var OMO_INTERNAL_INITIATOR_MARKER = "<!-- OMO_INTERNAL_INITIATOR -->";
+
+// ../plugin/src/shared/system-directive.ts
+var SYSTEM_DIRECTIVE_PREFIX = "[SYSTEM DIRECTIVE: MAGIC-CONTEXT";
+function isSystemDirective(text) {
+  return text.trimStart().startsWith(SYSTEM_DIRECTIVE_PREFIX);
+}
+function removeSystemReminders(text) {
+  return text.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, "").trim();
+}
+
+// ../plugin/src/hooks/magic-context/token-count-exact.ts
+var NO_RANK = 4294967295;
+var LONG_PIECE_BYTES = 256;
+var MERGE_CACHE_GENERATION = 50000;
+function hasInternals(tokenizer) {
+  const value = tokenizer;
+  return !!value && value.patternRegex instanceof RegExp && value.patternRegex.global && typeof value.stringRankEncoder === "object" && value.stringRankEncoder !== null && Array.isArray(value.binaryFirstByteIndex);
+}
+var textEncoder = new TextEncoder;
+var textDecoder = new TextDecoder("utf-8");
+function isValidUtf8(bytes) {
+  let index = 0;
+  while (index < bytes.length) {
+    const first = bytes[index];
+    let size = 0;
+    let codePoint = 0;
+    if (first <= 127) {
+      size = 1;
+      codePoint = first;
+    } else if ((first & 224) === 192) {
+      size = 2;
+      codePoint = first & 31;
+      if (first <= 193)
+        return false;
+    } else if ((first & 240) === 224) {
+      size = 3;
+      codePoint = first & 15;
+    } else if ((first & 248) === 240) {
+      size = 4;
+      codePoint = first & 7;
+      if (first > 244)
+        return false;
+    } else
+      return false;
+    if (index + size > bytes.length)
+      return false;
+    for (let offset = 1;offset < size; offset += 1) {
+      const byte = bytes[index + offset];
+      if ((byte & 192) !== 128)
+        return false;
+      codePoint = codePoint << 6 | byte & 63;
+    }
+    if (size === 2 && codePoint < 128)
+      return false;
+    if (size === 3 && codePoint < 2048)
+      return false;
+    if (size === 4 && codePoint < 65536)
+      return false;
+    if (codePoint >= 55296 && codePoint <= 57343)
+      return false;
+    if (codePoint > 1114111)
+      return false;
+    index += size;
+  }
+  return true;
+}
+function binarySearchBytes(sorted, key) {
+  let low = 0;
+  let high = sorted.length - 1;
+  while (low <= high) {
+    const mid = low + high >>> 1;
+    const midKey = sorted[mid][0];
+    let compare = 0;
+    const shared = Math.min(midKey.length, key.length);
+    for (let index = 0;index < shared; index += 1) {
+      compare = midKey[index] - key[index];
+      if (compare !== 0)
+        break;
+    }
+    if (compare === 0)
+      compare = midKey.length - key.length;
+    if (compare === 0)
+      return sorted[mid][1];
+    if (compare < 0)
+      low = mid + 1;
+    else
+      high = mid - 1;
+  }
+  return -1;
+}
+function rankOf(tokenizer, slice) {
+  if (isValidUtf8(slice)) {
+    const rank = tokenizer.stringRankEncoder[textDecoder.decode(slice)];
+    if (rank !== undefined)
+      return rank;
+  }
+  const bucket = tokenizer.binaryFirstByteIndex[slice[0]];
+  if (bucket !== null && bucket !== undefined) {
+    const rank = binarySearchBytes(bucket, slice);
+    if (rank !== -1)
+      return rank;
+  }
+  return NO_RANK;
+}
+function mergeByScanning(tokenizer, bytes) {
+  const starts = [];
+  const ranks = [];
+  const pairRank = (index) => {
+    const end = starts[index + 2];
+    return end === undefined ? NO_RANK : rankOf(tokenizer, bytes.subarray(starts[index], end));
+  };
+  for (let index = 0;index <= bytes.length; index += 1)
+    starts.push(index);
+  for (let index = 0;index < starts.length; index += 1)
+    ranks.push(pairRank(index));
+  while (starts.length > 1) {
+    let minRank = NO_RANK;
+    let minIndex = -1;
+    for (let index = 0;index < ranks.length - 1; index += 1) {
+      if (ranks[index] < minRank) {
+        minRank = ranks[index];
+        minIndex = index;
+      }
+    }
+    if (minIndex === -1)
+      break;
+    starts.splice(minIndex + 1, 1);
+    ranks.splice(minIndex, 1);
+    ranks[minIndex] = pairRank(minIndex);
+    if (minIndex > 0)
+      ranks[minIndex - 1] = pairRank(minIndex - 1);
+  }
+  const tokens = [];
+  for (let index = 0;index < starts.length - 1; index += 1) {
+    const rank = rankOf(tokenizer, bytes.subarray(starts[index], starts[index + 1]));
+    if (rank !== NO_RANK)
+      tokens.push(rank);
+  }
+  return tokens;
+}
+function mergeWithHeap(tokenizer, bytes) {
+  const length = bytes.length;
+  const next = new Int32Array(length + 1);
+  const prev = new Int32Array(length + 1);
+  const alive = new Uint8Array(length + 1).fill(1);
+  const rank = new Float64Array(length + 1).fill(NO_RANK);
+  for (let index = 0;index <= length; index += 1) {
+    next[index] = index + 1;
+    prev[index] = index - 1;
+  }
+  const heapRank = [];
+  const heapAt = [];
+  const less = (a, b) => heapRank[a] < heapRank[b] || heapRank[a] === heapRank[b] && heapAt[a] < heapAt[b];
+  const swap = (a, b) => {
+    [heapRank[a], heapRank[b]] = [heapRank[b], heapRank[a]];
+    [heapAt[a], heapAt[b]] = [heapAt[b], heapAt[a]];
+  };
+  const push = (value, at) => {
+    heapRank.push(value);
+    heapAt.push(at);
+    let child = heapRank.length - 1;
+    while (child > 0) {
+      const parent = child - 1 >> 1;
+      if (!less(child, parent))
+        break;
+      swap(child, parent);
+      child = parent;
+    }
+  };
+  const pop = () => {
+    const last = heapRank.length - 1;
+    swap(0, last);
+    heapRank.pop();
+    heapAt.pop();
+    let parent = 0;
+    for (;; ) {
+      const left = parent * 2 + 1;
+      const right = left + 1;
+      let smallest = parent;
+      if (left < heapRank.length && less(left, smallest))
+        smallest = left;
+      if (right < heapRank.length && less(right, smallest))
+        smallest = right;
+      if (smallest === parent)
+        break;
+      swap(parent, smallest);
+      parent = smallest;
+    }
+  };
+  const pairRank = (at) => {
+    const middle = next[at];
+    if (middle >= length)
+      return NO_RANK;
+    return rankOf(tokenizer, bytes.subarray(at, next[middle]));
+  };
+  const update = (at) => {
+    rank[at] = pairRank(at);
+    if (rank[at] !== NO_RANK)
+      push(rank[at], at);
+  };
+  for (let at = 0;at < length; at += 1)
+    update(at);
+  while (heapRank.length > 0) {
+    const value = heapRank[0];
+    const at = heapAt[0];
+    pop();
+    if (!alive[at] || rank[at] !== value)
+      continue;
+    const removed = next[at];
+    alive[removed] = 0;
+    next[at] = next[removed];
+    prev[next[removed]] = at;
+    update(at);
+    if (at > 0)
+      update(prev[at]);
+  }
+  const tokens = [];
+  for (let at = 0;at < length; at = next[at]) {
+    const value = rankOf(tokenizer, bytes.subarray(at, next[at]));
+    if (value !== NO_RANK)
+      tokens.push(value);
+  }
+  return tokens;
+}
+var mergeCaches = new WeakMap;
+var mergedPieces = 0;
+function mergePiece(tokenizer, piece) {
+  let cache = mergeCaches.get(tokenizer);
+  if (!cache) {
+    cache = { current: new Map, previous: new Map };
+    mergeCaches.set(tokenizer, cache);
+  }
+  let tokens = cache.current.get(piece);
+  if (tokens)
+    return tokens;
+  tokens = cache.previous.get(piece);
+  if (!tokens) {
+    const bytes = textEncoder.encode(piece);
+    mergedPieces += 1;
+    tokens = bytes.length > LONG_PIECE_BYTES ? mergeWithHeap(tokenizer, bytes) : mergeByScanning(tokenizer, bytes);
+  }
+  if (cache.current.size >= MERGE_CACHE_GENERATION) {
+    cache.previous = cache.current;
+    cache.current = new Map;
+  }
+  cache.current.set(piece, tokens);
+  return tokens;
+}
+function encodeTokensExactly(tokenizer, text) {
+  if (!hasInternals(tokenizer)) {
+    return tokenizer.encode(text, "all");
+  }
+  if (text.length < 10) {
+    const direct = tokenizer.stringRankEncoder[text];
+    if (direct !== undefined)
+      return [direct];
+  }
+  const tokens = [];
+  const regex = tokenizer.patternRegex;
+  regex.lastIndex = 0;
+  for (let match = regex.exec(text);match !== null; match = regex.exec(text)) {
+    const piece = match[0];
+    const direct = tokenizer.stringRankEncoder[piece];
+    if (direct !== undefined) {
+      tokens.push(direct);
+      continue;
+    }
+    for (const token of mergePiece(tokenizer, piece))
+      tokens.push(token);
+  }
+  return tokens;
+}
+function countTokensExactly(tokenizer, text) {
+  return encodeTokensExactly(tokenizer, text).length;
+}
+
 // ../plugin/src/hooks/magic-context/read-session-formatting.ts
 var MAX_COMMITS_PER_BLOCK = 5;
 function hasMeaningfulUserText2(parts) {
@@ -144,22 +410,30 @@ var tokenizerSerializedTableBytes;
 function tokenizerPackageRoots() {
   const cwd = process.cwd();
   const openCodeCache = join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "opencode");
-  const roots = [cwd, openCodeCache];
   const candidates = [];
-  for (const root of roots) {
+  const probeRoot = (root) => {
     for (const packageDir of TOKENIZER_PACKAGE_DIRS) {
       candidates.push(join(root, "node_modules", ...packageDir, "node_modules", "ai-tokenizer"));
     }
     candidates.push(join(root, "node_modules", "ai-tokenizer"));
+  };
+  for (const root of [cwd, openCodeCache]) {
+    probeRoot(root);
   }
-  let ancestor = process.argv[1] ? dirname(resolve(process.argv[1])) : cwd;
-  while (true) {
-    candidates.push(join(ancestor, "node_modules", "ai-tokenizer"));
-    const parent = dirname(ancestor);
-    if (parent === ancestor)
-      break;
-    ancestor = parent;
-  }
+  const pushAncestors = (startDir) => {
+    let ancestor = startDir;
+    while (true) {
+      candidates.push(join(ancestor, "node_modules", "ai-tokenizer"));
+      const parent = dirname(ancestor);
+      if (parent === ancestor)
+        break;
+      ancestor = parent;
+    }
+  };
+  pushAncestors(process.argv[1] ? dirname(resolve(process.argv[1])) : cwd);
+  const ownDir = dirname(fileURLToPath(new URL(import.meta.url)));
+  pushAncestors(ownDir);
+  probeRoot(join(homedir(), ".omp", "plugins"));
   return [...new Set(candidates)];
 }
 function packageImportTarget(value) {
@@ -222,7 +496,7 @@ function warnTokenizerFallback(error) {
     return;
   tokenizerWarningSent = true;
   const reason = error instanceof Error ? error.message : String(error);
-  log("[magic-context] ai-tokenizer is unavailable; using approximate character-based token counts for this process. Token budgets, persisted per-message counts, and protected-tail/compartment boundaries may be less accurate until restart:", reason);
+  log2("[magic-context] ai-tokenizer is unavailable; using approximate character-based token counts for this process. Token budgets, persisted per-message counts, and protected-tail/compartment boundaries may be less accurate until restart:", reason);
 }
 async function preloadTokenizer() {
   if (tokenizer)
@@ -297,7 +571,7 @@ function estimateTokens(text) {
   if (!activeTokenizer)
     return estimateTokensHeuristically(text);
   try {
-    return activeTokenizer.encode(text, "all").length;
+    return countTokensExactly(activeTokenizer, text);
   } catch (error) {
     tokenizer = undefined;
     tokenizerLoadAttempted = true;