@cortexkit/opencode-magic-context 0.36.0 → 0.36.1

package/dist/{index-z8v31d85.js → index-3zqcdwqr.js}

@@ -53,7 +53,11 @@ function removeSystemReminders(text) {
 }
 
 // src/hooks/magic-context/read-session-formatting.ts
+import { existsSync, readFileSync, realpathSync } from "node:fs";
 import { createRequire } from "node:module";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
 
 // src/shared/commit-detection.ts
 var HASH_HEX = "[0-9a-f]{7,12}";
@@ -146,23 +150,146 @@ function truncateArg(value, maxLen = 60) {
     return value;
   return `${value.slice(0, maxLen)}…`;
 }
-var requireFromThisModule = createRequire(import.meta.url);
+var TOKENIZER_PACKAGE_DIRS = [
+  ["@cortexkit", "opencode-magic-context"],
+  ["@cortexkit", "pi-magic-context"]
+];
 var tokenizer;
-function getTokenizer() {
+var tokenizerLoadAttempted = false;
+var tokenizerLoadPromise;
+var tokenizerWarningSent = false;
+function tokenizerPackageRoots() {
+  const cwd = process.cwd();
+  const openCodeCache = join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "opencode");
+  const roots = [cwd, openCodeCache];
+  const candidates = [];
+  for (const root of roots) {
+    for (const packageDir of TOKENIZER_PACKAGE_DIRS) {
+      candidates.push(join(root, "node_modules", ...packageDir, "node_modules", "ai-tokenizer"));
+    }
+    candidates.push(join(root, "node_modules", "ai-tokenizer"));
+  }
+  let ancestor = process.argv[1] ? dirname(resolve(process.argv[1])) : cwd;
+  while (true) {
+    candidates.push(join(ancestor, "node_modules", "ai-tokenizer"));
+    const parent = dirname(ancestor);
+    if (parent === ancestor)
+      break;
+    ancestor = parent;
+  }
+  return [...new Set(candidates)];
+}
+function packageImportTarget(value) {
+  if (typeof value === "string")
+    return value;
+  if (!value || typeof value !== "object")
+    return;
+  const conditions = value;
+  return packageImportTarget(conditions.import) ?? packageImportTarget(conditions.default);
+}
+function findTokenizerImportPaths() {
+  for (const packageRoot of tokenizerPackageRoots()) {
+    const packageJsonPath = join(packageRoot, "package.json");
+    if (!existsSync(packageJsonPath))
+      continue;
+    try {
+      const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8"));
+      const tokenizerTarget = packageImportTarget(packageJson.exports?.["."]) ?? (typeof packageJson.module === "string" ? packageJson.module : undefined) ?? (typeof packageJson.main === "string" ? packageJson.main : undefined);
+      const encodingTarget = packageImportTarget(packageJson.exports?.["./encoding/claude"]);
+      if (!tokenizerTarget || !encodingTarget)
+        continue;
+      return {
+        tokenizerPath: realpathSync(join(packageRoot, tokenizerTarget)),
+        encodingPath: realpathSync(join(packageRoot, encodingTarget))
+      };
+    } catch {}
+  }
+  return;
+}
+function constructTokenizer(tokenizerModule, claudeEncoding) {
+  const typedModule = tokenizerModule;
+  const Tokenizer = typedModule.default ?? typedModule.Tokenizer;
+  if (!Tokenizer) {
+    throw new Error("ai-tokenizer does not expose a Tokenizer constructor");
+  }
+  return new Tokenizer(claudeEncoding);
+}
+function loadTokenizer() {
+  const requireFromThisModule = createRequire(import.meta.url);
+  return constructTokenizer(requireFromThisModule("ai-" + "tokenizer"), requireFromThisModule("ai-tokenizer/encoding/" + "claude"));
+}
+async function loadTokenizerFromInstalledPackage() {
+  const installedPaths = findTokenizerImportPaths();
+  if (!installedPaths) {
+    throw new Error("ai-tokenizer was not found under the project, runtime, or OpenCode cache node_modules roots");
+  }
+  const [tokenizerModule, claudeEncoding] = await Promise.all([
+    import(pathToFileURL(installedPaths.tokenizerPath).href),
+    import(pathToFileURL(installedPaths.encodingPath).href)
+  ]);
+  return constructTokenizer(tokenizerModule, claudeEncoding);
+}
+function warnTokenizerFallback(error) {
+  if (tokenizerWarningSent)
+    return;
+  tokenizerWarningSent = true;
+  const reason = error instanceof Error ? error.message : String(error);
+  console.warn("[magic-context] ai-tokenizer is unavailable; using approximate character-based token counts for this process. Token budgets, persisted per-message counts, and protected-tail/compartment boundaries may be less accurate until restart:", reason);
+}
+async function preloadTokenizer() {
   if (tokenizer)
+    return true;
+  if (tokenizerLoadAttempted)
+    return false;
+  if (tokenizerLoadPromise)
+    return tokenizerLoadPromise;
+  tokenizerLoadPromise = (async () => {
+    try {
+      try {
+        tokenizer = loadTokenizer();
+      } catch {
+        tokenizer = await loadTokenizerFromInstalledPackage();
+      }
+      tokenizerLoadAttempted = true;
+      return true;
+    } catch (error) {
+      tokenizerLoadAttempted = true;
+      warnTokenizerFallback(error);
+      return false;
+    } finally {
+      tokenizerLoadPromise = undefined;
+    }
+  })();
+  return tokenizerLoadPromise;
+}
+function getTokenizer() {
+  if (tokenizer || tokenizerLoadAttempted)
     return tokenizer;
-  const tokenizerModule = requireFromThisModule("ai-" + "tokenizer");
-  const Tokenizer = tokenizerModule.default ?? tokenizerModule.Tokenizer;
-  if (!Tokenizer)
-    throw new Error("ai-tokenizer does not expose a Tokenizer constructor");
-  const claudeEncoding = requireFromThisModule("ai-tokenizer/encoding/" + "claude");
-  tokenizer = new Tokenizer(claudeEncoding);
+  tokenizerLoadAttempted = true;
+  try {
+    tokenizer = loadTokenizer();
+  } catch (error) {
+    warnTokenizerFallback(error);
+  }
   return tokenizer;
 }
+function estimateTokensHeuristically(text) {
+  return Math.ceil(text.length / 3.5);
+}
 function estimateTokens(text) {
   if (!text)
     return 0;
-  return getTokenizer().encode(text, "all").length;
+  const activeTokenizer = getTokenizer();
+  if (!activeTokenizer)
+    return estimateTokensHeuristically(text);
+  try {
+    return activeTokenizer.encode(text, "all").length;
+  } catch (error) {
+    tokenizer = undefined;
+    tokenizerLoadAttempted = true;
+    warnTokenizerFallback(error);
+    return estimateTokensHeuristically(text);
+  }
 }
 function normalizeText(text) {
   return text.replace(/\s+/g, " ").trim();
@@ -2889,4 +3016,4 @@ function clearSession(db, sessionId) {
   })();
 }
 
-export { textMentionsRecentCommit, hasMeaningfulUserText, extractTexts, extractToolCallSummaries, estimateTokens, normalizeText, stableStringify, clearCompressionDepth, clearCompressionDepthRange, ensureSessionMetaRow, persistCachedM0, clearCachedM0M1, isRawCompactionSummaryInfo, extractInMemoryMessageViews, buildInMemoryTailRawMessages, isRecord, completedToolArcCrossesBoundary, estimateTrueRawMessageTokens, buildToolArcs, fenceBoundaryForToolArcs, buildTrueRawTokenIndex, computeRawRangeFingerprint, invalidateTrueRawTokenCache, stripPersistedAssistantText, byteSize, stripTagPrefix, peelLeadingMcTagNotation, prependTag, isThinkingPart, isTextPart, isToolPartWithOutput, isFilePart, buildFileSourceContent, isEditTool, partHasCompletedResult, extractToolCallObservation, ToolMutationBatch, createToolDropTarget, setRawMessageProvider, cleanUserText, withRawSessionMessageCache, readRawSessionMessages, primeTailRawMessageCache, getCachedAbsoluteMessageCount, primeInMemoryTailRawMessageCache, readRawSessionMessageOrdinalPage, getRawSessionStoredMessageCount, readRawSessionMessagePartsById, readRawSessionMessageOrdinalById, readRawSessionMessageById, getRawSessionMessageCount, getRawSessionTagKeysThrough, getLegacyProtectedTailStartOrdinal, readSessionChunk, getMessageIndexSourceIdentity, isMessageIndexSourceCurrent, getLastIndexedOrdinal, getIndexedMessageCorpusSize, getMessageIndexReconciliationStartOrdinal, isMessageIndexReconciledThrough, deleteIndexedMessage, clearIndexedMessages, indexSingleMessage, indexMessagesAfterOrdinal, sweepOrphanedOpenCodeMessageIndexes, getOrCreateSessionMeta, updateSessionMeta, advanceToolReclaimWatermark, markSessionCleanupPending, retryPendingSessionCleanups, clearSession };
+export { textMentionsRecentCommit, hasMeaningfulUserText, extractTexts, extractToolCallSummaries, preloadTokenizer, estimateTokens, normalizeText, stableStringify, clearCompressionDepth, clearCompressionDepthRange, ensureSessionMetaRow, persistCachedM0, clearCachedM0M1, isRawCompactionSummaryInfo, extractInMemoryMessageViews, buildInMemoryTailRawMessages, isRecord, completedToolArcCrossesBoundary, estimateTrueRawMessageTokens, buildToolArcs, fenceBoundaryForToolArcs, buildTrueRawTokenIndex, computeRawRangeFingerprint, invalidateTrueRawTokenCache, stripPersistedAssistantText, byteSize, stripTagPrefix, peelLeadingMcTagNotation, prependTag, isThinkingPart, isTextPart, isToolPartWithOutput, isFilePart, buildFileSourceContent, isEditTool, partHasCompletedResult, extractToolCallObservation, ToolMutationBatch, createToolDropTarget, setRawMessageProvider, cleanUserText, withRawSessionMessageCache, readRawSessionMessages, primeTailRawMessageCache, getCachedAbsoluteMessageCount, primeInMemoryTailRawMessageCache, readRawSessionMessageOrdinalPage, getRawSessionStoredMessageCount, readRawSessionMessagePartsById, readRawSessionMessageOrdinalById, readRawSessionMessageById, getRawSessionMessageCount, getRawSessionTagKeysThrough, getLegacyProtectedTailStartOrdinal, readSessionChunk, getMessageIndexSourceIdentity, isMessageIndexSourceCurrent, getLastIndexedOrdinal, getIndexedMessageCorpusSize, getMessageIndexReconciliationStartOrdinal, isMessageIndexReconciledThrough, deleteIndexedMessage, clearIndexedMessages, indexSingleMessage, indexMessagesAfterOrdinal, sweepOrphanedOpenCodeMessageIndexes, getOrCreateSessionMeta, updateSessionMeta, advanceToolReclaimWatermark, markSessionCleanupPending, retryPendingSessionCleanups, clearSession };

package/dist/{index-0c9dmw9m.js → index-8mtsfhyg.js}

@@ -90,9 +90,39 @@ function stripTrailingCommas(content) {
   }
   return result;
 }
-function parseJsonc(content) {
+var PROTOTYPE_POLLUTION_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+function isPrototypePollutionKey(key) {
+  return PROTOTYPE_POLLUTION_KEYS.has(key);
+}
+function sanitizeParsedJson(value, options = {}, path = []) {
+  if (Array.isArray(value)) {
+    return value.map((entry, index) => sanitizeParsedJson(entry, options, [...path, index]));
+  }
+  if (value === null || typeof value !== "object")
+    return value;
+  const source = value;
+  const sourcePrototype = Object.getPrototypeOf(source);
+  if (sourcePrototype !== null && sourcePrototype !== Object.prototype) {
+    options.onRejectedKey?.([...path, "__proto__"]);
+  }
+  const sanitized = {};
+  for (const key of Object.keys(source)) {
+    if (isPrototypePollutionKey(key)) {
+      options.onRejectedKey?.([...path, key]);
+      continue;
+    }
+    Object.defineProperty(sanitized, key, {
+      value: sanitizeParsedJson(source[key], options, [...path, key]),
+      enumerable: true,
+      configurable: true,
+      writable: true
+    });
+  }
+  return sanitized;
+}
+function parseJsonc(content, options = {}) {
   const normalized = stripTrailingCommas(stripJsonComments(content));
-  return JSON.parse(normalized);
+  return sanitizeParsedJson(JSON.parse(normalized), options);
 }
 function readJsoncFile(filePath) {
   try {
@@ -113,4 +143,4 @@ function detectConfigFile(basePath) {
   return { format: "none", path: jsoncPath };
 }
 
-export { stripJsonComments, parseJsonc, readJsoncFile, detectConfigFile };
+export { stripJsonComments, isPrototypePollutionKey, parseJsonc, readJsoncFile, detectConfigFile };

package/dist/{index-37rt2bka.js → index-zp6x6s6a.js}

@@ -1,6 +1,6 @@
 import {
   readJsoncFile
-} from "./index-0c9dmw9m.js";
+} from "./index-8mtsfhyg.js";
 import {
   waitForSafeNotificationTarget
 } from "./index-tsgs7td8.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAkyBvE,QAAA,MAAM,MAAM,EAAE,YAGb,CAAC;AAEF,eAAe,MAAM,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAuyBvE,QAAA,MAAM,MAAM,EAAE,YAGb,CAAC;AAEF,eAAe,MAAM,CAAC"}