@mastra/memory 1.6.2 → 1.7.0-alpha.1

package/dist/{chunk-5W5463NI.cjs → chunk-SHID74TI.cjs}

@@ -11,14 +11,12 @@ var xxhash = require('xxhash-wasm');
 var crypto$1 = require('crypto');
 var async_hooks = require('async_hooks');
 var imageSize = require('image-size');
-var lite = require('js-tiktoken/lite');
-var o200k_base = require('js-tiktoken/ranks/o200k_base');
+var tokenx = require('tokenx');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 
 var xxhash__default = /*#__PURE__*/_interopDefault(xxhash);
 var imageSize__default = /*#__PURE__*/_interopDefault(imageSize);
-var o200k_base__default = /*#__PURE__*/_interopDefault(o200k_base);
 
 // src/processors/observational-memory/observational-memory.ts
 
@@ -1598,14 +1596,6 @@ function calculateProjectedMessageRemoval(chunks, bufferActivation, messageToken
   }
   return bestBoundaryMessageTokens;
 }
-var GLOBAL_TIKTOKEN_KEY = "__mastraTiktoken";
-function getDefaultEncoder() {
-  const cached = globalThis[GLOBAL_TIKTOKEN_KEY];
-  if (cached) return cached;
-  const encoder = new lite.Tiktoken(o200k_base__default.default);
-  globalThis[GLOBAL_TIKTOKEN_KEY] = encoder;
-  return encoder;
-}
 var IMAGE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
   "png",
   "jpg",
@@ -1619,7 +1609,7 @@ var IMAGE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
   "heif",
   "avif"
 ]);
-var TOKEN_ESTIMATE_CACHE_VERSION = 5;
+var TOKEN_ESTIMATE_CACHE_VERSION = 6;
 var DEFAULT_IMAGE_ESTIMATOR = {
   baseTokens: 85,
   tileTokens: 170,
@@ -1686,13 +1676,8 @@ function buildEstimateKey(kind, text) {
   const payloadHash = crypto$1.createHash("sha1").update(text).digest("hex");
   return `${kind}:${payloadHash}`;
 }
-function resolveEncodingId(encoding) {
-  if (!encoding) return "o200k_base";
-  try {
-    return `custom:${crypto$1.createHash("sha1").update(JSON.stringify(encoding)).digest("hex")}`;
-  } catch {
-    return "custom:unknown";
-  }
+function resolveEstimatorId() {
+  return "tokenx";
 }
 function isTokenEstimateEntry(value) {
   if (!value || typeof value !== "object") return false;
@@ -2391,20 +2376,17 @@ async function fetchGoogleAttachmentTokenEstimate(modelId, part) {
   }
 }
 var TokenCounter = class _TokenCounter {
-  encoder;
   cacheSource;
   defaultModelContext;
   modelContextStorage = new async_hooks.AsyncLocalStorage();
   inFlightAttachmentCounts = /* @__PURE__ */ new Map();
   // Per-message overhead: accounts for role tokens, message framing, and separators.
-  // Empirically derived from OpenAI's token counting guide (3 tokens per message base +
-  // fractional overhead from name/role encoding). 3.8 is a practical average across models.
+  // 3.8 remains a practical average across providers for OM thresholding.
   static TOKENS_PER_MESSAGE = 3.8;
   // Conversation-level overhead: system prompt framing, reply priming tokens, etc.
   static TOKENS_PER_CONVERSATION = 24;
-  constructor(encoding, options) {
-    this.encoder = encoding ? new lite.Tiktoken(encoding) : getDefaultEncoder();
-    this.cacheSource = `v${TOKEN_ESTIMATE_CACHE_VERSION}:${resolveEncodingId(encoding)}`;
+  constructor(options) {
+    this.cacheSource = `v${TOKEN_ESTIMATE_CACHE_VERSION}:${resolveEstimatorId()}`;
     this.defaultModelContext = parseModelContext(options?.model);
   }
   runWithModelContext(model, fn) {
@@ -2418,7 +2400,7 @@ var TokenCounter = class _TokenCounter {
    */
   countString(text) {
     if (!text) return 0;
-    return this.encoder.encode(text, "all").length;
+    return tokenx.estimateTokenCount(text);
   }
   readOrPersistPartEstimate(part, kind, payload) {
     const key = buildEstimateKey(kind, payload);
@@ -3314,7 +3296,7 @@ Async buffering is enabled by default \u2014 this opt-out is only needed when us
       ),
       instruction: config.reflection?.instruction
     };
-    this.tokenCounter = new TokenCounter(void 0, {
+    this.tokenCounter = new TokenCounter({
       model: typeof observationModel === "string" ? observationModel : void 0
     });
     this.onDebugEvent = config.onDebugEvent;
@@ -6739,5 +6721,5 @@ exports.formatMessagesForObserver = formatMessagesForObserver;
 exports.hasCurrentTaskSection = hasCurrentTaskSection;
 exports.optimizeObservationsForContext = optimizeObservationsForContext;
 exports.parseObserverOutput = parseObserverOutput;
-//# sourceMappingURL=chunk-5W5463NI.cjs.map
-//# sourceMappingURL=chunk-5W5463NI.cjs.map
+//# sourceMappingURL=chunk-SHID74TI.cjs.map
+//# sourceMappingURL=chunk-SHID74TI.cjs.map