nemoris 0.1.0 → 0.1.2

package/src/runtime/token-counter.js

@@ -1,134 +1,134 @@
-import { estimateInvocationTokens } from "./token-estimator.js";
-
-let jsTiktokenModulePromise = null;
-
-function normalizeModelId(modelId) {
-  return String(modelId || "").toLowerCase();
-}
-
-function extractOpenAiTokenizerModel(modelId) {
-  const normalized = normalizeModelId(modelId);
-  const candidates = [
-    "gpt-4.1-mini",
-    "gpt-4.1",
-    "gpt-4o-mini",
-    "gpt-4o",
-    "o3-mini",
-    "o3",
-    "o1-mini",
-    "o1",
-    "gpt-4-turbo",
-    "gpt-4",
-    "gpt-3.5-turbo"
-  ];
-
-  for (const candidate of candidates) {
-    if (normalized.includes(candidate)) {
-      return candidate;
-    }
-  }
-
-  return null;
-}
-
-function renderInvocationText(invocation) {
-  const sections = [];
-  if (invocation?.system) {
-    sections.push(`system\n${invocation.system}`);
-  }
-  for (const message of invocation?.messages || []) {
-    sections.push(`${message.role || "user"}\n${message.content || ""}`);
-  }
-  return sections.join("\n\n");
-}
-
-async function loadJsTiktoken() {
-  if (!jsTiktokenModulePromise) {
-    jsTiktokenModulePromise = import("js-tiktoken").catch(() => null);
-  }
-  return jsTiktokenModulePromise;
-}
-
-function countWithApproximateTokenizer(modelName, invocation) {
-  const rendered = renderInvocationText(invocation);
-  const textTokens = Math.max(1, Math.ceil(rendered.length / 4));
-  const framing = invocation?.messages?.length ? invocation.messages.length * 6 : 0;
-  return {
-    total: textTokens + framing,
-    mode: "model_specific",
-    source: `js-tiktoken:${modelName}`,
-    details: {
-      textTokens,
-      framing,
-      fallback: "approximate_model_specific",
-    }
-  };
-}
-
-async function countWithJsTiktoken(modelName, invocation) {
-  const jsTiktoken = await loadJsTiktoken();
-  if (!jsTiktoken?.encodingForModel) {
-    return countWithApproximateTokenizer(modelName, invocation);
-  }
-  const encoder = jsTiktoken.encodingForModel(modelName);
-  const rendered = renderInvocationText(invocation);
-  const textTokens = encoder.encode(rendered).length;
-  const framing = invocation?.messages?.length ? invocation.messages.length * 6 : 0;
-  return {
-    total: textTokens + framing,
-    mode: "model_specific",
-    source: `js-tiktoken:${modelName}`,
-    details: {
-      textTokens,
-      framing
-    }
-  };
-}
-
-export class TokenCounter {
-  async countInvocation({ providerId, invocation, adapter }) {
-    if (adapter && typeof adapter.countTokens === "function") {
-      try {
-        const total = await adapter.countTokens(invocation);
-        if (Number.isFinite(total) && total > 0) {
-          return {
-            total,
-            mode: "exact",
-            source: `${providerId}:count_tokens`,
-            details: {
-              providerId
-            }
-          };
-        }
-      } catch (error) {
-        const fallback = this.countHeuristic(invocation);
-        return {
-          ...fallback,
-          details: {
-            ...fallback.details,
-            fallbackReason: error.message
-          }
-        };
-      }
-    }
-
-    const tokenizerModel = extractOpenAiTokenizerModel(invocation?.model);
-    if (tokenizerModel) {
-      try {
-        return await countWithJsTiktoken(tokenizerModel, invocation);
-      } catch {}
-    }
-
-    return this.countHeuristic(invocation);
-  }
-
-  countHeuristic(invocation) {
-    const estimate = estimateInvocationTokens(invocation);
-    return {
-      total: estimate.total,
-      mode: "heuristic",
-      source: "multi_strategy_estimator",
-      details: estimate
-    };
-  }
-}
+import { estimateInvocationTokens } from "./token-estimator.js";
+
+let jsTiktokenModulePromise = null;
+
+function normalizeModelId(modelId) {
+  return String(modelId || "").toLowerCase();
+}
+
+function extractOpenAiTokenizerModel(modelId) {
+  const normalized = normalizeModelId(modelId);
+  const candidates = [
+    "gpt-4.1-mini",
+    "gpt-4.1",
+    "gpt-4o-mini",
+    "gpt-4o",
+    "o3-mini",
+    "o3",
+    "o1-mini",
+    "o1",
+    "gpt-4-turbo",
+    "gpt-4",
+    "gpt-3.5-turbo"
+  ];
+
+  for (const candidate of candidates) {
+    if (normalized.includes(candidate)) {
+      return candidate;
+    }
+  }
+
+  return null;
+}
+
+function renderInvocationText(invocation) {
+  const sections = [];
+  if (invocation?.system) {
+    sections.push(`system\n${invocation.system}`);
+  }
+  for (const message of invocation?.messages || []) {
+    sections.push(`${message.role || "user"}\n${message.content || ""}`);
+  }
+  return sections.join("\n\n");
+}
+
+async function loadJsTiktoken() {
+  if (!jsTiktokenModulePromise) {
+    jsTiktokenModulePromise = import("js-tiktoken").catch(() => null);
+  }
+  return jsTiktokenModulePromise;
+}
+
+function countWithApproximateTokenizer(modelName, invocation) {
+  const rendered = renderInvocationText(invocation);
+  const textTokens = Math.max(1, Math.ceil(rendered.length / 4));
+  const framing = invocation?.messages?.length ? invocation.messages.length * 6 : 0;
+  return {
+    total: textTokens + framing,
+    mode: "model_specific",
+    source: `js-tiktoken:${modelName}`,
+    details: {
+      textTokens,
+      framing,
+      fallback: "approximate_model_specific",
+    }
+  };
+}
+
+async function countWithJsTiktoken(modelName, invocation) {
+  const jsTiktoken = await loadJsTiktoken();
+  if (!jsTiktoken?.encodingForModel) {
+    return countWithApproximateTokenizer(modelName, invocation);
+  }
+  const encoder = jsTiktoken.encodingForModel(modelName);
+  const rendered = renderInvocationText(invocation);
+  const textTokens = encoder.encode(rendered).length;
+  const framing = invocation?.messages?.length ? invocation.messages.length * 6 : 0;
+  return {
+    total: textTokens + framing,
+    mode: "model_specific",
+    source: `js-tiktoken:${modelName}`,
+    details: {
+      textTokens,
+      framing
+    }
+  };
+}
+
+export class TokenCounter {
+  async countInvocation({ providerId, invocation, adapter }) {
+    if (adapter && typeof adapter.countTokens === "function") {
+      try {
+        const total = await adapter.countTokens(invocation);
+        if (Number.isFinite(total) && total > 0) {
+          return {
+            total,
+            mode: "exact",
+            source: `${providerId}:count_tokens`,
+            details: {
+              providerId
+            }
+          };
+        }
+      } catch (error) {
+        const fallback = this.countHeuristic(invocation);
+        return {
+          ...fallback,
+          details: {
+            ...fallback.details,
+            fallbackReason: error.message
+          }
+        };
+      }
+    }
+
+    const tokenizerModel = extractOpenAiTokenizerModel(invocation?.model);
+    if (tokenizerModel) {
+      try {
+        return await countWithJsTiktoken(tokenizerModel, invocation);
+      } catch {}
+    }
+
+    return this.countHeuristic(invocation);
+  }
+
+  countHeuristic(invocation) {
+    const estimate = estimateInvocationTokens(invocation);
+    return {
+      total: estimate.total,
+      mode: "heuristic",
+      source: "multi_strategy_estimator",
+      details: estimate
+    };
+  }
+}

package/src/runtime/token-estimator.js

@@ -1,59 +1,59 @@
-function countMatches(text, pattern) {
-  return (text.match(pattern) || []).length;
-}
-
-export function estimateTextTokens(text) {
-  const raw = String(text || "");
-  if (!raw) {
-    return {
-      total: 0,
-      strategies: {
-        charBased: 0,
-        wordBased: 0,
-        denseText: 0,
-        cjkAware: 0
-      }
-    };
-  }
-
-  const words = raw.trim().split(/\s+/).filter(Boolean).length;
-  const punctuation = countMatches(raw, /[{}[\]():;,.]/g);
-  const operators = countMatches(raw, /[=+\-/*<>|&]/g);
-  const quotes = countMatches(raw, /["'`]/g);
-  const newlines = countMatches(raw, /\n/g);
-  const cjk = countMatches(raw, /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu);
-  const symbolRatio = raw.length ? (punctuation + operators + quotes) / raw.length : 0;
-
-  const charBased = Math.ceil(raw.length / 4);
-  const wordBased = Math.ceil(words * 1.28 + punctuation * 0.18 + operators * 0.2 + newlines * 0.12);
-  const denseDivisor = symbolRatio >= 0.12 ? 3.1 : 3.5;
-  const denseText = Math.ceil(raw.length / denseDivisor + quotes * 0.08);
-  const cjkAware = cjk ? Math.ceil(cjk * 1.15 + (raw.length - cjk) / 4.4) : 0;
-
-  return {
-    total: Math.max(charBased, wordBased, denseText, cjkAware),
-    strategies: {
-      charBased,
-      wordBased,
-      denseText,
-      cjkAware
-    }
-  };
-}
-
-export function estimateInvocationTokens(invocation) {
-  const system = estimateTextTokens(invocation?.system || "");
-  const messages = (invocation?.messages || []).map((message) => ({
-    role: message.role || "user",
-    ...estimateTextTokens(message.content || "")
-  }));
-  const messageTotal = messages.reduce((sum, message) => sum + message.total, 0);
-  const framing = invocation?.messages?.length ? invocation.messages.length * 6 : 0;
-
-  return {
-    total: system.total + messageTotal + framing,
-    system,
-    messages,
-    framing
-  };
-}
+function countMatches(text, pattern) {
+  return (text.match(pattern) || []).length;
+}
+
+export function estimateTextTokens(text) {
+  const raw = String(text || "");
+  if (!raw) {
+    return {
+      total: 0,
+      strategies: {
+        charBased: 0,
+        wordBased: 0,
+        denseText: 0,
+        cjkAware: 0
+      }
+    };
+  }
+
+  const words = raw.trim().split(/\s+/).filter(Boolean).length;
+  const punctuation = countMatches(raw, /[{}[\]():;,.]/g);
+  const operators = countMatches(raw, /[=+\-/*<>|&]/g);
+  const quotes = countMatches(raw, /["'`]/g);
+  const newlines = countMatches(raw, /\n/g);
+  const cjk = countMatches(raw, /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu);
+  const symbolRatio = raw.length ? (punctuation + operators + quotes) / raw.length : 0;
+
+  const charBased = Math.ceil(raw.length / 4);
+  const wordBased = Math.ceil(words * 1.28 + punctuation * 0.18 + operators * 0.2 + newlines * 0.12);
+  const denseDivisor = symbolRatio >= 0.12 ? 3.1 : 3.5;
+  const denseText = Math.ceil(raw.length / denseDivisor + quotes * 0.08);
+  const cjkAware = cjk ? Math.ceil(cjk * 1.15 + (raw.length - cjk) / 4.4) : 0;
+
+  return {
+    total: Math.max(charBased, wordBased, denseText, cjkAware),
+    strategies: {
+      charBased,
+      wordBased,
+      denseText,
+      cjkAware
+    }
+  };
+}
+
+export function estimateInvocationTokens(invocation) {
+  const system = estimateTextTokens(invocation?.system || "");
+  const messages = (invocation?.messages || []).map((message) => ({
+    role: message.role || "user",
+    ...estimateTextTokens(message.content || "")
+  }));
+  const messageTotal = messages.reduce((sum, message) => sum + message.total, 0);
+  const framing = invocation?.messages?.length ? invocation.messages.length * 6 : 0;
+
+  return {
+    total: system.total + messageTotal + framing,
+    system,
+    messages,
+    framing
+  };
+}