ai-token-estimator 1.4.0 → 1.6.0

package/README.md

@@ -11,6 +11,7 @@ The best way to estimate **tokens + input cost** for LLM calls — with **exact
 ## Features
 
 - **Exact OpenAI tokenization** (tiktoken-compatible BPE): `encode()` / `decode()` / `openai_exact`
+- **Chat-aware tokenization**: `encodeChat()` returns exact token IDs for chat messages using ChatML format
 - **OpenAI chat completion token counting** (legacy `functions` API): `countChatCompletionTokens()` with optional per-message breakdown
 - **Pure TypeScript SentencePiece tokenizer** (no native dependencies):
   - Supports `.model` files (protobuf format)
@@ -64,6 +65,21 @@ console.log(countTokens({ text: 'Hello, world!', model: 'gpt-5.1' }));
 
 ## Quick Recipes
 
+### Encode chat messages to tokens (ChatML format)
+
+```ts
+import { encodeChat, decode } from 'ai-token-estimator';
+
+const tokens = encodeChat([
+  { role: 'system', content: 'You are helpful.' },
+  { role: 'user', content: 'Hello!' }
+], { model: 'gpt-4o' });
+
+console.log(tokens); // [200264, 9125, 200266, 2610, 525, 11190, 13, 200265, ...]
+console.log(decode(tokens, { encoding: 'o200k_base' }));
+// <|im_start|>system<|im_sep|>You are helpful.<|im_end|>...
+```
+
 ### OpenAI chat completion tokens (legacy functions API)
 
 ```ts
@@ -561,6 +577,129 @@ Encodes text into **OpenAI token IDs** using tiktoken-compatible BPE tokenizatio
 
 Decodes OpenAI token IDs back into text using the selected encoding/model.
 
+### `encodeChat(messages: ChatMessage[], options?: EncodeChatOptions): number[]`
+
+Encodes chat messages into **exact token IDs** using ChatML format. Returns the ChatML message prompt tokens (messages + optional assistant priming), including special delimiter tokens (`<|im_start|>`, `<|im_sep|>`, `<|im_end|>`).
+
+```ts
+import { encodeChat, decode } from 'ai-token-estimator';
+
+const tokens = encodeChat([
+  { role: 'system', content: 'You are helpful.' },
+  { role: 'user', content: 'Hello!' }
+], { model: 'gpt-4o' });
+
+// Tokens include ChatML structure:
+// <|im_start|>system<|im_sep|>You are helpful.<|im_end|>
+// <|im_start|>user<|im_sep|>Hello!<|im_end|>
+// <|im_start|>assistant<|im_sep|>  (priming)
+```
+
+**Parameters:**
+
+```typescript
+interface EncodeChatOptions {
+  model?: string;              // OpenAI model (e.g., 'gpt-4o')
+  encoding?: OpenAIEncoding;   // Explicit encoding override
+  primeAssistant?: boolean;    // Append assistant priming (default: true)
+}
+```
+
+**Supported encodings:**
+- `cl100k_base` (GPT-4, GPT-3.5-turbo)
+- `o200k_base` (GPT-4o, GPT-4o-mini)
+- `o200k_harmony` (experimental)
+
+**Limitations:**
+- **OpenAI models only** — throws for claude-*, gemini-*
+- **Legacy functions API only** — throws for tool_calls, tool_call_id
+- **Text content only** — throws for multimodal content (arrays)
+
+**Note on function_call:** Messages with `function_call` are encoded with the function name and arguments as content. The token count differs from `countChatCompletionTokens()` because the latter includes `FUNCTION_CALL_METADATA_TOKEN_OVERHEAD` (3 tokens) for API accounting. The exact difference depends on whether both name and arguments are present (2 token difference due to newline separator) or only one field is present (3 token difference).
+
+**Note on o200k_harmony:** Support for `o200k_harmony` encoding is experimental. The token structure may not match actual API behavior.
+
+### `isWithinTokenLimit(text, tokenLimit, options?): false | number`
+
+Checks if text is within a token limit with **early exit optimization**. Returns `false` if the limit is exceeded, or the actual token count if within limit.
+
+This is significantly faster than full tokenization when the limit is exceeded early in the text (up to 1000x+ faster for large texts with small limits).
+
+```typescript
+import { isWithinTokenLimit } from 'ai-token-estimator';
+
+// Returns token count if within limit
+const count = isWithinTokenLimit('Hello, world!', 100, { model: 'gpt-4o' });
+if (count !== false) {
+  console.log(`Text has ${count} tokens`);
+}
+
+// Returns false if exceeds limit (with early exit)
+const result = isWithinTokenLimit(longText, 10, { model: 'gpt-4o' });
+if (result === false) {
+  console.log('Text exceeds 10 tokens');
+}
+```
+
+**Parameters:**
+
+```typescript
+interface IsWithinTokenLimitOptions {
+  model?: string;              // OpenAI model (e.g., 'gpt-4o')
+  encoding?: OpenAIEncoding;   // Explicit encoding override
+  allowSpecial?: SpecialTokenHandling;  // How to handle special tokens
+}
+```
+
+**Throws:**
+- `Error` if `tokenLimit` is invalid (NaN, Infinity, negative, non-integer)
+- `Error` if `model` is a known non-OpenAI model (claude-*, gemini-*)
+
+### `isChatWithinTokenLimit(input): false | number`
+
+Checks if chat messages are within a token limit with **early exit optimization**. Returns `false` if exceeded, or the actual token count if within limit.
+
+Uses the same token counting logic as `countChatCompletionTokens()` but exits early when the limit is exceeded.
+
+```typescript
+import { isChatWithinTokenLimit } from 'ai-token-estimator';
+
+const result = isChatWithinTokenLimit({
+  messages: [
+    { role: 'system', content: 'You are a helpful assistant.' },
+    { role: 'user', content: 'Hello!' }
+  ],
+  model: 'gpt-4o',
+  tokenLimit: 100,
+  functions: [{ name: 'get_weather', parameters: { type: 'object' } }],
+});
+
+if (result === false) {
+  console.log('Messages exceed token limit');
+} else {
+  console.log(`Messages use ${result} tokens`);
+}
+```
+
+**Parameters:**
+
+```typescript
+interface IsChatWithinTokenLimitInput {
+  messages: ChatMessage[];
+  model: string;
+  tokenLimit: number;
+  encoding?: OpenAIEncoding;
+  functions?: FunctionDefinition[];
+  function_call?: FunctionCallOption;
+}
+```
+
+**Throws:**
+- `Error` if `tokenLimit` is invalid (NaN, Infinity, negative, non-integer)
+- `Error` if model is not an OpenAI model (unless encoding override provided)
+- `Error` if tools, tool_choice, tool_calls, or tool_call_id are present
+- `Error` if any message has non-string content
+
 ### `getModelConfig(model: string): ModelConfig`
 
 Returns the configuration for a specific model. Throws if the model is not found.

package/dist/index.cjs

@@ -44,6 +44,7 @@ __export(index_exports, {
   decodeSentencePiece: () => decodeSentencePiece,
   decodeSentencePieceAsync: () => decodeSentencePieceAsync,
   encode: () => encode,
+  encodeChat: () => encodeChat,
   encodeSentencePiece: () => encodeSentencePiece,
   encodeSentencePieceAsync: () => encodeSentencePieceAsync,
   ensureSentencePieceModel: () => ensureSentencePieceModel,
@@ -57,6 +58,8 @@ __export(index_exports, {
   getOpenAIEncoding: () => getOpenAIEncoding,
   getSentencePieceTokenizer: () => getSentencePieceTokenizer,
   getTotalCost: () => getTotalCost,
+  isChatWithinTokenLimit: () => isChatWithinTokenLimit,
+  isWithinTokenLimit: () => isWithinTokenLimit,
   loadSentencePieceTokenizer: () => loadSentencePieceTokenizer,
   parseModelProto: () => parseModelProto
 });
@@ -616,6 +619,83 @@ var BPETokenizer = class {
     }
     return tokens;
   }
+  /**
+   * Encode text with a token limit, returning early if the limit is exceeded.
+   * This is optimized for fast token-limit validation without full tokenization.
+   *
+   * @param text - The text to encode
+   * @param limit - Maximum number of tokens allowed
+   * @param allowedSpecial - Controls special token handling (same as encodeText)
+   * @returns Object with count and exceeded flag
+   */
+  encodeTextWithLimit(text, limit, allowedSpecial) {
+    if (!text) return { count: 0, exceeded: false };
+    if (limit < 0) return { count: 0, exceeded: true };
+    if (allowedSpecial === "skip") {
+      return this.encodeOrdinaryWithLimit(text, limit);
+    }
+    let count = 0;
+    if (this.specialTokenMap.size > 0) {
+      const parts = this.splitOnSpecialTokens(text, allowedSpecial);
+      for (const part of parts) {
+        if (part.isSpecial) {
+          count += 1;
+          if (count > limit) return { count, exceeded: true };
+        } else {
+          const result = this.encodeOrdinaryWithLimit(part.text, limit - count);
+          count += result.count;
+          if (result.exceeded) {
+            return { count, exceeded: true };
+          }
+        }
+      }
+    } else {
+      return this.encodeOrdinaryWithLimit(text, limit);
+    }
+    return { count, exceeded: false };
+  }
+  /**
+   * Incremental encoding with early exit.
+   * CRITICAL: Uses RegExp.exec() loop instead of text.match() to avoid
+   * allocating all pieces upfront. This enables true early exit.
+   */
+  encodeOrdinaryWithLimit(text, limit) {
+    if (!text) return { count: 0, exceeded: false };
+    if (limit < 0) return { count: 0, exceeded: true };
+    let count = 0;
+    const regex = new RegExp(
+      this.tokenSplitRegex.source,
+      this.tokenSplitRegex.flags.includes("g") ? this.tokenSplitRegex.flags : this.tokenSplitRegex.flags + "g"
+    );
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+      const piece = match[0];
+      if (piece.length === 0) {
+        regex.lastIndex++;
+        continue;
+      }
+      const cached = this.getFromCache(piece);
+      if (cached) {
+        count += cached.length;
+        if (count > limit) return { count, exceeded: true };
+        continue;
+      }
+      const pieceBytes = this.textEncoder.encode(piece);
+      const key = bytesToLatin1(pieceBytes);
+      const directRank = this.encoder.get(key);
+      if (directRank !== void 0) {
+        count += 1;
+        this.addToCache(piece, [directRank]);
+        if (count > limit) return { count, exceeded: true };
+        continue;
+      }
+      const pieceTokens = this.mergeBytePairs(pieceBytes);
+      count += pieceTokens.length;
+      this.addToCache(piece, pieceTokens);
+      if (count > limit) return { count, exceeded: true };
+    }
+    return { count, exceeded: false };
+  }
   /**
    * Core BPE merge algorithm.
    */
@@ -770,10 +850,18 @@ var CL100K_BASE_SPECIAL_TOKENS = [
   ["<|fim_prefix|>", 100258],
   ["<|fim_middle|>", 100259],
   ["<|fim_suffix|>", 100260],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 100264],
+  ["<|im_end|>", 100265],
+  ["<|im_sep|>", 100266],
   ["<|endofprompt|>", 100276]
 ];
 var O200K_BASE_SPECIAL_TOKENS = [
   ["<|endoftext|>", 199999],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 200264],
+  ["<|im_end|>", 200265],
+  ["<|im_sep|>", 200266],
   ["<|endofprompt|>", 200018]
 ];
 function buildO200kHarmonySpecialTokens() {
@@ -401892,7 +401980,8 @@ function getTokenizer(encoding) {
   }
   return {
     encode: (text, allowedSpecial) => tokenizer.encodeText(text, allowedSpecial),
-    decode: (tokens) => tokenizer.decodeTokens(tokens)
+    decode: (tokens) => tokenizer.decodeTokens(tokens),
+    encodeTextWithLimit: (text, limit, allowedSpecial) => tokenizer.encodeTextWithLimit(text, limit, allowedSpecial)
   };
 }
 function resolveEncoding(options) {
@@ -401932,6 +402021,39 @@ function decode(tokens, options) {
   const api = getTokenizer(encoding);
   return api.decode(tokens);
 }
+function validateTokenLimit(tokenLimit) {
+  if (!Number.isFinite(tokenLimit)) {
+    throw new Error("tokenLimit must be a finite number");
+  }
+  if (!Number.isInteger(tokenLimit)) {
+    throw new Error("tokenLimit must be an integer");
+  }
+  if (tokenLimit < 0) {
+    throw new Error("tokenLimit must be non-negative");
+  }
+}
+function rejectNonOpenAIModel(model) {
+  if (!model) return;
+  if (model.startsWith("claude-")) {
+    throw new Error(
+      `Model "${model}" is an Anthropic model. isWithinTokenLimit only supports OpenAI models. Use the Anthropic API's count_tokens endpoint via estimateAsync() instead.`
+    );
+  }
+  if (model.startsWith("gemini-")) {
+    throw new Error(
+      `Model "${model}" is a Google model. isWithinTokenLimit only supports OpenAI models. Use the Gemini API's countTokens endpoint via estimateAsync() instead.`
+    );
+  }
+}
+function isWithinTokenLimit(text, tokenLimit, options) {
+  validateTokenLimit(tokenLimit);
+  rejectNonOpenAIModel(options?.model);
+  const encoding = resolveEncoding(options);
+  const api = getTokenizer(encoding);
+  const allowedSpecial = resolveAllowedSpecial(options?.allowSpecial);
+  const result = api.encodeTextWithLimit(text, tokenLimit, allowedSpecial);
+  return result.exceeded ? false : result.count;
+}
 
 // src/token-counter.ts
 function isNonOpenAIModel(model) {
@@ -405208,6 +405330,250 @@ function countChatCompletionTokens(input) {
   }
   return result;
 }
+function validateTokenLimit2(tokenLimit) {
+  if (!Number.isFinite(tokenLimit)) {
+    throw new Error("tokenLimit must be a finite number");
+  }
+  if (!Number.isInteger(tokenLimit)) {
+    throw new Error("tokenLimit must be an integer");
+  }
+  if (tokenLimit < 0) {
+    throw new Error("tokenLimit must be non-negative");
+  }
+}
+function isChatWithinTokenLimit(input) {
+  const { messages, model, tokenLimit, encoding, functions, function_call } = input;
+  validateTokenLimit2(tokenLimit);
+  validateNoToolsApi(input);
+  validateMessages(messages);
+  validateOpenAIModel(model, encoding);
+  const resolvedEncoding = encoding ?? getOpenAIEncoding({ model });
+  const api = getTokenizer(resolvedEncoding);
+  let count = COMPLETION_REQUEST_TOKEN_OVERHEAD;
+  if (count > tokenLimit) return false;
+  const hasFunctions = Boolean(functions?.length);
+  const hasSystemMessage = messages.some((m) => m.role === "system");
+  if (hasFunctions && functions) {
+    const formatted = formatFunctionDefinitions(functions);
+    const funcResult = api.encodeTextWithLimit(
+      formatted,
+      tokenLimit - count,
+      "skip"
+    );
+    if (funcResult.exceeded) return false;
+    let funcOverhead = funcResult.count + FUNCTION_DEFINITION_TOKEN_OVERHEAD;
+    if (hasSystemMessage) {
+      funcOverhead -= SYSTEM_FUNCTION_TOKEN_DEDUCTION;
+    }
+    count += funcOverhead;
+    if (count > tokenLimit) return false;
+  }
+  if (function_call && function_call !== "auto") {
+    if (function_call === "none") {
+      count += FUNCTION_CALL_NONE_TOKEN_OVERHEAD;
+    } else if (typeof function_call === "object" && function_call.name) {
+      const fcNameResult = api.encodeTextWithLimit(
+        function_call.name,
+        tokenLimit - count,
+        "skip"
+      );
+      if (fcNameResult.exceeded) return false;
+      count += fcNameResult.count + FUNCTION_CALL_NAME_TOKEN_OVERHEAD;
+    }
+    if (count > tokenLimit) return false;
+  }
+  let systemPadded = false;
+  for (const message of messages) {
+    let overhead = MESSAGE_TOKEN_OVERHEAD;
+    if (message.role) {
+      const roleResult = api.encodeTextWithLimit(
+        message.role,
+        tokenLimit - count,
+        "skip"
+      );
+      if (roleResult.exceeded) return false;
+      count += roleResult.count;
+    }
+    let content = message.content ?? "";
+    if (hasFunctions && message.role === "system" && !systemPadded) {
+      if (content && !content.endsWith("\n")) {
+        content = content + "\n";
+      }
+      systemPadded = true;
+    }
+    if (content) {
+      const contentResult = api.encodeTextWithLimit(
+        content,
+        tokenLimit - count,
+        "skip"
+      );
+      if (contentResult.exceeded) return false;
+      count += contentResult.count;
+    }
+    if (message.name) {
+      const nameResult = api.encodeTextWithLimit(
+        message.name,
+        tokenLimit - count,
+        "skip"
+      );
+      if (nameResult.exceeded) return false;
+      count += nameResult.count;
+      overhead += MESSAGE_NAME_TOKEN_OVERHEAD;
+    }
+    if (message.function_call) {
+      if (message.function_call.name) {
+        const fcNameResult = api.encodeTextWithLimit(
+          message.function_call.name,
+          tokenLimit - count,
+          "skip"
+        );
+        if (fcNameResult.exceeded) return false;
+        count += fcNameResult.count;
+      }
+      if (message.function_call.arguments) {
+        const fcArgsResult = api.encodeTextWithLimit(
+          message.function_call.arguments,
+          tokenLimit - count,
+          "skip"
+        );
+        if (fcArgsResult.exceeded) return false;
+        count += fcArgsResult.count;
+      }
+      overhead += FUNCTION_CALL_METADATA_TOKEN_OVERHEAD;
+    }
+    if (message.role === "function") {
+      overhead -= FUNCTION_ROLE_TOKEN_DISCOUNT;
+    }
+    count += overhead;
+    if (count > tokenLimit) return false;
+  }
+  return count;
+}
+
+// src/encode-chat.ts
+var CHAT_TOKENS = {
+  cl100k_base: { imStart: 100264, imEnd: 100265, imSep: 100266 },
+  o200k_base: { imStart: 200264, imEnd: 200265, imSep: 200266 }
+};
+var HARMONY_TOKENS = {
+  start: 200006,
+  end: 200007,
+  message: 200008
+};
+function encodeChat(messages, options) {
+  const { model, encoding: encodingOverride, primeAssistant = true } = options ?? {};
+  validateChatModel(model, encodingOverride);
+  const encoding = encodingOverride ?? (model ? getOpenAIEncoding({ model }) : "o200k_base");
+  if (encoding === "o200k_harmony") {
+    console.warn(
+      "[ai-token-estimator] o200k_harmony support is experimental. Token structure may not match actual API behavior."
+    );
+  }
+  const chatTokens = getChatTokens(encoding);
+  if (!chatTokens) {
+    throw new Error(
+      `Encoding "${encoding}" does not support chat format. Use cl100k_base or o200k_base for chat models.`
+    );
+  }
+  const { imStart, imEnd, imSep } = chatTokens;
+  const tokens = [];
+  for (const message of messages) {
+    validateMessage(message);
+    tokens.push(imStart);
+    let roleStr;
+    if (message.role === "function" && message.name) {
+      roleStr = message.name;
+    } else if (message.name) {
+      roleStr = `${message.role}:${message.name}`;
+    } else {
+      roleStr = message.role;
+    }
+    tokens.push(...encode(roleStr, { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+    if (message.content) {
+      tokens.push(
+        ...encode(message.content, { encoding, allowSpecial: "none" })
+      );
+    }
+    if (message.function_call) {
+      const fcContent = formatFunctionCall(message.function_call);
+      tokens.push(...encode(fcContent, { encoding, allowSpecial: "none" }));
+    }
+    tokens.push(imEnd);
+  }
+  if (primeAssistant) {
+    tokens.push(imStart);
+    tokens.push(...encode("assistant", { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+  }
+  return tokens;
+}
+function validateChatModel(model, encodingOverride) {
+  if (model) {
+    if (isAnthropicModel(model)) {
+      throw new Error(
+        `Model "${model}" is an Anthropic model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isGoogleModel(model)) {
+      throw new Error(
+        `Model "${model}" is a Google model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isKnownModel(model) && !isChatModel(model)) {
+      throw new Error(
+        `Model "${model}" is not a chat completion model. encodeChat only supports chat models (e.g., gpt-4o, gpt-3.5-turbo).`
+      );
+    }
+  }
+  if (encodingOverride) {
+    return;
+  }
+  if (!model) {
+    throw new Error(
+      "Either model or encoding must be provided. Provide a known OpenAI chat model (e.g., gpt-4o) or an explicit encoding (e.g., o200k_base)."
+    );
+  }
+  if (!isChatModel(model)) {
+    throw new Error(
+      `Model "${model}" is not recognized as an OpenAI chat model. If this is a new OpenAI model, provide the encoding option explicitly (e.g., encoding: "o200k_base").`
+    );
+  }
+}
+function validateMessage(message) {
+  const msgAny = message;
+  if ("tool_calls" in msgAny && msgAny.tool_calls !== void 0) {
+    throw new Error(
+      "tool_calls is not supported. Use function_call with the legacy functions API."
+    );
+  }
+  if ("tool_call_id" in msgAny && msgAny.tool_call_id !== void 0) {
+    throw new Error(
+      "tool_call_id is not supported. Use the legacy functions API."
+    );
+  }
+  if (message.content !== null && message.content !== void 0 && typeof message.content !== "string") {
+    throw new Error(
+      "Multimodal content (arrays) is not supported. Only text content is supported."
+    );
+  }
+}
+function getChatTokens(encoding) {
+  if (encoding === "o200k_harmony") {
+    return {
+      imStart: HARMONY_TOKENS.start,
+      imEnd: HARMONY_TOKENS.end,
+      imSep: HARMONY_TOKENS.message
+    };
+  }
+  return CHAT_TOKENS[encoding] ?? null;
+}
+function formatFunctionCall(fc) {
+  const parts = [];
+  if (fc.name) parts.push(fc.name);
+  if (fc.arguments) parts.push(fc.arguments);
+  return parts.join("\n");
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   DEFAULT_MODELS,
@@ -405224,6 +405590,7 @@ function countChatCompletionTokens(input) {
   decodeSentencePiece,
   decodeSentencePieceAsync,
   encode,
+  encodeChat,
   encodeSentencePiece,
   encodeSentencePieceAsync,
   ensureSentencePieceModel,
@@ -405237,6 +405604,8 @@ function countChatCompletionTokens(input) {
   getOpenAIEncoding,
   getSentencePieceTokenizer,
   getTotalCost,
+  isChatWithinTokenLimit,
+  isWithinTokenLimit,
   loadSentencePieceTokenizer,
   parseModelProto
 });

package/dist/index.d.cts

@@ -55,6 +55,58 @@ declare function encode(text: string, options?: EncodeOptions): number[];
  * Decode OpenAI token IDs into text using tiktoken-compatible BPE encoding.
  */
 declare function decode(tokens: Iterable<number>, options?: Pick<EncodeOptions, 'encoding' | 'model'>): string;
+/**
+ * Options for isWithinTokenLimit.
+ */
+interface IsWithinTokenLimitOptions {
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     * Note: Non-OpenAI models (claude-*, gemini-*) are rejected.
+     */
+    model?: string;
+    /**
+     * How special tokens are handled.
+     * - `none_raise` (default): throw if special tokens appear
+     * - `none`: treat special tokens as regular text
+     * - `all`: allow special tokens and encode them as special token IDs
+     */
+    allowSpecial?: SpecialTokenHandling;
+}
+/**
+ * Check if text is within a token limit, with early exit optimization.
+ *
+ * Returns `false` if the token count exceeds the limit, otherwise returns the
+ * actual token count. This is significantly faster than full tokenization when
+ * the limit is exceeded early in the text.
+ *
+ * @param text - The text to check
+ * @param tokenLimit - Maximum allowed tokens (must be non-negative finite integer)
+ * @param options - Encoding options
+ * @returns `false` if exceeded, or the actual token count if within limit
+ * @throws Error if tokenLimit is invalid (NaN, Infinity, negative, non-integer)
+ * @throws Error if model is a known non-OpenAI model (claude-*, gemini-*)
+ *
+ * @example
+ * ```typescript
+ * // Returns token count if within limit
+ * const count = isWithinTokenLimit('Hello, world!', 100, { model: 'gpt-4o' });
+ * if (count !== false) {
+ *   console.log(`Text has ${count} tokens`);
+ * }
+ *
+ * // Returns false if exceeds limit
+ * const result = isWithinTokenLimit(longText, 10, { model: 'gpt-4o' });
+ * if (result === false) {
+ *   console.log('Text exceeds 10 tokens');
+ * }
+ * ```
+ */
+declare function isWithinTokenLimit(text: string, tokenLimit: number, options?: IsWithinTokenLimitOptions): false | number;
 
 /**
  * Configuration for a specific LLM model.
@@ -542,6 +594,100 @@ declare function countTokens(input: TokenCountInput): TokenCountOutput;
  * ```
  */
 declare function countChatCompletionTokens(input: ChatCompletionTokenCountInput): ChatCompletionTokenCountOutput;
+/**
+ * Input for isChatWithinTokenLimit.
+ * Object-style input to match countChatCompletionTokens API.
+ */
+interface IsChatWithinTokenLimitInput {
+    messages: ChatMessage[];
+    model: string;
+    tokenLimit: number;
+    encoding?: OpenAIEncoding;
+    functions?: FunctionDefinition[];
+    function_call?: FunctionCallOption;
+}
+/**
+ * Check if chat messages are within a token limit, with early exit optimization.
+ *
+ * Uses object-style input to match countChatCompletionTokens API.
+ * Returns `false` if the token count exceeds the limit, otherwise returns
+ * the actual token count.
+ *
+ * This is significantly faster than full tokenization when the limit is
+ * exceeded early in the input.
+ *
+ * @throws {Error} If tokenLimit is invalid (NaN, Infinity, negative, non-integer)
+ * @throws {Error} If model is not an OpenAI model (unless encoding override provided)
+ * @throws {Error} If tools, tool_choice, tool_calls, or tool_call_id are present
+ * @throws {Error} If any message has non-string content (arrays, numbers, objects)
+ *
+ * @example
+ * ```typescript
+ * const result = isChatWithinTokenLimit({
+ *   messages: [
+ *     { role: 'system', content: 'You are a helpful assistant.' },
+ *     { role: 'user', content: 'Hello!' }
+ *   ],
+ *   model: 'gpt-4o',
+ *   tokenLimit: 100,
+ * });
+ *
+ * if (result === false) {
+ *   console.log('Messages exceed token limit');
+ * } else {
+ *   console.log(`Messages use ${result} tokens`);
+ * }
+ * ```
+ */
+declare function isChatWithinTokenLimit(input: IsChatWithinTokenLimitInput): false | number;
+
+/**
+ * Chat-aware tokenization using ChatML format.
+ *
+ * Encodes chat messages into ChatML message prompt tokens including special
+ * delimiter tokens (<|im_start|>, <|im_sep|>, <|im_end|>).
+ */
+
+/**
+ * Options for encodeChat.
+ */
+interface EncodeChatOptions {
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     * Note: Non-OpenAI models (claude-*, gemini-*) are rejected.
+     */
+    model?: string;
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * Prime the output with the start of an assistant response.
+     * When true (default), appends <|im_start|>assistant<|im_sep|> at the end.
+     * Set to false to get just the messages without assistant priming.
+     */
+    primeAssistant?: boolean;
+}
+/**
+ * Encode chat messages into token IDs using ChatML format.
+ *
+ * Returns the exact token sequence that OpenAI models expect for chat
+ * completions, including special delimiter tokens.
+ *
+ * @param messages - Array of chat messages
+ * @param options - Encoding options
+ * @returns Token IDs representing the chat prompt
+ *
+ * @example
+ * ```typescript
+ * const tokens = encodeChat([
+ *   { role: 'system', content: 'You are helpful.' },
+ *   { role: 'user', content: 'Hello!' }
+ * ], { model: 'gpt-4o' });
+ * ```
+ */
+declare function encodeChat(messages: ChatMessage[], options?: EncodeChatOptions): number[];
 
 interface AnthropicCountTokensParams {
     /** Claude model id, e.g. `claude-sonnet-4-5` */
@@ -818,4 +964,4 @@ declare function clearModelCache(): void;
  */
 declare function parseModelProto(buffer: Uint8Array): ModelProto;
 
-export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, loadSentencePieceTokenizer, parseModelProto };
+export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeChatOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type IsChatWithinTokenLimitInput, type IsWithinTokenLimitOptions, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeChat, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, isChatWithinTokenLimit, isWithinTokenLimit, loadSentencePieceTokenizer, parseModelProto };

package/dist/index.d.ts

@@ -55,6 +55,58 @@ declare function encode(text: string, options?: EncodeOptions): number[];
  * Decode OpenAI token IDs into text using tiktoken-compatible BPE encoding.
  */
 declare function decode(tokens: Iterable<number>, options?: Pick<EncodeOptions, 'encoding' | 'model'>): string;
+/**
+ * Options for isWithinTokenLimit.
+ */
+interface IsWithinTokenLimitOptions {
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     * Note: Non-OpenAI models (claude-*, gemini-*) are rejected.
+     */
+    model?: string;
+    /**
+     * How special tokens are handled.
+     * - `none_raise` (default): throw if special tokens appear
+     * - `none`: treat special tokens as regular text
+     * - `all`: allow special tokens and encode them as special token IDs
+     */
+    allowSpecial?: SpecialTokenHandling;
+}
+/**
+ * Check if text is within a token limit, with early exit optimization.
+ *
+ * Returns `false` if the token count exceeds the limit, otherwise returns the
+ * actual token count. This is significantly faster than full tokenization when
+ * the limit is exceeded early in the text.
+ *
+ * @param text - The text to check
+ * @param tokenLimit - Maximum allowed tokens (must be non-negative finite integer)
+ * @param options - Encoding options
+ * @returns `false` if exceeded, or the actual token count if within limit
+ * @throws Error if tokenLimit is invalid (NaN, Infinity, negative, non-integer)
+ * @throws Error if model is a known non-OpenAI model (claude-*, gemini-*)
+ *
+ * @example
+ * ```typescript
+ * // Returns token count if within limit
+ * const count = isWithinTokenLimit('Hello, world!', 100, { model: 'gpt-4o' });
+ * if (count !== false) {
+ *   console.log(`Text has ${count} tokens`);
+ * }
+ *
+ * // Returns false if exceeds limit
+ * const result = isWithinTokenLimit(longText, 10, { model: 'gpt-4o' });
+ * if (result === false) {
+ *   console.log('Text exceeds 10 tokens');
+ * }
+ * ```
+ */
+declare function isWithinTokenLimit(text: string, tokenLimit: number, options?: IsWithinTokenLimitOptions): false | number;
 
 /**
  * Configuration for a specific LLM model.
@@ -542,6 +594,100 @@ declare function countTokens(input: TokenCountInput): TokenCountOutput;
  * ```
  */
 declare function countChatCompletionTokens(input: ChatCompletionTokenCountInput): ChatCompletionTokenCountOutput;
+/**
+ * Input for isChatWithinTokenLimit.
+ * Object-style input to match countChatCompletionTokens API.
+ */
+interface IsChatWithinTokenLimitInput {
+    messages: ChatMessage[];
+    model: string;
+    tokenLimit: number;
+    encoding?: OpenAIEncoding;
+    functions?: FunctionDefinition[];
+    function_call?: FunctionCallOption;
+}
+/**
+ * Check if chat messages are within a token limit, with early exit optimization.
+ *
+ * Uses object-style input to match countChatCompletionTokens API.
+ * Returns `false` if the token count exceeds the limit, otherwise returns
+ * the actual token count.
+ *
+ * This is significantly faster than full tokenization when the limit is
+ * exceeded early in the input.
+ *
+ * @throws {Error} If tokenLimit is invalid (NaN, Infinity, negative, non-integer)
+ * @throws {Error} If model is not an OpenAI model (unless encoding override provided)
+ * @throws {Error} If tools, tool_choice, tool_calls, or tool_call_id are present
+ * @throws {Error} If any message has non-string content (arrays, numbers, objects)
+ *
+ * @example
+ * ```typescript
+ * const result = isChatWithinTokenLimit({
+ *   messages: [
+ *     { role: 'system', content: 'You are a helpful assistant.' },
+ *     { role: 'user', content: 'Hello!' }
+ *   ],
+ *   model: 'gpt-4o',
+ *   tokenLimit: 100,
+ * });
+ *
+ * if (result === false) {
+ *   console.log('Messages exceed token limit');
+ * } else {
+ *   console.log(`Messages use ${result} tokens`);
+ * }
+ * ```
+ */
+declare function isChatWithinTokenLimit(input: IsChatWithinTokenLimitInput): false | number;
+
+/**
+ * Chat-aware tokenization using ChatML format.
+ *
+ * Encodes chat messages into ChatML message prompt tokens including special
+ * delimiter tokens (<|im_start|>, <|im_sep|>, <|im_end|>).
+ */
+
+/**
+ * Options for encodeChat.
+ */
+interface EncodeChatOptions {
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     * Note: Non-OpenAI models (claude-*, gemini-*) are rejected.
+     */
+    model?: string;
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * Prime the output with the start of an assistant response.
+     * When true (default), appends <|im_start|>assistant<|im_sep|> at the end.
+     * Set to false to get just the messages without assistant priming.
+     */
+    primeAssistant?: boolean;
+}
+/**
+ * Encode chat messages into token IDs using ChatML format.
+ *
+ * Returns the exact token sequence that OpenAI models expect for chat
+ * completions, including special delimiter tokens.
+ *
+ * @param messages - Array of chat messages
+ * @param options - Encoding options
+ * @returns Token IDs representing the chat prompt
+ *
+ * @example
+ * ```typescript
+ * const tokens = encodeChat([
+ *   { role: 'system', content: 'You are helpful.' },
+ *   { role: 'user', content: 'Hello!' }
+ * ], { model: 'gpt-4o' });
+ * ```
+ */
+declare function encodeChat(messages: ChatMessage[], options?: EncodeChatOptions): number[];
 
 interface AnthropicCountTokensParams {
     /** Claude model id, e.g. `claude-sonnet-4-5` */
@@ -818,4 +964,4 @@ declare function clearModelCache(): void;
  */
 declare function parseModelProto(buffer: Uint8Array): ModelProto;
 
-export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, loadSentencePieceTokenizer, parseModelProto };
+export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeChatOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type IsChatWithinTokenLimitInput, type IsWithinTokenLimitOptions, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeChat, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, isChatWithinTokenLimit, isWithinTokenLimit, loadSentencePieceTokenizer, parseModelProto };

package/dist/index.js

@@ -552,6 +552,83 @@ var BPETokenizer = class {
     }
     return tokens;
   }
+  /**
+   * Encode text with a token limit, returning early if the limit is exceeded.
+   * This is optimized for fast token-limit validation without full tokenization.
+   *
+   * @param text - The text to encode
+   * @param limit - Maximum number of tokens allowed
+   * @param allowedSpecial - Controls special token handling (same as encodeText)
+   * @returns Object with count and exceeded flag
+   */
+  encodeTextWithLimit(text, limit, allowedSpecial) {
+    if (!text) return { count: 0, exceeded: false };
+    if (limit < 0) return { count: 0, exceeded: true };
+    if (allowedSpecial === "skip") {
+      return this.encodeOrdinaryWithLimit(text, limit);
+    }
+    let count = 0;
+    if (this.specialTokenMap.size > 0) {
+      const parts = this.splitOnSpecialTokens(text, allowedSpecial);
+      for (const part of parts) {
+        if (part.isSpecial) {
+          count += 1;
+          if (count > limit) return { count, exceeded: true };
+        } else {
+          const result = this.encodeOrdinaryWithLimit(part.text, limit - count);
+          count += result.count;
+          if (result.exceeded) {
+            return { count, exceeded: true };
+          }
+        }
+      }
+    } else {
+      return this.encodeOrdinaryWithLimit(text, limit);
+    }
+    return { count, exceeded: false };
+  }
+  /**
+   * Incremental encoding with early exit.
+   * CRITICAL: Uses RegExp.exec() loop instead of text.match() to avoid
+   * allocating all pieces upfront. This enables true early exit.
+   */
+  encodeOrdinaryWithLimit(text, limit) {
+    if (!text) return { count: 0, exceeded: false };
+    if (limit < 0) return { count: 0, exceeded: true };
+    let count = 0;
+    const regex = new RegExp(
+      this.tokenSplitRegex.source,
+      this.tokenSplitRegex.flags.includes("g") ? this.tokenSplitRegex.flags : this.tokenSplitRegex.flags + "g"
+    );
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+      const piece = match[0];
+      if (piece.length === 0) {
+        regex.lastIndex++;
+        continue;
+      }
+      const cached = this.getFromCache(piece);
+      if (cached) {
+        count += cached.length;
+        if (count > limit) return { count, exceeded: true };
+        continue;
+      }
+      const pieceBytes = this.textEncoder.encode(piece);
+      const key = bytesToLatin1(pieceBytes);
+      const directRank = this.encoder.get(key);
+      if (directRank !== void 0) {
+        count += 1;
+        this.addToCache(piece, [directRank]);
+        if (count > limit) return { count, exceeded: true };
+        continue;
+      }
+      const pieceTokens = this.mergeBytePairs(pieceBytes);
+      count += pieceTokens.length;
+      this.addToCache(piece, pieceTokens);
+      if (count > limit) return { count, exceeded: true };
+    }
+    return { count, exceeded: false };
+  }
   /**
    * Core BPE merge algorithm.
    */
@@ -706,10 +783,18 @@ var CL100K_BASE_SPECIAL_TOKENS = [
   ["<|fim_prefix|>", 100258],
   ["<|fim_middle|>", 100259],
   ["<|fim_suffix|>", 100260],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 100264],
+  ["<|im_end|>", 100265],
+  ["<|im_sep|>", 100266],
   ["<|endofprompt|>", 100276]
 ];
 var O200K_BASE_SPECIAL_TOKENS = [
   ["<|endoftext|>", 199999],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 200264],
+  ["<|im_end|>", 200265],
+  ["<|im_sep|>", 200266],
   ["<|endofprompt|>", 200018]
 ];
 function buildO200kHarmonySpecialTokens() {
@@ -401828,7 +401913,8 @@ function getTokenizer(encoding) {
   }
   return {
     encode: (text, allowedSpecial) => tokenizer.encodeText(text, allowedSpecial),
-    decode: (tokens) => tokenizer.decodeTokens(tokens)
+    decode: (tokens) => tokenizer.decodeTokens(tokens),
+    encodeTextWithLimit: (text, limit, allowedSpecial) => tokenizer.encodeTextWithLimit(text, limit, allowedSpecial)
   };
 }
 function resolveEncoding(options) {
@@ -401868,6 +401954,39 @@ function decode(tokens, options) {
   const api = getTokenizer(encoding);
   return api.decode(tokens);
 }
+function validateTokenLimit(tokenLimit) {
+  if (!Number.isFinite(tokenLimit)) {
+    throw new Error("tokenLimit must be a finite number");
+  }
+  if (!Number.isInteger(tokenLimit)) {
+    throw new Error("tokenLimit must be an integer");
+  }
+  if (tokenLimit < 0) {
+    throw new Error("tokenLimit must be non-negative");
+  }
+}
+function rejectNonOpenAIModel(model) {
+  if (!model) return;
+  if (model.startsWith("claude-")) {
+    throw new Error(
+      `Model "${model}" is an Anthropic model. isWithinTokenLimit only supports OpenAI models. Use the Anthropic API's count_tokens endpoint via estimateAsync() instead.`
+    );
+  }
+  if (model.startsWith("gemini-")) {
+    throw new Error(
+      `Model "${model}" is a Google model. isWithinTokenLimit only supports OpenAI models. Use the Gemini API's countTokens endpoint via estimateAsync() instead.`
+    );
+  }
+}
+function isWithinTokenLimit(text, tokenLimit, options) {
+  validateTokenLimit(tokenLimit);
+  rejectNonOpenAIModel(options?.model);
+  const encoding = resolveEncoding(options);
+  const api = getTokenizer(encoding);
+  const allowedSpecial = resolveAllowedSpecial(options?.allowSpecial);
+  const result = api.encodeTextWithLimit(text, tokenLimit, allowedSpecial);
+  return result.exceeded ? false : result.count;
+}
 
 // src/token-counter.ts
 function isNonOpenAIModel(model) {
@@ -405144,6 +405263,250 @@ function countChatCompletionTokens(input) {
   }
   return result;
 }
+function validateTokenLimit2(tokenLimit) {
+  if (!Number.isFinite(tokenLimit)) {
+    throw new Error("tokenLimit must be a finite number");
+  }
+  if (!Number.isInteger(tokenLimit)) {
+    throw new Error("tokenLimit must be an integer");
+  }
+  if (tokenLimit < 0) {
+    throw new Error("tokenLimit must be non-negative");
+  }
+}
+function isChatWithinTokenLimit(input) {
+  const { messages, model, tokenLimit, encoding, functions, function_call } = input;
+  validateTokenLimit2(tokenLimit);
+  validateNoToolsApi(input);
+  validateMessages(messages);
+  validateOpenAIModel(model, encoding);
+  const resolvedEncoding = encoding ?? getOpenAIEncoding({ model });
+  const api = getTokenizer(resolvedEncoding);
+  let count = COMPLETION_REQUEST_TOKEN_OVERHEAD;
+  if (count > tokenLimit) return false;
+  const hasFunctions = Boolean(functions?.length);
+  const hasSystemMessage = messages.some((m) => m.role === "system");
+  if (hasFunctions && functions) {
+    const formatted = formatFunctionDefinitions(functions);
+    const funcResult = api.encodeTextWithLimit(
+      formatted,
+      tokenLimit - count,
+      "skip"
+    );
+    if (funcResult.exceeded) return false;
+    let funcOverhead = funcResult.count + FUNCTION_DEFINITION_TOKEN_OVERHEAD;
+    if (hasSystemMessage) {
+      funcOverhead -= SYSTEM_FUNCTION_TOKEN_DEDUCTION;
+    }
+    count += funcOverhead;
+    if (count > tokenLimit) return false;
+  }
+  if (function_call && function_call !== "auto") {
+    if (function_call === "none") {
+      count += FUNCTION_CALL_NONE_TOKEN_OVERHEAD;
+    } else if (typeof function_call === "object" && function_call.name) {
+      const fcNameResult = api.encodeTextWithLimit(
+        function_call.name,
+        tokenLimit - count,
+        "skip"
+      );
+      if (fcNameResult.exceeded) return false;
+      count += fcNameResult.count + FUNCTION_CALL_NAME_TOKEN_OVERHEAD;
+    }
+    if (count > tokenLimit) return false;
+  }
+  let systemPadded = false;
+  for (const message of messages) {
+    let overhead = MESSAGE_TOKEN_OVERHEAD;
+    if (message.role) {
+      const roleResult = api.encodeTextWithLimit(
+        message.role,
+        tokenLimit - count,
+        "skip"
+      );
+      if (roleResult.exceeded) return false;
+      count += roleResult.count;
+    }
+    let content = message.content ?? "";
+    if (hasFunctions && message.role === "system" && !systemPadded) {
+      if (content && !content.endsWith("\n")) {
+        content = content + "\n";
+      }
+      systemPadded = true;
+    }
+    if (content) {
+      const contentResult = api.encodeTextWithLimit(
+        content,
+        tokenLimit - count,
+        "skip"
+      );
+      if (contentResult.exceeded) return false;
+      count += contentResult.count;
+    }
+    if (message.name) {
+      const nameResult = api.encodeTextWithLimit(
+        message.name,
+        tokenLimit - count,
+        "skip"
+      );
+      if (nameResult.exceeded) return false;
+      count += nameResult.count;
+      overhead += MESSAGE_NAME_TOKEN_OVERHEAD;
+    }
+    if (message.function_call) {
+      if (message.function_call.name) {
+        const fcNameResult = api.encodeTextWithLimit(
+          message.function_call.name,
+          tokenLimit - count,
+          "skip"
+        );
+        if (fcNameResult.exceeded) return false;
+        count += fcNameResult.count;
+      }
+      if (message.function_call.arguments) {
+        const fcArgsResult = api.encodeTextWithLimit(
+          message.function_call.arguments,
+          tokenLimit - count,
+          "skip"
+        );
+        if (fcArgsResult.exceeded) return false;
+        count += fcArgsResult.count;
+      }
+      overhead += FUNCTION_CALL_METADATA_TOKEN_OVERHEAD;
+    }
+    if (message.role === "function") {
+      overhead -= FUNCTION_ROLE_TOKEN_DISCOUNT;
+    }
+    count += overhead;
+    if (count > tokenLimit) return false;
+  }
+  return count;
+}
+
+// src/encode-chat.ts
+var CHAT_TOKENS = {
+  cl100k_base: { imStart: 100264, imEnd: 100265, imSep: 100266 },
+  o200k_base: { imStart: 200264, imEnd: 200265, imSep: 200266 }
+};
+var HARMONY_TOKENS = {
+  start: 200006,
+  end: 200007,
+  message: 200008
+};
+function encodeChat(messages, options) {
+  const { model, encoding: encodingOverride, primeAssistant = true } = options ?? {};
+  validateChatModel(model, encodingOverride);
+  const encoding = encodingOverride ?? (model ? getOpenAIEncoding({ model }) : "o200k_base");
+  if (encoding === "o200k_harmony") {
+    console.warn(
+      "[ai-token-estimator] o200k_harmony support is experimental. Token structure may not match actual API behavior."
+    );
+  }
+  const chatTokens = getChatTokens(encoding);
+  if (!chatTokens) {
+    throw new Error(
+      `Encoding "${encoding}" does not support chat format. Use cl100k_base or o200k_base for chat models.`
+    );
+  }
+  const { imStart, imEnd, imSep } = chatTokens;
+  const tokens = [];
+  for (const message of messages) {
+    validateMessage(message);
+    tokens.push(imStart);
+    let roleStr;
+    if (message.role === "function" && message.name) {
+      roleStr = message.name;
+    } else if (message.name) {
+      roleStr = `${message.role}:${message.name}`;
+    } else {
+      roleStr = message.role;
+    }
+    tokens.push(...encode(roleStr, { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+    if (message.content) {
+      tokens.push(
+        ...encode(message.content, { encoding, allowSpecial: "none" })
+      );
+    }
+    if (message.function_call) {
+      const fcContent = formatFunctionCall(message.function_call);
+      tokens.push(...encode(fcContent, { encoding, allowSpecial: "none" }));
+    }
+    tokens.push(imEnd);
+  }
+  if (primeAssistant) {
+    tokens.push(imStart);
+    tokens.push(...encode("assistant", { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+  }
+  return tokens;
+}
+function validateChatModel(model, encodingOverride) {
+  if (model) {
+    if (isAnthropicModel(model)) {
+      throw new Error(
+        `Model "${model}" is an Anthropic model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isGoogleModel(model)) {
+      throw new Error(
+        `Model "${model}" is a Google model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isKnownModel(model) && !isChatModel(model)) {
+      throw new Error(
+        `Model "${model}" is not a chat completion model. encodeChat only supports chat models (e.g., gpt-4o, gpt-3.5-turbo).`
+      );
+    }
+  }
+  if (encodingOverride) {
+    return;
+  }
+  if (!model) {
+    throw new Error(
+      "Either model or encoding must be provided. Provide a known OpenAI chat model (e.g., gpt-4o) or an explicit encoding (e.g., o200k_base)."
+    );
+  }
+  if (!isChatModel(model)) {
+    throw new Error(
+      `Model "${model}" is not recognized as an OpenAI chat model. If this is a new OpenAI model, provide the encoding option explicitly (e.g., encoding: "o200k_base").`
+    );
+  }
+}
+function validateMessage(message) {
+  const msgAny = message;
+  if ("tool_calls" in msgAny && msgAny.tool_calls !== void 0) {
+    throw new Error(
+      "tool_calls is not supported. Use function_call with the legacy functions API."
+    );
+  }
+  if ("tool_call_id" in msgAny && msgAny.tool_call_id !== void 0) {
+    throw new Error(
+      "tool_call_id is not supported. Use the legacy functions API."
+    );
+  }
+  if (message.content !== null && message.content !== void 0 && typeof message.content !== "string") {
+    throw new Error(
+      "Multimodal content (arrays) is not supported. Only text content is supported."
+    );
+  }
+}
+function getChatTokens(encoding) {
+  if (encoding === "o200k_harmony") {
+    return {
+      imStart: HARMONY_TOKENS.start,
+      imEnd: HARMONY_TOKENS.end,
+      imSep: HARMONY_TOKENS.message
+    };
+  }
+  return CHAT_TOKENS[encoding] ?? null;
+}
+function formatFunctionCall(fc) {
+  const parts = [];
+  if (fc.name) parts.push(fc.name);
+  if (fc.arguments) parts.push(fc.arguments);
+  return parts.join("\n");
+}
 export {
   DEFAULT_MODELS,
   LAST_UPDATED,
@@ -405159,6 +405522,7 @@ export {
   decodeSentencePiece,
   decodeSentencePieceAsync,
   encode,
+  encodeChat,
   encodeSentencePiece,
   encodeSentencePieceAsync,
   ensureSentencePieceModel,
@@ -405172,6 +405536,8 @@ export {
   getOpenAIEncoding,
   getSentencePieceTokenizer,
   getTotalCost,
+  isChatWithinTokenLimit,
+  isWithinTokenLimit,
   loadSentencePieceTokenizer,
   parseModelProto
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-token-estimator",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "Estimate and count tokens (incl. exact OpenAI BPE) and input costs for LLM API calls",
   "type": "module",
   "main": "./dist/index.cjs",