ai-token-estimator 1.5.0 → 1.6.0

package/README.md

@@ -11,6 +11,7 @@ The best way to estimate **tokens + input cost** for LLM calls — with **exact
 ## Features
 
 - **Exact OpenAI tokenization** (tiktoken-compatible BPE): `encode()` / `decode()` / `openai_exact`
+- **Chat-aware tokenization**: `encodeChat()` returns exact token IDs for chat messages using ChatML format
 - **OpenAI chat completion token counting** (legacy `functions` API): `countChatCompletionTokens()` with optional per-message breakdown
 - **Pure TypeScript SentencePiece tokenizer** (no native dependencies):
   - Supports `.model` files (protobuf format)
@@ -64,6 +65,21 @@ console.log(countTokens({ text: 'Hello, world!', model: 'gpt-5.1' }));
 
 ## Quick Recipes
 
+### Encode chat messages to tokens (ChatML format)
+
+```ts
+import { encodeChat, decode } from 'ai-token-estimator';
+
+const tokens = encodeChat([
+  { role: 'system', content: 'You are helpful.' },
+  { role: 'user', content: 'Hello!' }
+], { model: 'gpt-4o' });
+
+console.log(tokens); // [200264, 9125, 200266, 2610, 525, 11190, 13, 200265, ...]
+console.log(decode(tokens, { encoding: 'o200k_base' }));
+// <|im_start|>system<|im_sep|>You are helpful.<|im_end|>...
+```
+
 ### OpenAI chat completion tokens (legacy functions API)
 
 ```ts
@@ -561,6 +577,48 @@ Encodes text into **OpenAI token IDs** using tiktoken-compatible BPE tokenizatio
 
 Decodes OpenAI token IDs back into text using the selected encoding/model.
 
+### `encodeChat(messages: ChatMessage[], options?: EncodeChatOptions): number[]`
+
+Encodes chat messages into **exact token IDs** using ChatML format. Returns the ChatML message prompt tokens (messages + optional assistant priming), including special delimiter tokens (`<|im_start|>`, `<|im_sep|>`, `<|im_end|>`).
+
+```ts
+import { encodeChat, decode } from 'ai-token-estimator';
+
+const tokens = encodeChat([
+  { role: 'system', content: 'You are helpful.' },
+  { role: 'user', content: 'Hello!' }
+], { model: 'gpt-4o' });
+
+// Tokens include ChatML structure:
+// <|im_start|>system<|im_sep|>You are helpful.<|im_end|>
+// <|im_start|>user<|im_sep|>Hello!<|im_end|>
+// <|im_start|>assistant<|im_sep|>  (priming)
+```
+
+**Parameters:**
+
+```typescript
+interface EncodeChatOptions {
+  model?: string;              // OpenAI model (e.g., 'gpt-4o')
+  encoding?: OpenAIEncoding;   // Explicit encoding override
+  primeAssistant?: boolean;    // Append assistant priming (default: true)
+}
+```
+
+**Supported encodings:**
+- `cl100k_base` (GPT-4, GPT-3.5-turbo)
+- `o200k_base` (GPT-4o, GPT-4o-mini)
+- `o200k_harmony` (experimental)
+
+**Limitations:**
+- **OpenAI models only** — throws for claude-*, gemini-*
+- **Legacy functions API only** — throws for tool_calls, tool_call_id
+- **Text content only** — throws for multimodal content (arrays)
+
+**Note on function_call:** Messages with `function_call` are encoded with the function name and arguments as content. The token count differs from `countChatCompletionTokens()` because the latter includes `FUNCTION_CALL_METADATA_TOKEN_OVERHEAD` (3 tokens) for API accounting. The exact difference depends on whether both name and arguments are present (2 token difference due to newline separator) or only one field is present (3 token difference).
+
+**Note on o200k_harmony:** Support for `o200k_harmony` encoding is experimental. The token structure may not match actual API behavior.
+
 ### `isWithinTokenLimit(text, tokenLimit, options?): false | number`
 
 Checks if text is within a token limit with **early exit optimization**. Returns `false` if the limit is exceeded, or the actual token count if within limit.

package/dist/index.cjs

@@ -44,6 +44,7 @@ __export(index_exports, {
   decodeSentencePiece: () => decodeSentencePiece,
   decodeSentencePieceAsync: () => decodeSentencePieceAsync,
   encode: () => encode,
+  encodeChat: () => encodeChat,
   encodeSentencePiece: () => encodeSentencePiece,
   encodeSentencePieceAsync: () => encodeSentencePieceAsync,
   ensureSentencePieceModel: () => ensureSentencePieceModel,
@@ -849,10 +850,18 @@ var CL100K_BASE_SPECIAL_TOKENS = [
   ["<|fim_prefix|>", 100258],
   ["<|fim_middle|>", 100259],
   ["<|fim_suffix|>", 100260],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 100264],
+  ["<|im_end|>", 100265],
+  ["<|im_sep|>", 100266],
   ["<|endofprompt|>", 100276]
 ];
 var O200K_BASE_SPECIAL_TOKENS = [
   ["<|endoftext|>", 199999],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 200264],
+  ["<|im_end|>", 200265],
+  ["<|im_sep|>", 200266],
   ["<|endofprompt|>", 200018]
 ];
 function buildO200kHarmonySpecialTokens() {
@@ -405440,6 +405449,131 @@ function isChatWithinTokenLimit(input) {
   }
   return count;
 }
+
+// src/encode-chat.ts
+var CHAT_TOKENS = {
+  cl100k_base: { imStart: 100264, imEnd: 100265, imSep: 100266 },
+  o200k_base: { imStart: 200264, imEnd: 200265, imSep: 200266 }
+};
+var HARMONY_TOKENS = {
+  start: 200006,
+  end: 200007,
+  message: 200008
+};
+function encodeChat(messages, options) {
+  const { model, encoding: encodingOverride, primeAssistant = true } = options ?? {};
+  validateChatModel(model, encodingOverride);
+  const encoding = encodingOverride ?? (model ? getOpenAIEncoding({ model }) : "o200k_base");
+  if (encoding === "o200k_harmony") {
+    console.warn(
+      "[ai-token-estimator] o200k_harmony support is experimental. Token structure may not match actual API behavior."
+    );
+  }
+  const chatTokens = getChatTokens(encoding);
+  if (!chatTokens) {
+    throw new Error(
+      `Encoding "${encoding}" does not support chat format. Use cl100k_base or o200k_base for chat models.`
+    );
+  }
+  const { imStart, imEnd, imSep } = chatTokens;
+  const tokens = [];
+  for (const message of messages) {
+    validateMessage(message);
+    tokens.push(imStart);
+    let roleStr;
+    if (message.role === "function" && message.name) {
+      roleStr = message.name;
+    } else if (message.name) {
+      roleStr = `${message.role}:${message.name}`;
+    } else {
+      roleStr = message.role;
+    }
+    tokens.push(...encode(roleStr, { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+    if (message.content) {
+      tokens.push(
+        ...encode(message.content, { encoding, allowSpecial: "none" })
+      );
+    }
+    if (message.function_call) {
+      const fcContent = formatFunctionCall(message.function_call);
+      tokens.push(...encode(fcContent, { encoding, allowSpecial: "none" }));
+    }
+    tokens.push(imEnd);
+  }
+  if (primeAssistant) {
+    tokens.push(imStart);
+    tokens.push(...encode("assistant", { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+  }
+  return tokens;
+}
+function validateChatModel(model, encodingOverride) {
+  if (model) {
+    if (isAnthropicModel(model)) {
+      throw new Error(
+        `Model "${model}" is an Anthropic model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isGoogleModel(model)) {
+      throw new Error(
+        `Model "${model}" is a Google model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isKnownModel(model) && !isChatModel(model)) {
+      throw new Error(
+        `Model "${model}" is not a chat completion model. encodeChat only supports chat models (e.g., gpt-4o, gpt-3.5-turbo).`
+      );
+    }
+  }
+  if (encodingOverride) {
+    return;
+  }
+  if (!model) {
+    throw new Error(
+      "Either model or encoding must be provided. Provide a known OpenAI chat model (e.g., gpt-4o) or an explicit encoding (e.g., o200k_base)."
+    );
+  }
+  if (!isChatModel(model)) {
+    throw new Error(
+      `Model "${model}" is not recognized as an OpenAI chat model. If this is a new OpenAI model, provide the encoding option explicitly (e.g., encoding: "o200k_base").`
+    );
+  }
+}
+function validateMessage(message) {
+  const msgAny = message;
+  if ("tool_calls" in msgAny && msgAny.tool_calls !== void 0) {
+    throw new Error(
+      "tool_calls is not supported. Use function_call with the legacy functions API."
+    );
+  }
+  if ("tool_call_id" in msgAny && msgAny.tool_call_id !== void 0) {
+    throw new Error(
+      "tool_call_id is not supported. Use the legacy functions API."
+    );
+  }
+  if (message.content !== null && message.content !== void 0 && typeof message.content !== "string") {
+    throw new Error(
+      "Multimodal content (arrays) is not supported. Only text content is supported."
+    );
+  }
+}
+function getChatTokens(encoding) {
+  if (encoding === "o200k_harmony") {
+    return {
+      imStart: HARMONY_TOKENS.start,
+      imEnd: HARMONY_TOKENS.end,
+      imSep: HARMONY_TOKENS.message
+    };
+  }
+  return CHAT_TOKENS[encoding] ?? null;
+}
+function formatFunctionCall(fc) {
+  const parts = [];
+  if (fc.name) parts.push(fc.name);
+  if (fc.arguments) parts.push(fc.arguments);
+  return parts.join("\n");
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   DEFAULT_MODELS,
@@ -405456,6 +405590,7 @@ function isChatWithinTokenLimit(input) {
   decodeSentencePiece,
   decodeSentencePieceAsync,
   encode,
+  encodeChat,
   encodeSentencePiece,
   encodeSentencePieceAsync,
   ensureSentencePieceModel,

package/dist/index.d.cts

@@ -641,6 +641,54 @@ interface IsChatWithinTokenLimitInput {
  */
 declare function isChatWithinTokenLimit(input: IsChatWithinTokenLimitInput): false | number;
 
+/**
+ * Chat-aware tokenization using ChatML format.
+ *
+ * Encodes chat messages into ChatML message prompt tokens including special
+ * delimiter tokens (<|im_start|>, <|im_sep|>, <|im_end|>).
+ */
+
+/**
+ * Options for encodeChat.
+ */
+interface EncodeChatOptions {
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     * Note: Non-OpenAI models (claude-*, gemini-*) are rejected.
+     */
+    model?: string;
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * Prime the output with the start of an assistant response.
+     * When true (default), appends <|im_start|>assistant<|im_sep|> at the end.
+     * Set to false to get just the messages without assistant priming.
+     */
+    primeAssistant?: boolean;
+}
+/**
+ * Encode chat messages into token IDs using ChatML format.
+ *
+ * Returns the exact token sequence that OpenAI models expect for chat
+ * completions, including special delimiter tokens.
+ *
+ * @param messages - Array of chat messages
+ * @param options - Encoding options
+ * @returns Token IDs representing the chat prompt
+ *
+ * @example
+ * ```typescript
+ * const tokens = encodeChat([
+ *   { role: 'system', content: 'You are helpful.' },
+ *   { role: 'user', content: 'Hello!' }
+ * ], { model: 'gpt-4o' });
+ * ```
+ */
+declare function encodeChat(messages: ChatMessage[], options?: EncodeChatOptions): number[];
+
 interface AnthropicCountTokensParams {
     /** Claude model id, e.g. `claude-sonnet-4-5` */
     model: string;
@@ -916,4 +964,4 @@ declare function clearModelCache(): void;
  */
 declare function parseModelProto(buffer: Uint8Array): ModelProto;
 
-export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type IsChatWithinTokenLimitInput, type IsWithinTokenLimitOptions, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, isChatWithinTokenLimit, isWithinTokenLimit, loadSentencePieceTokenizer, parseModelProto };
+export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeChatOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type IsChatWithinTokenLimitInput, type IsWithinTokenLimitOptions, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeChat, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, isChatWithinTokenLimit, isWithinTokenLimit, loadSentencePieceTokenizer, parseModelProto };

package/dist/index.d.ts

@@ -641,6 +641,54 @@ interface IsChatWithinTokenLimitInput {
  */
 declare function isChatWithinTokenLimit(input: IsChatWithinTokenLimitInput): false | number;
 
+/**
+ * Chat-aware tokenization using ChatML format.
+ *
+ * Encodes chat messages into ChatML message prompt tokens including special
+ * delimiter tokens (<|im_start|>, <|im_sep|>, <|im_end|>).
+ */
+
+/**
+ * Options for encodeChat.
+ */
+interface EncodeChatOptions {
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     * Note: Non-OpenAI models (claude-*, gemini-*) are rejected.
+     */
+    model?: string;
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * Prime the output with the start of an assistant response.
+     * When true (default), appends <|im_start|>assistant<|im_sep|> at the end.
+     * Set to false to get just the messages without assistant priming.
+     */
+    primeAssistant?: boolean;
+}
+/**
+ * Encode chat messages into token IDs using ChatML format.
+ *
+ * Returns the exact token sequence that OpenAI models expect for chat
+ * completions, including special delimiter tokens.
+ *
+ * @param messages - Array of chat messages
+ * @param options - Encoding options
+ * @returns Token IDs representing the chat prompt
+ *
+ * @example
+ * ```typescript
+ * const tokens = encodeChat([
+ *   { role: 'system', content: 'You are helpful.' },
+ *   { role: 'user', content: 'Hello!' }
+ * ], { model: 'gpt-4o' });
+ * ```
+ */
+declare function encodeChat(messages: ChatMessage[], options?: EncodeChatOptions): number[];
+
 interface AnthropicCountTokensParams {
     /** Claude model id, e.g. `claude-sonnet-4-5` */
     model: string;
@@ -916,4 +964,4 @@ declare function clearModelCache(): void;
  */
 declare function parseModelProto(buffer: Uint8Array): ModelProto;
 
-export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type IsChatWithinTokenLimitInput, type IsWithinTokenLimitOptions, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, isChatWithinTokenLimit, isWithinTokenLimit, loadSentencePieceTokenizer, parseModelProto };
+export { type AnthropicCountTokensParams, type ChatCompletionTokenCountInput, type ChatCompletionTokenCountOutput, type ChatMessage, type CostEstimate, DEFAULT_MODELS, type DataOptions, type DownloadOptions, type EncodeChatOptions, type EncodeOptions, type EstimateAsyncInput, type EstimateCostFromTextAsyncOptions, type EstimateCostFromTextOptions, type EstimateCostInput, type EstimateInput, type EstimateOutput, type FileOptions, type FunctionCallOption, type FunctionDefinition, type FunctionParameterProperty, type FunctionParameters, type GeminiCountTokensParams, type GemmaSentencePieceCountTokensParams, type IsChatWithinTokenLimitInput, type IsWithinTokenLimitOptions, type KnownTokenizer, LAST_UPDATED, type ModelConfig, type ModelInfo, type ModelProto, type NormalizerSpec, type OpenAIEncoding, type SentencePiece, type SentencePieceTokenizer, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, type TokenizerModeAsync, type TrainerSpec, clearModelCache, countAnthropicInputTokens, countChatCompletionTokens, countGeminiTokens, countGemmaSentencePieceTokens, countSentencePieceTokens, countSentencePieceTokensAsync, countTokens, decode, decodeSentencePiece, decodeSentencePieceAsync, encode, encodeChat, encodeSentencePiece, encodeSentencePieceAsync, ensureSentencePieceModel, estimate, estimateAsync, estimateCost, estimateCostFromText, estimateCostFromTextAsync, getAvailableModels, getModelConfig, getOpenAIEncoding, getSentencePieceTokenizer, getTotalCost, isChatWithinTokenLimit, isWithinTokenLimit, loadSentencePieceTokenizer, parseModelProto };

package/dist/index.js

@@ -783,10 +783,18 @@ var CL100K_BASE_SPECIAL_TOKENS = [
   ["<|fim_prefix|>", 100258],
   ["<|fim_middle|>", 100259],
   ["<|fim_suffix|>", 100260],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 100264],
+  ["<|im_end|>", 100265],
+  ["<|im_sep|>", 100266],
   ["<|endofprompt|>", 100276]
 ];
 var O200K_BASE_SPECIAL_TOKENS = [
   ["<|endoftext|>", 199999],
+  // ChatML tokens for chat completion
+  ["<|im_start|>", 200264],
+  ["<|im_end|>", 200265],
+  ["<|im_sep|>", 200266],
   ["<|endofprompt|>", 200018]
 ];
 function buildO200kHarmonySpecialTokens() {
@@ -405374,6 +405382,131 @@ function isChatWithinTokenLimit(input) {
   }
   return count;
 }
+
+// src/encode-chat.ts
+var CHAT_TOKENS = {
+  cl100k_base: { imStart: 100264, imEnd: 100265, imSep: 100266 },
+  o200k_base: { imStart: 200264, imEnd: 200265, imSep: 200266 }
+};
+var HARMONY_TOKENS = {
+  start: 200006,
+  end: 200007,
+  message: 200008
+};
+function encodeChat(messages, options) {
+  const { model, encoding: encodingOverride, primeAssistant = true } = options ?? {};
+  validateChatModel(model, encodingOverride);
+  const encoding = encodingOverride ?? (model ? getOpenAIEncoding({ model }) : "o200k_base");
+  if (encoding === "o200k_harmony") {
+    console.warn(
+      "[ai-token-estimator] o200k_harmony support is experimental. Token structure may not match actual API behavior."
+    );
+  }
+  const chatTokens = getChatTokens(encoding);
+  if (!chatTokens) {
+    throw new Error(
+      `Encoding "${encoding}" does not support chat format. Use cl100k_base or o200k_base for chat models.`
+    );
+  }
+  const { imStart, imEnd, imSep } = chatTokens;
+  const tokens = [];
+  for (const message of messages) {
+    validateMessage(message);
+    tokens.push(imStart);
+    let roleStr;
+    if (message.role === "function" && message.name) {
+      roleStr = message.name;
+    } else if (message.name) {
+      roleStr = `${message.role}:${message.name}`;
+    } else {
+      roleStr = message.role;
+    }
+    tokens.push(...encode(roleStr, { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+    if (message.content) {
+      tokens.push(
+        ...encode(message.content, { encoding, allowSpecial: "none" })
+      );
+    }
+    if (message.function_call) {
+      const fcContent = formatFunctionCall(message.function_call);
+      tokens.push(...encode(fcContent, { encoding, allowSpecial: "none" }));
+    }
+    tokens.push(imEnd);
+  }
+  if (primeAssistant) {
+    tokens.push(imStart);
+    tokens.push(...encode("assistant", { encoding, allowSpecial: "none" }));
+    tokens.push(imSep);
+  }
+  return tokens;
+}
+function validateChatModel(model, encodingOverride) {
+  if (model) {
+    if (isAnthropicModel(model)) {
+      throw new Error(
+        `Model "${model}" is an Anthropic model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isGoogleModel(model)) {
+      throw new Error(
+        `Model "${model}" is a Google model. encodeChat only supports OpenAI models.`
+      );
+    }
+    if (isKnownModel(model) && !isChatModel(model)) {
+      throw new Error(
+        `Model "${model}" is not a chat completion model. encodeChat only supports chat models (e.g., gpt-4o, gpt-3.5-turbo).`
+      );
+    }
+  }
+  if (encodingOverride) {
+    return;
+  }
+  if (!model) {
+    throw new Error(
+      "Either model or encoding must be provided. Provide a known OpenAI chat model (e.g., gpt-4o) or an explicit encoding (e.g., o200k_base)."
+    );
+  }
+  if (!isChatModel(model)) {
+    throw new Error(
+      `Model "${model}" is not recognized as an OpenAI chat model. If this is a new OpenAI model, provide the encoding option explicitly (e.g., encoding: "o200k_base").`
+    );
+  }
+}
+function validateMessage(message) {
+  const msgAny = message;
+  if ("tool_calls" in msgAny && msgAny.tool_calls !== void 0) {
+    throw new Error(
+      "tool_calls is not supported. Use function_call with the legacy functions API."
+    );
+  }
+  if ("tool_call_id" in msgAny && msgAny.tool_call_id !== void 0) {
+    throw new Error(
+      "tool_call_id is not supported. Use the legacy functions API."
+    );
+  }
+  if (message.content !== null && message.content !== void 0 && typeof message.content !== "string") {
+    throw new Error(
+      "Multimodal content (arrays) is not supported. Only text content is supported."
+    );
+  }
+}
+function getChatTokens(encoding) {
+  if (encoding === "o200k_harmony") {
+    return {
+      imStart: HARMONY_TOKENS.start,
+      imEnd: HARMONY_TOKENS.end,
+      imSep: HARMONY_TOKENS.message
+    };
+  }
+  return CHAT_TOKENS[encoding] ?? null;
+}
+function formatFunctionCall(fc) {
+  const parts = [];
+  if (fc.name) parts.push(fc.name);
+  if (fc.arguments) parts.push(fc.arguments);
+  return parts.join("\n");
+}
 export {
   DEFAULT_MODELS,
   LAST_UPDATED,
@@ -405389,6 +405522,7 @@ export {
   decodeSentencePiece,
   decodeSentencePieceAsync,
   encode,
+  encodeChat,
   encodeSentencePiece,
   encodeSentencePieceAsync,
   ensureSentencePieceModel,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-token-estimator",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Estimate and count tokens (incl. exact OpenAI BPE) and input costs for LLM API calls",
   "type": "module",
   "main": "./dist/index.cjs",