@dexto/core 1.1.4 → 1.1.6

package/dist/llm/services/vercel.js

@@ -0,0 +1,506 @@
+import {
+  generateText,
+  streamText,
+  APICallError,
+  stepCountIs
+} from "ai";
+import { logger } from "../../logger/index.js";
+import { jsonSchema } from "ai";
+import { ContextManager } from "../../context/manager.js";
+import { sanitizeToolResultToContent, summarizeToolContentForText } from "../../context/utils.js";
+import { getMaxInputTokensForModel, getEffectiveMaxInputTokens } from "../registry.js";
+import { DextoRuntimeError } from "../../errors/DextoRuntimeError.js";
+import { LLMErrorCode } from "../error-codes.js";
+import { ErrorScope, ErrorType } from "../../errors/types.js";
+import { toError } from "../../utils/error-conversion.js";
+import { ToolErrorCode } from "../../tools/error-codes.js";
+import { VercelMessageFormatter } from "../formatters/vercel.js";
+import { createTokenizer } from "../tokenizer/factory.js";
+class VercelLLMService {
+  model;
+  config;
+  toolManager;
+  contextManager;
+  sessionEventBus;
+  sessionId;
+  toolSupportCache = /* @__PURE__ */ new Map();
+  // Map of toolCallId -> queue of raw results to emit with that callId
+  rawResultsByCallId = /* @__PURE__ */ new Map();
+  /**
+   * Helper to extract model ID from LanguageModel union type (string | LanguageModelV2)
+   */
+  getModelId() {
+    return typeof this.model === "string" ? this.model : this.model.modelId;
+  }
+  constructor(toolManager, model, promptManager, historyProvider, sessionEventBus, config, sessionId) {
+    this.model = model;
+    this.config = config;
+    this.toolManager = toolManager;
+    this.sessionEventBus = sessionEventBus;
+    this.sessionId = sessionId;
+    const formatter = new VercelMessageFormatter();
+    const tokenizer = createTokenizer(config.provider, this.getModelId());
+    const maxInputTokens = getEffectiveMaxInputTokens(config);
+    this.contextManager = new ContextManager(
+      config,
+      formatter,
+      promptManager,
+      maxInputTokens,
+      tokenizer,
+      historyProvider,
+      sessionId
+    );
+    logger.debug(
+      `[VercelLLMService] Initialized for model: ${this.getModelId()}, provider: ${this.config.provider}, temperature: ${this.config.temperature}, maxOutputTokens: ${this.config.maxOutputTokens}`
+    );
+  }
+  getAllTools() {
+    return this.toolManager.getAllTools();
+  }
+  formatTools(tools) {
+    logger.debug(`Formatting tools for vercel`);
+    return Object.keys(tools).reduce((acc, toolName) => {
+      const tool = tools[toolName];
+      if (tool) {
+        acc[toolName] = {
+          inputSchema: jsonSchema(tool.parameters),
+          execute: async (args, options) => {
+            try {
+              const rawResult = await this.toolManager.executeTool(
+                toolName,
+                args,
+                this.sessionId
+              );
+              const callId = options.toolCallId;
+              const queue = this.rawResultsByCallId.get(callId) ?? [];
+              queue.push(rawResult);
+              this.rawResultsByCallId.set(callId, queue);
+              const safeContent = sanitizeToolResultToContent(rawResult);
+              const summaryText = summarizeToolContentForText(safeContent);
+              return summaryText;
+            } catch (err) {
+              if (err instanceof DextoRuntimeError && err.code === ToolErrorCode.EXECUTION_DENIED) {
+                return { error: err.message, denied: true };
+              }
+              if (err instanceof DextoRuntimeError && err.code === ToolErrorCode.CONFIRMATION_TIMEOUT) {
+                return { error: err.message, denied: true, timeout: true };
+              }
+              const message = err instanceof Error ? err.message : String(err);
+              return { error: message };
+            }
+          },
+          ...tool.description && { description: tool.description }
+        };
+      }
+      return acc;
+    }, {});
+  }
+  async validateToolSupport() {
+    const modelKey = `${this.config.provider}:${this.getModelId()}:${this.config.baseURL ?? ""}`;
+    if (this.toolSupportCache.has(modelKey)) {
+      return this.toolSupportCache.get(modelKey);
+    }
+    if (!this.config.baseURL) {
+      logger.debug(`Skipping tool validation for ${modelKey} - no custom baseURL`);
+      this.toolSupportCache.set(modelKey, true);
+      return true;
+    }
+    logger.debug(`Testing tool support for custom endpoint model: ${modelKey}`);
+    const testTool = {
+      test_tool: {
+        inputSchema: jsonSchema({
+          type: "object",
+          properties: {},
+          additionalProperties: false
+        }),
+        execute: async () => ({ result: "test" })
+      }
+    };
+    try {
+      await generateText({
+        model: this.model,
+        messages: [{ role: "user", content: "Hello" }],
+        tools: testTool,
+        stopWhen: stepCountIs(1)
+      });
+      this.toolSupportCache.set(modelKey, true);
+      logger.debug(`Model ${modelKey} supports tools`);
+      return true;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      if (errorMessage.includes("does not support tools")) {
+        this.toolSupportCache.set(modelKey, false);
+        logger.debug(`Model ${modelKey} does not support tools`);
+        return false;
+      }
+      logger.debug(
+        `Tool validation error for ${modelKey}, assuming supported: ${errorMessage}`
+      );
+      this.toolSupportCache.set(modelKey, true);
+      return true;
+    }
+  }
+  async completeTask(textInput, options, imageData, fileData, stream) {
+    logger.debug(
+      `[VercelLLMService] addUserMessage(text ~${textInput.length} chars, image=${Boolean(
+        imageData
+      )}, file=${Boolean(fileData)})`
+    );
+    await this.contextManager.addUserMessage(textInput, imageData, fileData);
+    const tools = await this.toolManager.getAllTools();
+    logger.silly(
+      `[VercelLLMService] Tools before formatting: ${JSON.stringify(tools, null, 2)}`
+    );
+    const formattedTools = this.formatTools(tools);
+    logger.silly(
+      `[VercelLLMService] Formatted tools: ${JSON.stringify(formattedTools, null, 2)}`
+    );
+    let fullResponse = "";
+    this.sessionEventBus.emit("llmservice:thinking");
+    const prepared = await this.contextManager.getFormattedMessagesWithCompression(
+      { mcpManager: this.toolManager.getMcpManager() },
+      { provider: this.config.provider, model: this.getModelId() }
+    );
+    const formattedMessages = prepared.formattedMessages;
+    const tokensUsed = prepared.tokensUsed;
+    logger.silly(
+      `Messages (potentially compressed): ${JSON.stringify(formattedMessages, null, 2)}`
+    );
+    logger.silly(`Tools: ${JSON.stringify(formattedTools, null, 2)}`);
+    logger.debug(`Estimated tokens being sent to Vercel provider: ${tokensUsed}`);
+    if (stream) {
+      fullResponse = await this.streamText(
+        formattedMessages,
+        formattedTools,
+        this.config.maxIterations,
+        options?.signal
+      );
+    } else {
+      fullResponse = await this.generateText(
+        formattedMessages,
+        formattedTools,
+        this.config.maxIterations,
+        options?.signal
+      );
+    }
+    if (options?.signal?.aborted) {
+      return fullResponse;
+    }
+    return fullResponse || `Reached maximum number of steps (${this.config.maxIterations}) without a final response.`;
+  }
+  async generateText(messages, tools, maxSteps = 50, signal) {
+    let stepIteration = 0;
+    const estimatedTokens = Math.ceil(JSON.stringify(messages, null, 2).length / 4);
+    logger.debug(
+      `vercel generateText:Generating text with messages (${estimatedTokens} estimated tokens)`
+    );
+    const temperature = this.config.temperature;
+    const maxOutputTokens = this.config.maxOutputTokens;
+    const supportsTools = await this.validateToolSupport();
+    const effectiveTools = supportsTools ? tools : {};
+    if (!supportsTools && Object.keys(tools).length > 0) {
+      logger.debug(
+        `Model ${this.getModelId()} does not support tools, using empty tools object for generation`
+      );
+    }
+    try {
+      const includeMaxOutputTokens = typeof maxOutputTokens === "number";
+      const response = await generateText({
+        model: this.model,
+        messages,
+        tools: effectiveTools,
+        ...signal ? { abortSignal: signal } : {},
+        onStepFinish: (step) => {
+          logger.debug(`Step iteration: ${stepIteration}`);
+          stepIteration++;
+          logger.debug(`Step finished, text: ${step.text}`);
+          logger.debug(
+            `Step finished, step tool calls: ${JSON.stringify(step.toolCalls, null, 2)}`
+          );
+          logger.debug(
+            `Step finished, step tool results: ${JSON.stringify(step.toolResults, null, 2)}`
+          );
+          if (step.toolCalls && step.toolCalls.length > 0) {
+            for (const toolCall of step.toolCalls) {
+              this.sessionEventBus.emit("llmservice:toolCall", {
+                toolName: toolCall.toolName,
+                args: toolCall.input,
+                callId: toolCall.toolCallId
+              });
+            }
+          }
+          if (step.toolResults && step.toolResults.length > 0) {
+            for (const toolResult of step.toolResults) {
+              const callId = toolResult.toolCallId;
+              const sanitized = toolResult.output;
+              let raw;
+              if (callId) {
+                const q = this.rawResultsByCallId.get(callId) ?? [];
+                raw = q.shift();
+                if (q.length > 0) this.rawResultsByCallId.set(callId, q);
+                else this.rawResultsByCallId.delete(callId);
+              }
+              this.sessionEventBus.emit("llmservice:toolResult", {
+                toolName: toolResult.toolName,
+                result: raw ?? sanitized,
+                callId,
+                success: true
+              });
+            }
+          }
+        },
+        stopWhen: stepCountIs(maxSteps),
+        ...includeMaxOutputTokens ? { maxOutputTokens } : {},
+        ...temperature !== void 0 && { temperature }
+      });
+      this.sessionEventBus.emit("llmservice:response", {
+        content: response.text,
+        ...response.reasoningText && { reasoning: response.reasoningText },
+        provider: this.config.provider,
+        model: this.getModelId(),
+        router: "vercel",
+        tokenUsage: {
+          ...response.totalUsage.inputTokens !== void 0 && {
+            inputTokens: response.totalUsage.inputTokens
+          },
+          ...response.totalUsage.outputTokens !== void 0 && {
+            outputTokens: response.totalUsage.outputTokens
+          },
+          ...response.totalUsage.reasoningTokens !== void 0 && {
+            reasoningTokens: response.totalUsage.reasoningTokens
+          },
+          ...response.totalUsage.totalTokens !== void 0 && {
+            totalTokens: response.totalUsage.totalTokens
+          }
+        }
+      });
+      await this.contextManager.processLLMResponse(response);
+      if (typeof response.totalUsage.totalTokens === "number") {
+        this.contextManager.updateActualTokenCount(response.totalUsage.totalTokens);
+      }
+      return response.text;
+    } catch (err) {
+      this.mapProviderError(err, "generate");
+    }
+  }
+  mapProviderError(err, phase) {
+    if (APICallError.isInstance?.(err)) {
+      const status = err.statusCode;
+      const headers = err.responseHeaders || {};
+      const retryAfter = headers["retry-after"] ? Number(headers["retry-after"]) : void 0;
+      const body = typeof err.responseBody === "string" ? err.responseBody : JSON.stringify(err.responseBody ?? "");
+      if (status === 429) {
+        throw new DextoRuntimeError(
+          LLMErrorCode.RATE_LIMIT_EXCEEDED,
+          ErrorScope.LLM,
+          ErrorType.RATE_LIMIT,
+          `Rate limit exceeded${body ? ` - ${body}` : ""}`,
+          {
+            sessionId: this.sessionId,
+            provider: this.config.provider,
+            model: this.getModelId(),
+            status,
+            retryAfter,
+            body,
+            phase
+          }
+        );
+      }
+      if (status === 408) {
+        throw new DextoRuntimeError(
+          LLMErrorCode.GENERATION_FAILED,
+          ErrorScope.LLM,
+          ErrorType.TIMEOUT,
+          `Provider timed out${body ? ` - ${body}` : ""}`,
+          {
+            sessionId: this.sessionId,
+            provider: this.config.provider,
+            model: this.getModelId(),
+            status,
+            body,
+            phase
+          }
+        );
+      }
+      throw new DextoRuntimeError(
+        LLMErrorCode.GENERATION_FAILED,
+        ErrorScope.LLM,
+        ErrorType.THIRD_PARTY,
+        `Provider error ${status}${body ? ` - ${body}` : ""}`,
+        {
+          sessionId: this.sessionId,
+          provider: this.config.provider,
+          model: this.getModelId(),
+          status,
+          body,
+          phase
+        }
+      );
+    }
+    throw err;
+  }
+  // Updated streamText to behave like generateText - returns string and handles message processing internally
+  async streamText(messages, tools, maxSteps = 10, signal) {
+    let stepIteration = 0;
+    const temperature = this.config.temperature;
+    const maxOutputTokens = this.config.maxOutputTokens;
+    const supportsTools = await this.validateToolSupport();
+    const effectiveTools = supportsTools ? tools : {};
+    if (!supportsTools && Object.keys(tools).length > 0) {
+      logger.debug(
+        `Model ${this.getModelId()} does not support tools, using empty tools object for streaming`
+      );
+    }
+    let streamErr;
+    const includeMaxOutputTokens = typeof maxOutputTokens === "number";
+    const response = streamText({
+      model: this.model,
+      messages,
+      tools: effectiveTools,
+      ...signal ? { abortSignal: signal } : {},
+      onChunk: (chunk) => {
+        logger.debug(`Chunk type: ${chunk.chunk.type}`);
+        if (chunk.chunk.type === "text-delta") {
+          this.sessionEventBus.emit("llmservice:chunk", {
+            type: "text",
+            content: chunk.chunk.text,
+            isComplete: false
+          });
+        } else if (chunk.chunk.type === "reasoning-delta") {
+          this.sessionEventBus.emit("llmservice:chunk", {
+            type: "reasoning",
+            content: chunk.chunk.text,
+            isComplete: false
+          });
+        }
+      },
+      // TODO: Add onAbort handler when we implement partial response handling.
+      // Vercel triggers onAbort instead of onError for cancelled streams.
+      // This is where cancellation logic should be handled properly.
+      onError: (error) => {
+        const err = toError(error);
+        logger.error(`Error in streamText: ${err?.stack ?? err}`, null, "red");
+        this.sessionEventBus.emit("llmservice:error", {
+          error: err,
+          context: "streamText",
+          recoverable: false
+        });
+        streamErr = error;
+      },
+      onStepFinish: (step) => {
+        logger.debug(`Step iteration: ${stepIteration}`);
+        stepIteration++;
+        logger.debug(`Step finished, text: ${step.text}`);
+        logger.debug(
+          `Step finished, step tool calls: ${JSON.stringify(step.toolCalls, null, 2)}`
+        );
+        logger.debug(
+          `Step finished, step tool results: ${JSON.stringify(step.toolResults, null, 2)}`
+        );
+        if (step.toolCalls && step.toolCalls.length > 0) {
+          for (const toolCall of step.toolCalls) {
+            this.sessionEventBus.emit("llmservice:toolCall", {
+              toolName: toolCall.toolName,
+              args: toolCall.input,
+              callId: toolCall.toolCallId
+            });
+          }
+        }
+        if (step.toolResults && step.toolResults.length > 0) {
+          for (const toolResult of step.toolResults) {
+            const callId = toolResult.toolCallId;
+            const sanitized = toolResult.output;
+            let raw;
+            if (callId) {
+              const q = this.rawResultsByCallId.get(callId) ?? [];
+              raw = q.shift();
+              if (q.length > 0) this.rawResultsByCallId.set(callId, q);
+              else this.rawResultsByCallId.delete(callId);
+            }
+            this.sessionEventBus.emit("llmservice:toolResult", {
+              toolName: toolResult.toolName,
+              result: raw ?? sanitized,
+              callId,
+              success: true
+            });
+          }
+        }
+      },
+      // No onFinish: we finalize after the stream completes below.
+      stopWhen: stepCountIs(maxSteps),
+      ...includeMaxOutputTokens ? { maxOutputTokens } : {},
+      ...temperature !== void 0 && { temperature }
+    });
+    const [finalText, usage, reasoningText] = await Promise.all([
+      response.text,
+      response.totalUsage,
+      response.reasoningText
+    ]);
+    response.totalUsage;
+    if (streamErr) {
+      return "";
+    }
+    this.sessionEventBus.emit("llmservice:response", {
+      content: finalText,
+      ...reasoningText && { reasoning: reasoningText },
+      provider: this.config.provider,
+      model: this.getModelId(),
+      router: "vercel",
+      tokenUsage: {
+        ...usage.inputTokens !== void 0 && { inputTokens: usage.inputTokens },
+        ...usage.outputTokens !== void 0 && { outputTokens: usage.outputTokens },
+        ...usage.reasoningTokens !== void 0 && {
+          reasoningTokens: usage.reasoningTokens
+        },
+        ...usage.totalTokens !== void 0 && { totalTokens: usage.totalTokens }
+      }
+    });
+    if (typeof usage.totalTokens === "number") {
+      this.contextManager.updateActualTokenCount(usage.totalTokens);
+    }
+    await this.contextManager.processLLMStreamResponse(response);
+    logger.silly(`streamText response object: ${JSON.stringify(response, null, 2)}`);
+    return finalText;
+  }
+  /**
+   * Get configuration information about the LLM service
+   * @returns Configuration object with provider and model information
+   */
+  getConfig() {
+    const configuredMaxTokens = this.contextManager.getMaxInputTokens();
+    let modelMaxInputTokens;
+    try {
+      modelMaxInputTokens = getMaxInputTokensForModel(
+        this.config.provider,
+        // Use our internal provider name
+        this.getModelId()
+      );
+    } catch (error) {
+      if (error instanceof DextoRuntimeError && error.code === LLMErrorCode.MODEL_UNKNOWN) {
+        modelMaxInputTokens = configuredMaxTokens;
+        logger.debug(
+          `Could not find model ${this.getModelId()} in LLM registry to get max tokens. Using configured max tokens: ${configuredMaxTokens}.`
+        );
+      } else {
+        throw error;
+      }
+    }
+    return {
+      router: "vercel",
+      provider: this.config.provider,
+      // Use our internal provider name
+      model: this.model,
+      configuredMaxInputTokens: configuredMaxTokens,
+      modelMaxInputTokens
+    };
+  }
+  /**
+   * Get the context manager for external access
+   */
+  getContextManager() {
+    return this.contextManager;
+  }
+}
+export {
+  VercelLLMService
+};

package/dist/llm/tokenizer/anthropic.cjs

@@ -0,0 +1,43 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var anthropic_exports = {};
+__export(anthropic_exports, {
+  AnthropicTokenizer: () => AnthropicTokenizer
+});
+module.exports = __toCommonJS(anthropic_exports);
+class AnthropicTokenizer {
+  /**
+   * TODO: Replace with a better implementation for anthropic
+   * Approximates token count for Anthropic models.
+   * Assumes roughly 4 characters per token.
+   * @param text Text content to count tokens for
+   * @returns Approximate number of tokens
+   */
+  countTokens(text) {
+    if (!text) return 0;
+    return Math.ceil(text.length / 4);
+  }
+  getProviderName() {
+    return "anthropic";
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AnthropicTokenizer
+});

package/dist/llm/tokenizer/anthropic.d.cts

@@ -0,0 +1,21 @@
+import { ITokenizer } from './types.cjs';
+
+/**
+ * Placeholder tokenizer for Anthropic models.
+ * Uses a rough approximation based on character count.
+ * TODO: IMPORTANT: This is not accurate and should be replaced
+ * with a better implementation or official library when available.
+ */
+declare class AnthropicTokenizer implements ITokenizer {
+    /**
+     * TODO: Replace with a better implementation for anthropic
+     * Approximates token count for Anthropic models.
+     * Assumes roughly 4 characters per token.
+     * @param text Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text: string): number;
+    getProviderName(): string;
+}
+
+export { AnthropicTokenizer };

package/dist/llm/tokenizer/anthropic.d.ts

@@ -0,0 +1,21 @@
+import { ITokenizer } from './types.js';
+
+/**
+ * Placeholder tokenizer for Anthropic models.
+ * Uses a rough approximation based on character count.
+ * TODO: IMPORTANT: This is not accurate and should be replaced
+ * with a better implementation or official library when available.
+ */
+declare class AnthropicTokenizer implements ITokenizer {
+    /**
+     * TODO: Replace with a better implementation for anthropic
+     * Approximates token count for Anthropic models.
+     * Assumes roughly 4 characters per token.
+     * @param text Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text: string): number;
+    getProviderName(): string;
+}
+
+export { AnthropicTokenizer };

package/dist/llm/tokenizer/anthropic.js

@@ -0,0 +1,19 @@
+class AnthropicTokenizer {
+  /**
+   * TODO: Replace with a better implementation for anthropic
+   * Approximates token count for Anthropic models.
+   * Assumes roughly 4 characters per token.
+   * @param text Text content to count tokens for
+   * @returns Approximate number of tokens
+   */
+  countTokens(text) {
+    if (!text) return 0;
+    return Math.ceil(text.length / 4);
+  }
+  getProviderName() {
+    return "anthropic";
+  }
+}
+export {
+  AnthropicTokenizer
+};

package/dist/llm/tokenizer/default.cjs

@@ -0,0 +1,41 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var default_exports = {};
+__export(default_exports, {
+  DefaultTokenizer: () => DefaultTokenizer
+});
+module.exports = __toCommonJS(default_exports);
+class DefaultTokenizer {
+  /**
+   * Assumes roughly 4 characters per token.
+   * @param text Text content to count tokens for
+   * @returns Approximate number of tokens
+   */
+  countTokens(text) {
+    if (!text) return 0;
+    return Math.ceil(text.length / 4);
+  }
+  getProviderName() {
+    return "default";
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DefaultTokenizer
+});

package/dist/llm/tokenizer/default.d.cts

@@ -0,0 +1,16 @@
+import { ITokenizer } from './types.cjs';
+
+/**
+ * Placeholder tokenizer for unknown models
+ */
+declare class DefaultTokenizer implements ITokenizer {
+    /**
+     * Assumes roughly 4 characters per token.
+     * @param text Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text: string): number;
+    getProviderName(): string;
+}
+
+export { DefaultTokenizer };

package/dist/llm/tokenizer/default.d.ts

@@ -0,0 +1,16 @@
+import { ITokenizer } from './types.js';
+
+/**
+ * Placeholder tokenizer for unknown models
+ */
+declare class DefaultTokenizer implements ITokenizer {
+    /**
+     * Assumes roughly 4 characters per token.
+     * @param text Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text: string): number;
+    getProviderName(): string;
+}
+
+export { DefaultTokenizer };