@dexto/core 1.3.0 → 1.4.0

package/dist/llm/services/openai.js

@@ -1,547 +0,0 @@
-import {
-  __decorateElement,
-  __decoratorStart,
-  __runInitializers
-} from "../../chunk-C6A6W6XS.js";
-var _OpenAIService_decorators, _init;
-import { DextoLogComponent } from "../../logger/v2/types.js";
-import { ContextManager } from "../../context/manager.js";
-import { getMaxInputTokensForModel, getEffectiveMaxInputTokens } from "../registry.js";
-import { DextoRuntimeError } from "../../errors/DextoRuntimeError.js";
-import { LLMErrorCode } from "../error-codes.js";
-import { OpenAIMessageFormatter } from "../formatters/openai.js";
-import { createTokenizer } from "../tokenizer/factory.js";
-import { shouldIncludeRawToolResult } from "../../utils/debug.js";
-import { InstrumentClass } from "../../telemetry/decorators.js";
-import { trace, context, propagation } from "@opentelemetry/api";
-_OpenAIService_decorators = [InstrumentClass({
-  prefix: "llm.openai",
-  excludeMethods: ["getAllTools", "formatToolsForOpenAI", "getConfig", "getContextManager"]
-})];
-class OpenAIService {
-  openai;
-  config;
-  toolManager;
-  contextManager;
-  sessionEventBus;
-  sessionId;
-  logger;
-  constructor(toolManager, openai, systemPromptManager, historyProvider, sessionEventBus, config, sessionId, resourceManager, logger) {
-    this.logger = logger.createChild(DextoLogComponent.LLM);
-    this.config = config;
-    this.openai = openai;
-    this.toolManager = toolManager;
-    this.sessionEventBus = sessionEventBus;
-    this.sessionId = sessionId;
-    const formatter = new OpenAIMessageFormatter(this.logger);
-    const tokenizer = createTokenizer(config.provider, config.model, this.logger);
-    const maxInputTokens = getEffectiveMaxInputTokens(config, this.logger);
-    this.contextManager = new ContextManager(
-      config,
-      formatter,
-      systemPromptManager,
-      maxInputTokens,
-      tokenizer,
-      historyProvider,
-      sessionId,
-      resourceManager,
-      this.logger
-      // compressionStrategies uses default
-    );
-  }
-  getAllTools() {
-    return this.toolManager.getAllTools();
-  }
-  async completeTask(textInput, options, imageData, fileData, stream) {
-    const activeSpan = trace.getActiveSpan();
-    const currentContext = context.active();
-    const provider = this.config.provider;
-    const model = this.config.model;
-    if (activeSpan) {
-      activeSpan.setAttribute("llm.provider", provider);
-      activeSpan.setAttribute("llm.model", model);
-    }
-    const existingBaggage = propagation.getBaggage(currentContext);
-    const baggageEntries = {};
-    if (existingBaggage) {
-      existingBaggage.getAllEntries().forEach(([key, entry]) => {
-        baggageEntries[key] = entry;
-      });
-    }
-    baggageEntries["llm.provider"] = { value: provider };
-    baggageEntries["llm.model"] = { value: model };
-    const updatedContext = propagation.setBaggage(
-      currentContext,
-      propagation.createBaggage(baggageEntries)
-    );
-    return await context.with(updatedContext, async () => {
-      await this.contextManager.addUserMessage(textInput, imageData, fileData);
-      const rawTools = await this.toolManager.getAllTools();
-      const formattedTools = this.formatToolsForOpenAI(rawTools);
-      this.logger.silly(`Formatted tools: ${JSON.stringify(formattedTools, null, 2)}`);
-      this.sessionEventBus.emit("llm:thinking");
-      let iterationCount = 0;
-      let totalTokens = 0;
-      let inputTokens = 0;
-      let outputTokens = 0;
-      let reasoningTokens = 0;
-      let fullResponse = "";
-      const contextStr = stream ? "OpenAI streaming API call" : "OpenAI API call";
-      try {
-        while (iterationCount < this.config.maxIterations) {
-          if (options?.signal?.aborted) {
-            throw Object.assign(new Error("Aborted"), {
-              name: "AbortError",
-              aborted: true
-            });
-          }
-          iterationCount++;
-          const { message, usage } = stream ? await this.getAIStreamingResponseWithRetries(
-            formattedTools,
-            options?.signal
-          ) : await this.getAIResponseWithRetries(formattedTools, options?.signal);
-          if (usage) {
-            totalTokens += usage.total_tokens ?? 0;
-            inputTokens += usage.prompt_tokens ?? 0;
-            outputTokens += usage.completion_tokens ?? 0;
-            reasoningTokens += usage.completion_tokens_details?.reasoning_tokens ?? 0;
-          }
-          if (!message.tool_calls || message.tool_calls.length === 0) {
-            const responseText = message.content || "";
-            const finalContent = stream ? fullResponse + responseText : responseText;
-            await this.contextManager.addAssistantMessage(finalContent, void 0, {
-              tokenUsage: totalTokens > 0 ? {
-                totalTokens,
-                inputTokens,
-                outputTokens,
-                reasoningTokens
-              } : void 0
-            });
-            if (totalTokens > 0) {
-              this.contextManager.updateActualTokenCount(totalTokens);
-            }
-            this.sessionEventBus.emit("llm:response", {
-              content: finalContent,
-              provider: this.config.provider,
-              model: this.config.model,
-              router: "in-built",
-              tokenUsage: { totalTokens, inputTokens, outputTokens, reasoningTokens }
-            });
-            const activeSpan3 = trace.getActiveSpan();
-            if (activeSpan3 && totalTokens > 0) {
-              const attributes = {
-                "gen_ai.usage.total_tokens": totalTokens
-              };
-              if (inputTokens > 0) {
-                attributes["gen_ai.usage.input_tokens"] = inputTokens;
-              }
-              if (outputTokens > 0) {
-                attributes["gen_ai.usage.output_tokens"] = outputTokens;
-              }
-              if (reasoningTokens > 0) {
-                attributes["gen_ai.usage.reasoning_tokens"] = reasoningTokens;
-              }
-              activeSpan3.setAttributes(attributes);
-            }
-            return finalContent;
-          }
-          const functionToolCalls = message.tool_calls.filter(
-            (toolCall) => {
-              this.logger.debug(`Tool call type received: ${toolCall.type}`);
-              return toolCall.type === "function";
-            }
-          );
-          if (message.tool_calls.length > functionToolCalls.length) {
-            this.logger.warn(
-              `Filtered out ${message.tool_calls.length - functionToolCalls.length} non-function tool calls`
-            );
-          }
-          await this.contextManager.addAssistantMessage(
-            message.content,
-            functionToolCalls.length > 0 ? functionToolCalls : void 0,
-            {}
-          );
-          if (stream && message.content) {
-            fullResponse += message.content;
-          }
-          for (const toolCall of functionToolCalls) {
-            if (options?.signal?.aborted) {
-              throw Object.assign(new Error("Aborted"), {
-                name: "AbortError",
-                aborted: true
-              });
-            }
-            this.logger.debug(`Tool call initiated - Type: ${toolCall.type}`);
-            this.logger.debug(
-              `Tool call details: ${JSON.stringify(toolCall, null, 2)}`
-            );
-            const toolName = toolCall.function.name;
-            let args = {};
-            try {
-              args = JSON.parse(toolCall.function.arguments);
-            } catch (e) {
-              this.logger.error(`Error parsing arguments for ${toolName}:`, {
-                error: e instanceof Error ? e.message : String(e)
-              });
-              const sanitized = await this.contextManager.addToolResult(
-                toolCall.id,
-                toolName,
-                { error: `Failed to parse arguments: ${e}` },
-                { success: false }
-              );
-              this.sessionEventBus.emit("llm:tool-result", {
-                toolName,
-                callId: toolCall.id,
-                success: false,
-                sanitized,
-                ...shouldIncludeRawToolResult() ? { rawResult: { error: `Failed to parse arguments: ${e}` } } : {}
-              });
-              continue;
-            }
-            this.sessionEventBus.emit("llm:tool-call", {
-              toolName,
-              args,
-              callId: toolCall.id
-            });
-            try {
-              const result = await this.toolManager.executeTool(
-                toolName,
-                args,
-                this.sessionId
-              );
-              const sanitized = await this.contextManager.addToolResult(
-                toolCall.id,
-                toolName,
-                result,
-                { success: true }
-              );
-              this.sessionEventBus.emit("llm:tool-result", {
-                toolName,
-                callId: toolCall.id,
-                success: true,
-                sanitized,
-                ...shouldIncludeRawToolResult() ? { rawResult: result } : {}
-              });
-            } catch (error) {
-              const errorMessage = error instanceof Error ? error.message : String(error);
-              this.logger.error(
-                `Tool execution error for ${toolName}: ${errorMessage}`
-              );
-              const sanitized = await this.contextManager.addToolResult(
-                toolCall.id,
-                toolName,
-                { error: errorMessage },
-                { success: false }
-              );
-              this.sessionEventBus.emit("llm:tool-result", {
-                toolName,
-                callId: toolCall.id,
-                success: false,
-                sanitized,
-                ...shouldIncludeRawToolResult() ? { rawResult: { error: errorMessage } } : {}
-              });
-            }
-          }
-        }
-        this.logger.warn(
-          `Reached maximum iterations (${this.config.maxIterations}) for task.`
-        );
-        const finalResponse = fullResponse || "Task completed but reached maximum tool call iterations.";
-        await this.contextManager.addAssistantMessage(finalResponse, void 0, {
-          tokenUsage: totalTokens > 0 ? {
-            totalTokens,
-            inputTokens,
-            outputTokens,
-            reasoningTokens
-          } : void 0
-        });
-        if (totalTokens > 0) {
-          this.contextManager.updateActualTokenCount(totalTokens);
-        }
-        this.sessionEventBus.emit("llm:response", {
-          content: finalResponse,
-          provider: this.config.provider,
-          model: this.config.model,
-          router: "in-built",
-          tokenUsage: { totalTokens, inputTokens, outputTokens, reasoningTokens }
-        });
-        const activeSpan2 = trace.getActiveSpan();
-        if (activeSpan2 && totalTokens > 0) {
-          const attributes = {
-            "gen_ai.usage.total_tokens": totalTokens
-          };
-          if (inputTokens > 0) {
-            attributes["gen_ai.usage.input_tokens"] = inputTokens;
-          }
-          if (outputTokens > 0) {
-            attributes["gen_ai.usage.output_tokens"] = outputTokens;
-          }
-          if (reasoningTokens > 0) {
-            attributes["gen_ai.usage.reasoning_tokens"] = reasoningTokens;
-          }
-          activeSpan2.setAttributes(attributes);
-        }
-        return finalResponse;
-      } catch (error) {
-        if (error instanceof Error && (error.name === "AbortError" || error.aborted === true)) {
-          throw error;
-        }
-        const errorMessage = error instanceof Error ? error.message : String(error);
-        this.logger.error(`Error in ${contextStr}: ${errorMessage}`, { error });
-        if (!stream) {
-          this.logger.warn(
-            `If this error is due to token overflow, consider configuring 'maxInputTokens' in your LLMConfig.`
-          );
-        }
-        this.sessionEventBus.emit("llm:error", {
-          error: error instanceof Error ? error : new Error(errorMessage),
-          context: contextStr,
-          recoverable: false
-        });
-        const errorResponse = `Error processing ${stream ? "streaming " : ""}request: ${errorMessage}`;
-        await this.contextManager.addAssistantMessage(errorResponse, void 0, {});
-        return errorResponse;
-      }
-    });
-  }
-  /**
-   * Get configuration information about the LLM service
-   * @returns Configuration object with provider and model information
-   */
-  getConfig() {
-    const configuredMaxInputTokens = this.contextManager.getMaxInputTokens();
-    let modelMaxInputTokens;
-    try {
-      modelMaxInputTokens = getMaxInputTokensForModel(
-        this.config.provider,
-        this.config.model,
-        this.logger
-      );
-    } catch (error) {
-      if (error instanceof DextoRuntimeError && error.code === LLMErrorCode.MODEL_UNKNOWN) {
-        modelMaxInputTokens = configuredMaxInputTokens;
-        this.logger.debug(
-          `Could not find model ${this.config.model} in LLM registry to get max tokens. Using configured max tokens: ${configuredMaxInputTokens}.`
-        );
-      } else {
-        throw error;
-      }
-    }
-    return {
-      router: "in-built",
-      provider: this.config.provider,
-      model: this.config.model,
-      configuredMaxInputTokens,
-      modelMaxInputTokens
-    };
-  }
-  // Helper methods
-  async getAIResponseWithRetries(tools, signal) {
-    let attempts = 0;
-    const MAX_ATTEMPTS = 3;
-    this.logger.debug(`Tools size in getAIResponseWithRetries: ${tools.length}`);
-    while (attempts < MAX_ATTEMPTS) {
-      attempts++;
-      try {
-        const {
-          formattedMessages,
-          systemPrompt: _systemPrompt,
-          tokensUsed
-        } = await this.contextManager.getFormattedMessagesWithCompression(
-          { mcpManager: this.toolManager.getMcpManager() },
-          { provider: this.config.provider, model: this.config.model }
-        );
-        this.logger.silly(
-          `Message history (potentially compressed) in getAIResponseWithRetries: ${JSON.stringify(formattedMessages, null, 2)}`
-        );
-        this.logger.debug(`Estimated tokens being sent to OpenAI: ${tokensUsed}`);
-        const response = await this.openai.chat.completions.create(
-          {
-            model: this.config.model,
-            messages: formattedMessages,
-            tools: attempts === 1 ? tools : [],
-            // Only offer tools on first attempt
-            tool_choice: attempts === 1 ? "auto" : "none"
-            // Disable tool choice on retry
-          },
-          signal ? { signal } : void 0
-        );
-        this.logger.silly(
-          `OPENAI CHAT COMPLETION RESPONSE: ${JSON.stringify(response, null, 2)}`
-        );
-        const message = response.choices[0]?.message;
-        if (!message) {
-          throw new Error("Received empty message from OpenAI API");
-        }
-        const usage = response.usage;
-        return { message, ...usage && { usage } };
-      } catch (error) {
-        if (error instanceof Error && (error.name === "AbortError" || error.aborted === true)) {
-          throw error;
-        }
-        const apiError = error;
-        this.logger.error(
-          `Error in OpenAI API call (Attempt ${attempts}/${MAX_ATTEMPTS}): ${apiError.message || JSON.stringify(apiError, null, 2)}`,
-          { status: apiError.status, headers: apiError.headers }
-        );
-        if (apiError.status === 400 && apiError.code === "context_length_exceeded") {
-          this.logger.warn(
-            `Context length exceeded. ContextManager compression might not be sufficient. Error details: ${JSON.stringify(apiError.error)}`
-          );
-        }
-        if (attempts >= MAX_ATTEMPTS) {
-          this.logger.error(
-            `Failed to get response from OpenAI after ${MAX_ATTEMPTS} attempts.`
-          );
-          throw error;
-        }
-        await new Promise((resolve) => setTimeout(resolve, 500 * attempts));
-      }
-    }
-    throw new Error("Failed to get response after maximum retry attempts");
-  }
-  async getAIStreamingResponseWithRetries(tools, signal) {
-    let attempts = 0;
-    const MAX_ATTEMPTS = 3;
-    while (attempts < MAX_ATTEMPTS) {
-      attempts++;
-      try {
-        const {
-          formattedMessages,
-          systemPrompt: _systemPrompt,
-          tokensUsed
-        } = await this.contextManager.getFormattedMessagesWithCompression(
-          { mcpManager: this.toolManager.getMcpManager() },
-          { provider: this.config.provider, model: this.config.model }
-        );
-        this.logger.silly(
-          `Streaming message history (potentially compressed): ${JSON.stringify(formattedMessages, null, 2)}`
-        );
-        this.logger.debug(`Estimated tokens being sent to OpenAI streaming: ${tokensUsed}`);
-        const stream = await this.openai.chat.completions.create(
-          {
-            model: this.config.model,
-            messages: formattedMessages,
-            tools: attempts === 1 ? tools : [],
-            // Only offer tools on first attempt
-            tool_choice: attempts === 1 ? "auto" : "none",
-            // Disable tool choice on retry
-            stream: true,
-            stream_options: {
-              include_usage: true
-            }
-          },
-          signal ? { signal } : void 0
-        );
-        let content = "";
-        let toolCalls = [];
-        let usage;
-        for await (const chunk of stream) {
-          if (signal?.aborted) {
-            throw Object.assign(new Error("Aborted"), {
-              name: "AbortError",
-              aborted: true
-            });
-          }
-          const delta = chunk.choices[0]?.delta;
-          if (delta?.content) {
-            content += delta.content;
-            this.sessionEventBus.emit("llm:chunk", {
-              chunkType: "text",
-              content: delta.content,
-              isComplete: false
-            });
-          }
-          if (delta?.tool_calls) {
-            for (const toolCall of delta.tool_calls) {
-              const index = toolCall.index;
-              if (typeof index !== "number") {
-                continue;
-              }
-              if (!toolCalls[index]) {
-                toolCalls[index] = {
-                  id: `pending-${index}`,
-                  type: "function",
-                  function: { name: "", arguments: "" }
-                };
-              }
-              if (toolCall.id) {
-                toolCalls[index].id = toolCall.id;
-              }
-              const existingToolCall = toolCalls[index];
-              if (existingToolCall.type === "function" && toolCall.function) {
-                if (toolCall.function.name) {
-                  existingToolCall.function.name = toolCall.function.name;
-                }
-                if (toolCall.function.arguments) {
-                  existingToolCall.function.arguments += toolCall.function.arguments;
-                }
-              }
-            }
-          }
-          if (chunk.usage) {
-            usage = chunk.usage;
-          }
-        }
-        if (content) {
-          this.sessionEventBus.emit("llm:chunk", {
-            chunkType: "text",
-            content: "",
-            isComplete: true
-          });
-        }
-        const message = {
-          role: "assistant",
-          content: content || null,
-          refusal: null,
-          ...toolCalls.length > 0 && { tool_calls: toolCalls }
-        };
-        this.logger.debug(`OpenAI streaming completed. Usage info: ${usage}`);
-        return { message, ...usage && { usage } };
-      } catch (error) {
-        const apiError = error;
-        this.logger.error(
-          `Error in OpenAI streaming API call (Attempt ${attempts}/${MAX_ATTEMPTS}): ${apiError.message || JSON.stringify(apiError, null, 2)}`,
-          { status: apiError.status, headers: apiError.headers }
-        );
-        if (apiError.status === 400 && apiError.message?.includes("maximum context length")) {
-          this.logger.warn(
-            `Context length exceeded in streaming. ContextManager compression might not be sufficient. Error details: ${JSON.stringify(apiError.error)}`
-          );
-        }
-        if (attempts >= MAX_ATTEMPTS) {
-          this.logger.error(
-            `Failed to get streaming response from OpenAI after ${MAX_ATTEMPTS} attempts.`
-          );
-          throw error;
-        }
-        await new Promise((resolve) => setTimeout(resolve, 500 * attempts));
-      }
-    }
-    throw new Error("Failed to get streaming response after maximum retry attempts");
-  }
-  formatToolsForOpenAI(tools) {
-    return Object.entries(tools).map(([name, tool]) => {
-      return {
-        type: "function",
-        function: {
-          name,
-          description: tool.description || "",
-          parameters: tool.parameters
-        }
-      };
-    });
-  }
-  /**
-   * Get the context manager for external access
-   */
-  getContextManager() {
-    return this.contextManager;
-  }
-}
-_init = __decoratorStart(null);
-OpenAIService = __decorateElement(_init, 0, "OpenAIService", _OpenAIService_decorators, OpenAIService);
-__runInitializers(_init, 1, OpenAIService);
-export {
-  OpenAIService
-};

package/dist/llm/tokenizer/anthropic.cjs

@@ -1,43 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var anthropic_exports = {};
-__export(anthropic_exports, {
-  AnthropicTokenizer: () => AnthropicTokenizer
-});
-module.exports = __toCommonJS(anthropic_exports);
-class AnthropicTokenizer {
-  /**
-   * TODO: Replace with a better implementation for anthropic
-   * Approximates token count for Anthropic models.
-   * Assumes roughly 4 characters per token.
-   * @param text Text content to count tokens for
-   * @returns Approximate number of tokens
-   */
-  countTokens(text) {
-    if (!text) return 0;
-    return Math.ceil(text.length / 4);
-  }
-  getProviderName() {
-    return "anthropic";
-  }
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  AnthropicTokenizer
-});

package/dist/llm/tokenizer/anthropic.d.ts

@@ -1,19 +0,0 @@
-import { ITokenizer } from './types.js';
-/**
- * Placeholder tokenizer for Anthropic models.
- * Uses a rough approximation based on character count.
- * TODO: IMPORTANT: This is not accurate and should be replaced
- * with a better implementation or official library when available.
- */
-export declare class AnthropicTokenizer implements ITokenizer {
-    /**
-     * TODO: Replace with a better implementation for anthropic
-     * Approximates token count for Anthropic models.
-     * Assumes roughly 4 characters per token.
-     * @param text Text content to count tokens for
-     * @returns Approximate number of tokens
-     */
-    countTokens(text: string): number;
-    getProviderName(): string;
-}
-//# sourceMappingURL=anthropic.d.ts.map

package/dist/llm/tokenizer/anthropic.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../src/llm/tokenizer/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC;;;;;GAKG;AACH,qBAAa,kBAAmB,YAAW,UAAU;IACjD;;;;;;OAMG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAMjC,eAAe,IAAI,MAAM;CAG5B"}

package/dist/llm/tokenizer/anthropic.js

@@ -1,20 +0,0 @@
-import "../../chunk-C6A6W6XS.js";
-class AnthropicTokenizer {
-  /**
-   * TODO: Replace with a better implementation for anthropic
-   * Approximates token count for Anthropic models.
-   * Assumes roughly 4 characters per token.
-   * @param text Text content to count tokens for
-   * @returns Approximate number of tokens
-   */
-  countTokens(text) {
-    if (!text) return 0;
-    return Math.ceil(text.length / 4);
-  }
-  getProviderName() {
-    return "anthropic";
-  }
-}
-export {
-  AnthropicTokenizer
-};

package/dist/llm/tokenizer/default.d.ts

@@ -1,14 +0,0 @@
-import { ITokenizer } from './types.js';
-/**
- * Placeholder tokenizer for unknown models
- */
-export declare class DefaultTokenizer implements ITokenizer {
-    /**
-     * Assumes roughly 4 characters per token.
-     * @param text Text content to count tokens for
-     * @returns Approximate number of tokens
-     */
-    countTokens(text: string): number;
-    getProviderName(): string;
-}
-//# sourceMappingURL=default.d.ts.map

package/dist/llm/tokenizer/default.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../../../src/llm/tokenizer/default.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC;;GAEG;AACH,qBAAa,gBAAiB,YAAW,UAAU;IAC/C;;;;OAIG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAKjC,eAAe,IAAI,MAAM;CAG5B"}

package/dist/llm/tokenizer/default.js

@@ -1,18 +0,0 @@
-import "../../chunk-C6A6W6XS.js";
-class DefaultTokenizer {
-  /**
-   * Assumes roughly 4 characters per token.
-   * @param text Text content to count tokens for
-   * @returns Approximate number of tokens
-   */
-  countTokens(text) {
-    if (!text) return 0;
-    return Math.ceil(text.length / 4);
-  }
-  getProviderName() {
-    return "default";
-  }
-}
-export {
-  DefaultTokenizer
-};