@dexto/core 1.3.0 → 1.4.0

package/dist/llm/tokenizer/factory.cjs

@@ -1,44 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var factory_exports = {};
-__export(factory_exports, {
-  createTokenizer: () => createTokenizer
-});
-module.exports = __toCommonJS(factory_exports);
-var import_openai = require("./openai.js");
-var import_anthropic = require("./anthropic.js");
-var import_google = require("./google.js");
-var import_default = require("./default.js");
-function createTokenizer(provider, model, logger) {
-  switch (provider) {
-    case "openai":
-      return new import_openai.OpenAITokenizer(model, logger);
-    case "anthropic":
-      return new import_anthropic.AnthropicTokenizer();
-    case "google":
-      return new import_google.GoogleTokenizer(model);
-    // Add cases for other providers here
-    default:
-      return new import_default.DefaultTokenizer();
-  }
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  createTokenizer
-});

package/dist/llm/tokenizer/factory.d.ts

@@ -1,12 +0,0 @@
-import { ITokenizer } from './types.js';
-import type { LLMProvider } from '../types.js';
-import type { IDextoLogger } from '../../logger/v2/types.js';
-/**
- * Creates the appropriate tokenizer for the specified provider and model
- * @param provider The LLM provider name (case-insensitive)
- * @param model The specific model name (used by some tokenizers)
- * @param logger The logger instance for logging
- * @returns An appropriate tokenizer implementation, or DefaultTokenizer if no specific implementation exists
- */
-export declare function createTokenizer(provider: LLMProvider, model: string, logger: IDextoLogger): ITokenizer;
-//# sourceMappingURL=factory.d.ts.map

package/dist/llm/tokenizer/factory.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../../src/llm/tokenizer/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAKxC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAE7D;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC3B,QAAQ,EAAE,WAAW,EACrB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,YAAY,GACrB,UAAU,CAcZ"}

package/dist/llm/tokenizer/factory.js

@@ -1,21 +0,0 @@
-import "../../chunk-C6A6W6XS.js";
-import { OpenAITokenizer } from "./openai.js";
-import { AnthropicTokenizer } from "./anthropic.js";
-import { GoogleTokenizer } from "./google.js";
-import { DefaultTokenizer } from "./default.js";
-function createTokenizer(provider, model, logger) {
-  switch (provider) {
-    case "openai":
-      return new OpenAITokenizer(model, logger);
-    case "anthropic":
-      return new AnthropicTokenizer();
-    case "google":
-      return new GoogleTokenizer(model);
-    // Add cases for other providers here
-    default:
-      return new DefaultTokenizer();
-  }
-}
-export {
-  createTokenizer
-};

package/dist/llm/tokenizer/google.cjs

@@ -1,52 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var google_exports = {};
-__export(google_exports, {
-  GoogleTokenizer: () => GoogleTokenizer
-});
-module.exports = __toCommonJS(google_exports);
-class GoogleTokenizer {
-  modelName;
-  /**
-   * Initializes the tokenizer for a specific Google Gemini model.
-   * Currently, the model name is not used due to the approximation.
-   * @param model The Google model name (e.g., 'gemini-pro')
-   */
-  constructor(model) {
-    this.modelName = model;
-  }
-  /**
-   * Approximates the token count for Google Gemini models.
-   * Uses a rough guess based on character count (similar to Anthropic's approximation).
-   * IMPORTANT: This is NOT accurate for Gemini models.
-   * @param text Text content to count tokens for
-   * @returns Approximate number of tokens
-   */
-  countTokens(text) {
-    if (!text) return 0;
-    return Math.ceil(text.length / 4);
-  }
-  getProviderName() {
-    return "google";
-  }
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  GoogleTokenizer
-});

package/dist/llm/tokenizer/google.d.ts

@@ -1,29 +0,0 @@
-import { ITokenizer } from './types.js';
-/**
- * Tokenizer for Google Gemini models - CURRENTLY USING APPROXIMATION.
- *
- * TODO: Replace this with a proper token counting implementation.
- * The official `@google/generative-ai` library's `countTokens` function
- * is asynchronous, which complicates synchronous use cases.
- * Consider using a WASM-based tokenizer or re-evaluating the need
- * for synchronous counting if a more accurate method is required.
- */
-export declare class GoogleTokenizer implements ITokenizer {
-    private modelName;
-    /**
-     * Initializes the tokenizer for a specific Google Gemini model.
-     * Currently, the model name is not used due to the approximation.
-     * @param model The Google model name (e.g., 'gemini-pro')
-     */
-    constructor(model: string);
-    /**
-     * Approximates the token count for Google Gemini models.
-     * Uses a rough guess based on character count (similar to Anthropic's approximation).
-     * IMPORTANT: This is NOT accurate for Gemini models.
-     * @param text Text content to count tokens for
-     * @returns Approximate number of tokens
-     */
-    countTokens(text: string): number;
-    getProviderName(): string;
-}
-//# sourceMappingURL=google.d.ts.map

package/dist/llm/tokenizer/google.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"google.d.ts","sourceRoot":"","sources":["../../../src/llm/tokenizer/google.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC;;;;;;;;GAQG;AACH,qBAAa,eAAgB,YAAW,UAAU;IAC9C,OAAO,CAAC,SAAS,CAAS;IAE1B;;;;OAIG;gBACS,KAAK,EAAE,MAAM;IAMzB;;;;;;OAMG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAOjC,eAAe,IAAI,MAAM;CAG5B"}

package/dist/llm/tokenizer/google.js

@@ -1,29 +0,0 @@
-import "../../chunk-C6A6W6XS.js";
-class GoogleTokenizer {
-  modelName;
-  /**
-   * Initializes the tokenizer for a specific Google Gemini model.
-   * Currently, the model name is not used due to the approximation.
-   * @param model The Google model name (e.g., 'gemini-pro')
-   */
-  constructor(model) {
-    this.modelName = model;
-  }
-  /**
-   * Approximates the token count for Google Gemini models.
-   * Uses a rough guess based on character count (similar to Anthropic's approximation).
-   * IMPORTANT: This is NOT accurate for Gemini models.
-   * @param text Text content to count tokens for
-   * @returns Approximate number of tokens
-   */
-  countTokens(text) {
-    if (!text) return 0;
-    return Math.ceil(text.length / 4);
-  }
-  getProviderName() {
-    return "google";
-  }
-}
-export {
-  GoogleTokenizer
-};

package/dist/llm/tokenizer/openai.cjs

@@ -1,115 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var openai_exports = {};
-__export(openai_exports, {
-  OpenAITokenizer: () => OpenAITokenizer
-});
-module.exports = __toCommonJS(openai_exports);
-var import_types = require("./types.js");
-var import_module = require("module");
-var import_types2 = require("../../logger/v2/types.js");
-const import_meta = {};
-const FALLBACK_ENCODING = "cl100k_base";
-class OpenAITokenizer {
-  modelName;
-  // Store original model name for context/logging
-  encoding;
-  // Tiktoken encoding instance
-  logger;
-  /**
-   * Initializes the tokenizer for a specific OpenAI model or compatible model.
-   * @param model The OpenAI model name (e.g., 'gpt-5') or a custom model name.
-   * @param logger The logger instance for logging.
-   * @throws TokenizationError if tiktoken initialization fails for both specific model and fallback.
-   */
-  constructor(model, logger) {
-    this.modelName = model;
-    this.logger = logger.createChild(import_types2.DextoLogComponent.LLM);
-    try {
-      const { encoding_for_model } = loadTiktoken();
-      this.encoding = encoding_for_model(model);
-      this.logger.debug(`Initialized tiktoken with specific encoding for model: ${model}`);
-    } catch (error) {
-      this.logger.warn(
-        `Could not get specific encoding for model '${this.modelName}'. Falling back to '${FALLBACK_ENCODING}'. Error: ${error instanceof Error ? error.message : String(error)}`
-      );
-      try {
-        const { get_encoding } = loadTiktoken();
-        this.encoding = get_encoding(FALLBACK_ENCODING);
-        this.logger.debug(
-          `Initialized tiktoken with fallback encoding: ${FALLBACK_ENCODING}`
-        );
-      } catch (fallbackError) {
-        this.logger.error(
-          `Failed to initialize tiktoken with specific model '${this.modelName}' or fallback '${FALLBACK_ENCODING}'.`,
-          {
-            error: fallbackError instanceof Error ? fallbackError.message : String(fallbackError)
-          }
-        );
-        throw new import_types.TokenizationError(
-          `Failed to initialize tiktoken for model '${this.modelName}' using specific or fallback encoding ('${FALLBACK_ENCODING}'): ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`
-        );
-      }
-    }
-  }
-  /**
-   * Counts the tokens in the text using the determined encoding.
-   * @param text Text content to count tokens for
-   * @returns Number of tokens
-   * @throws TokenizationError if encoding fails during counting.
-   */
-  countTokens(text) {
-    if (!text) return 0;
-    try {
-      const tokens = this.encoding.encode(text);
-      return tokens.length;
-    } catch (error) {
-      this.logger.error(
-        `Tiktoken encoding failed for model ${this.modelName} (using encoding: ${this.encoding.name}):`,
-        { error: error instanceof Error ? error.message : String(error) }
-      );
-      throw new import_types.TokenizationError(
-        `Encoding failed for text snippet using model ${this.modelName}.`
-      );
-    }
-  }
-  /**
-   * Cleans up the tiktoken encoding instance when done.
-   * Recommended by tiktoken library.
-   */
-  free() {
-    if (this.encoding) {
-      this.encoding.free();
-    }
-  }
-  getProviderName() {
-    return "openai";
-  }
-}
-let _tiktoken = null;
-function loadTiktoken() {
-  if (_tiktoken) return _tiktoken;
-  const req = (0, import_module.createRequire)(import_meta.url);
-  _tiktoken = req("tiktoken");
-  return _tiktoken;
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  OpenAITokenizer
-});

package/dist/llm/tokenizer/openai.d.ts

@@ -1,41 +0,0 @@
-import { ITokenizer } from './types.js';
-import type { IDextoLogger } from '../../logger/v2/types.js';
-/**
- * Tokenizer for OpenAI models using the tiktoken library.
- *
- * TODO: Make tiktoken an optional peer dependency to reduce package size by ~23MB
- * tiktoken includes WASM binaries (5.3MB) and encoder data files (16MB).
- * It's only needed for accurate pre-request token estimation for OpenAI models.
- * - All providers return actual token counts in API responses
- * - Anthropic/Google/others already use simple text.length/4 approximation
- * - Consider falling back to DefaultTokenizer if tiktoken is not installed
- * See: packages/core/src/llm/tokenizer/factory.ts
- * Attempts to use the specific model's encoding, falls back to a common base encoding ('cl100k_base')
- * for unknown or custom model names (often used with custom baseURLs).
- */
-export declare class OpenAITokenizer implements ITokenizer {
-    private modelName;
-    private encoding;
-    private logger;
-    /**
-     * Initializes the tokenizer for a specific OpenAI model or compatible model.
-     * @param model The OpenAI model name (e.g., 'gpt-5') or a custom model name.
-     * @param logger The logger instance for logging.
-     * @throws TokenizationError if tiktoken initialization fails for both specific model and fallback.
-     */
-    constructor(model: string, logger: IDextoLogger);
-    /**
-     * Counts the tokens in the text using the determined encoding.
-     * @param text Text content to count tokens for
-     * @returns Number of tokens
-     * @throws TokenizationError if encoding fails during counting.
-     */
-    countTokens(text: string): number;
-    /**
-     * Cleans up the tiktoken encoding instance when done.
-     * Recommended by tiktoken library.
-     */
-    free(): void;
-    getProviderName(): string;
-}
-//# sourceMappingURL=openai.d.ts.map

package/dist/llm/tokenizer/openai.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/llm/tokenizer/openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAqB,MAAM,YAAY,CAAC;AAG3D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAM7D;;;;;;;;;;;;GAYG;AACH,qBAAa,eAAgB,YAAW,UAAU;IAC9C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,MAAM,CAAe;IAE7B;;;;;OAKG;gBACS,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY;IAqC/C;;;;;OAKG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAgBjC;;;OAGG;IACH,IAAI,IAAI,IAAI;IAMZ,eAAe,IAAI,MAAM;CAG5B"}

package/dist/llm/tokenizer/openai.js

@@ -1,91 +0,0 @@
-import "../../chunk-C6A6W6XS.js";
-import { TokenizationError } from "./types.js";
-import { createRequire } from "module";
-import { DextoLogComponent } from "../../logger/v2/types.js";
-const FALLBACK_ENCODING = "cl100k_base";
-class OpenAITokenizer {
-  modelName;
-  // Store original model name for context/logging
-  encoding;
-  // Tiktoken encoding instance
-  logger;
-  /**
-   * Initializes the tokenizer for a specific OpenAI model or compatible model.
-   * @param model The OpenAI model name (e.g., 'gpt-5') or a custom model name.
-   * @param logger The logger instance for logging.
-   * @throws TokenizationError if tiktoken initialization fails for both specific model and fallback.
-   */
-  constructor(model, logger) {
-    this.modelName = model;
-    this.logger = logger.createChild(DextoLogComponent.LLM);
-    try {
-      const { encoding_for_model } = loadTiktoken();
-      this.encoding = encoding_for_model(model);
-      this.logger.debug(`Initialized tiktoken with specific encoding for model: ${model}`);
-    } catch (error) {
-      this.logger.warn(
-        `Could not get specific encoding for model '${this.modelName}'. Falling back to '${FALLBACK_ENCODING}'. Error: ${error instanceof Error ? error.message : String(error)}`
-      );
-      try {
-        const { get_encoding } = loadTiktoken();
-        this.encoding = get_encoding(FALLBACK_ENCODING);
-        this.logger.debug(
-          `Initialized tiktoken with fallback encoding: ${FALLBACK_ENCODING}`
-        );
-      } catch (fallbackError) {
-        this.logger.error(
-          `Failed to initialize tiktoken with specific model '${this.modelName}' or fallback '${FALLBACK_ENCODING}'.`,
-          {
-            error: fallbackError instanceof Error ? fallbackError.message : String(fallbackError)
-          }
-        );
-        throw new TokenizationError(
-          `Failed to initialize tiktoken for model '${this.modelName}' using specific or fallback encoding ('${FALLBACK_ENCODING}'): ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`
-        );
-      }
-    }
-  }
-  /**
-   * Counts the tokens in the text using the determined encoding.
-   * @param text Text content to count tokens for
-   * @returns Number of tokens
-   * @throws TokenizationError if encoding fails during counting.
-   */
-  countTokens(text) {
-    if (!text) return 0;
-    try {
-      const tokens = this.encoding.encode(text);
-      return tokens.length;
-    } catch (error) {
-      this.logger.error(
-        `Tiktoken encoding failed for model ${this.modelName} (using encoding: ${this.encoding.name}):`,
-        { error: error instanceof Error ? error.message : String(error) }
-      );
-      throw new TokenizationError(
-        `Encoding failed for text snippet using model ${this.modelName}.`
-      );
-    }
-  }
-  /**
-   * Cleans up the tiktoken encoding instance when done.
-   * Recommended by tiktoken library.
-   */
-  free() {
-    if (this.encoding) {
-      this.encoding.free();
-    }
-  }
-  getProviderName() {
-    return "openai";
-  }
-}
-let _tiktoken = null;
-function loadTiktoken() {
-  if (_tiktoken) return _tiktoken;
-  const req = createRequire(import.meta.url);
-  _tiktoken = req("tiktoken");
-  return _tiktoken;
-}
-export {
-  OpenAITokenizer
-};

package/dist/llm/tokenizer/types.d.ts

@@ -1,18 +0,0 @@
-export interface ITokenizer {
-    /**
-     * Counts the number of tokens in the provided text according
-     * to the specific LLM provider's tokenization rules
-     * @param text Text content to count tokens for
-     * @returns Number of tokens in the text
-     */
-    countTokens(text: string): number;
-    /**
-     * Gets the name of the LLM provider this tokenizer is for
-     * @returns Provider name (e.g., "openai", "anthropic")
-     */
-    getProviderName(): string;
-}
-export declare class TokenizationError extends Error {
-    constructor(message: string);
-}
-//# sourceMappingURL=types.d.ts.map

package/dist/llm/tokenizer/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/llm/tokenizer/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACvB;;;;;OAKG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAElC;;;OAGG;IACH,eAAe,IAAI,MAAM,CAAC;CAC7B;AAED,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,EAAE,MAAM;CAI9B"}

package/dist/llm/tokenizer/types.js

@@ -1,10 +0,0 @@
-import "../../chunk-C6A6W6XS.js";
-class TokenizationError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "TokenizationError";
-  }
-}
-export {
-  TokenizationError
-};