@xpert-ai/plugin-sdk 3.6.5 → 3.6.7

package/index.cjs.js

@@ -25,7 +25,7 @@ var documents = require('@langchain/core/documents');
 var chat_models = require('@langchain/core/language_models/chat_models');
 var messages = require('@langchain/core/messages');
 var env = require('@langchain/core/utils/env');
-var jsTiktoken = require('js-tiktoken');
+require('js-tiktoken');
 
 function _interopNamespaceDefault(e) {
 	var n = Object.create(null);
@@ -1108,7 +1108,7 @@ exports.DBCreateTableMode = void 0;
 (function(DBCreateTableMode) {
     DBCreateTableMode["ERROR"] = "error";
     DBCreateTableMode["IGNORE"] = "ignore";
-    DBCreateTableMode["UPGRADE"] = "upgrade"; // 表已存在 → 自动升级表结构
+    DBCreateTableMode["UPGRADE"] = "upgrade"; // automatically upgrade table structure
 })(exports.DBCreateTableMode || (exports.DBCreateTableMode = {}));
 exports.DBTableDataAction = void 0;
 (function(DBTableDataAction) {
@@ -2041,56 +2041,12 @@ class Speech2TextChatModel extends chat_models.BaseChatModel {
     }
 }
 
-/**
- * Fallback token estimation method.
- *
- * 思路:
- * - 英文：约 4 chars / token
- * - 中文：约 1.5 chars / token
- * - Mixed：按字符区分
- */ function estimateTokens(text) {
-    if (!text) return 0;
-    let cn = 0;
-    let en = 0;
-    for (const ch of text){
-        if (/[\u4e00-\u9fa5]/.test(ch)) {
-            cn++;
-        } else {
-            en++;
-        }
-    }
-    // 中文 1 token ≈ 1.5 chars
-    const cnTokens = cn / 1.5;
-    // 英文 1 token ≈ 4 chars
-    const enTokens = en / 4;
-    return Math.ceil(cnTokens + enTokens);
-}
 /**
  * Count tokens in text
  * 1) Preferred: js-tiktoken precise encoding
  * 2) Fallback: estimated token count
  */ function countTokensSafe(text, opts) {
-    if (!text) return 0;
-    let resolvedEncoding;
-    // Decide encoding name
-    if (opts == null ? void 0 : opts.encodingName) {
-        resolvedEncoding = String(opts.encodingName);
-    } else if (opts == null ? void 0 : opts.model) {
-        var _getEncodingNameForModel;
-        resolvedEncoding = (_getEncodingNameForModel = jsTiktoken.getEncodingNameForModel(opts.model)) != null ? _getEncodingNameForModel : 'cl100k_base';
-    } else {
-        resolvedEncoding = 'cl100k_base';
-    }
-    try {
-        // Prefer exact token count via js-tiktoken
-        const enc = (opts == null ? void 0 : opts.model) ? jsTiktoken.encodingForModel(opts.model) : jsTiktoken.getEncoding(resolvedEncoding);
-        const tokens = enc.encode(text);
-        return tokens.length;
-    } catch (e) {
-        // Fallback
-        console.warn('[countTokensSafe] tiktoken failed, fallback estimate', e);
-        return estimateTokens(text);
-    }
+    return 0;
 }
 
 Object.defineProperty(exports, "IColumnDef", {

package/index.esm.js

@@ -27,7 +27,7 @@ import { Document } from '@langchain/core/documents';
 import { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import { AIMessage } from '@langchain/core/messages';
 import { getEnvironmentVariable } from '@langchain/core/utils/env';
-import { getEncodingNameForModel, encodingForModel, getEncoding } from 'js-tiktoken';
+import 'js-tiktoken';
 
 /**
  * Metadata keys used in plugins for defining various aspects like entities, subscribers, and configurations.
@@ -1088,7 +1088,7 @@ var DBCreateTableMode;
 (function(DBCreateTableMode) {
     DBCreateTableMode["ERROR"] = "error";
     DBCreateTableMode["IGNORE"] = "ignore";
-    DBCreateTableMode["UPGRADE"] = "upgrade"; // 表已存在 → 自动升级表结构
+    DBCreateTableMode["UPGRADE"] = "upgrade"; // automatically upgrade table structure
 })(DBCreateTableMode || (DBCreateTableMode = {}));
 var DBTableDataAction;
 (function(DBTableDataAction) {
@@ -2021,56 +2021,12 @@ class Speech2TextChatModel extends BaseChatModel {
     }
 }
 
-/**
- * Fallback token estimation method.
- *
- * 思路:
- * - 英文：约 4 chars / token
- * - 中文：约 1.5 chars / token
- * - Mixed：按字符区分
- */ function estimateTokens(text) {
-    if (!text) return 0;
-    let cn = 0;
-    let en = 0;
-    for (const ch of text){
-        if (/[\u4e00-\u9fa5]/.test(ch)) {
-            cn++;
-        } else {
-            en++;
-        }
-    }
-    // 中文 1 token ≈ 1.5 chars
-    const cnTokens = cn / 1.5;
-    // 英文 1 token ≈ 4 chars
-    const enTokens = en / 4;
-    return Math.ceil(cnTokens + enTokens);
-}
 /**
  * Count tokens in text
  * 1) Preferred: js-tiktoken precise encoding
  * 2) Fallback: estimated token count
  */ function countTokensSafe(text, opts) {
-    if (!text) return 0;
-    let resolvedEncoding;
-    // Decide encoding name
-    if (opts == null ? void 0 : opts.encodingName) {
-        resolvedEncoding = String(opts.encodingName);
-    } else if (opts == null ? void 0 : opts.model) {
-        var _getEncodingNameForModel;
-        resolvedEncoding = (_getEncodingNameForModel = getEncodingNameForModel(opts.model)) != null ? _getEncodingNameForModel : 'cl100k_base';
-    } else {
-        resolvedEncoding = 'cl100k_base';
-    }
-    try {
-        // Prefer exact token count via js-tiktoken
-        const enc = (opts == null ? void 0 : opts.model) ? encodingForModel(opts.model) : getEncoding(resolvedEncoding);
-        const tokens = enc.encode(text);
-        return tokens.length;
-    } catch (e) {
-        // Fallback
-        console.warn('[countTokensSafe] tiktoken failed, fallback estimate', e);
-        return estimateTokens(text);
-    }
+    return 0;
 }
 
 export { AIModelProviderNotFoundException, AIModelProviderRegistry, AIModelProviderStrategy, AI_MODEL_PROVIDER, AdapterDataSourceStrategy, AiModelNotFoundException, BaseHTTPQueryRunner, BaseQueryRunner, BaseSQLQueryRunner, BaseTool, BaseToolset, BuiltinToolset, ChatOAICompatReasoningModel, CommonParameterRules, CredentialsValidateFailedError, DATASOURCE_STRATEGY, DBCreateTableMode, DBProtocolEnum, DBSyntaxEnum, DBTableAction, DBTableDataAction, DOCUMENT_SOURCE_STRATEGY, DOCUMENT_TRANSFORMER_STRATEGY, DataSourceStrategy, DataSourceStrategyRegistry, DocumentSourceRegistry, DocumentSourceStrategy, DocumentTransformerRegistry, DocumentTransformerStrategy, IMAGE_UNDERSTANDING_STRATEGY, INTEGRATION_STRATEGY, ImageUnderstandingRegistry, ImageUnderstandingStrategy, IntegrationStrategyKey, IntegrationStrategyRegistry, KNOWLEDGE_STRATEGY, KnowledgeStrategyKey, KnowledgeStrategyRegistry, LLMUsage, LargeLanguageModel, ModelProvider, OpenAICompatibleReranker, PLUGIN_METADATA, PROVIDE_AI_MODEL_LLM, PROVIDE_AI_MODEL_MODERATION, PROVIDE_AI_MODEL_RERANK, PROVIDE_AI_MODEL_SPEECH2TEXT, PROVIDE_AI_MODEL_TEXT_EMBEDDING, PROVIDE_AI_MODEL_TTS, RETRIEVER_STRATEGY, RequestContext, RequestContextMiddleware, RerankModel, RetrieverRegistry, RetrieverStrategy, Speech2TextChatModel, SpeechToTextModel, TEXT_SPLITTER_STRATEGY, TOOLSET_STRATEGY, TextEmbeddingModelManager, TextSplitterRegistry, TextSplitterStrategy, TextToSpeechModel, ToolsetRegistry, ToolsetStrategy, VECTOR_STORE_STRATEGY, VectorStoreRegistry, VectorStoreStrategy, WORKFLOW_NODE_STRATEGY, WORKFLOW_TRIGGER_STRATEGY, WorkflowNodeRegistry, WorkflowNodeStrategy, WorkflowTriggerRegistry, WorkflowTriggerStrategy, XpFileSystem, XpertServerPlugin, als, calcTokenUsage, countTokensSafe, createI18nInstance, createPluginLogger, downloadRemoteFile, getErrorMessage, getPositionList, getPositionMap, getRequestContext, isRemoteFile, loadYamlFile, mergeCredentials, mergeParentChildChunks, runWithRequestContext, sumTokenUsage };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xpert-ai/plugin-sdk",
-  "version": "3.6.5",
+  "version": "3.6.7",
   "license": "AGPL-3.0",
   "repository": {
     "type": "git",

package/src/lib/data/datasource/types.d.ts

@@ -306,9 +306,12 @@ export interface DBIndexDefinition {
     unique?: boolean;
     type?: 'btree' | 'hash' | 'gin' | 'bitmap' | 'fulltext' | string;
 }
+/**
+ * Modes for creating a table, if the table already exists
+ */
 export declare enum DBCreateTableMode {
-    ERROR = "error",// 表已存在 → 报错
-    IGNORE = "ignore",// 表已存在 → 什么也不做
+    ERROR = "error",// throw error
+    IGNORE = "ignore",// do nothing
     UPGRADE = "upgrade"
 }
 export declare enum DBTableDataAction {