@hibanacloud/core 0.3.15

package/dist/src/utils/request-tokenizer/textTokenizer.js

@@ -0,0 +1,88 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { get_encoding } from 'tiktoken';
+/**
+ * Text tokenizer for calculating text tokens using tiktoken
+ */
+export class TextTokenizer {
+    encoding = null;
+    encodingName;
+    constructor(encodingName = 'cl100k_base') {
+        this.encodingName = encodingName;
+    }
+    /**
+     * Initialize the tokenizer (lazy loading)
+     */
+    async ensureEncoding() {
+        if (this.encoding)
+            return;
+        try {
+            // Use type assertion since we know the encoding name is valid
+            this.encoding = get_encoding(this.encodingName);
+        }
+        catch (error) {
+            console.warn(`Failed to load tiktoken with encoding ${this.encodingName}:`, error);
+            this.encoding = null;
+        }
+    }
+    /**
+     * Calculate tokens for text content
+     */
+    async calculateTokens(text) {
+        if (!text)
+            return 0;
+        await this.ensureEncoding();
+        if (this.encoding) {
+            try {
+                return this.encoding.encode(text).length;
+            }
+            catch (error) {
+                console.warn('Error encoding text with tiktoken:', error);
+            }
+        }
+        // Fallback: rough approximation using character count
+        // This is a conservative estimate: 1 token ≈ 4 characters for most languages
+        return Math.ceil(text.length / 4);
+    }
+    /**
+     * Calculate tokens for multiple text strings in parallel
+     */
+    async calculateTokensBatch(texts) {
+        await this.ensureEncoding();
+        if (this.encoding) {
+            try {
+                return texts.map((text) => {
+                    if (!text)
+                        return 0;
+                    // this.encoding may be null, add a null check to satisfy lint
+                    return this.encoding ? this.encoding.encode(text).length : 0;
+                });
+            }
+            catch (error) {
+                console.warn('Error encoding texts with tiktoken:', error);
+                // In case of error, return fallback estimation for all texts
+                return texts.map((text) => Math.ceil((text || '').length / 4));
+            }
+        }
+        // Fallback for batch processing
+        return texts.map((text) => Math.ceil((text || '').length / 4));
+    }
+    /**
+     * Dispose of resources
+     */
+    dispose() {
+        if (this.encoding) {
+            try {
+                this.encoding.free();
+            }
+            catch (error) {
+                console.warn('Error freeing tiktoken encoding:', error);
+            }
+            this.encoding = null;
+        }
+    }
+}
+//# sourceMappingURL=textTokenizer.js.map

package/dist/src/utils/request-tokenizer/textTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textTokenizer.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/textTokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC;;GAEG;AACH,MAAM,OAAO,aAAa;IAChB,QAAQ,GAAoB,IAAI,CAAC;IACjC,YAAY,CAAS;IAE7B,YAAY,eAAuB,aAAa;QAC9C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,IAAI,CAAC;YACH,8DAA8D;YAC9D,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,YAAgC,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CACV,yCAAyC,IAAI,CAAC,YAAY,GAAG,EAC7D,KAAK,CACN,CAAC;YACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAEpB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YAC3C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,sDAAsD;QACtD,6EAA6E;QAC7E,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,KAAe;QACxC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBACxB,IAAI,CAAC,IAAI;wBAAE,OAAO,CAAC,CAAC;oBACpB,8DAA8D;oBAC9D,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/D,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;gBAC3D,6DAA6D;gBAC7D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;CACF"}

package/dist/src/utils/request-tokenizer/textTokenizer.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/utils/request-tokenizer/textTokenizer.test.js

@@ -0,0 +1,253 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { TextTokenizer } from './textTokenizer.js';
+// Mock tiktoken at the top level with hoisted functions
+const mockEncode = vi.hoisted(() => vi.fn());
+const mockFree = vi.hoisted(() => vi.fn());
+const mockGetEncoding = vi.hoisted(() => vi.fn());
+vi.mock('tiktoken', () => ({
+    get_encoding: mockGetEncoding,
+}));
+describe('TextTokenizer', () => {
+    let tokenizer;
+    let consoleWarnSpy;
+    beforeEach(() => {
+        vi.resetAllMocks();
+        consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => { });
+        // Default mock implementation
+        mockGetEncoding.mockReturnValue({
+            encode: mockEncode,
+            free: mockFree,
+        });
+    });
+    afterEach(() => {
+        vi.restoreAllMocks();
+        tokenizer?.dispose();
+    });
+    describe('constructor', () => {
+        it('should create tokenizer with default encoding', () => {
+            tokenizer = new TextTokenizer();
+            expect(tokenizer).toBeInstanceOf(TextTokenizer);
+        });
+        it('should create tokenizer with custom encoding', () => {
+            tokenizer = new TextTokenizer('gpt2');
+            expect(tokenizer).toBeInstanceOf(TextTokenizer);
+        });
+    });
+    describe('calculateTokens', () => {
+        beforeEach(() => {
+            tokenizer = new TextTokenizer();
+        });
+        it('should return 0 for empty text', async () => {
+            const result = await tokenizer.calculateTokens('');
+            expect(result).toBe(0);
+        });
+        it('should return 0 for null/undefined text', async () => {
+            const result1 = await tokenizer.calculateTokens(null);
+            const result2 = await tokenizer.calculateTokens(undefined);
+            expect(result1).toBe(0);
+            expect(result2).toBe(0);
+        });
+        it('should calculate tokens using tiktoken when available', async () => {
+            const testText = 'Hello, world!';
+            const mockTokens = [1, 2, 3, 4, 5]; // 5 tokens
+            mockEncode.mockReturnValue(mockTokens);
+            const result = await tokenizer.calculateTokens(testText);
+            expect(mockGetEncoding).toHaveBeenCalledWith('cl100k_base');
+            expect(mockEncode).toHaveBeenCalledWith(testText);
+            expect(result).toBe(5);
+        });
+        it('should use fallback calculation when tiktoken fails to load', async () => {
+            mockGetEncoding.mockImplementation(() => {
+                throw new Error('Failed to load tiktoken');
+            });
+            const testText = 'Hello, world!'; // 13 characters
+            const result = await tokenizer.calculateTokens(testText);
+            expect(consoleWarnSpy).toHaveBeenCalledWith('Failed to load tiktoken with encoding cl100k_base:', expect.any(Error));
+            // Fallback: Math.ceil(13 / 4) = 4
+            expect(result).toBe(4);
+        });
+        it('should use fallback calculation when encoding fails', async () => {
+            mockEncode.mockImplementation(() => {
+                throw new Error('Encoding failed');
+            });
+            const testText = 'Hello, world!'; // 13 characters
+            const result = await tokenizer.calculateTokens(testText);
+            expect(consoleWarnSpy).toHaveBeenCalledWith('Error encoding text with tiktoken:', expect.any(Error));
+            // Fallback: Math.ceil(13 / 4) = 4
+            expect(result).toBe(4);
+        });
+        it('should handle very long text', async () => {
+            const longText = 'a'.repeat(10000);
+            const mockTokens = new Array(2500); // 2500 tokens
+            mockEncode.mockReturnValue(mockTokens);
+            const result = await tokenizer.calculateTokens(longText);
+            expect(result).toBe(2500);
+        });
+        it('should handle unicode characters', async () => {
+            const unicodeText = '你好世界 🌍';
+            const mockTokens = [1, 2, 3, 4, 5, 6];
+            mockEncode.mockReturnValue(mockTokens);
+            const result = await tokenizer.calculateTokens(unicodeText);
+            expect(result).toBe(6);
+        });
+        it('should use custom encoding when specified', async () => {
+            tokenizer = new TextTokenizer('gpt2');
+            const testText = 'Hello, world!';
+            const mockTokens = [1, 2, 3];
+            mockEncode.mockReturnValue(mockTokens);
+            const result = await tokenizer.calculateTokens(testText);
+            expect(mockGetEncoding).toHaveBeenCalledWith('gpt2');
+            expect(result).toBe(3);
+        });
+    });
+    describe('calculateTokensBatch', () => {
+        beforeEach(() => {
+            tokenizer = new TextTokenizer();
+        });
+        it('should process multiple texts and return token counts', async () => {
+            const texts = ['Hello', 'world', 'test'];
+            mockEncode
+                .mockReturnValueOnce([1, 2]) // 2 tokens for 'Hello'
+                .mockReturnValueOnce([3, 4, 5]) // 3 tokens for 'world'
+                .mockReturnValueOnce([6]); // 1 token for 'test'
+            const result = await tokenizer.calculateTokensBatch(texts);
+            expect(result).toEqual([2, 3, 1]);
+            expect(mockEncode).toHaveBeenCalledTimes(3);
+        });
+        it('should handle empty array', async () => {
+            const result = await tokenizer.calculateTokensBatch([]);
+            expect(result).toEqual([]);
+        });
+        it('should handle array with empty strings', async () => {
+            const texts = ['', 'hello', ''];
+            mockEncode.mockReturnValue([1, 2, 3]); // Only called for 'hello'
+            const result = await tokenizer.calculateTokensBatch(texts);
+            expect(result).toEqual([0, 3, 0]);
+            expect(mockEncode).toHaveBeenCalledTimes(1);
+            expect(mockEncode).toHaveBeenCalledWith('hello');
+        });
+        it('should use fallback calculation when tiktoken fails to load', async () => {
+            mockGetEncoding.mockImplementation(() => {
+                throw new Error('Failed to load tiktoken');
+            });
+            const texts = ['Hello', 'world']; // 5 and 5 characters
+            const result = await tokenizer.calculateTokensBatch(texts);
+            expect(consoleWarnSpy).toHaveBeenCalledWith('Failed to load tiktoken with encoding cl100k_base:', expect.any(Error));
+            // Fallback: Math.ceil(5/4) = 2 for both
+            expect(result).toEqual([2, 2]);
+        });
+        it('should use fallback calculation when encoding fails during batch processing', async () => {
+            mockEncode.mockImplementation(() => {
+                throw new Error('Encoding failed');
+            });
+            const texts = ['Hello', 'world']; // 5 and 5 characters
+            const result = await tokenizer.calculateTokensBatch(texts);
+            expect(consoleWarnSpy).toHaveBeenCalledWith('Error encoding texts with tiktoken:', expect.any(Error));
+            // Fallback: Math.ceil(5/4) = 2 for both
+            expect(result).toEqual([2, 2]);
+        });
+        it('should handle null and undefined values in batch', async () => {
+            const texts = [null, 'hello', undefined, 'world'];
+            mockEncode
+                .mockReturnValueOnce([1, 2, 3]) // 3 tokens for 'hello'
+                .mockReturnValueOnce([4, 5]); // 2 tokens for 'world'
+            const result = await tokenizer.calculateTokensBatch(texts);
+            expect(result).toEqual([0, 3, 0, 2]);
+        });
+    });
+    describe('dispose', () => {
+        beforeEach(() => {
+            tokenizer = new TextTokenizer();
+        });
+        it('should free tiktoken encoding when disposing', async () => {
+            // Initialize the encoding by calling calculateTokens
+            await tokenizer.calculateTokens('test');
+            tokenizer.dispose();
+            expect(mockFree).toHaveBeenCalled();
+        });
+        it('should handle disposal when encoding is not initialized', () => {
+            expect(() => tokenizer.dispose()).not.toThrow();
+            expect(mockFree).not.toHaveBeenCalled();
+        });
+        it('should handle disposal when encoding is null', async () => {
+            // Force encoding to be null by making tiktoken fail
+            mockGetEncoding.mockImplementation(() => {
+                throw new Error('Failed to load');
+            });
+            await tokenizer.calculateTokens('test');
+            expect(() => tokenizer.dispose()).not.toThrow();
+            expect(mockFree).not.toHaveBeenCalled();
+        });
+        it('should handle errors during disposal gracefully', async () => {
+            await tokenizer.calculateTokens('test');
+            mockFree.mockImplementation(() => {
+                throw new Error('Free failed');
+            });
+            tokenizer.dispose();
+            expect(consoleWarnSpy).toHaveBeenCalledWith('Error freeing tiktoken encoding:', expect.any(Error));
+        });
+        it('should allow multiple calls to dispose', async () => {
+            await tokenizer.calculateTokens('test');
+            tokenizer.dispose();
+            tokenizer.dispose(); // Second call should not throw
+            expect(mockFree).toHaveBeenCalledTimes(1);
+        });
+    });
+    describe('lazy initialization', () => {
+        beforeEach(() => {
+            tokenizer = new TextTokenizer();
+        });
+        it('should not initialize tiktoken until first use', () => {
+            expect(mockGetEncoding).not.toHaveBeenCalled();
+        });
+        it('should initialize tiktoken on first calculateTokens call', async () => {
+            await tokenizer.calculateTokens('test');
+            expect(mockGetEncoding).toHaveBeenCalledTimes(1);
+        });
+        it('should not reinitialize tiktoken on subsequent calls', async () => {
+            await tokenizer.calculateTokens('test1');
+            await tokenizer.calculateTokens('test2');
+            expect(mockGetEncoding).toHaveBeenCalledTimes(1);
+        });
+        it('should initialize tiktoken on first calculateTokensBatch call', async () => {
+            await tokenizer.calculateTokensBatch(['test']);
+            expect(mockGetEncoding).toHaveBeenCalledTimes(1);
+        });
+    });
+    describe('edge cases', () => {
+        beforeEach(() => {
+            tokenizer = new TextTokenizer();
+        });
+        it('should handle very short text', async () => {
+            const result = await tokenizer.calculateTokens('a');
+            if (mockGetEncoding.mock.calls.length > 0) {
+                // If tiktoken was called, use its result
+                expect(mockEncode).toHaveBeenCalledWith('a');
+            }
+            else {
+                // If tiktoken failed, should use fallback: Math.ceil(1/4) = 1
+                expect(result).toBe(1);
+            }
+        });
+        it('should handle text with only whitespace', async () => {
+            const whitespaceText = '   \n\t  ';
+            const mockTokens = [1];
+            mockEncode.mockReturnValue(mockTokens);
+            const result = await tokenizer.calculateTokens(whitespaceText);
+            expect(result).toBe(1);
+        });
+        it('should handle special characters and symbols', async () => {
+            const specialText = '!@#$%^&*()_+-=[]{}|;:,.<>?';
+            const mockTokens = new Array(10);
+            mockEncode.mockReturnValue(mockTokens);
+            const result = await tokenizer.calculateTokens(specialText);
+            expect(result).toBe(10);
+        });
+    });
+});
+//# sourceMappingURL=textTokenizer.test.js.map

package/dist/src/utils/request-tokenizer/textTokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textTokenizer.test.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/textTokenizer.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,wDAAwD;AACxD,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7C,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAC3C,MAAM,eAAe,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAElD,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;IACzB,YAAY,EAAE,eAAe;CAC9B,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,IAAI,SAAwB,CAAC;IAC7B,IAAI,cAA2C,CAAC;IAEhD,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,cAAc,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAExE,8BAA8B;QAC9B,eAAe,CAAC,eAAe,CAAC;YAC9B,MAAM,EAAE,UAAU;YAClB,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,EAAE,CAAC,eAAe,EAAE,CAAC;QACrB,SAAS,EAAE,OAAO,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,CAAC,SAAS,CAAC,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,SAAS,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,CAAC,SAAS,CAAC,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,GAAG,EAAE;YACd,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,eAAe,CAC7C,IAAyB,CAC1B,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,eAAe,CAC7C,SAA8B,CAC/B,CAAC;YACF,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,QAAQ,GAAG,eAAe,CAAC;YACjC,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW;YAC/C,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAEzD,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YAC5D,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,eAAe,CAAC,kBAAkB,CAAC,GAAG,EAAE;gBACtC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,gBAAgB;YAClD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAEzD,MAAM,CAAC,cAAc,CAAC,CAAC,oBAAoB,CACzC,oDAAoD,EACpD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAClB,CAAC;YACF,kCAAkC;YAClC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;YACnE,UAAU,CAAC,kBAAkB,CAAC,GAAG,EAAE;gBACjC,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,gBAAgB;YAClD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAEzD,MAAM,CAAC,cAAc,CAAC,CAAC,oBAAoB,CACzC,oCAAoC,EACpC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAClB,CAAC;YACF,kCAAkC;YAClC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc;YAClD,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAEzD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,WAAW,GAAG,SAAS,CAAC;YAC9B,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACtC,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YAE5D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,SAAS,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,eAAe,CAAC;YACjC,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC7B,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAEzD,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,UAAU,CAAC,GAAG,EAAE;YACd,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzC,UAAU;iBACP,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,uBAAuB;iBACnD,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,uBAAuB;iBACtD,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB;YAElD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,KAAK,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;YAChC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,0BAA0B;YAEjE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,eAAe,CAAC,kBAAkB,CAAC,GAAG,EAAE;gBACtC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,qBAAqB;YACvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE3D,MAAM,CAAC,cAAc,CAAC,CAAC,oBAAoB,CACzC,oDAAoD,EACpD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAClB,CAAC;YACF,wCAAwC;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;YAC3F,UAAU,CAAC,kBAAkB,CAAC,GAAG,EAAE;gBACjC,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,qBAAqB;YACvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE3D,MAAM,CAAC,cAAc,CAAC,CAAC,oBAAoB,CACzC,qCAAqC,EACrC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAClB,CAAC;YACF,wCAAwC;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAwB,CAAC;YACzE,UAAU;iBACP,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,uBAAuB;iBACtD,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB;YAEvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,UAAU,CAAC,GAAG,EAAE;YACd,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,qDAAqD;YACrD,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YAExC,SAAS,CAAC,OAAO,EAAE,CAAC;YAEpB,MAAM,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;YACjE,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAChD,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,oDAAoD;YACpD,eAAe,CAAC,kBAAkB,CAAC,GAAG,EAAE;gBACtC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;YAEH,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YAExC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAChD,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YAExC,QAAQ,CAAC,kBAAkB,CAAC,GAAG,EAAE;gBAC/B,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,SAAS,CAAC,OAAO,EAAE,CAAC;YAEpB,MAAM,CAAC,cAAc,CAAC,CAAC,oBAAoB,CACzC,kCAAkC,EAClC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YAExC,SAAS,CAAC,OAAO,EAAE,CAAC;YACpB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,+BAA+B;YAEpD,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,UAAU,CAAC,GAAG,EAAE;YACd,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,CAAC,eAAe,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACzC,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAEzC,MAAM,CAAC,eAAe,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,MAAM,SAAS,CAAC,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,eAAe,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,UAAU,CAAC,GAAG,EAAE;YACd,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,yCAAyC;gBACzC,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,8DAA8D;gBAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,cAAc,GAAG,WAAW,CAAC;YACnC,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC;YACvB,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,WAAW,GAAG,4BAA4B,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC;YACjC,UAAU,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YAE5D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/utils/request-tokenizer/types.d.ts

@@ -0,0 +1,55 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import type { CountTokensParameters } from '@google/genai';
+/**
+ * Token calculation result for different content types
+ */
+export interface TokenCalculationResult {
+    /** Total tokens calculated */
+    totalTokens: number;
+    /** Breakdown by content type */
+    breakdown: {
+        textTokens: number;
+        imageTokens: number;
+        audioTokens: number;
+        otherTokens: number;
+    };
+    /** Processing time in milliseconds */
+    processingTime: number;
+}
+/**
+ * Configuration for token calculation
+ */
+export interface TokenizerConfig {
+    /** Custom text tokenizer encoding (defaults to cl100k_base) */
+    textEncoding?: string;
+}
+/**
+ * Image metadata extracted from base64 data
+ */
+export interface ImageMetadata {
+    /** Image width in pixels */
+    width: number;
+    /** Image height in pixels */
+    height: number;
+    /** MIME type of the image */
+    mimeType: string;
+    /** Size of the base64 data in bytes */
+    dataSize: number;
+}
+/**
+ * Request tokenizer interface
+ */
+export interface RequestTokenizer {
+    /**
+     * Calculate tokens for a request
+     */
+    calculateTokens(request: CountTokensParameters, config?: TokenizerConfig): Promise<TokenCalculationResult>;
+    /**
+     * Dispose of resources (worker threads, etc.)
+     */
+    dispose(): Promise<void>;
+}

package/dist/src/utils/request-tokenizer/types.js

@@ -0,0 +1,7 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/src/utils/request-tokenizer/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/src/utils/retry.d.ts

@@ -0,0 +1,32 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import type { GenerateContentResponse } from '@google/genai';
+export interface HttpError extends Error {
+    status?: number;
+}
+export interface RetryOptions {
+    maxAttempts: number;
+    initialDelayMs: number;
+    maxDelayMs: number;
+    shouldRetryOnError: (error: Error) => boolean;
+    shouldRetryOnContent?: (content: GenerateContentResponse) => boolean;
+    onPersistent429?: (authType?: string, error?: unknown) => Promise<string | boolean | null>;
+    authType?: string;
+}
+/**
+ * Retries a function with exponential backoff and jitter.
+ * @param fn The asynchronous function to retry.
+ * @param options Optional retry configuration.
+ * @returns A promise that resolves with the result of the function if successful.
+ * @throws The last error encountered if all attempts fail.
+ */
+export declare function retryWithBackoff<T>(fn: () => Promise<T>, options?: Partial<RetryOptions>): Promise<T>;
+/**
+ * Extracts the HTTP status code from an error object.
+ * @param error The error object.
+ * @returns The HTTP status code, or undefined if not found.
+ */
+export declare function getErrorStatus(error: unknown): number | undefined;