@blackbox_ai/blackbox-cli-core 0.0.4

package/dist/src/utils/request-tokenizer/requestTokenizer.test.js

@@ -0,0 +1,245 @@
+/**
+ * @license
+ * Copyright 2025 Blackbox
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { DefaultRequestTokenizer } from './requestTokenizer.js';
+describe('DefaultRequestTokenizer', () => {
+    let tokenizer;
+    beforeEach(() => {
+        tokenizer = new DefaultRequestTokenizer();
+    });
+    afterEach(async () => {
+        await tokenizer.dispose();
+    });
+    describe('text token calculation', () => {
+        it('should calculate tokens for simple text content', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [{ text: 'Hello, world!' }],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+            expect(result.breakdown.imageTokens).toBe(0);
+            expect(result.processingTime).toBeGreaterThan(0);
+        });
+        it('should handle multiple text parts', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            { text: 'First part' },
+                            { text: 'Second part' },
+                            { text: 'Third part' },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+        });
+        it('should handle string content', async () => {
+            const request = {
+                model: 'test-model',
+                contents: ['Simple string content'],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+        });
+    });
+    describe('image token calculation', () => {
+        it('should calculate tokens for image content', async () => {
+            // Create a simple 1x1 PNG image in base64
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThanOrEqual(4); // Minimum 4 tokens per image
+            expect(result.breakdown.imageTokens).toBeGreaterThanOrEqual(4);
+            expect(result.breakdown.textTokens).toBe(0);
+        });
+        it('should handle multiple images', async () => {
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThanOrEqual(8); // At least 4 tokens per image
+            expect(result.breakdown.imageTokens).toBeGreaterThanOrEqual(8);
+        });
+    });
+    describe('mixed content', () => {
+        it('should handle text and image content together', async () => {
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            { text: 'Here is an image:' },
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                            { text: 'What do you see?' },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(4);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+            expect(result.breakdown.imageTokens).toBeGreaterThanOrEqual(4);
+        });
+    });
+    describe('function content', () => {
+        it('should handle function calls', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                functionCall: {
+                                    name: 'test_function',
+                                    args: { param1: 'value1', param2: 42 },
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.otherTokens).toBeGreaterThan(0);
+        });
+    });
+    describe('empty content', () => {
+        it('should handle empty request', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBe(0);
+            expect(result.breakdown.textTokens).toBe(0);
+            expect(result.breakdown.imageTokens).toBe(0);
+        });
+        it('should handle undefined contents', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBe(0);
+        });
+    });
+    describe('configuration', () => {
+        it('should use custom text encoding', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [{ text: 'Test text for encoding' }],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request, {
+                textEncoding: 'cl100k_base',
+            });
+            expect(result.totalTokens).toBeGreaterThan(0);
+        });
+        it('should process multiple images serially', async () => {
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: Array(10).fill({
+                            inlineData: {
+                                mimeType: 'image/png',
+                                data: pngBase64,
+                            },
+                        }),
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThanOrEqual(60); // At least 6 tokens per image * 10 images
+        });
+    });
+    describe('error handling', () => {
+        it('should handle malformed image data gracefully', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: 'invalid-base64-data',
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            // Should still return some tokens (fallback to minimum)
+            expect(result.totalTokens).toBeGreaterThanOrEqual(4);
+        });
+    });
+});
+//# sourceMappingURL=requestTokenizer.test.js.map

package/dist/src/utils/request-tokenizer/requestTokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"requestTokenizer.test.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/requestTokenizer.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAGhE,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,IAAI,SAAkC,CAAC;IAEvC,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,GAAG,IAAI,uBAAuB,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACnC;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL,EAAE,IAAI,EAAE,YAAY,EAAE;4BACtB,EAAE,IAAI,EAAE,aAAa,EAAE;4BACvB,EAAE,IAAI,EAAE,YAAY,EAAE;yBACvB;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE,CAAC,uBAAuB,CAAC;aACpC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,0CAA0C;YAC1C,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,6BAA6B;YACnF,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;4BACD;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,8BAA8B;YACpF,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL,EAAE,IAAI,EAAE,mBAAmB,EAAE;4BAC7B;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;4BACD,EAAE,IAAI,EAAE,kBAAkB,EAAE;yBAC7B;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,YAAY,EAAE;oCACZ,IAAI,EAAE,eAAe;oCACrB,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;iCACvC;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE,EAAE;aACb,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE,EAAE;aACb,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC;qBAC5C;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE;gBACtD,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC;4BACpB,UAAU,EAAE;gCACV,QAAQ,EAAE,WAAW;gCACrB,IAAI,EAAE,SAAS;6BAChB;yBACF,CAAC;qBACH;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,0CAA0C;QACnG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,qBAAqB;iCAC5B;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,wDAAwD;YACxD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/utils/request-tokenizer/supportedImageFormats.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @license
+ * Copyright 2025 Blackbox
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Supported image MIME types for vision models
+ * These formats are supported by the vision model and can be processed by the image tokenizer
+ */
+export declare const SUPPORTED_IMAGE_MIME_TYPES: readonly ["image/bmp", "image/jpeg", "image/jpg", "image/png", "image/tiff", "image/webp", "image/heic"];
+/**
+ * Type for supported image MIME types
+ */
+export type SupportedImageMimeType = (typeof SUPPORTED_IMAGE_MIME_TYPES)[number];
+/**
+ * Check if a MIME type is supported for vision processing
+ * @param mimeType The MIME type to check
+ * @returns True if the MIME type is supported
+ */
+export declare function isSupportedImageMimeType(mimeType: string): mimeType is SupportedImageMimeType;
+/**
+ * Get a human-readable list of supported image formats
+ * @returns Comma-separated string of supported formats
+ */
+export declare function getSupportedImageFormatsString(): string;
+/**
+ * Get warning message for unsupported image formats
+ * @returns Warning message string
+ */
+export declare function getUnsupportedImageFormatWarning(): string;

package/dist/src/utils/request-tokenizer/supportedImageFormats.js

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright 2025 Blackbox
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Supported image MIME types for vision models
+ * These formats are supported by the vision model and can be processed by the image tokenizer
+ */
+export const SUPPORTED_IMAGE_MIME_TYPES = [
+    'image/bmp',
+    'image/jpeg',
+    'image/jpg', // Alternative MIME type for JPEG
+    'image/png',
+    'image/tiff',
+    'image/webp',
+    'image/heic',
+];
+/**
+ * Check if a MIME type is supported for vision processing
+ * @param mimeType The MIME type to check
+ * @returns True if the MIME type is supported
+ */
+export function isSupportedImageMimeType(mimeType) {
+    return SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType);
+}
+/**
+ * Get a human-readable list of supported image formats
+ * @returns Comma-separated string of supported formats
+ */
+export function getSupportedImageFormatsString() {
+    return SUPPORTED_IMAGE_MIME_TYPES.map((type) => type.replace('image/', '').toUpperCase()).join(', ');
+}
+/**
+ * Get warning message for unsupported image formats
+ * @returns Warning message string
+ */
+export function getUnsupportedImageFormatWarning() {
+    return `Only the following image formats are supported: ${getSupportedImageFormatsString()}. Other formats may not work as expected.`;
+}
+//# sourceMappingURL=supportedImageFormats.js.map

package/dist/src/utils/request-tokenizer/supportedImageFormats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"supportedImageFormats.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/supportedImageFormats.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,WAAW;IACX,YAAY;IACZ,WAAW,EAAE,iCAAiC;IAC9C,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,YAAY;CACJ,CAAC;AAQX;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CACtC,QAAgB;IAEhB,OAAO,0BAA0B,CAAC,QAAQ,CACxC,QAAkC,CACnC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,8BAA8B;IAC5C,OAAO,0BAA0B,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CACzC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gCAAgC;IAC9C,OAAO,mDAAmD,8BAA8B,EAAE,2CAA2C,CAAC;AACxI,CAAC"}

package/dist/src/utils/request-tokenizer/textTokenizer.d.ts

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright 2025 Blackbox
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Text tokenizer for calculating text tokens using tiktoken
+ */
+export declare class TextTokenizer {
+    private encoding;
+    private encodingName;
+    constructor(encodingName?: string);
+    /**
+     * Initialize the tokenizer (lazy loading)
+     */
+    private ensureEncoding;
+    /**
+     * Calculate tokens for text content
+     */
+    calculateTokens(text: string): Promise<number>;
+    /**
+     * Calculate tokens for multiple text strings in parallel
+     */
+    calculateTokensBatch(texts: string[]): Promise<number[]>;
+    /**
+     * Dispose of resources
+     */
+    dispose(): void;
+}

package/dist/src/utils/request-tokenizer/textTokenizer.js

@@ -0,0 +1,88 @@
+/**
+ * @license
+ * Copyright 2025 Blackbox
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { get_encoding } from 'tiktoken';
+/**
+ * Text tokenizer for calculating text tokens using tiktoken
+ */
+export class TextTokenizer {
+    encoding = null;
+    encodingName;
+    constructor(encodingName = 'cl100k_base') {
+        this.encodingName = encodingName;
+    }
+    /**
+     * Initialize the tokenizer (lazy loading)
+     */
+    async ensureEncoding() {
+        if (this.encoding)
+            return;
+        try {
+            // Use type assertion since we know the encoding name is valid
+            this.encoding = get_encoding(this.encodingName);
+        }
+        catch (error) {
+            console.warn(`Failed to load tiktoken with encoding ${this.encodingName}:`, error);
+            this.encoding = null;
+        }
+    }
+    /**
+     * Calculate tokens for text content
+     */
+    async calculateTokens(text) {
+        if (!text)
+            return 0;
+        await this.ensureEncoding();
+        if (this.encoding) {
+            try {
+                return this.encoding.encode(text).length;
+            }
+            catch (error) {
+                console.warn('Error encoding text with tiktoken:', error);
+            }
+        }
+        // Fallback: rough approximation using character count
+        // This is a conservative estimate: 1 token ≈ 4 characters for most languages
+        return Math.ceil(text.length / 4);
+    }
+    /**
+     * Calculate tokens for multiple text strings in parallel
+     */
+    async calculateTokensBatch(texts) {
+        await this.ensureEncoding();
+        if (this.encoding) {
+            try {
+                return texts.map((text) => {
+                    if (!text)
+                        return 0;
+                    // this.encoding may be null, add a null check to satisfy lint
+                    return this.encoding ? this.encoding.encode(text).length : 0;
+                });
+            }
+            catch (error) {
+                console.warn('Error encoding texts with tiktoken:', error);
+                // In case of error, return fallback estimation for all texts
+                return texts.map((text) => Math.ceil((text || '').length / 4));
+            }
+        }
+        // Fallback for batch processing
+        return texts.map((text) => Math.ceil((text || '').length / 4));
+    }
+    /**
+     * Dispose of resources
+     */
+    dispose() {
+        if (this.encoding) {
+            try {
+                this.encoding.free();
+            }
+            catch (error) {
+                console.warn('Error freeing tiktoken encoding:', error);
+            }
+            this.encoding = null;
+        }
+    }
+}
+//# sourceMappingURL=textTokenizer.js.map

package/dist/src/utils/request-tokenizer/textTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textTokenizer.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/textTokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC;;GAEG;AACH,MAAM,OAAO,aAAa;IAChB,QAAQ,GAAoB,IAAI,CAAC;IACjC,YAAY,CAAS;IAE7B,YAAY,eAAuB,aAAa;QAC9C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,IAAI,CAAC;YACH,8DAA8D;YAC9D,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,YAAgC,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CACV,yCAAyC,IAAI,CAAC,YAAY,GAAG,EAC7D,KAAK,CACN,CAAC;YACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAEpB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YAC3C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,sDAAsD;QACtD,6EAA6E;QAC7E,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,KAAe;QACxC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBACxB,IAAI,CAAC,IAAI;wBAAE,OAAO,CAAC,CAAC;oBACpB,8DAA8D;oBAC9D,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/D,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;gBAC3D,6DAA6D;gBAC7D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;CACF"}

package/dist/src/utils/request-tokenizer/textTokenizer.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2025 Blackbox
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};