@google/gemini-cli-core 0.19.0-preview.0 → 0.20.0-nightly.20251130.576fda18e

package/dist/src/utils/tokenCalculation.test.js

@@ -0,0 +1,78 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { calculateRequestTokenCount } from './tokenCalculation.js';
+describe('calculateRequestTokenCount', () => {
+    const mockContentGenerator = {
+        countTokens: vi.fn(),
+    };
+    const model = 'gemini-pro';
+    it('should use countTokens API for media requests (images/files)', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockResolvedValue({
+            totalTokens: 100,
+        });
+        const request = [{ inlineData: { mimeType: 'image/png', data: 'data' } }];
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(100);
+        expect(mockContentGenerator.countTokens).toHaveBeenCalled();
+    });
+    it('should estimate tokens locally for tool calls', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        const request = [{ functionCall: { name: 'foo', args: { bar: 'baz' } } }];
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        // Estimation logic: JSON.stringify(part).length / 4
+        // JSON: {"functionCall":{"name":"foo","args":{"bar":"baz"}}}
+        // Length: ~53 chars. 53 / 4 = 13.25 -> 13.
+        expect(count).toBeGreaterThan(0);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should estimate tokens locally for simple ASCII text', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        // 12 chars. 12 * 0.25 = 3 tokens.
+        const request = 'Hello world!';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(3);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should estimate tokens locally for CJK text with higher weight', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        // 2 chars. 2 * 1.3 = 2.6 -> floor(2.6) = 2.
+        // Old logic would be 2/4 = 0.5 -> 0.
+        const request = '你好';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBeGreaterThanOrEqual(2);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should handle mixed content', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        // 'Hi': 2 * 0.25 = 0.5
+        // '你好': 2 * 1.3 = 2.6
+        // Total: 3.1 -> 3
+        const request = 'Hi你好';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(3);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should handle empty text', async () => {
+        const request = '';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(0);
+    });
+    it('should fallback to local estimation when countTokens API fails', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockRejectedValue(new Error('API error'));
+        const request = [
+            { text: 'Hello' },
+            { inlineData: { mimeType: 'image/png', data: 'data' } },
+        ];
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        // Should fallback to estimation:
+        // 'Hello': 5 chars * 0.25 = 1.25
+        // inlineData: JSON.stringify length / 4
+        expect(count).toBeGreaterThan(0);
+        expect(mockContentGenerator.countTokens).toHaveBeenCalled();
+    });
+});
+//# sourceMappingURL=tokenCalculation.test.js.map

package/dist/src/utils/tokenCalculation.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenCalculation.test.js","sourceRoot":"","sources":["../../../src/utils/tokenCalculation.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAGnE,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,MAAM,oBAAoB,GAAG;QAC3B,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE;KACU,CAAC;IAEjC,MAAM,KAAK,GAAG,YAAY,CAAC;IAE3B,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAAC;YAC5D,WAAW,EAAE,GAAG;SACjB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1E,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,gBAAgB,EAAE,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1E,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,oDAAoD;QACpD,6DAA6D;QAC7D,2CAA2C;QAC3C,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,kCAAkC;QAClC,MAAM,OAAO,GAAG,cAAc,CAAC;QAE/B,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAC9E,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,4CAA4C;QAC5C,qCAAqC;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC;QAErB,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,uBAAuB;QACvB,sBAAsB;QACtB,kBAAkB;QAClB,MAAM,OAAO,GAAG,MAAM,CAAC;QAEvB,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;QACxC,MAAM,OAAO,GAAG,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAC9E,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAC3D,IAAI,KAAK,CAAC,WAAW,CAAC,CACvB,CAAC;QACF,MAAM,OAAO,GAAG;YACd,EAAE,IAAI,EAAE,OAAO,EAAE;YACjB,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;SACxD,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,iCAAiC;QACjC,iCAAiC;QACjC,wCAAwC;QACxC,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,gBAAgB,EAAE,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}