@google/gemini-cli-core 0.19.0-preview.0 → 0.20.0-nightly.20251127.5bed97064

package/dist/src/utils/tokenCalculation.js

@@ -0,0 +1,70 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+// Token estimation constants
+// ASCII characters (0-127) are roughly 4 chars per token
+const ASCII_TOKENS_PER_CHAR = 0.25;
+// Non-ASCII characters (including CJK) are often 1-2 tokens per char.
+// We use 1.3 as a conservative estimate to avoid underestimation.
+const NON_ASCII_TOKENS_PER_CHAR = 1.3;
+/**
+ * Estimates token count for parts synchronously using a heuristic.
+ * - Text: character-based heuristic (ASCII vs CJK).
+ * - Non-text (Tools, etc): JSON string length / 4.
+ */
+export function estimateTokenCountSync(parts) {
+    let totalTokens = 0;
+    for (const part of parts) {
+        if (typeof part.text === 'string') {
+            for (const char of part.text) {
+                if (char.codePointAt(0) <= 127) {
+                    totalTokens += ASCII_TOKENS_PER_CHAR;
+                }
+                else {
+                    totalTokens += NON_ASCII_TOKENS_PER_CHAR;
+                }
+            }
+        }
+        else {
+            // For non-text parts (functionCall, functionResponse, executableCode, etc.),
+            // we fallback to the JSON string length heuristic.
+            // Note: This is an approximation.
+            totalTokens += JSON.stringify(part).length / 4;
+        }
+    }
+    return Math.floor(totalTokens);
+}
+/**
+ * Calculates the token count of the request.
+ * If the request contains only text or tools, it estimates the token count locally.
+ * If the request contains media (images, files), it uses the countTokens API.
+ */
+export async function calculateRequestTokenCount(request, contentGenerator, model) {
+    const parts = Array.isArray(request)
+        ? request.map((p) => (typeof p === 'string' ? { text: p } : p))
+        : typeof request === 'string'
+            ? [{ text: request }]
+            : [request];
+    // Use countTokens API only for heavy media parts that are hard to estimate.
+    const hasMedia = parts.some((p) => {
+        const isMedia = 'inlineData' in p || 'fileData' in p;
+        return isMedia;
+    });
+    if (hasMedia) {
+        try {
+            const response = await contentGenerator.countTokens({
+                model,
+                contents: [{ role: 'user', parts }],
+            });
+            return response.totalTokens ?? 0;
+        }
+        catch {
+            // Fallback to local estimation if the API call fails
+            return estimateTokenCountSync(parts);
+        }
+    }
+    return estimateTokenCountSync(parts);
+}
+//# sourceMappingURL=tokenCalculation.js.map

package/dist/src/utils/tokenCalculation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenCalculation.js","sourceRoot":"","sources":["../../../src/utils/tokenCalculation.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,6BAA6B;AAC7B,yDAAyD;AACzD,MAAM,qBAAqB,GAAG,IAAI,CAAC;AACnC,sEAAsE;AACtE,kEAAkE;AAClE,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAEtC;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAAa;IAClD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7B,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,GAAG,EAAE,CAAC;oBAChC,WAAW,IAAI,qBAAqB,CAAC;gBACvC,CAAC;qBAAM,CAAC;oBACN,WAAW,IAAI,yBAAyB,CAAC;gBAC3C,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,6EAA6E;YAC7E,mDAAmD;YACnD,kCAAkC;YAClC,WAAW,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,OAAsB,EACtB,gBAAkC,EAClC,KAAa;IAEb,MAAM,KAAK,GAAW,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAC1C,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ;YAC3B,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;YACrB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAEhB,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,MAAM,OAAO,GAAG,YAAY,IAAI,CAAC,IAAI,UAAU,IAAI,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC;gBAClD,KAAK;gBACL,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;aACpC,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,WAAW,IAAI,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,qDAAqD;YACrD,OAAO,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,sBAAsB,CAAC,KAAK,CAAC,CAAC;AACvC,CAAC"}

package/dist/src/utils/tokenCalculation.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/utils/tokenCalculation.test.js

@@ -0,0 +1,78 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { calculateRequestTokenCount } from './tokenCalculation.js';
+describe('calculateRequestTokenCount', () => {
+    const mockContentGenerator = {
+        countTokens: vi.fn(),
+    };
+    const model = 'gemini-pro';
+    it('should use countTokens API for media requests (images/files)', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockResolvedValue({
+            totalTokens: 100,
+        });
+        const request = [{ inlineData: { mimeType: 'image/png', data: 'data' } }];
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(100);
+        expect(mockContentGenerator.countTokens).toHaveBeenCalled();
+    });
+    it('should estimate tokens locally for tool calls', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        const request = [{ functionCall: { name: 'foo', args: { bar: 'baz' } } }];
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        // Estimation logic: JSON.stringify(part).length / 4
+        // JSON: {"functionCall":{"name":"foo","args":{"bar":"baz"}}}
+        // Length: ~53 chars. 53 / 4 = 13.25 -> 13.
+        expect(count).toBeGreaterThan(0);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should estimate tokens locally for simple ASCII text', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        // 12 chars. 12 * 0.25 = 3 tokens.
+        const request = 'Hello world!';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(3);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should estimate tokens locally for CJK text with higher weight', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        // 2 chars. 2 * 1.3 = 2.6 -> floor(2.6) = 2.
+        // Old logic would be 2/4 = 0.5 -> 0.
+        const request = '你好';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBeGreaterThanOrEqual(2);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should handle mixed content', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockClear();
+        // 'Hi': 2 * 0.25 = 0.5
+        // '你好': 2 * 1.3 = 2.6
+        // Total: 3.1 -> 3
+        const request = 'Hi你好';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(3);
+        expect(mockContentGenerator.countTokens).not.toHaveBeenCalled();
+    });
+    it('should handle empty text', async () => {
+        const request = '';
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        expect(count).toBe(0);
+    });
+    it('should fallback to local estimation when countTokens API fails', async () => {
+        vi.mocked(mockContentGenerator.countTokens).mockRejectedValue(new Error('API error'));
+        const request = [
+            { text: 'Hello' },
+            { inlineData: { mimeType: 'image/png', data: 'data' } },
+        ];
+        const count = await calculateRequestTokenCount(request, mockContentGenerator, model);
+        // Should fallback to estimation:
+        // 'Hello': 5 chars * 0.25 = 1.25
+        // inlineData: JSON.stringify length / 4
+        expect(count).toBeGreaterThan(0);
+        expect(mockContentGenerator.countTokens).toHaveBeenCalled();
+    });
+});
+//# sourceMappingURL=tokenCalculation.test.js.map

package/dist/src/utils/tokenCalculation.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenCalculation.test.js","sourceRoot":"","sources":["../../../src/utils/tokenCalculation.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAGnE,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,MAAM,oBAAoB,GAAG;QAC3B,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE;KACU,CAAC;IAEjC,MAAM,KAAK,GAAG,YAAY,CAAC;IAE3B,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAAC;YAC5D,WAAW,EAAE,GAAG;SACjB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1E,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,gBAAgB,EAAE,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1E,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,oDAAoD;QACpD,6DAA6D;QAC7D,2CAA2C;QAC3C,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,kCAAkC;QAClC,MAAM,OAAO,GAAG,cAAc,CAAC;QAE/B,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAC9E,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,4CAA4C;QAC5C,qCAAqC;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC;QAErB,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC;QACxD,uBAAuB;QACvB,sBAAsB;QACtB,kBAAkB;QAClB,MAAM,OAAO,GAAG,MAAM,CAAC;QAEvB,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;QACxC,MAAM,OAAO,GAAG,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAC9E,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAC3D,IAAI,KAAK,CAAC,WAAW,CAAC,CACvB,CAAC;QACF,MAAM,OAAO,GAAG;YACd,EAAE,IAAI,EAAE,OAAO,EAAE;YACjB,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;SACxD,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAC5C,OAAO,EACP,oBAAoB,EACpB,KAAK,CACN,CAAC;QAEF,iCAAiC;QACjC,iCAAiC;QACjC,wCAAwC;QACxC,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,gBAAgB,EAAE,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}