@machina.ai/cell-cli-core 1.41.1-rc2 → 1.45.1-rc2

package/dist/src/context/utils/adaptiveTokenCalculator.js

@@ -0,0 +1,138 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { StaticTokenCalculator, } from './contextTokenCalculator.js';
+import { debugLogger } from '../../utils/debugLogger.js';
+/**
+ * An Adaptive Token Calculator that dynamically learns the true token cost of the user's
+ * conversation by applying an Exponential Moving Average (EMA) gradient descent to
+ * real usage metadata returned from the Gemini API.
+ *
+ * It wraps the deterministic `StaticTokenCalculator` base heuristic to ensure
+ * immutable node cost caching while still surfacing a self-corrected estimate
+ * to the pipeline processors.
+ */
+export class AdaptiveTokenCalculator {
+    getOverheadTokens;
+    learnedWeight = 1.0;
+    baseCalculator;
+    learningRate;
+    maxStep;
+    constructor(charsPerToken, registry, eventBus, getOverheadTokens, config) {
+        this.getOverheadTokens = getOverheadTokens;
+        this.learningRate = config?.learningRate ?? 0.2;
+        this.maxStep = config?.maxStep ?? 0.15;
+        this.baseCalculator = new StaticTokenCalculator(charsPerToken, registry);
+        eventBus.onTokenGroundTruth((event) => {
+            this.handleGroundTruth(event.actualTokens, event.promptBaseUnits);
+        });
+    }
+    handleGroundTruth(actualTokens, promptBaseUnits) {
+        if (promptBaseUnits <= 0)
+            return;
+        const overheadTokens = this.getOverheadTokens
+            ? this.getOverheadTokens()
+            : 0;
+        // The Gemini API token count includes the static overhead (system instruction + tools)
+        // and the dynamic chat history (which we measure as promptBaseUnits).
+        // We subtract the overhead so the adaptive calculator is comparing "apples to apples"
+        // when learning the weight multiplier for the graph nodes.
+        const actualGraphTokens = Math.max(0, actualTokens - overheadTokens);
+        // Determine what ratio we should have used
+        const rawTargetWeight = actualGraphTokens / promptBaseUnits;
+        const oldWeight = this.learnedWeight;
+        // Dampen extreme outliers *before* applying the EMA by capping the target weight
+        // to a reasonable multiple of the current weight. This prevents a single massive
+        // anomaly from destroying the running average.
+        const targetWeight = Math.max(oldWeight * 0.5, Math.min(rawTargetWeight, oldWeight * 2.0));
+        // Apply Momentum (Learning Rate)
+        let newWeight = oldWeight * (1 - this.learningRate) + targetWeight * this.learningRate;
+        // Hard limit the maximum step size per turn to prevent violent oscillation
+        if (newWeight > oldWeight + this.maxStep)
+            newWeight = oldWeight + this.maxStep;
+        if (newWeight < oldWeight - this.maxStep)
+            newWeight = oldWeight - this.maxStep;
+        // Clamp to reasonable absolute safety bounds
+        this.learnedWeight = Math.max(0.5, Math.min(newWeight, 2.0));
+        debugLogger.log(`[AdaptiveTokenCalculator] Learned weight updated to ${this.learnedWeight.toFixed(3)} ` +
+            `(API Tokens: ${actualTokens}, Overhead: ${overheadTokens}, Graph Tokens: ${actualGraphTokens}, Base Units: ${promptBaseUnits}, Target Ratio: ${targetWeight.toFixed(3)})`);
+    }
+    /**
+     * Retrieves the current learned weight multiplier.
+     */
+    getLearnedWeight() {
+        return this.learnedWeight;
+    }
+    /**
+     * Returns the exact, unweighted Base Heuristic Units for the graph.
+     * This is used exactly once per interaction to capture the baseline sent to the API.
+     */
+    getRawBaseUnits(nodes) {
+        return this.baseCalculator.calculateConcreteListTokens(nodes);
+    }
+    /**
+     * Returns the exact, unweighted Base Heuristic Units for a raw content chunk.
+     */
+    getRawBaseUnitsForContent(content) {
+        return this.baseCalculator.calculateContentTokens(content);
+    }
+    calculateTokensAndBaseUnits(nodes) {
+        const baseUnits = this.baseCalculator.calculateConcreteListTokens(nodes);
+        return {
+            tokens: Math.round(baseUnits * this.learnedWeight),
+            baseUnits,
+        };
+    }
+    calculateContentTokensAndBaseUnits(content) {
+        const baseUnits = this.baseCalculator.calculateContentTokens(content);
+        return {
+            tokens: Math.round(baseUnits * this.learnedWeight),
+            baseUnits,
+        };
+    }
+    // --- Delegation and Weighting ---
+    garbageCollectCache(liveNodeIds) {
+        this.baseCalculator.garbageCollectCache(liveNodeIds);
+    }
+    cacheNodeTokens(node) {
+        return this.baseCalculator.cacheNodeTokens(node);
+    }
+    calculateTokenBreakdown(nodes) {
+        const raw = this.baseCalculator.calculateTokenBreakdown(nodes);
+        return {
+            text: Math.round(raw.text * this.learnedWeight),
+            media: Math.round(raw.media * this.learnedWeight),
+            tool: Math.round(raw.tool * this.learnedWeight),
+            overhead: Math.round(raw.overhead * this.learnedWeight),
+            total: Math.round(raw.total * this.learnedWeight),
+        };
+    }
+    estimateTokensForParts(parts) {
+        const baseUnits = this.baseCalculator.estimateTokensForParts(parts);
+        return Math.round(baseUnits * this.learnedWeight);
+    }
+    getTokenCost(node) {
+        const baseUnits = this.baseCalculator.getTokenCost(node);
+        return Math.round(baseUnits * this.learnedWeight);
+    }
+    calculateConcreteListTokens(nodes) {
+        const baseUnits = this.baseCalculator.calculateConcreteListTokens(nodes);
+        return Math.round(baseUnits * this.learnedWeight);
+    }
+    calculateContentTokens(content) {
+        const baseUnits = this.baseCalculator.calculateContentTokens(content);
+        return Math.round(baseUnits * this.learnedWeight);
+    }
+    estimateTokensForString(text) {
+        const baseUnits = this.baseCalculator.estimateTokensForString(text);
+        return Math.round(baseUnits * this.learnedWeight);
+    }
+    tokensToChars(tokens) {
+        // If weight is > 1.0 (we are inflating tokens), a single returned token is worth fewer chars.
+        // We reverse the math: convert requested tokens to target base units, then get chars.
+        return this.baseCalculator.tokensToChars(tokens / this.learnedWeight);
+    }
+}
+//# sourceMappingURL=adaptiveTokenCalculator.js.map

package/dist/src/context/utils/adaptiveTokenCalculator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptiveTokenCalculator.js","sourceRoot":"","sources":["../../../../src/context/utils/adaptiveTokenCalculator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EACL,qBAAqB,GAEtB,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AASzD;;;;;;;;GAQG;AACH,MAAM,OAAO,uBAAuB;IAUf;IATX,aAAa,GAAG,GAAG,CAAC;IACX,cAAc,CAAwB;IACtC,YAAY,CAAS;IACrB,OAAO,CAAS;IAEjC,YACE,aAAqB,EACrB,QAA8B,EAC9B,QAAyB,EACR,iBAAgC,EACjD,MAA+B;QADd,sBAAiB,GAAjB,iBAAiB,CAAe;QAGjD,IAAI,CAAC,YAAY,GAAG,MAAM,EAAE,YAAY,IAAI,GAAG,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,IAAI,qBAAqB,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QACzE,QAAQ,CAAC,kBAAkB,CAAC,CAAC,KAA4B,EAAE,EAAE;YAC3D,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,iBAAiB,CAAC,YAAoB,EAAE,eAAuB;QACrE,IAAI,eAAe,IAAI,CAAC;YAAE,OAAO;QAEjC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB;YAC3C,CAAC,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC1B,CAAC,CAAC,CAAC,CAAC;QAEN,uFAAuF;QACvF,sEAAsE;QACtE,sFAAsF;QACtF,2DAA2D;QAC3D,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,cAAc,CAAC,CAAC;QAErE,2CAA2C;QAC3C,MAAM,eAAe,GAAG,iBAAiB,GAAG,eAAe,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;QAErC,iFAAiF;QACjF,iFAAiF;QACjF,+CAA+C;QAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,SAAS,GAAG,GAAG,EACf,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,GAAG,GAAG,CAAC,CAC3C,CAAC;QAEF,iCAAiC;QACjC,IAAI,SAAS,GACX,SAAS,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEzE,2EAA2E;QAC3E,IAAI,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,OAAO;YACtC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC;QACvC,IAAI,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,OAAO;YACtC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC;QAEvC,6CAA6C;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QAE7D,WAAW,CAAC,GAAG,CACb,uDAAuD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YACrF,gBAAgB,YAAY,eAAe,cAAc,mBAAmB,iBAAiB,iBAAiB,eAAe,mBAAmB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC7K,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,KAA8B;QAC5C,OAAO,IAAI,CAAC,cAAc,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,yBAAyB,CAAC,OAAgB;QACxC,OAAO,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAED,2BAA2B,CAAC,KAA8B;QAIxD,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;QACzE,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;YAClD,SAAS;SACV,CAAC;IACJ,CAAC;IAED,kCAAkC,CAAC,OAAgB;QAIjD,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACtE,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;YAClD,SAAS;SACV,CAAC;IACJ,CAAC;IAED,mCAAmC;IAEnC,mBAAmB,CAAC,WAAgC;QAClD,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;IAED,eAAe,CAAC,IAAkB;QAChC,OAAO,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IAED,uBAAuB,CAAC,KAA8B;QAOpD,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QAC/D,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC;YAC/C,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;YACjD,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC;YAC/C,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC;YACvD,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;SAClD,CAAC;IACJ,CAAC;IAED,sBAAsB,CAAC,KAAa;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACpD,CAAC;IAED,YAAY,CAAC,IAAkB;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACpD,CAAC;IAED,2BAA2B,CAAC,KAA8B;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACpD,CAAC;IAED,sBAAsB,CAAC,OAAgB;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACpD,CAAC;IAED,uBAAuB,CAAC,IAAY;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACpD,CAAC;IAED,aAAa,CAAC,MAAc;QAC1B,8FAA8F;QAC9F,sFAAsF;QACtF,OAAO,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACxE,CAAC;CACF"}

package/dist/src/context/utils/adaptiveTokenCalculator.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/context/utils/adaptiveTokenCalculator.test.js

@@ -0,0 +1,129 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect } from 'vitest';
+import { AdaptiveTokenCalculator } from './adaptiveTokenCalculator.js';
+import { NodeBehaviorRegistry } from '../graph/behaviorRegistry.js';
+import { registerBuiltInBehaviors } from '../graph/builtinBehaviors.js';
+import { ContextEventBus } from '../eventBus.js';
+import { createDummyNode } from '../testing/contextTestUtils.js';
+import { NodeType } from '../graph/types.js';
+describe('AdaptiveTokenCalculator', () => {
+    const registry = new NodeBehaviorRegistry();
+    registerBuiltInBehaviors(registry);
+    const charsPerToken = 1; // Simplifies math
+    it('should initialize with a learned weight of 1.0', () => {
+        const eventBus = new ContextEventBus();
+        const calculator = new AdaptiveTokenCalculator(charsPerToken, registry, eventBus);
+        expect(calculator.getLearnedWeight()).toBe(1.0);
+    });
+    it('should dynamically update learned weight based on token ground truth events', () => {
+        const eventBus = new ContextEventBus();
+        const calculator = new AdaptiveTokenCalculator(charsPerToken, registry, eventBus);
+        // Initial state: weight = 1.0
+        // Simulate an event where the API reported fewer tokens than our base units
+        // targetWeight = 50 / 100 = 0.5
+        // newWeight = 1.0 * 0.8 + 0.5 * 0.2 = 0.8 + 0.1 = 0.9
+        eventBus.emitTokenGroundTruth({
+            actualTokens: 50,
+            promptBaseUnits: 100,
+        });
+        // JavaScript floating point precision means we should use toBeCloseTo
+        expect(calculator.getLearnedWeight()).toBeCloseTo(0.9, 5);
+        // Simulate another event
+        // newWeight = 0.9 * 0.8 + (150 / 100) * 0.2 = 0.72 + 0.3 = 1.02
+        eventBus.emitTokenGroundTruth({
+            actualTokens: 150,
+            promptBaseUnits: 100,
+        });
+        expect(calculator.getLearnedWeight()).toBeCloseTo(1.02, 5);
+    });
+    it('should clamp the learned weight between 0.5 and 2.0', () => {
+        const eventBus = new ContextEventBus();
+        const calculator = new AdaptiveTokenCalculator(charsPerToken, registry, eventBus);
+        // Push weight up extremely high (API returns 10x tokens)
+        for (let i = 0; i < 20; i++) {
+            eventBus.emitTokenGroundTruth({
+                actualTokens: 1000,
+                promptBaseUnits: 100,
+            });
+        }
+        expect(calculator.getLearnedWeight()).toBe(2.0);
+        // Push weight down extremely low (API returns 0 tokens)
+        for (let i = 0; i < 20; i++) {
+            eventBus.emitTokenGroundTruth({ actualTokens: 0, promptBaseUnits: 100 });
+        }
+        expect(calculator.getLearnedWeight()).toBe(0.5);
+    });
+    it('should correctly apply the learned weight to node calculations while keeping raw base units stable', () => {
+        const eventBus = new ContextEventBus();
+        const calculator = new AdaptiveTokenCalculator(charsPerToken, registry, eventBus);
+        // Decrease the weight to exactly 0.5
+        for (let i = 0; i < 20; i++) {
+            eventBus.emitTokenGroundTruth({ actualTokens: 0, promptBaseUnits: 100 });
+        }
+        const turn1Id = 'turn-1';
+        const node1 = createDummyNode(turn1Id, NodeType.USER_PROMPT);
+        // Get raw base units directly
+        const rawTokens = calculator.calculateTokensAndBaseUnits([node1]).baseUnits;
+        // Get adjusted tokens
+        const adjustedTokens = calculator.calculateConcreteListTokens([node1]);
+        expect(adjustedTokens).toBe(Math.round(rawTokens * 0.5));
+    });
+    it('should ignore ground truth events with 0 promptBaseUnits to prevent division by zero', () => {
+        const eventBus = new ContextEventBus();
+        const calculator = new AdaptiveTokenCalculator(charsPerToken, registry, eventBus);
+        eventBus.emitTokenGroundTruth({
+            actualTokens: 100,
+            promptBaseUnits: 0,
+        });
+        expect(calculator.getLearnedWeight()).toBe(1.0);
+    });
+    it('should subtract overhead tokens from actual tokens when determining target weight', () => {
+        const eventBus = new ContextEventBus();
+        const getOverheadTokens = () => 40;
+        const calculator = new AdaptiveTokenCalculator(charsPerToken, registry, eventBus, getOverheadTokens);
+        // Initial state: weight = 1.0
+        // Simulate an event where the API reported 100 tokens, and our base units were 100
+        // But overhead is 40.
+        // actualGraphTokens = 100 - 40 = 60
+        // rawTargetWeight = 60 / 100 = 0.6
+        // targetWeight = Math.max(0.5, 0.6) = 0.6
+        // newWeight = 1.0 * 0.8 + 0.6 * 0.2 = 0.8 + 0.12 = 0.92
+        eventBus.emitTokenGroundTruth({
+            actualTokens: 100,
+            promptBaseUnits: 100,
+        });
+        expect(calculator.getLearnedWeight()).toBeCloseTo(0.92, 5);
+    });
+    it('should enforce the maxStep limit to prevent violent oscillation from massive outliers', () => {
+        const eventBus = new ContextEventBus();
+        const maxStep = 0.05; // Tight limit
+        const calculator = new AdaptiveTokenCalculator(charsPerToken, registry, eventBus, undefined, { maxStep });
+        // Initial state: weight = 1.0
+        // Simulate a massive outlier where the API reports 10,000 tokens for 100 base units.
+        // rawTargetWeight = 100
+        // targetWeight = Math.min(100, 1.0 * 2.0) = 2.0
+        // emaWeight = 1.0 * 0.8 + 2.0 * 0.2 = 1.2
+        // BUT maxStep is 0.05, so the actual step is clamped.
+        // finalWeight = 1.0 + 0.05 = 1.05
+        eventBus.emitTokenGroundTruth({
+            actualTokens: 10000,
+            promptBaseUnits: 100,
+        });
+        expect(calculator.getLearnedWeight()).toBeCloseTo(1.05, 5);
+        // Simulate a massive under-estimation
+        // rawTargetWeight = 0
+        // targetWeight = Math.max(0, 1.05 * 0.5) = 0.525
+        // emaWeight = 1.05 * 0.8 + 0.525 * 0.2 = 0.84 + 0.105 = 0.945
+        // BUT maxStep is 0.05, so step is clamped: 1.05 - 0.05 = 1.0
+        eventBus.emitTokenGroundTruth({
+            actualTokens: 0,
+            promptBaseUnits: 100,
+        });
+        expect(calculator.getLearnedWeight()).toBeCloseTo(1.0, 5);
+    });
+});
+//# sourceMappingURL=adaptiveTokenCalculator.test.js.map

package/dist/src/context/utils/adaptiveTokenCalculator.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptiveTokenCalculator.test.js","sourceRoot":"","sources":["../../../../src/context/utils/adaptiveTokenCalculator.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,MAAM,QAAQ,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC5C,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,CAAC,CAAC,CAAC,kBAAkB;IAE3C,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,uBAAuB,CAC5C,aAAa,EACb,QAAQ,EACR,QAAQ,CACT,CAAC;QACF,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,GAAG,EAAE;QACrF,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,uBAAuB,CAC5C,aAAa,EACb,QAAQ,EACR,QAAQ,CACT,CAAC;QAEF,8BAA8B;QAE9B,4EAA4E;QAC5E,gCAAgC;QAChC,sDAAsD;QACtD,QAAQ,CAAC,oBAAoB,CAAC;YAC5B,YAAY,EAAE,EAAE;YAChB,eAAe,EAAE,GAAG;SACrB,CAAC,CAAC;QAEH,sEAAsE;QACtE,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAE1D,yBAAyB;QACzB,gEAAgE;QAChE,QAAQ,CAAC,oBAAoB,CAAC;YAC5B,YAAY,EAAE,GAAG;YACjB,eAAe,EAAE,GAAG;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,uBAAuB,CAC5C,aAAa,EACb,QAAQ,EACR,QAAQ,CACT,CAAC;QAEF,yDAAyD;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,QAAQ,CAAC,oBAAoB,CAAC;gBAC5B,YAAY,EAAE,IAAI;gBAClB,eAAe,EAAE,GAAG;aACrB,CAAC,CAAC;QACL,CAAC;QACD,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhD,wDAAwD;QACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,QAAQ,CAAC,oBAAoB,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oGAAoG,EAAE,GAAG,EAAE;QAC5G,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,uBAAuB,CAC5C,aAAa,EACb,QAAQ,EACR,QAAQ,CACT,CAAC;QAEF,qCAAqC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,QAAQ,CAAC,oBAAoB,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC;QACzB,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QAE7D,8BAA8B;QAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,2BAA2B,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAE5E,sBAAsB;QACtB,MAAM,cAAc,GAAG,UAAU,CAAC,2BAA2B,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAEvE,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sFAAsF,EAAE,GAAG,EAAE;QAC9F,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,uBAAuB,CAC5C,aAAa,EACb,QAAQ,EACR,QAAQ,CACT,CAAC;QAEF,QAAQ,CAAC,oBAAoB,CAAC;YAC5B,YAAY,EAAE,GAAG;YACjB,eAAe,EAAE,CAAC;SACnB,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,GAAG,EAAE;QAC3F,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACvC,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,IAAI,uBAAuB,CAC5C,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,iBAAiB,CAClB,CAAC;QAEF,8BAA8B;QAE9B,mFAAmF;QACnF,sBAAsB;QACtB,oCAAoC;QACpC,mCAAmC;QACnC,0CAA0C;QAC1C,wDAAwD;QACxD,QAAQ,CAAC,oBAAoB,CAAC;YAC5B,YAAY,EAAE,GAAG;YACjB,eAAe,EAAE,GAAG;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uFAAuF,EAAE,GAAG,EAAE;QAC/F,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,cAAc;QACpC,MAAM,UAAU,GAAG,IAAI,uBAAuB,CAC5C,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,EAAE,OAAO,EAAE,CACZ,CAAC;QAEF,8BAA8B;QAE9B,qFAAqF;QACrF,wBAAwB;QACxB,gDAAgD;QAChD,0CAA0C;QAC1C,sDAAsD;QACtD,kCAAkC;QAClC,QAAQ,CAAC,oBAAoB,CAAC;YAC5B,YAAY,EAAE,KAAK;YACnB,eAAe,EAAE,GAAG;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAE3D,sCAAsC;QACtC,sBAAsB;QACtB,iDAAiD;QACjD,8DAA8D;QAC9D,6DAA6D;QAC7D,QAAQ,CAAC,oBAAoB,CAAC;YAC5B,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,GAAG;SACrB,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/context/utils/contextTokenCalculator.d.ts

@@ -3,14 +3,46 @@
  * Copyright 2026 Google LLC
  * SPDX-License-Identifier: Apache-2.0
  */
-import type { Part } from '@google/genai';
+import type { Part, Content } from '@google/genai';
 import type { ConcreteNode } from '../graph/types.js';
 import type { NodeBehaviorRegistry } from '../graph/behaviorRegistry.js';
 /**
  * The flat token cost assigned to a single multi-modal asset (like an image tile)
  * by the Gemini API. We use this as a baseline heuristic for inlineData/fileData.
  */
-export declare class ContextTokenCalculator {
+export interface ContextTokenCalculator {
+    estimateTokensForString(text: string): number;
+    tokensToChars(tokens: number): number;
+    garbageCollectCache(liveNodeIds: ReadonlySet<string>): void;
+    cacheNodeTokens(node: ConcreteNode): number;
+    getTokenCost(node: ConcreteNode): number;
+    calculateTokenBreakdown(nodes: readonly ConcreteNode[]): {
+        text: number;
+        media: number;
+        tool: number;
+        overhead: number;
+        total: number;
+    };
+    calculateConcreteListTokens(nodes: readonly ConcreteNode[]): number;
+    calculateContentTokens(content: Content): number;
+    estimateTokensForParts(parts: Part[]): number;
+}
+export interface AdvancedTokenCalculator extends ContextTokenCalculator {
+    getRawBaseUnits(nodes: readonly ConcreteNode[]): number;
+    getRawBaseUnitsForContent(content: Content): number;
+    calculateTokensAndBaseUnits(nodes: readonly ConcreteNode[]): {
+        tokens: number;
+        baseUnits: number;
+    };
+    calculateContentTokensAndBaseUnits(content: Content): {
+        tokens: number;
+        baseUnits: number;
+    };
+}
+/**
+ * A fast, deterministic token heuristic calculator.
+ */
+export declare class StaticTokenCalculator implements AdvancedTokenCalculator {
     private readonly charsPerToken;
     private readonly registry;
     private readonly tokenCache;
@@ -40,11 +72,40 @@ export declare class ContextTokenCalculator {
      * If it misses the cache, it computes it and caches it.
      */
     getTokenCost(node: ConcreteNode): number;
+    /**
+     * Calculates a detailed breakdown of tokens by category for a list of nodes.
+     * Useful for calibration tracing and debugging overestimation.
+     */
+    calculateTokenBreakdown(nodes: readonly ConcreteNode[]): {
+        total: number;
+        text: number;
+        media: number;
+        tool: number;
+        overhead: number;
+    };
+    /**
+     * For the static calculator, Raw Base Units are exactly the same as the final tokens,
+     * because there is no dynamic learned weight (the multiplier is effectively 1.0).
+     */
+    getRawBaseUnits(nodes: readonly ConcreteNode[]): number;
+    getRawBaseUnitsForContent(content: Content): number;
+    calculateTokensAndBaseUnits(nodes: readonly ConcreteNode[]): {
+        tokens: number;
+        baseUnits: number;
+    };
+    calculateContentTokensAndBaseUnits(content: Content): {
+        tokens: number;
+        baseUnits: number;
+    };
     /**
      * Fast calculation for a flat array of ConcreteNodes (The Nodes).
      * It relies entirely on the O(1) sidecar token cache.
      */
     calculateConcreteListTokens(nodes: readonly ConcreteNode[]): number;
+    /**
+     * Calculates the token cost for a single Gemini Content object.
+     */
+    calculateContentTokens(content: Content): number;
     /**
      * Slower, precise estimation for a Gemini Content/Part graph.
      * Deeply inspects the nested structure and uses the base tokenization math.

package/dist/src/context/utils/contextTokenCalculator.js

@@ -3,12 +3,11 @@
  * Copyright 2026 Google LLC
  * SPDX-License-Identifier: Apache-2.0
  */
-import { estimateTokenCountSync } from '../../utils/tokenCalculation.js';
+import { estimateTokenCountSync, MSG_OVERHEAD_TOKENS, } from '../../utils/tokenCalculation.js';
 /**
- * The flat token cost assigned to a single multi-modal asset (like an image tile)
- * by the Gemini API. We use this as a baseline heuristic for inlineData/fileData.
+ * A fast, deterministic token heuristic calculator.
  */
-export class ContextTokenCalculator {
+export class StaticTokenCalculator {
     charsPerToken;
     registry;
     tokenCache = new Map();
@@ -62,17 +61,93 @@ export class ContextTokenCalculator {
             return cached;
         return this.cacheNodeTokens(node);
     }
+    /**
+     * Calculates a detailed breakdown of tokens by category for a list of nodes.
+     * Useful for calibration tracing and debugging overestimation.
+     */
+    calculateTokenBreakdown(nodes) {
+        const breakdown = { total: 0, text: 0, media: 0, tool: 0, overhead: 0 };
+        const seenIds = new Set();
+        const seenTurnIds = new Set();
+        for (const node of nodes) {
+            if (seenIds.has(node.id))
+                continue;
+            seenIds.add(node.id);
+            if (node.turnId) {
+                if (!seenTurnIds.has(node.turnId)) {
+                    seenTurnIds.add(node.turnId);
+                    breakdown.overhead += MSG_OVERHEAD_TOKENS;
+                    breakdown.total += MSG_OVERHEAD_TOKENS;
+                }
+            }
+            const cost = this.getTokenCost(node);
+            breakdown.total += cost;
+            const behavior = this.registry.get(node.type);
+            const parts = behavior.getEstimatableParts(node);
+            for (const part of parts) {
+                if (typeof part.text === 'string') {
+                    breakdown.text += estimateTokenCountSync([part], 0, this.charsPerToken);
+                }
+                else if (part.inlineData?.mimeType?.startsWith('image/') ||
+                    part.fileData?.mimeType?.startsWith('image/')) {
+                    breakdown.media += estimateTokenCountSync([part], 0, this.charsPerToken);
+                }
+                else if (part.functionCall || part.functionResponse) {
+                    breakdown.tool += estimateTokenCountSync([part], 0, this.charsPerToken);
+                }
+                else {
+                    breakdown.overhead += estimateTokenCountSync([part], 0, this.charsPerToken);
+                }
+            }
+        }
+        return breakdown;
+    }
+    /**
+     * For the static calculator, Raw Base Units are exactly the same as the final tokens,
+     * because there is no dynamic learned weight (the multiplier is effectively 1.0).
+     */
+    getRawBaseUnits(nodes) {
+        return this.calculateConcreteListTokens(nodes);
+    }
+    getRawBaseUnitsForContent(content) {
+        return this.calculateContentTokens(content);
+    }
+    calculateTokensAndBaseUnits(nodes) {
+        const baseUnits = this.calculateConcreteListTokens(nodes);
+        return { tokens: baseUnits, baseUnits };
+    }
+    calculateContentTokensAndBaseUnits(content) {
+        const baseUnits = this.calculateContentTokens(content);
+        return { tokens: baseUnits, baseUnits };
+    }
     /**
      * Fast calculation for a flat array of ConcreteNodes (The Nodes).
      * It relies entirely on the O(1) sidecar token cache.
      */
     calculateConcreteListTokens(nodes) {
         let tokens = 0;
+        const seenIds = new Set();
+        const seenTurnIds = new Set();
         for (const node of nodes) {
-            tokens += this.getTokenCost(node);
+            if (!seenIds.has(node.id)) {
+                seenIds.add(node.id);
+                tokens += this.getTokenCost(node);
+                if (node.turnId) {
+                    if (!seenTurnIds.has(node.turnId)) {
+                        seenTurnIds.add(node.turnId);
+                        tokens += MSG_OVERHEAD_TOKENS;
+                    }
+                }
+            }
         }
         return tokens;
     }
+    /**
+     * Calculates the token cost for a single Gemini Content object.
+     */
+    calculateContentTokens(content) {
+        return (this.estimateTokensForParts(content.parts || []) + MSG_OVERHEAD_TOKENS);
+    }
     /**
      * Slower, precise estimation for a Gemini Content/Part graph.
      * Deeply inspects the nested structure and uses the base tokenization math.

package/dist/src/context/utils/contextTokenCalculator.js.map

@@ -1 +1 @@
-{"version":3,"file":"contextTokenCalculator.js","sourceRoot":"","sources":["../../../../src/context/utils/contextTokenCalculator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAIzE;;;GAGG;AAEH,MAAM,OAAO,sBAAsB;IAId;IACA;IAJF,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExD,YACmB,aAAqB,EACrB,QAA8B;QAD9B,kBAAa,GAAb,aAAa,CAAQ;QACrB,aAAQ,GAAR,QAAQ,CAAsB;IAC9C,CAAC;IAEJ;;;OAGG;IACH,uBAAuB,CAAC,IAAY;QAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACrD,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,OAAO,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;IACrC,CAAC;IAED;;;OAGG;IAEH;;;OAGG;IACH,mBAAmB,CAAC,WAAgC;QAClD,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAED,eAAe,CAAC,IAAkB;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACrC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,IAAkB;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,2BAA2B,CAAC,KAA8B;QACxD,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACc,cAAc,GAAG,IAAI,OAAO,EAAkB,CAAC;IAEhE,sBAAsB,CAAC,KAAa;QAClC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC9C,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,IAAI,GAAG,sBAAsB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC7D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACtC,CAAC;gBACD,KAAK,IAAI,IAAI,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,sBAAsB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
+{"version":3,"file":"contextTokenCalculator.js","sourceRoot":"","sources":["../../../../src/context/utils/contextTokenCalculator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EACL,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AAwCzC;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAIb;IACA;IAJF,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExD,YACmB,aAAqB,EACrB,QAA8B;QAD9B,kBAAa,GAAb,aAAa,CAAQ;QACrB,aAAQ,GAAR,QAAQ,CAAsB;IAC9C,CAAC;IAEJ;;;OAGG;IACH,uBAAuB,CAAC,IAAY;QAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IACrD,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,OAAO,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;IACrC,CAAC;IAED;;;OAGG;IAEH;;;OAGG;IACH,mBAAmB,CAAC,WAAgC;QAClD,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAED,eAAe,CAAC,IAAkB;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACrC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,IAAkB;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,uBAAuB,CAAC,KAA8B;QAOpD,MAAM,SAAS,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACxE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAAE,SAAS;YACnC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAErB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC7B,SAAS,CAAC,QAAQ,IAAI,mBAAmB,CAAC;oBAC1C,SAAS,CAAC,KAAK,IAAI,mBAAmB,CAAC;gBACzC,CAAC;YACH,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACrC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;YAEjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAClC,SAAS,CAAC,IAAI,IAAI,sBAAsB,CACtC,CAAC,IAAI,CAAC,EACN,CAAC,EACD,IAAI,CAAC,aAAa,CACnB,CAAC;gBACJ,CAAC;qBAAM,IACL,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC;oBAC/C,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,EAC7C,CAAC;oBACD,SAAS,CAAC,KAAK,IAAI,sBAAsB,CACvC,CAAC,IAAI,CAAC,EACN,CAAC,EACD,IAAI,CAAC,aAAa,CACnB,CAAC;gBACJ,CAAC;qBAAM,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACtD,SAAS,CAAC,IAAI,IAAI,sBAAsB,CACtC,CAAC,IAAI,CAAC,EACN,CAAC,EACD,IAAI,CAAC,aAAa,CACnB,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,SAAS,CAAC,QAAQ,IAAI,sBAAsB,CAC1C,CAAC,IAAI,CAAC,EACN,CAAC,EACD,IAAI,CAAC,aAAa,CACnB,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,KAA8B;QAC5C,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;IAED,yBAAyB,CAAC,OAAgB;QACxC,OAAO,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,2BAA2B,CAAC,KAA8B;QAIxD,MAAM,SAAS,GAAG,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED,kCAAkC,CAAC,OAAgB;QAIjD,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACvD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACH,2BAA2B,CAAC,KAA8B;QACxD,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;gBAElC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;wBAClC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC7B,MAAM,IAAI,mBAAmB,CAAC;oBAChC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,OAAgB;QACrC,OAAO,CACL,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,GAAG,mBAAmB,CACvE,CAAC;IACJ,CAAC;IAED;;;OAGG;IACc,cAAc,GAAG,IAAI,OAAO,EAAkB,CAAC;IAEhE,sBAAsB,CAAC,KAAa;QAClC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC9C,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,IAAI,GAAG,sBAAsB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC7D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACtC,CAAC;gBACD,KAAK,IAAI,IAAI,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,sBAAsB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/src/context/utils/contextTokenCalculator.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/context/utils/contextTokenCalculator.test.js

@@ -0,0 +1,54 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect } from 'vitest';
+import { StaticTokenCalculator } from './contextTokenCalculator.js';
+import { NodeBehaviorRegistry } from '../graph/behaviorRegistry.js';
+import { registerBuiltInBehaviors } from '../graph/builtinBehaviors.js';
+import { createDummyNode } from '../testing/contextTestUtils.js';
+import { MSG_OVERHEAD_TOKENS } from '../../utils/tokenCalculation.js';
+import { NodeType } from '../graph/types.js';
+describe('ContextTokenCalculator', () => {
+    const registry = new NodeBehaviorRegistry();
+    registerBuiltInBehaviors(registry);
+    const charsPerToken = 1; // Simplifies math for text nodes in tests
+    const calculator = new StaticTokenCalculator(charsPerToken, registry);
+    it('should include structural overhead for each unique turn', () => {
+        const turn1Id = 'turn-1';
+        const turn2Id = 'turn-2';
+        const node1 = createDummyNode(turn1Id, NodeType.USER_PROMPT);
+        const node2 = createDummyNode(turn1Id, NodeType.USER_PROMPT); // Same turn
+        const node3 = createDummyNode(turn2Id, NodeType.AGENT_THOUGHT); // Different turn
+        const nodes = [node1, node2, node3];
+        // Estimated tokens (using charsPerToken = 1):
+        // node1: 17 chars / 1 = 17 tokens
+        // node2: 17 chars / 1 = 17 tokens
+        // node3: 19 chars / 1 = 19 tokens
+        // Turn 1 overhead: 5 tokens
+        // Turn 2 overhead: 5 tokens
+        // Total: 17 + 17 + 19 + 5 + 5 = 63
+        const total = calculator.calculateConcreteListTokens(nodes);
+        expect(total).toBe(63);
+    });
+    it('should handle categorical breakdown with overhead', () => {
+        const turn1Id = 'turn-1';
+        const node = createDummyNode(turn1Id, NodeType.USER_PROMPT);
+        const breakdown = calculator.calculateTokenBreakdown([node]);
+        expect(breakdown.overhead).toBe(MSG_OVERHEAD_TOKENS);
+        expect(breakdown.total).toBe(calculator.getTokenCost(node) + MSG_OVERHEAD_TOKENS);
+    });
+    it('should not double-count overhead for duplicate turn IDs in separate nodes', () => {
+        const turn1Id = 'turn-1';
+        const node1 = createDummyNode(turn1Id, NodeType.USER_PROMPT);
+        const node2 = createDummyNode(turn1Id, NodeType.USER_PROMPT);
+        const total = calculator.calculateConcreteListTokens([node1, node2]);
+        // cost(node1) + cost(node2) + 1 * overhead
+        const expected = calculator.getTokenCost(node1) +
+            calculator.getTokenCost(node2) +
+            MSG_OVERHEAD_TOKENS;
+        expect(total).toBe(expected);
+    });
+});
+//# sourceMappingURL=contextTokenCalculator.test.js.map

package/dist/src/context/utils/contextTokenCalculator.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextTokenCalculator.test.js","sourceRoot":"","sources":["../../../../src/context/utils/contextTokenCalculator.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,MAAM,QAAQ,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC5C,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,CAAC,CAAC,CAAC,0CAA0C;IACnE,MAAM,UAAU,GAAG,IAAI,qBAAqB,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IAEtE,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,OAAO,GAAG,QAAQ,CAAC;QACzB,MAAM,OAAO,GAAG,QAAQ,CAAC;QAEzB,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY;QAC1E,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,iBAAiB;QAEjF,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAEpC,8CAA8C;QAC9C,kCAAkC;QAClC,kCAAkC;QAClC,kCAAkC;QAClC,4BAA4B;QAC5B,4BAA4B;QAC5B,mCAAmC;QAEnC,MAAM,KAAK,GAAG,UAAU,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC;QACzB,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QAE5D,MAAM,SAAS,GAAG,UAAU,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAE7D,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrD,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAC1B,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,mBAAmB,CACpD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,MAAM,OAAO,GAAG,QAAQ,CAAC;QACzB,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QAE7D,MAAM,KAAK,GAAG,UAAU,CAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAErE,2CAA2C;QAC3C,MAAM,QAAQ,GACZ,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC;YAC9B,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC;YAC9B,mBAAmB,CAAC;QACtB,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/context/utils/formatNodesForLlm.d.ts

@@ -0,0 +1,21 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import type { ConcreteNode } from '../graph/types.js';
+export interface FormatNodesOptions {
+    /**
+     * The maximum number of characters to retain from a tool response.
+     * Tool responses larger than this will be truncated to preserve LLM attention span
+     * and avoid context limits during summarization operations.
+     * Defaults to 2000.
+     */
+    maxToolResponseChars?: number;
+}
+/**
+ * Formats a sequence of Context Graph nodes into a dense, human/LLM-readable text transcript.
+ * This is used by summarization processors (like SnapshotGenerator and RollingSummaryProcessor)
+ * to serialize the graph before passing it to an LLM.
+ */
+export declare function formatNodesForLlm(nodes: readonly ConcreteNode[], options?: FormatNodesOptions): string;