@tuanhung303/opencode-acp 2.2.3 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/commands/stats.d.ts.map +1 -1
- package/dist/lib/commands/stats.js +2 -0
- package/dist/lib/commands/stats.js.map +1 -1
- package/dist/lib/config-schema.d.ts +88 -0
- package/dist/lib/config-schema.d.ts.map +1 -1
- package/dist/lib/config-schema.js +36 -0
- package/dist/lib/config-schema.js.map +1 -1
- package/dist/lib/config.d.ts +52 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +89 -0
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/hooks.d.ts.map +1 -1
- package/dist/lib/hooks.js +101 -17
- package/dist/lib/hooks.js.map +1 -1
- package/dist/lib/messages/index.d.ts +1 -1
- package/dist/lib/messages/index.d.ts.map +1 -1
- package/dist/lib/messages/index.js +1 -1
- package/dist/lib/messages/index.js.map +1 -1
- package/dist/lib/messages/prune.d.ts +8 -0
- package/dist/lib/messages/prune.d.ts.map +1 -1
- package/dist/lib/messages/prune.js +112 -0
- package/dist/lib/messages/prune.js.map +1 -1
- package/dist/lib/prompts/system/both.d.ts +1 -1
- package/dist/lib/prompts/system/both.d.ts.map +1 -1
- package/dist/lib/prompts/system/both.js +40 -42
- package/dist/lib/prompts/system/both.js.map +1 -1
- package/dist/lib/prompts/system/discard.d.ts +1 -1
- package/dist/lib/prompts/system/discard.d.ts.map +1 -1
- package/dist/lib/prompts/system/discard.js +33 -30
- package/dist/lib/prompts/system/discard.js.map +1 -1
- package/dist/lib/prompts/system/extract.d.ts +1 -1
- package/dist/lib/prompts/system/extract.d.ts.map +1 -1
- package/dist/lib/prompts/system/extract.js +27 -29
- package/dist/lib/prompts/system/extract.js.map +1 -1
- package/dist/lib/state/persistence.d.ts +2 -0
- package/dist/lib/state/persistence.d.ts.map +1 -1
- package/dist/lib/state/persistence.js +4 -1
- package/dist/lib/state/persistence.js.map +1 -1
- package/dist/lib/state/state.d.ts.map +1 -1
- package/dist/lib/state/state.js +22 -0
- package/dist/lib/state/state.js.map +1 -1
- package/dist/lib/state/types.d.ts +121 -13
- package/dist/lib/state/types.d.ts.map +1 -1
- package/dist/lib/state/utils.d.ts +43 -0
- package/dist/lib/state/utils.d.ts.map +1 -1
- package/dist/lib/state/utils.js +138 -0
- package/dist/lib/state/utils.js.map +1 -1
- package/dist/lib/strategies/deduplication.d.ts.map +1 -1
- package/dist/lib/strategies/deduplication.js +19 -3
- package/dist/lib/strategies/deduplication.js.map +1 -1
- package/dist/lib/strategies/error-chain-collapse.d.ts +9 -0
- package/dist/lib/strategies/error-chain-collapse.d.ts.map +1 -0
- package/dist/lib/strategies/error-chain-collapse.js +118 -0
- package/dist/lib/strategies/error-chain-collapse.js.map +1 -0
- package/dist/lib/strategies/image-pruning.d.ts +16 -0
- package/dist/lib/strategies/image-pruning.d.ts.map +1 -0
- package/dist/lib/strategies/image-pruning.js +251 -0
- package/dist/lib/strategies/image-pruning.js.map +1 -0
- package/dist/lib/strategies/index.d.ts +5 -0
- package/dist/lib/strategies/index.d.ts.map +1 -1
- package/dist/lib/strategies/index.js +5 -0
- package/dist/lib/strategies/index.js.map +1 -1
- package/dist/lib/strategies/purge-errors.d.ts.map +1 -1
- package/dist/lib/strategies/purge-errors.js +15 -3
- package/dist/lib/strategies/purge-errors.js.map +1 -1
- package/dist/lib/strategies/stale-read-pruning.d.ts +16 -0
- package/dist/lib/strategies/stale-read-pruning.d.ts.map +1 -0
- package/dist/lib/strategies/stale-read-pruning.js +132 -0
- package/dist/lib/strategies/stale-read-pruning.js.map +1 -0
- package/dist/lib/strategies/supersede-writes.d.ts.map +1 -1
- package/dist/lib/strategies/supersede-writes.js +15 -3
- package/dist/lib/strategies/supersede-writes.js.map +1 -1
- package/dist/lib/strategies/thinking-compression.d.ts +16 -0
- package/dist/lib/strategies/thinking-compression.d.ts.map +1 -0
- package/dist/lib/strategies/thinking-compression.js +194 -0
- package/dist/lib/strategies/thinking-compression.js.map +1 -0
- package/dist/lib/strategies/tools.d.ts.map +1 -1
- package/dist/lib/strategies/tools.js +94 -53
- package/dist/lib/strategies/tools.js.map +1 -1
- package/dist/lib/strategies/truncation.d.ts +17 -0
- package/dist/lib/strategies/truncation.d.ts.map +1 -0
- package/dist/lib/strategies/truncation.js +172 -0
- package/dist/lib/strategies/truncation.js.map +1 -0
- package/dist/lib/strategies/utils.d.ts +1 -1
- package/dist/lib/strategies/utils.d.ts.map +1 -1
- package/dist/lib/strategies/utils.js +34 -20
- package/dist/lib/strategies/utils.js.map +1 -1
- package/dist/lib/ui/notification.d.ts +1 -1
- package/dist/lib/ui/notification.d.ts.map +1 -1
- package/dist/lib/ui/notification.js +12 -7
- package/dist/lib/ui/notification.js.map +1 -1
- package/dist/lib/ui/utils.d.ts +1 -1
- package/dist/lib/ui/utils.d.ts.map +1 -1
- package/dist/lib/ui/utils.js +16 -2
- package/dist/lib/ui/utils.js.map +1 -1
- package/package.json +1 -1
package/dist/lib/state/utils.js
CHANGED
|
@@ -7,4 +7,142 @@ export async function isSubAgentSession(client, sessionID) {
|
|
|
7
7
|
return false;
|
|
8
8
|
}
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Gets or creates cached prune sets for O(1) lookups.
|
|
12
|
+
* The sets are computed once and cached until prune arrays change.
|
|
13
|
+
*/
|
|
14
|
+
export function getPruneSets(state) {
|
|
15
|
+
// Return cached sets if they exist
|
|
16
|
+
if (state.pruneSets) {
|
|
17
|
+
return state.pruneSets;
|
|
18
|
+
}
|
|
19
|
+
// Create new sets from arrays
|
|
20
|
+
const sets = {
|
|
21
|
+
toolIds: new Set(state.prune.toolIds),
|
|
22
|
+
messagePartIds: new Set(state.prune.messagePartIds),
|
|
23
|
+
};
|
|
24
|
+
// Cache them
|
|
25
|
+
state.pruneSets = sets;
|
|
26
|
+
return sets;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Invalidates the cached prune sets.
|
|
30
|
+
* Call this whenever prune arrays are modified.
|
|
31
|
+
*/
|
|
32
|
+
export function invalidatePruneSets(state) {
|
|
33
|
+
state.pruneSets = undefined;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Checks if a tool ID is pruned using O(1) Set lookup.
|
|
37
|
+
*/
|
|
38
|
+
export function isToolPruned(state, toolId) {
|
|
39
|
+
return getPruneSets(state).toolIds.has(toolId);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Checks if a message part ID is pruned using O(1) Set lookup.
|
|
43
|
+
*/
|
|
44
|
+
export function isMessagePartPruned(state, partId) {
|
|
45
|
+
return getPruneSets(state).messagePartIds.has(partId);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Adds a tool ID to prune list and invalidates cache.
|
|
49
|
+
*/
|
|
50
|
+
export function addPrunedTool(state, toolId) {
|
|
51
|
+
state.prune.toolIds.push(toolId);
|
|
52
|
+
invalidatePruneSets(state);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Adds a message part ID to prune list and invalidates cache.
|
|
56
|
+
*/
|
|
57
|
+
export function addPrunedMessagePart(state, partId) {
|
|
58
|
+
state.prune.messagePartIds.push(partId);
|
|
59
|
+
invalidatePruneSets(state);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Creates a new token count cache.
|
|
63
|
+
*/
|
|
64
|
+
export function createTokenCountCache(maxSize = 1000) {
|
|
65
|
+
return {
|
|
66
|
+
cache: new Map(),
|
|
67
|
+
maxSize,
|
|
68
|
+
hits: 0,
|
|
69
|
+
misses: 0,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Gets cached token count or calculates and caches it.
|
|
74
|
+
*/
|
|
75
|
+
export function getCachedTokenCount(state, content, calculateFn) {
|
|
76
|
+
// Initialize cache if needed
|
|
77
|
+
if (!state.tokenCache) {
|
|
78
|
+
state.tokenCache = createTokenCountCache();
|
|
79
|
+
}
|
|
80
|
+
const cache = state.tokenCache;
|
|
81
|
+
const hash = hashContent(content);
|
|
82
|
+
// Check cache
|
|
83
|
+
const cached = cache.cache.get(hash);
|
|
84
|
+
if (cached !== undefined) {
|
|
85
|
+
cache.hits++;
|
|
86
|
+
return cached;
|
|
87
|
+
}
|
|
88
|
+
// Calculate and cache
|
|
89
|
+
cache.misses++;
|
|
90
|
+
const count = calculateFn(content);
|
|
91
|
+
cache.cache.set(hash, count);
|
|
92
|
+
// Evict oldest entries if cache is full (simple LRU)
|
|
93
|
+
if (cache.cache.size > cache.maxSize) {
|
|
94
|
+
const firstKey = cache.cache.keys().next().value;
|
|
95
|
+
if (firstKey !== undefined) {
|
|
96
|
+
cache.cache.delete(firstKey);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return count;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Simple hash function for content.
|
|
103
|
+
*/
|
|
104
|
+
function hashContent(content) {
|
|
105
|
+
let hash = 0;
|
|
106
|
+
for (let i = 0; i < content.length; i++) {
|
|
107
|
+
const char = content.charCodeAt(i);
|
|
108
|
+
hash = (hash << 5) - hash + char;
|
|
109
|
+
hash = hash & hash; // Convert to 32bit integer
|
|
110
|
+
}
|
|
111
|
+
return hash.toString(16);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Creates performance metrics for a session.
|
|
115
|
+
*/
|
|
116
|
+
export function createPerformanceMetrics() {
|
|
117
|
+
return {
|
|
118
|
+
totalStrategyTime: 0,
|
|
119
|
+
strategyTimes: {
|
|
120
|
+
deduplication: 0,
|
|
121
|
+
supersedeWrites: 0,
|
|
122
|
+
purgeErrors: 0,
|
|
123
|
+
truncation: 0,
|
|
124
|
+
thinkingCompression: 0,
|
|
125
|
+
},
|
|
126
|
+
strategyInvocations: {
|
|
127
|
+
deduplication: 0,
|
|
128
|
+
supersedeWrites: 0,
|
|
129
|
+
purgeErrors: 0,
|
|
130
|
+
truncation: 0,
|
|
131
|
+
thinkingCompression: 0,
|
|
132
|
+
},
|
|
133
|
+
lastExecutionTime: 0,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Records strategy execution time.
|
|
138
|
+
*/
|
|
139
|
+
export function recordStrategyTime(state, strategy, durationMs) {
|
|
140
|
+
if (!state.performanceMetrics) {
|
|
141
|
+
state.performanceMetrics = createPerformanceMetrics();
|
|
142
|
+
}
|
|
143
|
+
state.performanceMetrics.strategyTimes[strategy] += durationMs;
|
|
144
|
+
state.performanceMetrics.strategyInvocations[strategy]++;
|
|
145
|
+
state.performanceMetrics.totalStrategyTime += durationMs;
|
|
146
|
+
state.performanceMetrics.lastExecutionTime = Date.now();
|
|
147
|
+
}
|
|
10
148
|
//# sourceMappingURL=utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../lib/state/utils.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAW,EAAE,SAAiB;IAClE,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAA;QACpE,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAA;IAClC,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,OAAO,KAAK,CAAA;IAChB,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../lib/state/utils.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAW,EAAE,SAAiB;IAClE,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAA;QACpE,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAA;IAClC,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,OAAO,KAAK,CAAA;IAChB,CAAC;AACL,CAAC;AAID;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,KAAmB;IAC5C,mCAAmC;IACnC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC,SAAS,CAAA;IAC1B,CAAC;IAED,8BAA8B;IAC9B,MAAM,IAAI,GAAc;QACpB,OAAO,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;QACrC,cAAc,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC;KACtD,CAAA;IAED,aAAa;IACb,KAAK,CAAC,SAAS,GAAG,IAAI,CAAA;IACtB,OAAO,IAAI,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAmB;IACnD,KAAK,CAAC,SAAS,GAAG,SAAS,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,MAAc;IAC5D,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAmB,EAAE,MAAc;IACnE,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;AACzD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,KAAmB,EAAE,MAAc;IAC7D,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAChC,mBAAmB,CAAC,KAAK,CAAC,CAAA;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAmB,EAAE,MAAc;IACpE,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACvC,mBAAmB,CAAC,KAAK,CAAC,CAAA;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAO,GAAG,IAAI;IAChD,OAAO;QACH,KAAK,EAAE,IAAI,GAAG,EAAkB;QAChC,OAAO;QACP,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;KACZ,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAC/B,KAAmB,EACnB,OAAe,EACf,WAAwC;IAExC,6BAA6B;IAC7B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QACpB,KAAK,CAAC,UAAU,GAAG,qBAAqB,EAAE,CAAA;IAC9C,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAA;IAC9B,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;IAEjC,cAAc;IACd,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,EAAE,CAAA;QACZ,OAAO,MAAM,CAAA;IACjB,CAAC;IAED,sBAAsB;IACtB,KAAK,CAAC,MAAM,EAAE,CAAA;IACd,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;IAClC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAE5B,qDAAqD;IACrD,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;QAChD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAChC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAA;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,OAAe;IAChC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAClC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;QAChC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAA,CAAC,2BAA2B;IAClD,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACpC,OAAO;QACH,iBAAiB,EAAE,CAAC;QACpB,aAAa,EAAE;YACX,aAAa,EAAE,CAAC;YAChB,eAAe,EAAE,CAAC;YAClB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,mBAAmB,EAAE,CAAC;SACzB;QACD,mBAAmB,EAAE;YACjB,aAAa,EAAE,CAAC;YAChB,eAAe,EAAE,CAAC;YAClB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,mBAAmB,EAAE,CAAC;SACzB;QACD,iBAAiB,EAAE,CAAC;KACvB,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAC9B,KAAmB,EACnB,QAAmD,EACnD,UAAkB;IAElB,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC5B,KAAK,CAAC,kBAAkB,GAAG,wBAAwB,EAAE,CAAA;IACzD,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAA;IAC9D,KAAK,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAA;IACxD,KAAK,CAAC,kBAAkB,CAAC,iBAAiB,IAAI,UAAU,CAAA;IACxD,KAAK,CAAC,kBAAkB,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;AAC3D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deduplication.d.ts","sourceRoot":"","sources":["../../../lib/strategies/deduplication.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;
|
|
1
|
+
{"version":3,"file":"deduplication.d.ts","sourceRoot":"","sources":["../../../lib/strategies/deduplication.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AAgBvD;;;;;GAKG;AACH,eAAO,MAAM,WAAW,GACpB,OAAO,YAAY,EACnB,QAAQ,MAAM,EACd,QAAQ,YAAY,EACpB,UAAU,SAAS,EAAE,KACtB,IAiGF,CAAA"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { buildToolIdList } from "../messages/utils";
|
|
2
2
|
import { getFilePathFromParameters, isProtectedFilePath } from "../protected-file-patterns";
|
|
3
3
|
import { calculateTokensSaved } from "./utils";
|
|
4
|
+
import { getPruneSets, addPrunedTool, recordStrategyTime } from "../state/utils";
|
|
4
5
|
/**
|
|
5
6
|
* Deduplication strategy - prunes older tool calls that have identical
|
|
6
7
|
* tool name and parameters, keeping only the most recent occurrence.
|
|
@@ -8,16 +9,25 @@ import { calculateTokensSaved } from "./utils";
|
|
|
8
9
|
* Modifies the session state in place to add pruned tool call IDs.
|
|
9
10
|
*/
|
|
10
11
|
export const deduplicate = (state, logger, config, messages) => {
|
|
12
|
+
const startTime = performance.now();
|
|
11
13
|
if (!config.strategies.deduplication.enabled) {
|
|
12
14
|
return;
|
|
13
15
|
}
|
|
16
|
+
// Early exit: if no messages, nothing to deduplicate
|
|
17
|
+
if (messages.length === 0) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
14
20
|
// Build list of all tool call IDs from messages (chronological order)
|
|
15
21
|
const allToolIds = buildToolIdList(state, messages, logger);
|
|
16
22
|
if (allToolIds.length === 0) {
|
|
17
23
|
return;
|
|
18
24
|
}
|
|
19
|
-
//
|
|
20
|
-
|
|
25
|
+
// Early exit: if only one tool, no duplicates possible
|
|
26
|
+
if (allToolIds.length < 2) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
// Filter out IDs already pruned using O(1) Set lookup
|
|
30
|
+
const alreadyPruned = getPruneSets(state).toolIds;
|
|
21
31
|
const unprunedIds = allToolIds.filter((id) => !alreadyPruned.has(id));
|
|
22
32
|
if (unprunedIds.length === 0) {
|
|
23
33
|
return;
|
|
@@ -63,13 +73,19 @@ export const deduplicate = (state, logger, config, messages) => {
|
|
|
63
73
|
logger.debug(`Marked ${overlappingReads.length} overlapping file reads for pruning`);
|
|
64
74
|
}
|
|
65
75
|
if (allDeduplicatedIds.length > 0) {
|
|
66
|
-
state.prune.toolIds.push(...allDeduplicatedIds);
|
|
67
76
|
const tokensSaved = calculateTokensSaved(state, messages, allDeduplicatedIds);
|
|
68
77
|
state.stats.totalPruneTokens += tokensSaved;
|
|
69
78
|
state.stats.strategyStats.deduplication.count += allDeduplicatedIds.length;
|
|
70
79
|
state.stats.strategyStats.deduplication.tokens += tokensSaved;
|
|
80
|
+
// Use addPrunedTool to maintain cache consistency
|
|
81
|
+
for (const id of allDeduplicatedIds) {
|
|
82
|
+
addPrunedTool(state, id);
|
|
83
|
+
}
|
|
71
84
|
logger.debug(`Marked ${allDeduplicatedIds.length} duplicate tool calls for pruning`);
|
|
72
85
|
}
|
|
86
|
+
// Record performance metrics
|
|
87
|
+
const duration = performance.now() - startTime;
|
|
88
|
+
recordStrategyTime(state, "deduplication", duration);
|
|
73
89
|
};
|
|
74
90
|
/**
|
|
75
91
|
* Find overlapping file reads that can be safely pruned.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deduplication.js","sourceRoot":"","sources":["../../../lib/strategies/deduplication.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAA;
|
|
1
|
+
{"version":3,"file":"deduplication.js","sourceRoot":"","sources":["../../../lib/strategies/deduplication.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAA;AAC9C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAYhF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CACvB,KAAmB,EACnB,MAAc,EACd,MAAoB,EACpB,QAAqB,EACjB,EAAE;IACN,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAEnC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QAC3C,OAAM;IACV,CAAC;IAED,qDAAqD;IACrD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAM;IACV,CAAC;IAED,sEAAsE;IACtE,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAA;IAC3D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAM;IACV,CAAC;IAED,uDAAuD;IACvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAM;IACV,CAAC;IAED,sDAAsD;IACtD,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAA;IACjD,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAErE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAM;IACV,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,cAAc,CAAA;IAErE,yDAAyD;IACzD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAA;IAEhD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,0DAA0D;YAC1D,SAAQ;QACZ,CAAC;QAED,uBAAuB;QACvB,IAAI,cAAc,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,SAAQ;QACZ,CAAC;QAED,MAAM,QAAQ,GAAG,yBAAyB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;QAC/D,IAAI,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC9D,SAAQ;QACZ,CAAC;QAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAA;QACzE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;QACnC,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACzC,CAAC;IAED,mEAAmE;IACnE,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjB,iDAAiD;YACjD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACpC,WAAW,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAA;QACpC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAA;IAElF,MAAM,kBAAkB,GAAa,CAAC,GAAG,WAAW,CAAC,CAAA;IAErD,0DAA0D;IAC1D,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,CAAC,CAAA;IACzF,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,kBAAkB,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAA;QAC5C,MAAM,CAAC,KAAK,CAAC,UAAU,gBAAgB,CAAC,MAAM,qCAAqC,CAAC,CAAA;IACxF,CAAC;IAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;QAC7E,KAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,WAAW,CAAA;QAC3C,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC,KAAK,IAAI,kBAAkB,CAAC,MAAM,CAAA;QAC1E,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC,MAAM,IAAI,WAAW,CAAA;QAC7D,kDAAkD;QAClD,KAAK,MAAM,EAAE,IAAI,kBAAkB,EAAE,CAAC;YAClC,aAAa,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC5B,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,UAAU,kBAAkB,CAAC,MAAM,mCAAmC,CAAC,CAAA;IACxF,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;IAC9C,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,QAAQ,CAAC,CAAA;AACxD,CAAC,CAAA;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CACzB,KAAmB,EACnB,OAAiB,EACjB,cAAwB,EACxB,MAAoB;IAEpB,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAA;IAEpD,8CAA8C;IAC9C,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC7C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM;YAAE,SAAQ;QACnD,IAAI,cAAc,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAQ;QAEpD,MAAM,QAAQ,GAAG,yBAAyB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;QAC/D,IAAI,CAAC,QAAQ,IAAI,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,qBAAqB,CAAC;YAAE,SAAQ;QAEtF,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAA;QACxC,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QACpE,MAAM,KAAK,GAAG,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAA;QAEzE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAC/B,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;IAClE,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,8CAA8C;IAC9C,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAQ;QAE9B,yEAAyE;QACzE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAChB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;gBAAE,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;YACrD,8EAA8E;YAC9E,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAA;YACnC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC,CAAA;YACpC,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAA;QAC5B,CAAC,CAAC,CAAA;QAEF,yFAAyF;QACzF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YACvB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA,CAAC,gCAAgC;YAEvE,IAAI,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE;gBAAE,SAAQ;YAEnC,2DAA2D;YAC3D,IAAI,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;gBACjC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAC1B,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAA;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,CAAgB,EAAE,CAAgB;IACxD,+DAA+D;IAC/D,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAA;IAC/B,CAAC;IAED,kDAAkD;IAClD,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,KAAK,CAAA;IAChB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAA;IAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAA;IAE/B,wEAAwE;IACxE,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAA;AAC/C,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,UAAgB;IACvD,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,OAAO,IAAI,CAAA;IACf,CAAC;IACD,MAAM,UAAU,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAA;IAClD,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;IACzC,OAAO,GAAG,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAA;AAC/C,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAW;IACpC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,MAAM,CAAA;IAChE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAA;IAExC,MAAM,UAAU,GAAQ,EAAE,CAAA;IAC1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACxC,UAAU,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;QAC3B,CAAC;IACL,CAAC;IACD,OAAO,UAAU,CAAA;AACrB,CAAC;AAED,SAAS,cAAc,CAAC,GAAQ;IAC5B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,GAAG,CAAA;IACvD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAEtD,MAAM,MAAM,GAAQ,EAAE,CAAA;IACtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IACD,OAAO,MAAM,CAAA;AACjB,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { PluginConfig } from "../config";
|
|
2
|
+
import { Logger } from "../logger";
|
|
3
|
+
import type { SessionState, WithParts } from "../state";
|
|
4
|
+
/**
|
|
5
|
+
* Collapses chains of related error tools.
|
|
6
|
+
* Modifies the session state in place to add pruned tool call IDs.
|
|
7
|
+
*/
|
|
8
|
+
export declare const collapseErrorChains: (state: SessionState, logger: Logger, config: PluginConfig, messages: WithParts[]) => void;
|
|
9
|
+
//# sourceMappingURL=error-chain-collapse.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"error-chain-collapse.d.ts","sourceRoot":"","sources":["../../../lib/strategies/error-chain-collapse.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AA2BvD;;;GAGG;AACH,eAAO,MAAM,mBAAmB,GAC5B,OAAO,YAAY,EACnB,QAAQ,MAAM,EACd,QAAQ,YAAY,EACpB,UAAU,SAAS,EAAE,KACtB,IA2EF,CAAA"}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { getPruneSets, addPrunedTool, recordStrategyTime } from "../state/utils";
|
|
2
|
+
import { buildToolIdList } from "../messages/utils";
|
|
3
|
+
import { calculateTokensSaved } from "./utils";
|
|
4
|
+
/**
|
|
5
|
+
* Collapses chains of related error tools.
|
|
6
|
+
* Modifies the session state in place to add pruned tool call IDs.
|
|
7
|
+
*/
|
|
8
|
+
export const collapseErrorChains = (state, logger, config, messages) => {
|
|
9
|
+
const startTime = performance.now();
|
|
10
|
+
if (!config.strategies.errorChainCollapse?.enabled) {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
// Early exit: if no messages, nothing to process
|
|
14
|
+
if (messages.length === 0) {
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
// Build list of all tool call IDs from messages (chronological order)
|
|
18
|
+
const allToolIds = buildToolIdList(state, messages, logger);
|
|
19
|
+
if (allToolIds.length === 0) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
// Filter out IDs already pruned using O(1) Set lookup
|
|
23
|
+
const alreadyPruned = getPruneSets(state).toolIds;
|
|
24
|
+
const unprunedIds = allToolIds.filter((id) => !alreadyPruned.has(id));
|
|
25
|
+
if (unprunedIds.length === 0) {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
const minChainLength = config.strategies.errorChainCollapse.minChainLength ?? 3;
|
|
29
|
+
const minTurnsOld = config.strategies.errorChainCollapse.minTurnsOld ?? 1;
|
|
30
|
+
// Find error chains
|
|
31
|
+
const errorChains = findErrorChains(state, unprunedIds, minChainLength, minTurnsOld);
|
|
32
|
+
if (errorChains.length === 0) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
// Collapse each chain (prune all but the root error)
|
|
36
|
+
const newPruneIds = [];
|
|
37
|
+
for (const chain of errorChains) {
|
|
38
|
+
// Keep the root error, prune the rest of the chain
|
|
39
|
+
for (const errorId of chain.chain) {
|
|
40
|
+
if (errorId !== chain.rootError) {
|
|
41
|
+
newPruneIds.push(errorId);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
logger.debug(`Collapsing error chain`, {
|
|
45
|
+
rootError: chain.rootError,
|
|
46
|
+
chainLength: chain.chain.length,
|
|
47
|
+
tokensInChain: chain.totalTokens,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
if (newPruneIds.length > 0) {
|
|
51
|
+
const tokensSaved = calculateTokensSaved(state, messages, newPruneIds);
|
|
52
|
+
state.stats.totalPruneTokens += tokensSaved;
|
|
53
|
+
state.stats.strategyStats.errorChainCollapse = {
|
|
54
|
+
count: (state.stats.strategyStats.errorChainCollapse?.count || 0) + newPruneIds.length,
|
|
55
|
+
tokens: (state.stats.strategyStats.errorChainCollapse?.tokens || 0) + tokensSaved,
|
|
56
|
+
};
|
|
57
|
+
// Use addPrunedTool to maintain cache consistency
|
|
58
|
+
for (const id of newPruneIds) {
|
|
59
|
+
addPrunedTool(state, id);
|
|
60
|
+
}
|
|
61
|
+
logger.info(`Collapsed ${errorChains.length} error chain(s), pruned ${newPruneIds.length} tool(s)`);
|
|
62
|
+
}
|
|
63
|
+
// Record performance metrics
|
|
64
|
+
const duration = performance.now() - startTime;
|
|
65
|
+
recordStrategyTime(state, "errorChainCollapse", duration);
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* Finds chains of related error tools.
|
|
69
|
+
*/
|
|
70
|
+
function findErrorChains(state, toolIds, minChainLength, minTurnsOld) {
|
|
71
|
+
const chains = [];
|
|
72
|
+
const processed = new Set();
|
|
73
|
+
// Group consecutive errors
|
|
74
|
+
let currentChain = [];
|
|
75
|
+
let currentRoot = null;
|
|
76
|
+
for (const id of toolIds) {
|
|
77
|
+
if (processed.has(id)) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
const metadata = state.toolParameters.get(id);
|
|
81
|
+
if (!metadata) {
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
// Check age threshold
|
|
85
|
+
const toolAge = state.currentTurn - metadata.turn;
|
|
86
|
+
if (toolAge < minTurnsOld) {
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (metadata.status === "error") {
|
|
90
|
+
if (currentChain.length === 0) {
|
|
91
|
+
currentRoot = id;
|
|
92
|
+
}
|
|
93
|
+
currentChain.push(id);
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
// End of error chain
|
|
97
|
+
if (currentChain.length >= minChainLength) {
|
|
98
|
+
chains.push({
|
|
99
|
+
rootError: currentRoot,
|
|
100
|
+
chain: [...currentChain],
|
|
101
|
+
totalTokens: 0, // Will be calculated later
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
currentChain = [];
|
|
105
|
+
currentRoot = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// Check if there's a chain at the end
|
|
109
|
+
if (currentChain.length >= minChainLength) {
|
|
110
|
+
chains.push({
|
|
111
|
+
rootError: currentRoot,
|
|
112
|
+
chain: [...currentChain],
|
|
113
|
+
totalTokens: 0,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return chains;
|
|
117
|
+
}
|
|
118
|
+
//# sourceMappingURL=error-chain-collapse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"error-chain-collapse.js","sourceRoot":"","sources":["../../../lib/strategies/error-chain-collapse.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAChF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAA;AAwB9C;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAC/B,KAAmB,EACnB,MAAc,EACd,MAAoB,EACpB,QAAqB,EACjB,EAAE;IACN,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAEnC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,kBAAkB,EAAE,OAAO,EAAE,CAAC;QACjD,OAAM;IACV,CAAC;IAED,iDAAiD;IACjD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAM;IACV,CAAC;IAED,sEAAsE;IACtE,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAA;IAC3D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAM;IACV,CAAC;IAED,sDAAsD;IACtD,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAA;IACjD,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAErE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAM;IACV,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,cAAc,IAAI,CAAC,CAAA;IAC/E,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,WAAW,IAAI,CAAC,CAAA;IAEzE,oBAAoB;IACpB,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,CAAC,CAAA;IAEpF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAM;IACV,CAAC;IAED,qDAAqD;IACrD,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;QAC9B,mDAAmD;QACnD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,OAAO,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC9B,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7B,CAAC;QACL,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;YACnC,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;YAC/B,aAAa,EAAE,KAAK,CAAC,WAAW;SACnC,CAAC,CAAA;IACN,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAA;QACtE,KAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,WAAW,CAAA;QAC3C,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,kBAAkB,GAAG;YAC3C,KAAK,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,kBAAkB,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM;YACtF,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,kBAAkB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,WAAW;SACpF,CAAA;QAED,kDAAkD;QAClD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC3B,aAAa,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC5B,CAAC;QAED,MAAM,CAAC,IAAI,CACP,aAAa,WAAW,CAAC,MAAM,2BAA2B,WAAW,CAAC,MAAM,UAAU,CACzF,CAAA;IACL,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;IAC9C,kBAAkB,CAAC,KAAK,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAA;AAC7D,CAAC,CAAA;AAED;;GAEG;AACH,SAAS,eAAe,CACpB,KAAmB,EACnB,OAAiB,EACjB,cAAsB,EACtB,WAAmB;IAEnB,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;IAEnC,2BAA2B;IAC3B,IAAI,YAAY,GAAa,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QACvB,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,SAAQ;QACZ,CAAC;QAED,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,SAAQ;QACZ,CAAC;QAED,sBAAsB;QACtB,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAA;QACjD,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;YACxB,SAAQ;QACZ,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;YAC9B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,WAAW,GAAG,EAAE,CAAA;YACpB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACzB,CAAC;aAAM,CAAC;YACJ,qBAAqB;YACrB,IAAI,YAAY,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACR,SAAS,EAAE,WAAY;oBACvB,KAAK,EAAE,CAAC,GAAG,YAAY,CAAC;oBACxB,WAAW,EAAE,CAAC,EAAE,2BAA2B;iBAC9C,CAAC,CAAA;YACN,CAAC;YACD,YAAY,GAAG,EAAE,CAAA;YACjB,WAAW,GAAG,IAAI,CAAA;QACtB,CAAC;IACL,CAAC;IAED,sCAAsC;IACtC,IAAI,YAAY,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC;YACR,SAAS,EAAE,WAAY;YACvB,KAAK,EAAE,CAAC,GAAG,YAAY,CAAC;YACxB,WAAW,EAAE,CAAC;SACjB,CAAC,CAAA;IACN,CAAC;IAED,OAAO,MAAM,CAAA;AACjB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { PluginConfig } from "../config";
|
|
2
|
+
import { Logger } from "../logger";
|
|
3
|
+
import type { SessionState, WithParts } from "../state";
|
|
4
|
+
/**
|
|
5
|
+
* Prunes base64 image data from tool outputs.
|
|
6
|
+
* Modifies message parts in-place.
|
|
7
|
+
*/
|
|
8
|
+
export declare const pruneBase64Images: (state: SessionState, logger: Logger, config: PluginConfig, messages: WithParts[]) => void;
|
|
9
|
+
/**
|
|
10
|
+
* Estimates potential savings from image pruning.
|
|
11
|
+
*/
|
|
12
|
+
export declare function estimateImagePruningSavings(messages: WithParts[], minBase64Length?: number): {
|
|
13
|
+
count: number;
|
|
14
|
+
estimatedSavings: number;
|
|
15
|
+
};
|
|
16
|
+
//# sourceMappingURL=image-pruning.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"image-pruning.d.ts","sourceRoot":"","sources":["../../../lib/strategies/image-pruning.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AAiCvD;;;GAGG;AACH,eAAO,MAAM,iBAAiB,GAC1B,OAAO,YAAY,EACnB,QAAQ,MAAM,EACd,QAAQ,YAAY,EACpB,UAAU,SAAS,EAAE,KACtB,IA0GF,CAAA;AAoHD;;GAEG;AACH,wBAAgB,2BAA2B,CACvC,QAAQ,EAAE,SAAS,EAAE,EACrB,eAAe,GAAE,MAAa,GAC/B;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,CA2C7C"}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { getPruneSets, recordStrategyTime } from "../state/utils";
|
|
2
|
+
import { isMessageCompacted } from "../shared-utils";
|
|
3
|
+
import { countTokens } from "./utils";
|
|
4
|
+
/**
|
|
5
|
+
* Image/Base64 Pruning Strategy
|
|
6
|
+
*
|
|
7
|
+
* Detects and prunes large base64-encoded image data from tool outputs.
|
|
8
|
+
* Images can consume massive amounts of context (10k-100k+ tokens).
|
|
9
|
+
* This strategy replaces them with a placeholder while preserving the
|
|
10
|
+
* context that an image was present.
|
|
11
|
+
*
|
|
12
|
+
* Features:
|
|
13
|
+
* - Detects base64 image data in tool outputs
|
|
14
|
+
* - Configurable size threshold for pruning
|
|
15
|
+
* - Preserves image metadata (filename, dimensions if available)
|
|
16
|
+
* - Replaces with compact placeholder
|
|
17
|
+
*/
|
|
18
|
+
// Regex to detect base64 image data
|
|
19
|
+
const BASE64_IMAGE_REGEX = /data:image\/[^;]+;base64,[A-Za-z0-9+/=]{100,}/g;
|
|
20
|
+
// Regex to detect standalone base64 strings that look like images
|
|
21
|
+
const STANDALONE_BASE64_REGEX = /[A-Za-z0-9+/=]{1000,}/g;
|
|
22
|
+
/**
|
|
23
|
+
* Prunes base64 image data from tool outputs.
|
|
24
|
+
* Modifies message parts in-place.
|
|
25
|
+
*/
|
|
26
|
+
export const pruneBase64Images = (state, logger, config, messages) => {
|
|
27
|
+
const startTime = performance.now();
|
|
28
|
+
if (!config.strategies.imagePruning?.enabled) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
// Early exit: if no messages, nothing to process
|
|
32
|
+
if (messages.length === 0) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
const minTurnsOld = config.strategies.imagePruning.minTurnsOld ?? 1;
|
|
36
|
+
const minBase64Length = config.strategies.imagePruning.minBase64Length ?? 1000;
|
|
37
|
+
const targetTools = config.strategies.imagePruning.targetTools ?? [
|
|
38
|
+
"read",
|
|
39
|
+
"webfetch",
|
|
40
|
+
"pdf-reader",
|
|
41
|
+
];
|
|
42
|
+
// Create a Set for O(1) lookup of target tools
|
|
43
|
+
const targetToolsSet = new Set(targetTools);
|
|
44
|
+
let totalTokensSaved = 0;
|
|
45
|
+
let prunedCount = 0;
|
|
46
|
+
// Get pruned tool IDs using O(1) Set lookup
|
|
47
|
+
const prunedToolIds = getPruneSets(state).toolIds;
|
|
48
|
+
for (const msg of messages) {
|
|
49
|
+
if (isMessageCompacted(state, msg)) {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
const parts = Array.isArray(msg.parts) ? msg.parts : [];
|
|
53
|
+
for (const part of parts) {
|
|
54
|
+
if (part.type !== "tool" || !part.callID) {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
// Skip if already pruned using O(1) Set lookup
|
|
58
|
+
if (prunedToolIds.has(part.callID)) {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
// Only process specific tools
|
|
62
|
+
if (!targetToolsSet.has(part.tool)) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
// Only process completed tools
|
|
66
|
+
if (part.state.status !== "completed") {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
// Check tool age
|
|
70
|
+
const toolMeta = state.toolParameters.get(part.callID);
|
|
71
|
+
if (toolMeta) {
|
|
72
|
+
const toolAge = state.currentTurn - toolMeta.turn;
|
|
73
|
+
if (toolAge < minTurnsOld) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
// Get output
|
|
78
|
+
const output = part.state.output;
|
|
79
|
+
if (!output || typeof output !== "string") {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
// Skip if already processed
|
|
83
|
+
if (output.includes("[Image content pruned")) {
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
// Find and prune base64 images
|
|
87
|
+
const result = pruneBase64FromString(output, minBase64Length);
|
|
88
|
+
if (result.changed) {
|
|
89
|
+
const tokensBefore = countTokens(output);
|
|
90
|
+
const tokensAfter = countTokens(result.content);
|
|
91
|
+
const tokensSaved = tokensBefore - tokensAfter;
|
|
92
|
+
part.state.output = result.content;
|
|
93
|
+
totalTokensSaved += tokensSaved;
|
|
94
|
+
prunedCount += result.prunedImages;
|
|
95
|
+
logger.debug(`Pruned base64 images from ${part.tool} output`, {
|
|
96
|
+
callId: part.callID,
|
|
97
|
+
imagesPruned: result.prunedImages,
|
|
98
|
+
tokensSaved,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (prunedCount > 0) {
|
|
104
|
+
state.stats.strategyStats.imagePruning = {
|
|
105
|
+
count: (state.stats.strategyStats.imagePruning?.count || 0) + prunedCount,
|
|
106
|
+
tokens: (state.stats.strategyStats.imagePruning?.tokens || 0) + totalTokensSaved,
|
|
107
|
+
};
|
|
108
|
+
logger.info(`Pruned ${prunedCount} image(s), saved ~${totalTokensSaved} tokens`);
|
|
109
|
+
}
|
|
110
|
+
// Record performance metrics
|
|
111
|
+
const duration = performance.now() - startTime;
|
|
112
|
+
recordStrategyTime(state, "imagePruning", duration);
|
|
113
|
+
};
|
|
114
|
+
/**
|
|
115
|
+
* Prunes base64 image data from a string.
|
|
116
|
+
* Returns the modified content and metadata about changes.
|
|
117
|
+
*/
|
|
118
|
+
function pruneBase64FromString(content, minLength) {
|
|
119
|
+
let changed = false;
|
|
120
|
+
let prunedImages = 0;
|
|
121
|
+
let result = content;
|
|
122
|
+
// Find data URI base64 images
|
|
123
|
+
const dataUriMatches = findAllMatches(result, BASE64_IMAGE_REGEX);
|
|
124
|
+
for (const match of dataUriMatches.reverse()) {
|
|
125
|
+
// Reverse to process from end (preserves indices)
|
|
126
|
+
if (match.estimatedTokens > 50) {
|
|
127
|
+
// Only prune if substantial
|
|
128
|
+
const placeholder = `[Image content pruned: ~${match.estimatedTokens} tokens saved]`;
|
|
129
|
+
result =
|
|
130
|
+
result.substring(0, match.startIndex) +
|
|
131
|
+
placeholder +
|
|
132
|
+
result.substring(match.endIndex);
|
|
133
|
+
prunedImages++;
|
|
134
|
+
changed = true;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
// Find standalone base64 strings (potential images without data URI prefix)
|
|
138
|
+
if (prunedImages === 0) {
|
|
139
|
+
const standaloneMatches = findAllMatches(result, STANDALONE_BASE64_REGEX);
|
|
140
|
+
for (const match of standaloneMatches.reverse()) {
|
|
141
|
+
if (match.fullMatch.length >= minLength) {
|
|
142
|
+
// Check if it looks like base64 image data (high entropy, no spaces)
|
|
143
|
+
if (isLikelyBase64Image(match.fullMatch)) {
|
|
144
|
+
const placeholder = `[Base64 content pruned: ~${match.estimatedTokens} tokens saved]`;
|
|
145
|
+
result =
|
|
146
|
+
result.substring(0, match.startIndex) +
|
|
147
|
+
placeholder +
|
|
148
|
+
result.substring(match.endIndex);
|
|
149
|
+
prunedImages++;
|
|
150
|
+
changed = true;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return { content: result, changed, prunedImages };
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Finds all regex matches in a string with their positions.
|
|
159
|
+
*/
|
|
160
|
+
function findAllMatches(content, regex) {
|
|
161
|
+
const matches = [];
|
|
162
|
+
let match;
|
|
163
|
+
// Reset regex lastIndex
|
|
164
|
+
regex.lastIndex = 0;
|
|
165
|
+
while ((match = regex.exec(content)) !== null) {
|
|
166
|
+
const fullMatch = match[0];
|
|
167
|
+
const estimatedTokens = Math.ceil(fullMatch.length / 4); // Rough estimate: 4 chars ≈ 1 token
|
|
168
|
+
matches.push({
|
|
169
|
+
fullMatch,
|
|
170
|
+
startIndex: match.index,
|
|
171
|
+
endIndex: match.index + fullMatch.length,
|
|
172
|
+
estimatedTokens,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
return matches;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Checks if a base64 string is likely to be image data.
|
|
179
|
+
* Uses heuristics to avoid pruning non-image base64.
|
|
180
|
+
*/
|
|
181
|
+
function isLikelyBase64Image(base64String) {
|
|
182
|
+
// Check length (images are typically large)
|
|
183
|
+
if (base64String.length < 1000) {
|
|
184
|
+
return false;
|
|
185
|
+
}
|
|
186
|
+
// Check for high entropy (images have random-looking data)
|
|
187
|
+
const uniqueChars = new Set(base64String).size;
|
|
188
|
+
const entropy = uniqueChars / base64String.length;
|
|
189
|
+
if (entropy < 0.5) {
|
|
190
|
+
// Too low entropy for image data
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
// Check for common image signatures in base64
|
|
194
|
+
const imageSignatures = [
|
|
195
|
+
"/9j/", // JPEG
|
|
196
|
+
"iVBORw0KGgo", // PNG
|
|
197
|
+
"R0lGOD", // GIF
|
|
198
|
+
"SUkqAA", // TIFF
|
|
199
|
+
"Qk0", // BMP
|
|
200
|
+
];
|
|
201
|
+
for (const sig of imageSignatures) {
|
|
202
|
+
if (base64String.startsWith(sig) || base64String.includes(sig)) {
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
// If very long and high entropy, likely an image
|
|
207
|
+
return base64String.length > 5000 && entropy > 0.6;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Estimates potential savings from image pruning.
|
|
211
|
+
*/
|
|
212
|
+
export function estimateImagePruningSavings(messages, minBase64Length = 1000) {
|
|
213
|
+
let count = 0;
|
|
214
|
+
let estimatedSavings = 0;
|
|
215
|
+
for (const msg of messages) {
|
|
216
|
+
const parts = Array.isArray(msg.parts) ? msg.parts : [];
|
|
217
|
+
for (const part of parts) {
|
|
218
|
+
if (part.type !== "tool") {
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const output = part.state?.output;
|
|
222
|
+
if (!output || typeof output !== "string") {
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
// Count data URI images
|
|
226
|
+
const dataUriMatches = output.match(BASE64_IMAGE_REGEX);
|
|
227
|
+
if (dataUriMatches) {
|
|
228
|
+
for (const match of dataUriMatches) {
|
|
229
|
+
const tokens = Math.ceil(match.length / 4);
|
|
230
|
+
if (tokens > 50) {
|
|
231
|
+
count++;
|
|
232
|
+
estimatedSavings += tokens;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// Count standalone base64
|
|
237
|
+
const standaloneMatches = output.match(STANDALONE_BASE64_REGEX);
|
|
238
|
+
if (standaloneMatches) {
|
|
239
|
+
for (const match of standaloneMatches) {
|
|
240
|
+
if (match.length >= minBase64Length && isLikelyBase64Image(match)) {
|
|
241
|
+
const tokens = Math.ceil(match.length / 4);
|
|
242
|
+
count++;
|
|
243
|
+
estimatedSavings += tokens;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return { count, estimatedSavings };
|
|
250
|
+
}
|
|
251
|
+
//# sourceMappingURL=image-pruning.js.map
|