@compilr-dev/agents 0.3.12 → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +19 -1
- package/dist/agent.js +75 -10
- package/dist/context/file-tracker.d.ts +59 -1
- package/dist/context/file-tracker.js +96 -1
- package/dist/context/file-tracking-hook.js +9 -4
- package/dist/context/index.d.ts +1 -1
- package/dist/context/tool-result-delegator.js +12 -3
- package/dist/index.d.ts +1 -1
- package/dist/mcp/client.js +6 -2
- package/dist/providers/claude.js +3 -1
- package/dist/providers/gemini-native.js +7 -0
- package/dist/providers/openai-compatible.js +11 -0
- package/dist/providers/types.d.ts +5 -0
- package/dist/tools/builtin/task.js +1 -0
- package/dist/tools/builtin/todo.d.ts +8 -0
- package/dist/tools/builtin/todo.js +18 -2
- package/dist/utils/tokenizer.d.ts +2 -1
- package/dist/utils/tokenizer.js +38 -1
- package/package.json +5 -3
package/dist/agent.d.ts
CHANGED
|
@@ -576,6 +576,19 @@ export interface AgentConfig {
|
|
|
576
576
|
* ```
|
|
577
577
|
*/
|
|
578
578
|
enableFileTracking?: boolean;
|
|
579
|
+
/**
|
|
580
|
+
* Options for file context restoration after compaction.
|
|
581
|
+
*
|
|
582
|
+
* Controls how file contents are re-injected into the context after
|
|
583
|
+
* compaction. Small files get their content inlined; large files get
|
|
584
|
+
* reference-only hints.
|
|
585
|
+
*
|
|
586
|
+
* Only applies when `enableFileTracking` is true.
|
|
587
|
+
*/
|
|
588
|
+
fileRestoration?: {
|
|
589
|
+
/** Max total tokens for inline file content after compaction (default: 4000) */
|
|
590
|
+
maxInlineTokens?: number;
|
|
591
|
+
};
|
|
579
592
|
}
|
|
580
593
|
/**
|
|
581
594
|
* Options for a single run
|
|
@@ -861,6 +874,10 @@ export declare class Agent {
|
|
|
861
874
|
* File access tracker for context restoration hints
|
|
862
875
|
*/
|
|
863
876
|
private readonly fileTracker?;
|
|
877
|
+
/**
|
|
878
|
+
* File restoration options for post-compaction content injection
|
|
879
|
+
*/
|
|
880
|
+
private readonly fileRestorationConfig?;
|
|
864
881
|
constructor(config: AgentConfig);
|
|
865
882
|
/**
|
|
866
883
|
* Create an agent with project memory loaded from files.
|
|
@@ -1278,7 +1295,8 @@ export declare class Agent {
|
|
|
1278
1295
|
formatRestorationHints(): string;
|
|
1279
1296
|
/**
|
|
1280
1297
|
* Inject context restoration hints into messages after compaction/summarization.
|
|
1281
|
-
*
|
|
1298
|
+
* Uses content-aware format: small files get content inlined, large files get
|
|
1299
|
+
* reference-only notes. Each file is injected as a separate user message.
|
|
1282
1300
|
*
|
|
1283
1301
|
* @internal
|
|
1284
1302
|
*/
|
package/dist/agent.js
CHANGED
|
@@ -93,6 +93,10 @@ export class Agent {
|
|
|
93
93
|
* File access tracker for context restoration hints
|
|
94
94
|
*/
|
|
95
95
|
fileTracker;
|
|
96
|
+
/**
|
|
97
|
+
* File restoration options for post-compaction content injection
|
|
98
|
+
*/
|
|
99
|
+
fileRestorationConfig;
|
|
96
100
|
constructor(config) {
|
|
97
101
|
this.provider = config.provider;
|
|
98
102
|
this.systemPrompt = config.systemPrompt ?? '';
|
|
@@ -215,6 +219,7 @@ export class Agent {
|
|
|
215
219
|
// File tracking for context restoration hints
|
|
216
220
|
if (config.enableFileTracking && config.contextManager) {
|
|
217
221
|
this.fileTracker = new FileAccessTracker();
|
|
222
|
+
this.fileRestorationConfig = config.fileRestoration;
|
|
218
223
|
const trackingHook = createFileTrackingHook(this.fileTracker);
|
|
219
224
|
hooksConfig.afterTool = hooksConfig.afterTool ?? [];
|
|
220
225
|
hooksConfig.afterTool.push(trackingHook);
|
|
@@ -825,7 +830,8 @@ export class Agent {
|
|
|
825
830
|
}
|
|
826
831
|
/**
|
|
827
832
|
* Inject context restoration hints into messages after compaction/summarization.
|
|
828
|
-
*
|
|
833
|
+
* Uses content-aware format: small files get content inlined, large files get
|
|
834
|
+
* reference-only notes. Each file is injected as a separate user message.
|
|
829
835
|
*
|
|
830
836
|
* @internal
|
|
831
837
|
*/
|
|
@@ -833,17 +839,22 @@ export class Agent {
|
|
|
833
839
|
if (!this.fileTracker || this.fileTracker.size === 0) {
|
|
834
840
|
return;
|
|
835
841
|
}
|
|
836
|
-
const hints = this.
|
|
837
|
-
|
|
842
|
+
const hints = this.fileTracker.formatRestorationHintsWithContent({
|
|
843
|
+
maxInlineTokens: this.fileRestorationConfig?.maxInlineTokens ?? 4000,
|
|
844
|
+
});
|
|
845
|
+
if (hints.length === 0) {
|
|
838
846
|
return;
|
|
839
847
|
}
|
|
840
|
-
//
|
|
848
|
+
// Insert after system prompt, each as a separate user message
|
|
841
849
|
const systemIndex = messages.findIndex((m) => m.role === 'system');
|
|
842
850
|
const insertIndex = systemIndex >= 0 ? systemIndex + 1 : 0;
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
851
|
+
// Insert in reverse order so they end up in the correct order
|
|
852
|
+
for (let i = hints.length - 1; i >= 0; i--) {
|
|
853
|
+
messages.splice(insertIndex, 0, {
|
|
854
|
+
role: 'user',
|
|
855
|
+
content: hints[i].text,
|
|
856
|
+
});
|
|
857
|
+
}
|
|
847
858
|
}
|
|
848
859
|
// ==========================================================================
|
|
849
860
|
// Context Compaction
|
|
@@ -1532,6 +1543,7 @@ export class Agent {
|
|
|
1532
1543
|
this.contextManager.updateCategoryUsage('system', systemTokens);
|
|
1533
1544
|
}
|
|
1534
1545
|
// Check if we need to manage context before starting
|
|
1546
|
+
// Order: emergency (95%) → compaction (50%/20 turns) → warning (90%)
|
|
1535
1547
|
if (this.contextManager.needsEmergencySummarization()) {
|
|
1536
1548
|
emit({
|
|
1537
1549
|
type: 'context_warning',
|
|
@@ -1550,6 +1562,27 @@ export class Agent {
|
|
|
1550
1562
|
rounds: result.rounds,
|
|
1551
1563
|
});
|
|
1552
1564
|
}
|
|
1565
|
+
else if (this.contextManager.needsCompaction()) {
|
|
1566
|
+
// Proactive compaction (50% utilization or 20+ turns since last compaction)
|
|
1567
|
+
// compact() reads this.conversationHistory which has all previous runs' messages
|
|
1568
|
+
const compactResult = await this.compact({ useSmartCompaction: true });
|
|
1569
|
+
if (compactResult.success) {
|
|
1570
|
+
// Rebuild messages: compacted history + current userMsg (still in newMessages)
|
|
1571
|
+
messages = this.systemPrompt
|
|
1572
|
+
? [
|
|
1573
|
+
{ role: 'system', content: this.systemPrompt },
|
|
1574
|
+
...this.conversationHistory,
|
|
1575
|
+
...newMessages,
|
|
1576
|
+
]
|
|
1577
|
+
: [...this.conversationHistory, ...newMessages];
|
|
1578
|
+
// Don't touch newMessages — it still has [userMsg] which finally needs to append
|
|
1579
|
+
emit({
|
|
1580
|
+
type: 'context_compacted',
|
|
1581
|
+
tokensBefore: compactResult.originalTokens,
|
|
1582
|
+
tokensAfter: compactResult.summaryTokens,
|
|
1583
|
+
});
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1553
1586
|
else if (this.contextManager.needsSummarization()) {
|
|
1554
1587
|
emit({
|
|
1555
1588
|
type: 'context_warning',
|
|
@@ -2077,6 +2110,36 @@ export class Agent {
|
|
|
2077
2110
|
completedWithText: false,
|
|
2078
2111
|
});
|
|
2079
2112
|
}
|
|
2113
|
+
// Auto-compaction: check if context needs compaction after this iteration
|
|
2114
|
+
if (this.contextManager && this.autoContextManagement) {
|
|
2115
|
+
await this.contextManager.updateTokenCount(messages);
|
|
2116
|
+
if (this.contextManager.needsCompaction()) {
|
|
2117
|
+
// Flush current run's messages to conversationHistory so compact() can see them.
|
|
2118
|
+
// compact() reads this.conversationHistory, which normally only updates in finally.
|
|
2119
|
+
this.conversationHistory.push(...newMessages);
|
|
2120
|
+
newMessages.length = 0;
|
|
2121
|
+
const compactResult = await this.compact({ useSmartCompaction: true });
|
|
2122
|
+
if (compactResult.success) {
|
|
2123
|
+
// compact() already replaced this.conversationHistory with compacted version.
|
|
2124
|
+
// Rebuild local messages array from it.
|
|
2125
|
+
messages = this.systemPrompt
|
|
2126
|
+
? [
|
|
2127
|
+
{ role: 'system', content: this.systemPrompt },
|
|
2128
|
+
...this.conversationHistory,
|
|
2129
|
+
]
|
|
2130
|
+
: [...this.conversationHistory];
|
|
2131
|
+
// newMessages stays empty — subsequent iterations will push new messages into it,
|
|
2132
|
+
// and finally block will correctly append only post-compaction messages.
|
|
2133
|
+
emit({
|
|
2134
|
+
type: 'context_compacted',
|
|
2135
|
+
tokensBefore: compactResult.originalTokens,
|
|
2136
|
+
tokensAfter: compactResult.summaryTokens,
|
|
2137
|
+
});
|
|
2138
|
+
}
|
|
2139
|
+
// If compact() failed, messages are already flushed to conversationHistory.
|
|
2140
|
+
// newMessages is empty so finally won't double-push.
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2080
2143
|
emit({ type: 'iteration_end', iteration: iterations });
|
|
2081
2144
|
// Check if we're about to hit the iteration limit
|
|
2082
2145
|
// If callback is defined, ask if we should continue
|
|
@@ -2302,13 +2365,15 @@ export class Agent {
|
|
|
2302
2365
|
* @param signal - Optional abort signal
|
|
2303
2366
|
*/
|
|
2304
2367
|
chatWithRetry(messages, options, emit, signal) {
|
|
2368
|
+
// Merge signal into chat options so providers can abort the request
|
|
2369
|
+
const chatOptions = signal ? { ...options, signal } : options;
|
|
2305
2370
|
// If retry is disabled, return the raw provider stream
|
|
2306
2371
|
if (!this.retryConfig.enabled) {
|
|
2307
|
-
return this.provider.chat(messages,
|
|
2372
|
+
return this.provider.chat(messages, chatOptions);
|
|
2308
2373
|
}
|
|
2309
2374
|
const providerName = this.provider.name;
|
|
2310
2375
|
const { maxAttempts, baseDelayMs, maxDelayMs } = this.retryConfig;
|
|
2311
|
-
return withRetryGenerator(() => this.provider.chat(messages,
|
|
2376
|
+
return withRetryGenerator(() => this.provider.chat(messages, chatOptions), {
|
|
2312
2377
|
maxAttempts,
|
|
2313
2378
|
baseDelayMs,
|
|
2314
2379
|
maxDelayMs,
|
|
@@ -32,6 +32,10 @@ export interface FileAccess {
|
|
|
32
32
|
lineCount?: number;
|
|
33
33
|
/** Optional summary of what was found/changed */
|
|
34
34
|
summary?: string;
|
|
35
|
+
/** Stored file content (only for small reads, used for post-compaction restoration) */
|
|
36
|
+
content?: string;
|
|
37
|
+
/** Token count of stored content */
|
|
38
|
+
tokenCount?: number;
|
|
35
39
|
}
|
|
36
40
|
/**
|
|
37
41
|
* Options for FileAccessTracker constructor
|
|
@@ -47,6 +51,18 @@ export interface FileAccessTrackerOptions {
|
|
|
47
51
|
* @default true
|
|
48
52
|
*/
|
|
49
53
|
deduplicateReferences?: boolean;
|
|
54
|
+
/**
|
|
55
|
+
* Maximum line count for a file to have its content stored inline.
|
|
56
|
+
* Files with more lines than this threshold will only store path/lineCount.
|
|
57
|
+
* @default 200
|
|
58
|
+
*/
|
|
59
|
+
inlineThreshold?: number;
|
|
60
|
+
/**
|
|
61
|
+
* Maximum number of files that can have stored content at once.
|
|
62
|
+
* When exceeded, oldest content entries are evicted (path still tracked).
|
|
63
|
+
* @default 10
|
|
64
|
+
*/
|
|
65
|
+
maxContentEntries?: number;
|
|
50
66
|
}
|
|
51
67
|
/**
|
|
52
68
|
* Options for formatting restoration hints
|
|
@@ -65,6 +81,19 @@ export interface FormatHintsOptions {
|
|
|
65
81
|
/** Verbosity level (adjusts format automatically) */
|
|
66
82
|
verbosityLevel?: VerbosityLevel;
|
|
67
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* A single restoration hint message for post-compaction context injection
|
|
86
|
+
*/
|
|
87
|
+
export interface RestorationHintMessage {
|
|
88
|
+
/** File path */
|
|
89
|
+
path: string;
|
|
90
|
+
/** Whether content is inlined or just referenced */
|
|
91
|
+
type: 'inline' | 'reference';
|
|
92
|
+
/** The formatted hint text */
|
|
93
|
+
text: string;
|
|
94
|
+
/** Estimated token count of this hint */
|
|
95
|
+
tokens: number;
|
|
96
|
+
}
|
|
68
97
|
/**
|
|
69
98
|
* Statistics about file accesses
|
|
70
99
|
*/
|
|
@@ -95,11 +124,13 @@ export declare class FileAccessTracker {
|
|
|
95
124
|
private readonly accesses;
|
|
96
125
|
private readonly maxEntries;
|
|
97
126
|
private readonly deduplicateReferences;
|
|
127
|
+
private readonly inlineThreshold;
|
|
128
|
+
private readonly maxContentEntries;
|
|
98
129
|
constructor(options?: FileAccessTrackerOptions);
|
|
99
130
|
/**
|
|
100
131
|
* Track a file that was fully read
|
|
101
132
|
*/
|
|
102
|
-
trackRead(filePath: string, lineCount: number, summary?: string): void;
|
|
133
|
+
trackRead(filePath: string, lineCount: number, summary?: string, content?: string): void;
|
|
103
134
|
/**
|
|
104
135
|
* Track a file that was referenced (e.g., appeared in grep/glob results)
|
|
105
136
|
*/
|
|
@@ -135,6 +166,23 @@ export declare class FileAccessTracker {
|
|
|
135
166
|
* Format restoration hints for injection after compaction
|
|
136
167
|
*/
|
|
137
168
|
formatRestorationHints(options?: FormatHintsOptions): string;
|
|
169
|
+
/**
|
|
170
|
+
* Format restoration hints with inline file content (Claude Code style).
|
|
171
|
+
*
|
|
172
|
+
* Small files with stored content are inlined up to a token budget.
|
|
173
|
+
* Large files or files exceeding the budget get reference-only hints.
|
|
174
|
+
* Each file produces a separate hint message for individual injection.
|
|
175
|
+
*
|
|
176
|
+
* Priority order for inlining:
|
|
177
|
+
* 1. Modified files (most critical context)
|
|
178
|
+
* 2. Read files, most recent first
|
|
179
|
+
* 3. Once budget exhausted → remaining become reference-only
|
|
180
|
+
* 4. Referenced-only files → always reference-only
|
|
181
|
+
*/
|
|
182
|
+
formatRestorationHintsWithContent(options?: {
|
|
183
|
+
/** Total token budget for all inline content (default: 4000) */
|
|
184
|
+
maxInlineTokens?: number;
|
|
185
|
+
}): RestorationHintMessage[];
|
|
138
186
|
/**
|
|
139
187
|
* Clear all tracked accesses
|
|
140
188
|
*/
|
|
@@ -143,6 +191,16 @@ export declare class FileAccessTracker {
|
|
|
143
191
|
* Get the number of tracked files
|
|
144
192
|
*/
|
|
145
193
|
get size(): number;
|
|
194
|
+
/**
|
|
195
|
+
* Rough token estimate (4 chars ≈ 1 token). Avoids heavy tiktoken dependency
|
|
196
|
+
* in the tracker — exact counts aren't critical for budget enforcement.
|
|
197
|
+
*/
|
|
198
|
+
private estimateTokens;
|
|
199
|
+
/**
|
|
200
|
+
* Evict oldest stored content when maxContentEntries is exceeded.
|
|
201
|
+
* Only drops the `content`/`tokenCount` fields — the access entry itself is preserved.
|
|
202
|
+
*/
|
|
203
|
+
private enforceMaxContentEntries;
|
|
146
204
|
private normalizePath;
|
|
147
205
|
private enforceMaxEntries;
|
|
148
206
|
private getEffectiveMaxFiles;
|
|
@@ -38,9 +38,13 @@ export class FileAccessTracker {
|
|
|
38
38
|
accesses = new Map();
|
|
39
39
|
maxEntries;
|
|
40
40
|
deduplicateReferences;
|
|
41
|
+
inlineThreshold;
|
|
42
|
+
maxContentEntries;
|
|
41
43
|
constructor(options = {}) {
|
|
42
44
|
this.maxEntries = options.maxEntries ?? 100;
|
|
43
45
|
this.deduplicateReferences = options.deduplicateReferences ?? true;
|
|
46
|
+
this.inlineThreshold = options.inlineThreshold ?? 200;
|
|
47
|
+
this.maxContentEntries = options.maxContentEntries ?? 10;
|
|
44
48
|
}
|
|
45
49
|
// ==========================================================================
|
|
46
50
|
// Track Methods
|
|
@@ -48,13 +52,15 @@ export class FileAccessTracker {
|
|
|
48
52
|
/**
|
|
49
53
|
* Track a file that was fully read
|
|
50
54
|
*/
|
|
51
|
-
trackRead(filePath, lineCount, summary) {
|
|
55
|
+
trackRead(filePath, lineCount, summary, content) {
|
|
52
56
|
const normalizedPath = this.normalizePath(filePath);
|
|
53
57
|
// Read supersedes reference - remove any existing reference
|
|
54
58
|
const existing = this.accesses.get(normalizedPath);
|
|
55
59
|
if (existing && existing.type === 'referenced') {
|
|
56
60
|
this.accesses.delete(normalizedPath);
|
|
57
61
|
}
|
|
62
|
+
// Store content only for small files
|
|
63
|
+
const shouldStoreContent = content !== undefined && lineCount <= this.inlineThreshold;
|
|
58
64
|
// Update or create read entry
|
|
59
65
|
this.accesses.set(normalizedPath, {
|
|
60
66
|
path: normalizedPath,
|
|
@@ -62,7 +68,12 @@ export class FileAccessTracker {
|
|
|
62
68
|
timestamp: Date.now(),
|
|
63
69
|
lineCount,
|
|
64
70
|
summary,
|
|
71
|
+
content: shouldStoreContent ? content : undefined,
|
|
72
|
+
tokenCount: shouldStoreContent ? this.estimateTokens(content) : undefined,
|
|
65
73
|
});
|
|
74
|
+
if (shouldStoreContent) {
|
|
75
|
+
this.enforceMaxContentEntries();
|
|
76
|
+
}
|
|
66
77
|
this.enforceMaxEntries();
|
|
67
78
|
}
|
|
68
79
|
/**
|
|
@@ -184,6 +195,65 @@ export class FileAccessTracker {
|
|
|
184
195
|
includeTimestamp,
|
|
185
196
|
});
|
|
186
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* Format restoration hints with inline file content (Claude Code style).
|
|
200
|
+
*
|
|
201
|
+
* Small files with stored content are inlined up to a token budget.
|
|
202
|
+
* Large files or files exceeding the budget get reference-only hints.
|
|
203
|
+
* Each file produces a separate hint message for individual injection.
|
|
204
|
+
*
|
|
205
|
+
* Priority order for inlining:
|
|
206
|
+
* 1. Modified files (most critical context)
|
|
207
|
+
* 2. Read files, most recent first
|
|
208
|
+
* 3. Once budget exhausted → remaining become reference-only
|
|
209
|
+
* 4. Referenced-only files → always reference-only
|
|
210
|
+
*/
|
|
211
|
+
formatRestorationHintsWithContent(options) {
|
|
212
|
+
if (this.accesses.size === 0) {
|
|
213
|
+
return [];
|
|
214
|
+
}
|
|
215
|
+
const maxInlineTokens = options?.maxInlineTokens ?? 4000;
|
|
216
|
+
const allAccesses = this.getAccesses();
|
|
217
|
+
const hints = [];
|
|
218
|
+
let usedTokens = 0;
|
|
219
|
+
// Sort by priority: modified first, then read (most recent first), then referenced
|
|
220
|
+
const prioritized = [...allAccesses].sort((a, b) => {
|
|
221
|
+
const typePriority = { modified: 0, read: 1, referenced: 2 };
|
|
222
|
+
const aPriority = typePriority[a.type];
|
|
223
|
+
const bPriority = typePriority[b.type];
|
|
224
|
+
if (aPriority !== bPriority)
|
|
225
|
+
return aPriority - bPriority;
|
|
226
|
+
return b.timestamp - a.timestamp; // most recent first within same type
|
|
227
|
+
});
|
|
228
|
+
for (const access of prioritized) {
|
|
229
|
+
// Referenced files are always reference-only (no content was read)
|
|
230
|
+
if (access.type === 'referenced') {
|
|
231
|
+
continue; // Skip referenced files — they add noise without value
|
|
232
|
+
}
|
|
233
|
+
const canInline = access.content !== undefined &&
|
|
234
|
+
access.tokenCount !== undefined &&
|
|
235
|
+
usedTokens + access.tokenCount <= maxInlineTokens;
|
|
236
|
+
if (canInline) {
|
|
237
|
+
// Inline: simulate a read_file tool call result
|
|
238
|
+
// canInline guarantees content and tokenCount are defined
|
|
239
|
+
const fileContent = access.content;
|
|
240
|
+
const tokens = access.tokenCount;
|
|
241
|
+
const text = `Note: ${access.path} was read before the last conversation was compacted.\n` +
|
|
242
|
+
`<file_content>\n${fileContent}\n</file_content>`;
|
|
243
|
+
usedTokens += tokens;
|
|
244
|
+
hints.push({ path: access.path, type: 'inline', text, tokens });
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
// Reference-only
|
|
248
|
+
const lineInfo = access.lineCount !== undefined ? ` (${String(access.lineCount)} lines)` : '';
|
|
249
|
+
const text = `Note: ${access.path}${lineInfo} was read before the last conversation was compacted, ` +
|
|
250
|
+
`but the contents are too large to include. Use read_file if you need to access it.`;
|
|
251
|
+
const tokens = this.estimateTokens(text);
|
|
252
|
+
hints.push({ path: access.path, type: 'reference', text, tokens });
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return hints;
|
|
256
|
+
}
|
|
187
257
|
// ==========================================================================
|
|
188
258
|
// Lifecycle Methods
|
|
189
259
|
// ==========================================================================
|
|
@@ -202,6 +272,31 @@ export class FileAccessTracker {
|
|
|
202
272
|
// ==========================================================================
|
|
203
273
|
// Private Helpers
|
|
204
274
|
// ==========================================================================
|
|
275
|
+
/**
|
|
276
|
+
* Rough token estimate (4 chars ≈ 1 token). Avoids heavy tiktoken dependency
|
|
277
|
+
* in the tracker — exact counts aren't critical for budget enforcement.
|
|
278
|
+
*/
|
|
279
|
+
estimateTokens(text) {
|
|
280
|
+
if (!text)
|
|
281
|
+
return 0;
|
|
282
|
+
return Math.ceil(text.length / 4);
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Evict oldest stored content when maxContentEntries is exceeded.
|
|
286
|
+
* Only drops the `content`/`tokenCount` fields — the access entry itself is preserved.
|
|
287
|
+
*/
|
|
288
|
+
enforceMaxContentEntries() {
|
|
289
|
+
const entriesWithContent = Array.from(this.accesses.values())
|
|
290
|
+
.filter((a) => a.content !== undefined)
|
|
291
|
+
.sort((a, b) => a.timestamp - b.timestamp); // oldest first
|
|
292
|
+
while (entriesWithContent.length > this.maxContentEntries) {
|
|
293
|
+
const oldest = entriesWithContent.shift();
|
|
294
|
+
if (oldest) {
|
|
295
|
+
oldest.content = undefined;
|
|
296
|
+
oldest.tokenCount = undefined;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
205
300
|
normalizePath(filePath) {
|
|
206
301
|
// Resolve to absolute path
|
|
207
302
|
return path.resolve(filePath);
|
|
@@ -32,18 +32,23 @@ export function createFileTrackingHook(tracker) {
|
|
|
32
32
|
case TOOL_NAMES.READ_FILE: {
|
|
33
33
|
const readInput = input;
|
|
34
34
|
const readResult = result;
|
|
35
|
-
const
|
|
36
|
-
|
|
35
|
+
const content = readResult.result?.content;
|
|
36
|
+
// totalLines is set when maxLines/startLine used or content truncated;
|
|
37
|
+
// linesReturned is the actual slice size; fall back to counting content lines
|
|
38
|
+
const lineCount = readResult.result?.totalLines ??
|
|
39
|
+
readResult.result?.linesReturned ??
|
|
40
|
+
(content ? content.split('\n').length : 0);
|
|
41
|
+
tracker.trackRead(readInput.path, lineCount, undefined, content);
|
|
37
42
|
break;
|
|
38
43
|
}
|
|
39
44
|
case TOOL_NAMES.WRITE_FILE: {
|
|
40
45
|
const writeInput = input;
|
|
41
|
-
tracker.trackModification(writeInput.
|
|
46
|
+
tracker.trackModification(writeInput.path, 'File written');
|
|
42
47
|
break;
|
|
43
48
|
}
|
|
44
49
|
case TOOL_NAMES.EDIT: {
|
|
45
50
|
const editInput = input;
|
|
46
|
-
tracker.trackModification(editInput.
|
|
51
|
+
tracker.trackModification(editInput.filePath, 'File edited');
|
|
47
52
|
break;
|
|
48
53
|
}
|
|
49
54
|
case TOOL_NAMES.GREP: {
|
package/dist/context/index.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
export { ContextManager, DEFAULT_CONTEXT_CONFIG } from './manager.js';
|
|
12
12
|
export type { ContextManagerOptions } from './manager.js';
|
|
13
13
|
export { FileAccessTracker } from './file-tracker.js';
|
|
14
|
-
export type { FileAccessType, FileAccess, FileAccessTrackerOptions, FormatHintsOptions, FileAccessStats, } from './file-tracker.js';
|
|
14
|
+
export type { FileAccessType, FileAccess, FileAccessTrackerOptions, FormatHintsOptions, FileAccessStats, RestorationHintMessage, } from './file-tracker.js';
|
|
15
15
|
export { createFileTrackingHook, TRACKED_TOOLS } from './file-tracking-hook.js';
|
|
16
16
|
export type { ContextCategory, BudgetAllocation, CategoryBudgetInfo, PreflightResult, VerbosityLevel, VerbosityConfig, ContextConfig, FilteringConfig, CompactionConfig, SummarizationConfig, CompactionResult, SummarizationResult, FilteringResult, ContextEvent, ContextEventHandler, ContextStats, CategorizedMessages, SmartCompactOptions, SmartCompactionResult, } from './types.js';
|
|
17
17
|
export { DelegatedResultStore } from './delegated-result-store.js';
|
|
@@ -70,6 +70,10 @@ export class ToolResultDelegator {
|
|
|
70
70
|
* Perform the actual delegation (async path).
|
|
71
71
|
*/
|
|
72
72
|
async delegateResult(context, content, tokens, toolConfig) {
|
|
73
|
+
// If already aborted, skip delegation entirely — return original result
|
|
74
|
+
if (context.signal?.aborted) {
|
|
75
|
+
return { result: context.result };
|
|
76
|
+
}
|
|
73
77
|
// Generate delegation ID
|
|
74
78
|
const id = this.store.generateId();
|
|
75
79
|
this.onEvent?.({
|
|
@@ -84,7 +88,7 @@ export class ToolResultDelegator {
|
|
|
84
88
|
let summary;
|
|
85
89
|
let usedStrategy = strategy;
|
|
86
90
|
if (strategy === 'llm') {
|
|
87
|
-
const llmSummary = await this.summarizeLLM(content, context.toolName);
|
|
91
|
+
const llmSummary = await this.summarizeLLM(content, context.toolName, context.signal);
|
|
88
92
|
if (llmSummary !== null) {
|
|
89
93
|
summary = llmSummary;
|
|
90
94
|
usedStrategy = 'llm';
|
|
@@ -97,7 +101,7 @@ export class ToolResultDelegator {
|
|
|
97
101
|
}
|
|
98
102
|
else if (toolConfig.strategy === 'auto') {
|
|
99
103
|
// Auto: try LLM first, fall back to extractive
|
|
100
|
-
const llmSummary = await this.summarizeLLM(content, context.toolName);
|
|
104
|
+
const llmSummary = await this.summarizeLLM(content, context.toolName, context.signal);
|
|
101
105
|
if (llmSummary !== null) {
|
|
102
106
|
summary = llmSummary;
|
|
103
107
|
usedStrategy = 'llm';
|
|
@@ -261,8 +265,10 @@ export class ToolResultDelegator {
|
|
|
261
265
|
* LLM-based summarization using the provider.
|
|
262
266
|
* Returns null if the LLM call fails.
|
|
263
267
|
*/
|
|
264
|
-
async summarizeLLM(content, toolName) {
|
|
268
|
+
async summarizeLLM(content, toolName, signal) {
|
|
265
269
|
try {
|
|
270
|
+
if (signal?.aborted)
|
|
271
|
+
return null;
|
|
266
272
|
const messages = [
|
|
267
273
|
{
|
|
268
274
|
role: 'system',
|
|
@@ -278,7 +284,10 @@ export class ToolResultDelegator {
|
|
|
278
284
|
maxTokens: this.config.summaryMaxTokens,
|
|
279
285
|
temperature: 0,
|
|
280
286
|
model: undefined, // Use the provider's default model
|
|
287
|
+
signal,
|
|
281
288
|
})) {
|
|
289
|
+
if (signal?.aborted)
|
|
290
|
+
break;
|
|
282
291
|
if (chunk.type === 'text' && chunk.text) {
|
|
283
292
|
text += chunk.text;
|
|
284
293
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -40,7 +40,7 @@ export { generateId, sleep, retry, truncate, withRetryGenerator, calculateBackof
|
|
|
40
40
|
export type { RetryConfig as LLMRetryConfig, WithRetryOptions } from './utils/index.js';
|
|
41
41
|
export { AgentError, ProviderError, ToolError, ToolTimeoutError, ToolLoopError, ValidationError, MaxIterationsError, AbortError, ContextOverflowError, isAgentError, isProviderError, isToolError, isToolTimeoutError, isToolLoopError, isContextOverflowError, wrapError, } from './errors.js';
|
|
42
42
|
export { ContextManager, DEFAULT_CONTEXT_CONFIG, FileAccessTracker, createFileTrackingHook, TRACKED_TOOLS, DelegatedResultStore, ToolResultDelegator, DELEGATION_SYSTEM_PROMPT, DEFAULT_DELEGATION_CONFIG, } from './context/index.js';
|
|
43
|
-
export type { ContextManagerOptions, ContextCategory, BudgetAllocation, CategoryBudgetInfo, PreflightResult, VerbosityLevel, VerbosityConfig, ContextConfig, FilteringConfig, CompactionConfig, SummarizationConfig, CompactionResult, SummarizationResult, FilteringResult, ContextEvent, ContextEventHandler, ContextStats, FileAccessType, FileAccess, FileAccessTrackerOptions, FormatHintsOptions, FileAccessStats, DelegatedResultStoreStats, ToolResultDelegatorOptions, DelegationConfig, StoredResult, DelegationEvent, } from './context/index.js';
|
|
43
|
+
export type { ContextManagerOptions, ContextCategory, BudgetAllocation, CategoryBudgetInfo, PreflightResult, VerbosityLevel, VerbosityConfig, ContextConfig, FilteringConfig, CompactionConfig, SummarizationConfig, CompactionResult, SummarizationResult, FilteringResult, ContextEvent, ContextEventHandler, ContextStats, FileAccessType, FileAccess, FileAccessTrackerOptions, FormatHintsOptions, FileAccessStats, RestorationHintMessage, DelegatedResultStoreStats, ToolResultDelegatorOptions, DelegationConfig, StoredResult, DelegationEvent, } from './context/index.js';
|
|
44
44
|
export { SkillRegistry, defineSkill, createSkillRegistry, builtinSkills, getDefaultSkillRegistry, resetDefaultSkillRegistry, } from './skills/index.js';
|
|
45
45
|
export type { Skill, SkillInvocationResult, SkillInvokeOptions } from './skills/index.js';
|
|
46
46
|
export { JsonSerializer, CompactJsonSerializer, defaultSerializer, MemoryCheckpointer, FileCheckpointer, StateError, StateErrorCode, CURRENT_STATE_VERSION, } from './state/index.js';
|
package/dist/mcp/client.js
CHANGED
|
@@ -20,15 +20,17 @@ async function getMCPSDK() {
|
|
|
20
20
|
return sdkCache;
|
|
21
21
|
try {
|
|
22
22
|
// Dynamic imports to keep SDK optional
|
|
23
|
-
const [clientModule, stdioModule, httpModule] = await Promise.all([
|
|
23
|
+
const [clientModule, stdioModule, httpModule, typesModule] = await Promise.all([
|
|
24
24
|
import('@modelcontextprotocol/sdk/client/index.js'),
|
|
25
25
|
import('@modelcontextprotocol/sdk/client/stdio.js'),
|
|
26
26
|
import('@modelcontextprotocol/sdk/client/streamableHttp.js'),
|
|
27
|
+
import('@modelcontextprotocol/sdk/types.js'),
|
|
27
28
|
]);
|
|
28
29
|
sdkCache = {
|
|
29
30
|
Client: clientModule.Client,
|
|
30
31
|
StdioClientTransport: stdioModule.StdioClientTransport,
|
|
31
32
|
StreamableHTTPClientTransport: httpModule.StreamableHTTPClientTransport,
|
|
33
|
+
ToolListChangedNotificationSchema: typesModule.ToolListChangedNotificationSchema,
|
|
32
34
|
};
|
|
33
35
|
// Non-null assertion safe: we just assigned sdkCache above
|
|
34
36
|
return sdkCache;
|
|
@@ -112,6 +114,7 @@ export class MCPClient {
|
|
|
112
114
|
args: this.config.stdio.args,
|
|
113
115
|
env: this.config.stdio.env,
|
|
114
116
|
cwd: this.config.stdio.cwd,
|
|
117
|
+
stderr: 'pipe', // Capture stderr to prevent bleeding into host terminal
|
|
115
118
|
});
|
|
116
119
|
}
|
|
117
120
|
else if (this.config.transport === 'http' && this.config.http) {
|
|
@@ -123,7 +126,8 @@ export class MCPClient {
|
|
|
123
126
|
version: this.config.clientVersion ?? '1.0.0',
|
|
124
127
|
});
|
|
125
128
|
// Set up notification handler for tools changed
|
|
126
|
-
|
|
129
|
+
// SDK 1.26+ requires Zod schema instead of string for notification handlers
|
|
130
|
+
this.client.setNotificationHandler(sdk.ToolListChangedNotificationSchema, () => {
|
|
127
131
|
this.cachedTools = null; // Invalidate cache
|
|
128
132
|
this.emit({ type: 'tools_changed', serverName: this.name });
|
|
129
133
|
});
|
package/dist/providers/claude.js
CHANGED
|
@@ -74,7 +74,9 @@ export class ClaudeProvider {
|
|
|
74
74
|
if (thinking) {
|
|
75
75
|
Object.assign(params, { thinking });
|
|
76
76
|
}
|
|
77
|
-
|
|
77
|
+
// Pass abort signal to SDK for immediate cancellation
|
|
78
|
+
const requestOptions = options?.signal ? { signal: options.signal } : undefined;
|
|
79
|
+
const stream = this.client.messages.stream(params, requestOptions);
|
|
78
80
|
const model = options?.model ?? this.defaultModel;
|
|
79
81
|
let currentToolId = '';
|
|
80
82
|
let currentToolName = '';
|
|
@@ -67,6 +67,10 @@ export class GeminiNativeProvider {
|
|
|
67
67
|
if (tools.length > 0) {
|
|
68
68
|
config.tools = [{ functionDeclarations: tools }];
|
|
69
69
|
}
|
|
70
|
+
// Check abort before starting the request
|
|
71
|
+
if (options?.signal?.aborted) {
|
|
72
|
+
throw new Error('Aborted');
|
|
73
|
+
}
|
|
70
74
|
// Request streaming response
|
|
71
75
|
const streamResponse = await this.client.models.generateContentStream({
|
|
72
76
|
model,
|
|
@@ -82,6 +86,9 @@ export class GeminiNativeProvider {
|
|
|
82
86
|
let lastUsageMetadata;
|
|
83
87
|
// Process stream chunks
|
|
84
88
|
for await (const chunk of streamResponse) {
|
|
89
|
+
// Check abort between chunks
|
|
90
|
+
if (options?.signal?.aborted)
|
|
91
|
+
break;
|
|
85
92
|
const streamChunks = this.processChunk(chunk, currentToolId, currentToolName, toolInputJson, inThinkingBlock);
|
|
86
93
|
for (const streamChunk of streamChunks) {
|
|
87
94
|
// Update tracking state
|
|
@@ -99,6 +99,17 @@ export class OpenAICompatibleProvider {
|
|
|
99
99
|
let usage;
|
|
100
100
|
try {
|
|
101
101
|
const controller = new AbortController();
|
|
102
|
+
// Chain user abort signal to our controller
|
|
103
|
+
if (options?.signal) {
|
|
104
|
+
if (options.signal.aborted) {
|
|
105
|
+
controller.abort();
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
options.signal.addEventListener('abort', () => {
|
|
109
|
+
controller.abort();
|
|
110
|
+
}, { once: true });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
102
113
|
const timeoutId = setTimeout(() => {
|
|
103
114
|
controller.abort();
|
|
104
115
|
}, this.timeout);
|
|
@@ -147,6 +147,11 @@ export interface ChatOptions {
|
|
|
147
147
|
* ```
|
|
148
148
|
*/
|
|
149
149
|
thinking?: ThinkingConfig;
|
|
150
|
+
/**
|
|
151
|
+
* AbortSignal for cancelling the LLM request.
|
|
152
|
+
* When aborted, the provider should stop streaming and throw/return immediately.
|
|
153
|
+
*/
|
|
154
|
+
signal?: AbortSignal;
|
|
150
155
|
/**
|
|
151
156
|
* Enable prompt caching for system prompt and tools (Claude-specific)
|
|
152
157
|
*
|
|
@@ -183,6 +183,7 @@ export function createTaskTool(options) {
|
|
|
183
183
|
};
|
|
184
184
|
}
|
|
185
185
|
const resultPromise = parentAgent.runSubAgent(subAgentName, prompt, {
|
|
186
|
+
signal: context?.abortSignal,
|
|
186
187
|
onEvent: eventHandler,
|
|
187
188
|
});
|
|
188
189
|
const result = await Promise.race([resultPromise, timeoutPromise]);
|
|
@@ -38,6 +38,11 @@ export interface TodoItem {
|
|
|
38
38
|
* If undefined/null, the task is unassigned
|
|
39
39
|
*/
|
|
40
40
|
owner?: string;
|
|
41
|
+
/**
|
|
42
|
+
* 1-based task numbers this task is blocked by.
|
|
43
|
+
* References positions in the todo array.
|
|
44
|
+
*/
|
|
45
|
+
blockedBy?: number[];
|
|
41
46
|
/**
|
|
42
47
|
* Creation timestamp
|
|
43
48
|
*/
|
|
@@ -60,6 +65,7 @@ export interface TodoWriteInput {
|
|
|
60
65
|
activeForm?: string;
|
|
61
66
|
priority?: number;
|
|
62
67
|
owner?: string;
|
|
68
|
+
blockedBy?: number[];
|
|
63
69
|
}>;
|
|
64
70
|
}
|
|
65
71
|
/**
|
|
@@ -110,6 +116,7 @@ export declare class TodoStore {
|
|
|
110
116
|
activeForm?: string;
|
|
111
117
|
priority?: number;
|
|
112
118
|
owner?: string;
|
|
119
|
+
blockedBy?: number[];
|
|
113
120
|
}>): void;
|
|
114
121
|
/**
|
|
115
122
|
* Add a single todo
|
|
@@ -120,6 +127,7 @@ export declare class TodoStore {
|
|
|
120
127
|
activeForm?: string;
|
|
121
128
|
priority?: number;
|
|
122
129
|
owner?: string;
|
|
130
|
+
blockedBy?: number[];
|
|
123
131
|
}): TodoItem;
|
|
124
132
|
/**
|
|
125
133
|
* Update a todo by ID
|
|
@@ -64,6 +64,7 @@ export class TodoStore {
|
|
|
64
64
|
activeForm: todo.activeForm,
|
|
65
65
|
priority: todo.priority,
|
|
66
66
|
owner: todo.owner,
|
|
67
|
+
blockedBy: todo.blockedBy,
|
|
67
68
|
createdAt: now,
|
|
68
69
|
updatedAt: now,
|
|
69
70
|
});
|
|
@@ -82,6 +83,7 @@ export class TodoStore {
|
|
|
82
83
|
activeForm: todo.activeForm,
|
|
83
84
|
priority: todo.priority,
|
|
84
85
|
owner: todo.owner,
|
|
86
|
+
blockedBy: todo.blockedBy,
|
|
85
87
|
createdAt: now,
|
|
86
88
|
updatedAt: now,
|
|
87
89
|
};
|
|
@@ -158,7 +160,8 @@ export const todoWriteTool = defineTool({
|
|
|
158
160
|
name: 'todo_write',
|
|
159
161
|
description: 'Update the SESSION todo list (shown in CLI footer). These are ephemeral tasks for the current session only - ' +
|
|
160
162
|
'NOT persistent backlog items. For persistent project backlog, use workitem_* tools. ' +
|
|
161
|
-
'Provide the complete list to replace current todos.'
|
|
163
|
+
'Provide the complete list to replace current todos. ' +
|
|
164
|
+
'Use blockedBy to declare task dependencies (1-based positions in the array).',
|
|
162
165
|
inputSchema: {
|
|
163
166
|
type: 'object',
|
|
164
167
|
properties: {
|
|
@@ -189,6 +192,11 @@ export const todoWriteTool = defineTool({
|
|
|
189
192
|
type: 'string',
|
|
190
193
|
description: 'Owner agent ID (e.g., "dev", "pm", "arch"). Omit for unassigned tasks.',
|
|
191
194
|
},
|
|
195
|
+
blockedBy: {
|
|
196
|
+
type: 'array',
|
|
197
|
+
items: { type: 'number' },
|
|
198
|
+
description: 'Array of 1-based task numbers this task depends on (e.g., [3, 4] means blocked by task #3 and #4)',
|
|
199
|
+
},
|
|
192
200
|
},
|
|
193
201
|
required: ['content', 'status'],
|
|
194
202
|
},
|
|
@@ -268,6 +276,7 @@ export const todoReadTool = defineTool({
|
|
|
268
276
|
activeForm: t.activeForm,
|
|
269
277
|
priority: t.priority,
|
|
270
278
|
owner: t.owner,
|
|
279
|
+
blockedBy: t.blockedBy,
|
|
271
280
|
})),
|
|
272
281
|
counts,
|
|
273
282
|
total: todos.length,
|
|
@@ -282,7 +291,8 @@ export function createTodoTools(store) {
|
|
|
282
291
|
const todoWrite = defineTool({
|
|
283
292
|
name: 'todo_write',
|
|
284
293
|
description: 'Update the SESSION todo list (shown in CLI footer). These are ephemeral tasks for the current session only - ' +
|
|
285
|
-
'NOT persistent backlog items. For persistent project backlog, use workitem_* tools.'
|
|
294
|
+
'NOT persistent backlog items. For persistent project backlog, use workitem_* tools. ' +
|
|
295
|
+
'Use blockedBy to declare task dependencies (1-based positions in the array).',
|
|
286
296
|
inputSchema: {
|
|
287
297
|
type: 'object',
|
|
288
298
|
properties: {
|
|
@@ -297,6 +307,11 @@ export function createTodoTools(store) {
|
|
|
297
307
|
activeForm: { type: 'string' },
|
|
298
308
|
priority: { type: 'number' },
|
|
299
309
|
owner: { type: 'string' },
|
|
310
|
+
blockedBy: {
|
|
311
|
+
type: 'array',
|
|
312
|
+
items: { type: 'number' },
|
|
313
|
+
description: '1-based task numbers this task depends on',
|
|
314
|
+
},
|
|
300
315
|
},
|
|
301
316
|
required: ['content', 'status'],
|
|
302
317
|
},
|
|
@@ -368,6 +383,7 @@ export function createTodoTools(store) {
|
|
|
368
383
|
activeForm: t.activeForm,
|
|
369
384
|
priority: t.priority,
|
|
370
385
|
owner: t.owner,
|
|
386
|
+
blockedBy: t.blockedBy,
|
|
371
387
|
})),
|
|
372
388
|
counts,
|
|
373
389
|
total: todos.length,
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { Message } from '../providers/types.js';
|
|
8
8
|
/**
|
|
9
|
-
* Count tokens in a text string using tiktoken
|
|
9
|
+
* Count tokens in a text string using tiktoken.
|
|
10
|
+
* Falls back to chars/4 heuristic for very large or pathologically repetitive strings.
|
|
10
11
|
*/
|
|
11
12
|
export declare function countTokens(text: string): number;
|
|
12
13
|
/**
|
package/dist/utils/tokenizer.js
CHANGED
|
@@ -14,11 +14,48 @@ function getEncoder() {
|
|
|
14
14
|
return encoder;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
|
-
*
|
|
17
|
+
* Maximum text length for tiktoken encoding.
|
|
18
|
+
* Beyond this, fall back to chars/4 heuristic to avoid pathological BPE performance.
|
|
19
|
+
*/
|
|
20
|
+
const MAX_TIKTOKEN_LENGTH = 10000;
|
|
21
|
+
/**
|
|
22
|
+
* Minimum length at which we check for low-entropy (repetitive) content.
|
|
23
|
+
* Repetitive single-character strings like 'a'.repeat(5000) cause O(n²+)
|
|
24
|
+
* performance in js-tiktoken's BPE merge loop.
|
|
25
|
+
*/
|
|
26
|
+
const ENTROPY_CHECK_THRESHOLD = 2000;
|
|
27
|
+
/**
|
|
28
|
+
* Check if a string is low-entropy (highly repetitive), which causes
|
|
29
|
+
* pathological BPE performance in js-tiktoken.
|
|
30
|
+
*
|
|
31
|
+
* Samples characters from the string and checks unique character ratio.
|
|
32
|
+
* A ratio below 0.01 (e.g., 'aaaa...') triggers the heuristic fallback.
|
|
33
|
+
*/
|
|
34
|
+
function isLowEntropy(text) {
|
|
35
|
+
// Sample up to 200 characters evenly across the string
|
|
36
|
+
const sampleSize = Math.min(200, text.length);
|
|
37
|
+
const step = Math.max(1, Math.floor(text.length / sampleSize));
|
|
38
|
+
const seen = new Set();
|
|
39
|
+
for (let i = 0; i < text.length && seen.size < 20; i += step) {
|
|
40
|
+
seen.add(text[i]);
|
|
41
|
+
}
|
|
42
|
+
// If unique chars / length ratio is very low, it's repetitive
|
|
43
|
+
return seen.size / Math.min(sampleSize, text.length) < 0.02;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Count tokens in a text string using tiktoken.
|
|
47
|
+
* Falls back to chars/4 heuristic for very large or pathologically repetitive strings.
|
|
18
48
|
*/
|
|
19
49
|
export function countTokens(text) {
|
|
20
50
|
if (!text)
|
|
21
51
|
return 0;
|
|
52
|
+
if (text.length > MAX_TIKTOKEN_LENGTH) {
|
|
53
|
+
return Math.ceil(text.length / 4);
|
|
54
|
+
}
|
|
55
|
+
// Detect low-entropy strings that cause O(n²+) BPE performance
|
|
56
|
+
if (text.length >= ENTROPY_CHECK_THRESHOLD && isLowEntropy(text)) {
|
|
57
|
+
return Math.ceil(text.length / 4);
|
|
58
|
+
}
|
|
22
59
|
return getEncoder().encode(text).length;
|
|
23
60
|
}
|
|
24
61
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@compilr-dev/agents",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.14",
|
|
4
4
|
"description": "Lightweight multi-LLM agent library for building CLI AI assistants",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -77,10 +77,12 @@
|
|
|
77
77
|
"vitest": "^4.0.18"
|
|
78
78
|
},
|
|
79
79
|
"dependencies": {
|
|
80
|
-
"@google/genai": "^1.
|
|
80
|
+
"@google/genai": "^1.42.0",
|
|
81
81
|
"js-tiktoken": "^1.0.21"
|
|
82
82
|
},
|
|
83
83
|
"overrides": {
|
|
84
|
-
"hono": "^4.11.
|
|
84
|
+
"hono": "^4.11.10",
|
|
85
|
+
"minimatch": ">=10.2.1",
|
|
86
|
+
"glob": ">=11.0.0"
|
|
85
87
|
}
|
|
86
88
|
}
|