@link-assistant/hive-mind 1.64.1 → 1.64.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/package.json +1 -1
- package/src/agent-commander.lib.mjs +47 -5
- package/src/agent-token-usage.lib.mjs +15 -1
- package/src/claude.budget-stats.lib.mjs +72 -27
- package/src/claude.lib.mjs +12 -1
- package/src/codex.lib.mjs +22 -1
- package/src/context-fill.lib.mjs +71 -0
- package/src/gemini.lib.mjs +22 -7
- package/src/github.lib.mjs +2 -2
- package/src/interactive-mode.lib.mjs +104 -8
- package/src/lib.mjs +3 -3
- package/src/post-finish-sanitization-sweep.lib.mjs +201 -0
- package/src/qwen.lib.mjs +191 -9
- package/src/solve.config.lib.mjs +15 -0
- package/src/solve.results.lib.mjs +52 -0
- package/src/telegram-bot.mjs +40 -0
- package/src/telegram-leak-notifier.lib.mjs +79 -0
- package/src/telegram-tokens-command.lib.mjs +151 -0
- package/src/token-sanitization.lib.mjs +355 -18
- package/src/tool-comments.lib.mjs +6 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 1.64.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- dd52682: Sanitize all user-facing output to prevent token leaks (#1745).
|
|
8
|
+
- All comment-posting paths (`postComment`, `editComment`, `postTrackedComment`) run bodies through `sanitizeOutput` (canonical name) / `sanitizeCommentBody` (active-token wrapper). `sanitizeLogContent` is preserved as a backward-compatible alias.
|
|
9
|
+
- `KNOWN_LOCAL_TOKEN_ENV_VARS` registry masks tokens by exact env value (Telegram, GitHub, Anthropic/Claude, OpenAI/Codex, Gemini/Google, Qwen/Dashscope, OpenCode, AgentCLI, HuggingFace).
|
|
10
|
+
- Three independent CLI flags: `--dangerously-skip-output-sanitization`, `--dangerously-skip-code-output-sanitization`, `--dangerously-skip-active-tokens-output-sanitization` — all default false; active-tokens skip stays separate so the broad skip flag still keeps active-token masking on.
|
|
11
|
+
- Process-wide sanitization counters (`getSanitizationStats`, `formatSanitizationSummary`) print a one-line summary at the end of each run with a hint to use `--dangerously-skip-output-sanitization` when masking blocks the user's workflow.
|
|
12
|
+
- `extractTokensFromUserContent` carve-out helper: tokens already present in user-provided content (issue body, non-bot comments, pre-existing code) are passed as `excludeTokens` so the sanitizer leaves them untouched while still masking active local tokens.
|
|
13
|
+
- Post-finish sweep (`runPostFinishSweep`) re-reads bot-authored PR comments and the PR description after the AI session completes and edits in place if a leak slipped past the live sanitizer.
|
|
14
|
+
- ESLint guardrail (`gh-rate-limit/require-sanitized-output`) flags raw `gh pr comment`, `gh issue comment`, `gh pr edit`, and `gh api .../comments` calls that bypass the sanitizer.
|
|
15
|
+
- Out-of-band Telegram leak DM with masked summaries when a known-local token is detected in an outbound comment.
|
|
16
|
+
- Hidden owner-only `/tokens` Telegram command lists configured tokens (always masked, private chat only).
|
|
17
|
+
- `maskToken` defaults to 3+3 characters per issue requirements.
|
|
18
|
+
- secretlint preset (best-of-breed) runs alongside our custom patterns; mismatch warnings surface gaps.
|
|
19
|
+
|
|
20
|
+
## 1.64.2
|
|
21
|
+
|
|
22
|
+
### Patch Changes
|
|
23
|
+
|
|
24
|
+
- 320ca42: Fix budget stats sub-agent context-fill calculation so cumulative-only rows (e.g. Claude Haiku 4.5 sub-agent calls that never appear in the parent JSONL) use `input + cache_creation` instead of `input + cache_creation + cache_read`. The previous formula double-counted the cached prefix replayed across calls and produced impossible percentages such as `1.2M / 200K (583%)`.
|
|
25
|
+
|
|
3
26
|
## 1.64.1
|
|
4
27
|
|
|
5
28
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -11,6 +11,7 @@ import { resolveCodexReasoningEffort } from './codex.options.lib.mjs';
|
|
|
11
11
|
import { mapModelForTool } from './models/index.mjs';
|
|
12
12
|
import { buildCodexDisable1mContextConfigArgs, buildCodexSubSessionSizeConfigArgs, parseSubSessionSize } from './sub-session-size.lib.mjs';
|
|
13
13
|
import { detectUsageLimit } from './usage-limit.lib.mjs';
|
|
14
|
+
import { getCacheReadTokenCount, getCumulativeContextInputTokens, getOutputTokenCount } from './context-fill.lib.mjs';
|
|
14
15
|
|
|
15
16
|
export const AGENT_COMMANDER_TOOLS = new Set(['claude', 'codex', 'opencode', 'agent', 'qwen', 'gemini']);
|
|
16
17
|
|
|
@@ -222,10 +223,45 @@ const extractResultSummary = (messages, plainOutput) => {
|
|
|
222
223
|
|
|
223
224
|
const hasErrorMessage = messages => messages.some(message => message?.is_error === true || message?.type === 'error' || message?.type === 'step_error' || message?.error);
|
|
224
225
|
|
|
226
|
+
const normalizeAgentCommanderTokenUsage = usage => {
|
|
227
|
+
if (!usage || typeof usage !== 'object') return null;
|
|
228
|
+
const normalized = {
|
|
229
|
+
...usage,
|
|
230
|
+
contextFillInputTokens: usage.contextFillInputTokens ?? getCumulativeContextInputTokens(usage),
|
|
231
|
+
};
|
|
232
|
+
const cacheReadTokens = getCacheReadTokenCount(normalized);
|
|
233
|
+
const hasTokenCounts = getCumulativeContextInputTokens(normalized) > 0 || getOutputTokenCount(normalized) > 0 || cacheReadTokens > 0;
|
|
234
|
+
if (!hasTokenCounts) return null;
|
|
235
|
+
if (!normalized.stepCount) normalized.stepCount = 1;
|
|
236
|
+
return normalized;
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
const enrichPricingInfoWithTokenUsage = ({ pricingInfo = null, usage = null, tool = null, publicPricingEstimate = null }) => {
|
|
240
|
+
const tokenUsage = normalizeAgentCommanderTokenUsage(pricingInfo?.tokenUsage || usage);
|
|
241
|
+
if (!tokenUsage) return pricingInfo || null;
|
|
242
|
+
|
|
243
|
+
return {
|
|
244
|
+
source: 'agent-commander',
|
|
245
|
+
...(pricingInfo || {}),
|
|
246
|
+
provider: pricingInfo?.provider || tool || 'agent-commander',
|
|
247
|
+
modelId: pricingInfo?.modelId || tokenUsage.respondedModelId || tokenUsage.requestedModelId || null,
|
|
248
|
+
modelName: pricingInfo?.modelName || tokenUsage.respondedModelId || tokenUsage.requestedModelId || null,
|
|
249
|
+
totalCostUSD: pricingInfo?.totalCostUSD ?? publicPricingEstimate ?? null,
|
|
250
|
+
tokenUsage,
|
|
251
|
+
};
|
|
252
|
+
};
|
|
253
|
+
|
|
225
254
|
export const summarizeAgentCommanderResult = ({ result, tool }) => {
|
|
226
255
|
const plainOutput = result?.output?.plain || '';
|
|
227
256
|
if (result?.metadata && typeof result.metadata === 'object') {
|
|
228
257
|
const metadata = result.metadata;
|
|
258
|
+
const streamTokenUsage = metadata.streamTokenUsage || result.usage || null;
|
|
259
|
+
const pricingInfo = enrichPricingInfoWithTokenUsage({
|
|
260
|
+
pricingInfo: metadata.pricingInfo || null,
|
|
261
|
+
usage: streamTokenUsage,
|
|
262
|
+
tool,
|
|
263
|
+
publicPricingEstimate: metadata.publicPricingEstimate ?? metadata.pricingInfo?.totalCostUSD ?? null,
|
|
264
|
+
});
|
|
229
265
|
return {
|
|
230
266
|
success: metadata.success === true,
|
|
231
267
|
sessionId: metadata.sessionId || result.sessionId || null,
|
|
@@ -233,11 +269,11 @@ export const summarizeAgentCommanderResult = ({ result, tool }) => {
|
|
|
233
269
|
limitResetTime: metadata.limitResetTime || null,
|
|
234
270
|
limitTimezone: metadata.limitTimezone || null,
|
|
235
271
|
anthropicTotalCostUSD: metadata.anthropicTotalCostUSD ?? null,
|
|
236
|
-
publicPricingEstimate: metadata.publicPricingEstimate ??
|
|
237
|
-
pricingInfo
|
|
272
|
+
publicPricingEstimate: metadata.publicPricingEstimate ?? pricingInfo?.totalCostUSD ?? null,
|
|
273
|
+
pricingInfo,
|
|
238
274
|
resultSummary: metadata.resultSummary || null,
|
|
239
275
|
resultModelUsage: metadata.resultModelUsage || null,
|
|
240
|
-
streamTokenUsage
|
|
276
|
+
streamTokenUsage,
|
|
241
277
|
subAgentCalls: metadata.subAgentCalls || null,
|
|
242
278
|
errorDuringExecution: metadata.errorDuringExecution === true || result?.exitCode !== 0,
|
|
243
279
|
result: plainOutput,
|
|
@@ -250,6 +286,12 @@ export const summarizeAgentCommanderResult = ({ result, tool }) => {
|
|
|
250
286
|
const resultMessage = [...messages].reverse().find(message => message?.type === 'result') || null;
|
|
251
287
|
const totalCost = typeof resultMessage?.total_cost_usd === 'number' ? resultMessage.total_cost_usd : null;
|
|
252
288
|
const publicPricingEstimate = tool === 'agent' && typeof usage?.totalCost === 'number' ? usage.totalCost : null;
|
|
289
|
+
const pricingInfo = enrichPricingInfoWithTokenUsage({
|
|
290
|
+
pricingInfo: publicPricingEstimate !== null ? { totalCostUSD: publicPricingEstimate, source: 'agent-commander' } : null,
|
|
291
|
+
usage,
|
|
292
|
+
tool,
|
|
293
|
+
publicPricingEstimate,
|
|
294
|
+
});
|
|
253
295
|
|
|
254
296
|
return {
|
|
255
297
|
success: result?.exitCode === 0 && !usageLimit.isUsageLimit && !hasErrorMessage(messages),
|
|
@@ -258,8 +300,8 @@ export const summarizeAgentCommanderResult = ({ result, tool }) => {
|
|
|
258
300
|
limitResetTime: usageLimit.resetTime,
|
|
259
301
|
limitTimezone: usageLimit.timezone,
|
|
260
302
|
anthropicTotalCostUSD: tool === 'claude' ? totalCost : null,
|
|
261
|
-
publicPricingEstimate,
|
|
262
|
-
pricingInfo
|
|
303
|
+
publicPricingEstimate: publicPricingEstimate ?? pricingInfo?.totalCostUSD ?? null,
|
|
304
|
+
pricingInfo,
|
|
263
305
|
resultSummary: extractResultSummary(messages, plainOutput),
|
|
264
306
|
resultModelUsage: null,
|
|
265
307
|
streamTokenUsage: usage,
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import Decimal from 'decimal.js-light';
|
|
4
4
|
import { sanitizeObjectStrings } from './unicode-sanitization.lib.mjs';
|
|
5
|
+
import { getCumulativeContextInputTokens, getRestoredContextInputTokens } from './context-fill.lib.mjs';
|
|
5
6
|
|
|
6
7
|
export const createTokenFieldAvailability = () => ({
|
|
7
8
|
inputTokens: false,
|
|
@@ -23,6 +24,7 @@ export const createAgentTokenUsage = () => ({
|
|
|
23
24
|
respondedModelId: null,
|
|
24
25
|
contextLimit: null,
|
|
25
26
|
outputLimit: null,
|
|
27
|
+
contextFillInputTokens: 0,
|
|
26
28
|
peakContextUsage: 0,
|
|
27
29
|
tokenFieldAvailability: createTokenFieldAvailability(),
|
|
28
30
|
});
|
|
@@ -61,10 +63,22 @@ export const accumulateAgentStepFinishUsage = (usage, data) => {
|
|
|
61
63
|
if (data.part.model.respondedModelID) usage.respondedModelId = data.part.model.respondedModelID;
|
|
62
64
|
}
|
|
63
65
|
|
|
66
|
+
const stepContextFill = getCumulativeContextInputTokens({
|
|
67
|
+
inputTokens: getTokenCount(tokens.input),
|
|
68
|
+
cacheWriteTokens: getTokenCount(tokens.cache?.write),
|
|
69
|
+
});
|
|
70
|
+
if (stepContextFill > (usage.contextFillInputTokens || 0)) {
|
|
71
|
+
usage.contextFillInputTokens = stepContextFill;
|
|
72
|
+
}
|
|
73
|
+
|
|
64
74
|
if (data.part.context) {
|
|
65
75
|
if (data.part.context.contextLimit) usage.contextLimit = data.part.context.contextLimit;
|
|
66
76
|
if (data.part.context.outputLimit) usage.outputLimit = data.part.context.outputLimit;
|
|
67
|
-
const stepContextUsage =
|
|
77
|
+
const stepContextUsage = getRestoredContextInputTokens({
|
|
78
|
+
inputTokens: getTokenCount(tokens.input),
|
|
79
|
+
cacheWriteTokens: getTokenCount(tokens.cache?.write),
|
|
80
|
+
cacheReadTokens: getTokenCount(tokens.cache?.read),
|
|
81
|
+
});
|
|
68
82
|
if (stepContextUsage > (usage.peakContextUsage || 0)) {
|
|
69
83
|
usage.peakContextUsage = stepContextUsage;
|
|
70
84
|
}
|
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|
import { formatNumber } from './claude.lib.mjs';
|
|
6
6
|
import Decimal from 'decimal.js-light';
|
|
7
|
+
import { getCacheReadTokenCount, getCacheWriteTokenCount, getCumulativeContextInputTokens, getDisplayContextInputTokens, getExplicitContextFillInputTokens, getInputTokenCount, getOutputTokenCount, getRestoredContextInputTokens } from './context-fill.lib.mjs';
|
|
8
|
+
|
|
9
|
+
export { getCumulativeContextInputTokens, getRestoredContextInputTokens };
|
|
7
10
|
|
|
8
11
|
/**
|
|
9
12
|
* Helper: creates a fresh sub-session usage object for tracking tokens between compactification events
|
|
@@ -19,9 +22,9 @@ export const createEmptySubSessionUsage = () => ({
|
|
|
19
22
|
peakOutputUsage: 0,
|
|
20
23
|
});
|
|
21
24
|
|
|
22
|
-
export const getRawRequestInputTokens = usage => (usage
|
|
25
|
+
export const getRawRequestInputTokens = usage => getRestoredContextInputTokens(usage);
|
|
23
26
|
|
|
24
|
-
export const getUsageInputTokens = usage => (usage
|
|
27
|
+
export const getUsageInputTokens = usage => getRestoredContextInputTokens(usage);
|
|
25
28
|
|
|
26
29
|
/**
|
|
27
30
|
* Helper: accumulates token usage from a JSONL entry into a model usage map
|
|
@@ -184,6 +187,7 @@ export const dumpBudgetTrace = async (usage, tokenUsage, log) => {
|
|
|
184
187
|
const reads = usage.cacheReadTokens || 0;
|
|
185
188
|
const inputs = usage.inputTokens || 0;
|
|
186
189
|
const outputs = usage.outputTokens || 0;
|
|
190
|
+
const explicitContextFill = getExplicitContextFillInputTokens(usage);
|
|
187
191
|
const webSearches = usage.webSearchRequests || 0;
|
|
188
192
|
const subSessionCount = (tokenUsage?.subSessions || []).length;
|
|
189
193
|
const source = usage._sourceResultJson ? 'jsonl + result-event' : 'jsonl';
|
|
@@ -194,6 +198,14 @@ export const dumpBudgetTrace = async (usage, tokenUsage, log) => {
|
|
|
194
198
|
// buckets split for cost and accounting review.
|
|
195
199
|
await log(` peak input: ${formatNumber(peak)}${limit.context ? ` / ${formatNumber(limit.context)} context` : ''} (largest request input + cache_creation + cache_read)`, { verbose: true });
|
|
196
200
|
await log(` cumulative: input ${formatNumber(inputs)}, cache_write ${formatNumber(writes)} (5m ${formatNumber(writes5m)} / 1h ${formatNumber(writes1h)}), cache_read ${formatNumber(reads)}, output ${formatNumber(outputs)}`, { verbose: true });
|
|
201
|
+
// Issue #1741: when peak is 0 (sub-agent only seen via result event), the
|
|
202
|
+
// detail row falls back to the cumulative-context proxy `input + cache_write`
|
|
203
|
+
// (cache_read is excluded because it represents the same cached prefix replayed
|
|
204
|
+
// across calls and would inflate the percentage past 100%).
|
|
205
|
+
if (explicitContextFill !== null || peak === 0) {
|
|
206
|
+
const contextFill = explicitContextFill ?? getCumulativeContextInputTokens(usage);
|
|
207
|
+
await log(` context fill: ${formatNumber(contextFill)}${limit.context ? ` / ${formatNumber(limit.context)} context` : ''} (input + cache_write; cache_read excluded — issue #1741)`, { verbose: true });
|
|
208
|
+
}
|
|
197
209
|
// Issue #1710 R1: web_search is now billed in calculateModelCost. The trace
|
|
198
210
|
// still surfaces the implied dollar cost so the residual remains debuggable
|
|
199
211
|
// from the saved log even if a future model lacks pricing data.
|
|
@@ -305,17 +317,25 @@ export const mergeResultModelUsage = (modelUsage, resultModelUsage) => {
|
|
|
305
317
|
if (!resultModelUsage || typeof resultModelUsage !== 'object') return;
|
|
306
318
|
for (const [modelId, resultUsage] of Object.entries(resultModelUsage)) {
|
|
307
319
|
if (modelId.startsWith('<') && modelId.endsWith('>')) continue;
|
|
320
|
+
const inputTokens = getInputTokenCount(resultUsage);
|
|
321
|
+
const cacheCreationTokens = getCacheWriteTokenCount(resultUsage);
|
|
322
|
+
const cacheReadTokens = getCacheReadTokenCount(resultUsage);
|
|
323
|
+
const outputTokens = getOutputTokenCount(resultUsage);
|
|
324
|
+
const explicitContextFill = getExplicitContextFillInputTokens(resultUsage);
|
|
308
325
|
if (!modelUsage[modelId]) {
|
|
309
326
|
modelUsage[modelId] = {
|
|
310
|
-
inputTokens
|
|
311
|
-
cacheCreationTokens
|
|
327
|
+
inputTokens,
|
|
328
|
+
cacheCreationTokens,
|
|
312
329
|
cacheCreation5mTokens: 0,
|
|
313
330
|
cacheCreation1hTokens: 0,
|
|
314
|
-
cacheReadTokens
|
|
315
|
-
outputTokens
|
|
331
|
+
cacheReadTokens,
|
|
332
|
+
outputTokens,
|
|
316
333
|
webSearchRequests: resultUsage.webSearchRequests || 0,
|
|
317
334
|
_sourceResultJson: true,
|
|
318
335
|
};
|
|
336
|
+
if (explicitContextFill !== null) {
|
|
337
|
+
modelUsage[modelId].contextFillInputTokens = explicitContextFill;
|
|
338
|
+
}
|
|
319
339
|
if (resultUsage.costUSD != null) {
|
|
320
340
|
modelUsage[modelId]._resultCostUSD = resultUsage.costUSD;
|
|
321
341
|
}
|
|
@@ -331,13 +351,16 @@ export const mergeResultModelUsage = (modelUsage, resultModelUsage) => {
|
|
|
331
351
|
} else {
|
|
332
352
|
const jsonlUsage = modelUsage[modelId];
|
|
333
353
|
const jsonlTotal = jsonlUsage.inputTokens + jsonlUsage.cacheCreationTokens + jsonlUsage.cacheReadTokens + jsonlUsage.outputTokens;
|
|
334
|
-
const resultTotal =
|
|
354
|
+
const resultTotal = inputTokens + cacheCreationTokens + cacheReadTokens + outputTokens;
|
|
335
355
|
if (resultTotal > jsonlTotal) {
|
|
336
|
-
jsonlUsage.inputTokens =
|
|
337
|
-
jsonlUsage.cacheCreationTokens =
|
|
338
|
-
jsonlUsage.cacheReadTokens =
|
|
339
|
-
jsonlUsage.outputTokens =
|
|
356
|
+
jsonlUsage.inputTokens = inputTokens;
|
|
357
|
+
jsonlUsage.cacheCreationTokens = cacheCreationTokens;
|
|
358
|
+
jsonlUsage.cacheReadTokens = cacheReadTokens;
|
|
359
|
+
jsonlUsage.outputTokens = outputTokens;
|
|
340
360
|
jsonlUsage._sourceResultJson = true;
|
|
361
|
+
if (explicitContextFill !== null) {
|
|
362
|
+
jsonlUsage.contextFillInputTokens = explicitContextFill;
|
|
363
|
+
}
|
|
341
364
|
}
|
|
342
365
|
if (resultUsage.costUSD != null) {
|
|
343
366
|
jsonlUsage._resultCostUSD = resultUsage.costUSD;
|
|
@@ -573,7 +596,7 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
|
|
|
573
596
|
stats += `\n\n**${modelName}:** (${subSessions.length} sub-sessions)`;
|
|
574
597
|
}
|
|
575
598
|
|
|
576
|
-
const peakContext = usage
|
|
599
|
+
const peakContext = getDisplayContextInputTokens(usage);
|
|
577
600
|
|
|
578
601
|
if (showSubSessions) {
|
|
579
602
|
// Issue #1600: Unified format — no "Context window:" prefix, same format as sub-agent calls
|
|
@@ -587,10 +610,14 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
|
|
|
587
610
|
// so peakContext stays at 0; without this fallback the rendered comment loses
|
|
588
611
|
// the sub-agent's input-token information entirely. The detail line is
|
|
589
612
|
// deliberately simple; the Total line below keeps the cache split.
|
|
613
|
+
// Issue #1741: For result-event-only rows we have cumulative totals, not a
|
|
614
|
+
// per-request peak, so the detail-line numerator must exclude cache_reads
|
|
615
|
+
// (which are the same cached prefix replayed across calls and would inflate
|
|
616
|
+
// the percentage past 100%). The Total line keeps the full split.
|
|
590
617
|
const parts = [];
|
|
591
618
|
const isResultSingleCall = usage._sourceResultJson || callCount > 0;
|
|
592
619
|
const inputPart = isResultSingleCall
|
|
593
|
-
? formatInputContextPart(
|
|
620
|
+
? formatInputContextPart(getCumulativeContextInputTokens(usage), contextLimit, formatTokensCompact)
|
|
594
621
|
: buildCumulativeInputPhrase({
|
|
595
622
|
input: usage.inputTokens || 0,
|
|
596
623
|
cacheWrites: usage.cacheCreationTokens || 0,
|
|
@@ -636,7 +663,12 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
|
|
|
636
663
|
for (let i = 0; i < matchingCalls.length; i++) {
|
|
637
664
|
const call = matchingCalls[i];
|
|
638
665
|
const cu = call.usage || {};
|
|
639
|
-
|
|
666
|
+
// Issue #1741: per-call usage is itself cumulative across the
|
|
667
|
+
// sub-agent's internal API requests (each Anthropic Agent call
|
|
668
|
+
// can run a tool loop), so cache_reads grow with the loop length
|
|
669
|
+
// and would push the displayed fill past 100%. Use the same
|
|
670
|
+
// input + cache_creation proxy as the result-event-only fallback.
|
|
671
|
+
const callInput = getCumulativeContextInputTokens(cu);
|
|
640
672
|
const callOutput = cu.outputTokens || 0;
|
|
641
673
|
const parts = [];
|
|
642
674
|
if (contextLimit) {
|
|
@@ -655,9 +687,13 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
|
|
|
655
687
|
}
|
|
656
688
|
} else {
|
|
657
689
|
// Estimated per-call breakdown when sub-agent stream tracking did not capture
|
|
658
|
-
// per-call usage.
|
|
659
|
-
//
|
|
660
|
-
|
|
690
|
+
// per-call usage. Issue #1741: cumulative cache_read tokens grow without
|
|
691
|
+
// bound across calls (the same cached prefix is replayed on every call),
|
|
692
|
+
// so we mustn't add them when projecting an average per-call fill —
|
|
693
|
+
// doing so would routinely exceed 100% of the context window. The
|
|
694
|
+
// estimate uses input + cache_creation (cache reads stay in the Total
|
|
695
|
+
// line below).
|
|
696
|
+
const aggregateInput = getCumulativeContextInputTokens(usage);
|
|
661
697
|
const avgInput = Math.round(aggregateInput / callCount);
|
|
662
698
|
const avgOutput = Math.round(usage.outputTokens / callCount);
|
|
663
699
|
for (let i = 0; i < matchingCalls.length; i++) {
|
|
@@ -696,7 +732,14 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
|
|
|
696
732
|
* @returns {Object|null} Budget stats data compatible with buildBudgetStatsString, or null if no data
|
|
697
733
|
*/
|
|
698
734
|
export const buildAgentBudgetStats = (tokenUsage, pricingInfo) => {
|
|
699
|
-
if (!tokenUsage
|
|
735
|
+
if (!tokenUsage) return null;
|
|
736
|
+
|
|
737
|
+
const inputTokens = getInputTokenCount(tokenUsage);
|
|
738
|
+
const cacheWriteTokens = getCacheWriteTokenCount(tokenUsage);
|
|
739
|
+
const cacheReadTokens = getCacheReadTokenCount(tokenUsage);
|
|
740
|
+
const outputTokens = getOutputTokenCount(tokenUsage);
|
|
741
|
+
const hasTokens = inputTokens > 0 || cacheWriteTokens > 0 || cacheReadTokens > 0 || outputTokens > 0;
|
|
742
|
+
if ((tokenUsage.stepCount || 0) === 0 && !hasTokens) return null;
|
|
700
743
|
|
|
701
744
|
const modelName = pricingInfo?.modelName || tokenUsage.respondedModelId || tokenUsage.requestedModelId || 'Unknown';
|
|
702
745
|
const modelId = tokenUsage.respondedModelId || tokenUsage.requestedModelId || pricingInfo?.modelId || 'unknown';
|
|
@@ -704,14 +747,16 @@ export const buildAgentBudgetStats = (tokenUsage, pricingInfo) => {
|
|
|
704
747
|
// Use context limits from step_finish events if available, otherwise from pricing model info
|
|
705
748
|
const contextLimit = tokenUsage.contextLimit || pricingInfo?.modelInfo?.limit?.context || null;
|
|
706
749
|
const outputLimit = tokenUsage.outputLimit || pricingInfo?.modelInfo?.limit?.output || null;
|
|
750
|
+
const contextFillInputTokens = getExplicitContextFillInputTokens(tokenUsage) ?? getCumulativeContextInputTokens({ inputTokens, cacheWriteTokens });
|
|
707
751
|
|
|
708
752
|
const modelUsageEntry = {
|
|
709
|
-
inputTokens
|
|
710
|
-
cacheCreationTokens:
|
|
711
|
-
cacheReadTokens
|
|
712
|
-
outputTokens
|
|
753
|
+
inputTokens,
|
|
754
|
+
cacheCreationTokens: cacheWriteTokens,
|
|
755
|
+
cacheReadTokens,
|
|
756
|
+
outputTokens,
|
|
713
757
|
modelName,
|
|
714
758
|
modelInfo: contextLimit || outputLimit ? { limit: { context: contextLimit, output: outputLimit } } : null,
|
|
759
|
+
contextFillInputTokens,
|
|
715
760
|
peakContextUsage: tokenUsage.peakContextUsage || 0,
|
|
716
761
|
costUSD: pricingInfo?.totalCostUSD ?? null,
|
|
717
762
|
};
|
|
@@ -719,11 +764,11 @@ export const buildAgentBudgetStats = (tokenUsage, pricingInfo) => {
|
|
|
719
764
|
return {
|
|
720
765
|
modelUsage: { [modelId]: modelUsageEntry },
|
|
721
766
|
subSessions: [],
|
|
722
|
-
inputTokens
|
|
723
|
-
cacheCreationTokens:
|
|
724
|
-
cacheReadTokens
|
|
725
|
-
outputTokens
|
|
726
|
-
totalTokens:
|
|
767
|
+
inputTokens,
|
|
768
|
+
cacheCreationTokens: cacheWriteTokens,
|
|
769
|
+
cacheReadTokens,
|
|
770
|
+
outputTokens,
|
|
771
|
+
totalTokens: inputTokens + cacheWriteTokens + outputTokens,
|
|
727
772
|
};
|
|
728
773
|
};
|
|
729
774
|
|
package/src/claude.lib.mjs
CHANGED
|
@@ -682,7 +682,18 @@ export const executeClaudeCommand = async params => {
|
|
|
682
682
|
let interactiveHandler = null;
|
|
683
683
|
if (argv.interactiveMode && owner && repo && prNumber) {
|
|
684
684
|
await log('🔌 Interactive mode: Creating handler for real-time PR comments', { verbose: true });
|
|
685
|
-
interactiveHandler = createInteractiveHandler({
|
|
685
|
+
interactiveHandler = createInteractiveHandler({
|
|
686
|
+
owner,
|
|
687
|
+
repo,
|
|
688
|
+
prNumber,
|
|
689
|
+
$,
|
|
690
|
+
log,
|
|
691
|
+
verbose: argv.verbose,
|
|
692
|
+
// Issue #1745: thread the three independent dangerous-skip flags through
|
|
693
|
+
// so the comment-posting path can honor them; flags default to false.
|
|
694
|
+
skipOutputSanitization: argv['dangerously-skip-output-sanitization'] === true,
|
|
695
|
+
skipActiveTokensOutputSanitization: argv['dangerously-skip-active-tokens-output-sanitization'] === true,
|
|
696
|
+
});
|
|
686
697
|
} else if (argv.interactiveMode) {
|
|
687
698
|
await log('⚠️ Interactive mode: Disabled - missing PR info (owner/repo/prNumber)', { verbose: true });
|
|
688
699
|
}
|
package/src/codex.lib.mjs
CHANGED
|
@@ -26,6 +26,7 @@ import { fetchModelInfo } from './model-info.lib.mjs';
|
|
|
26
26
|
import { defaultModels } from './models/index.mjs';
|
|
27
27
|
import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
|
|
28
28
|
import { parseSubSessionSize, buildCodexSubSessionSizeConfigArgs, buildCodexDisable1mContextConfigArgs } from './sub-session-size.lib.mjs'; // Issue #1706
|
|
29
|
+
import { getCumulativeContextInputTokens } from './context-fill.lib.mjs';
|
|
29
30
|
import Decimal from 'decimal.js-light';
|
|
30
31
|
|
|
31
32
|
const CODEX_USAGE_FIELD_NAMES = ['input_tokens', 'cached_input_tokens', 'output_tokens', 'cache_write_tokens', 'cache_creation_input_tokens', 'reasoning_tokens', 'input_tokens_details.cached_tokens', 'input_tokens_details.cache_read_tokens', 'input_tokens_details.cache_write_tokens', 'input_tokens_details.cache_creation_tokens', 'input_tokens_details.cache_creation_input_tokens', 'output_tokens_details.reasoning_tokens'];
|
|
@@ -85,6 +86,7 @@ export const createCodexTokenUsage = requestedModelId => ({
|
|
|
85
86
|
respondedModelId: requestedModelId || null,
|
|
86
87
|
contextLimit: null,
|
|
87
88
|
outputLimit: null,
|
|
89
|
+
contextFillInputTokens: 0,
|
|
88
90
|
peakContextUsage: 0,
|
|
89
91
|
tokenFieldAvailability: createCodexTokenFieldAvailability(),
|
|
90
92
|
});
|
|
@@ -346,6 +348,13 @@ export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId =
|
|
|
346
348
|
if (turnContextUsage > (nextState.tokenUsage.peakContextUsage || 0)) {
|
|
347
349
|
nextState.tokenUsage.peakContextUsage = turnContextUsage;
|
|
348
350
|
}
|
|
351
|
+
const turnContextFill = getCumulativeContextInputTokens({
|
|
352
|
+
inputTokens: nonCachedInputTokens,
|
|
353
|
+
cacheWriteTokens,
|
|
354
|
+
});
|
|
355
|
+
if (turnContextFill > (nextState.tokenUsage.contextFillInputTokens || 0)) {
|
|
356
|
+
nextState.tokenUsage.contextFillInputTokens = turnContextFill;
|
|
357
|
+
}
|
|
349
358
|
|
|
350
359
|
const usageFieldSet = CODEX_USAGE_FIELD_NAMES.filter(fieldName => hasOwnPath(data.usage, fieldName));
|
|
351
360
|
if (usageFieldSet.length > 0) nextState.observedUsageFieldSets.push(usageFieldSet);
|
|
@@ -407,6 +416,7 @@ export const buildCodexResultModelUsage = (modelId, tokenUsage, pricingInfo = nu
|
|
|
407
416
|
outputTokens: tokenUsage.outputTokens || 0,
|
|
408
417
|
modelName: pricingInfo?.modelName || modelId,
|
|
409
418
|
modelInfo: pricingInfo?.modelInfo || null,
|
|
419
|
+
contextFillInputTokens: tokenUsage.contextFillInputTokens || getCumulativeContextInputTokens(tokenUsage),
|
|
410
420
|
peakContextUsage: tokenUsage.peakContextUsage || 0,
|
|
411
421
|
costUSD: pricingInfo?.totalCostUSD ?? null,
|
|
412
422
|
},
|
|
@@ -782,7 +792,18 @@ export const executeCodexCommand = async params => {
|
|
|
782
792
|
let interactiveHandler = null;
|
|
783
793
|
if (argv.interactiveMode && owner && repo && prNumber) {
|
|
784
794
|
await log('🔌 Interactive mode: Creating handler for real-time PR comments', { verbose: true });
|
|
785
|
-
interactiveHandler = createInteractiveHandler({
|
|
795
|
+
interactiveHandler = createInteractiveHandler({
|
|
796
|
+
owner,
|
|
797
|
+
repo,
|
|
798
|
+
prNumber,
|
|
799
|
+
$,
|
|
800
|
+
log,
|
|
801
|
+
verbose: argv.verbose,
|
|
802
|
+
// Issue #1745: pass the three independent dangerous-skip flags so the
|
|
803
|
+
// comment-posting path can honor them. All default to false.
|
|
804
|
+
skipOutputSanitization: argv['dangerously-skip-output-sanitization'] === true,
|
|
805
|
+
skipActiveTokensOutputSanitization: argv['dangerously-skip-active-tokens-output-sanitization'] === true,
|
|
806
|
+
});
|
|
786
807
|
} else if (argv.interactiveMode) {
|
|
787
808
|
await log('⚠️ Interactive mode: Disabled - missing PR info (owner/repo/prNumber)', { verbose: true });
|
|
788
809
|
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// Shared context-window fill helpers.
|
|
2
|
+
|
|
3
|
+
const TOKEN_FIELD_ALIASES = {
|
|
4
|
+
input: ['inputTokens', 'input_tokens', 'input', 'promptTokens', 'prompt_tokens', 'prompt'],
|
|
5
|
+
output: ['outputTokens', 'output_tokens', 'output', 'completionTokens', 'completion_tokens', 'completion'],
|
|
6
|
+
cacheWrite: ['cacheCreationTokens', 'cacheWriteTokens', 'cacheCreationInputTokens', 'cache_creation_input_tokens', 'cache_write_tokens', 'cacheWrite'],
|
|
7
|
+
cacheRead: ['cacheReadTokens', 'cacheReadInputTokens', 'cache_read_input_tokens', 'cache_read_tokens', 'cachedInputTokens', 'cached_input_tokens', 'cacheRead'],
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export const toTokenCount = value => {
|
|
11
|
+
if (Number.isFinite(value)) return Math.max(0, value);
|
|
12
|
+
if (typeof value === 'string' && value.trim()) {
|
|
13
|
+
const parsed = Number(value);
|
|
14
|
+
if (Number.isFinite(parsed)) return Math.max(0, parsed);
|
|
15
|
+
}
|
|
16
|
+
return 0;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const getFirstTokenField = (usage, fieldNames) => {
|
|
20
|
+
if (!usage || typeof usage !== 'object') return 0;
|
|
21
|
+
for (const fieldName of fieldNames) {
|
|
22
|
+
if (Object.hasOwn(usage, fieldName)) return toTokenCount(usage[fieldName]);
|
|
23
|
+
}
|
|
24
|
+
return 0;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export const getInputTokenCount = usage => getFirstTokenField(usage, TOKEN_FIELD_ALIASES.input);
|
|
28
|
+
|
|
29
|
+
export const getOutputTokenCount = usage => getFirstTokenField(usage, TOKEN_FIELD_ALIASES.output);
|
|
30
|
+
|
|
31
|
+
export const getCacheWriteTokenCount = usage => {
|
|
32
|
+
const direct = getFirstTokenField(usage, TOKEN_FIELD_ALIASES.cacheWrite);
|
|
33
|
+
if (direct > 0 || !usage?.cache || typeof usage.cache !== 'object') return direct;
|
|
34
|
+
return toTokenCount(usage.cache.write);
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export const getCacheReadTokenCount = usage => {
|
|
38
|
+
const direct = getFirstTokenField(usage, TOKEN_FIELD_ALIASES.cacheRead);
|
|
39
|
+
if (direct > 0 || !usage?.cache || typeof usage.cache !== 'object') return direct;
|
|
40
|
+
return toTokenCount(usage.cache.read);
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Issue #1741: context-fill from cumulative/session usage.
|
|
45
|
+
*
|
|
46
|
+
* Cache reads are intentionally excluded. They are the same cached prefix replayed
|
|
47
|
+
* across requests, so summing them in a cumulative row can exceed the model's
|
|
48
|
+
* context window even though no single sub-session filled that much context.
|
|
49
|
+
*/
|
|
50
|
+
export const getCumulativeContextInputTokens = usage => getInputTokenCount(usage) + getCacheWriteTokenCount(usage);
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Issue #1737: restored prompt size for one concrete request/turn.
|
|
54
|
+
*
|
|
55
|
+
* Use this only when the source row is a single request or a tool-specific
|
|
56
|
+
* per-turn value. For cumulative model rows, use getCumulativeContextInputTokens.
|
|
57
|
+
*/
|
|
58
|
+
export const getRestoredContextInputTokens = usage => getInputTokenCount(usage) + getCacheWriteTokenCount(usage) + getCacheReadTokenCount(usage);
|
|
59
|
+
|
|
60
|
+
export const getExplicitContextFillInputTokens = usage => {
|
|
61
|
+
if (!usage || typeof usage !== 'object') return null;
|
|
62
|
+
if (Object.hasOwn(usage, 'contextFillInputTokens')) return toTokenCount(usage.contextFillInputTokens);
|
|
63
|
+
if (Object.hasOwn(usage, 'cumulativeContextInputTokens')) return toTokenCount(usage.cumulativeContextInputTokens);
|
|
64
|
+
return null;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
export const getDisplayContextInputTokens = usage => {
|
|
68
|
+
const explicitContextFill = getExplicitContextFillInputTokens(usage);
|
|
69
|
+
if (explicitContextFill !== null) return explicitContextFill;
|
|
70
|
+
return toTokenCount(usage?.peakContextUsage);
|
|
71
|
+
};
|
package/src/gemini.lib.mjs
CHANGED
|
@@ -20,6 +20,7 @@ import { sanitizeObjectStrings } from './unicode-sanitization.lib.mjs';
|
|
|
20
20
|
import { defaultModels, geminiModels } from './models/index.mjs';
|
|
21
21
|
import { checkPlaywrightMcpPackageAvailability } from './playwright-mcp.lib.mjs';
|
|
22
22
|
import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
|
|
23
|
+
import { getCumulativeContextInputTokens, toTokenCount } from './context-fill.lib.mjs';
|
|
23
24
|
|
|
24
25
|
const shellQuote = value => `"${String(value).replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`;
|
|
25
26
|
|
|
@@ -46,20 +47,34 @@ const extractGeminiTextContent = value => {
|
|
|
46
47
|
return '';
|
|
47
48
|
};
|
|
48
49
|
|
|
49
|
-
const
|
|
50
|
+
const pickTokenValue = (...values) => {
|
|
51
|
+
for (const value of values) {
|
|
52
|
+
if (value !== undefined && value !== null) return toTokenCount(value);
|
|
53
|
+
}
|
|
54
|
+
return 0;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export const buildGeminiResultModelUsage = (modelId, stats = null) => {
|
|
50
58
|
const modelStats = stats?.models && typeof stats.models === 'object' ? stats.models : null;
|
|
51
59
|
if (modelStats) {
|
|
52
60
|
const usage = {};
|
|
53
61
|
for (const [id, data] of Object.entries(modelStats)) {
|
|
54
62
|
const tokens = data?.tokens || {};
|
|
63
|
+
const inputTokens = pickTokenValue(tokens.input, tokens.prompt);
|
|
64
|
+
const cacheCreationTokens = pickTokenValue(tokens.cacheWrite, tokens.cache_write, tokens.cacheCreationTokens);
|
|
65
|
+
const cacheReadTokens = pickTokenValue(tokens.cacheRead, tokens.cache_read, tokens.cacheReadTokens);
|
|
66
|
+
const outputTokens = pickTokenValue(tokens.output, tokens.completion);
|
|
67
|
+
const contextLimit = pickTokenValue(tokens.contextLimit, tokens.context_limit, data?.contextLimit, data?.limit?.context);
|
|
68
|
+
const outputLimit = pickTokenValue(tokens.outputLimit, tokens.output_limit, data?.outputLimit, data?.limit?.output);
|
|
55
69
|
usage[id] = {
|
|
56
|
-
inputTokens
|
|
57
|
-
cacheCreationTokens
|
|
58
|
-
cacheReadTokens
|
|
59
|
-
outputTokens
|
|
70
|
+
inputTokens,
|
|
71
|
+
cacheCreationTokens,
|
|
72
|
+
cacheReadTokens,
|
|
73
|
+
outputTokens,
|
|
60
74
|
modelName: data?.name || id,
|
|
61
|
-
modelInfo: null,
|
|
62
|
-
|
|
75
|
+
modelInfo: contextLimit || outputLimit ? { limit: { context: contextLimit || null, output: outputLimit || null } } : null,
|
|
76
|
+
contextFillInputTokens: getCumulativeContextInputTokens({ inputTokens, cacheCreationTokens }),
|
|
77
|
+
peakContextUsage: pickTokenValue(tokens.total),
|
|
63
78
|
costUSD: null,
|
|
64
79
|
};
|
|
65
80
|
}
|
package/src/github.lib.mjs
CHANGED
|
@@ -6,8 +6,8 @@ import { log, maskToken, cleanErrorMessage, isENOSPC, ghCmdRetry } from './lib.m
|
|
|
6
6
|
import { reportError } from './sentry.lib.mjs';
|
|
7
7
|
import { githubLimits, timeouts } from './config.lib.mjs';
|
|
8
8
|
import { batchCheckPullRequestsForIssues as batchCheckPRs, batchCheckArchivedRepositories as batchCheckArchived } from './github.batch.lib.mjs';
|
|
9
|
-
import { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeLogContent } from './token-sanitization.lib.mjs';
|
|
10
|
-
export { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeLogContent }; // Re-export for backward compatibility
|
|
9
|
+
import { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeOutput, sanitizeLogContent } from './token-sanitization.lib.mjs';
|
|
10
|
+
export { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeOutput, sanitizeLogContent }; // Re-export for backward compatibility
|
|
11
11
|
import { uploadLogWithGhUploadLog } from './log-upload.lib.mjs';
|
|
12
12
|
import { formatResetTimeWithRelative } from './usage-limit.lib.mjs'; // See: https://github.com/link-assistant/hive-mind/issues/1236
|
|
13
13
|
// Import model info helpers (Issue #1225)
|