@link-assistant/hive-mind 1.64.1 → 1.64.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,28 @@
1
1
  # @link-assistant/hive-mind
2
2
 
3
+ ## 1.64.3
4
+
5
+ ### Patch Changes
6
+
7
+ - dd52682: Sanitize all user-facing output to prevent token leaks (#1745).
8
+ - All comment-posting paths (`postComment`, `editComment`, `postTrackedComment`) run bodies through `sanitizeOutput` (canonical name) / `sanitizeCommentBody` (active-token wrapper). `sanitizeLogContent` is preserved as a backward-compatible alias.
9
+ - `KNOWN_LOCAL_TOKEN_ENV_VARS` registry masks tokens by exact env value (Telegram, GitHub, Anthropic/Claude, OpenAI/Codex, Gemini/Google, Qwen/Dashscope, OpenCode, AgentCLI, HuggingFace).
10
+ - Three independent CLI flags: `--dangerously-skip-output-sanitization`, `--dangerously-skip-code-output-sanitization`, `--dangerously-skip-active-tokens-output-sanitization` — all default false; active-tokens skip stays separate so the broad skip flag still keeps active-token masking on.
11
+ - Process-wide sanitization counters (`getSanitizationStats`, `formatSanitizationSummary`) print a one-line summary at the end of each run with a hint to use `--dangerously-skip-output-sanitization` when masking blocks the user's workflow.
12
+ - `extractTokensFromUserContent` carve-out helper: tokens already present in user-provided content (issue body, non-bot comments, pre-existing code) are passed as `excludeTokens` so the sanitizer leaves them untouched while still masking active local tokens.
13
+ - Post-finish sweep (`runPostFinishSweep`) re-reads bot-authored PR comments and the PR description after the AI session completes and edits in place if a leak slipped past the live sanitizer.
14
+ - ESLint guardrail (`gh-rate-limit/require-sanitized-output`) flags raw `gh pr comment`, `gh issue comment`, `gh pr edit`, and `gh api .../comments` calls that bypass the sanitizer.
15
+ - Out-of-band Telegram leak DM with masked summaries when a known-local token is detected in an outbound comment.
16
+ - Hidden owner-only `/tokens` Telegram command lists configured tokens (always masked, private chat only).
17
+ - `maskToken` defaults to 3+3 characters per issue requirements.
18
+ - secretlint preset (best-of-breed) runs alongside our custom patterns; mismatch warnings surface gaps.
19
+
20
+ ## 1.64.2
21
+
22
+ ### Patch Changes
23
+
24
+ - 320ca42: Fix budget stats sub-agent context-fill calculation so cumulative-only rows (e.g. Claude Haiku 4.5 sub-agent calls that never appear in the parent JSONL) use `input + cache_creation` instead of `input + cache_creation + cache_read`. The previous formula double-counted the cached prefix replayed across calls and produced impossible percentages such as `1.2M / 200K (583%)`.
25
+
3
26
  ## 1.64.1
4
27
 
5
28
  ### Patch Changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@link-assistant/hive-mind",
3
- "version": "1.64.1",
3
+ "version": "1.64.3",
4
4
  "description": "AI-powered issue solver and hive mind for collaborative problem solving",
5
5
  "main": "src/hive.mjs",
6
6
  "type": "module",
@@ -11,6 +11,7 @@ import { resolveCodexReasoningEffort } from './codex.options.lib.mjs';
11
11
  import { mapModelForTool } from './models/index.mjs';
12
12
  import { buildCodexDisable1mContextConfigArgs, buildCodexSubSessionSizeConfigArgs, parseSubSessionSize } from './sub-session-size.lib.mjs';
13
13
  import { detectUsageLimit } from './usage-limit.lib.mjs';
14
+ import { getCacheReadTokenCount, getCumulativeContextInputTokens, getOutputTokenCount } from './context-fill.lib.mjs';
14
15
 
15
16
  export const AGENT_COMMANDER_TOOLS = new Set(['claude', 'codex', 'opencode', 'agent', 'qwen', 'gemini']);
16
17
 
@@ -222,10 +223,45 @@ const extractResultSummary = (messages, plainOutput) => {
222
223
 
223
224
  const hasErrorMessage = messages => messages.some(message => message?.is_error === true || message?.type === 'error' || message?.type === 'step_error' || message?.error);
224
225
 
226
+ const normalizeAgentCommanderTokenUsage = usage => {
227
+ if (!usage || typeof usage !== 'object') return null;
228
+ const normalized = {
229
+ ...usage,
230
+ contextFillInputTokens: usage.contextFillInputTokens ?? getCumulativeContextInputTokens(usage),
231
+ };
232
+ const cacheReadTokens = getCacheReadTokenCount(normalized);
233
+ const hasTokenCounts = getCumulativeContextInputTokens(normalized) > 0 || getOutputTokenCount(normalized) > 0 || cacheReadTokens > 0;
234
+ if (!hasTokenCounts) return null;
235
+ if (!normalized.stepCount) normalized.stepCount = 1;
236
+ return normalized;
237
+ };
238
+
239
+ const enrichPricingInfoWithTokenUsage = ({ pricingInfo = null, usage = null, tool = null, publicPricingEstimate = null }) => {
240
+ const tokenUsage = normalizeAgentCommanderTokenUsage(pricingInfo?.tokenUsage || usage);
241
+ if (!tokenUsage) return pricingInfo || null;
242
+
243
+ return {
244
+ source: 'agent-commander',
245
+ ...(pricingInfo || {}),
246
+ provider: pricingInfo?.provider || tool || 'agent-commander',
247
+ modelId: pricingInfo?.modelId || tokenUsage.respondedModelId || tokenUsage.requestedModelId || null,
248
+ modelName: pricingInfo?.modelName || tokenUsage.respondedModelId || tokenUsage.requestedModelId || null,
249
+ totalCostUSD: pricingInfo?.totalCostUSD ?? publicPricingEstimate ?? null,
250
+ tokenUsage,
251
+ };
252
+ };
253
+
225
254
  export const summarizeAgentCommanderResult = ({ result, tool }) => {
226
255
  const plainOutput = result?.output?.plain || '';
227
256
  if (result?.metadata && typeof result.metadata === 'object') {
228
257
  const metadata = result.metadata;
258
+ const streamTokenUsage = metadata.streamTokenUsage || result.usage || null;
259
+ const pricingInfo = enrichPricingInfoWithTokenUsage({
260
+ pricingInfo: metadata.pricingInfo || null,
261
+ usage: streamTokenUsage,
262
+ tool,
263
+ publicPricingEstimate: metadata.publicPricingEstimate ?? metadata.pricingInfo?.totalCostUSD ?? null,
264
+ });
229
265
  return {
230
266
  success: metadata.success === true,
231
267
  sessionId: metadata.sessionId || result.sessionId || null,
@@ -233,11 +269,11 @@ export const summarizeAgentCommanderResult = ({ result, tool }) => {
233
269
  limitResetTime: metadata.limitResetTime || null,
234
270
  limitTimezone: metadata.limitTimezone || null,
235
271
  anthropicTotalCostUSD: metadata.anthropicTotalCostUSD ?? null,
236
- publicPricingEstimate: metadata.publicPricingEstimate ?? metadata.pricingInfo?.totalCostUSD ?? null,
237
- pricingInfo: metadata.pricingInfo || null,
272
+ publicPricingEstimate: metadata.publicPricingEstimate ?? pricingInfo?.totalCostUSD ?? null,
273
+ pricingInfo,
238
274
  resultSummary: metadata.resultSummary || null,
239
275
  resultModelUsage: metadata.resultModelUsage || null,
240
- streamTokenUsage: metadata.streamTokenUsage || result.usage || null,
276
+ streamTokenUsage,
241
277
  subAgentCalls: metadata.subAgentCalls || null,
242
278
  errorDuringExecution: metadata.errorDuringExecution === true || result?.exitCode !== 0,
243
279
  result: plainOutput,
@@ -250,6 +286,12 @@ export const summarizeAgentCommanderResult = ({ result, tool }) => {
250
286
  const resultMessage = [...messages].reverse().find(message => message?.type === 'result') || null;
251
287
  const totalCost = typeof resultMessage?.total_cost_usd === 'number' ? resultMessage.total_cost_usd : null;
252
288
  const publicPricingEstimate = tool === 'agent' && typeof usage?.totalCost === 'number' ? usage.totalCost : null;
289
+ const pricingInfo = enrichPricingInfoWithTokenUsage({
290
+ pricingInfo: publicPricingEstimate !== null ? { totalCostUSD: publicPricingEstimate, source: 'agent-commander' } : null,
291
+ usage,
292
+ tool,
293
+ publicPricingEstimate,
294
+ });
253
295
 
254
296
  return {
255
297
  success: result?.exitCode === 0 && !usageLimit.isUsageLimit && !hasErrorMessage(messages),
@@ -258,8 +300,8 @@ export const summarizeAgentCommanderResult = ({ result, tool }) => {
258
300
  limitResetTime: usageLimit.resetTime,
259
301
  limitTimezone: usageLimit.timezone,
260
302
  anthropicTotalCostUSD: tool === 'claude' ? totalCost : null,
261
- publicPricingEstimate,
262
- pricingInfo: publicPricingEstimate !== null ? { totalCostUSD: publicPricingEstimate, source: 'agent-commander' } : null,
303
+ publicPricingEstimate: publicPricingEstimate ?? pricingInfo?.totalCostUSD ?? null,
304
+ pricingInfo,
263
305
  resultSummary: extractResultSummary(messages, plainOutput),
264
306
  resultModelUsage: null,
265
307
  streamTokenUsage: usage,
@@ -2,6 +2,7 @@
2
2
 
3
3
  import Decimal from 'decimal.js-light';
4
4
  import { sanitizeObjectStrings } from './unicode-sanitization.lib.mjs';
5
+ import { getCumulativeContextInputTokens, getRestoredContextInputTokens } from './context-fill.lib.mjs';
5
6
 
6
7
  export const createTokenFieldAvailability = () => ({
7
8
  inputTokens: false,
@@ -23,6 +24,7 @@ export const createAgentTokenUsage = () => ({
23
24
  respondedModelId: null,
24
25
  contextLimit: null,
25
26
  outputLimit: null,
27
+ contextFillInputTokens: 0,
26
28
  peakContextUsage: 0,
27
29
  tokenFieldAvailability: createTokenFieldAvailability(),
28
30
  });
@@ -61,10 +63,22 @@ export const accumulateAgentStepFinishUsage = (usage, data) => {
61
63
  if (data.part.model.respondedModelID) usage.respondedModelId = data.part.model.respondedModelID;
62
64
  }
63
65
 
66
+ const stepContextFill = getCumulativeContextInputTokens({
67
+ inputTokens: getTokenCount(tokens.input),
68
+ cacheWriteTokens: getTokenCount(tokens.cache?.write),
69
+ });
70
+ if (stepContextFill > (usage.contextFillInputTokens || 0)) {
71
+ usage.contextFillInputTokens = stepContextFill;
72
+ }
73
+
64
74
  if (data.part.context) {
65
75
  if (data.part.context.contextLimit) usage.contextLimit = data.part.context.contextLimit;
66
76
  if (data.part.context.outputLimit) usage.outputLimit = data.part.context.outputLimit;
67
- const stepContextUsage = getTokenCount(tokens.input) + getTokenCount(tokens.cache?.read);
77
+ const stepContextUsage = getRestoredContextInputTokens({
78
+ inputTokens: getTokenCount(tokens.input),
79
+ cacheWriteTokens: getTokenCount(tokens.cache?.write),
80
+ cacheReadTokens: getTokenCount(tokens.cache?.read),
81
+ });
68
82
  if (stepContextUsage > (usage.peakContextUsage || 0)) {
69
83
  usage.peakContextUsage = stepContextUsage;
70
84
  }
@@ -4,6 +4,9 @@
4
4
 
5
5
  import { formatNumber } from './claude.lib.mjs';
6
6
  import Decimal from 'decimal.js-light';
7
+ import { getCacheReadTokenCount, getCacheWriteTokenCount, getCumulativeContextInputTokens, getDisplayContextInputTokens, getExplicitContextFillInputTokens, getInputTokenCount, getOutputTokenCount, getRestoredContextInputTokens } from './context-fill.lib.mjs';
8
+
9
+ export { getCumulativeContextInputTokens, getRestoredContextInputTokens };
7
10
 
8
11
  /**
9
12
  * Helper: creates a fresh sub-session usage object for tracking tokens between compactification events
@@ -19,9 +22,9 @@ export const createEmptySubSessionUsage = () => ({
19
22
  peakOutputUsage: 0,
20
23
  });
21
24
 
22
- export const getRawRequestInputTokens = usage => (usage?.input_tokens || 0) + (usage?.cache_creation_input_tokens || 0) + (usage?.cache_read_input_tokens || 0);
25
+ export const getRawRequestInputTokens = usage => getRestoredContextInputTokens(usage);
23
26
 
24
- export const getUsageInputTokens = usage => (usage?.inputTokens || 0) + (usage?.cacheCreationTokens || 0) + (usage?.cacheReadTokens || 0);
27
+ export const getUsageInputTokens = usage => getRestoredContextInputTokens(usage);
25
28
 
26
29
  /**
27
30
  * Helper: accumulates token usage from a JSONL entry into a model usage map
@@ -184,6 +187,7 @@ export const dumpBudgetTrace = async (usage, tokenUsage, log) => {
184
187
  const reads = usage.cacheReadTokens || 0;
185
188
  const inputs = usage.inputTokens || 0;
186
189
  const outputs = usage.outputTokens || 0;
190
+ const explicitContextFill = getExplicitContextFillInputTokens(usage);
187
191
  const webSearches = usage.webSearchRequests || 0;
188
192
  const subSessionCount = (tokenUsage?.subSessions || []).length;
189
193
  const source = usage._sourceResultJson ? 'jsonl + result-event' : 'jsonl';
@@ -194,6 +198,14 @@ export const dumpBudgetTrace = async (usage, tokenUsage, log) => {
194
198
  // buckets split for cost and accounting review.
195
199
  await log(` peak input: ${formatNumber(peak)}${limit.context ? ` / ${formatNumber(limit.context)} context` : ''} (largest request input + cache_creation + cache_read)`, { verbose: true });
196
200
  await log(` cumulative: input ${formatNumber(inputs)}, cache_write ${formatNumber(writes)} (5m ${formatNumber(writes5m)} / 1h ${formatNumber(writes1h)}), cache_read ${formatNumber(reads)}, output ${formatNumber(outputs)}`, { verbose: true });
201
+ // Issue #1741: when peak is 0 (sub-agent only seen via result event), the
202
+ // detail row falls back to the cumulative-context proxy `input + cache_write`
203
+ // (cache_read is excluded because it represents the same cached prefix replayed
204
+ // across calls and would inflate the percentage past 100%).
205
+ if (explicitContextFill !== null || peak === 0) {
206
+ const contextFill = explicitContextFill ?? getCumulativeContextInputTokens(usage);
207
+ await log(` context fill: ${formatNumber(contextFill)}${limit.context ? ` / ${formatNumber(limit.context)} context` : ''} (input + cache_write; cache_read excluded — issue #1741)`, { verbose: true });
208
+ }
197
209
  // Issue #1710 R1: web_search is now billed in calculateModelCost. The trace
198
210
  // still surfaces the implied dollar cost so the residual remains debuggable
199
211
  // from the saved log even if a future model lacks pricing data.
@@ -305,17 +317,25 @@ export const mergeResultModelUsage = (modelUsage, resultModelUsage) => {
305
317
  if (!resultModelUsage || typeof resultModelUsage !== 'object') return;
306
318
  for (const [modelId, resultUsage] of Object.entries(resultModelUsage)) {
307
319
  if (modelId.startsWith('<') && modelId.endsWith('>')) continue;
320
+ const inputTokens = getInputTokenCount(resultUsage);
321
+ const cacheCreationTokens = getCacheWriteTokenCount(resultUsage);
322
+ const cacheReadTokens = getCacheReadTokenCount(resultUsage);
323
+ const outputTokens = getOutputTokenCount(resultUsage);
324
+ const explicitContextFill = getExplicitContextFillInputTokens(resultUsage);
308
325
  if (!modelUsage[modelId]) {
309
326
  modelUsage[modelId] = {
310
- inputTokens: resultUsage.inputTokens || 0,
311
- cacheCreationTokens: resultUsage.cacheCreationInputTokens || 0,
327
+ inputTokens,
328
+ cacheCreationTokens,
312
329
  cacheCreation5mTokens: 0,
313
330
  cacheCreation1hTokens: 0,
314
- cacheReadTokens: resultUsage.cacheReadInputTokens || 0,
315
- outputTokens: resultUsage.outputTokens || 0,
331
+ cacheReadTokens,
332
+ outputTokens,
316
333
  webSearchRequests: resultUsage.webSearchRequests || 0,
317
334
  _sourceResultJson: true,
318
335
  };
336
+ if (explicitContextFill !== null) {
337
+ modelUsage[modelId].contextFillInputTokens = explicitContextFill;
338
+ }
319
339
  if (resultUsage.costUSD != null) {
320
340
  modelUsage[modelId]._resultCostUSD = resultUsage.costUSD;
321
341
  }
@@ -331,13 +351,16 @@ export const mergeResultModelUsage = (modelUsage, resultModelUsage) => {
331
351
  } else {
332
352
  const jsonlUsage = modelUsage[modelId];
333
353
  const jsonlTotal = jsonlUsage.inputTokens + jsonlUsage.cacheCreationTokens + jsonlUsage.cacheReadTokens + jsonlUsage.outputTokens;
334
- const resultTotal = (resultUsage.inputTokens || 0) + (resultUsage.cacheCreationInputTokens || 0) + (resultUsage.cacheReadInputTokens || 0) + (resultUsage.outputTokens || 0);
354
+ const resultTotal = inputTokens + cacheCreationTokens + cacheReadTokens + outputTokens;
335
355
  if (resultTotal > jsonlTotal) {
336
- jsonlUsage.inputTokens = resultUsage.inputTokens || 0;
337
- jsonlUsage.cacheCreationTokens = resultUsage.cacheCreationInputTokens || 0;
338
- jsonlUsage.cacheReadTokens = resultUsage.cacheReadInputTokens || 0;
339
- jsonlUsage.outputTokens = resultUsage.outputTokens || 0;
356
+ jsonlUsage.inputTokens = inputTokens;
357
+ jsonlUsage.cacheCreationTokens = cacheCreationTokens;
358
+ jsonlUsage.cacheReadTokens = cacheReadTokens;
359
+ jsonlUsage.outputTokens = outputTokens;
340
360
  jsonlUsage._sourceResultJson = true;
361
+ if (explicitContextFill !== null) {
362
+ jsonlUsage.contextFillInputTokens = explicitContextFill;
363
+ }
341
364
  }
342
365
  if (resultUsage.costUSD != null) {
343
366
  jsonlUsage._resultCostUSD = resultUsage.costUSD;
@@ -573,7 +596,7 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
573
596
  stats += `\n\n**${modelName}:** (${subSessions.length} sub-sessions)`;
574
597
  }
575
598
 
576
- const peakContext = usage.peakContextUsage || 0;
599
+ const peakContext = getDisplayContextInputTokens(usage);
577
600
 
578
601
  if (showSubSessions) {
579
602
  // Issue #1600: Unified format — no "Context window:" prefix, same format as sub-agent calls
@@ -587,10 +610,14 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
587
610
  // so peakContext stays at 0; without this fallback the rendered comment loses
588
611
  // the sub-agent's input-token information entirely. The detail line is
589
612
  // deliberately simple; the Total line below keeps the cache split.
613
+ // Issue #1741: For result-event-only rows we have cumulative totals, not a
614
+ // per-request peak, so the detail-line numerator must exclude cache_reads
615
+ // (which are the same cached prefix replayed across calls and would inflate
616
+ // the percentage past 100%). The Total line keeps the full split.
590
617
  const parts = [];
591
618
  const isResultSingleCall = usage._sourceResultJson || callCount > 0;
592
619
  const inputPart = isResultSingleCall
593
- ? formatInputContextPart(getUsageInputTokens(usage), contextLimit, formatTokensCompact)
620
+ ? formatInputContextPart(getCumulativeContextInputTokens(usage), contextLimit, formatTokensCompact)
594
621
  : buildCumulativeInputPhrase({
595
622
  input: usage.inputTokens || 0,
596
623
  cacheWrites: usage.cacheCreationTokens || 0,
@@ -636,7 +663,12 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
636
663
  for (let i = 0; i < matchingCalls.length; i++) {
637
664
  const call = matchingCalls[i];
638
665
  const cu = call.usage || {};
639
- const callInput = (cu.inputTokens || 0) + (cu.cacheCreationTokens || 0) + (cu.cacheReadTokens || 0);
666
+ // Issue #1741: per-call usage is itself cumulative across the
667
+ // sub-agent's internal API requests (each Anthropic Agent call
668
+ // can run a tool loop), so cache_reads grow with the loop length
669
+ // and would push the displayed fill past 100%. Use the same
670
+ // input + cache_creation proxy as the result-event-only fallback.
671
+ const callInput = getCumulativeContextInputTokens(cu);
640
672
  const callOutput = cu.outputTokens || 0;
641
673
  const parts = [];
642
674
  if (contextLimit) {
@@ -655,9 +687,13 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
655
687
  }
656
688
  } else {
657
689
  // Estimated per-call breakdown when sub-agent stream tracking did not capture
658
- // per-call usage. Includes everything the model actually saw:
659
- // input + cache_creation (writes) + cache_read.
660
- const aggregateInput = (usage.inputTokens || 0) + (usage.cacheCreationTokens || 0) + (usage.cacheReadTokens || 0);
690
+ // per-call usage. Issue #1741: cumulative cache_read tokens grow without
691
+ // bound across calls (the same cached prefix is replayed on every call),
692
+ // so we mustn't add them when projecting an average per-call fill
693
+ // doing so would routinely exceed 100% of the context window. The
694
+ // estimate uses input + cache_creation (cache reads stay in the Total
695
+ // line below).
696
+ const aggregateInput = getCumulativeContextInputTokens(usage);
661
697
  const avgInput = Math.round(aggregateInput / callCount);
662
698
  const avgOutput = Math.round(usage.outputTokens / callCount);
663
699
  for (let i = 0; i < matchingCalls.length; i++) {
@@ -696,7 +732,14 @@ export const buildBudgetStatsString = (tokenUsage, subAgentCalls = null) => {
696
732
  * @returns {Object|null} Budget stats data compatible with buildBudgetStatsString, or null if no data
697
733
  */
698
734
  export const buildAgentBudgetStats = (tokenUsage, pricingInfo) => {
699
- if (!tokenUsage || tokenUsage.stepCount === 0) return null;
735
+ if (!tokenUsage) return null;
736
+
737
+ const inputTokens = getInputTokenCount(tokenUsage);
738
+ const cacheWriteTokens = getCacheWriteTokenCount(tokenUsage);
739
+ const cacheReadTokens = getCacheReadTokenCount(tokenUsage);
740
+ const outputTokens = getOutputTokenCount(tokenUsage);
741
+ const hasTokens = inputTokens > 0 || cacheWriteTokens > 0 || cacheReadTokens > 0 || outputTokens > 0;
742
+ if ((tokenUsage.stepCount || 0) === 0 && !hasTokens) return null;
700
743
 
701
744
  const modelName = pricingInfo?.modelName || tokenUsage.respondedModelId || tokenUsage.requestedModelId || 'Unknown';
702
745
  const modelId = tokenUsage.respondedModelId || tokenUsage.requestedModelId || pricingInfo?.modelId || 'unknown';
@@ -704,14 +747,16 @@ export const buildAgentBudgetStats = (tokenUsage, pricingInfo) => {
704
747
  // Use context limits from step_finish events if available, otherwise from pricing model info
705
748
  const contextLimit = tokenUsage.contextLimit || pricingInfo?.modelInfo?.limit?.context || null;
706
749
  const outputLimit = tokenUsage.outputLimit || pricingInfo?.modelInfo?.limit?.output || null;
750
+ const contextFillInputTokens = getExplicitContextFillInputTokens(tokenUsage) ?? getCumulativeContextInputTokens({ inputTokens, cacheWriteTokens });
707
751
 
708
752
  const modelUsageEntry = {
709
- inputTokens: tokenUsage.inputTokens,
710
- cacheCreationTokens: tokenUsage.cacheWriteTokens || 0,
711
- cacheReadTokens: tokenUsage.cacheReadTokens || 0,
712
- outputTokens: tokenUsage.outputTokens,
753
+ inputTokens,
754
+ cacheCreationTokens: cacheWriteTokens,
755
+ cacheReadTokens,
756
+ outputTokens,
713
757
  modelName,
714
758
  modelInfo: contextLimit || outputLimit ? { limit: { context: contextLimit, output: outputLimit } } : null,
759
+ contextFillInputTokens,
715
760
  peakContextUsage: tokenUsage.peakContextUsage || 0,
716
761
  costUSD: pricingInfo?.totalCostUSD ?? null,
717
762
  };
@@ -719,11 +764,11 @@ export const buildAgentBudgetStats = (tokenUsage, pricingInfo) => {
719
764
  return {
720
765
  modelUsage: { [modelId]: modelUsageEntry },
721
766
  subSessions: [],
722
- inputTokens: tokenUsage.inputTokens,
723
- cacheCreationTokens: tokenUsage.cacheWriteTokens || 0,
724
- cacheReadTokens: tokenUsage.cacheReadTokens || 0,
725
- outputTokens: tokenUsage.outputTokens,
726
- totalTokens: tokenUsage.inputTokens + (tokenUsage.cacheWriteTokens || 0) + tokenUsage.outputTokens,
767
+ inputTokens,
768
+ cacheCreationTokens: cacheWriteTokens,
769
+ cacheReadTokens,
770
+ outputTokens,
771
+ totalTokens: inputTokens + cacheWriteTokens + outputTokens,
727
772
  };
728
773
  };
729
774
 
@@ -682,7 +682,18 @@ export const executeClaudeCommand = async params => {
682
682
  let interactiveHandler = null;
683
683
  if (argv.interactiveMode && owner && repo && prNumber) {
684
684
  await log('🔌 Interactive mode: Creating handler for real-time PR comments', { verbose: true });
685
- interactiveHandler = createInteractiveHandler({ owner, repo, prNumber, $, log, verbose: argv.verbose });
685
+ interactiveHandler = createInteractiveHandler({
686
+ owner,
687
+ repo,
688
+ prNumber,
689
+ $,
690
+ log,
691
+ verbose: argv.verbose,
692
+ // Issue #1745: thread the three independent dangerous-skip flags through
693
+ // so the comment-posting path can honor them; flags default to false.
694
+ skipOutputSanitization: argv['dangerously-skip-output-sanitization'] === true,
695
+ skipActiveTokensOutputSanitization: argv['dangerously-skip-active-tokens-output-sanitization'] === true,
696
+ });
686
697
  } else if (argv.interactiveMode) {
687
698
  await log('⚠️ Interactive mode: Disabled - missing PR info (owner/repo/prNumber)', { verbose: true });
688
699
  }
package/src/codex.lib.mjs CHANGED
@@ -26,6 +26,7 @@ import { fetchModelInfo } from './model-info.lib.mjs';
26
26
  import { defaultModels } from './models/index.mjs';
27
27
  import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
28
28
  import { parseSubSessionSize, buildCodexSubSessionSizeConfigArgs, buildCodexDisable1mContextConfigArgs } from './sub-session-size.lib.mjs'; // Issue #1706
29
+ import { getCumulativeContextInputTokens } from './context-fill.lib.mjs';
29
30
  import Decimal from 'decimal.js-light';
30
31
 
31
32
  const CODEX_USAGE_FIELD_NAMES = ['input_tokens', 'cached_input_tokens', 'output_tokens', 'cache_write_tokens', 'cache_creation_input_tokens', 'reasoning_tokens', 'input_tokens_details.cached_tokens', 'input_tokens_details.cache_read_tokens', 'input_tokens_details.cache_write_tokens', 'input_tokens_details.cache_creation_tokens', 'input_tokens_details.cache_creation_input_tokens', 'output_tokens_details.reasoning_tokens'];
@@ -85,6 +86,7 @@ export const createCodexTokenUsage = requestedModelId => ({
85
86
  respondedModelId: requestedModelId || null,
86
87
  contextLimit: null,
87
88
  outputLimit: null,
89
+ contextFillInputTokens: 0,
88
90
  peakContextUsage: 0,
89
91
  tokenFieldAvailability: createCodexTokenFieldAvailability(),
90
92
  });
@@ -346,6 +348,13 @@ export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId =
346
348
  if (turnContextUsage > (nextState.tokenUsage.peakContextUsage || 0)) {
347
349
  nextState.tokenUsage.peakContextUsage = turnContextUsage;
348
350
  }
351
+ const turnContextFill = getCumulativeContextInputTokens({
352
+ inputTokens: nonCachedInputTokens,
353
+ cacheWriteTokens,
354
+ });
355
+ if (turnContextFill > (nextState.tokenUsage.contextFillInputTokens || 0)) {
356
+ nextState.tokenUsage.contextFillInputTokens = turnContextFill;
357
+ }
349
358
 
350
359
  const usageFieldSet = CODEX_USAGE_FIELD_NAMES.filter(fieldName => hasOwnPath(data.usage, fieldName));
351
360
  if (usageFieldSet.length > 0) nextState.observedUsageFieldSets.push(usageFieldSet);
@@ -407,6 +416,7 @@ export const buildCodexResultModelUsage = (modelId, tokenUsage, pricingInfo = nu
407
416
  outputTokens: tokenUsage.outputTokens || 0,
408
417
  modelName: pricingInfo?.modelName || modelId,
409
418
  modelInfo: pricingInfo?.modelInfo || null,
419
+ contextFillInputTokens: tokenUsage.contextFillInputTokens || getCumulativeContextInputTokens(tokenUsage),
410
420
  peakContextUsage: tokenUsage.peakContextUsage || 0,
411
421
  costUSD: pricingInfo?.totalCostUSD ?? null,
412
422
  },
@@ -782,7 +792,18 @@ export const executeCodexCommand = async params => {
782
792
  let interactiveHandler = null;
783
793
  if (argv.interactiveMode && owner && repo && prNumber) {
784
794
  await log('🔌 Interactive mode: Creating handler for real-time PR comments', { verbose: true });
785
- interactiveHandler = createInteractiveHandler({ owner, repo, prNumber, $, log, verbose: argv.verbose });
795
+ interactiveHandler = createInteractiveHandler({
796
+ owner,
797
+ repo,
798
+ prNumber,
799
+ $,
800
+ log,
801
+ verbose: argv.verbose,
802
+ // Issue #1745: pass the three independent dangerous-skip flags so the
803
+ // comment-posting path can honor them. All default to false.
804
+ skipOutputSanitization: argv['dangerously-skip-output-sanitization'] === true,
805
+ skipActiveTokensOutputSanitization: argv['dangerously-skip-active-tokens-output-sanitization'] === true,
806
+ });
786
807
  } else if (argv.interactiveMode) {
787
808
  await log('⚠️ Interactive mode: Disabled - missing PR info (owner/repo/prNumber)', { verbose: true });
788
809
  }
@@ -0,0 +1,71 @@
1
+ // Shared context-window fill helpers.
2
+
3
+ const TOKEN_FIELD_ALIASES = {
4
+ input: ['inputTokens', 'input_tokens', 'input', 'promptTokens', 'prompt_tokens', 'prompt'],
5
+ output: ['outputTokens', 'output_tokens', 'output', 'completionTokens', 'completion_tokens', 'completion'],
6
+ cacheWrite: ['cacheCreationTokens', 'cacheWriteTokens', 'cacheCreationInputTokens', 'cache_creation_input_tokens', 'cache_write_tokens', 'cacheWrite'],
7
+ cacheRead: ['cacheReadTokens', 'cacheReadInputTokens', 'cache_read_input_tokens', 'cache_read_tokens', 'cachedInputTokens', 'cached_input_tokens', 'cacheRead'],
8
+ };
9
+
10
+ export const toTokenCount = value => {
11
+ if (Number.isFinite(value)) return Math.max(0, value);
12
+ if (typeof value === 'string' && value.trim()) {
13
+ const parsed = Number(value);
14
+ if (Number.isFinite(parsed)) return Math.max(0, parsed);
15
+ }
16
+ return 0;
17
+ };
18
+
19
+ const getFirstTokenField = (usage, fieldNames) => {
20
+ if (!usage || typeof usage !== 'object') return 0;
21
+ for (const fieldName of fieldNames) {
22
+ if (Object.hasOwn(usage, fieldName)) return toTokenCount(usage[fieldName]);
23
+ }
24
+ return 0;
25
+ };
26
+
27
+ export const getInputTokenCount = usage => getFirstTokenField(usage, TOKEN_FIELD_ALIASES.input);
28
+
29
+ export const getOutputTokenCount = usage => getFirstTokenField(usage, TOKEN_FIELD_ALIASES.output);
30
+
31
+ export const getCacheWriteTokenCount = usage => {
32
+ const direct = getFirstTokenField(usage, TOKEN_FIELD_ALIASES.cacheWrite);
33
+ if (direct > 0 || !usage?.cache || typeof usage.cache !== 'object') return direct;
34
+ return toTokenCount(usage.cache.write);
35
+ };
36
+
37
+ export const getCacheReadTokenCount = usage => {
38
+ const direct = getFirstTokenField(usage, TOKEN_FIELD_ALIASES.cacheRead);
39
+ if (direct > 0 || !usage?.cache || typeof usage.cache !== 'object') return direct;
40
+ return toTokenCount(usage.cache.read);
41
+ };
42
+
43
+ /**
44
+ * Issue #1741: context-fill from cumulative/session usage.
45
+ *
46
+ * Cache reads are intentionally excluded. They are the same cached prefix replayed
47
+ * across requests, so summing them in a cumulative row can exceed the model's
48
+ * context window even though no single sub-session filled that much context.
49
+ */
50
+ export const getCumulativeContextInputTokens = usage => getInputTokenCount(usage) + getCacheWriteTokenCount(usage);
51
+
52
+ /**
53
+ * Issue #1737: restored prompt size for one concrete request/turn.
54
+ *
55
+ * Use this only when the source row is a single request or a tool-specific
56
+ * per-turn value. For cumulative model rows, use getCumulativeContextInputTokens.
57
+ */
58
+ export const getRestoredContextInputTokens = usage => getInputTokenCount(usage) + getCacheWriteTokenCount(usage) + getCacheReadTokenCount(usage);
59
+
60
+ export const getExplicitContextFillInputTokens = usage => {
61
+ if (!usage || typeof usage !== 'object') return null;
62
+ if (Object.hasOwn(usage, 'contextFillInputTokens')) return toTokenCount(usage.contextFillInputTokens);
63
+ if (Object.hasOwn(usage, 'cumulativeContextInputTokens')) return toTokenCount(usage.cumulativeContextInputTokens);
64
+ return null;
65
+ };
66
+
67
+ export const getDisplayContextInputTokens = usage => {
68
+ const explicitContextFill = getExplicitContextFillInputTokens(usage);
69
+ if (explicitContextFill !== null) return explicitContextFill;
70
+ return toTokenCount(usage?.peakContextUsage);
71
+ };
@@ -20,6 +20,7 @@ import { sanitizeObjectStrings } from './unicode-sanitization.lib.mjs';
20
20
  import { defaultModels, geminiModels } from './models/index.mjs';
21
21
  import { checkPlaywrightMcpPackageAvailability } from './playwright-mcp.lib.mjs';
22
22
  import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
23
+ import { getCumulativeContextInputTokens, toTokenCount } from './context-fill.lib.mjs';
23
24
 
24
25
  const shellQuote = value => `"${String(value).replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`;
25
26
 
@@ -46,20 +47,34 @@ const extractGeminiTextContent = value => {
46
47
  return '';
47
48
  };
48
49
 
49
- const buildGeminiResultModelUsage = (modelId, stats = null) => {
50
+ const pickTokenValue = (...values) => {
51
+ for (const value of values) {
52
+ if (value !== undefined && value !== null) return toTokenCount(value);
53
+ }
54
+ return 0;
55
+ };
56
+
57
+ export const buildGeminiResultModelUsage = (modelId, stats = null) => {
50
58
  const modelStats = stats?.models && typeof stats.models === 'object' ? stats.models : null;
51
59
  if (modelStats) {
52
60
  const usage = {};
53
61
  for (const [id, data] of Object.entries(modelStats)) {
54
62
  const tokens = data?.tokens || {};
63
+ const inputTokens = pickTokenValue(tokens.input, tokens.prompt);
64
+ const cacheCreationTokens = pickTokenValue(tokens.cacheWrite, tokens.cache_write, tokens.cacheCreationTokens);
65
+ const cacheReadTokens = pickTokenValue(tokens.cacheRead, tokens.cache_read, tokens.cacheReadTokens);
66
+ const outputTokens = pickTokenValue(tokens.output, tokens.completion);
67
+ const contextLimit = pickTokenValue(tokens.contextLimit, tokens.context_limit, data?.contextLimit, data?.limit?.context);
68
+ const outputLimit = pickTokenValue(tokens.outputLimit, tokens.output_limit, data?.outputLimit, data?.limit?.output);
55
69
  usage[id] = {
56
- inputTokens: tokens.input || tokens.prompt || 0,
57
- cacheCreationTokens: tokens.cacheWrite || 0,
58
- cacheReadTokens: tokens.cacheRead || 0,
59
- outputTokens: tokens.output || tokens.completion || 0,
70
+ inputTokens,
71
+ cacheCreationTokens,
72
+ cacheReadTokens,
73
+ outputTokens,
60
74
  modelName: data?.name || id,
61
- modelInfo: null,
62
- peakContextUsage: tokens.total || 0,
75
+ modelInfo: contextLimit || outputLimit ? { limit: { context: contextLimit || null, output: outputLimit || null } } : null,
76
+ contextFillInputTokens: getCumulativeContextInputTokens({ inputTokens, cacheCreationTokens }),
77
+ peakContextUsage: pickTokenValue(tokens.total),
63
78
  costUSD: null,
64
79
  };
65
80
  }
@@ -6,8 +6,8 @@ import { log, maskToken, cleanErrorMessage, isENOSPC, ghCmdRetry } from './lib.m
6
6
  import { reportError } from './sentry.lib.mjs';
7
7
  import { githubLimits, timeouts } from './config.lib.mjs';
8
8
  import { batchCheckPullRequestsForIssues as batchCheckPRs, batchCheckArchivedRepositories as batchCheckArchived } from './github.batch.lib.mjs';
9
- import { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeLogContent } from './token-sanitization.lib.mjs';
10
- export { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeLogContent }; // Re-export for backward compatibility
9
+ import { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeOutput, sanitizeLogContent } from './token-sanitization.lib.mjs';
10
+ export { isSafeToken, isHexInSafeContext, getGitHubTokensFromFiles, getGitHubTokensFromCommand, sanitizeOutput, sanitizeLogContent }; // Re-export for backward compatibility
11
11
  import { uploadLogWithGhUploadLog } from './log-upload.lib.mjs';
12
12
  import { formatResetTimeWithRelative } from './usage-limit.lib.mjs'; // See: https://github.com/link-assistant/hive-mind/issues/1236
13
13
  // Import model info helpers (Issue #1225)