mixdog 0.9.3 → 0.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (329) hide show
  1. package/package.json +7 -3
  2. package/scripts/bench/lead-review-tasks-r3.json +20 -0
  3. package/scripts/bench/lead-review-tasks.json +20 -0
  4. package/scripts/bench/r4-mixed-tasks.json +20 -0
  5. package/scripts/bench/review-tasks.json +20 -0
  6. package/scripts/bench/round-codex.json +114 -0
  7. package/scripts/bench/round-mixdog-lead-r3.json +269 -0
  8. package/scripts/bench/round-mixdog-lead.json +269 -0
  9. package/scripts/bench/round-mixdog.json +126 -0
  10. package/scripts/bench/round-r10-bigsample.json +679 -0
  11. package/scripts/bench/round-r11-codexalign.json +257 -0
  12. package/scripts/bench/round-r4-codex.json +114 -0
  13. package/scripts/bench/round-r4-mixed.json +225 -0
  14. package/scripts/bench/round-r5-gpt-lead.json +259 -0
  15. package/scripts/bench/round-r6-codex.json +114 -0
  16. package/scripts/bench/round-r6-solo.json +257 -0
  17. package/scripts/bench/round-r7-full.json +254 -0
  18. package/scripts/bench/round-r8-fulldefault.json +255 -0
  19. package/scripts/bench-run.mjs +215 -29
  20. package/scripts/freevar-smoke.mjs +95 -0
  21. package/scripts/internal-comms-bench.mjs +1 -0
  22. package/scripts/internal-comms-smoke.mjs +10 -9
  23. package/scripts/mouse-probe.mjs +45 -0
  24. package/scripts/output-style-bench.mjs +13 -6
  25. package/scripts/output-style-smoke.mjs +4 -4
  26. package/scripts/provider-toolcall-test.mjs +7 -3
  27. package/scripts/recall-usecase-cases.json +18 -0
  28. package/scripts/recall-usecase-probe.json +6 -0
  29. package/scripts/session-bench.mjs +152 -6
  30. package/scripts/tool-smoke.mjs +23 -63
  31. package/scripts/tui-render-smoke.mjs +90 -0
  32. package/scripts/webhook-smoke.mjs +208 -0
  33. package/src/agents/debugger/AGENT.md +4 -1
  34. package/src/agents/heavy-worker/AGENT.md +6 -5
  35. package/src/agents/maintainer/AGENT.md +4 -0
  36. package/src/agents/reviewer/AGENT.md +2 -1
  37. package/src/agents/worker/AGENT.md +8 -4
  38. package/src/lib/rules-builder.cjs +4 -0
  39. package/src/mixdog-session-runtime.mjs +632 -2042
  40. package/src/output-styles/default.md +34 -9
  41. package/src/output-styles/{oneline.md → extreme-minimal.md} +5 -4
  42. package/src/output-styles/minimal.md +4 -1
  43. package/src/output-styles/simple.md +22 -7
  44. package/src/rules/agent/00-common.md +2 -0
  45. package/src/rules/lead/lead-brief.md +12 -0
  46. package/src/rules/lead/lead-tool.md +0 -11
  47. package/src/runtime/agent/orchestrator/agent-runtime/agent-loop-policy.mjs +25 -0
  48. package/src/runtime/agent/orchestrator/agent-runtime/cache-strategy.mjs +100 -23
  49. package/src/runtime/agent/orchestrator/agent-runtime/session-builder.mjs +6 -15
  50. package/src/runtime/agent/orchestrator/agent-trace-format.mjs +362 -0
  51. package/src/runtime/agent/orchestrator/agent-trace-io.mjs +410 -0
  52. package/src/runtime/agent/orchestrator/agent-trace.mjs +16 -735
  53. package/src/runtime/agent/orchestrator/config.mjs +69 -2
  54. package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +62 -20
  55. package/src/runtime/agent/orchestrator/providers/anthropic-model-resolve.mjs +209 -0
  56. package/src/runtime/agent/orchestrator/providers/anthropic-oauth-credentials.mjs +489 -0
  57. package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +81 -1281
  58. package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +607 -0
  59. package/src/runtime/agent/orchestrator/providers/anthropic.mjs +32 -3
  60. package/src/runtime/agent/orchestrator/providers/codex-client-meta.mjs +81 -0
  61. package/src/runtime/agent/orchestrator/providers/gemini-cache.mjs +248 -0
  62. package/src/runtime/agent/orchestrator/providers/gemini-schema.mjs +303 -0
  63. package/src/runtime/agent/orchestrator/providers/gemini-stream.mjs +505 -0
  64. package/src/runtime/agent/orchestrator/providers/gemini.mjs +43 -1013
  65. package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +17 -3
  66. package/src/runtime/agent/orchestrator/providers/model-catalog.mjs +86 -11
  67. package/src/runtime/agent/orchestrator/providers/model-list-sanitize.mjs +348 -0
  68. package/src/runtime/agent/orchestrator/providers/openai-codex-model.mjs +108 -0
  69. package/src/runtime/agent/orchestrator/providers/openai-compat-trace.mjs +58 -0
  70. package/src/runtime/agent/orchestrator/providers/openai-compat-wire.mjs +368 -0
  71. package/src/runtime/agent/orchestrator/providers/openai-compat-xai.mjs +760 -0
  72. package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +40 -1143
  73. package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +732 -0
  74. package/src/runtime/agent/orchestrator/providers/openai-oauth-login.mjs +193 -0
  75. package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +297 -2123
  76. package/src/runtime/agent/orchestrator/providers/openai-oauth.mjs +130 -1002
  77. package/src/runtime/agent/orchestrator/providers/openai-ws-delta.mjs +227 -0
  78. package/src/runtime/agent/orchestrator/providers/openai-ws-events.mjs +67 -0
  79. package/src/runtime/agent/orchestrator/providers/openai-ws-pool.mjs +436 -0
  80. package/src/runtime/agent/orchestrator/providers/openai-ws-stream.mjs +1105 -0
  81. package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +2 -1
  82. package/src/runtime/agent/orchestrator/session/compact/budget.mjs +288 -0
  83. package/src/runtime/agent/orchestrator/session/compact/constants.mjs +85 -0
  84. package/src/runtime/agent/orchestrator/session/compact/engine.mjs +749 -0
  85. package/src/runtime/agent/orchestrator/session/compact/messages.mjs +82 -0
  86. package/src/runtime/agent/orchestrator/session/compact/summary-schema.mjs +315 -0
  87. package/src/runtime/agent/orchestrator/session/compact/summary.mjs +643 -0
  88. package/src/runtime/agent/orchestrator/session/compact/text-utils.mjs +326 -0
  89. package/src/runtime/agent/orchestrator/session/compact.mjs +40 -2282
  90. package/src/runtime/agent/orchestrator/session/loop/compact-policy.mjs +14 -2
  91. package/src/runtime/agent/orchestrator/session/loop/completion-guards.mjs +61 -0
  92. package/src/runtime/agent/orchestrator/session/loop/pre-dispatch-deny.mjs +1 -3
  93. package/src/runtime/agent/orchestrator/session/loop/recall-fasttrack.mjs +182 -0
  94. package/src/runtime/agent/orchestrator/session/loop/steering-ladder.mjs +173 -0
  95. package/src/runtime/agent/orchestrator/session/loop/termination.mjs +58 -0
  96. package/src/runtime/agent/orchestrator/session/loop/tool-exec.mjs +239 -0
  97. package/src/runtime/agent/orchestrator/session/loop.mjs +251 -397
  98. package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs +471 -0
  99. package/src/runtime/agent/orchestrator/session/manager/context-meta.mjs +7 -4
  100. package/src/runtime/agent/orchestrator/session/manager/prompt-utils.mjs +12 -0
  101. package/src/runtime/agent/orchestrator/session/manager/runtime-liveness.mjs +406 -0
  102. package/src/runtime/agent/orchestrator/session/manager/status-telemetry.mjs +80 -0
  103. package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +210 -0
  104. package/src/runtime/agent/orchestrator/session/manager.mjs +166 -1087
  105. package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +189 -0
  106. package/src/runtime/agent/orchestrator/session/store.mjs +74 -179
  107. package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.mjs +70 -20
  108. package/src/runtime/agent/orchestrator/tools/builtin/bash-tool.mjs +22 -2
  109. package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +32 -41
  110. package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +40 -0
  111. package/src/runtime/agent/orchestrator/tools/builtin/rg-runner.mjs +29 -0
  112. package/src/runtime/agent/orchestrator/tools/builtin/search-builders.mjs +8 -0
  113. package/src/runtime/agent/orchestrator/tools/builtin/search-path-diagnostics.mjs +126 -0
  114. package/src/runtime/agent/orchestrator/tools/builtin/search-tool.mjs +81 -92
  115. package/src/runtime/agent/orchestrator/tools/builtin/shell-job-paths.mjs +161 -0
  116. package/src/runtime/agent/orchestrator/tools/builtin/shell-job-process.mjs +108 -0
  117. package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +28 -265
  118. package/src/runtime/agent/orchestrator/tools/builtin.mjs +0 -6
  119. package/src/runtime/agent/orchestrator/tools/code-graph/dispatch.mjs +57 -3
  120. package/src/runtime/agent/orchestrator/tools/code-graph/keyword-match.mjs +82 -0
  121. package/src/runtime/agent/orchestrator/tools/code-graph/search.mjs +10 -122
  122. package/src/runtime/agent/orchestrator/tools/code-graph/text-columns.mjs +45 -0
  123. package/src/runtime/agent/orchestrator/tools/code-graph-tool-defs.mjs +6 -6
  124. package/src/runtime/agent/orchestrator/tools/graph-binary-fetcher.mjs +6 -3
  125. package/src/runtime/agent/orchestrator/tools/patch/constants.mjs +9 -0
  126. package/src/runtime/agent/orchestrator/tools/patch/dispatch.mjs +171 -0
  127. package/src/runtime/agent/orchestrator/tools/patch/matcher.mjs +471 -0
  128. package/src/runtime/agent/orchestrator/tools/patch/native-server.mjs +436 -0
  129. package/src/runtime/agent/orchestrator/tools/patch/orchestrator.mjs +342 -0
  130. package/src/runtime/agent/orchestrator/tools/patch/parsing.mjs +359 -0
  131. package/src/runtime/agent/orchestrator/tools/patch/paths.mjs +340 -0
  132. package/src/runtime/agent/orchestrator/tools/patch/v4a-convert.mjs +643 -0
  133. package/src/runtime/agent/orchestrator/tools/patch.mjs +36 -2959
  134. package/src/runtime/agent/orchestrator/tools/progress-message.mjs +0 -21
  135. package/src/runtime/agent/orchestrator/tools/shell-command.mjs +9 -72
  136. package/src/runtime/agent/orchestrator/tools/shell-powershell.mjs +77 -0
  137. package/src/runtime/agent/orchestrator/tools/shell-state.mjs +154 -0
  138. package/src/runtime/channels/backends/discord-access.mjs +32 -0
  139. package/src/runtime/channels/backends/discord-attachments.mjs +65 -0
  140. package/src/runtime/channels/backends/discord-gateway.mjs +233 -0
  141. package/src/runtime/channels/backends/discord.mjs +12 -292
  142. package/src/runtime/channels/index.mjs +229 -663
  143. package/src/runtime/channels/lib/backend-dispatch.mjs +44 -0
  144. package/src/runtime/channels/lib/event-pipeline.mjs +18 -1
  145. package/src/runtime/channels/lib/event-queue.mjs +63 -4
  146. package/src/runtime/channels/lib/inbound-routing.mjs +111 -0
  147. package/src/runtime/channels/lib/output-forwarder.mjs +1 -1
  148. package/src/runtime/channels/lib/owner-heartbeat.mjs +75 -0
  149. package/src/runtime/channels/lib/parent-bridge.mjs +88 -0
  150. package/src/runtime/channels/lib/runtime-paths.mjs +14 -4
  151. package/src/runtime/channels/lib/session-discovery.mjs +56 -4
  152. package/src/runtime/channels/lib/tool-dispatch.mjs +158 -0
  153. package/src/runtime/channels/lib/tool-format.mjs +1 -1
  154. package/src/runtime/channels/lib/transcript-discovery.mjs +4 -4
  155. package/src/runtime/channels/lib/voice-runtime-fetcher.mjs +6 -3
  156. package/src/runtime/channels/lib/voice-transcription.mjs +179 -0
  157. package/src/runtime/channels/lib/webhook/deliveries.mjs +312 -0
  158. package/src/runtime/channels/lib/webhook/log.mjs +42 -0
  159. package/src/runtime/channels/lib/webhook/ngrok.mjs +181 -0
  160. package/src/runtime/channels/lib/webhook/signature.mjs +60 -0
  161. package/src/runtime/channels/lib/webhook.mjs +36 -570
  162. package/src/runtime/channels/tool-defs.mjs +11 -130
  163. package/src/runtime/memory/index.mjs +201 -1948
  164. package/src/runtime/memory/lib/cycle-llm-adapters.mjs +58 -0
  165. package/src/runtime/memory/lib/cycle-scheduler.mjs +497 -0
  166. package/src/runtime/memory/lib/embedding-warmup.mjs +58 -0
  167. package/src/runtime/memory/lib/memory-config-flags.mjs +91 -0
  168. package/src/runtime/memory/lib/memory-cycle.mjs +1 -1
  169. package/src/runtime/memory/lib/memory-cycle2-gate.mjs +515 -0
  170. package/src/runtime/memory/lib/memory-cycle2-mutations.mjs +324 -0
  171. package/src/runtime/memory/lib/memory-cycle2-shared.mjs +18 -0
  172. package/src/runtime/memory/lib/memory-cycle2.mjs +24 -842
  173. package/src/runtime/memory/lib/memory-embed.mjs +149 -0
  174. package/src/runtime/memory/lib/memory-process-lock.mjs +162 -0
  175. package/src/runtime/memory/lib/memory-recall-store.mjs +22 -2
  176. package/src/runtime/memory/lib/pg/supervisor.mjs +1 -1
  177. package/src/runtime/memory/lib/query-handlers.mjs +780 -0
  178. package/src/runtime/memory/lib/recall-format.mjs +55 -0
  179. package/src/runtime/memory/lib/runtime-fetcher.mjs +8 -3
  180. package/src/runtime/memory/lib/transcript-ingest.mjs +425 -0
  181. package/src/runtime/memory/tool-defs.mjs +5 -13
  182. package/src/runtime/search/lib/http-fetch.mjs +274 -0
  183. package/src/runtime/search/lib/ssrf-guard.mjs +333 -0
  184. package/src/runtime/search/lib/web-tools.mjs +24 -602
  185. package/src/runtime/shared/atomic-file.mjs +26 -1
  186. package/src/runtime/shared/launcher-control.mjs +2 -2
  187. package/src/runtime/shared/tool-primitives.mjs +308 -0
  188. package/src/runtime/shared/tool-result-summary.mjs +515 -0
  189. package/src/runtime/shared/tool-surface.mjs +80 -898
  190. package/src/runtime/shared/transcript-writer.mjs +23 -0
  191. package/src/runtime/shared/update-checker.mjs +7 -4
  192. package/src/session-runtime/config-helpers.mjs +84 -2
  193. package/src/session-runtime/config-lifecycle.mjs +232 -0
  194. package/src/session-runtime/cwd-plugins.mjs +226 -0
  195. package/src/session-runtime/mcp-glue.mjs +177 -0
  196. package/src/session-runtime/model-recency.mjs +111 -0
  197. package/src/session-runtime/native-search.mjs +247 -0
  198. package/src/session-runtime/output-styles.mjs +11 -9
  199. package/src/session-runtime/prewarm.mjs +142 -0
  200. package/src/session-runtime/provider-models.mjs +278 -0
  201. package/src/session-runtime/provider-usage.mjs +120 -0
  202. package/src/session-runtime/quick-model-rows.mjs +170 -0
  203. package/src/session-runtime/quick-search-models.mjs +46 -0
  204. package/src/session-runtime/session-hooks.mjs +93 -0
  205. package/src/session-runtime/settings-api.mjs +319 -0
  206. package/src/session-runtime/tool-catalog.mjs +29 -29
  207. package/src/session-runtime/tool-defs.mjs +84 -0
  208. package/src/session-runtime/warmup-schedulers.mjs +201 -0
  209. package/src/standalone/agent-tool/helpers.mjs +237 -0
  210. package/src/standalone/agent-tool/notify.mjs +107 -0
  211. package/src/standalone/agent-tool/provider-init.mjs +143 -0
  212. package/src/standalone/agent-tool/render.mjs +152 -0
  213. package/src/standalone/agent-tool/tool-def.mjs +55 -0
  214. package/src/standalone/agent-tool.mjs +110 -671
  215. package/src/standalone/channel-worker.mjs +4 -7
  216. package/src/standalone/explore-tool.mjs +30 -9
  217. package/src/standalone/hook-bus/config.mjs +207 -0
  218. package/src/standalone/hook-bus/constants.mjs +90 -0
  219. package/src/standalone/hook-bus/handlers.mjs +481 -0
  220. package/src/standalone/hook-bus/payload.mjs +31 -0
  221. package/src/standalone/hook-bus/rules.mjs +77 -0
  222. package/src/standalone/hook-bus.mjs +77 -870
  223. package/src/standalone/memory-runtime-proxy.mjs +7 -0
  224. package/src/standalone/opencode-go-login.mjs +5 -1
  225. package/src/standalone/provider-admin.mjs +1 -16
  226. package/src/standalone/usage-dashboard.mjs +3 -1
  227. package/src/tui/App.jsx +945 -8094
  228. package/src/tui/app/app-format.mjs +206 -0
  229. package/src/tui/app/channel-pickers.mjs +510 -0
  230. package/src/tui/app/clipboard.mjs +67 -0
  231. package/src/tui/app/core-memory-picker.mjs +210 -0
  232. package/src/tui/app/extension-pickers.mjs +506 -0
  233. package/src/tui/app/input-parsers.mjs +193 -0
  234. package/src/tui/app/maintenance-pickers.mjs +324 -0
  235. package/src/tui/app/model-options.mjs +330 -0
  236. package/src/tui/app/model-picker.mjs +365 -0
  237. package/src/tui/app/onboarding-steps.mjs +400 -0
  238. package/src/tui/app/project-picker.mjs +247 -0
  239. package/src/tui/app/provider-setup-picker.mjs +580 -0
  240. package/src/tui/app/resume-picker.mjs +55 -0
  241. package/src/tui/app/route-pickers.mjs +419 -0
  242. package/src/tui/app/settings-picker.mjs +490 -0
  243. package/src/tui/app/slash-commands.mjs +101 -0
  244. package/src/tui/app/slash-dispatch.mjs +427 -0
  245. package/src/tui/app/text-layout.mjs +46 -0
  246. package/src/tui/app/theme-effort-pickers.mjs +154 -0
  247. package/src/tui/app/transcript-window.mjs +671 -0
  248. package/src/tui/app/use-mouse-input.mjs +460 -0
  249. package/src/tui/app/use-prompt-handlers.mjs +310 -0
  250. package/src/tui/app/use-transcript-scroll.mjs +510 -0
  251. package/src/tui/app/use-transcript-window.mjs +589 -0
  252. package/src/tui/components/ConfirmBar.jsx +1 -1
  253. package/src/tui/components/Picker.jsx +32 -4
  254. package/src/tui/components/PromptInput.jsx +23 -101
  255. package/src/tui/components/SlashCommandPalette.jsx +8 -1
  256. package/src/tui/components/StatusLine.jsx +63 -12
  257. package/src/tui/components/TextEntryPanel.jsx +11 -0
  258. package/src/tui/components/ToolExecution.jsx +52 -594
  259. package/src/tui/components/TranscriptItem.jsx +105 -0
  260. package/src/tui/components/UsagePanel.jsx +18 -4
  261. package/src/tui/components/prompt-input/edit-helpers.mjs +72 -0
  262. package/src/tui/components/prompt-input/voice-indicator.mjs +39 -0
  263. package/src/tui/components/tool-execution/ResultBody.jsx +56 -0
  264. package/src/tui/components/tool-execution/surface-detail.mjs +405 -0
  265. package/src/tui/components/tool-execution/text-format.mjs +161 -0
  266. package/src/tui/display-width.mjs +20 -3
  267. package/src/tui/dist/index.mjs +19652 -18630
  268. package/src/tui/engine/agent-job-feed.mjs +133 -0
  269. package/src/tui/engine/notification-plan.mjs +76 -0
  270. package/src/tui/engine/render-timing.mjs +17 -0
  271. package/src/tui/engine/tool-approval.mjs +94 -0
  272. package/src/tui/engine/tool-card-results.mjs +234 -0
  273. package/src/tui/engine/tool-result-status.mjs +135 -0
  274. package/src/tui/engine.mjs +122 -562
  275. package/src/tui/figures.mjs +5 -0
  276. package/src/tui/index.jsx +105 -0
  277. package/src/tui/input-editing.mjs +2 -2
  278. package/src/tui/markdown/format-token.mjs +4 -1
  279. package/src/tui/statusline-ansi-bridge.mjs +11 -3
  280. package/src/tui/theme.mjs +6 -0
  281. package/src/ui/statusline-agents.mjs +213 -0
  282. package/src/ui/statusline-format.mjs +146 -0
  283. package/src/ui/statusline-segments.mjs +148 -0
  284. package/src/ui/statusline.mjs +67 -501
  285. package/src/ui/tool-card.mjs +0 -1
  286. package/src/vendor/statusline/bin/statusline-route.mjs +15 -2
  287. package/src/workflows/default/WORKFLOW.md +1 -1
  288. package/src/workflows/sequential/WORKFLOW.md +1 -1
  289. package/vendor/ink/build/display-width.js +19 -3
  290. package/vendor/ink/build/ink.js +103 -6
  291. package/vendor/ink/build/log-update.js +17 -3
  292. package/vendor/ink/build/wrap-text.js +125 -0
  293. package/scripts/_test-folder-dialog.mjs +0 -30
  294. package/scripts/fix-brief-fn.mjs +0 -35
  295. package/scripts/fix-format-tool-surface.mjs +0 -24
  296. package/scripts/fix-tool-exec-visible.mjs +0 -42
  297. package/scripts/patch-agent-brief.mjs +0 -48
  298. package/scripts/patch-app.mjs +0 -21
  299. package/scripts/patch-app2.mjs +0 -18
  300. package/scripts/patch-dist-brief.mjs +0 -96
  301. package/scripts/patch-tool-exec.mjs +0 -70
  302. package/src/examples/schedules/SCHEDULE.example.md +0 -32
  303. package/src/examples/webhooks/WEBHOOK.example.md +0 -40
  304. package/src/runtime/agent/orchestrator/session/manager.reactive-persist.test.mjs +0 -107
  305. package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.test.mjs +0 -143
  306. package/src/runtime/agent/orchestrator/tools/builtin/diagnostics-tool.mjs +0 -285
  307. package/src/runtime/agent/orchestrator/tools/builtin/external-tool-adapters.test.mjs +0 -162
  308. package/src/runtime/agent/orchestrator/tools/builtin/open-config-tool.mjs +0 -26
  309. package/src/runtime/shared/channel-notification-routing.test.mjs +0 -45
  310. package/src/runtime/shared/task-notification-envelope.test.mjs +0 -107
  311. package/src/runtime/shared/tool-execution-contract.test.mjs +0 -183
  312. package/src/standalone/agent-task-status.test.mjs +0 -76
  313. package/src/tui/components/tool-output-format.test.mjs +0 -399
  314. package/src/tui/display-width.test.mjs +0 -35
  315. package/src/tui/engine-runtime-notification.test.mjs +0 -115
  316. package/src/tui/engine-tool-result-text.test.mjs +0 -75
  317. package/src/tui/input-editing.selection.test.mjs +0 -75
  318. package/src/tui/markdown/format-token.test.mjs +0 -354
  319. package/src/tui/markdown/render-ansi.test.mjs +0 -108
  320. package/src/tui/markdown/stream-fence.test.mjs +0 -26
  321. package/src/tui/markdown/streaming-markdown.test.mjs +0 -70
  322. package/src/tui/paste-fix.test.mjs +0 -119
  323. package/src/tui/prompt-history-store.test.mjs +0 -52
  324. package/src/tui/statusline-ansi-bridge.test.mjs +0 -159
  325. package/src/tui/transcript-tool-failures.test.mjs +0 -111
  326. package/src/ui/markdown.test.mjs +0 -70
  327. package/src/ui/statusline-context-label.test.mjs +0 -15
  328. package/src/vendor/statusline/bin/statusline-lib.mjs +0 -186
  329. package/src/vendor/statusline/bin/statusline-route.test.mjs +0 -80
@@ -1,26 +1,17 @@
1
- import { createHash } from 'node:crypto';
2
- import {
3
- sanitizeToolPairs,
4
- dedupToolResultBodies,
5
- reconcileDedupStubs,
6
- estimateMessagesTokens,
7
- DEFAULT_COMPACTION_BUFFER_TOKENS,
8
- DEFAULT_COMPACTION_BUFFER_RATIO,
9
- MAX_COMPACTION_BUFFER_RATIO,
10
- DEFAULT_COMPACTION_KEEP_TOKENS,
11
- normalizeCompactionBufferRatio,
12
- compactionBufferTokensForBoundary,
13
- } from './context-utils.mjs';
14
-
15
- export const SUMMARY_PREFIX = 'A previous model worked on this task and produced the compacted handoff summary below. Build on the work already done and avoid duplicating it; treat the summary as authoritative context for continuing the task. You also retain the preserved recent turns that follow.';
1
+ // Facade for the compaction module. The god file was split into cohesive
2
+ // modules under session/compact/ (behavior-preserving); this file re-exports
3
+ // the identical public surface so every existing importer
4
+ // (loop.mjs, manager.mjs + subdirs, agent-runtime/session-builder.mjs,
5
+ // memory/index.mjs, scripts/compact-*.mjs) keeps working unchanged.
6
+ //
16
7
  // Default auto-compact trigger sits below the effective compact boundary by a
17
8
  // compaction buffer (10% of boundary, capped at MAX_COMPACTION_BUFFER_RATIO).
18
- // That headroom lets semantic compact run before the transcript is already at the
19
- // hard limit (zero buffer caused overflow_failed with no room to summarize).
20
- // Operators may still set compaction.bufferTokens / bufferPercent / bufferRatio,
21
- // to tune headroom. Telemetry-persisted bufferTokens/bufferRatio of zero is not
22
- // operator config; loop/manager strip it and reapply this default (see
23
- // compactBufferConfigForBoundary).
9
+ // That headroom lets semantic compact run before the transcript is already at
10
+ // the hard limit (zero buffer caused overflow_failed with no room to
11
+ // summarize). Operators may still set compaction.bufferTokens / bufferPercent /
12
+ // bufferRatio to tune headroom. Telemetry-persisted bufferTokens/bufferRatio of
13
+ // zero is not operator config; loop/manager strip it and reapply this default
14
+ // (see compactBufferConfigForBoundary).
24
15
  export {
25
16
  DEFAULT_COMPACTION_BUFFER_TOKENS,
26
17
  DEFAULT_COMPACTION_BUFFER_RATIO,
@@ -28,2270 +19,37 @@ export {
28
19
  DEFAULT_COMPACTION_KEEP_TOKENS,
29
20
  normalizeCompactionBufferRatio,
30
21
  compactionBufferTokensForBoundary,
31
- };
32
- export const SUMMARY_OUTPUT_TOKENS = 4_096;
33
- // Minimum room the generated summary needs after the mandatory (system +
34
- // preserved tail) cost is accounted for. When the configured target budget is
35
- // smaller than the mandatory cost (e.g. the preserved recent turn carries a
36
- // large tool result), the compaction MUST still proceed: the old head is the
37
- // part being summarized away, so dropping it already shrinks the transcript.
38
- // Refusing with "exceeds budget" here is what surfaced as auto-clear / overflow
39
- // compact failures. Floor the working budget to mandatory + this room instead.
40
- export const COMPACT_SUMMARY_MIN_ROOM_TOKENS = 4_000;
41
-
42
- const TOOL_CALL_ARGS_MAX_CHARS = 260;
43
- const TOOL_CALL_FACT_ARGS_MAX_CHARS = 140;
44
- const TOOL_CALLS_MAX = 4;
45
- const TOOL_ARG_STRING_MAX_CHARS = 360;
46
- const TOOL_ARG_ARRAY_MAX_ITEMS = 8;
47
- const TOOL_ARG_MAX_DEPTH = 4;
48
- const SENSITIVE_TOOL_ARG_KEY_RE = /(?:^|[_-])(?:api[_-]?key|authorization|auth|cookie|credential|passwd|password|refresh[_-]?token|secret|token)(?:$|[_-])/i;
49
- // Word alternation for raw-string (non-JSON) secret redaction. Mirrors the
50
- // keys in SENSITIVE_TOOL_ARG_KEY_RE and the session-ingest redactor so a raw
51
- // tool-call argument string like `authorization: Bearer abc.def` or
52
- // `password="abc def"` never reaches preserved facts or the compaction prompt.
53
- const SENSITIVE_TOOL_ARG_KEY_WORD = '(?:api[_-]?key|authorization|auth|cookie|credential|passwd|password|refresh[_-]?token|secret|token)';
54
- // Full key matcher: the sensitive WORD may carry a prefix and/or suffix segment
55
- // joined by `_`/`-` so prefixed variants like `access_token`, `access-token`,
56
- // `x-api-key`, and `bearer_token` are matched as whole keys (not just the bare
57
- // word at key start). Prefix/suffix are bounded by a `_`/`-` separator so the
58
- // word stays at an identifier boundary.
59
- const SENSITIVE_TOOL_ARG_KEY_FULL = `(?:[A-Za-z0-9_-]*[_-])?${SENSITIVE_TOOL_ARG_KEY_WORD}(?:[_-][A-Za-z0-9_-]*)?`;
60
-
61
- // Redact `key: value` / `key=value` secret pairs inside a raw (non-JSON)
62
- // string. Consumes the WHOLE value after the key — spaces, `Bearer `/`Basic `
63
- // scheme words, quoted values with internal spaces, and `;`-separated cookie
64
- // pairs — so no secret fragment survives. Kept local to compact-core to avoid a
65
- // cross-module dependency on the memory lib; logic matches session-ingest's
66
- // redactRawArgString.
67
- function redactRawSecretString(text) {
68
- const value = String(text ?? '');
69
- if (!value) return value;
70
- const keyRe = new RegExp(`((?:^|[\\s,{(])["']?${SENSITIVE_TOOL_ARG_KEY_FULL}["']?\\s*[:=]\\s*)`, 'gi');
71
- let out = '';
72
- let last = 0;
73
- let match;
74
- while ((match = keyRe.exec(value)) !== null) {
75
- const prefixEnd = match.index + match[0].length;
76
- out += value.slice(last, prefixEnd);
77
- let i = prefixEnd;
78
- const quote = value[i] === '"' || value[i] === "'" ? value[i] : '';
79
- if (quote) {
80
- i += 1;
81
- while (i < value.length && value[i] !== quote) i += 1;
82
- if (i < value.length) i += 1; // include closing quote
83
- } else {
84
- while (i < value.length && !/[,)}\n]/.test(value[i])) i += 1;
85
- }
86
- out += '[redacted]';
87
- last = i;
88
- keyRe.lastIndex = i;
89
- }
90
- out += value.slice(last);
91
- return out;
92
- }
93
-
94
- function sha16(value) {
95
- const text = typeof value === 'string' ? value : JSON.stringify(value ?? null);
96
- return createHash('sha256').update(text).digest('hex').slice(0, 16);
97
- }
98
-
99
- function roleCounts(messages) {
100
- const counts = new Map();
101
- for (const m of messages) counts.set(m?.role || 'unknown', (counts.get(m?.role || 'unknown') || 0) + 1);
102
- return [...counts.entries()].map(([role, count]) => `${role}:${count}`).join(', ');
103
- }
104
-
105
- function extractText(m) {
106
- if (!m || typeof m !== 'object') return '';
107
- if (typeof m.content === 'string') return m.content;
108
- if (Array.isArray(m.content)) {
109
- return m.content
110
- .map((item) => {
111
- if (!item || typeof item !== 'object') return '';
112
- if (typeof item.text === 'string') return item.text;
113
- if (typeof item.content === 'string') return item.content;
114
- return '';
115
- })
116
- .filter(Boolean)
117
- .join('\n');
118
- }
119
- try { return JSON.stringify(m.content ?? ''); } catch { return ''; }
120
- }
121
-
122
- function truncateMiddle(text, maxChars) {
123
- const value = String(text ?? '').replace(/\r\n/g, '\n');
124
- if (maxChars <= 0 || value.length === 0) return '';
125
- if (value.length <= maxChars) return value;
126
- if (maxChars <= 8) return value.slice(0, maxChars);
127
- const head = Math.ceil((maxChars - 5) / 2);
128
- const tail = Math.floor((maxChars - 5) / 2);
129
- return `${value.slice(0, head)} … ${value.slice(value.length - tail)}`;
130
- }
131
-
132
- function normalizeToolArgValue(value, key = '', depth = 0) {
133
- if (SENSITIVE_TOOL_ARG_KEY_RE.test(String(key || ''))) return '[redacted]';
134
- if (typeof value === 'bigint') return String(value);
135
- // Defense-in-depth: a non-sensitive KEY can still carry a secret embedded
136
- // in its string VALUE (e.g. a freeform `headers` string). Redact raw
137
- // key:value secret pairs before truncating.
138
- if (typeof value === 'string') return truncateMiddle(redactRawSecretString(value), TOOL_ARG_STRING_MAX_CHARS);
139
- if (!value || typeof value !== 'object') return value ?? null;
140
- if (depth >= TOOL_ARG_MAX_DEPTH) return Array.isArray(value) ? `[array:${value.length}]` : '[object]';
141
- if (Array.isArray(value)) {
142
- const out = value.slice(0, TOOL_ARG_ARRAY_MAX_ITEMS)
143
- .map((item, index) => normalizeToolArgValue(item, String(index), depth + 1));
144
- if (value.length > TOOL_ARG_ARRAY_MAX_ITEMS) out.push(`+${value.length - TOOL_ARG_ARRAY_MAX_ITEMS} more`);
145
- return out;
146
- }
147
- const out = {};
148
- for (const k of Object.keys(value).sort()) {
149
- out[k] = normalizeToolArgValue(value[k], k, depth + 1);
150
- }
151
- return out;
152
- }
153
-
154
- function stableToolArgJson(value) {
155
- if (value == null) return '';
156
- if (typeof value === 'string') {
157
- const text = value.trim();
158
- if (!text) return '';
159
- if (/^[\[{]/.test(text)) {
160
- try { return JSON.stringify(normalizeToolArgValue(JSON.parse(text))); } catch { /* keep raw */ }
161
- }
162
- // Non-JSON raw string: redact secret key:value pairs before truncating
163
- // so toolCallSummary / preserved facts / compaction prompt metadata
164
- // never leak `authorization: Bearer ...`, passwords, cookies, tokens.
165
- return truncateMiddle(redactRawSecretString(text), TOOL_ARG_STRING_MAX_CHARS);
166
- }
167
- try {
168
- return JSON.stringify(normalizeToolArgValue(value));
169
- } catch {
170
- return truncateMiddle(redactRawSecretString(String(value || '')), TOOL_ARG_STRING_MAX_CHARS);
171
- }
172
- }
173
-
174
- function toolCallArgsText(tc, maxChars = TOOL_CALL_ARGS_MAX_CHARS) {
175
- if (!(maxChars > 0)) return '';
176
- const raw = tc?.arguments ?? tc?.function?.arguments;
177
- const text = stableToolArgJson(raw);
178
- return text ? truncateMiddle(text, maxChars) : '';
179
- }
180
-
181
- function summarizeToolCall(tc, maxArgChars = TOOL_CALL_ARGS_MAX_CHARS) {
182
- const name = tc?.name || tc?.function?.name || tc?.id || '?';
183
- const args = toolCallArgsText(tc, maxArgChars);
184
- return args ? `${name}(${args})` : name;
185
- }
186
-
187
- function toolCallArgBudget(perMessageChars) {
188
- const chars = Number(perMessageChars);
189
- if (!Number.isFinite(chars) || chars <= 0) return 0;
190
- return Math.min(TOOL_CALL_ARGS_MAX_CHARS, Math.max(32, Math.floor(chars * 0.5)));
191
- }
192
-
193
- function toolCallSummary(m, maxArgChars = TOOL_CALL_ARGS_MAX_CHARS) {
194
- if (!Array.isArray(m?.toolCalls) || m.toolCalls.length === 0) return '';
195
- const calls = m.toolCalls
196
- .slice(0, TOOL_CALLS_MAX)
197
- .map(tc => summarizeToolCall(tc, maxArgChars));
198
- if (m.toolCalls.length > TOOL_CALLS_MAX) calls.push(`+${m.toolCalls.length - TOOL_CALLS_MAX} more`);
199
- return ` tool_calls=${calls.join(';')}`;
200
- }
201
-
202
- function toolResultId(m) {
203
- return m?.role === 'tool' && m.toolCallId ? ` tool_result=${m.toolCallId}` : '';
204
- }
205
-
206
- function compactHeader(oldHistory) {
207
- const encoded = JSON.stringify(oldHistory ?? []);
208
- return [
209
- SUMMARY_PREFIX,
210
- `messages=${oldHistory.length} sha256=${sha16(encoded)} roles=${roleCounts(oldHistory) || 'none'}`,
211
- ];
212
- }
213
-
214
- function makeSummaryMessage(content) {
215
- return { role: 'user', content };
216
- }
217
-
218
- // A compact summary message is a synthetic role:'user' message carrying the
219
- // SUMMARY_PREFIX anchor. It is NOT a real user turn: it must be excluded from
220
- // real user-turn boundary calculations and treated as merge input, otherwise
221
- // an old summary can sit in the preserved tail as a live user message,
222
- // duplicate, or fail to merge across repeated compaction.
223
- function isSummaryMessage(m) {
224
- return m?.role === 'user'
225
- && typeof m.content === 'string'
226
- && m.content.startsWith(SUMMARY_PREFIX);
227
- }
228
-
229
- function isProtectedContextUserMessage(m) {
230
- if (m?.role !== 'user' || typeof m.content !== 'string') return false;
231
- return m.content.trimStart().startsWith('<system-reminder>');
232
- }
233
-
234
- // An injected Skill-body user message (the general newMessages channel carries
235
- // the full SKILL.md body as a role:'user' message after the Skill tool_result).
236
- // Like isSummaryMessage / isProtectedContextUserMessage, it is detected by
237
- // content prefix (the `<skill>` envelope from buildSkillResultEnvelope) so the
238
- // check survives even if the synthetic `meta` field is dropped during a tail
239
- // rebuild. It is NOT the human's latest prompt and must be excluded from
240
- // "latest human request" selection (deriveCurrentRequest /
241
- // buildRecallFastTrackQuery). The `meta:'skill'` marker is also honoured.
242
- function isInjectedSkillBodyMessage(m) {
243
- if (m?.role !== 'user') return false;
244
- if (m.meta === 'skill') return true;
245
- return typeof m.content === 'string' && m.content.trimStart().startsWith('<skill>');
246
- }
247
-
248
- function isProtectedContextAckMessage(m) {
249
- return m?.role === 'assistant'
250
- && typeof m.content === 'string'
251
- && m.content.trim() === '.'
252
- && !Array.isArray(m.toolCalls);
253
- }
254
-
255
- function splitProtectedContext(messages) {
256
- const protectedPrefix = [];
257
- const conversation = [];
258
- let prefixMode = true;
259
- let previousWasProtectedContext = false;
260
- for (const m of messages || []) {
261
- if (m?.role === 'system') {
262
- protectedPrefix.push(m);
263
- previousWasProtectedContext = false;
264
- continue;
265
- }
266
- if (prefixMode && isProtectedContextUserMessage(m)) {
267
- protectedPrefix.push(m);
268
- previousWasProtectedContext = true;
269
- continue;
270
- }
271
- if (prefixMode && previousWasProtectedContext && isProtectedContextAckMessage(m)) {
272
- protectedPrefix.push(m);
273
- previousWasProtectedContext = false;
274
- continue;
275
- }
276
- prefixMode = false;
277
- previousWasProtectedContext = false;
278
- conversation.push(m);
279
- }
280
- return { protectedPrefix, conversation };
281
- }
282
-
283
- // Redaction-ONLY recursive walk for tool-call argument VALUES kept verbatim
284
- // through compaction. Unlike normalizeToolArgValue (which is for prompt-side
285
- // summaries and truncates/summarizes/key-sorts), this preserves shape exactly:
286
- // - sensitive KEY -> '[redacted]'
287
- // - string value -> redactRawSecretString(value) (no truncation/middle-cut)
288
- // - array -> same length, items walked in order (no slicing/caps)
289
- // - object -> same keys in INSERTION order (no sorting/depth caps)
290
- // - other primitive (number/boolean/bigint/null) -> returned unchanged
291
- // Returns { value, changed } so callers can preserve byte-exact input when the
292
- // walk altered nothing. Only sensitive-key values and embedded raw secret pairs
293
- // inside strings are altered; everything else is structure identical.
294
- function redactToolArgValueOnly(value, key = '') {
295
- if (SENSITIVE_TOOL_ARG_KEY_RE.test(String(key || ''))) {
296
- return { value: '[redacted]', changed: value !== '[redacted]' };
297
- }
298
- if (value == null) return { value, changed: false };
299
- if (typeof value === 'string') {
300
- const redacted = redactRawSecretString(value);
301
- return { value: redacted, changed: redacted !== value };
302
- }
303
- if (Array.isArray(value)) {
304
- let changed = false;
305
- const out = value.map((item) => {
306
- const r = redactToolArgValueOnly(item, '');
307
- if (r.changed) changed = true;
308
- return r.value;
309
- });
310
- return { value: changed ? out : value, changed };
311
- }
312
- if (typeof value === 'object') {
313
- let changed = false;
314
- const out = {};
315
- for (const k of Object.keys(value)) {
316
- const r = redactToolArgValueOnly(value[k], k);
317
- if (r.changed) changed = true;
318
- out[k] = r.value;
319
- }
320
- return { value: changed ? out : value, changed };
321
- }
322
- return { value, changed: false };
323
- }
324
-
325
- // Redact sensitive values inside a single tool-call arguments payload while
326
- // preserving its original shape (string stays a string, object stays an
327
- // object) so the provider-valid tool_call structure is not broken. Used to
328
- // scrub messages that survive compaction VERBATIM (preserved tail / mandatory
329
- // context), where prompt-side redaction does not apply. Redaction-only — no
330
- // truncation, summarization, key sorting, or depth/array caps.
331
- function redactToolCallArgumentsValue(rawArgs) {
332
- if (rawArgs == null) return rawArgs;
333
- if (typeof rawArgs === 'string') {
334
- const trimmed = rawArgs.trim();
335
- if (/^[\[{]/.test(trimmed)) {
336
- try {
337
- // Parse, redaction-only walk; only reserialize when the walk
338
- // actually changed a sensitive value. When nothing changed,
339
- // return the ORIGINAL string byte-exact (no JSON re-formatting).
340
- const { value, changed } = redactToolArgValueOnly(JSON.parse(trimmed));
341
- return changed ? JSON.stringify(value) : rawArgs;
342
- } catch { /* fall through to raw redaction */ }
343
- }
344
- return redactRawSecretString(rawArgs);
345
- }
346
- if (typeof rawArgs === 'object') {
347
- try {
348
- const { value, changed } = redactToolArgValueOnly(rawArgs);
349
- return changed ? value : rawArgs;
350
- } catch { return rawArgs; }
351
- }
352
- return rawArgs;
353
- }
354
-
355
- // Return a copy of a message with sensitive tool-call argument values redacted,
356
- // keeping role / content / toolCallId / tool names+ids and non-sensitive args
357
- // intact. Non-tool-bearing messages are returned unchanged (same reference).
358
- function redactMessageToolCallSecrets(m) {
359
- if (!m || typeof m !== 'object' || !Array.isArray(m.toolCalls) || m.toolCalls.length === 0) {
360
- return m;
361
- }
362
- let changed = false;
363
- const toolCalls = m.toolCalls.map((tc) => {
364
- if (!tc || typeof tc !== 'object') return tc;
365
- const out = { ...tc };
366
- if ('arguments' in tc && tc.arguments != null) {
367
- const redacted = redactToolCallArgumentsValue(tc.arguments);
368
- if (redacted !== tc.arguments) { out.arguments = redacted; changed = true; }
369
- }
370
- if (tc.function && typeof tc.function === 'object' && tc.function.arguments != null) {
371
- const redacted = redactToolCallArgumentsValue(tc.function.arguments);
372
- if (redacted !== tc.function.arguments) {
373
- out.function = { ...tc.function, arguments: redacted };
374
- changed = true;
375
- }
376
- }
377
- return out;
378
- });
379
- if (!changed) return m;
380
- return { ...m, toolCalls };
381
- }
382
-
383
- // Scrub an array of messages that are kept verbatim through compaction so a
384
- // recent assistant tool call carrying a secret (e.g. `authorization: Bearer
385
- // ...`) cannot survive into the returned compacted transcript. Only assistant
386
- // toolCalls argument VALUES are touched; structure/order/pairing is preserved.
387
- export function redactToolCallSecretsInMessages(messages) {
388
- if (!Array.isArray(messages)) return messages;
389
- return messages.map((m) => redactMessageToolCallSecrets(m));
390
- }
391
-
392
- // Floor for the reserve-adjusted compact budget. When the tool-schema/request
393
- // reserve rivals the whole budget (huge agent tool surfaces), subtracting the
394
- // full reserve could leave a degenerate target; keep enough room to attempt a
395
- // summary and let the final fit check decide. Logged as degraded because a
396
- // floored budget can still overflow on the next send.
397
- const MIN_EFFECTIVE_COMPACT_BUDGET_TOKENS = 1024;
398
- export function effectiveBudget(budgetTokens, opts) {
399
- if (!(budgetTokens > 0)) throw new Error('compact: budgetTokens must be > 0');
400
- const reserve = Number(opts?.reserveTokens) || 0;
401
- if (reserve <= 0) return budgetTokens;
402
- // Subtract the FULL reserve so an accepted compact actually fits next to
403
- // the request reserve on the following send. The previous 50%-of-budget cap
404
- // under-reserved large tool surfaces (agent sessions): a compact could be
405
- // "accepted" at budget/2 while the true remaining room was smaller, then
406
- // overflow immediately on the next request.
407
- const remaining = budgetTokens - reserve;
408
- if (remaining >= MIN_EFFECTIVE_COMPACT_BUDGET_TOKENS) return remaining;
409
- const floored = Math.max(1, Math.min(budgetTokens, MIN_EFFECTIVE_COMPACT_BUDGET_TOKENS));
410
- try { process.stderr.write(`[compact] degraded budget: reserve=${reserve} leaves ${remaining} of budget=${budgetTokens}; flooring to ${floored}\n`); } catch { /* best-effort */ }
411
- return floored;
412
- }
413
-
414
- const PRUNE_TOOL_OUTPUT_MAX_CHARS = 2_000;
415
- const PRUNE_TOOL_OUTPUT_HEAD_CHARS = 1_000;
416
- const PRUNE_TOOL_OUTPUT_TAIL_CHARS = 600;
417
- const PRUNE_TAIL_TURNS = 2;
418
- const DEFAULT_TAIL_TURNS = 2;
419
- const MIN_PRESERVE_RECENT_TOKENS = 2_000;
420
- const COMPACTION_INPUT_MAX_CHARS = 2_000;
421
- const COMPACTION_PROMPT_HEADROOM = 0.85;
422
- const PRESERVED_FACTS_MAX_CHARS = 600;
423
-
424
- export const COMPACT_TYPE_SEMANTIC = 'semantic';
425
- export const COMPACT_TYPE_RECALL_FASTTRACK = 'recall-fasttrack';
426
- export const DEFAULT_COMPACT_TYPE = COMPACT_TYPE_SEMANTIC;
427
- export const COMPACT_TYPES = Object.freeze([
22
+ SUMMARY_PREFIX,
23
+ SUMMARY_OUTPUT_TOKENS,
24
+ CONTEXT_SHARE_RATIO,
25
+ RECALL_TOKEN_CAP_FLOOR_TOKENS,
26
+ COMPACT_SUMMARY_MIN_ROOM_TOKENS,
428
27
  COMPACT_TYPE_SEMANTIC,
429
28
  COMPACT_TYPE_RECALL_FASTTRACK,
430
- ]);
431
-
432
- export function normalizeCompactType(value, fallback = DEFAULT_COMPACT_TYPE) {
433
- const raw = String(value ?? '').trim().toLowerCase().replace(/_/g, '-');
434
- if (!raw) return fallback;
435
- if (raw === '1' || raw === 'type1' || raw === 'type-1' || raw === 'bench1' || raw === 'bench-1' || raw === 'semantic' || raw === 'summary') {
436
- return COMPACT_TYPE_SEMANTIC;
437
- }
438
- // Recall fast-track aliases. `replace(/_/g,'-')` above already folds
439
- // snake_case (fast_track -> fast-track), but list both dash/no-dash forms
440
- // explicitly so callers passing either spelling resolve deterministically.
441
- if (raw === '2' || raw === 'type2' || raw === 'type-2' || raw === 'recall' || raw === 'recall-fast' || raw === 'recall-fasttrack' || raw === 'recall-fast-track' || raw === 'fasttrack' || raw === 'fast-track') {
442
- return COMPACT_TYPE_RECALL_FASTTRACK;
443
- }
444
- // Unknown / unrecognized value: fall back to the caller-provided default
445
- // (semantic by default). Callers that need to detect an unknown value
446
- // should compare the input against COMPACT_TYPES before normalizing.
447
- return fallback;
448
- }
449
-
450
- export function compactTypeIsSemantic(value) {
451
- return normalizeCompactType(value) === COMPACT_TYPE_SEMANTIC;
452
- }
453
-
454
- export function compactTypeIsRecallFastTrack(value) {
455
- return normalizeCompactType(value) === COMPACT_TYPE_RECALL_FASTTRACK;
456
- }
457
-
458
- function compactDebugEnabled() {
459
- return String(process.env.MIXDOG_COMPACT_DEBUG || '').trim() === '1';
460
- }
461
-
462
- function compactDebugLog(scope, details = {}) {
463
- if (!compactDebugEnabled()) return;
464
- try {
465
- process.stderr.write(`[compact] ${scope} ${JSON.stringify(details)}\n`);
466
- } catch { /* best-effort diagnostics only */ }
467
- }
468
-
469
- function safeEstimateMessagesTokens(messages) {
470
- try { return estimateMessagesTokens(messages); }
471
- catch { return null; }
472
- }
473
-
474
- function textByteLength(text) {
475
- try { return Buffer.byteLength(String(text || ''), 'utf8'); }
476
- catch { return String(text || '').length; }
477
- }
478
-
479
- function messageContentHasMarker(m, marker) {
480
- if (!m || !marker) return false;
481
- if (typeof m.content === 'string') return m.content.includes(marker);
482
- if (Array.isArray(m.content)) {
483
- return m.content.some((part) => {
484
- if (!part || typeof part !== 'object') return false;
485
- return String(part.text || part.content || '').includes(marker);
486
- });
487
- }
488
- return false;
489
- }
490
-
491
- // Count raw (unchunked) pending rows still present in a dump_session_roots
492
- // payload. recall-fasttrack must keep cycle1-chunking until this reaches 0 so
493
- // the injected root is the chunked summary, not the raw transcript tail.
494
- export function countRawPendingRows(dumpText) {
495
- const text = String(dumpText || '');
496
- const matches = text.match(/(?:^|\n)# raw_pending\s+\d+\s+id=/gi);
497
- return matches ? matches.length : 0;
498
- }
499
-
500
- // Drain a single session's cycle1 in fixed window×concurrency units, looping
501
- // until no raw rows remain (or a pass stops making progress / the deadline
502
- // elapses). This replaces the previous single-pass cycle1 so large sessions
503
- // get fully chunked before their root is injected into the compacted context.
504
- export async function drainSessionCycle1(runTool, { sessionId, cycleArgs = {}, dumpArgs, maxPasses = 0, deadlineMs = 0 } = {}) {
505
- if (typeof runTool !== 'function') throw new Error('drainSessionCycle1: runTool is required');
506
- if (!sessionId) throw new Error('drainSessionCycle1: sessionId is required');
507
- if (!dumpArgs) throw new Error('drainSessionCycle1: dumpArgs is required');
508
- const startedAt = Date.now();
509
- const hardPasses = Math.max(1, Number(maxPasses) || 50);
510
- const lines = [];
511
- let recallText = await runTool('memory', dumpArgs);
512
- let rawRemaining = countRawPendingRows(recallText);
513
- let pass = 0;
514
- while (rawRemaining > 0 && pass < hardPasses) {
515
- if (deadlineMs > 0 && (Date.now() - startedAt) >= deadlineMs) break;
516
- pass += 1;
517
- const passDeadline = deadlineMs > 0
518
- ? Math.max(1, deadlineMs - (Date.now() - startedAt))
519
- : 0;
520
- let passText = '';
521
- try {
522
- passText = await runTool('memory', {
523
- action: 'cycle1',
524
- sessionId,
525
- ...cycleArgs,
526
- ...(passDeadline > 0 ? { _callerDeadlineMs: passDeadline } : {}),
527
- });
528
- } catch (err) {
529
- lines.push(`cycle1 pass=${pass} error=${err?.message || err}`);
530
- break;
531
- }
532
- if (passText) lines.push(`cycle1 pass=${pass}: ${String(passText).trim()}`);
533
- recallText = await runTool('memory', dumpArgs);
534
- const nextRaw = countRawPendingRows(recallText);
535
- // No forward progress (raw not shrinking) — stop instead of spinning.
536
- if (nextRaw >= rawRemaining) {
537
- rawRemaining = nextRaw;
538
- break;
539
- }
540
- rawRemaining = nextRaw;
541
- }
542
- return {
543
- recallText,
544
- cycle1Text: lines.join('\n'),
545
- passes: pass,
546
- rawRemaining,
547
- };
548
- }
549
-
550
- function preservedFactHints() {
551
- return String(process.env.MIXDOG_COMPACT_FACT_HINTS || '')
552
- .split(/[\n,;]+/u)
553
- .map(s => s.trim())
554
- .filter(Boolean)
555
- .map(s => s.toLocaleLowerCase());
556
- }
557
-
558
- const PRESERVED_FACT_HINTS = preservedFactHints();
559
-
560
- function hasConfiguredPreservedFactHint(text) {
561
- const lower = String(text || '').toLocaleLowerCase();
562
- return PRESERVED_FACT_HINTS.some(hint => hint && lower.includes(hint));
563
- }
564
-
565
- const COMPACTION_SYSTEM_PROMPT = [
566
- 'You are an anchored context summarization assistant for coding sessions.',
567
- '',
568
- 'Summarize only the conversation history you are given. The newest turns may be kept verbatim outside your summary, so focus on the older context that still matters for continuing the work.',
569
- '',
570
- 'If the prompt includes a <previous-summary> block, treat it as the current anchored summary. Update it with the new history by preserving still-true details, removing stale details, and merging in new facts.',
571
- '',
572
- 'Always follow the exact output structure requested by the user prompt. Keep every section, preserve exact file paths and identifiers when known, and prefer terse bullets over paragraphs.',
573
- '',
574
- 'Do not answer the conversation itself. Do not mention that you are summarizing, compacting, or merging context. Respond in the same language as the conversation.',
575
- ].join('\n');
576
- const SUMMARY_TEMPLATE = `Output exactly the Markdown structure shown inside <template> and keep the section order unchanged. Do not include the <template> tags in your response.
577
- <template>
578
- ## Goal
579
- - [single-sentence task summary]
580
-
581
- ## Constraints & Preferences
582
- - [user constraints, preferences, specs, or "(none)"]
583
-
584
- ## Progress
585
- ### Done
586
- - [completed work or "(none)"]
587
-
588
- ### In Progress
589
- - [current work or "(none)"]
590
-
591
- ### Blocked
592
- - [blockers or "(none)"]
593
-
594
- ## Key Decisions
595
- - [decision and why, or "(none)"]
596
-
597
- ## Next Steps
598
- - [ordered next actions or "(none)"]
599
-
600
- ## Critical Context
601
- - [important technical facts, errors, open questions, or "(none)"]
602
-
603
- ## Relevant Files
604
- - [file or directory path: why it matters, or "(none)"]
605
- </template>
606
-
607
- Rules:
608
- - Keep every section, even when empty.
609
- - Use terse bullets, not prose paragraphs.
610
- - Preserve exact file paths, commands, error strings, and identifiers when known.
611
- - Use the same language as the active user thread when it is clear.
612
- - Do not mention the summary process or that context was compacted.`;
613
-
614
- function extractPreservedFacts(messages) {
615
- if (!messages || messages.length === 0) return '';
616
- const candidates = [];
617
- const seenSignatures = new Set();
618
- const add = (prefix, text, score, messageIndex, lineIndex = 0) => {
619
- const clean = text.replace(/`/g, '').trim();
620
- if (clean.length < 10) return;
621
- const sig = clean.slice(0, 80).toLowerCase().replace(/\s+/g, ' ');
622
- if (seenSignatures.has(sig)) return;
623
- seenSignatures.add(sig);
624
- candidates.push({ prefix, text: clean, score, messageIndex, lineIndex });
625
- };
626
- const classifyLine = (t) => {
627
- // Language-neutral structural cues: exact assignments, paths, URLs,
628
- // symbolic error/constant identifiers, and optional user-configured
629
- // locale/domain hints. Do not bake human-language keyword lists here.
630
- if (/[\p{L}\p{N}_$./:-]{2,}\s*=\s*\S+/u.test(t)) return { prefix: '•', score: 100 };
631
- if (/(?:^|[\s('"`])https?:\/\/[^\s'"`<>]+/iu.test(t)) return { prefix: '•', score: 95 };
632
- if (/(?:^|[\s('"`])(?:[A-Za-z]:[\\/]|\.{1,2}[\\/]|~?[\\/][^\s'"`<>]+|[\p{L}\p{N}_$.-]+[\\/][^\s'"`<>]+|[\p{L}\p{N}_$.-]+\.(?:mjs|cjs|js|jsx|ts|tsx|json|md|rs|go|py|java|kt|cs|cpp|c|h|hpp|css|html|yml|yaml|toml|lock|sh|ps1)\b)/iu.test(t)) return { prefix: '•', score: 95 };
633
- if (/\b[A-Z][A-Z0-9_:-]{2,}\b/.test(t)) return { prefix: '•', score: 90 };
634
- if (hasConfiguredPreservedFactHint(t)) return { prefix: '!', score: 70 };
635
- return null;
636
- };
637
- for (let mi = 0; mi < messages.length; mi += 1) {
638
- const m = messages[mi];
639
- const text = extractText(m);
640
- if (!text) continue;
641
- const lines = text.split('\n');
642
- for (let li = 0; li < lines.length; li += 1) {
643
- const line = lines[li];
644
- const t = line.trim();
645
- if (t.length < 10 || t.length > 250) continue;
646
- const cls = classifyLine(t);
647
- if (cls) add(cls.prefix, t, cls.score, mi, li);
648
- }
649
- if (m?.role === 'assistant' && Array.isArray(m.toolCalls)) {
650
- for (const tc of m.toolCalls) {
651
- const name = tc?.function?.name || tc?.name || '';
652
- if (name) {
653
- const summary = summarizeToolCall(tc, TOOL_CALL_FACT_ARGS_MAX_CHARS);
654
- const sig = `tool:${summary.slice(0, 160).toLowerCase()}`;
655
- if (seenSignatures.has(sig)) continue;
656
- seenSignatures.add(sig);
657
- candidates.push({ prefix: '•', text: `Tool: ${summary}`, score: 50, messageIndex: mi, lineIndex: Number.MAX_SAFE_INTEGER });
658
- }
659
- }
660
- }
661
- }
662
- if (candidates.length === 0) return '';
663
- candidates.sort((a, b) =>
664
- (b.score - a.score)
665
- || (b.messageIndex - a.messageIndex)
666
- || (b.lineIndex - a.lineIndex)
667
- );
668
- let result = '## Preserved Facts\n';
669
- let kept = 0;
670
- for (const c of candidates) {
671
- if (kept >= 25) break;
672
- let line = `- ${c.prefix} ${c.text}\n`;
673
- if (result.length + line.length > PRESERVED_FACTS_MAX_CHARS) {
674
- const room = PRESERVED_FACTS_MAX_CHARS - result.length - 8;
675
- if (kept === 0 && room > 32) line = `- ${c.prefix} ${c.text.slice(0, room)}…\n`;
676
- else continue;
677
- }
678
- result += line;
679
- kept += 1;
680
- }
681
- return kept > 0 ? result : '';
682
- }
683
-
684
- function protectedTailStart(messages, tailTurns = PRUNE_TAIL_TURNS) {
685
- let seenUsers = 0;
686
- for (let i = messages.length - 1; i >= 0; i -= 1) {
687
- if (messages[i]?.role !== 'user') continue;
688
- seenUsers += 1;
689
- if (seenUsers >= tailTurns) return i;
690
- }
691
- return 0;
692
- }
693
-
694
- function pruneToolOutputText(text, maxChars, toolCallId) {
695
- const value = String(text ?? '').replace(/\r\n/g, '\n');
696
- if (value.length <= maxChars) return value;
697
- const hash = createHash('sha256').update(value).digest('hex').slice(0, 16);
698
- const head = value.slice(0, PRUNE_TOOL_OUTPUT_HEAD_CHARS);
699
- const tail = value.slice(-PRUNE_TOOL_OUTPUT_TAIL_CHARS);
700
- return [
701
- `[mixdog pruned old tool output: ${value.length} chars, sha256:${hash}${toolCallId ? `, tool_use_id=${toolCallId}` : ''}]`,
702
- head,
703
- '... [old tool output omitted during worker compaction] ...',
704
- tail,
705
- ].join('\n');
706
- }
707
-
708
- export function pruneToolOutputs(messages, budgetTokens, opts = {}) {
709
- const budget = effectiveBudget(budgetTokens, opts);
710
- let result = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(messages)));
711
- if (estimateMessagesTokens(result) <= budget) return result;
712
-
713
- const maxChars = Math.max(256, Number(opts?.maxToolOutputChars) || PRUNE_TOOL_OUTPUT_MAX_CHARS);
714
- const protectFrom = protectedTailStart(result, Number(opts?.tailTurns) || PRUNE_TAIL_TURNS);
715
- const candidates = [];
716
- for (let i = 0; i < protectFrom; i += 1) {
717
- const m = result[i];
718
- if (m?.role !== 'tool' || typeof m.content !== 'string') continue;
719
- if (m.content.length <= maxChars) continue;
720
- candidates.push({ index: i, length: m.content.length });
721
- }
722
- candidates.sort((a, b) => b.length - a.length);
723
- for (const c of candidates) {
724
- const m = result[c.index];
725
- result[c.index] = {
726
- ...m,
727
- content: pruneToolOutputText(m.content, maxChars, m.toolCallId),
728
- compacted: true,
729
- compactedKind: 'tool_output_prune',
730
- };
731
- if (estimateMessagesTokens(result) <= budget) break;
732
- }
733
- return reconcileDedupStubs(result);
734
- }
735
-
736
- // Anchor-independent tool-output prune (loop overflow safety net).
737
- //
738
- // pruneToolOutputs protects the most-recent tailTurns of USER-anchored history,
739
- // so a single-turn transcript with no user boundary yields protectFrom=0 and
740
- // prunes nothing. This variant needs no user anchor: it middle-truncates the
741
- // OLDEST oversized tool_result bodies first, walking forward, until the
742
- // transcript fits the budget. The newest tool_result is truncated last (and
743
- // only if still necessary) so fresh state is preserved as long as possible.
744
- // Structure/pairing is preserved (only string content shrinks), and the result
745
- // is re-reconciled so tool pairing stays provider-valid.
746
- export function pruneToolOutputsUnanchored(messages, budgetTokens, opts = {}) {
747
- const budget = effectiveBudget(budgetTokens, opts);
748
- let result = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(messages)));
749
- if (estimateMessagesTokens(result) <= budget) return result;
750
-
751
- const maxChars = Math.max(256, Number(opts?.maxToolOutputChars) || PRUNE_TOOL_OUTPUT_MAX_CHARS);
752
- // Oldest -> newest so recent tool output survives longest. No user-turn
753
- // protection: every oversized tool_result is a candidate.
754
- for (let i = 0; i < result.length; i += 1) {
755
- const m = result[i];
756
- if (m?.role !== 'tool' || typeof m.content !== 'string') continue;
757
- if (m.content.length <= maxChars) continue;
758
- result[i] = {
759
- ...m,
760
- content: pruneToolOutputText(m.content, maxChars, m.toolCallId),
761
- compacted: true,
762
- compactedKind: 'tool_output_prune',
763
- };
764
- if (estimateMessagesTokens(result) <= budget) break;
765
- }
766
- return reconcileDedupStubs(result);
767
- }
768
-
769
- function preserveRecentBudget(budget, opts = {}) {
770
- const maxForBudget = Math.max(1, Math.floor(Number(budget || 0) * 0.8));
771
- const explicit = Number(opts.preserveRecentTokens ?? opts.keepTokens);
772
- if (Number.isFinite(explicit) && explicit > 0) {
773
- return Math.max(1, Math.min(Math.floor(explicit), maxForBudget));
774
- }
775
- return Math.max(
776
- 1,
777
- Math.min(
778
- DEFAULT_COMPACTION_KEEP_TOKENS,
779
- Math.max(MIN_PRESERVE_RECENT_TOKENS, Math.floor(budget * 0.25)),
780
- maxForBudget,
781
- ),
782
- );
783
- }
784
-
785
- function userIndexes(messages) {
786
- const out = [];
787
- for (let i = 0; i < messages.length; i += 1) {
788
- if (messages[i]?.role === 'user') out.push(i);
789
- }
790
- return out;
791
- }
792
-
793
- function indexLiveTurns(live) {
794
- const turns = splitTailIntoTurns(live);
795
- const indexed = [];
796
- let scan = 0;
797
- for (const messages of turns) {
798
- while (scan < live.length && live[scan] !== messages[0]) scan += 1;
799
- const start = scan;
800
- const end = start + messages.length;
801
- indexed.push({ start, end, messages });
802
- scan = end;
803
- }
804
- return indexed;
805
- }
806
-
807
- function splitTurnStartIndexForBudget(turn, budget) {
808
- const { start, end, messages } = turn;
809
- for (let i = 0; i < messages.length; i += 1) {
810
- const suffixStart = start + i;
811
- const suffix = messages.slice(i);
812
- if (suffix.length > 0 && estimateMessagesTokens(suffix) <= budget) {
813
- return suffixStart;
814
- }
815
- }
816
- return end;
817
- }
818
-
819
- function splitLiveCompactionContext(messages) {
820
- const sanitized = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(messages)));
821
- const { protectedPrefix, conversation: nonSystem } = splitProtectedContext(sanitized);
822
- let previousSummary = null;
823
- for (let i = nonSystem.length - 1; i >= 0; i -= 1) {
824
- if (isSummaryMessage(nonSystem[i])) {
825
- previousSummary = nonSystem[i].content;
826
- break;
827
- }
828
- }
829
- const live = nonSystem.filter((m) => !isSummaryMessage(m));
830
- return { system: protectedPrefix, live, previousSummary, sanitized };
831
- }
832
-
833
- // A tail may begin ONLY at an index that is not a tool result: a tool result
834
- // must stay paired with the assistant tool_call that precedes it, so it can
835
- // never be the first message of the preserved tail. Every other role (user /
836
- // assistant / developer / ...) is a valid tail boundary. This replaces the old
837
- // "the tail must begin at a real user turn" rule, which threw whenever the
838
- // recent window carried no user message (single-turn agent sessions whose tail
839
- // is assistant/tool only).
840
- function findValidCutIndices(live) {
841
- const out = [];
842
- for (let i = 0; i < live.length; i += 1) {
843
- if (live[i]?.role === 'tool') continue;
844
- out.push(i);
845
- }
846
- return out;
847
- }
848
-
849
- // User-anchored path (unchanged behaviour): keep up to tailTurns recent turns
850
- // bounded by recentBudget, splitting the newest turn's suffix when it alone is
851
- // too large. Preserved verbatim so Lead / normal sessions with real user turns
852
- // compact exactly as before.
853
- function selectTailStartByTurns(live, recentBudget, tailTurns, previousSummary, opts) {
854
- const indexedTurns = indexLiveTurns(live);
855
- if (indexedTurns.length === 0) return live.length;
856
-
857
- let tailStartIdx = live.length;
858
- let keptTurns = 0;
859
-
860
- for (let t = indexedTurns.length - 1; t >= 0; t -= 1) {
861
- if (keptTurns >= tailTurns) break;
862
- const turn = indexedTurns[t];
863
- const tailFromTurn = live.slice(turn.start);
864
- if (keptTurns === 0) {
865
- if (estimateMessagesTokens(tailFromTurn) <= recentBudget) {
866
- tailStartIdx = turn.start;
867
- keptTurns += 1;
868
- continue;
869
- }
870
- const splitIdx = splitTurnStartIndexForBudget(turn, recentBudget);
871
- if (splitIdx < turn.end) {
872
- tailStartIdx = splitIdx;
873
- keptTurns += 1;
874
- break;
875
- }
876
- // Newest turn has no fitting suffix: keep entire live transcript in head for summarization.
877
- tailStartIdx = live.length;
878
- keptTurns = 0;
879
- break;
880
- }
881
- const candidateStart = turn.start;
882
- const candidateTail = live.slice(candidateStart);
883
- if (estimateMessagesTokens(candidateTail) <= recentBudget) {
884
- tailStartIdx = candidateStart;
885
- keptTurns += 1;
886
- continue;
887
- }
888
- break;
889
- }
890
-
891
- if (opts.force === true && !previousSummary && tailStartIdx <= 0) {
892
- if (indexedTurns.length >= 2) {
893
- tailStartIdx = indexedTurns[1].start;
894
- } else if (indexedTurns.length === 1) {
895
- const onlyTurn = indexedTurns[0];
896
- const splitIdx = splitTurnStartIndexForBudget(onlyTurn, recentBudget);
897
- if (splitIdx > onlyTurn.start && splitIdx < onlyTurn.end) {
898
- tailStartIdx = splitIdx;
899
- } else if (onlyTurn.end > onlyTurn.start + 1) {
900
- tailStartIdx = onlyTurn.start + 1;
901
- }
902
- }
903
- }
904
- return tailStartIdx;
905
- }
906
-
907
- // No-user path: pick the tail boundary from valid cut points. Walk newest ->
908
- // oldest, growing the tail across valid cut points while its suffix still fits
909
- // recentBudget, and stop before it overflows. Never anchors on a user turn, so
910
- // an assistant/tool-only single-turn transcript still yields a head to
911
- // summarize and a paired tail to keep.
912
- function selectTailStartByCutPoint(live, recentBudget, previousSummary) {
913
- const validCuts = findValidCutIndices(live);
914
- if (validCuts.length === 0) return live.length; // degenerate: only tool results
915
-
916
- let chosen = null;
917
- for (let k = validCuts.length - 1; k >= 0; k -= 1) {
918
- const idx = validCuts[k];
919
- if (estimateMessagesTokens(live.slice(idx)) <= recentBudget) {
920
- chosen = idx; // fits — try to grow the tail toward an older cut
921
- continue;
922
- }
923
- break; // this cut overflows recentBudget; keep the previous (newer) choice
924
- }
925
-
926
- if (chosen === null) {
927
- // Even the newest valid cut's suffix exceeds recentBudget (a single huge
928
- // message run). Keep the minimal tail from the newest valid cut so a head
929
- // remains to summarize; if that cut is at index 0 there is nothing to
930
- // split off, so keep everything in the head instead. The oversized tail
931
- // is tolerated downstream (mandatory-cost budget raise) rather than
932
- // throwing.
933
- const newestCut = validCuts[validCuts.length - 1];
934
- return newestCut > 0 ? newestCut : live.length;
935
- }
936
-
937
- if (chosen <= 0) {
938
- // Whole transcript would become the tail => nothing to compact. With no
939
- // prior summary to build on, pull the tail start forward to the next
940
- // valid cut so the leading message(s) become the compactable head.
941
- if (!previousSummary && validCuts.length >= 2) return validCuts[1];
942
- // Only ONE valid cut (or a leading tool run before it) and no prior
943
- // summary: there is no older cut to pull forward to. Returning 0 would
944
- // make the whole transcript the tail with an empty head, and
945
- // semanticCompactMessages throws on head.length===0 && !previousSummary.
946
- // Keep everything in the HEAD instead (empty tail) so a head remains to
947
- // summarize; an empty tail is valid downstream (mandatory = system+tail).
948
- if (!previousSummary && validCuts.length < 2) return live.length;
949
- return chosen;
950
- }
951
- return chosen;
952
- }
953
-
954
- function selectCompactionWindow(messages, budget, opts = {}) {
955
- const { system, live, previousSummary } = splitLiveCompactionContext(messages);
956
- const tailTurns = Math.max(1, Number(opts.tailTurns) || DEFAULT_TAIL_TURNS);
957
- const recentBudget = preserveRecentBudget(budget, opts);
958
-
959
- const tailStartIdx = userIndexes(live).length
960
- ? selectTailStartByTurns(live, recentBudget, tailTurns, previousSummary, opts)
961
- : selectTailStartByCutPoint(live, recentBudget, previousSummary);
962
-
963
- const head = live.slice(0, tailStartIdx);
964
- let tail = live.slice(tailStartIdx);
965
- // sanitizeToolPairs/dedup/reconcile repairs any orphan tool_result the cut
966
- // may have left; because valid cut points never start on a tool result, an
967
- // assistant tool_call and its trailing tool_results always land on the same
968
- // side of the boundary, so pairing stays provider-valid.
969
- tail = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(tail)));
970
-
971
- // Only a genuinely empty live window is unrecoverable. Absence of a user
972
- // turn in the tail is no longer an error.
973
- if (!head.length && !tail.length) {
974
- throw new Error('semanticCompactMessages: nothing to compact (empty live window)');
975
- }
976
-
977
- const preservedFacts = extractPreservedFacts(head);
978
- const originalHead = head;
979
- return {
980
- system,
981
- head,
982
- tail,
983
- previousSummary,
984
- originalHead,
985
- preservedFacts,
986
- };
987
- }
988
-
989
- function transcriptLineForCompaction(m, index, perMessageChars) {
990
- const role = m?.role || 'unknown';
991
- const text = truncateMiddle(extractText(m).trim(), perMessageChars);
992
- const meta = `${toolCallSummary(m, toolCallArgBudget(perMessageChars))}${toolResultId(m)}`;
993
- if (!text) return `${index + 1}. ${role}${meta}`;
994
- return `${index + 1}. ${role}${meta}:\n${text}`;
995
- }
996
-
997
- function buildCompactionPrompt({ head, previousSummary, preservedFacts }, perMessageChars) {
998
- const lines = [
999
- previousSummary
1000
- ? 'Update the anchored summary below using the conversation history that follows. Preserve still-true details, remove stale details, and merge in the new facts.'
1001
- : 'Create a new anchored summary from the conversation history below.',
1002
- SUMMARY_TEMPLATE,
1003
- ];
1004
- if (previousSummary) {
1005
- lines.push('', '<previous-summary>', previousSummary, '</previous-summary>');
1006
- }
1007
- if (preservedFacts) {
1008
- lines.push('', '<preserved-facts>', preservedFacts, '</preserved-facts>');
1009
- }
1010
- lines.push('', '<conversation-history>');
1011
- if (head.length === 0) {
1012
- lines.push('[No additional older messages before the preserved recent tail.]');
1013
- } else {
1014
- for (let i = 0; i < head.length; i += 1) {
1015
- lines.push(transcriptLineForCompaction(head[i], i, perMessageChars));
1016
- }
1017
- }
1018
- lines.push('</conversation-history>');
1019
- return lines.join('\n');
1020
- }
29
+ DEFAULT_COMPACT_TYPE,
30
+ COMPACT_TYPES,
31
+ normalizeCompactType,
32
+ compactTypeIsSemantic,
33
+ compactTypeIsRecallFastTrack,
34
+ } from './compact/constants.mjs';
1021
35
 
1022
- function estimateCompactionPromptTokens(input, perMessageChars) {
1023
- const prompt = buildCompactionPrompt(input, perMessageChars);
1024
- return estimateMessagesTokens([
1025
- { role: 'system', content: COMPACTION_SYSTEM_PROMPT },
1026
- { role: 'user', content: prompt },
1027
- ]);
1028
- }
36
+ export { redactToolCallSecretsInMessages } from './compact/text-utils.mjs';
1029
37
 
1030
- function previousSummaryBodyForCompactionPrompt(previousSummary) {
1031
- const text = String(previousSummary || '').trim();
1032
- if (!text) return '';
1033
- return stripNestedSummaryHeaderLines(text);
1034
- }
1035
-
1036
- function priorSummaryNeedsNormalization(text) {
1037
- const body = String(text || '').trim();
1038
- if (!body) return false;
1039
- if (!/^##\s+/m.test(body)) return true;
1040
- if (!summaryIsSchemaValid(body)) return true;
1041
- return summaryHasUnrecognizedHeadings(body);
1042
- }
1043
-
1044
- function normalizePriorSummaryForCompactionPrompt(fullBody) {
1045
- const text = String(fullBody || '').trim();
1046
- if (!text) return '';
1047
- if (!priorSummaryNeedsNormalization(text)) return text;
1048
- return repairSemanticSummary(text, { head: [], tail: [] });
1049
- }
1050
-
1051
- // Shrink or drop a prior anchored summary so the compaction provider prompt fits
1052
- // the call budget. Unstructured/legacy priors are repaired first; section
1053
- // anchors are preserved via truncateSummaryBySections;
1054
- // the last resort is omitting <previous-summary> entirely.
1055
- function fitPreviousSummaryForCompactionPrompt(input, perMessageChars, targetTokens) {
1056
- if (!input?.previousSummary) return input;
1057
- const fullBody = normalizePriorSummaryForCompactionPrompt(
1058
- previousSummaryBodyForCompactionPrompt(input.previousSummary),
1059
- );
1060
- const withSummary = (summaryText) => {
1061
- const trimmed = String(summaryText || '').trim();
1062
- if (!trimmed) return { ...input, previousSummary: null };
1063
- return { ...input, previousSummary: trimmed };
1064
- };
1065
-
1066
- if (estimateCompactionPromptTokens(withSummary(fullBody), perMessageChars) <= targetTokens) {
1067
- return withSummary(fullBody);
1068
- }
1069
-
1070
- if (fullBody) {
1071
- let lo = 0;
1072
- let hi = fullBody.length;
1073
- let bestChars = -1;
1074
- while (lo <= hi) {
1075
- const mid = Math.floor((lo + hi) / 2);
1076
- const truncated = truncateSummaryBySections(fullBody, mid);
1077
- const candidate = withSummary(truncated);
1078
- if (estimateCompactionPromptTokens(candidate, perMessageChars) <= targetTokens) {
1079
- bestChars = mid;
1080
- lo = mid + 1;
1081
- } else {
1082
- hi = mid - 1;
1083
- }
1084
- }
1085
- if (bestChars >= 0) {
1086
- return withSummary(truncateSummaryBySections(fullBody, bestChars));
1087
- }
1088
- }
1089
-
1090
- const minimalPrior = minimalSchemaSummary();
1091
- if (estimateCompactionPromptTokens(withSummary(minimalPrior), perMessageChars) <= targetTokens) {
1092
- return withSummary(minimalPrior);
1093
- }
1094
-
1095
- const withoutPrior = withSummary(null);
1096
- if (estimateCompactionPromptTokens(withoutPrior, perMessageChars) <= targetTokens) {
1097
- return withoutPrior;
1098
- }
1099
-
1100
- return null;
1101
- }
1102
-
1103
- function fitCompactionPrompt(input, targetTokens) {
1104
- const tryFit = (withFacts) => {
1105
- const baseInp = withFacts ? input : { ...input, preservedFacts: null };
1106
-
1107
- const fitAt = (perMessageChars) => {
1108
- let inp = baseInp;
1109
- if (estimateCompactionPromptTokens(inp, perMessageChars) > targetTokens) {
1110
- const fitted = fitPreviousSummaryForCompactionPrompt(inp, perMessageChars, targetTokens);
1111
- if (!fitted) return null;
1112
- inp = fitted;
1113
- if (estimateCompactionPromptTokens(inp, perMessageChars) > targetTokens) return null;
1114
- }
1115
- return buildCompactionPrompt(inp, perMessageChars);
1116
- };
1117
-
1118
- const minimalPrompt = fitAt(0);
1119
- if (!minimalPrompt) return null;
1120
-
1121
- let maxText = 0;
1122
- for (const m of baseInp.head) maxText = Math.max(maxText, extractText(m).length);
1123
- let lo = 0;
1124
- let hi = Math.min(COMPACTION_INPUT_MAX_CHARS, Math.max(maxText, 0));
1125
- let best = minimalPrompt;
1126
- while (lo <= hi) {
1127
- const mid = Math.floor((lo + hi) / 2);
1128
- const candidate = fitAt(mid);
1129
- if (candidate) {
1130
- best = candidate;
1131
- lo = mid + 1;
1132
- } else {
1133
- hi = mid - 1;
1134
- }
1135
- }
1136
- return best;
1137
- };
1138
- if (input.preservedFacts) {
1139
- const withFacts = tryFit(true);
1140
- if (withFacts && estimateMessagesTokens([
1141
- { role: 'system', content: COMPACTION_SYSTEM_PROMPT },
1142
- { role: 'user', content: withFacts },
1143
- ]) <= targetTokens) return withFacts;
1144
- }
1145
- const fitted = tryFit(false);
1146
- if (fitted) return fitted;
1147
-
1148
- // Emergency deterministic reduction: even at perMessageChars=0 the prompt can
1149
- // overflow when the head carries a very large NUMBER of messages (each still
1150
- // emits a `N. role` line). Keep only the newest K head messages and collapse
1151
- // the rest into a single `[K older messages omitted]` stub line, binary
1152
- // searching the largest K that fits. This bounds the head by COUNT, not just
1153
- // per-message chars, so a huge-head transcript still yields a minimal prompt
1154
- // instead of null (which surfaced as a hard compaction throw).
1155
- const head = Array.isArray(input.head) ? input.head : [];
1156
- const baseNoFacts = { ...input, preservedFacts: null };
1157
- const buildReduced = (k) => {
1158
- const kept = k > 0 ? head.slice(head.length - k) : [];
1159
- const omitted = head.length - kept.length;
1160
- const stubHead = omitted > 0
1161
- ? [{ role: 'user', content: `[${omitted} older messages omitted]` }, ...kept]
1162
- : kept;
1163
- let inp = { ...baseNoFacts, head: stubHead };
1164
- // Also shrink/drop a prior <previous-summary> (same as the normal fitAt
1165
- // path) — a large prior summary can keep the prompt over budget even at
1166
- // K=0. fitPreviousSummaryForCompactionPrompt is a no-op when there is no
1167
- // previousSummary, so this is safe for the summary-less case.
1168
- if (estimateCompactionPromptTokens(inp, 0) > targetTokens) {
1169
- const fitted = fitPreviousSummaryForCompactionPrompt(inp, 0, targetTokens);
1170
- if (!fitted) return null;
1171
- inp = fitted;
1172
- if (estimateCompactionPromptTokens(inp, 0) > targetTokens) return null;
1173
- }
1174
- return buildCompactionPrompt(inp, 0);
1175
- };
1176
- let lo = 0;
1177
- let hi = head.length;
1178
- let best = null;
1179
- while (lo <= hi) {
1180
- const mid = Math.floor((lo + hi) / 2);
1181
- const candidate = buildReduced(mid);
1182
- if (candidate) { best = candidate; lo = mid + 1; }
1183
- else hi = mid - 1;
1184
- }
1185
- return best;
1186
- }
1187
-
1188
- function extractResponseText(response) {
1189
- if (!response) return '';
1190
- if (typeof response.content === 'string') return response.content.trim();
1191
- if (Array.isArray(response.content)) {
1192
- return response.content
1193
- .map((item) => {
1194
- if (typeof item === 'string') return item;
1195
- if (typeof item?.text === 'string') return item.text;
1196
- if (typeof item?.content === 'string') return item.content;
1197
- return '';
1198
- })
1199
- .filter(Boolean)
1200
- .join('\n')
1201
- .trim();
1202
- }
1203
- return '';
1204
- }
1205
-
1206
- // Canonical section anchors the semantic summary template (SUMMARY_TEMPLATE)
1207
- // must contain. Used for lightweight schema validation of provider output.
1208
- const REQUIRED_SUMMARY_SECTIONS = Object.freeze([
1209
- '## Goal',
1210
- '## Constraints',
1211
- '## Progress',
1212
- '## Key Decisions',
1213
- '## Next Steps',
1214
- '## Critical Context',
1215
- '## Relevant Files',
1216
- ]);
1217
-
1218
- // Collect actual top-level (`## `, not `### `) heading lines from a summary.
1219
- // Validation is heading-anchor based (not substring includes) so prose or code
1220
- // that merely mentions "## Relevant Files" inside a bullet body cannot satisfy
1221
- // a section anchor.
1222
- function summaryHeadingLines(summary) {
1223
- const out = [];
1224
- for (const rawLine of String(summary || '').split('\n')) {
1225
- const line = rawLine.trim();
1226
- if (/^##\s+\S/.test(line) && !/^###\s+/.test(line)) out.push(line);
1227
- }
1228
- return out;
1229
- }
1230
-
1231
- // An anchor matches a heading when the heading title equals the anchor title or
1232
- // extends it at a word/punctuation boundary. This lets `## Constraints &
1233
- // Preferences` satisfy the `## Constraints` anchor while NOT letting an
1234
- // unrelated `## Goalkeeper` heading satisfy `## Goal`. Requires a real `## `
1235
- // heading line (not a substring buried in prose).
1236
- function headingMatchesAnchor(heading, anchor) {
1237
- const anchorTitle = anchor.replace(/^##\s+/, '').trim().toLowerCase();
1238
- const headingTitle = heading.replace(/^##\s+/, '').trim().toLowerCase();
1239
- if (headingTitle === anchorTitle) return true;
1240
- if (!headingTitle.startsWith(anchorTitle)) return false;
1241
- // Next char after the anchor title must be a boundary (space or &/punct),
1242
- // not a continuation letter/digit.
1243
- const nextChar = headingTitle.charAt(anchorTitle.length);
1244
- return /[\s&:(-]/.test(nextChar);
1245
- }
1246
-
1247
- function summarySchemaScore(summary) {
1248
- const headings = summaryHeadingLines(summary);
1249
- let hits = 0;
1250
- for (const anchor of REQUIRED_SUMMARY_SECTIONS) {
1251
- if (headings.some((h) => headingMatchesAnchor(h, anchor))) hits += 1;
1252
- }
1253
- return hits;
1254
- }
1255
-
1256
- // A summary is schema-valid only when EVERY required section anchor is present
1257
- // as a real heading. A partial summary (e.g. missing Critical Context /
1258
- // Relevant Files) must be repaired rather than injected unchanged.
1259
- function summaryIsSchemaValid(summary) {
1260
- if (summarySchemaScore(summary) !== REQUIRED_SUMMARY_SECTIONS.length) return false;
1261
- return !summaryHasUnrecognizedHeadings(summary);
1262
- }
1263
-
1264
- function deriveRelevantFilesBullets(head) {
1265
- const seen = new Set();
1266
- const out = [];
1267
- const fileRe = /(?:[A-Za-z]:[\\/]|\.{1,2}[\\/]|[\w$.-]+[\\/])?[\w$.-]+\.(?:mjs|cjs|js|jsx|ts|tsx|json|md|rs|go|py|java|kt|cs|cpp|c|h|hpp|css|html|yml|yaml|toml|lock|sh|ps1)\b/gi;
1268
- for (const m of Array.isArray(head) ? head : []) {
1269
- const text = extractText(m);
1270
- if (!text) continue;
1271
- let match;
1272
- while ((match = fileRe.exec(text)) && out.length < 8) {
1273
- const file = match[0];
1274
- const key = file.toLowerCase();
1275
- if (seen.has(key)) continue;
1276
- seen.add(key);
1277
- out.push(`- ${file}`);
1278
- }
1279
- if (out.length >= 8) break;
1280
- }
1281
- return out;
1282
- }
1283
-
1284
- function deriveCurrentRequest(messages) {
1285
- for (let i = (Array.isArray(messages) ? messages.length : 0) - 1; i >= 0; i -= 1) {
1286
- const m = messages[i];
1287
- if (m?.role === 'user' && !isProtectedContextUserMessage(m) && !isInjectedSkillBodyMessage(m)) {
1288
- const text = truncateMiddle(extractText(m).trim(), 400);
1289
- if (text) return text;
1290
- }
1291
- }
1292
- return '';
1293
- }
1294
-
1295
- // Canonical ordered section headings the structured summary scaffold emits.
1296
- // Each `## ` heading maps to one REQUIRED_SUMMARY_SECTIONS anchor; Progress
1297
- // additionally carries its three `### ` sub-headings.
1298
- const SUMMARY_SECTION_LAYOUT = Object.freeze([
1299
- { heading: '## Goal', anchor: '## Goal' },
1300
- { heading: '## Constraints & Preferences', anchor: '## Constraints' },
1301
- { heading: '## Progress', anchor: '## Progress', sub: ['### Done', '### In Progress', '### Blocked'] },
1302
- { heading: '## Key Decisions', anchor: '## Key Decisions' },
1303
- { heading: '## Next Steps', anchor: '## Next Steps' },
1304
- { heading: '## Critical Context', anchor: '## Critical Context' },
1305
- { heading: '## Relevant Files', anchor: '## Relevant Files' },
1306
- ]);
1307
-
1308
- // Split a markdown summary into a map of top-level `## ` heading -> body lines.
1309
- function parseSummarySections(text) {
1310
- const map = new Map();
1311
- let current = null;
1312
- for (const rawLine of String(text || '').split('\n')) {
1313
- const trimmed = rawLine.trim();
1314
- const line = rawLine.replace(/\s+$/, '');
1315
- if (/^##\s+/.test(trimmed) && !/^###\s+/.test(trimmed)) {
1316
- current = trimmed;
1317
- if (!map.has(current)) map.set(current, []);
1318
- continue;
1319
- }
1320
- if (current) map.get(current).push(line);
1321
- }
1322
- return map;
1323
- }
1324
-
1325
- function summarySectionIsRecognized(heading) {
1326
- for (const section of SUMMARY_SECTION_LAYOUT) {
1327
- if (headingMatchesAnchor(heading, section.anchor)) return true;
1328
- }
1329
- return false;
1330
- }
1331
-
1332
- function summaryHasUnrecognizedHeadings(summary) {
1333
- for (const heading of parseSummarySections(summary).keys()) {
1334
- if (!summarySectionIsRecognized(heading)) return true;
1335
- }
1336
- return false;
1337
- }
1338
-
1339
- function summaryLinesToBullets(text) {
1340
- return String(text || '')
1341
- .split('\n')
1342
- .map((l) => l.trim())
1343
- .filter(Boolean)
1344
- .map((l) => (l.startsWith('-') ? l : `- ${l}`));
1345
- }
1346
-
1347
- function unrecognizedSummarySectionText(present) {
1348
- const chunks = [];
1349
- for (const [heading, body] of present) {
1350
- if (summarySectionIsRecognized(heading)) continue;
1351
- const lines = [heading];
1352
- for (const line of body || []) {
1353
- const trimmed = String(line).trim();
1354
- if (trimmed) lines.push(line);
1355
- }
1356
- chunks.push(lines.join('\n'));
1357
- }
1358
- return chunks.join('\n\n').trim();
1359
- }
1360
-
1361
- // Deterministic schema repair for a non-empty but malformed/partial semantic
1362
- // summary. Preserve every section the provider DID supply (matched by anchor),
1363
- // and scaffold the missing required sections so downstream consumers always
1364
- // receive the full structured anchored shape. Content that lives outside any
1365
- // recognized section is routed into Critical Context so nothing is dropped.
1366
- // Lightly backfill Goal / Relevant Files from the transcript when empty.
1367
- function repairSemanticSummary(summary, { head = [], tail = [] } = {}) {
1368
- const raw = String(summary || '').trim();
1369
- const present = parseSummarySections(raw);
1370
- // Capture any leading content before the first recognized `## ` heading so
1371
- // an entirely unstructured blob is preserved rather than silently dropped.
1372
- let preamble = '';
1373
- if (raw) {
1374
- const firstHeading = raw.search(/(^|\n)\s*##\s+/);
1375
- preamble = firstHeading === -1 ? raw : raw.slice(0, firstHeading);
1376
- preamble = preamble.trim();
1377
- }
1378
- const orphanText = unrecognizedSummarySectionText(present);
1379
- const extraContextParts = [];
1380
- if (preamble) extraContextParts.push(preamble);
1381
- if (orphanText) extraContextParts.push(orphanText);
1382
- const extraContext = extraContextParts.join('\n\n').trim();
1383
- const bulletize = (lines) => {
1384
- const cleaned = (Array.isArray(lines) ? lines : [])
1385
- .map((l) => String(l).trim())
1386
- .filter(Boolean);
1387
- return cleaned.length ? cleaned : null;
1388
- };
1389
- const findPresent = (anchor) => {
1390
- for (const [heading, body] of present) {
1391
- if (headingMatchesAnchor(heading, anchor)) return body;
1392
- }
1393
- return null;
1394
- };
1395
- const goal = deriveCurrentRequest(tail) || deriveCurrentRequest(head);
1396
- const files = deriveRelevantFilesBullets(head);
1397
- const out = [];
1398
- for (const section of SUMMARY_SECTION_LAYOUT) {
1399
- if (out.length) out.push('');
1400
- out.push(section.heading);
1401
- const body = bulletize(findPresent(section.anchor));
1402
- if (section.sub) {
1403
- // Progress: keep provider sub-bodies when present, else scaffold.
1404
- if (body) {
1405
- out.push(...body);
1406
- } else {
1407
- for (const sub of section.sub) {
1408
- out.push(sub, '- (none)');
1409
- }
1410
- }
1411
- continue;
1412
- }
1413
- if (section.anchor === '## Critical Context') {
1414
- const ccLines = [];
1415
- if (body) ccLines.push(...body);
1416
- for (const line of summaryLinesToBullets(extraContext)) {
1417
- if (!ccLines.some((existing) => existing.trim() === line.trim())) ccLines.push(line);
1418
- }
1419
- if (ccLines.some((line) => line.trim() !== '- (none)')) {
1420
- const withoutPlaceholder = ccLines.filter((line) => line.trim() !== '- (none)');
1421
- out.push(...(withoutPlaceholder.length ? withoutPlaceholder : ccLines));
1422
- } else {
1423
- out.push(...(ccLines.length ? ccLines : ['- (none)']));
1424
- }
1425
- continue;
1426
- }
1427
- if (body) {
1428
- out.push(...body);
1429
- continue;
1430
- }
1431
- if (section.anchor === '## Goal') {
1432
- out.push(goal ? `- ${goal}` : '- (none)');
1433
- } else if (section.anchor === '## Relevant Files') {
1434
- out.push(...(files.length ? files : ['- (none)']));
1435
- } else {
1436
- out.push('- (none)');
1437
- }
1438
- }
1439
- return out.join('\n');
1440
- }
1441
-
1442
- // Validate the provider summary against the required template sections; when it
1443
- // is missing ANY required section anchor (fully or partially malformed) repair
1444
- // it deterministically so a non-empty-but-broken response is never injected as
1445
- // the sole summary. Returns { summary, repaired }.
1446
- function enforceSemanticSummarySchema(summary, ctx = {}) {
1447
- const text = String(summary || '').trim();
1448
- if (!text) return { summary: text, repaired: false };
1449
- if (summaryIsSchemaValid(text)) {
1450
- return { summary: text, repaired: false };
1451
- }
1452
- return { summary: repairSemanticSummary(text, ctx), repaired: true };
1453
- }
1454
-
1455
- function makeSemanticSummaryMessage(oldHistory, summary, semanticMeta = {}, preservedFacts = '') {
1456
- const header = compactHeader(oldHistory);
1457
- header.push(`compact_type=${COMPACT_TYPE_SEMANTIC}`);
1458
- header.push(`semantic=true provider=${semanticMeta.provider || 'unknown'} model=${semanticMeta.model || 'unknown'}`);
1459
- const facts = String(preservedFacts || '').trim();
1460
- const body = String(summary || '').trim();
1461
- const parts = [header.join('\n')];
1462
- if (facts) parts.push(facts);
1463
- if (body) parts.push(body);
1464
- return makeSummaryMessage(parts.join('\n\n'));
1465
- }
1466
-
1467
- export function buildRecallFastTrackQuery(messages, opts = {}) {
1468
- const maxChars = Math.max(200, Number(opts.maxChars) || 2_000);
1469
- const hints = String(opts.hints || 'current task decisions constraints file paths changed files verification failures next steps').trim();
1470
- let latestUser = '';
1471
- const recent = [];
1472
- const input = Array.isArray(messages) ? messages : [];
1473
- for (let i = input.length - 1; i >= 0; i -= 1) {
1474
- const m = input[i];
1475
- const text = extractText(m).trim();
1476
- if (!text) continue;
1477
- if (recent.length < 6) recent.unshift(text);
1478
- if (!latestUser && m?.role === 'user' && !isProtectedContextUserMessage(m) && !isInjectedSkillBodyMessage(m)) {
1479
- latestUser = text;
1480
- }
1481
- if (latestUser && recent.length >= 6) break;
1482
- }
1483
- const parts = [latestUser, hints, recent.join('\n')]
1484
- .map((s) => String(s || '').trim())
1485
- .filter(Boolean);
1486
- return truncateMiddle([...new Set(parts)].join('\n'), maxChars);
1487
- }
1488
-
1489
- // A headings-only structured summary: every required `## ` (and Progress `### `)
1490
- // anchor present with `- (none)` bodies. This is the minimal schema-valid shape
1491
- // the fitter can fall back to when token pressure cannot hold real section
1492
- // bodies — it still passes summaryIsSchemaValid so the injected message is
1493
- // never partial.
1494
- function minimalSchemaSummary() {
1495
- const out = [];
1496
- for (const section of SUMMARY_SECTION_LAYOUT) {
1497
- if (out.length) out.push('');
1498
- out.push(section.heading);
1499
- if (section.sub) {
1500
- for (const sub of section.sub) out.push(sub, '- (none)');
1501
- } else {
1502
- out.push('- (none)');
1503
- }
1504
- }
1505
- return out.join('\n');
1506
- }
1507
-
1508
- // Section-aware truncation: keep EVERY `## ` heading and Progress `### `
1509
- // sub-heading intact, trimming only section bodies to `perSectionChars`. Unlike
1510
- // a raw text.slice(0, n) this never drops a trailing required section, so the
1511
- // result stays schema-valid (all anchors present) at any budget.
1512
- function truncateSummaryBySections(summary, perSectionChars) {
1513
- const sections = parseSummarySections(summary);
1514
- const out = [];
1515
- for (const section of SUMMARY_SECTION_LAYOUT) {
1516
- if (out.length) out.push('');
1517
- out.push(section.heading);
1518
- let body = null;
1519
- for (const [heading, lines] of sections) {
1520
- if (headingMatchesAnchor(heading, section.anchor)) { body = lines; break; }
1521
- }
1522
- const bodyText = (Array.isArray(body) ? body : [])
1523
- .map((l) => String(l).trim())
1524
- .filter(Boolean)
1525
- .join('\n');
1526
- if (!bodyText) {
1527
- if (section.sub) for (const sub of section.sub) out.push(sub, '- (none)');
1528
- else out.push('- (none)');
1529
- continue;
1530
- }
1531
- const trimmed = perSectionChars > 0 ? truncateMiddle(bodyText, perSectionChars) : '';
1532
- if (trimmed) out.push(trimmed);
1533
- else if (section.sub) for (const sub of section.sub) out.push(sub, '- (none)');
1534
- else out.push('- (none)');
1535
- }
1536
- return out.join('\n');
1537
- }
1538
-
1539
- // Fit the structured semantic summary into the remaining token budget WITHOUT
1540
- // dropping any required section. The incoming `summary` is already schema-valid
1541
- // (enforceSemanticSummarySchema ran upstream); here we shrink section bodies via
1542
- // section-aware truncation, fall back to a headings-only schema-valid summary,
1543
- // and finally revalidate so the injected SUMMARY_PREFIX message always carries
1544
- // every required anchor. Returns null only when even the minimal schema-valid
1545
- // summary cannot fit (caller throws).
1546
- function fitSemanticSummaryMessage(oldHistory, summary, remainingTokens, semanticMeta, preservedFacts = '') {
1547
- const tryFit = (factsText) => {
1548
- const text = String(summary || '').trim();
1549
- // Minimal schema-valid body (headings + "(none)"). If even this does
1550
- // not fit, this facts variant cannot produce a valid message.
1551
- const minimalBody = text ? minimalSchemaSummary() : '';
1552
- const minimal = makeSemanticSummaryMessage(oldHistory, minimalBody, semanticMeta, factsText);
1553
- if (estimateMessagesTokens([minimal]) > remainingTokens) return null;
1554
- if (!text) return minimal;
1555
- // Binary search the per-section body budget; keep all anchors intact.
1556
- let lo = 0;
1557
- let hi = text.length;
1558
- let best = minimal;
1559
- while (lo <= hi) {
1560
- const mid = Math.floor((lo + hi) / 2);
1561
- const body = truncateSummaryBySections(text, mid);
1562
- const candidate = makeSemanticSummaryMessage(oldHistory, body, semanticMeta, factsText);
1563
- if (estimateMessagesTokens([candidate]) <= remainingTokens && summaryIsSchemaValid(body)) {
1564
- best = candidate;
1565
- lo = mid + 1;
1566
- } else {
1567
- hi = mid - 1;
1568
- }
1569
- }
1570
- return best;
1571
- };
1572
- let result = null;
1573
- if (preservedFacts) result = tryFit(preservedFacts);
1574
- if (!result) result = tryFit('');
1575
- return result;
1576
- }
1577
-
1578
- // Recall fast-track (compact type 2) does no LLM summarization — it just emits
1579
- // the chunked history in order. Keep the message clean: the mandatory anchor
1580
- // header (so selectCompactionWindow / clear-preserve / TUI can recognize the
1581
- // compact message) plus the chunk text itself. No "Preserved Facts" extraction,
1582
- // no "Recall Fast-Track Context" heading, no "(no recall hits)" filler.
1583
- function makeRecallFastTrackSummaryMessage(oldHistory, recallText, recallMeta = {}) {
1584
- return makeRecallFastTrackSummaryMessageParts(oldHistory, recallText, '', recallMeta);
1585
- }
1586
-
1587
- const RECALL_TAIL_TRUNCATION_MARKER = '[... truncated during recall tail preservation ...]';
1588
- const RECALL_TAIL_SHORT_TRUNCATION_MARKER = '[truncated]';
1589
-
1590
- const PRIOR_COMPACTED_CONTEXT_OPEN = '<prior-compacted-context>';
1591
- const PRIOR_COMPACTED_CONTEXT_CLOSE = '</prior-compacted-context>';
1592
-
1593
- function formatPriorCompactedContextBlock(priorText) {
1594
- const prior = String(priorText || '').trim();
1595
- if (!prior) return '';
1596
- return `${PRIOR_COMPACTED_CONTEXT_OPEN}\n${prior}\n${PRIOR_COMPACTED_CONTEXT_CLOSE}`;
1597
- }
1598
-
1599
- function makeRecallFastTrackSummaryMessageParts(oldHistory, recallPart, priorPart, recallMeta = {}) {
1600
- const header = compactHeader(oldHistory);
1601
- header.push(`compact_type=${COMPACT_TYPE_RECALL_FASTTRACK} source=recall-fasttrack query_sha=${recallMeta.querySha || 'none'}`);
1602
- const parts = [header.join('\n')];
1603
- const priorBlock = formatPriorCompactedContextBlock(priorPart);
1604
- if (priorBlock) parts.push(priorBlock);
1605
- const recall = String(recallPart || '').trim();
1606
- if (recall) parts.push(recall);
1607
- return makeSummaryMessage(parts.join('\n\n'));
1608
- }
1609
-
1610
- function fitRecallFastTrackSummaryMessage(oldHistory, recallText, remainingTokens, recallMeta = {}, priorPart = '') {
1611
- const recall = String(recallText || '').trim();
1612
- const prior = String(priorPart || '').trim();
1613
-
1614
- let fittedPrior = prior;
1615
- if (prior) {
1616
- let lo = 0;
1617
- let hi = prior.length;
1618
- let bestPriorLen = 0;
1619
- while (lo <= hi) {
1620
- const mid = Math.floor((lo + hi) / 2);
1621
- const candidate = makeRecallFastTrackSummaryMessageParts(oldHistory, '', prior.slice(0, mid), recallMeta);
1622
- if (estimateMessagesTokens([candidate]) <= remainingTokens) {
1623
- bestPriorLen = mid;
1624
- lo = mid + 1;
1625
- } else {
1626
- hi = mid - 1;
1627
- }
1628
- }
1629
- fittedPrior = prior.slice(0, bestPriorLen);
1630
- if (!fittedPrior && prior) {
1631
- const markerOnly = makeRecallFastTrackSummaryMessageParts(oldHistory, '', RECALL_TAIL_TRUNCATION_MARKER, recallMeta);
1632
- if (estimateMessagesTokens([markerOnly]) <= remainingTokens) {
1633
- fittedPrior = RECALL_TAIL_TRUNCATION_MARKER;
1634
- }
1635
- }
1636
- }
1637
-
1638
- const minimal = makeRecallFastTrackSummaryMessageParts(oldHistory, '', fittedPrior, recallMeta);
1639
- if (estimateMessagesTokens([minimal]) > remainingTokens) return null;
1640
- if (!recall) return minimal;
1641
-
1642
- let lo = 0;
1643
- let hi = recall.length;
1644
- let best = minimal;
1645
- while (lo <= hi) {
1646
- const mid = Math.floor((lo + hi) / 2);
1647
- const candidate = makeRecallFastTrackSummaryMessageParts(oldHistory, recall.slice(0, mid), fittedPrior, recallMeta);
1648
- if (estimateMessagesTokens([candidate]) <= remainingTokens) {
1649
- best = candidate;
1650
- lo = mid + 1;
1651
- } else {
1652
- hi = mid - 1;
1653
- }
1654
- }
1655
- return best;
1656
- }
1657
-
1658
- // --- Smart-compact root-based fitting (arrival-time replacement) -----------
1659
- //
1660
- // dump_session_roots (memory/index.mjs dumpSessionRootChunks) renders chunks
1661
- // TIME-ORDERED (oldest first; chunks.sort by sourceTurn/ts/id ascending),
1662
- // each root/raw block starting with one of:
1663
- // # chunk N root=ID[ category=X]
1664
- // # raw_pending N id=ID
1665
- // # raw_terminal N id=ID
1666
- // and blocks joined by "\n\n". runRecallFastTrackForSession additionally
1667
- // prepends a "session_id=..." / cycle1-drain-status preamble before the dump
1668
- // text (also "\n\n"-joined) — preserved verbatim as a non-block segment.
1669
- //
1670
- // Unlike fitRecallFastTrackSummaryMessage (character-slice binary search,
1671
- // used by the LLM-summary-free but still-mid-turn recall-fasttrack compact
1672
- // path), the smart-compact arrival path must never cut a root block
1673
- // mid-entry — losing half a root's content silently corrupts that entry.
1674
- // This splitter finds block boundaries by the label pattern (robust to
1675
- // blank lines inside member/raw content, since it anchors on the distinctive
1676
- // "# chunk /raw_pending/raw_terminal" line rather than a blank-line split).
1677
- const RECALL_ROOT_BLOCK_HEADER_RE = /^# (?:chunk \d+ root=\d+(?: category=\S+)?|raw_pending \d+ id=\d+|raw_terminal \d+ id=\d+)[ \t]*$/;
1678
-
1679
- export function splitRecallRootBlocks(text) {
1680
- const value = String(text || '');
1681
- if (!value.trim()) return { preamble: '', blocks: [] };
1682
- const re = new RegExp(RECALL_ROOT_BLOCK_HEADER_RE.source, 'gm');
1683
- const starts = [];
1684
- let m;
1685
- while ((m = re.exec(value)) !== null) {
1686
- starts.push(m.index);
1687
- if (re.lastIndex === m.index) re.lastIndex += 1; // zero-width guard, defensive
1688
- }
1689
- if (starts.length === 0) return { preamble: value.trim(), blocks: [] };
1690
- const preamble = value.slice(0, starts[0]).trim();
1691
- const blocks = [];
1692
- for (let i = 0; i < starts.length; i += 1) {
1693
- const start = starts[i];
1694
- const end = i + 1 < starts.length ? starts[i + 1] : value.length;
1695
- const raw = value.slice(start, end).trim();
1696
- if (raw) blocks.push(raw);
1697
- }
1698
- return { preamble, blocks };
1699
- }
1700
-
1701
- // Minimal-header summary-message wrapper for the smart-compact roots path.
1702
- // Keeps the SUMMARY_PREFIX anchor (isSummaryMessage / selectCompactionWindow
1703
- // / clear-preserve / TUI all key off startsWith(SUMMARY_PREFIX)) but skips the
1704
- // full sha256/roleCounts header line that fitRecallFastTrackSummaryMessage
1705
- // computes — smart-arrival replacement is a lightweight prefix swap, not the
1706
- // anchored LLM-summary compact, so a heavy per-call header is unneeded cost.
1707
- function makeRecallRootsSummaryMessageParts(oldHistory, rootsPart, priorPart, recallMeta = {}) {
1708
- const header = `${SUMMARY_PREFIX}\nmessages=${(oldHistory || []).length} compact_type=${COMPACT_TYPE_RECALL_FASTTRACK} source=smart-arrival query_sha=${recallMeta.querySha || 'none'}`;
1709
- const parts = [header];
1710
- const priorBlock = formatPriorCompactedContextBlock(priorPart);
1711
- if (priorBlock) parts.push(priorBlock);
1712
- const roots = String(rootsPart || '').trim();
1713
- if (roots) parts.push(roots);
1714
- return makeSummaryMessage(parts.join('\n\n'));
1715
- }
1716
-
1717
- // Root-block-aware fit for the smart-compact arrival path (Step1). Mirrors
1718
- // fitRecallFastTrackSummaryMessage's prior-block binary-search fit, but the
1719
- // recall body is fit at ROOT-BLOCK granularity: when the full set of root
1720
- // blocks (kept in original time order) exceeds remainingTokens, the OLDEST
1721
- // blocks are dropped WHOLE (never character-truncated mid-block) until the
1722
- // remaining (newest-biased) suffix fits. Because dropping more leading
1723
- // blocks can only shrink (never grow) the serialized size, the minimal-drop
1724
- // threshold is found via binary search on the drop count.
1725
- export function fitRecallRootsMessage(oldHistory, recallText, remainingTokens, recallMeta = {}, priorPart = '') {
1726
- const prior = String(priorPart || '').trim();
1727
-
1728
- let fittedPrior = prior;
1729
- if (prior) {
1730
- let lo = 0;
1731
- let hi = prior.length;
1732
- let bestPriorLen = 0;
1733
- while (lo <= hi) {
1734
- const mid = Math.floor((lo + hi) / 2);
1735
- const candidate = makeRecallRootsSummaryMessageParts(oldHistory, '', prior.slice(0, mid), recallMeta);
1736
- if (estimateMessagesTokens([candidate]) <= remainingTokens) {
1737
- bestPriorLen = mid;
1738
- lo = mid + 1;
1739
- } else {
1740
- hi = mid - 1;
1741
- }
1742
- }
1743
- fittedPrior = prior.slice(0, bestPriorLen);
1744
- if (!fittedPrior && prior) {
1745
- const markerOnly = makeRecallRootsSummaryMessageParts(oldHistory, '', RECALL_TAIL_TRUNCATION_MARKER, recallMeta);
1746
- if (estimateMessagesTokens([markerOnly]) <= remainingTokens) {
1747
- fittedPrior = RECALL_TAIL_TRUNCATION_MARKER;
1748
- }
1749
- }
1750
- }
1751
-
1752
- const minimal = makeRecallRootsSummaryMessageParts(oldHistory, '', fittedPrior, recallMeta);
1753
- if (estimateMessagesTokens([minimal]) > remainingTokens) return null;
1754
-
1755
- const { preamble, blocks } = splitRecallRootBlocks(recallText);
1756
- if (blocks.length === 0) {
1757
- // No parseable root-block boundaries (empty / non-dump recallText) —
1758
- // degrade to keep-whole-if-it-fits, else drop entirely. Never
1759
- // mid-truncates: a non-block body is treated as a single atomic unit.
1760
- const whole = String(recallText || '').trim();
1761
- if (!whole) return minimal;
1762
- const full = makeRecallRootsSummaryMessageParts(oldHistory, whole, fittedPrior, recallMeta);
1763
- if (estimateMessagesTokens([full]) <= remainingTokens) return full;
1764
- return minimal;
1765
- }
1766
-
1767
- let loB = 0;
1768
- let hiB = blocks.length;
1769
- let bestLo = -1;
1770
- while (loB <= hiB) {
1771
- const mid = Math.floor((loB + hiB) / 2);
1772
- const kept = blocks.slice(mid);
1773
- const body = [preamble, ...kept].filter(Boolean).join('\n\n');
1774
- const candidate = makeRecallRootsSummaryMessageParts(oldHistory, body, fittedPrior, recallMeta);
1775
- if (estimateMessagesTokens([candidate]) <= remainingTokens) {
1776
- bestLo = mid;
1777
- hiB = mid - 1;
1778
- } else {
1779
- loB = mid + 1;
1780
- }
1781
- }
1782
- if (bestLo >= 0) {
1783
- const kept = blocks.slice(bestLo);
1784
- const body = [preamble, ...kept].filter(Boolean).join('\n\n');
1785
- return makeRecallRootsSummaryMessageParts(oldHistory, body, fittedPrior, recallMeta);
1786
- }
1787
- // Even zero root blocks (preamble alone) overflows alongside the fitted
1788
- // prior — try preamble alone, else the no-recall minimal header.
1789
- if (preamble) {
1790
- const preambleOnly = makeRecallRootsSummaryMessageParts(oldHistory, preamble, fittedPrior, recallMeta);
1791
- if (estimateMessagesTokens([preambleOnly]) <= remainingTokens) return preambleOnly;
1792
- }
1793
- return minimal;
1794
- }
1795
-
1796
- function combinedSignal(parent, timeoutMs) {
1797
- const ms = Number(timeoutMs);
1798
- if (!Number.isFinite(ms) || ms <= 0) return parent || undefined;
1799
- const timeout = AbortSignal.timeout(Math.floor(ms));
1800
- if (parent && typeof AbortSignal.any === 'function') return AbortSignal.any([parent, timeout]);
1801
- return timeout;
1802
- }
1803
-
1804
- export async function semanticCompactMessages(provider, messages, model, budgetTokens, opts = {}) {
1805
- if (!provider || typeof provider.send !== 'function') {
1806
- throw new Error('semanticCompactMessages: provider.send is required');
1807
- }
1808
- const startedAt = Date.now();
1809
- let budget = effectiveBudget(budgetTokens, opts);
1810
- const baseSanitized = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(messages)));
1811
- const baseTokens = safeEstimateMessagesTokens(baseSanitized);
1812
- // No-op fast path: if the original sanitized transcript already fits and we
1813
- // are not forced, return it UNCHANGED (no preserved-tail redaction applied)
1814
- // to keep prior no-compaction semantics.
1815
- if (baseTokens != null && baseTokens <= budget && opts.force !== true) {
1816
- return {
1817
- messages: baseSanitized,
1818
- usage: null,
1819
- semantic: false,
1820
- compactType: COMPACT_TYPE_SEMANTIC,
1821
- diagnostics: {
1822
- noOp: true,
1823
- reason: 'fits_budget',
1824
- inputMessages: Array.isArray(messages) ? messages.length : 0,
1825
- baseMessages: baseSanitized.length,
1826
- baseTokens,
1827
- budgetTokens: budget,
1828
- durationMs: Date.now() - startedAt,
1829
- },
1830
- };
1831
- }
1832
- // Compaction will proceed: redact sensitive tool-call argument VALUES before
1833
- // window selection so the preserved tail/system that survive verbatim are
1834
- // measured AND emitted in their redacted form. Head prompt normalizers
1835
- // (toolCallSummary/normalizeToolArgValue) still apply on top for the
1836
- // summarized head. Redaction is shape-preserving, so tool-pair structure
1837
- // stays provider-valid.
1838
- const sanitized = redactToolCallSecretsInMessages(baseSanitized);
1839
-
1840
- const selected = selectCompactionWindow(sanitized, budget, opts);
1841
- if (selected.head.length === 0 && !selected.previousSummary) {
1842
- throw new Error('semanticCompactMessages: no compactable prior history before preserved tail');
1843
- }
1844
-
1845
- const mandatory = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs([...selected.system, ...selected.tail])));
1846
- const mandatoryCost = estimateMessagesTokens(mandatory);
1847
- const originalBudget = budget;
1848
- // The preserved tail is kept verbatim and the head is replaced by a much
1849
- // smaller summary, so the compacted result is always smaller than the
1850
- // input regardless of how the configured target budget compares to the
1851
- // mandatory cost. When the budget cannot even hold what we must keep, raise
1852
- // it to fit (mandatory + summary room) rather than refusing — a refusal
1853
- // here was the source of auto-clear / overflow compact failures.
1854
- if (mandatoryCost + COMPACT_SUMMARY_MIN_ROOM_TOKENS > budget) {
1855
- budget = mandatoryCost + COMPACT_SUMMARY_MIN_ROOM_TOKENS;
1856
- }
1857
- const budgetRaisedBy = Math.max(0, budget - originalBudget);
1858
-
1859
- const callBudget = Math.max(1, Math.floor((opts.compactionInputBudgetTokens || budget) * COMPACTION_PROMPT_HEADROOM));
1860
- const prompt = fitCompactionPrompt(selected, callBudget);
1861
- if (!prompt) {
1862
- throw new Error(`semanticCompactMessages: compaction prompt cannot fit call budget=${callBudget}`);
1863
- }
1864
- const compactModel = model;
1865
- const sendOpts = {
1866
- ...(opts.sendOpts || {}),
1867
- thinkingBudgetTokens: undefined,
1868
- xaiReasoningEffort: undefined,
1869
- reasoningEffort: undefined,
1870
- effort: 'low',
1871
- fast: opts.fast ?? opts.sendOpts?.fast ?? true,
1872
- maxOutputTokens: opts.maxOutputTokens || SUMMARY_OUTPUT_TOKENS,
1873
- providerState: undefined,
1874
- onToolCall: undefined,
1875
- onToolResult: undefined,
1876
- onTextDelta: undefined,
1877
- onReasoningDelta: undefined,
1878
- onUsageDelta: undefined,
1879
- onStreamDelta: undefined,
1880
- onStageChange: undefined,
1881
- drainSteering: undefined,
1882
- onSteerMessage: undefined,
1883
- signal: combinedSignal(opts.signal || opts.sendOpts?.signal || null, opts.timeoutMs || 30_000),
1884
- };
1885
- if (opts.sessionId) sendOpts.sessionId = `${opts.sessionId}:compact`;
1886
- if (opts.promptCacheKey || opts.sendOpts?.promptCacheKey) {
1887
- sendOpts.promptCacheKey = `${opts.promptCacheKey || opts.sendOpts.promptCacheKey}:compact`;
1888
- }
1889
- if (opts.providerCacheKey || opts.sendOpts?.providerCacheKey) {
1890
- sendOpts.providerCacheKey = `${opts.providerCacheKey || opts.sendOpts.providerCacheKey}:compact`;
1891
- }
1892
-
1893
- const response = await provider.send([
1894
- { role: 'system', content: COMPACTION_SYSTEM_PROMPT },
1895
- { role: 'user', content: prompt },
1896
- ], compactModel, undefined, sendOpts);
1897
- const rawSummary = extractResponseText(response);
1898
- if (!rawSummary) throw new Error('semanticCompactMessages: compaction agent returned empty summary');
1899
- // Lightweight schema enforcement: a non-empty but malformed provider
1900
- // response (missing the required template sections) is deterministically
1901
- // repaired into the structured anchored shape rather than injected blindly.
1902
- const enforced = enforceSemanticSummarySchema(rawSummary, { head: selected.head, tail: selected.tail });
1903
- const summary = enforced.summary;
1904
-
1905
- const oldHistory = selected.originalHead;
1906
- const semanticMeta = {
1907
- provider: opts.providerName || provider.name || null,
1908
- model: compactModel,
1909
- };
1910
- const summaryMessage = fitSemanticSummaryMessage(oldHistory, summary, budget - mandatoryCost, semanticMeta, selected.preservedFacts);
1911
- if (!summaryMessage) {
1912
- throw new Error(`semanticCompactMessages: summary cannot fit remaining budget=${budget - mandatoryCost}`);
1913
- }
1914
-
1915
- // selected.system / selected.tail already carry redacted tool-call args
1916
- // (sanitized was redacted before window selection), so the preserved tail
1917
- // is both measured and emitted in redacted form.
1918
- let result = sanitizeToolPairs([...selected.system, summaryMessage, ...selected.tail]);
1919
- result = reconcileDedupStubs(dedupToolResultBodies(result));
1920
- const finalTokens = estimateMessagesTokens(result);
1921
- if (finalTokens > budget) {
1922
- throw new Error(`semanticCompactMessages: compacted result exceeds budget=${budget} (result=${finalTokens})`);
1923
- }
1924
- const diagnostics = {
1925
- noOp: false,
1926
- inputMessages: Array.isArray(messages) ? messages.length : 0,
1927
- baseMessages: baseSanitized.length,
1928
- baseTokens,
1929
- systemMessages: selected.system.length,
1930
- headMessages: selected.head.length,
1931
- originalHeadMessages: selected.originalHead.length,
1932
- tailMessages: selected.tail.length,
1933
- mandatoryMessages: mandatory.length,
1934
- finalMessages: result.length,
1935
- systemTokens: safeEstimateMessagesTokens(selected.system),
1936
- headTokens: safeEstimateMessagesTokens(selected.head),
1937
- tailTokens: safeEstimateMessagesTokens(selected.tail),
1938
- mandatoryCost,
1939
- finalTokens,
1940
- originalBudgetTokens: originalBudget,
1941
- budgetTokens: budget,
1942
- budgetRaised: budgetRaisedBy > 0,
1943
- budgetRaisedBy,
1944
- remainingTokens: budget - mandatoryCost,
1945
- callBudgetTokens: callBudget,
1946
- promptChars: String(prompt || '').length,
1947
- promptBytes: textByteLength(prompt),
1948
- promptTokens: safeEstimateMessagesTokens([
1949
- { role: 'system', content: COMPACTION_SYSTEM_PROMPT },
1950
- { role: 'user', content: prompt },
1951
- ]),
1952
- summaryChars: String(summary || '').length,
1953
- rawSummaryChars: String(rawSummary || '').length,
1954
- summaryRepaired: enforced.repaired === true,
1955
- previousSummary: !!selected.previousSummary,
1956
- durationMs: Date.now() - startedAt,
1957
- };
1958
- compactDebugLog('semantic result', diagnostics);
1959
- return {
1960
- messages: result,
1961
- usage: response?.usage || null,
1962
- providerState: response?.providerState,
1963
- semantic: true,
1964
- compactType: COMPACT_TYPE_SEMANTIC,
1965
- summary,
1966
- summaryRepaired: enforced.repaired === true,
1967
- diagnostics,
1968
- };
1969
- }
1970
-
1971
- export function recallFastTrackCompactMessages(messages, budgetTokens, opts = {}) {
1972
- return _recallFastTrackCompactMessages(messages, budgetTokens, opts);
1973
- }
1974
-
1975
- // Recall fast-track (type 2) tail policy: preserve the most recent turns of the
1976
- // live conversation VERBATIM and STRUCTURED, keeping role semantics for
1977
- // user / assistant / tool / system / developer instead of collapsing the tail
1978
- // to user-only. The chunk summary anchors older history; the preserved tail
1979
- // keeps recent assistant reasoning, tool_calls, and tool_results so fresh
1980
- // state is not silently dropped.
1981
- //
1982
- // Turns are anchored on user-role boundaries: each turn = a user message plus
1983
- // the assistant/tool/system/developer messages that follow it (a leading run of
1984
- // non-user messages before the first user boundary is treated as its own
1985
- // partial turn so nothing is lost). We keep the newest RECALL_TAIL_USER_MAX
1986
- // turns; if the kept set exceeds RECALL_TAIL_TOKEN_CAP we drop whole oldest
1987
- // turns first, then middle-truncate the oldest surviving messages' string
1988
- // content so the set fits while leaving the newest message whole.
1989
- //
1990
- // Partial tool_call/tool_result pairs that truncation might leave behind are
1991
- // repaired by sanitizeToolPairs/reconcileDedupStubs in the caller, so pairing
1992
- // stays valid even after trimming.
1993
- const RECALL_TAIL_USER_MAX = 2;
1994
- const RECALL_TAIL_TOKEN_CAP = DEFAULT_COMPACTION_KEEP_TOKENS; // 8k
1995
- // Rough chars-per-token used only to size a truncation target; the real fit is
1996
- // re-checked with estimateMessagesTokens below.
1997
- const RECALL_TAIL_CHARS_PER_TOKEN = 4;
1998
-
1999
- function splitTailIntoTurns(messages) {
2000
- const turns = [];
2001
- let current = null;
2002
- for (const m of messages) {
2003
- if (isSummaryMessage(m)) continue;
2004
- if (m?.role === 'user') {
2005
- if (current) turns.push(current);
2006
- current = [m];
2007
- } else {
2008
- if (!current) current = [];
2009
- current.push(m);
2010
- }
2011
- }
2012
- if (current && current.length) turns.push(current);
2013
- return turns;
2014
- }
2015
-
2016
- function truncateMessageForRecallTail(text, maxChars) {
2017
- const marker = RECALL_TAIL_TRUNCATION_MARKER;
2018
- const value = String(text ?? '').replace(/\r\n/g, '\n');
2019
- if (value.length <= maxChars) return value;
2020
- if (maxChars <= 0) return RECALL_TAIL_SHORT_TRUNCATION_MARKER;
2021
- if (maxChars < marker.length) return RECALL_TAIL_SHORT_TRUNCATION_MARKER;
2022
- const room = maxChars - marker.length;
2023
- const head = Math.ceil(room * 0.35);
2024
- const tailPart = Math.floor(room * 0.65);
2025
- return `${value.slice(0, head)}${marker}${value.slice(value.length - tailPart)}`;
2026
- }
2027
-
2028
- function fitRecallUserMessageToCap(userMsg, cap, following = []) {
2029
- const followCost = estimateMessagesTokens(following);
2030
- const room = Math.max(1, cap - followCost);
2031
- const base = { ...userMsg };
2032
- const raw = typeof base.content === 'string' ? base.content : extractText(base);
2033
- if (estimateMessagesTokens([{ ...base, content: raw }]) <= room) return { ...base, content: raw };
2034
-
2035
- let lo = 0;
2036
- let hi = raw.length;
2037
- let best = truncateMessageForRecallTail(raw, 0);
2038
- while (lo <= hi) {
2039
- const mid = Math.floor((lo + hi) / 2);
2040
- const candidateText = truncateMessageForRecallTail(raw, mid);
2041
- const candidate = { ...base, content: candidateText };
2042
- if (estimateMessagesTokens([candidate]) <= room) {
2043
- best = candidateText;
2044
- lo = mid + 1;
2045
- } else {
2046
- hi = mid - 1;
2047
- }
2048
- }
2049
- return { ...base, content: best };
2050
- }
2051
-
2052
- function fitSingleRecallTurnToCap(turn, cap) {
2053
- const userIdx = turn.findIndex((m) => m?.role === 'user');
2054
- if (userIdx < 0) {
2055
- return truncateTailToCap(turn, cap);
2056
- }
2057
- const userMsg = turn[userIdx];
2058
- const following = turn.slice(userIdx + 1);
2059
- const fittedUser = fitRecallUserMessageToCap(userMsg, cap, following);
2060
- let out = [fittedUser];
2061
- for (const m of following) {
2062
- const candidate = [...out, m];
2063
- if (estimateMessagesTokens(candidate) <= cap) out.push(m);
2064
- }
2065
- return reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(out)));
2066
- }
2067
-
2068
- function truncateTailToCap(messages, cap) {
2069
- const turn = Array.isArray(messages) ? messages : [];
2070
- if (turn.length === 0) return [];
2071
- // No user anchor in this turn: keep the NEWEST messages that fit `cap`,
2072
- // walking backward. (Previously this delegated back to
2073
- // fitSingleRecallTurnToCap, which re-entered here on a no-user turn —
2074
- // infinite mutual recursion. Unreachable while a no-user tail threw upstream;
2075
- // now that a no-user tail is allowed, this path must terminate on its own.)
2076
- let out = [];
2077
- let startIdx = turn.length; // index in `turn` where `out` begins
2078
- for (let i = turn.length - 1; i >= 0; i -= 1) {
2079
- const candidate = [turn[i], ...out];
2080
- if (estimateMessagesTokens(candidate) <= cap) {
2081
- out = candidate;
2082
- startIdx = i;
2083
- continue;
2084
- }
2085
- if (out.length === 0) {
2086
- // Even the newest single message exceeds cap: middle-truncate its
2087
- // string content so at least one message survives.
2088
- const m = turn[i];
2089
- const text = typeof m?.content === 'string' ? m.content : extractText(m);
2090
- const truncated = truncateMessageForRecallTail(text, Math.max(1, cap * RECALL_TAIL_CHARS_PER_TOKEN));
2091
- out = [{ ...m, content: truncated }];
2092
- startIdx = i;
2093
- }
2094
- break;
2095
- }
2096
- // A leading tool_result with no preceding assistant tool_call is an orphan
2097
- // that sanitizeToolPairs drops — which could empty the whole tail. Extend the
2098
- // window backward to swallow the preceding non-tool boundary (the assistant
2099
- // that owns the tool_call), so the pair survives sanitize. Bounded by
2100
- // startIdx so it always terminates.
2101
- while (startIdx > 0 && out[0]?.role === 'tool') {
2102
- startIdx -= 1;
2103
- out = [turn[startIdx], ...out];
2104
- }
2105
- let sanitized = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(out)));
2106
- // Final guard: if sanitize still emptied the tail but the turn has a non-tool
2107
- // message, rebuild from the newest non-tool message forward so the tail is
2108
- // never empty when preservable content exists.
2109
- if (sanitized.length === 0) {
2110
- let nt = -1;
2111
- for (let i = turn.length - 1; i >= 0; i -= 1) {
2112
- if (turn[i]?.role !== 'tool') { nt = i; break; }
2113
- }
2114
- if (nt >= 0) {
2115
- sanitized = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(turn.slice(nt))));
2116
- }
2117
- }
2118
- return sanitized;
2119
- }
2120
-
2121
- function stripNestedSummaryHeaderLines(text) {
2122
- const lines = String(text ?? '').split('\n');
2123
- const out = [];
2124
- for (const line of lines) {
2125
- if (line.startsWith(SUMMARY_PREFIX)) continue;
2126
- if (/^messages=\d+\s+sha256=/.test(line.trim())) continue;
2127
- out.push(line);
2128
- }
2129
- return out.join('\n').trim();
2130
- }
2131
-
2132
- function splitRecallFitInputs(recallText, previousSummary) {
2133
- return {
2134
- recall: String(recallText || '').trim(),
2135
- prior: previousSummary ? stripNestedSummaryHeaderLines(previousSummary) : '',
2136
- };
2137
- }
2138
-
2139
- function recallTailStartIndex(live, tail) {
2140
- if (!tail.length) return live.length;
2141
- const first = tail[0];
2142
- const idx = live.indexOf(first);
2143
- if (idx >= 0) return idx;
2144
- return Math.max(0, live.length - tail.length);
2145
- }
2146
-
2147
- function selectRecallPreservedTail(live, opts = {}) {
2148
- const msgs = (Array.isArray(live) ? live : []).filter((m) => m && !isSummaryMessage(m));
2149
- if (msgs.length === 0) return { tail: [], head: [], tailStartIdx: 0 };
2150
- const maxTurns = Math.max(1, Number(opts.maxUsers) || RECALL_TAIL_USER_MAX);
2151
- const cap = Math.max(1, Number(opts.tokenCap) || RECALL_TAIL_TOKEN_CAP);
2152
- const turns = splitTailIntoTurns(msgs);
2153
- if (turns.length === 0) return { tail: [], head: msgs, tailStartIdx: 0 };
2154
-
2155
- let kept = turns.slice(-maxTurns);
2156
- while (kept.length > 1 && estimateMessagesTokens(kept.flat()) > cap) {
2157
- kept = kept.slice(1);
2158
- }
2159
-
2160
- let tail;
2161
- if (estimateMessagesTokens(kept.flat()) <= cap) {
2162
- tail = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(kept.flat())));
2163
- } else {
2164
- tail = fitSingleRecallTurnToCap(kept[kept.length - 1], cap);
2165
- }
2166
-
2167
- // A no-user tail is valid: a single-turn agent session may keep only
2168
- // assistant/tool structure recently. Mirror the semantic cut-point model —
2169
- // preserve the recent structured turn(s) verbatim without demanding a user
2170
- // anchor rather than throwing. tool-pairing is already reconciled above.
2171
- const tailStartIdx = recallTailStartIndex(msgs, tail);
2172
- const head = msgs.slice(0, tailStartIdx);
2173
- return { tail, head, tailStartIdx };
2174
- }
2175
-
2176
- // Kept name as the type-2 tail anchor; behavior now preserves whole structured
2177
- // turns (all roles) rather than only role=user messages.
2178
- function selectRecallTailUserMessages(tail, opts = {}) {
2179
- const msgs = (Array.isArray(tail) ? tail : []).filter((m) => m && !isSummaryMessage(m));
2180
- return selectRecallPreservedTail(msgs, opts).tail;
2181
- }
2182
-
2183
- function _recallFastTrackCompactMessages(messages, budgetTokens, opts = {}) {
2184
- const startedAt = Date.now();
2185
- let budget = effectiveBudget(budgetTokens, opts);
2186
- const baseSanitized = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs(messages)));
2187
- const baseTokens = safeEstimateMessagesTokens(baseSanitized);
2188
- if (baseTokens != null && baseTokens <= budget && opts.force !== true) {
2189
- return {
2190
- messages: baseSanitized,
2191
- recallFastTrack: false,
2192
- compactType: COMPACT_TYPE_RECALL_FASTTRACK,
2193
- query: opts.query || '',
2194
- diagnostics: {
2195
- noOp: true,
2196
- reason: 'fits_budget',
2197
- inputMessages: Array.isArray(messages) ? messages.length : 0,
2198
- baseMessages: baseSanitized.length,
2199
- baseTokens,
2200
- budgetTokens: budget,
2201
- durationMs: Date.now() - startedAt,
2202
- },
2203
- };
2204
- }
2205
- const sanitized = redactToolCallSecretsInMessages(baseSanitized);
2206
-
2207
- const { system: safeSystem, live, previousSummary } = splitLiveCompactionContext(sanitized);
2208
- const recallTailOpts = {
2209
- maxUsers: opts.recallTailMaxUsers ?? opts.tailTurns ?? RECALL_TAIL_USER_MAX,
2210
- tokenCap: opts.recallTailTokenCap ?? preserveRecentBudget(budget, opts),
2211
- };
2212
- const { tail: recallTail, head: recallHead } = selectRecallPreservedTail(live, recallTailOpts);
2213
- const recallFit = splitRecallFitInputs(opts.recallText, previousSummary);
2214
- if (recallHead.length === 0 && !previousSummary
2215
- && !(recallFit.recall || recallFit.prior || opts.allowEmptyRecall === true)) {
2216
- throw new Error('recallFastTrackCompactMessages: no compactable prior history before preserved tail');
2217
- }
2218
-
2219
- const mandatory = reconcileDedupStubs(dedupToolResultBodies(sanitizeToolPairs([...safeSystem, ...recallTail])));
2220
- const mandatoryCost = estimateMessagesTokens(mandatory);
2221
- const originalBudget = budget;
2222
- if (mandatoryCost + COMPACT_SUMMARY_MIN_ROOM_TOKENS > budget) {
2223
- budget = mandatoryCost + COMPACT_SUMMARY_MIN_ROOM_TOKENS;
2224
- }
2225
- const budgetRaisedBy = Math.max(0, budget - originalBudget);
38
+ export {
39
+ effectiveBudget,
40
+ countRawPendingRows,
41
+ drainSessionCycle1,
42
+ pruneToolOutputs,
43
+ pruneToolOutputsUnanchored,
44
+ } from './compact/budget.mjs';
2226
45
 
2227
- if (!recallFit.recall && !recallFit.prior && opts.allowEmptyRecall !== true) {
2228
- throw new Error('recallFastTrackCompactMessages: recall text is empty');
2229
- }
2230
- const oldHistory = recallHead;
2231
- const recallMeta = {
2232
- querySha: opts.querySha || null,
2233
- };
2234
- const summaryMessage = fitRecallFastTrackSummaryMessage(
2235
- oldHistory,
2236
- recallFit.recall,
2237
- budget - mandatoryCost,
2238
- recallMeta,
2239
- recallFit.prior,
2240
- );
2241
- if (!summaryMessage) {
2242
- throw new Error(`recallFastTrackCompactMessages: summary cannot fit remaining budget=${budget - mandatoryCost}`);
2243
- }
46
+ export {
47
+ buildRecallFastTrackQuery,
48
+ splitRecallRootBlocks,
49
+ fitRecallRootsMessage,
50
+ } from './compact/summary.mjs';
2244
51
 
2245
- let result = sanitizeToolPairs([...safeSystem, summaryMessage, ...recallTail]);
2246
- result = reconcileDedupStubs(dedupToolResultBodies(result));
2247
- const finalTokens = estimateMessagesTokens(result);
2248
- if (finalTokens > budget) {
2249
- throw new Error(`recallFastTrackCompactMessages: compacted result exceeds budget=${budget} (result=${finalTokens})`);
2250
- }
2251
- const summaryContent = String(summaryMessage?.content || '');
2252
- const diagnostics = {
2253
- noOp: false,
2254
- inputMessages: Array.isArray(messages) ? messages.length : 0,
2255
- baseMessages: baseSanitized.length,
2256
- baseTokens,
2257
- systemMessages: safeSystem.length,
2258
- liveMessages: live.length,
2259
- headMessages: recallHead.length,
2260
- tailMessages: recallTail.length,
2261
- mandatoryMessages: mandatory.length,
2262
- finalMessages: result.length,
2263
- systemTokens: safeEstimateMessagesTokens(safeSystem),
2264
- liveTokens: safeEstimateMessagesTokens(live),
2265
- headTokens: safeEstimateMessagesTokens(recallHead),
2266
- tailTokens: safeEstimateMessagesTokens(recallTail),
2267
- mandatoryCost,
2268
- finalTokens,
2269
- originalBudgetTokens: originalBudget,
2270
- budgetTokens: budget,
2271
- budgetRaised: budgetRaisedBy > 0,
2272
- budgetRaisedBy,
2273
- remainingTokens: budget - mandatoryCost,
2274
- recallChars: recallFit.recall.length,
2275
- recallBytes: textByteLength(recallFit.recall),
2276
- priorChars: recallFit.prior.length,
2277
- priorBytes: textByteLength(recallFit.prior),
2278
- summaryMessageChars: summaryContent.length,
2279
- summaryMessageBytes: textByteLength(summaryContent),
2280
- recallEmpty: !recallFit.recall,
2281
- priorEmpty: !recallFit.prior,
2282
- recallTruncatedInSummary: !!recallFit.recall && !summaryContent.includes(recallFit.recall),
2283
- priorTruncatedInSummary: !!recallFit.prior && !summaryContent.includes(recallFit.prior),
2284
- tailTruncated: recallTail.some((m) => messageContentHasMarker(m, RECALL_TAIL_TRUNCATION_MARKER) || messageContentHasMarker(m, RECALL_TAIL_SHORT_TRUNCATION_MARKER)),
2285
- tailOptions: recallTailOpts,
2286
- previousSummary: !!previousSummary,
2287
- durationMs: Date.now() - startedAt,
2288
- };
2289
- compactDebugLog('recall-fasttrack result', diagnostics);
2290
- return {
2291
- messages: result,
2292
- recallFastTrack: true,
2293
- compactType: COMPACT_TYPE_RECALL_FASTTRACK,
2294
- query: opts.query || '',
2295
- diagnostics,
2296
- };
2297
- }
52
+ export {
53
+ semanticCompactMessages,
54
+ recallFastTrackCompactMessages,
55
+ } from './compact/engine.mjs';