@bitkyc08/opencodex 2.41.0 → 2.43.0-preview.20260906

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (260) hide show
  1. package/AGENTS_INSTALL.md +2 -2
  2. package/README.md +31 -0
  3. package/bin/ocx.mjs +10 -1
  4. package/gui/dist/assets/index-DS1NE4Jn.css +1 -0
  5. package/gui/dist/assets/index-VGeQEZ_v.js +112 -0
  6. package/gui/dist/index.html +2 -2
  7. package/package.json +1 -1
  8. package/src/adapters/anthropic-image-codec.ts +304 -0
  9. package/src/adapters/anthropic-image-normalize.ts +8 -298
  10. package/src/adapters/anthropic.ts +30 -7
  11. package/src/adapters/command-code.ts +7 -5
  12. package/src/adapters/cursor/desktop-executor-contract.ts +15 -0
  13. package/src/adapters/cursor/images.ts +36 -6
  14. package/src/adapters/cursor/live-transport.ts +7 -2
  15. package/src/adapters/cursor/native-exec-desktop.ts +2 -15
  16. package/src/adapters/cursor/protobuf-request.ts +54 -24
  17. package/src/adapters/cursor/tool-definitions.ts +5 -670
  18. package/src/adapters/cursor/tool-guidance.ts +236 -0
  19. package/src/adapters/cursor/tool-naming.ts +252 -0
  20. package/src/adapters/cursor/tool-schemas.ts +195 -0
  21. package/src/adapters/cursor/types.ts +6 -3
  22. package/src/adapters/exec-tool-result-normalize.ts +1 -1
  23. package/src/adapters/google-errors.ts +9 -1
  24. package/src/adapters/google.ts +1 -0
  25. package/src/adapters/identity.ts +8 -2
  26. package/src/adapters/kiro-calibration.ts +181 -0
  27. package/src/adapters/kiro.ts +135 -3
  28. package/src/adapters/openai-responses.ts +259 -29
  29. package/src/adapters/responses-code-mode.ts +59 -0
  30. package/src/adapters/tool-catalog-nudge.ts +1 -1
  31. package/src/adapters/xai-schema-analysis.ts +86 -0
  32. package/src/adapters/xai-tool-schema.ts +2 -87
  33. package/src/adapters/xai-web-search.ts +1 -1
  34. package/src/bridge.ts +47 -13
  35. package/src/chat/inbound.ts +11 -3
  36. package/src/claude/inbound-content-options.ts +60 -0
  37. package/src/claude/inbound-model-options.ts +142 -0
  38. package/src/claude/inbound-records.ts +7 -0
  39. package/src/claude/inbound.ts +10 -202
  40. package/src/claude/model-info.ts +45 -0
  41. package/src/cli/account-auth.ts +49 -9
  42. package/src/cli/account-extended.ts +7 -1
  43. package/src/cli/capabilities.ts +15 -4
  44. package/src/cli/claude.ts +232 -39
  45. package/src/cli/config-command.ts +9 -1
  46. package/src/cli/dispatch.ts +5 -1
  47. package/src/cli/doctor.ts +10 -0
  48. package/src/cli/effort.ts +372 -0
  49. package/src/cli/export-command.ts +3 -9
  50. package/src/cli/help.ts +1 -0
  51. package/src/cli/index.ts +13 -0
  52. package/src/cli/init.ts +4 -0
  53. package/src/cli/model-selection-guidance.ts +30 -0
  54. package/src/cli/models-runtime.ts +3 -2
  55. package/src/cli/models.ts +8 -3
  56. package/src/cli/observe.ts +3 -1
  57. package/src/cli/opencode.ts +4 -1
  58. package/src/cli/provider-runtime.ts +65 -0
  59. package/src/cli/provider.ts +8 -0
  60. package/src/cli/registry.ts +16 -2
  61. package/src/cli/runtime-api.ts +3 -1
  62. package/src/cli/star-prompt.ts +22 -6
  63. package/src/cli/status-probes.ts +168 -0
  64. package/src/cli/status.ts +5 -168
  65. package/src/clients/config-export/constants.ts +69 -0
  66. package/src/clients/config-export/contracts.ts +154 -0
  67. package/src/clients/config-export/dsh.ts +132 -0
  68. package/src/clients/config-export/fast-models.ts +29 -0
  69. package/src/clients/config-export/mcode.ts +83 -0
  70. package/src/clients/config-export/model-metadata.ts +108 -0
  71. package/src/clients/config-export/omp.ts +104 -0
  72. package/src/clients/config-export/zcode.ts +92 -0
  73. package/src/clients/config-export.ts +18 -710
  74. package/src/codex/account-lifecycle.ts +20 -3
  75. package/src/codex/account-usability.ts +2 -0
  76. package/src/codex/auth-api.ts +272 -32
  77. package/src/codex/auth-context.ts +328 -24
  78. package/src/codex/catalog/effort.ts +44 -5
  79. package/src/codex/catalog/metadata.ts +149 -14
  80. package/src/codex/catalog/native-models.ts +116 -4
  81. package/src/codex/catalog/parsing.ts +122 -8
  82. package/src/codex/catalog/provider-fetch.ts +154 -23
  83. package/src/codex/catalog/reserve.ts +52 -0
  84. package/src/codex/catalog/sync.ts +89 -16
  85. package/src/codex/catalog.ts +1 -1
  86. package/src/codex/convergence-types.ts +1 -0
  87. package/src/codex/convergence.ts +2 -0
  88. package/src/codex/data/upstream-models.json +169 -0
  89. package/src/codex/forward-transport-headers.ts +25 -0
  90. package/src/codex/inject.ts +99 -34
  91. package/src/codex/injected-marker.ts +30 -4
  92. package/src/codex/journal.ts +14 -0
  93. package/src/codex/legacy-config-keys.ts +68 -0
  94. package/src/codex/log-guard/inspect-schema.ts +137 -0
  95. package/src/codex/log-guard/inspect.ts +2 -134
  96. package/src/codex/loopback-target.ts +54 -0
  97. package/src/codex/main-account-cache.ts +63 -1
  98. package/src/codex/main-account-hard-lock.ts +52 -0
  99. package/src/codex/main-account.ts +3 -1
  100. package/src/codex/management-convergence.ts +3 -0
  101. package/src/codex/model-entitlements.ts +54 -4
  102. package/src/codex/project-config-warnings.ts +92 -2
  103. package/src/codex/prompt-layers/encoding.ts +80 -0
  104. package/src/codex/prompt-layers/paths.ts +54 -0
  105. package/src/codex/prompt-layers/revision.ts +55 -0
  106. package/src/codex/prompt-layers/toml-edit.ts +163 -0
  107. package/src/codex/prompt-layers/toml-read.ts +181 -0
  108. package/src/codex/prompt-layers.ts +14 -520
  109. package/src/codex/quota-auto-refresh-state.ts +16 -0
  110. package/src/codex/quota-auto-refresh.ts +219 -0
  111. package/src/codex/quota-types.ts +51 -0
  112. package/src/codex/quota.ts +252 -93
  113. package/src/codex/reserve-availability.ts +177 -0
  114. package/src/codex/routing.ts +28 -9
  115. package/src/codex/shim.ts +53 -11
  116. package/src/codex/subagent-model-fallback.ts +23 -3
  117. package/src/combos/failover.ts +125 -7
  118. package/src/combos/identifiers.ts +89 -0
  119. package/src/combos/index.ts +4 -0
  120. package/src/combos/resolve.ts +80 -9
  121. package/src/combos/types.ts +20 -93
  122. package/src/config/subagent-models.ts +24 -0
  123. package/src/config.ts +156 -13
  124. package/src/generated/compatibility-version.json +474 -178
  125. package/src/generated/model-metadata.ts +1 -1
  126. package/src/integrations/journal.ts +65 -4
  127. package/src/integrations/store.ts +5 -0
  128. package/src/lab/events/limits.ts +4 -0
  129. package/src/lib/destination-policy.ts +31 -2
  130. package/src/lib/errors.ts +39 -0
  131. package/src/lib/provider-outbound.ts +69 -3
  132. package/src/lib/proxy-env.ts +22 -0
  133. package/src/lib/redact-folding.ts +176 -0
  134. package/src/lib/redact.ts +2 -175
  135. package/src/lib/state-store-sweeper.ts +20 -6
  136. package/src/lib/token-estimate.ts +94 -27
  137. package/src/lib/windows-user-principal.ts +53 -5
  138. package/src/oauth/account-quota-rank.ts +40 -1
  139. package/src/oauth/anthropic-routing.ts +99 -3
  140. package/src/oauth/chatgpt-device.ts +187 -0
  141. package/src/oauth/chatgpt.ts +31 -4
  142. package/src/oauth/generic-account-failover.ts +36 -13
  143. package/src/oauth/index.ts +140 -29
  144. package/src/oauth/log.ts +3 -0
  145. package/src/oauth/login-cli.ts +5 -0
  146. package/src/oauth/meta-muse.ts +117 -15
  147. package/src/oauth/pool-settings-capability.ts +15 -4
  148. package/src/providers/api-keys.ts +8 -10
  149. package/src/providers/default-aliases.ts +39 -0
  150. package/src/providers/derive.ts +10 -2
  151. package/src/providers/fastwire.ts +36 -7
  152. package/src/providers/initial-model-selection-runtime.ts +90 -0
  153. package/src/providers/initial-model-selection.ts +120 -0
  154. package/src/providers/key-failover.ts +134 -54
  155. package/src/providers/key-store.ts +11 -1
  156. package/src/providers/label.ts +1 -1
  157. package/src/providers/model-discovery.ts +76 -0
  158. package/src/providers/model-rename-startup.ts +72 -8
  159. package/src/providers/muse-subscription-usage.ts +95 -0
  160. package/src/providers/openai-sidecar.ts +17 -5
  161. package/src/providers/openai-tiers-destination.ts +102 -0
  162. package/src/providers/openai-tiers.ts +2 -99
  163. package/src/providers/opencode-go-transport.ts +41 -0
  164. package/src/providers/quota-key-accounts.ts +141 -0
  165. package/src/providers/quota-types.ts +9 -0
  166. package/src/providers/quota.ts +625 -98
  167. package/src/providers/registry.ts +60 -17
  168. package/src/providers/xai-responses-opt-in.ts +31 -5
  169. package/src/quota/reset-activation.ts +81 -0
  170. package/src/quota/reset-detector.ts +305 -0
  171. package/src/quota/reset-notify-config.ts +162 -0
  172. package/src/quota/reset-observer.ts +125 -0
  173. package/src/quota/reset-poller.ts +160 -0
  174. package/src/quota/reset-seen-store.ts +385 -0
  175. package/src/quota/reset-sinks.ts +199 -0
  176. package/src/quota/window-mapping.ts +106 -0
  177. package/src/responses/apply-patch-envelope.ts +46 -0
  178. package/src/responses/code-mode-helper-compat.ts +39 -1
  179. package/src/responses/custom-tool-compat.ts +10 -4
  180. package/src/responses/hosted-tool-policy.ts +12 -4
  181. package/src/responses/parser-content.ts +133 -0
  182. package/src/responses/parser-text-format.ts +24 -0
  183. package/src/responses/parser-tools.ts +188 -0
  184. package/src/responses/parser.ts +3 -326
  185. package/src/responses/state.ts +124 -28
  186. package/src/router.ts +48 -13
  187. package/src/routing/analytics.ts +1 -0
  188. package/src/routing/capability.ts +17 -4
  189. package/src/server/auth-cors.ts +7 -1
  190. package/src/server/background-lifecycle.ts +23 -1
  191. package/src/server/chat-completions.ts +25 -3
  192. package/src/server/claude-messages.ts +62 -5
  193. package/src/server/effort-row.ts +1 -1
  194. package/src/server/fast-row.ts +295 -0
  195. package/src/server/gui-static.ts +30 -4
  196. package/src/server/index.ts +122 -28
  197. package/src/server/live.ts +18 -4
  198. package/src/server/management/agent-settings-routes.ts +2 -2
  199. package/src/server/management/combo-routes.ts +37 -9
  200. package/src/server/management/config-routes.ts +93 -2
  201. package/src/server/management/integration-routes.ts +108 -0
  202. package/src/server/management/model-routes.ts +13 -3
  203. package/src/server/management/model-rows.ts +20 -1
  204. package/src/server/management/native-integration-routes.ts +4 -1
  205. package/src/server/management/oauth-account-routes.ts +45 -10
  206. package/src/server/management/provider-routes.ts +34 -3
  207. package/src/server/management/quota-reset-routes.ts +57 -0
  208. package/src/server/management/route-registry.ts +7 -4
  209. package/src/server/management/shared.ts +19 -5
  210. package/src/server/management/system-routes.ts +3 -2
  211. package/src/server/management-api.ts +14 -2
  212. package/src/server/ports.ts +12 -2
  213. package/src/server/relay-eager.ts +38 -23
  214. package/src/server/relay.ts +4 -0
  215. package/src/server/request-log.ts +6 -0
  216. package/src/server/responses/codex-ws-correlation.ts +65 -0
  217. package/src/server/responses/codex-ws-exchange.ts +261 -0
  218. package/src/server/responses/codex-ws-metadata.ts +134 -0
  219. package/src/server/responses/codex-ws-pool.ts +162 -0
  220. package/src/server/responses/codex-ws-request.ts +87 -0
  221. package/src/server/responses/codex-ws-session.ts +93 -0
  222. package/src/server/responses/codex-ws-wire.ts +144 -0
  223. package/src/server/responses/collaboration.ts +41 -1
  224. package/src/server/responses/compact.ts +105 -12
  225. package/src/server/responses/core.ts +510 -57
  226. package/src/server/responses/empty-completion-guard.ts +4 -0
  227. package/src/server/responses/fetch-helpers.ts +10 -3
  228. package/src/server/responses/input-admission.ts +16 -9
  229. package/src/server/responses/responses-field-backfill.ts +1 -1
  230. package/src/server/responses/ws-upstream.ts +34 -318
  231. package/src/server/responses-custom-tool-repair.ts +20 -4
  232. package/src/server/responses-undeclared-tool-guard.ts +100 -8
  233. package/src/server/safe-response-headers.ts +23 -0
  234. package/src/server/search.ts +9 -0
  235. package/src/server/subagent-models-startup.ts +27 -0
  236. package/src/server/system-env-shell.ts +238 -0
  237. package/src/server/system-env.ts +7 -234
  238. package/src/server/ws-bridge.ts +3 -25
  239. package/src/server/xai-responses-startup.ts +21 -0
  240. package/src/service-manager-probe.ts +1 -1
  241. package/src/service.ts +55 -16
  242. package/src/types/config.ts +108 -12
  243. package/src/types/provider.ts +36 -7
  244. package/src/types/request.ts +8 -0
  245. package/src/types/tools.ts +26 -1
  246. package/src/types.ts +2 -0
  247. package/src/update/notify.ts +8 -2
  248. package/src/usage/cost.ts +38 -28
  249. package/src/usage/expected-prices.ts +34 -15
  250. package/src/usage/log.ts +2 -0
  251. package/src/usage/model-identity.ts +26 -0
  252. package/src/usage/summary.ts +15 -1
  253. package/src/vision/describe.ts +6 -0
  254. package/src/vision/image-rewrite.ts +108 -0
  255. package/src/vision/index.ts +19 -306
  256. package/src/vision/plan.ts +205 -0
  257. package/src/web-search/executor.ts +6 -0
  258. package/src/web-search/index.ts +8 -1
  259. package/gui/dist/assets/index-B2YjLA-i.css +0 -1
  260. package/gui/dist/assets/index-aPup8CKb.js +0 -112
package/src/lib/redact.ts CHANGED
@@ -1,3 +1,5 @@
1
+ import { foldForMatching } from "./redact-folding";
2
+
1
3
  export const REDACTED_SECRET = "[REDACTED]";
2
4
 
3
5
  /**
@@ -43,76 +45,6 @@ const CREDENTIAL_HEADER_LABEL_RAW = "x-api-key|x-goog-api-key|x-amz-security-tok
43
45
  const CREDENTIAL_HEADER_LABEL = CREDENTIAL_HEADER_LABEL_RAW
44
46
  .replace(/(?<![\[\\])([A-Za-z])(?![\]\-])/g, "[$1\u0001]");
45
47
 
46
- /**
47
- * Characters that render as a colon separator. Folded to `:` in the matching
48
- * view so a look-alike cannot hide a header from the label pattern.
49
- */
50
- const COLON_CONFUSABLES = new Set([
51
- "\uFF1A", "\uFE55", "\uFE13", "\uA789", "\u02D0", "\u2236",
52
- "\u205A", "\u0589", "\u1361", "\u16EC", "\u1803", "\u2982", "\u2AF6", "\uFE30",
53
- ]);
54
-
55
- /**
56
- * Characters dropped from the matching view: anything with no visible width
57
- * that could split a label into pieces the pattern no longer recognizes.
58
- * `\p{Default_Ignorable_Code_Point}` is the systematic answer — it covers the
59
- * zero-width set, the bidi isolates and marks, the Mongolian vowel separator,
60
- * and the variation selectors in one property instead of a list that review
61
- * keeps finding another member of. `\p{Cf}` and combining marks are folded too.
62
- */
63
- const INVISIBLE_FORMAT = /[\p{Default_Ignorable_Code_Point}\p{Cf}\p{Mn}\p{Me}]/u;
64
-
65
- /**
66
- * HTML named character references.
67
- *
68
- * A hand-picked list is a coverage promise nobody can keep — review found
69
- * `&ii;`, `&ee;`, and `&DifferentialD;` decoding to compatibility letters that
70
- * NFKD already maps onto `i`, `e`, and `d`, and the WHATWG table holds roughly
71
- * 2200 entries. Neither Bun nor Node exposes that table, and pulling in a
72
- * dependency to spell a header name is the wrong trade for this path.
73
- *
74
- * So names are not resolved at all. A named reference sitting inside a
75
- * credential label is folded to a single placeholder character of unknown
76
- * identity, and the label alternation accepts that placeholder wherever a
77
- * letter may appear. Every named entity is covered, present and future,
78
- * without claiming to know what any of them mean.
79
- */
80
- const NAMED_ENTITY_PLACEHOLDER = "\u0001";
81
-
82
- /**
83
- * The handful of named references that spell a SEPARATOR rather than a letter.
84
- * These have to resolve exactly, because the placeholder stands in for a letter
85
- * position and a separator is structure, not a character of the name.
86
- */
87
- const SEPARATOR_ENTITIES = new Map<string, string>([
88
- ["colon", ":"], ["semi", ";"], ["equals", "="], ["quot", '"'], ["apos", "'"],
89
- ["lt", "<"], ["gt", ">"], ["amp", "&"], ["sol", "/"], ["lowbar", "_"],
90
- ["hyphen", "-"], ["dash", "-"], ["ndash", "-"], ["mdash", "-"], ["minus", "-"],
91
- ["period", "."], ["comma", ","], ["num", "#"], ["nbsp", " "],
92
- ]);
93
-
94
- /**
95
- * Latin look-alikes for the ASCII letters that appear in credential labels.
96
- * Cyrillic `а`/`е`, Greek `ο`, fullwidth forms and the mathematical alphabets
97
- * all render as the label to a human, so the matching view folds them back.
98
- * NFKD handles the width/font variants; this table covers the cross-script
99
- * homoglyphs NFKD deliberately leaves alone.
100
- */
101
- const LETTER_CONFUSABLES = new Map<string, string>([
102
- // Cyrillic
103
- ["\u0430", "a"], ["\u0435", "e"], ["\u043E", "o"], ["\u0440", "p"], ["\u0441", "c"],
104
- ["\u0445", "x"], ["\u0443", "y"], ["\u04BB", "h"], ["\u0455", "s"], ["\u0456", "i"],
105
- ["\u0458", "j"], ["\u043A", "k"], ["\u0442", "t"], ["\u0432", "b"], ["\u043C", "m"],
106
- ["\u043D", "h"], ["\u0501", "d"], ["\u0503", "g"], ["\u051B", "q"], ["\u051D", "w"],
107
- ["\u04CF", "l"], ["\u0261", "g"], ["\u04AB", "c"], ["\u04BD", "e"], ["\u0459", "k"],
108
- // Greek
109
- ["\u03B1", "a"], ["\u03BF", "o"], ["\u03C1", "p"], ["\u03BD", "v"], ["\u03BA", "k"],
110
- ["\u03B5", "e"], ["\u03C4", "t"], ["\u03B9", "i"], ["\u03C5", "u"], ["\u03C7", "x"],
111
- ["\u03B7", "n"], ["\u03BC", "u"], ["\u03C3", "o"], ["\u03B2", "b"], ["\u03B3", "y"],
112
- // Latin extended / other
113
- ["\u0131", "i"], ["\u0269", "i"], ["\u1D0F", "o"], ["\u0280", "r"], ["\u01BF", "p"],
114
- ["\u0578", "n"], ["\u057D", "u"], ["\u0585", "o"], ["\u0581", "g"], ["\u2044", "/"],
115
- ]);
116
48
 
117
49
  // `\b` is the wrong left boundary for a header name: it matches after a `-` or
118
50
  // `_`, so `not-authorization:` and `internal_token:` were treated as the
@@ -240,111 +172,6 @@ function maskOtherFramingsOnce(value: string, decodeEscapes: boolean): string {
240
172
  return current;
241
173
  }
242
174
 
243
- /**
244
- * Build a folded copy plus an index map back to the original string, so the
245
- * match runs on normalized text while the output keeps every byte the match did
246
- * not cover.
247
- */
248
- function foldForMatching(value: string, decodeEscapes = true): { folded: string; map: number[] } {
249
- let folded = "";
250
- const map: number[] = [];
251
- // Serialization escapes are ALIASES for the label, not decoration: a JSON
252
- // `\u0069`, a percent-encoded `%69`, and an XML `&#105;` all spell the same
253
- // field name to whatever parses the body, while spelling something else to a
254
- // literal matcher. Decode them into the matching view (one folded character
255
- // per escape, with the whole escape mapped back to its start) so
256
- // `author\u0069zation`, `author%69zation`, and `author&#105;zation` are the
257
- // label they claim to be.
258
- const decodeEscape = (at: number): { ch: string; width: number } | null => {
259
- // JSON `\uXXXX`, INCLUDING a surrogate pair. Decoding the halves
260
- // independently left `\uD835\uDD69` as two lone surrogates, so the
261
- // mathematical letter they spell was never normalized as one code point.
262
- const json = /^\\u([0-9a-fA-F]{4})/.exec(value.slice(at, at + 6));
263
- if (json) {
264
- const high = parseInt(json[1]!, 16);
265
- if (high >= 0xd800 && high <= 0xdbff) {
266
- const low = /^\\u([0-9a-fA-F]{4})/.exec(value.slice(at + 6, at + 12));
267
- const lowCode = low ? parseInt(low[1]!, 16) : NaN;
268
- if (lowCode >= 0xdc00 && lowCode <= 0xdfff) {
269
- return { ch: String.fromCharCode(high, lowCode), width: 12 };
270
- }
271
- }
272
- return { ch: String.fromCharCode(high), width: 6 };
273
- }
274
- // Percent encoding is UTF-8: consecutive `%XX` bytes form ONE character.
275
- // Decoding each byte on its own turned `%D0%B5` into two unrelated
276
- // Latin-1 characters instead of the Cyrillic `е` the fold would have
277
- // recognized.
278
- const pct = /^(?:%[0-9a-fA-F]{2})+/.exec(value.slice(at, at + 24));
279
- if (pct) {
280
- try {
281
- const decoded = decodeURIComponent(pct[0]);
282
- if (decoded.length >= 1) {
283
- // Consume only the bytes that produced the FIRST character, so the
284
- // rest of the sequence is decoded on the next iteration.
285
- const first = String.fromCodePoint(decoded.codePointAt(0)!);
286
- const bytes = new TextEncoder().encode(first).length;
287
- return { ch: first, width: bytes * 3 };
288
- }
289
- } catch {
290
- const single = parseInt(pct[0].slice(1, 3), 16);
291
- return { ch: String.fromCharCode(single), width: 3 };
292
- }
293
- }
294
- const xml = /^&#(x[0-9a-fA-F]{1,6}|[0-9]{1,7});/.exec(value.slice(at, at + 11));
295
- if (xml) {
296
- const raw = xml[1]!;
297
- const code = raw[0] === "x" || raw[0] === "X"
298
- ? parseInt(raw.slice(1), 16)
299
- : parseInt(raw, 10);
300
- if (Number.isFinite(code) && code > 0 && code <= 0x10ffff) {
301
- return { ch: String.fromCodePoint(code), width: xml[0].length };
302
- }
303
- }
304
- // HTML named references. `&colon;` and the other separator names are
305
- // resolved exactly; anything else folds to the opaque placeholder so the
306
- // label still matches without pretending to know the character.
307
- const named = /^&([A-Za-z][A-Za-z0-9]{1,31});/.exec(value.slice(at, at + 34));
308
- if (named) {
309
- const separator = SEPARATOR_ENTITIES.get(named[1]!.toLowerCase());
310
- return { ch: separator ?? NAMED_ENTITY_PLACEHOLDER, width: named[0].length };
311
- }
312
- return null;
313
- };
314
- // Iterate by CODE POINT, not UTF-16 code unit: a supplementary character
315
- // (mathematical letters, variation selectors above the BMP) is two units, so
316
- // a per-unit loop hands each half to the property tests separately and
317
- // neither half matches anything. `𝕩-api-key` and a U+E0100 inside a label
318
- // both walked straight past the fold that way.
319
- let i = 0;
320
- while (i < value.length) {
321
- const escaped = decodeEscapes ? decodeEscape(i) : null;
322
- const ch = escaped ? escaped.ch : String.fromCodePoint(value.codePointAt(i)!);
323
- const width = escaped ? escaped.width : ch.length;
324
- if (INVISIBLE_FORMAT.test(ch)) {
325
- i += width;
326
- continue;
327
- }
328
- const mapped = COLON_CONFUSABLES.has(ch)
329
- ? ":"
330
- : LETTER_CONFUSABLES.get(ch.toLowerCase())
331
- // NFKD collapses fullwidth, circled, and mathematical letter variants
332
- // onto their ASCII base.
333
- ?? (ch.normalize("NFKD").length === 1 ? ch.normalize("NFKD") : ch);
334
- // One folded unit per source code point keeps the offset map aligned; a
335
- // multi-unit fold would desynchronize it, so those keep the original.
336
- folded += mapped.length === 1 ? mapped : ch;
337
- // One map entry per EMITTED UTF-16 unit. An escaped supplementary
338
- // character emits two units, and giving it one entry desynchronized every
339
- // later offset — the mask then landed mid-token and left part of the
340
- // credential behind.
341
- const emittedText = mapped.length === 1 ? mapped : ch;
342
- for (let k = 0; k < emittedText.length; k += 1) map.push(i);
343
- i += width;
344
- }
345
- map.push(value.length);
346
- return { folded, map };
347
- }
348
175
 
349
176
  /**
350
177
  * Run the header rule over BOTH matching views and take the union.
@@ -35,7 +35,13 @@ export interface StateStoreSweeperOptions {
35
35
  }
36
36
 
37
37
  const registrations = new Map<string, StateStoreRegistration>();
38
- const afterTickRegistrations = new Map<string, StateSweepAfterTickRegistration>();
38
+ interface AfterTickRegistrationNode {
39
+ registration: StateSweepAfterTickRegistration;
40
+ previous: AfterTickRegistrationNode | null;
41
+ active: boolean;
42
+ }
43
+
44
+ const afterTickRegistrations = new Map<string, AfterTickRegistrationNode>();
39
45
  let configGeneration = 0;
40
46
  let attemptSequence = 0;
41
47
  let generationContextBuilder: (() => GenerationContext) | null = null;
@@ -56,11 +62,19 @@ export function registerStateStore(registration: StateStoreRegistration): () =>
56
62
  }
57
63
 
58
64
  export function registerStateSweepAfterTick(registration: StateSweepAfterTickRegistration): () => void {
59
- afterTickRegistrations.set(registration.name, registration);
65
+ const node: AfterTickRegistrationNode = {
66
+ registration,
67
+ previous: afterTickRegistrations.get(registration.name) ?? null,
68
+ active: true,
69
+ };
70
+ afterTickRegistrations.set(registration.name, node);
60
71
  return () => {
61
- if (afterTickRegistrations.get(registration.name) === registration) {
62
- afterTickRegistrations.delete(registration.name);
63
- }
72
+ node.active = false;
73
+ if (afterTickRegistrations.get(registration.name) !== node) return;
74
+ let previous = node.previous;
75
+ while (previous && !previous.active) previous = previous.previous;
76
+ if (previous) afterTickRegistrations.set(registration.name, previous);
77
+ else afterTickRegistrations.delete(registration.name);
64
78
  };
65
79
  }
66
80
 
@@ -106,7 +120,7 @@ export function sweepLiveness(): StateSweepResult {
106
120
  }
107
121
 
108
122
  function runAfterTickCallbacks(): void {
109
- for (const registration of afterTickRegistrations.values()) {
123
+ for (const { registration } of afterTickRegistrations.values()) {
110
124
  try {
111
125
  registration.afterTick();
112
126
  } catch {
@@ -9,50 +9,112 @@
9
9
  * ~GPT 3.6, ~Gemini 3.8 chars/token (within ~10%). Code / JSON / tool-args (the dominant Codex
10
10
  * traffic) pack MORE tokens per char, so a lower chars-per-token ratio is used for those models.
11
11
  * Over-counting fails safe (auto-compact fires earlier); under-counting risks context overflow.
12
+ *
13
+ * ## Why the estimate is segmented by script rather than a single divisor
14
+ *
15
+ * A single chars-per-token divisor cannot describe mixed text. Latin prose and code run near
16
+ * 3.2 chars/token; Hangul and Han run near 1.2-1.5, because a Latin-trained BPE spends a token
17
+ * on roughly every CJK character. That is a ~2.5x spread inside one blob, and essentially all
18
+ * real agent traffic is mixed: English code and JSON framing with Korean prose threaded through
19
+ * it.
20
+ *
21
+ * The earlier model divided the whole blob by one ratio and then clamped to a denser ratio only
22
+ * when a SAMPLED CJK share crossed 30%. Two failures followed from that shape. The clamp was a
23
+ * cliff: 29% CJK counted at the sparse ratio, 31% at the dense one, and the band where real
24
+ * traffic actually sits (roughly 1-30% CJK) never triggered it at all. And the sampler read
25
+ * every stride-th character on long blobs, so the share it measured was noisy — a payload of
26
+ * fixed-width records could sample as 100% CJK while being 1.6% CJK, a case still documented in
27
+ * `server/responses/input-admission.ts`.
28
+ *
29
+ * Counting the two scripts separately and adding them removes both problems at once. There is no
30
+ * threshold to sit beside, no sampling error, and the result is continuous in the CJK share, so a
31
+ * blob that gains one Korean character gains a fraction of a token instead of jumping a ratio.
12
32
  */
13
33
 
14
34
  /** Generic English-prose fallback ratio (chars per token). */
15
35
  const DEFAULT_CHARS_PER_TOKEN = 4;
16
36
 
17
37
  /**
18
- * Kiro routes code/JSON-heavy agent traffic whose true ratio is ~3.0-3.3 chars/token. 3.5 keeps a
19
- * small safety margin (slight over-count) without wildly inflating; tune toward 3.3 if overflow is
20
- * ever observed. All kiro models are text LLMs, so a single ratio applies to the whole family.
38
+ * Kiro routes code/JSON-heavy agent traffic: diffs, file paths, tool arguments, identifiers.
39
+ * That material tokenizes markedly denser than the "4 chars per token" English-prose rule.
40
+ *
41
+ * Measured two independent ways, which agree. Recorded conversations pair exact text with the
42
+ * provider's authoritative input-token count; over 5,799 pure-Latin samples that aggregates to
43
+ * 2.80 chars/token. Separately, recorded Kiro request bodies charge ~2.43 bytes per token, and
44
+ * those bodies run ~1.12 bytes per counted character of JSON escaping and framing, which implies
45
+ * ~2.17 chars/token at the wire.
46
+ *
47
+ * 2.8 takes the conservative end of that pair. All kiro models are text LLMs, so one Latin ratio
48
+ * applies to the whole family.
49
+ *
50
+ * Applied ONLY to `kiro/`-prefixed ids. The measurement is Kiro's charge for Kiro's payload
51
+ * shape, and the id families below (`claude`, `deepseek`, `qwen`, ...) are also routed by
52
+ * Cursor, Anthropic direct and Antigravity, whose consumers read this same helper to size
53
+ * admission ceilings, `count_tokens` answers, and overflow-vs-429 classification. Widening a
54
+ * Kiro-derived constant to those callers would retune three unrelated subsystems from evidence
55
+ * that says nothing about them.
56
+ */
57
+ const KIRO_CHARS_PER_TOKEN = 2.8;
58
+
59
+ /**
60
+ * Ratio for the model families Kiro shares with other providers, when NOT routed through Kiro.
61
+ *
62
+ * These are code-heavy agent models, so 3.5 remains right for them; it is the value every
63
+ * non-Kiro consumer of this helper was calibrated against.
21
64
  */
22
- const KIRO_CHARS_PER_TOKEN = 3.5;
65
+ const AGENT_MODEL_CHARS_PER_TOKEN = 3.5;
23
66
 
24
- const KIRO_MODEL_PREFIXES = ["kiro", "claude", "deepseek", "minimax", "glm", "qwen"];
67
+ const AGENT_MODEL_PREFIXES = ["kiro", "claude", "deepseek", "minimax", "glm", "qwen"];
25
68
 
26
- /** Model-aware chars-per-token ratio. Unknown models fall back to the generic English ratio. */
69
+ /**
70
+ * Model-aware chars-per-token ratio for LATIN text. Unknown models fall back to the generic
71
+ * English ratio.
72
+ *
73
+ * This is the sparse-script ratio only. It stays exported, and keeps returning a single number,
74
+ * because callers and tests use it to compare model families; the CJK component is applied by
75
+ * `estimateTokens` per character rather than by rewriting this ratio.
76
+ */
27
77
  export function charsPerToken(modelId?: string): number {
28
78
  if (!modelId) return DEFAULT_CHARS_PER_TOKEN;
29
79
  const id = modelId.toLowerCase();
30
- if (KIRO_MODEL_PREFIXES.some(p => id.startsWith(p))) return KIRO_CHARS_PER_TOKEN;
80
+ // `estimateKiroTokens` always prefixes `kiro/`, so the Kiro-measured ratio reaches Kiro traffic
81
+ // and only Kiro traffic.
82
+ if (id.startsWith("kiro")) return KIRO_CHARS_PER_TOKEN;
83
+ if (AGENT_MODEL_PREFIXES.some(p => id.startsWith(p))) return AGENT_MODEL_CHARS_PER_TOKEN;
31
84
  return DEFAULT_CHARS_PER_TOKEN;
32
85
  }
33
86
 
34
87
  /**
35
- * CJK-aware ratio (devlog 260712 B3, audit R2#7): Korean/Chinese/Japanese text packs
36
- * roughly one token per 1.5-3 chars, so a CJK-heavy blob estimated at English ratios
37
- * badly undercounts. When >30% of chars are CJK, clamp DOWN to 2.5 chars/token
38
- * `min(model ratio, 2.5)` so per-model ratios (Claude 3.5, Kiro family) never rise.
88
+ * Chars per token for CJK text, applied to the CJK characters alone.
89
+ *
90
+ * Solved from the same recorded ground truth: holding the Latin ratio fixed and attributing the
91
+ * remainder of CJK-heavy samples to their Hangul/Han characters gives ~1.50 chars/token. A Korean
92
+ * character therefore costs roughly TWICE what a Latin character does, which is precisely what a
93
+ * single blended divisor cannot express.
39
94
  */
40
- const CJK_CHARS_PER_TOKEN = 2.5;
41
- const CJK_RATIO_THRESHOLD = 0.3;
42
- // Hangul syllables/jamo, CJK unified ideographs (+ext A), hiragana/katakana.
43
- const CJK_RE = /[\uAC00-\uD7A3\u1100-\u11FF\u3130-\u318F\u4E00-\u9FFF\u3400-\u4DBF\u3040-\u30FF]/;
95
+ const CJK_CHARS_PER_TOKEN = 1.5;
44
96
 
45
- function cjkRatio(text: string): number {
46
- if (text.length === 0) return 0;
47
- // Sample long blobs for O(1) cost: every char up to 2k, then a stride.
48
- const stride = text.length > 2048 ? Math.ceil(text.length / 2048) : 1;
97
+ /**
98
+ * Count CJK characters exactly, in one pass, with no allocation and no regex object per char.
99
+ *
100
+ * Ranges: Hangul syllables and jamo, CJK unified ideographs and extension A, hiragana/katakana.
101
+ * Surrogate pairs (rare CJK extensions beyond the BMP) are counted as their two code units,
102
+ * which slightly over-counts them — again the safe direction.
103
+ */
104
+ function countCjk(text: string): number {
49
105
  let cjk = 0;
50
- let sampled = 0;
51
- for (let i = 0; i < text.length; i += stride) {
52
- sampled++;
53
- if (CJK_RE.test(text[i]!)) cjk++;
106
+ for (let i = 0; i < text.length; i++) {
107
+ const c = text.charCodeAt(i);
108
+ if (
109
+ (c >= 0xac00 && c <= 0xd7a3)
110
+ || (c >= 0x1100 && c <= 0x11ff)
111
+ || (c >= 0x3130 && c <= 0x318f)
112
+ || (c >= 0x4e00 && c <= 0x9fff)
113
+ || (c >= 0x3400 && c <= 0x4dbf)
114
+ || (c >= 0x3040 && c <= 0x30ff)
115
+ ) cjk++;
54
116
  }
55
- return sampled === 0 ? 0 : cjk / sampled;
117
+ return cjk;
56
118
  }
57
119
 
58
120
  /**
@@ -80,7 +142,12 @@ export function estimateTokens(text: string, modelId?: string, contextWindow?: n
80
142
  if (!text) return 0;
81
143
  const len = text.length;
82
144
  if (len === 0) return 0;
83
- let ratio = charsPerToken(modelId);
84
- if (cjkRatio(text) > CJK_RATIO_THRESHOLD) ratio = Math.min(ratio, CJK_CHARS_PER_TOKEN);
85
- return capEstimateAtContextWindow(Math.max(1, Math.ceil(len / ratio)), contextWindow);
145
+ const latinRatio = charsPerToken(modelId);
146
+ const cjk = countCjk(text);
147
+ // Continuous in the CJK share: no threshold, so one added Korean character moves the estimate
148
+ // by a fraction of a token instead of switching the whole blob to a different divisor.
149
+ const estimate = cjk === 0
150
+ ? Math.ceil(len / latinRatio)
151
+ : Math.ceil((len - cjk) / latinRatio + cjk / CJK_CHARS_PER_TOKEN);
152
+ return capEstimateAtContextWindow(Math.max(1, estimate), contextWindow);
86
153
  }
@@ -23,6 +23,7 @@
23
23
  import { existsSync } from "node:fs";
24
24
  import { win32 as windowsPath } from "node:path";
25
25
  import { waitForSubprocessExit } from "./bounded-subprocess";
26
+ import { decodeWindowsTextBytes } from "./windows-text";
26
27
 
27
28
  import {
28
29
  resolveTrustedWindowsPowerShellExe,
@@ -98,7 +99,12 @@ export interface WindowsPrincipalLookupResult {
98
99
  success: boolean;
99
100
  exitCode: number | null;
100
101
  timedOut: boolean;
101
- stdout: string;
102
+ /**
103
+ * Raw child stdout. Bytes are allowed because `powershell.exe` writes the console
104
+ * output code page, not UTF-8, and the decode below is the thing under test: a seam
105
+ * that only carried a decoded string could never exercise it.
106
+ */
107
+ stdout: string | Uint8Array;
102
108
  }
103
109
 
104
110
  export type WindowsPrincipalRunner = (
@@ -138,7 +144,10 @@ function defaultWindowsPrincipalRunner(timeoutMs: number): WindowsPrincipalLooku
138
144
  success: result.success,
139
145
  exitCode: result.exitCode,
140
146
  timedOut: result.exitedDueToTimeout ?? false,
141
- stdout: result.stdout ? result.stdout.toString() : "",
147
+ // Bytes, NOT .toString(): that is UTF-8, and Windows PowerShell 5.1 emits the
148
+ // console output code page. A non-ASCII account name decoded as UTF-8 becomes
149
+ // U+FFFD and is then frozen into the identity cache.
150
+ stdout: result.stdout ?? new Uint8Array(),
142
151
  };
143
152
  }
144
153
 
@@ -152,8 +161,9 @@ async function defaultAsyncWindowsPrincipalRunner(
152
161
  windowsHide: true,
153
162
  });
154
163
  const { exitCode, timedOut } = await waitForSubprocessExit(proc, timeoutMs);
155
- const stdout = !timedOut && proc.stdout
156
- ? await new Response(proc.stdout).text().catch(() => "")
164
+ // `.bytes()` rather than `.text()`, for the same reason as the sync runner above.
165
+ const stdout: string | Uint8Array = !timedOut && proc.stdout
166
+ ? await new Response(proc.stdout).bytes().catch(() => new Uint8Array())
157
167
  : "";
158
168
  return {
159
169
  success: !timedOut && exitCode === 0,
@@ -165,6 +175,24 @@ async function defaultAsyncWindowsPrincipalRunner(
165
175
 
166
176
  let principalRunner: WindowsPrincipalRunner = defaultWindowsPrincipalRunner;
167
177
  let asyncPrincipalRunner: AsyncWindowsPrincipalRunner = defaultAsyncWindowsPrincipalRunner;
178
+ let principalLocaleForTests: string | undefined;
179
+
180
+ /**
181
+ * Decode child stdout the way the rest of this repository already decodes Windows
182
+ * console output: UTF-16 with or without a BOM, then STRICT UTF-8, then the locale's
183
+ * legacy code page. Strict-UTF-8-first is what keeps an ordinary UTF-8 host unaffected.
184
+ *
185
+ * The SID on the first line is ASCII by construction and survives either way, which is
186
+ * why this corruption stayed silent: only the account name on the second line breaks.
187
+ */
188
+ function decodePrincipalStdout(stdout: string | Uint8Array): string {
189
+ if (typeof stdout === "string") return stdout;
190
+ return decodeWindowsTextBytes(
191
+ stdout,
192
+ principalLocaleForTests ? { locale: principalLocaleForTests } : {},
193
+ );
194
+ }
195
+
168
196
  export interface WindowsPrincipalIdentity {
169
197
  readonly sid: string;
170
198
  readonly name: string;
@@ -220,7 +248,7 @@ function identityFromResult(result: WindowsPrincipalLookupResult): WindowsPrinci
220
248
  ? "timed out"
221
249
  : `exited ${result.exitCode ?? "null"}`);
222
250
  }
223
- const lines = result.stdout.trim().split(/\r?\n/);
251
+ const lines = decodePrincipalStdout(result.stdout).trim().split(/\r?\n/);
224
252
  const sid = lines[0]?.trim() ?? "";
225
253
  const name = lines[1]?.trim() ?? "";
226
254
  if (!SID_PATTERN.test(sid)) {
@@ -341,6 +369,26 @@ export function setAsyncWindowsPrincipalRunnerForTests(
341
369
  cachedIdentity = null;
342
370
  }
343
371
 
372
+ /**
373
+ * Test seam: pin the locale that selects the legacy code page.
374
+ *
375
+ * Required rather than convenient. `decodeWindowsTextBytes` picks ONE legacy encoding
376
+ * from the ambient locale, so CP949, CP932 and CP936 fixtures cannot all decode
377
+ * correctly in a single process without being told which to expect. Production passes
378
+ * nothing and keeps the ambient locale.
379
+ *
380
+ * Clears the cache and refuses mid-flight for the same reasons the runner setters do:
381
+ * a successful identity is returned from cache BEFORE any decode, and the async path
382
+ * decodes after its runner resolves.
383
+ */
384
+ export function setWindowsPrincipalLocaleForTests(locale: string | null): void {
385
+ if (asyncLookupInFlight) {
386
+ throw new Error("Cannot change the Windows principal locale while a lookup is in flight.");
387
+ }
388
+ principalLocaleForTests = locale ?? undefined;
389
+ cachedIdentity = null;
390
+ }
391
+
344
392
  /** Test seam: clear only process-local principal state. */
345
393
  export function resetWindowsPrincipalForTests(): void {
346
394
  if (asyncLookupInFlight) {
@@ -10,7 +10,7 @@
10
10
  * categories answer the only question rotation asks — "which of these is most likely to
11
11
  * serve the retry" — and within the healthy group a simple headroom sort is enough.
12
12
  */
13
- import { getCachedProviderAccountQuota } from "../providers/quota";
13
+ import { getCachedProviderAccountQuota, hasPassiveAccountQuota } from "../providers/quota";
14
14
  import { getKiroAccountExhaustion } from "../providers/kiro-usage";
15
15
 
16
16
  /** Lower sorts earlier. Unknown sits between measured-healthy and measured-empty. */
@@ -27,6 +27,24 @@ interface Ranked {
27
27
  index: number;
28
28
  }
29
29
 
30
+ /**
31
+ * How old a PASSIVELY observed quota may be and still steer routing.
32
+ *
33
+ * A probed row is fresh by construction: it exists only because a probe wrote it, and
34
+ * `fetchAccountQuota` re-probes once `ACCOUNT_QUOTA_TTL_MS` has passed. So no caller has
35
+ * ever needed an explicit age check, and `getCachedProviderAccountQuota` does not apply
36
+ * one.
37
+ *
38
+ * A passive row breaks that invariant — nothing re-probes it, so it can be hours or days
39
+ * old. Routing on such a reading is worse than routing on none: the unranked ring at
40
+ * least rotates, while a stale ranking sends every first attempt to an account that may
41
+ * have been spent since. The bound is longer than the probe TTL (an hour-old reading of
42
+ * a five-hour window is still informative) and far shorter than the six-hour disk
43
+ * horizon, which exists to preserve a value for DISPLAY — where the age is shown to the
44
+ * user and no automatic decision rides on it.
45
+ */
46
+ const PASSIVE_HEADROOM_MAX_AGE_MS = 60 * 60_000;
47
+
30
48
  /**
31
49
  * Remaining headroom across every window the provider reports.
32
50
  *
@@ -36,6 +54,9 @@ interface Ranked {
36
54
  function headroomOf(provider: string, accountId: string): number | null {
37
55
  const quota = getCachedProviderAccountQuota(provider, accountId);
38
56
  if (!quota) return null;
57
+ // Null, not a low rank: this must reproduce "no evidence" so a stale roster degrades to
58
+ // the unranked ring rather than to a differently wrong answer.
59
+ if (hasPassiveAccountQuota(provider) && Date.now() - quota.updatedAt > PASSIVE_HEADROOM_MAX_AGE_MS) return null;
39
60
  const percents = [
40
61
  quota.fiveHourPercent,
41
62
  quota.weeklyPercent,
@@ -56,6 +77,12 @@ export function rankAccountsByHeadroom(provider: string, ring: readonly string[]
56
77
  if (ring.length < 2) return [...ring];
57
78
 
58
79
  let sawEvidence = false;
80
+ // Same rule as hasHeadroomEvidence: a passive provider's partial roster must not rank
81
+ // at all. The failover path calls this directly (selectFailoverAccount), so the guard
82
+ // cannot live only in the pre-dispatch predicate.
83
+ if (hasPassiveAccountQuota(provider) && !ring.every(id => headroomOf(provider, id) !== null)) {
84
+ return [...ring];
85
+ }
59
86
  const ranked: Ranked[] = ring.map((id, index) => {
60
87
  // A provider-declared exhaustion verdict outranks the percentage: an account may sit at
61
88
  // 100% and still be servable when overage is enabled, and the verdict knows that.
@@ -84,6 +111,18 @@ export function rankAccountsByHeadroom(provider: string, ring: readonly string[]
84
111
  * can decline to act on a roster it knows nothing about.
85
112
  */
86
113
  export function hasHeadroomEvidence(provider: string, ids: readonly string[]): boolean {
114
+ // A PASSIVE provider needs evidence for EVERY candidate, not any one of them.
115
+ //
116
+ // A probe fills the whole roster in one pass (fetchProviderAccountQuotas), so "any"
117
+ // and "every" coincide there. An observation arrives one account at a time, so the
118
+ // normal passive state is "one measured, N unknown" -- and RANK_UNKNOWN (1) sorts
119
+ // AFTER RANK_HEALTHY (0), including behind a measured row sitting at 100% with zero
120
+ // headroom. Accepting partial evidence would therefore redirect the first attempt
121
+ // AWAY from an unmeasured account and TOWARD the one account known to be spent, which
122
+ // is the exact inversion of what ranking is for.
123
+ if (hasPassiveAccountQuota(provider)) {
124
+ return ids.length > 0 && ids.every(id => headroomOf(provider, id) !== null);
125
+ }
87
126
  return ids.some(id =>
88
127
  headroomOf(provider, id) !== null
89
128
  || (provider === "kiro" && getKiroAccountExhaustion(`${provider}\u0000${id}`) !== null));