@sema-agent/server 7.141.0 → 7.143.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (341) hide show
  1. package/MIGRATION.md +657 -0
  2. package/USAGE.md +102 -30
  3. package/dist/adoption/plan.js +22 -0
  4. package/dist/approval-ask-audit-store.d.ts +61 -43
  5. package/dist/approval-ask-audit-store.js +69 -44
  6. package/dist/approval-ask-machine.d.ts +15 -7
  7. package/dist/approval-ask-machine.js +10 -4
  8. package/dist/approval-card.d.ts +126 -27
  9. package/dist/approval-card.js +103 -23
  10. package/dist/approval-crash-convergence.d.ts +17 -7
  11. package/dist/approval-crash-convergence.js +94 -22
  12. package/dist/approval-deny-reasons.d.ts +34 -30
  13. package/dist/approval-deny-reasons.js +21 -6
  14. package/dist/approval-policy-settlement.d.ts +1 -1
  15. package/dist/approval-queue-kind.d.ts +27 -0
  16. package/dist/approval-queue-kind.js +6 -0
  17. package/dist/approval-reconciler.d.ts +41 -9
  18. package/dist/approval-reconciler.js +45 -42
  19. package/dist/audit.js +4 -2
  20. package/dist/auth-bridge.d.ts +9 -6
  21. package/dist/auth-bridge.js +9 -3
  22. package/dist/boot/adoption.js +2 -1
  23. package/dist/boot/budget-tracing.d.ts +17 -1
  24. package/dist/boot/budget-tracing.js +10 -6
  25. package/dist/boot/device-lane.d.ts +1 -1
  26. package/dist/boot/execution-env.d.ts +24 -4
  27. package/dist/boot/execution-env.js +26 -23
  28. package/dist/boot/execution-lane.js +2 -1
  29. package/dist/boot/exit-boundary.js +5 -4
  30. package/dist/boot/governance-seams.d.ts +26 -41
  31. package/dist/boot/governance-seams.js +61 -28
  32. package/dist/boot/late-binds.d.ts +7 -4
  33. package/dist/boot/late-binds.js +3 -1
  34. package/dist/boot/leader.js +2 -4
  35. package/dist/boot/memory-consolidation.d.ts +0 -8
  36. package/dist/boot/memory-consolidation.js +5 -4
  37. package/dist/boot/parked-revive-gate.js +2 -1
  38. package/dist/boot/permission-rules-audit.d.ts +0 -28
  39. package/dist/boot/permission-rules-audit.js +2 -1
  40. package/dist/boot/read-deny-anchor-audit.js +2 -1
  41. package/dist/boot/reapers.js +9 -7
  42. package/dist/boot/resolve-spec.d.ts +1 -1
  43. package/dist/boot/resolve-spec.js +19 -10
  44. package/dist/boot/retention-lane.js +5 -3
  45. package/dist/boot/runner-deps.d.ts +10 -7
  46. package/dist/boot/runner-deps.js +20 -29
  47. package/dist/boot/runtime-caps.js +3 -2
  48. package/dist/boot/session-faces.js +4 -3
  49. package/dist/boot/shutdown.d.ts +9 -0
  50. package/dist/boot/shutdown.js +19 -4
  51. package/dist/boot/stage-02-web-admin-pull.js +3 -0
  52. package/dist/boot/stage-03-store-backend.d.ts +4 -1
  53. package/dist/boot/stage-03-store-backend.js +1 -2
  54. package/dist/boot/stage-04-stores.d.ts +5 -2
  55. package/dist/boot/stage-04-stores.js +6 -7
  56. package/dist/boot/stage-05-execution-env.d.ts +6 -3
  57. package/dist/boot/stage-05-execution-env.js +8 -6
  58. package/dist/boot/stage-06-runners.d.ts +6 -3
  59. package/dist/boot/stage-06-runners.js +3 -4
  60. package/dist/boot/stage-07-capability-layer.d.ts +6 -3
  61. package/dist/boot/stage-07-capability-layer.js +7 -2
  62. package/dist/boot/stage-08-reapers.d.ts +6 -3
  63. package/dist/boot/stage-08-reapers.js +3 -2
  64. package/dist/boot/stage-09-leader.d.ts +6 -3
  65. package/dist/boot/stage-10-http-server.d.ts +6 -3
  66. package/dist/boot/stage-10-http-server.js +14 -5
  67. package/dist/boot/stores.d.ts +5 -2
  68. package/dist/boot/stores.js +10 -9
  69. package/dist/boot/web-admin.d.ts +5 -1
  70. package/dist/boot/web-admin.js +48 -36
  71. package/dist/boot/webfetch-summarize-lane.js +2 -1
  72. package/dist/boot/workflow-orchestration.js +3 -2
  73. package/dist/capabilities/builtin-tools.d.ts +2 -1
  74. package/dist/capabilities/builtin-tools.js +2 -4
  75. package/dist/capabilities/composite-members.d.ts +2 -1
  76. package/dist/capabilities/composite-members.js +2 -1
  77. package/dist/capabilities/hands-lane.d.ts +9 -39
  78. package/dist/capabilities/hands-lane.js +8 -13
  79. package/dist/capabilities/mcp-probe.d.ts +3 -2
  80. package/dist/capabilities/scenarios.d.ts +11 -1
  81. package/dist/capabilities/scenarios.js +1 -0
  82. package/dist/capabilities/skills.d.ts +1 -17
  83. package/dist/capabilities/skills.js +0 -8
  84. package/dist/capability-facts.d.ts +31 -7
  85. package/dist/capability-facts.js +27 -4
  86. package/dist/card-void.d.ts +6 -9
  87. package/dist/config-catalog.js +2 -1
  88. package/dist/config-dir-trust.js +58 -29
  89. package/dist/config-invariants.d.ts +7 -1
  90. package/dist/config-invariants.js +7 -0
  91. package/dist/config-provider.d.ts +10 -0
  92. package/dist/config-provider.js +15 -4
  93. package/dist/config-root-migration.d.ts +13 -0
  94. package/dist/config-root-migration.js +99 -0
  95. package/dist/config-types.d.ts +19 -8
  96. package/dist/config.d.ts +10 -3
  97. package/dist/config.js +18 -4
  98. package/dist/delegated-session-owner.d.ts +5 -3
  99. package/dist/delegated-session-owner.js +1 -1
  100. package/dist/device-ws-protocol.d.ts +1 -0
  101. package/dist/device-ws-protocol.js +2 -1
  102. package/dist/env-model-table-gate.d.ts +53 -0
  103. package/dist/env-model-table-gate.js +53 -0
  104. package/dist/execution-lane-caps.d.ts +2 -1
  105. package/dist/fleet/fleet-bus.d.ts +34 -55
  106. package/dist/fleet/fleet-bus.js +35 -100
  107. package/dist/fleet/fleet-reconciler.d.ts +4 -4
  108. package/dist/fleet/fleet-reconciler.js +1 -1
  109. package/dist/fleet/subagent-tail-bus.d.ts +14 -11
  110. package/dist/fleet/subagent-tail-bus.js +15 -28
  111. package/dist/hooks/hook-runner.d.ts +6 -0
  112. package/dist/hooks/hook-runner.js +58 -8
  113. package/dist/host-decision.d.ts +54 -20
  114. package/dist/host-decision.js +42 -14
  115. package/dist/http/admission.d.ts +1 -1
  116. package/dist/http/admission.js +7 -3
  117. package/dist/http/approval-sweeps.js +11 -8
  118. package/dist/http/caller-trust.d.ts +1 -1
  119. package/dist/http/dispatch.js +4 -2
  120. package/dist/http/durable-run-leg.js +2 -2
  121. package/dist/http/engine-refusal.d.ts +16 -0
  122. package/dist/http/engine-refusal.js +17 -0
  123. package/dist/http/execution-context.d.ts +2 -2
  124. package/dist/http/idempotency.d.ts +1 -1
  125. package/dist/http/memory-compliance-failure-reply.d.ts +1 -3
  126. package/dist/http/memory-compliance-failure-reply.js +4 -6
  127. package/dist/http/owner-access.d.ts +11 -0
  128. package/dist/http/owner-access.js +13 -0
  129. package/dist/http/principal-gate.d.ts +3 -1
  130. package/dist/http/principal-gate.js +17 -8
  131. package/dist/http/resume-legs.d.ts +21 -7
  132. package/dist/http/resume-legs.js +76 -58
  133. package/dist/http/resume-restatement.d.ts +39 -0
  134. package/dist/http/resume-restatement.js +25 -0
  135. package/dist/http/route-ctx.d.ts +35 -10
  136. package/dist/http/route-ctx.js +1 -1
  137. package/dist/http/route-table.js +2 -0
  138. package/dist/http/routes/approvals-assistant.d.ts +9 -1
  139. package/dist/http/routes/approvals-assistant.js +142 -31
  140. package/dist/http/routes/attachments.js +18 -0
  141. package/dist/http/routes/capabilities.js +16 -4
  142. package/dist/http/routes/memory-policy.d.ts +2 -2
  143. package/dist/http/routes/memory-policy.js +27 -7
  144. package/dist/http/routes/notify-wake.js +23 -25
  145. package/dist/http/routes/rules.js +48 -13
  146. package/dist/http/routes/runs.js +12 -11
  147. package/dist/http/routes/session-memory-sync.d.ts +91 -0
  148. package/dist/http/routes/session-memory-sync.js +174 -0
  149. package/dist/http/routes/session-sync.js +13 -20
  150. package/dist/http/routes/sessions-list.d.ts +2 -1
  151. package/dist/http/routes/sessions.d.ts +2 -2
  152. package/dist/http/routes/sessions.js +22 -23
  153. package/dist/http/routes/tasks.js +34 -54
  154. package/dist/http/run-read-face.d.ts +7 -0
  155. package/dist/http/run-read-face.js +9 -3
  156. package/dist/http/send.d.ts +5 -0
  157. package/dist/http/send.js +9 -2
  158. package/dist/http/server.d.ts +27 -15
  159. package/dist/http/server.js +4 -6
  160. package/dist/http/stored-body-replay.js +15 -4
  161. package/dist/http/submit-replay.d.ts +4 -3
  162. package/dist/http/submit-replay.js +2 -2
  163. package/dist/http/sync-submit-body.d.ts +21 -4
  164. package/dist/http/sync-submit-body.js +13 -0
  165. package/dist/http/wire-types.d.ts +17 -7
  166. package/dist/identity-source.d.ts +21 -2
  167. package/dist/identity-source.js +6 -0
  168. package/dist/leader/leader.js +5 -2
  169. package/dist/leader/wire.d.ts +7 -1
  170. package/dist/leader/wire.js +6 -5
  171. package/dist/leg-facts.d.ts +37 -0
  172. package/dist/leg-facts.js +15 -0
  173. package/dist/memory-bundle-engine.d.ts +1 -4
  174. package/dist/memory-bundle-engine.js +2 -2
  175. package/dist/memory-incident-sink.d.ts +19 -6
  176. package/dist/memory-incident-sink.js +8 -4
  177. package/dist/memory-operator-faces.d.ts +30 -66
  178. package/dist/memory-operator-faces.js +17 -88
  179. package/dist/memory-scope-stores.d.ts +38 -40
  180. package/dist/memory-scope-stores.js +51 -36
  181. package/dist/memory-scope.d.ts +0 -9
  182. package/dist/memory-scope.js +0 -9
  183. package/dist/memory-sync-client.d.ts +21 -21
  184. package/dist/memory-sync-client.js +62 -43
  185. package/dist/memory-sync-identity.d.ts +24 -0
  186. package/dist/memory-sync-identity.js +57 -0
  187. package/dist/memory-sync.d.ts +27 -5
  188. package/dist/memory-sync.js +29 -7
  189. package/dist/model-plane-face.d.ts +13 -7
  190. package/dist/model-plane-face.js +2 -1
  191. package/dist/observability/fail-open.d.ts +47 -14
  192. package/dist/observability/fail-open.js +37 -11
  193. package/dist/observability/run-terminal.d.ts +92 -34
  194. package/dist/observability/run-terminal.js +171 -162
  195. package/dist/orchestration/workflow-completion-inbox.d.ts +5 -19
  196. package/dist/orchestration/workflow-completion-inbox.js +0 -12
  197. package/dist/orchestration/workflow-notify-journal.d.ts +3 -2
  198. package/dist/orchestration/workflow-park-decision.d.ts +8 -3
  199. package/dist/orchestration/workflow-park-decision.js +5 -3
  200. package/dist/org-delivery-gate.d.ts +11 -0
  201. package/dist/org-delivery-gate.js +19 -1
  202. package/dist/parked-decide.d.ts +13 -9
  203. package/dist/parked-decide.js +7 -7
  204. package/dist/parked-revive-surface.js +1 -1
  205. package/dist/peer-inbound-face.d.ts +127 -0
  206. package/dist/peer-inbound-face.js +158 -0
  207. package/dist/permission-rule-list.d.ts +13 -9
  208. package/dist/permission-rule-list.js +7 -3
  209. package/dist/plugins/approval-ask-store-file.d.ts +14 -4
  210. package/dist/plugins/approval-ask-store-file.js +64 -11
  211. package/dist/plugins/approval-ask-store-memory.d.ts +2 -1
  212. package/dist/plugins/approval-ask-store-memory.js +17 -14
  213. package/dist/plugins/approval-ask-store-sql.d.ts +70 -23
  214. package/dist/plugins/approval-ask-store-sql.js +79 -30
  215. package/dist/plugins/checkpoint-store-sql.d.ts +4 -4
  216. package/dist/plugins/checkpoint-store-sql.js +9 -7
  217. package/dist/plugins/file-history-store-sql.d.ts +24 -18
  218. package/dist/plugins/file-history-store-sql.js +24 -19
  219. package/dist/plugins/file-run-store.d.ts +35 -1
  220. package/dist/plugins/file-run-store.js +108 -9
  221. package/dist/plugins/imported-layer-root-retirement.d.ts +82 -0
  222. package/dist/plugins/imported-layer-root-retirement.js +96 -0
  223. package/dist/plugins/local-checkpoint-store.d.ts +1 -1
  224. package/dist/plugins/local-checkpoint-store.js +2 -2
  225. package/dist/plugins/local-session-store.d.ts +42 -9
  226. package/dist/plugins/local-session-store.js +83 -33
  227. package/dist/plugins/local-task-attachment-store.d.ts +12 -2
  228. package/dist/plugins/local-task-attachment-store.js +67 -43
  229. package/dist/plugins/mailbox-recipient-lifecycle.d.ts +9 -31
  230. package/dist/plugins/mailbox-recipient-lifecycle.js +4 -0
  231. package/dist/plugins/mailbox-store-sql.d.ts +12 -8
  232. package/dist/plugins/mailbox-store-sql.js +76 -7
  233. package/dist/plugins/memory-key-guards.d.ts +3 -3
  234. package/dist/plugins/memory-run-store.d.ts +6 -1
  235. package/dist/plugins/memory-run-store.js +11 -1
  236. package/dist/plugins/memory-sync-store-pg.d.ts +15 -0
  237. package/dist/plugins/memory-sync-store-pg.js +21 -0
  238. package/dist/plugins/memory-sync-store-tidb.d.ts +2 -0
  239. package/dist/plugins/memory-sync-store-tidb.js +13 -1
  240. package/dist/plugins/one-time-migrations.d.ts +58 -2
  241. package/dist/plugins/one-time-migrations.js +161 -4
  242. package/dist/plugins/org-delivery-quota-store.d.ts +1 -1
  243. package/dist/plugins/org-delivery-quota-store.js +2 -12
  244. package/dist/plugins/park-commit-watch.d.ts +1 -1
  245. package/dist/plugins/permission-rule-store-file.d.ts +10 -1
  246. package/dist/plugins/permission-rule-store-file.js +15 -2
  247. package/dist/plugins/permission-rule-store-sql.d.ts +31 -1
  248. package/dist/plugins/permission-rule-store-sql.js +35 -2
  249. package/dist/plugins/remote-env-device.d.ts +8 -11
  250. package/dist/plugins/remote-env-device.js +6 -3
  251. package/dist/plugins/remote-env-e2b.d.ts +23 -4
  252. package/dist/plugins/remote-env-e2b.js +111 -28
  253. package/dist/plugins/remote-env-k8s.d.ts +61 -22
  254. package/dist/plugins/remote-env-k8s.js +133 -33
  255. package/dist/plugins/remote-env-local-docker.d.ts +15 -3
  256. package/dist/plugins/remote-env-local-docker.js +40 -12
  257. package/dist/plugins/retention-store-sql.d.ts +2 -2
  258. package/dist/plugins/run-store-sql.d.ts +9 -1
  259. package/dist/plugins/run-store-sql.js +9 -1
  260. package/dist/plugins/sandbox-read-floor.d.ts +5 -7
  261. package/dist/plugins/sandbox-read-floor.js +2 -7
  262. package/dist/plugins/sql-errors.d.ts +10 -1
  263. package/dist/plugins/sql-errors.js +13 -0
  264. package/dist/plugins/store-backend.d.ts +33 -35
  265. package/dist/plugins/store-backend.js +10 -1
  266. package/dist/plugins/store-contracts.d.ts +50 -1
  267. package/dist/plugins/store-contracts.js +6 -0
  268. package/dist/plugins/task-attachment-store.d.ts +112 -14
  269. package/dist/plugins/task-attachment-store.js +207 -29
  270. package/dist/plugins/tool-result-store-sql.d.ts +9 -8
  271. package/dist/plugins/tool-result-store-sql.js +23 -2
  272. package/dist/plugins/wiring-section-codec.d.ts +32 -0
  273. package/dist/plugins/wiring-section-codec.js +55 -0
  274. package/dist/plugins/workspace-fill.d.ts +60 -0
  275. package/dist/plugins/workspace-fill.js +106 -0
  276. package/dist/project-bundle.d.ts +38 -0
  277. package/dist/project-bundle.js +609 -0
  278. package/dist/rule-entry-list.d.ts +3 -2
  279. package/dist/rule-entry-list.js +5 -1
  280. package/dist/rules-consent.d.ts +28 -5
  281. package/dist/rules-consent.js +40 -28
  282. package/dist/run-local.js +11 -13
  283. package/dist/runs.d.ts +12 -8
  284. package/dist/runs.js +6 -8
  285. package/dist/runtime-governance.d.ts +1 -1
  286. package/dist/sandbox-egress-policy.d.ts +4 -1
  287. package/dist/sandbox-egress-policy.js +3 -1
  288. package/dist/sandbox-git-plane.d.ts +31 -12
  289. package/dist/sandbox-git-plane.js +24 -4
  290. package/dist/security.d.ts +9 -0
  291. package/dist/security.js +8 -0
  292. package/dist/server-secret-env.d.ts +4 -0
  293. package/dist/session-error-wire.d.ts +36 -0
  294. package/dist/session-error-wire.js +13 -0
  295. package/dist/session-incarnation.d.ts +1 -1
  296. package/dist/session-notify-door.d.ts +106 -0
  297. package/dist/session-notify-door.js +93 -0
  298. package/dist/session-sync.d.ts +9 -14
  299. package/dist/session-sync.js +5 -5
  300. package/dist/session-workspace.d.ts +3 -13
  301. package/dist/session-workspace.js +0 -9
  302. package/dist/standing-answer-grant.d.ts +5 -0
  303. package/dist/standing-answer-grant.js +10 -0
  304. package/dist/stores-sql/session-policy-store-sql.d.ts +4 -2
  305. package/dist/stores-sql/session-policy-store-sql.js +3 -13
  306. package/dist/task-mcp.d.ts +1 -1
  307. package/dist/task-settings.d.ts +21 -10
  308. package/dist/task-settings.js +46 -13
  309. package/dist/terminal.d.ts +39 -19
  310. package/dist/terminal.js +65 -22
  311. package/dist/tool-approval.d.ts +56 -37
  312. package/dist/tool-approval.js +78 -71
  313. package/dist/trace/core-keyset-guard.d.ts +134 -14
  314. package/dist/trace/engine-notice-wire.d.ts +9 -3
  315. package/dist/trace/engine-notice-wire.js +7 -3
  316. package/dist/trace/exit-seat.d.ts +9 -2
  317. package/dist/trace/exit-seat.js +1 -0
  318. package/dist/trace/ledger-sink.d.ts +10 -6
  319. package/dist/trace/ledger-sink.js +2 -2
  320. package/dist/trace/model-segment.d.ts +55 -25
  321. package/dist/trace/model-segment.js +101 -33
  322. package/dist/trace/project.d.ts +25 -5
  323. package/dist/trace/project.js +22 -0
  324. package/dist/trace/redact.d.ts +61 -9
  325. package/dist/trace/redact.js +1490 -468
  326. package/dist/trace/self-minted-wire.d.ts +745 -24
  327. package/dist/trace/self-minted-wire.js +239 -9
  328. package/dist/trace/wire-projection-faces.d.ts +2 -2
  329. package/dist/trace/wire-projection-faces.js +2 -2
  330. package/dist/verifier-pause-card.d.ts +33 -5
  331. package/dist/verifier-pause-card.js +15 -10
  332. package/dist/web-admin-client/apply-effective.d.ts +12 -16
  333. package/dist/web-admin-client/apply-effective.js +109 -9
  334. package/dist/web-admin-client/effective-keys.js +2 -0
  335. package/dist/web-admin-client/types.d.ts +4 -0
  336. package/dist/workspace-space.d.ts +19 -3
  337. package/dist/workspace-space.js +8 -4
  338. package/docs/DEVICE-EXECUTOR-PROTOCOL.md +3 -1
  339. package/package.json +3 -2
  340. package/wire-changes/server@7.142.0.json +639 -0
  341. package/wire-changes/server@7.143.0.json +1167 -0
@@ -461,9 +461,11 @@ let urlSegmentRuns = 0;
461
461
  let urlSegmentReplays = 0;
462
462
  let ssh2ScanBytes = 0;
463
463
  let mappedUnits = 0;
464
+ let coverageVisits = 0;
465
+ let rowVisits = 0;
464
466
  let readings = 0;
465
467
  export function armPoolBudgetForTest() {
466
- return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings };
468
+ return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings, coverageVisits, rowVisits };
467
469
  }
468
470
  export function resetArmPoolBudgetForTest() {
469
471
  poolSpanCount = 0;
@@ -478,6 +480,8 @@ export function resetArmPoolBudgetForTest() {
478
480
  ssh2ScanBytes = 0;
479
481
  mappedUnits = 0;
480
482
  readings = 0;
483
+ coverageVisits = 0;
484
+ rowVisits = 0;
481
485
  }
482
486
  const unitBit = (t, c) => (t[c >>> 5] >>> (c & 31)) & 1;
483
487
  let codeUnitDomain;
@@ -707,6 +711,166 @@ function deriveLiteralHead(re) {
707
711
  first: unionMembers(per.map((p) => p.first)),
708
712
  };
709
713
  }
714
+ const repeatLen = (k, len) => (k === 0 ? 0 : k * len);
715
+ function readReachAlternatives(src, from) {
716
+ const alts = [];
717
+ let i = from;
718
+ for (;;) {
719
+ const one = readReachSequence(src, i);
720
+ alts.push(one.reading);
721
+ i = one.next;
722
+ if (src[i] !== "|")
723
+ break;
724
+ i += 1;
725
+ }
726
+ return { alts, next: i };
727
+ }
728
+ function readReachSequence(src, from) {
729
+ const fail = (at, why) => {
730
+ throw new Error(`redact: decision-reach derivation cannot read ${why} at ${at} in /${src}/`);
731
+ };
732
+ let pos = 0;
733
+ let reach = 0;
734
+ let beyond = false;
735
+ let wallReach = 0;
736
+ const goesBeyond = () => {
737
+ beyond = true;
738
+ wallReach = Math.max(wallReach, reach);
739
+ };
740
+ let i = from;
741
+ while (i < src.length) {
742
+ const c = src[i];
743
+ if (c === "|" || c === ")")
744
+ break;
745
+ let atom;
746
+ if (c === "(") {
747
+ const open = /^\((\?:|\?=|\?!|\?<=|\?<!)/.exec(src.slice(i));
748
+ if (open === null)
749
+ return fail(i, "a capturing or named group");
750
+ const mark = open[1];
751
+ const inner = readReachAlternatives(src, i + 1 + mark.length);
752
+ if (src[inner.next] !== ")")
753
+ return fail(i, "an unbalanced group");
754
+ i = inner.next + 1;
755
+ atom = {
756
+ len: Math.max(...inner.alts.map((a) => a.len)),
757
+ reach: Math.max(...inner.alts.map((a) => a.reach)),
758
+ beyond: inner.alts.some((a) => a.beyond),
759
+ wallReach: Math.max(0, ...inner.alts.filter((a) => a.beyond).map((a) => a.wallReach)),
760
+ };
761
+ if (mark === "?<=" || mark === "?<!")
762
+ continue;
763
+ if (mark === "?=" || mark === "?!") {
764
+ if (pos === Infinity)
765
+ goesBeyond();
766
+ else {
767
+ reach = Math.max(reach, pos + atom.reach);
768
+ if (atom.beyond) {
769
+ beyond = true;
770
+ wallReach = Math.max(wallReach, pos + atom.wallReach);
771
+ }
772
+ }
773
+ continue;
774
+ }
775
+ }
776
+ else if (c === "[") {
777
+ let k = i + 1;
778
+ while (k < src.length && src[k] !== "]")
779
+ k += src[k] === "\\" ? 2 : 1;
780
+ if (k >= src.length)
781
+ return fail(i, "an unterminated character class");
782
+ atom = { len: 1, reach: 1, beyond: false, wallReach: 0 };
783
+ i = k + 1;
784
+ }
785
+ else if (c === "\\") {
786
+ const e = src[i + 1];
787
+ if (e === undefined)
788
+ return fail(i, "a dangling escape");
789
+ if (e === "b" || e === "B") {
790
+ if (pos === Infinity)
791
+ goesBeyond();
792
+ else
793
+ reach = Math.max(reach, pos + 1);
794
+ i += 2;
795
+ continue;
796
+ }
797
+ if (/[1-9k]/.test(e))
798
+ return fail(i, "a back-reference");
799
+ const wide = /^\\(?:u[0-9A-Fa-f]{4}|u\{[0-9A-Fa-f]+\}|x[0-9A-Fa-f]{2}|[pP]\{[^}]+\}|c[A-Za-z]|.)/.exec(src.slice(i));
800
+ atom = { len: 1, reach: 1, beyond: false, wallReach: 0 };
801
+ i += wide[0].length;
802
+ }
803
+ else if (c === "^" || c === "$") {
804
+ return fail(i, "an anchor");
805
+ }
806
+ else if (c === "*" || c === "+" || c === "?" || c === "{") {
807
+ return fail(i, "a quantifier without an atom");
808
+ }
809
+ else {
810
+ atom = { len: 1, reach: 1, beyond: false, wallReach: 0 };
811
+ i += 1;
812
+ }
813
+ let lo = 1;
814
+ let hi = 1;
815
+ const q = src[i];
816
+ if (q === "*") {
817
+ lo = 0;
818
+ hi = Infinity;
819
+ i += 1;
820
+ }
821
+ else if (q === "+") {
822
+ lo = 1;
823
+ hi = Infinity;
824
+ i += 1;
825
+ }
826
+ else if (q === "?") {
827
+ lo = 0;
828
+ hi = 1;
829
+ i += 1;
830
+ }
831
+ else if (q === "{") {
832
+ const m = /^\{(\d+)(,(\d*))?\}/.exec(src.slice(i));
833
+ if (m === null)
834
+ return fail(i, "a quantifier");
835
+ lo = Number(m[1]);
836
+ hi = m[2] === undefined ? lo : m[3] === "" ? Infinity : Number(m[3]);
837
+ i += m[0].length;
838
+ }
839
+ if (src[i] === "?")
840
+ return fail(i, "a lazy quantifier");
841
+ if (pos === Infinity) {
842
+ if (lo > 0)
843
+ goesBeyond();
844
+ continue;
845
+ }
846
+ const counted = hi === Infinity ? lo : hi;
847
+ if (counted > 0) {
848
+ reach = Math.max(reach, pos + repeatLen(counted - 1, atom.len) + atom.reach);
849
+ if (atom.beyond) {
850
+ beyond = true;
851
+ wallReach = Math.max(wallReach, pos + repeatLen(counted - 1, atom.len) + atom.wallReach);
852
+ }
853
+ }
854
+ pos = hi === Infinity || atom.len === Infinity ? Infinity : pos + repeatLen(hi, atom.len);
855
+ }
856
+ return { reading: { len: pos, reach, beyond, wallReach }, next: i };
857
+ }
858
+ function regexDecisionReach(re, lexical) {
859
+ if (/[uv]/.test(re.flags))
860
+ throw new Error(`redact: decision-reach derivation does not support the u/v flags on /${re.source}/`);
861
+ const read = readReachAlternatives(re.source, 0);
862
+ if (read.next !== re.source.length)
863
+ throw new Error(`redact: decision-reach derivation stopped at ${read.next} in /${re.source}/`);
864
+ let reach = 0;
865
+ for (const alt of read.alts) {
866
+ if (alt.beyond && !lexical)
867
+ throw new Error(`redact: arm /${re.source}/ decides beyond an unbounded run but registers no undecided tail`);
868
+ reach = Math.max(reach, alt.reach, alt.beyond ? alt.wallReach : 0);
869
+ }
870
+ if (!Number.isFinite(reach))
871
+ throw new Error(`redact: decision-reach derivation read no finite reach for /${re.source}/`);
872
+ return reach;
873
+ }
710
874
  function scanResumeAfterHit(start, end, back) {
711
875
  return Math.max(start + 1, end - back);
712
876
  }
@@ -715,7 +879,7 @@ function collectRegexSpans(a, text, out) {
715
879
  g.lastIndex = 0;
716
880
  for (let m = g.exec(text); m !== null; m = g.exec(text)) {
717
881
  const end = m.index + m[0].length;
718
- out.push({ start: m.index, end, repl: a.repl, rank: a.rank, order: a.order });
882
+ out.push({ start: m.index, end, repl: a.repl, rank: a.rank, order: a.order, anchor: m.index, closeAt: NOT_CLOSED });
719
883
  g.lastIndex = scanResumeAfterHit(m.index, end, a.back);
720
884
  }
721
885
  }
@@ -792,7 +956,7 @@ function collectValueArmSpans(arm, text, out) {
792
956
  }
793
957
  else
794
958
  break;
795
- out.push({ start: idx, end, repl, rank, order });
959
+ out.push({ start: idx, end, repl, rank, order, anchor: idx, closeAt: NOT_CLOSED });
796
960
  pos = scanResumeAfterHit(idx, end, back);
797
961
  }
798
962
  }
@@ -808,7 +972,7 @@ function collectStickySpans(a, text, at, out) {
808
972
  if (m === null)
809
973
  continue;
810
974
  const end = p + m[0].length;
811
- out.push({ start: p, end, repl, rank, order });
975
+ out.push({ start: p, end, repl, rank, order, anchor: p, closeAt: NOT_CLOSED });
812
976
  pos = scanResumeAfterHit(p, end, back);
813
977
  }
814
978
  }
@@ -871,13 +1035,13 @@ function undecidedKeywordStart(text, t) {
871
1035
  return -1;
872
1036
  }
873
1037
  const CREDENTIAL_NAME_ANCHORS = {
874
- api_key: "structure-only", apikey: "structure-only", "api-key": "structure-only", key: "structure-only",
875
- token: "structure-only", access_token: "structure-only", refresh_token: "structure-only", id_token: "structure-only",
1038
+ api_key: "declarative", apikey: "declarative", "api-key": "declarative", key: "structure-only",
1039
+ token: "structure-only", access_token: "declarative", refresh_token: "declarative", id_token: "declarative",
876
1040
  auth: "structure-only", authorization: "auth-scheme", "proxy-authorization": "auth-scheme",
877
1041
  password: "declarative", passwd: "any-form", pwd: "any-form",
878
- secret: "declarative", client_secret: "declarative", private_key: "structure-only",
1042
+ secret: "declarative", client_secret: "declarative", private_key: "declarative",
879
1043
  signature: "structure-only", sig: "structure-only",
880
- "x-amz-signature": "structure-only", "x-amz-security-token": "structure-only", "x-amz-credential": "structure-only",
1044
+ "x-amz-signature": "declarative", "x-amz-security-token": "declarative", "x-amz-credential": "declarative",
881
1045
  sas: "structure-only",
882
1046
  };
883
1047
  const CREDENTIAL_NAME_ALT = (() => {
@@ -921,6 +1085,8 @@ const STRUCTURAL_SECRET_PATTERNS = [
921
1085
  [/:(?<=[A-Za-z][A-Za-z0-9+.-]*:)\/\//g, "$1«redacted»"],
922
1086
  [new RegExp(buildJsonSecretFieldSource(), "gi"), '$1"«redacted»"'],
923
1087
  ];
1088
+ const PPK_FILE_HEAD = "PuTTY-User-Key-File-";
1089
+ const PPK_LINES_FIELD = "Private-Lines: ";
924
1090
  const PPK_CONTAINER_TOKEN = "«redacted:private-key»";
925
1091
  const PEM_FRAGMENT_TOKEN = "«redacted:private-key:partial»";
926
1092
  const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
@@ -962,7 +1128,7 @@ function skipPemSeps(scan, s, i, limit) {
962
1128
  for (;;) {
963
1129
  if (i >= limit)
964
1130
  return i;
965
- if (opaqueUnitEnd(scan, i) >= 0)
1131
+ if (tokenEnd(scan, i) >= 0)
966
1132
  return i;
967
1133
  const c = s.charCodeAt(i);
968
1134
  if (isSepCode(c)) {
@@ -1006,16 +1172,16 @@ function pemInlineSepLen(s, i, limit) {
1006
1172
  return 0;
1007
1173
  }
1008
1174
  function probePemHeaderToken(scan, s, p, limit) {
1009
- if (p >= limit || opaqueUnitEnd(scan, p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
1175
+ if (p >= limit || tokenEnd(scan, p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
1010
1176
  return -1;
1011
1177
  p += 1;
1012
1178
  let identLen = 1;
1013
- while (identLen < PEM_HEADER_IDENT_MAX && p < limit && opaqueUnitEnd(scan, p) < 0 && isIdentCode(s.charCodeAt(p))) {
1179
+ while (identLen < PEM_HEADER_IDENT_MAX && p < limit && tokenEnd(scan, p) < 0 && isIdentCode(s.charCodeAt(p))) {
1014
1180
  p += 1;
1015
1181
  identLen += 1;
1016
1182
  }
1017
1183
  for (;;) {
1018
- if (p >= limit || opaqueUnitEnd(scan, p) >= 0)
1184
+ if (p >= limit || tokenEnd(scan, p) >= 0)
1019
1185
  return -1;
1020
1186
  const c = s.charCodeAt(p);
1021
1187
  if (c === CH_SPACE || c === CH_TAB) {
@@ -1047,7 +1213,7 @@ function parsePemHeaderLine(scan, s, i, limit, allowBare, json) {
1047
1213
  for (;;) {
1048
1214
  if (p >= limit)
1049
1215
  return setPemHdr(scan, start, p, p, true);
1050
- const unit = opaqueUnitEnd(scan, p);
1216
+ const unit = crossEnd(scan, p);
1051
1217
  if (unit >= 0) {
1052
1218
  p = Math.min(unit, limit);
1053
1219
  continue;
@@ -1077,7 +1243,7 @@ function parsePemHeaderLine(scan, s, i, limit, allowBare, json) {
1077
1243
  }
1078
1244
  let q = p;
1079
1245
  for (;;) {
1080
- if (opaqueUnitEnd(scan, q) >= 0)
1246
+ if (tokenEnd(scan, q) >= 0)
1081
1247
  break;
1082
1248
  const b = pemLineBreakLen(s, q, limit);
1083
1249
  if (b === 1)
@@ -1104,7 +1270,7 @@ function parsePemBody(scan, s, i, limit) {
1104
1270
  for (;;) {
1105
1271
  const q = skipPemSeps(scan, s, p, limit);
1106
1272
  let r = q;
1107
- while (r < limit && opaqueUnitEnd(scan, r) < 0 && isBase64Code(s.charCodeAt(r)))
1273
+ while (r < limit && tokenEnd(scan, r) < 0 && isBase64Code(s.charCodeAt(r)))
1108
1274
  r += 1;
1109
1275
  if (r - q < PEM_BODY_RUN_MIN)
1110
1276
  break;
@@ -1113,20 +1279,34 @@ function parsePemBody(scan, s, i, limit) {
1113
1279
  }
1114
1280
  return end;
1115
1281
  }
1116
- function rightmostPemRunEnd(scan, s, from, to) {
1282
+ function rightmostPemRunEnd(scan, s, from, to, crossEndsRun) {
1117
1283
  let best = -1;
1118
1284
  let runStart = -1;
1119
1285
  let plainBackslashTill = -1;
1120
1286
  for (let p = from; p <= to; p++) {
1121
1287
  if (p < to) {
1122
- const unit = opaqueUnitEnd(scan, p);
1123
- if (unit > 0) {
1288
+ const tok = tokenEnd(scan, p);
1289
+ if (tok > 0) {
1124
1290
  if (runStart >= 0 && p - runStart >= PEM_BODY_RUN_MIN)
1125
1291
  best = p;
1126
1292
  runStart = -1;
1127
- if (unit > to)
1293
+ if (tok > to)
1128
1294
  return best;
1129
- p = unit - 1;
1295
+ p = tok - 1;
1296
+ continue;
1297
+ }
1298
+ const cross = crossEnd(scan, p);
1299
+ if (cross > 0) {
1300
+ if (crossEndsRun) {
1301
+ if (runStart >= 0 && p - runStart >= PEM_BODY_RUN_MIN)
1302
+ best = p;
1303
+ runStart = -1;
1304
+ if (cross >= to)
1305
+ return best;
1306
+ }
1307
+ else if (cross >= to)
1308
+ return runStart >= 0 && to - runStart >= PEM_BODY_RUN_MIN ? to : best;
1309
+ p = cross - 1;
1130
1310
  continue;
1131
1311
  }
1132
1312
  if (p >= plainBackslashTill && s.charCodeAt(p) === CH_BACKSLASH) {
@@ -1193,7 +1373,7 @@ function findPemFence(scan, text, from) {
1193
1373
  if (!isPemFenceLabel(text, labelFrom, close))
1194
1374
  continue;
1195
1375
  const after = close + PEM_FENCE_CLOSE.length;
1196
- if (anchorHeld(scan, fence, after))
1376
+ if (lineHeld(scan, fence, after, readingKey(familyCode("pem-armor"), fence)))
1197
1377
  continue;
1198
1378
  return { fence, after };
1199
1379
  }
@@ -1201,131 +1381,112 @@ function findPemFence(scan, text, from) {
1201
1381
  const SSH2_FENCE_OPEN = "---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----";
1202
1382
  const SSH2_FENCE_CLOSE = "---- END SSH2 ENCRYPTED PRIVATE KEY ----";
1203
1383
  const SSH2_BODY_MAX = 8192;
1204
- function unitCountWithin(scan, from, to, max, t = scan.table) {
1205
- if (to - from <= max)
1206
- return true;
1207
- noteEvidenceRead(scan, from);
1208
- noteEvidenceRead(scan, to);
1209
- const lo = firstCredentialSpanFrom(t, from);
1210
- let a = lo;
1211
- let b = t.ends.length;
1212
- while (a < b) {
1213
- const mid = (a + b) >>> 1;
1214
- if (t.ends[mid] <= to)
1215
- a = mid + 1;
1216
- else
1217
- b = mid;
1218
- }
1219
- if (a === lo)
1220
- return false;
1221
- const saved = unitSavings(t);
1222
- return to - from - (saved[a] - saved[lo]) <= max;
1384
+ function armorCloseAt(scan, text, kind, bodyFrom, max) {
1385
+ const near = firstDelim(scan, text, kind, bodyFrom, false);
1386
+ if (near < 0)
1387
+ return -1;
1388
+ const free = firstDelim(scan, text, kind, bodyFrom, true);
1389
+ const far = free >= 0 ? free : lastNotHiddenDelim(scan, text, kind, near, text.length);
1390
+ if (far > near && blockWithin(scan, text, kind, bodyFrom, far, max))
1391
+ return far;
1392
+ return blockWithin(scan, text, kind, bodyFrom, near, max) ? near : -1;
1223
1393
  }
1224
- const UNIT_SAVINGS = new WeakMap();
1225
- function unitSavings(t) {
1226
- let saved = UNIT_SAVINGS.get(t);
1227
- if (saved === undefined) {
1228
- const n = t.starts.length;
1229
- saved = new Float64Array(n + 1);
1230
- for (let k = 0; k < n; k++)
1231
- saved[k + 1] = saved[k] + (t.ends[k] - t.starts[k] - 1);
1232
- UNIT_SAVINGS.set(t, saved);
1233
- }
1234
- return saved;
1394
+ function closeLineEnd(text, kind, close) {
1395
+ return kind === "ssh2-close" ? close + SSH2_FENCE_CLOSE.length : pemFenceLineEnd(text, close + PEM_END_OPEN.length);
1235
1396
  }
1236
- export function unitCountWithinForTest(table, from, to, max) {
1237
- const t = { starts: Int32Array.from(table, (x) => x[0]), ends: Int32Array.from(table, (x) => x[1]) };
1238
- return unitCountWithin(buildForensicScan(t, 1), from, to, max);
1397
+ function blockWithin(scan, text, kind, bodyFrom, close, max) {
1398
+ const lineEnd = closeLineEnd(text, kind, close);
1399
+ return measureWithin(scan, bodyFrom, lineEnd, max + (lineEnd - close));
1239
1400
  }
1240
1401
  function collectSsh2FenceSpans(scan, text, json, span, isHead) {
1241
- let cachedFrom = 0;
1242
- let cachedAt;
1243
- const nextClose = (from) => {
1244
- if (cachedAt !== undefined && cachedFrom <= from && (cachedAt < 0 || cachedAt >= from))
1245
- return cachedAt;
1246
- let p = text.indexOf(SSH2_FENCE_CLOSE, from);
1247
- ssh2ScanBytes += (p < 0 ? text.length : p) - from;
1248
- while (p >= 0 && inOpaqueUnit(scan, p)) {
1249
- const q = text.indexOf(SSH2_FENCE_CLOSE, p + 1);
1250
- ssh2ScanBytes += (q < 0 ? text.length : q) - (p + 1);
1251
- p = q;
1252
- }
1253
- cachedFrom = from;
1254
- cachedAt = p;
1255
- return p;
1256
- };
1257
- let freeFrom = 0;
1258
- let freeAt;
1259
- const freeClose = (from) => {
1260
- if (freeAt !== undefined && freeFrom <= from && (freeAt < 0 || freeAt >= from))
1261
- return freeAt;
1262
- let p = text.indexOf(SSH2_FENCE_CLOSE, from);
1263
- ssh2ScanBytes += (p < 0 ? text.length : p) - from;
1264
- while (p >= 0 && anchorHeld(scan, p, p + 1)) {
1265
- const q = text.indexOf(SSH2_FENCE_CLOSE, p + 1);
1266
- ssh2ScanBytes += (q < 0 ? text.length : q) - (p + 1);
1267
- p = q;
1268
- }
1269
- freeFrom = from;
1270
- freeAt = p;
1271
- return p;
1272
- };
1273
- const truncatedEnd = createTruncatedBlockEnd(scan, text, json, freeClose);
1402
+ const fam = familyCode("ssh2-armor");
1403
+ const opens = delimPositions(scan, text, "ssh2-open");
1404
+ const closes = delimPositions(scan, text, "ssh2-close");
1405
+ const blockEnds = [];
1406
+ let firstLive = 0;
1407
+ let freshFrom = -1;
1408
+ let freshEnd = -1;
1409
+ for (let oi = 0, ci = 0, rooted = false, prevEnd = -1; ci < closes.length;) {
1410
+ const c = closes[ci];
1411
+ if (oi < opens.length && opens[oi] < c) {
1412
+ rooted = true;
1413
+ prevEnd = -1;
1414
+ oi += 1;
1415
+ continue;
1416
+ }
1417
+ if (rooted && prevEnd >= 0) {
1418
+ scan.self = readingKey(fam, c);
1419
+ while (firstLive < blockEnds.length && uncoveredWithin(scan, blockEnds[firstLive], c + SSH2_FENCE_CLOSE.length) > SSH2_BODY_MAX + SSH2_FENCE_CLOSE.length)
1420
+ firstLive += 1;
1421
+ for (let j = firstLive; j < blockEnds.length; j++) {
1422
+ const from = blockEnds[j];
1423
+ if (from > c)
1424
+ break;
1425
+ if ((from === freshFrom && blockWithin(scan, text, "ssh2-close", freshEnd, c, SSH2_BODY_MAX - 1)) || blockWithin(scan, text, "ssh2-close", from, c, SSH2_BODY_MAX)) {
1426
+ span(from, c + SSH2_FENCE_CLOSE.length, c, NOT_CLOSED);
1427
+ freshFrom = from;
1428
+ freshEnd = c + SSH2_FENCE_CLOSE.length;
1429
+ break;
1430
+ }
1431
+ }
1432
+ }
1433
+ prevEnd = c + SSH2_FENCE_CLOSE.length;
1434
+ if (rooted)
1435
+ blockEnds.push(prevEnd);
1436
+ ci += 1;
1437
+ }
1274
1438
  let openFrom = 0;
1275
1439
  let openAt;
1276
1440
  const nextFreeOpen = (from) => {
1277
1441
  if (openAt !== undefined && openFrom <= from && (openAt < 0 || openAt >= from))
1278
1442
  return openAt;
1279
- let p = text.indexOf(SSH2_FENCE_OPEN, from);
1280
- ssh2ScanBytes += (p < 0 ? text.length : p) - from;
1281
- while (p >= 0 && anchorHeld(scan, p, p + SSH2_FENCE_OPEN.length)) {
1282
- const q = text.indexOf(SSH2_FENCE_OPEN, p + 1);
1283
- ssh2ScanBytes += (q < 0 ? text.length : q) - (p + 1);
1284
- p = q;
1285
- }
1443
+ let k = lowerBound(opens, from);
1444
+ while (k < opens.length && lineHeld(scan, opens[k], opens[k] + SSH2_FENCE_OPEN.length, readingKey(fam, opens[k])))
1445
+ k += 1;
1286
1446
  openFrom = from;
1287
- openAt = p;
1288
- return p;
1447
+ openAt = k < opens.length ? opens[k] : -1;
1448
+ return openAt;
1289
1449
  };
1290
- for (let from = 0;;) {
1291
- const fence = text.indexOf(SSH2_FENCE_OPEN, from);
1292
- if (fence < 0)
1293
- return;
1450
+ const truncatedEnd = createTruncatedBlockEnd(scan, text, json, (from) => firstDelim(scan, text, "ssh2-close", from, true));
1451
+ for (let oi = 0; oi < opens.length;) {
1452
+ const fence = opens[oi];
1294
1453
  const bodyFrom = fence + SSH2_FENCE_OPEN.length;
1295
- if (anchorHeld(scan, fence, bodyFrom)) {
1296
- from = fence + 1;
1454
+ if (lineHeld(scan, fence, bodyFrom, readingKey(fam, fence))) {
1455
+ oi += 1;
1297
1456
  continue;
1298
1457
  }
1299
- if (isHead && unitCountWithin(scan, bodyFrom, text.length, SSH2_BODY_MAX)) {
1300
- span(fence, text.length, true);
1458
+ scan.self = readingKey(fam, fence);
1459
+ if (isHead && measureWithin(scan, bodyFrom, text.length, SSH2_BODY_MAX)) {
1460
+ span(fence, text.length, fence, NOT_CLOSED, true);
1301
1461
  return;
1302
1462
  }
1303
1463
  let resume = fence + 1;
1304
1464
  if (!isHead) {
1305
1465
  const open = nextFreeOpen(bodyFrom);
1306
1466
  const limit = open < 0 ? text.length : open;
1307
- const t = truncatedEnd(bodyFrom, limit);
1308
- const bodyEnd = t < 0 ? -1 : armorBodyEnd(scan, text, json, bodyFrom, limit, false);
1309
- const end = bodyEnd < 0 ? -1 : Math.max(bodyEnd, t);
1310
- if (end >= 0 && unitCountWithin(scan, bodyFrom, end, SSH2_BODY_MAX, othersOf(scan))) {
1311
- span(fence, end);
1467
+ const readEnd = (crossEndsRun) => {
1468
+ const t = truncatedEnd(bodyFrom, limit, crossEndsRun);
1469
+ const bodyEnd = t < 0 ? -1 : armorBodyEnd(scan, text, json, bodyFrom, limit, false, crossEndsRun);
1470
+ return bodyEnd < 0 ? -1 : Math.max(bodyEnd, t);
1471
+ };
1472
+ const far = readEnd(false);
1473
+ const near = far >= 0 && !measureWithin(scan, bodyFrom, far, SSH2_BODY_MAX) ? readEnd(true) : -1;
1474
+ const end = near >= 0 && near < far ? near : far;
1475
+ if (end >= 0 && measureWithin(scan, bodyFrom, end, SSH2_BODY_MAX)) {
1476
+ span(fence, end, fence, NOT_CLOSED);
1312
1477
  resume = end;
1313
1478
  }
1314
1479
  }
1315
- const close = nextClose(bodyFrom);
1480
+ const close = armorCloseAt(scan, text, "ssh2-close", bodyFrom, SSH2_BODY_MAX);
1316
1481
  if (close < 0) {
1317
- if (isHead)
1482
+ if (isHead && firstDelim(scan, text, "ssh2-close", bodyFrom, false) < 0)
1318
1483
  return;
1319
- from = resume;
1320
- continue;
1321
- }
1322
- if (!unitCountWithin(scan, bodyFrom, close, SSH2_BODY_MAX)) {
1323
- from = resume;
1484
+ oi = lowerBound(opens, resume);
1324
1485
  continue;
1325
1486
  }
1326
1487
  const end = close + SSH2_FENCE_CLOSE.length;
1327
- span(fence, end);
1328
- from = Math.max(resume, scanResumeAfterHit(fence, end, SSH2_FENCE_OPEN.length));
1488
+ span(fence, end, fence, close);
1489
+ oi = lowerBound(opens, delimClass(scan, close) === DELIM_FREE ? Math.max(resume, scanResumeAfterHit(fence, end, SSH2_FENCE_OPEN.length)) : resume);
1329
1490
  }
1330
1491
  }
1331
1492
  const PEM_END_OPEN = "-----END";
@@ -1344,24 +1505,7 @@ function pemFenceLineEnd(text, labelFrom) {
1344
1505
  return isPemFenceLabel(text, labelFrom, close) ? close + PEM_FENCE_CLOSE.length : PEM_LINE_INVALID;
1345
1506
  }
1346
1507
  function collectPemFenceSpans(scan, text, span, isHead) {
1347
- let endFrom = -1;
1348
- let endAt = -1;
1349
- let endAfter = -1;
1350
- const visibleEndLine = (b) => {
1351
- const after = pemFenceLineEnd(text, b + PEM_END_OPEN.length);
1352
- return after >= 0 && !inOpaqueUnit(scan, b) ? after : -1;
1353
- };
1354
- const earliestEndFence = (from) => {
1355
- if (endFrom >= 0 && from >= endFrom && (endAt < 0 || endAt >= from))
1356
- return;
1357
- let b = text.indexOf(PEM_END_OPEN, from);
1358
- let after = -1;
1359
- while (b >= 0 && (after = visibleEndLine(b)) < 0)
1360
- b = text.indexOf(PEM_END_OPEN, b + 1);
1361
- endFrom = from;
1362
- endAt = b;
1363
- endAfter = after;
1364
- };
1508
+ const fam = familyCode("pem-armor");
1365
1509
  for (let from = 0;;) {
1366
1510
  const fence = text.indexOf(PEM_FENCE_OPEN, from);
1367
1511
  if (fence < 0)
@@ -1371,23 +1515,25 @@ function collectPemFenceSpans(scan, text, span, isHead) {
1371
1515
  from = fence + 1;
1372
1516
  continue;
1373
1517
  }
1374
- if (anchorHeld(scan, fence, bodyFrom === PEM_LINE_OPEN ? text.length : bodyFrom)) {
1518
+ if (lineHeld(scan, fence, bodyFrom === PEM_LINE_OPEN ? text.length : bodyFrom, readingKey(fam, fence))) {
1375
1519
  from = fence + 1;
1376
1520
  continue;
1377
1521
  }
1378
- if (bodyFrom !== PEM_LINE_OPEN)
1379
- earliestEndFence(bodyFrom);
1380
- if (bodyFrom === PEM_LINE_OPEN || endAt < 0) {
1522
+ scan.self = readingKey(fam, fence);
1523
+ const near = bodyFrom === PEM_LINE_OPEN ? -1 : firstDelim(scan, text, "pem-end", bodyFrom, false);
1524
+ if (near < 0 || (isHead && firstDelim(scan, text, "pem-end", bodyFrom, true) < 0)) {
1381
1525
  if (isHead)
1382
- span(fence, text.length, true);
1526
+ span(fence, text.length, fence, NOT_CLOSED, true);
1383
1527
  return;
1384
1528
  }
1385
- if (!unitCountWithin(scan, bodyFrom, endAt, PEM_FENCE_BODY_MAX)) {
1529
+ const endAt = armorCloseAt(scan, text, "pem-end", bodyFrom, PEM_FENCE_BODY_MAX);
1530
+ if (endAt < 0) {
1386
1531
  from = fence + 1;
1387
1532
  continue;
1388
1533
  }
1389
- span(fence, endAfter);
1390
- from = scanResumeAfterHit(fence, endAfter, PEM_FENCE_OPEN.length);
1534
+ const endAfter = pemFenceLineEnd(text, endAt + PEM_END_OPEN.length);
1535
+ span(fence, endAfter, fence, endAt);
1536
+ from = delimClass(scan, endAt) === DELIM_FREE ? scanResumeAfterHit(fence, endAfter, PEM_FENCE_OPEN.length) : fence + 1;
1391
1537
  }
1392
1538
  }
1393
1539
  function truncationWindowEnd(text, from, json) {
@@ -1404,13 +1550,13 @@ function truncationWindowEnd(text, from, json) {
1404
1550
  }
1405
1551
  return text.length;
1406
1552
  }
1407
- function armorBodyEnd(scan, text, json, from, limit, allowBare) {
1553
+ function armorBodyEnd(scan, text, json, from, limit, allowBare, crossEndsRun) {
1408
1554
  let pos = from;
1409
1555
  let bare = allowBare;
1410
1556
  let hitLimit = false;
1411
1557
  let rightmost = -1;
1412
1558
  while (parsePemHeaderLine(scan, text, pos, limit, bare, json)) {
1413
- const e = rightmostPemRunEnd(scan, text, scan.hdr.search, scan.hdr.contentEnd);
1559
+ const e = rightmostPemRunEnd(scan, text, scan.hdr.search, scan.hdr.contentEnd, crossEndsRun);
1414
1560
  if (e > rightmost)
1415
1561
  rightmost = e;
1416
1562
  pos = scan.hdr.next;
@@ -1429,7 +1575,7 @@ function armorBodyEnd(scan, text, json, from, limit, allowBare) {
1429
1575
  function createTruncatedBlockEnd(scan, text, json, closeAt) {
1430
1576
  let winFrom = -1;
1431
1577
  let winAt = -1;
1432
- return (from, limit) => {
1578
+ return (from, limit, crossEndsRun) => {
1433
1579
  if (!(winFrom >= 0 && from >= winFrom && from <= winAt)) {
1434
1580
  winFrom = from;
1435
1581
  winAt = truncationWindowEnd(text, from, json);
@@ -1437,38 +1583,28 @@ function createTruncatedBlockEnd(scan, text, json, closeAt) {
1437
1583
  const close = closeAt(from);
1438
1584
  if (close >= 0 && close < winAt)
1439
1585
  return -1;
1440
- return rightmostPemRunEnd(scan, text, from, Math.min(limit, winAt));
1586
+ return rightmostPemRunEnd(scan, text, from, Math.min(limit, winAt), crossEndsRun);
1441
1587
  };
1442
1588
  }
1443
1589
  function collectPemFragmentSpans(scan, text, json, span, isHead) {
1590
+ const fam = familyCode("pem-armor");
1444
1591
  let cur = findPemFence(scan, text, 0);
1445
1592
  let inherited = false;
1446
- let closeFrom = -1;
1447
- let closeAt = -1;
1448
- const freeClose = (from) => {
1449
- if (closeFrom >= 0 && from >= closeFrom && (closeAt < 0 || closeAt >= from))
1450
- return closeAt;
1451
- let b = text.indexOf(PEM_END_OPEN, from);
1452
- while (b >= 0 && (pemFenceLineEnd(text, b + PEM_END_OPEN.length) < 0 || anchorHeld(scan, b, b + 1)))
1453
- b = text.indexOf(PEM_END_OPEN, b + 1);
1454
- closeFrom = from;
1455
- closeAt = b;
1456
- return b;
1457
- };
1458
- const truncatedEnd = createTruncatedBlockEnd(scan, text, json, freeClose);
1593
+ const truncatedEnd = createTruncatedBlockEnd(scan, text, json, (from) => firstDelim(scan, text, "pem-end", from, true));
1459
1594
  while (cur !== undefined) {
1460
1595
  const next = findPemFence(scan, text, cur.after);
1461
1596
  const limit = next === undefined ? text.length : next.fence;
1462
- const bodyEnd = armorBodyEnd(scan, text, json, cur.after, limit, inherited);
1597
+ scan.self = readingKey(fam, cur.fence);
1598
+ const bodyEnd = armorBodyEnd(scan, text, json, cur.after, limit, inherited, false);
1463
1599
  inherited = scan.hdr.hitLimit;
1464
- const end = isHead || bodyEnd < 0 ? bodyEnd : Math.max(bodyEnd, truncatedEnd(cur.after, limit));
1600
+ const end = isHead || bodyEnd < 0 ? bodyEnd : Math.max(bodyEnd, truncatedEnd(cur.after, limit, false));
1465
1601
  if (end >= 0)
1466
- span(cur.fence, end);
1602
+ span(cur.fence, end, cur.fence, NOT_CLOSED);
1467
1603
  cur = next;
1468
1604
  }
1469
1605
  }
1470
1606
  function collectHeadlessPpkSpans(scan, text, span, isHead) {
1471
- const KEY = "Private-Lines: ";
1607
+ const KEY = PPK_LINES_FIELD;
1472
1608
  const n = text.length;
1473
1609
  const LINE_NONE = -1;
1474
1610
  const LINE_OPEN = -2;
@@ -1476,19 +1612,20 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
1476
1612
  if (!isHead)
1477
1613
  return false;
1478
1614
  let k = i;
1479
- while (k < n && opaqueUnitEnd(scan, k) < 0 && text.charCodeAt(k) === 92)
1615
+ while (k < n && tokenEnd(scan, k) < 0 && text.charCodeAt(k) === 92)
1480
1616
  k += 1;
1481
1617
  return k >= n;
1482
1618
  };
1483
1619
  const isB64 = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 43 || c === 47 || c === 61;
1484
- const b64At = (i) => opaqueUnitEnd(scan, i) < 0 && isB64(text.charCodeAt(i));
1485
- const skipHs = (i) => { while (i < n && opaqueUnitEnd(scan, i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
1620
+ const b64At = (i) => tokenEnd(scan, i) < 0 && isB64(text.charCodeAt(i));
1621
+ const skipHs = (i) => { while (i < n && tokenEnd(scan, i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
1486
1622
  i += 1; return i; };
1623
+ const lineEndHidden = (i) => i < n && delimClass(scan, i) === DELIM_HIDDEN;
1487
1624
  const skipNewlines = (i) => {
1488
1625
  let q = skipHs(i);
1489
1626
  let seen = false;
1490
1627
  for (;;) {
1491
- if (opaqueUnitEnd(scan, q) >= 0)
1628
+ if (lineEndHidden(q))
1492
1629
  break;
1493
1630
  let any = false;
1494
1631
  if (text.charCodeAt(q) === 13) {
@@ -1523,7 +1660,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
1523
1660
  return seen ? q : LINE_NONE;
1524
1661
  };
1525
1662
  const escapedNewlineLen = (i) => {
1526
- if (opaqueUnitEnd(scan, i) >= 0)
1663
+ if (lineEndHidden(i))
1527
1664
  return 0;
1528
1665
  let k = i;
1529
1666
  while (k < n && text.charCodeAt(k) === 92)
@@ -1537,7 +1674,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
1537
1674
  const b = skipHs(i);
1538
1675
  if (b >= n || openAt(b))
1539
1676
  return true;
1540
- if (opaqueUnitEnd(scan, b) >= 0)
1677
+ if (tokenEnd(scan, b) >= 0 || lineEndHidden(b))
1541
1678
  return false;
1542
1679
  const c = text.charCodeAt(b);
1543
1680
  return c === 10 || c === 13 || c === 34 || escapedNewlineLen(b) > 0;
@@ -1547,7 +1684,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
1547
1684
  if (q0 < 0)
1548
1685
  return q0;
1549
1686
  let q = q0;
1550
- const mk = opaqueUnitEnd(scan, q) < 0 ? text.charCodeAt(q) : -1;
1687
+ const mk = tokenEnd(scan, q) < 0 ? text.charCodeAt(q) : -1;
1551
1688
  if (mk === 43 || mk === 45)
1552
1689
  q = skipHs(q + 1);
1553
1690
  let r = q;
@@ -1571,16 +1708,17 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
1571
1708
  break;
1572
1709
  let p = at + KEY.length;
1573
1710
  const d0 = p;
1574
- while (p < n && opaqueUnitEnd(scan, p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
1711
+ while (p < n && tokenEnd(scan, p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
1575
1712
  p += 1;
1576
1713
  if (p === d0 && p < n) {
1577
1714
  search = at + KEY.length;
1578
1715
  continue;
1579
1716
  }
1580
- if (anchorHeld(scan, at, p)) {
1717
+ if (lineHeld(scan, at, p, readingKey(familyCode("putty-ppk"), at))) {
1581
1718
  search = at + KEY.length;
1582
1719
  continue;
1583
1720
  }
1721
+ scan.self = readingKey(familyCode("putty-ppk"), at);
1584
1722
  let end = -1;
1585
1723
  let open = false;
1586
1724
  for (let from = p;;) {
@@ -1598,7 +1736,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
1598
1736
  search = p;
1599
1737
  continue;
1600
1738
  }
1601
- span(at, end, open);
1739
+ span(at, end, at, NOT_CLOSED, open);
1602
1740
  search = end;
1603
1741
  }
1604
1742
  }
@@ -1735,8 +1873,8 @@ function tokenUnitEnd(text, p) {
1735
1873
  TOKEN_UNIT_Y.lastIndex = p;
1736
1874
  return TOKEN_UNIT_Y.test(text) ? TOKEN_UNIT_Y.lastIndex : -1;
1737
1875
  }
1738
- const NO_CREDENTIAL_SPANS = { starts: new Int32Array(0), ends: new Int32Array(0) };
1739
- function credentialSpans(text) {
1876
+ const NO_TOKEN_UNITS = { starts: new Int32Array(0), ends: new Int32Array(0) };
1877
+ function tokenUnits(text) {
1740
1878
  const iv = [];
1741
1879
  for (let i = text.indexOf("«"); i >= 0; i = text.indexOf("«", i + 1)) {
1742
1880
  const e = tokenUnitEnd(text, i);
@@ -1746,8 +1884,7 @@ function credentialSpans(text) {
1746
1884
  }
1747
1885
  }
1748
1886
  if (iv.length === 0)
1749
- return NO_CREDENTIAL_SPANS;
1750
- iv.sort((a, b) => a[0] - b[0]);
1887
+ return NO_TOKEN_UNITS;
1751
1888
  const starts = new Int32Array(iv.length);
1752
1889
  const ends = new Int32Array(iv.length);
1753
1890
  let k = 0;
@@ -1769,133 +1906,442 @@ function credentialSpans(text) {
1769
1906
  k += 1;
1770
1907
  return { starts: starts.subarray(0, k), ends: ends.subarray(0, k) };
1771
1908
  }
1909
+ function familyCode(f) {
1910
+ switch (f) {
1911
+ case "token": return 0;
1912
+ case "pem-armor": return 1;
1913
+ case "ssh2-armor": return 2;
1914
+ case "putty-ppk": return 3;
1915
+ case "url": return 4;
1916
+ case "json-field": return 5;
1917
+ case "anchor": return 6;
1918
+ case "prose": return 7;
1919
+ }
1920
+ }
1921
+ const FAMILY_SLOTS = 8;
1922
+ const TOKEN_FAMILY = familyCode("token");
1923
+ const readingKey = (family, anchor) => anchor * FAMILY_SLOTS + family;
1924
+ const NOT_CLOSED = -1;
1925
+ const NO_SELF = -1;
1926
+ const NO_ROWS = { starts: new Int32Array(0), ends: new Int32Array(0), family: new Uint8Array(0), anchor: new Int32Array(0), closeAt: new Int32Array(0) };
1927
+ function buildEvidenceRows(text) {
1928
+ const s = [];
1929
+ const e = [];
1930
+ for (let i = text.indexOf("«"); i >= 0; i = text.indexOf("«", i + 1)) {
1931
+ const end = tokenUnitEnd(text, i);
1932
+ if (end > 0) {
1933
+ s.push(i);
1934
+ e.push(end);
1935
+ i = end - 1;
1936
+ }
1937
+ }
1938
+ if (s.length === 0)
1939
+ return NO_ROWS;
1940
+ const n = s.length;
1941
+ const family = new Uint8Array(n).fill(TOKEN_FAMILY);
1942
+ const ends = Int32Array.from(e);
1943
+ return { starts: Int32Array.from(s), ends, family, anchor: Int32Array.from(s), closeAt: ends.map((x) => x - 1) };
1944
+ }
1945
+ const NO_DIRTY = { starts: new Int32Array(0), ends: new Int32Array(0) };
1772
1946
  let forensicRescanMode = "incremental";
1773
1947
  export function setForensicRescanModeForTest(mode) {
1774
1948
  forensicRescanMode = mode;
1775
1949
  }
1776
- function foldCredentialSpans(prev, spans, from, admit) {
1950
+ const FORENSIC_LEVEL_MAX = 32;
1951
+ function compareRow(s1, f1, a1, c1, s2, f2, a2, c2) {
1952
+ return s1 - s2 || f1 - f2 || a1 - a2 || (c1 === c2 ? 0 : c1 ? 1 : -1);
1953
+ }
1954
+ function foldEvidenceRows(prev, spans, from, familyOf) {
1777
1955
  const cap = spans.length - from;
1778
1956
  if (cap === 0)
1779
- return { next: prev, dirty: NO_CREDENTIAL_SPANS };
1780
- const Sa = new Int32Array(cap);
1781
- const Ea = new Int32Array(cap);
1782
- let m = 0;
1783
- for (let i = from; i < spans.length; i++) {
1784
- const sp = spans[i];
1785
- if (admit !== undefined && !admit(sp))
1786
- continue;
1787
- const k = firstCredentialSpanFrom(prev, sp.start + 1) - 1;
1788
- if (k >= 0 && prev.ends[k] >= sp.end)
1789
- continue;
1790
- Sa[m] = sp.start;
1791
- Ea[m] = sp.end;
1792
- m += 1;
1793
- }
1794
- if (m === 0)
1795
- return { next: prev, dirty: NO_CREDENTIAL_SPANS };
1796
- const S = Sa.subarray(0, m).sort();
1797
- const E = Ea.subarray(0, m).sort();
1798
- const cs = new Int32Array(m);
1799
- const ce = new Int32Array(m);
1800
- let C = 0;
1801
- for (let i = 0, j = 0, depth = 0, open = 0; i < m || j < m;) {
1802
- if (i < m && (j >= m || S[i] <= E[j])) {
1803
- if (depth === 0)
1804
- open = S[i];
1805
- depth += 1;
1806
- i += 1;
1957
+ return { next: prev, dirty: NO_DIRTY };
1958
+ const idx = Array.from({ length: cap }, (_, i) => from + i);
1959
+ const fam = new Uint8Array(spans.length);
1960
+ for (let i = from; i < spans.length; i++)
1961
+ fam[i] = familyOf(spans[i].order);
1962
+ idx.sort((x, y) => {
1963
+ const a = spans[x];
1964
+ const b = spans[y];
1965
+ return compareRow(a.start, fam[x], a.anchor, a.closeAt >= 0, b.start, fam[y], b.anchor, b.closeAt >= 0) || b.end - a.end || b.closeAt - a.closeAt;
1966
+ });
1967
+ const P = prev.starts.length;
1968
+ const total = P + cap;
1969
+ const oS = new Int32Array(total);
1970
+ const oE = new Int32Array(total);
1971
+ const oF = new Uint8Array(total);
1972
+ const oA = new Int32Array(total);
1973
+ const oC = new Int32Array(total);
1974
+ const dS = [];
1975
+ const dE = [];
1976
+ let on = 0;
1977
+ let pi = 0;
1978
+ let ni = 0;
1979
+ const push = (s, e, f, a, c) => {
1980
+ oS[on] = s;
1981
+ oE[on] = e;
1982
+ oF[on] = f;
1983
+ oA[on] = a;
1984
+ oC[on] = c;
1985
+ on += 1;
1986
+ };
1987
+ while (pi < P || ni < cap) {
1988
+ let take;
1989
+ let sp;
1990
+ let nf = 0;
1991
+ if (ni < cap) {
1992
+ sp = spans[idx[ni]];
1993
+ nf = fam[idx[ni]];
1807
1994
  }
1995
+ if (pi >= P)
1996
+ take = "new";
1997
+ else if (sp === undefined)
1998
+ take = "prev";
1808
1999
  else {
1809
- depth -= 1;
1810
- if (depth === 0) {
1811
- cs[C] = open;
1812
- ce[C] = E[j];
1813
- C += 1;
2000
+ const c = compareRow(prev.starts[pi], prev.family[pi], prev.anchor[pi], prev.closeAt[pi] >= 0, sp.start, nf, sp.anchor, sp.closeAt >= 0);
2001
+ take = c < 0 ? "prev" : c > 0 ? "new" : "both";
2002
+ }
2003
+ if (take === "prev") {
2004
+ push(prev.starts[pi], prev.ends[pi], prev.family[pi], prev.anchor[pi], prev.closeAt[pi]);
2005
+ pi += 1;
2006
+ continue;
2007
+ }
2008
+ const s = sp.start;
2009
+ let e = sp.end;
2010
+ let c = sp.closeAt;
2011
+ const a = sp.anchor;
2012
+ let nj = ni + 1;
2013
+ while (nj < cap) {
2014
+ const t = spans[idx[nj]];
2015
+ if (compareRow(t.start, fam[idx[nj]], t.anchor, t.closeAt >= 0, s, nf, a, c >= 0) !== 0)
2016
+ break;
2017
+ nj += 1;
2018
+ }
2019
+ ni = nj;
2020
+ if (take === "both") {
2021
+ const pe = prev.ends[pi];
2022
+ const pc = prev.closeAt[pi];
2023
+ pi += 1;
2024
+ if (pe > e || (pe === e && pc >= c)) {
2025
+ push(s, pe, nf, a, pc);
2026
+ continue;
1814
2027
  }
1815
- j += 1;
2028
+ dS.push(s);
2029
+ dE.push(Math.max(e, pe));
1816
2030
  }
2031
+ else {
2032
+ dS.push(s);
2033
+ dE.push(e);
2034
+ }
2035
+ push(s, e, nf, a, c);
1817
2036
  }
1818
- const P = prev.starts.length;
1819
- const oS = new Int32Array(P + C);
1820
- const oE = new Int32Array(P + C);
1821
- const dS = new Int32Array(C + P);
1822
- const dE = new Int32Array(C + P);
1823
- let on = 0;
1824
- let dn = 0;
1825
- let curS = 0;
1826
- let curE = 0;
1827
- let sources = 0;
1828
- let onlyPrev = false;
1829
- const flush = () => {
1830
- if (sources === 0)
1831
- return;
1832
- oS[on] = curS;
1833
- oE[on] = curE;
1834
- on += 1;
1835
- if (!(sources === 1 && onlyPrev)) {
1836
- dS[dn] = curS;
1837
- dE[dn] = curE;
1838
- dn += 1;
2037
+ if (dS.length === 0)
2038
+ return { next: prev, dirty: NO_DIRTY };
2039
+ const next = { starts: oS.subarray(0, on), ends: oE.subarray(0, on), family: oF.subarray(0, on), anchor: oA.subarray(0, on), closeAt: oC.subarray(0, on) };
2040
+ const ixP = rowIndexOf(prev);
2041
+ const ixN = rowIndexOf(next);
2042
+ for (let a = 0, b = 0; a < P && b < on;) {
2043
+ const c = compareRow(prev.starts[a], prev.family[a], prev.anchor[a], prev.closeAt[a] >= 0, next.starts[b], next.family[b], next.anchor[b], next.closeAt[b] >= 0);
2044
+ if (c > 0) {
2045
+ b += 1;
2046
+ continue;
1839
2047
  }
1840
- };
1841
- for (let pi = 0, ci = 0; pi < P || ci < C;) {
1842
- const takePrev = ci >= C || (pi < P && prev.starts[pi] <= cs[ci]);
1843
- const s = takePrev ? prev.starts[pi] : cs[ci];
1844
- const e = takePrev ? prev.ends[pi] : ce[ci];
1845
- if (takePrev)
1846
- pi += 1;
1847
- else
1848
- ci += 1;
1849
- if (sources > 0 && s <= curE) {
1850
- if (e > curE)
1851
- curE = e;
1852
- sources += 1;
2048
+ if (c === 0 && ixP.closedFor[a] !== ixN.closedFor[b]) {
2049
+ dS.push(next.starts[b]);
2050
+ dE.push(Math.max(next.closeAt[b], prev.closeAt[a]));
1853
2051
  }
2052
+ a += 1;
2053
+ if (c === 0)
2054
+ b += 1;
2055
+ }
2056
+ return { next, dirty: mergeClosedIntervals(dS, dE) };
2057
+ }
2058
+ function mergeClosedIntervals(s, e) {
2059
+ const order = Array.from(s.keys()).sort((x, y) => s[x] - s[y]);
2060
+ const os = [];
2061
+ const oe = [];
2062
+ for (const k of order) {
2063
+ const a = s[k];
2064
+ const b = e[k];
2065
+ const last = oe.length - 1;
2066
+ if (last >= 0 && a <= oe[last])
2067
+ oe[last] = Math.max(oe[last], b);
1854
2068
  else {
1855
- flush();
1856
- curS = s;
1857
- curE = e;
1858
- sources = 1;
1859
- onlyPrev = takePrev;
2069
+ os.push(a);
2070
+ oe.push(b);
1860
2071
  }
1861
2072
  }
1862
- flush();
1863
- return { next: { starts: oS.subarray(0, on), ends: oE.subarray(0, on) }, dirty: { starts: dS.subarray(0, dn), ends: dE.subarray(0, dn) } };
2073
+ return { starts: Int32Array.from(os), ends: Int32Array.from(oe) };
2074
+ }
2075
+ export function evidenceRowsFoldForTest(prev, add) {
2076
+ const sorted = [...prev].sort((x, y) => compareRow(x[0], x[2], x[3], x[4] >= 0, y[0], y[2], y[3], y[4] >= 0));
2077
+ const table = {
2078
+ starts: Int32Array.from(sorted, (x) => x[0]),
2079
+ ends: Int32Array.from(sorted, (x) => x[1]),
2080
+ family: Uint8Array.from(sorted, (x) => x[2]),
2081
+ anchor: Int32Array.from(sorted, (x) => x[3]),
2082
+ closeAt: Int32Array.from(sorted, (x) => x[4]),
2083
+ };
2084
+ const fams = add.map((x) => x[2]);
2085
+ const spans = add.map(([start, end, , anchor, closeAt], order) => ({ start, end, repl: "", rank: 0, order, anchor, closeAt }));
2086
+ const { next, dirty } = foldEvidenceRows(table, spans, 0, (order) => fams[order]);
2087
+ return {
2088
+ next: Array.from(next.starts, (s, i) => [s, next.ends[i], next.family[i], next.anchor[i], next.closeAt[i]]),
2089
+ dirty: Array.from(dirty.starts, (s, i) => [s, dirty.ends[i]]),
2090
+ };
1864
2091
  }
1865
- export function credentialSpansFoldForTest(prev, add) {
1866
- const table = (iv) => ({ starts: Int32Array.from(iv, (x) => x[0]), ends: Int32Array.from(iv, (x) => x[1]) });
1867
- const back = (t) => Array.from(t.starts, (s, i) => [s, t.ends[i]]);
1868
- const spans = add.map(([start, end]) => ({ start, end, repl: "", rank: 0, order: 0 }));
1869
- const { next, dirty } = foldCredentialSpans(table(prev), spans, 0);
1870
- return { next: back(next), dirty: back(dirty) };
2092
+ function lowerBound(a, x) {
2093
+ let lo = 0;
2094
+ let hi = a.length;
2095
+ while (lo < hi) {
2096
+ const mid = (lo + hi) >>> 1;
2097
+ if (a[mid] < x)
2098
+ lo = mid + 1;
2099
+ else
2100
+ hi = mid;
2101
+ }
2102
+ return lo;
1871
2103
  }
1872
- function firstCredentialSpanFrom(cs, from) {
2104
+ function upperBound(a, x) {
1873
2105
  let lo = 0;
1874
- let hi = cs.starts.length;
2106
+ let hi = a.length;
1875
2107
  while (lo < hi) {
1876
2108
  const mid = (lo + hi) >>> 1;
1877
- if (cs.starts[mid] < from)
2109
+ if (a[mid] <= x)
1878
2110
  lo = mid + 1;
1879
2111
  else
1880
2112
  hi = mid;
1881
2113
  }
1882
2114
  return lo;
1883
2115
  }
1884
- function credentialSpansCover(scan, from, to) {
1885
- noteEvidenceRead(scan, from);
1886
- noteEvidenceRead(scan, to);
1887
- const cs = scan.table;
1888
- let k = firstCredentialSpanFrom(cs, from + 1) - 1;
1889
- if (k < 0)
1890
- k = 0;
1891
- return k < cs.starts.length && cs.starts[k] <= from && cs.ends[k] >= to;
2116
+ const CLOSED_FOR_ALL = -1;
2117
+ const CLOSED_FOR_NONE = -2;
2118
+ const ROW_INDEX = new WeakMap();
2119
+ function rowIndexOf(t) {
2120
+ const hit = ROW_INDEX.get(t);
2121
+ if (hit !== undefined)
2122
+ return hit;
2123
+ const m = t.starts.length;
2124
+ const e1 = new Int32Array(m);
2125
+ const ek1 = new Float64Array(m);
2126
+ const e2 = new Int32Array(m);
2127
+ const ek2 = new Float64Array(m);
2128
+ const e3 = new Int32Array(m);
2129
+ const c1 = new Int32Array(m);
2130
+ const ck1 = new Float64Array(m);
2131
+ const c2 = new Int32Array(m);
2132
+ let v1 = -1, k1 = -1, v2 = -1, k2 = -1, v3 = -1, w1 = -1, j1 = -1, w2 = -1;
2133
+ const dsL = [];
2134
+ const dK1L = [];
2135
+ const dE1L = [];
2136
+ const dE2L = [];
2137
+ for (let r = 0; r < m; r++) {
2138
+ const key = readingKey(t.family[r], t.anchor[r]);
2139
+ const e = t.ends[r];
2140
+ if (key === k1)
2141
+ v1 = Math.max(v1, e);
2142
+ else if (key === k2) {
2143
+ if (e > v2)
2144
+ v2 = e;
2145
+ if (v2 > v1) {
2146
+ const tv = v1;
2147
+ v1 = v2;
2148
+ v2 = tv;
2149
+ const tk = k1;
2150
+ k1 = k2;
2151
+ k2 = tk;
2152
+ }
2153
+ }
2154
+ else if (e > v1) {
2155
+ v3 = v2;
2156
+ v2 = v1;
2157
+ k2 = k1;
2158
+ v1 = e;
2159
+ k1 = key;
2160
+ }
2161
+ else if (e > v2) {
2162
+ v3 = v2;
2163
+ v2 = e;
2164
+ k2 = key;
2165
+ }
2166
+ else if (e > v3)
2167
+ v3 = e;
2168
+ e1[r] = v1;
2169
+ ek1[r] = k1;
2170
+ e2[r] = v2;
2171
+ ek2[r] = k2;
2172
+ e3[r] = v3;
2173
+ const s = t.starts[r];
2174
+ const d = dsL.length - 1;
2175
+ if (d < 0 || dsL[d] !== s) {
2176
+ dsL.push(s);
2177
+ dK1L.push(key);
2178
+ dE1L.push(e);
2179
+ dE2L.push(-1);
2180
+ }
2181
+ else if (key === dK1L[d])
2182
+ dE1L[d] = Math.max(dE1L[d], e);
2183
+ else if (e > dE1L[d]) {
2184
+ dE2L[d] = dE1L[d];
2185
+ dE1L[d] = e;
2186
+ dK1L[d] = key;
2187
+ }
2188
+ else if (e > dE2L[d])
2189
+ dE2L[d] = e;
2190
+ }
2191
+ const closedFor = new Float64Array(m);
2192
+ const closeForL = new Map();
2193
+ for (let r = 0; r < m; r++) {
2194
+ const c = t.closeAt[r];
2195
+ let f = CLOSED_FOR_NONE;
2196
+ if (c >= 0) {
2197
+ const key = readingKey(t.family[r], t.anchor[r]);
2198
+ const q = upperBound(t.starts, c) - 1;
2199
+ let a, ka, b;
2200
+ if (ek1[q] === key) {
2201
+ a = e2[q];
2202
+ ka = ek2[q];
2203
+ b = e3[q];
2204
+ }
2205
+ else if (ek2[q] === key) {
2206
+ a = e1[q];
2207
+ ka = ek1[q];
2208
+ b = e3[q];
2209
+ }
2210
+ else {
2211
+ a = e1[q];
2212
+ ka = ek1[q];
2213
+ b = e2[q];
2214
+ }
2215
+ f = a <= c ? CLOSED_FOR_ALL : b <= c ? ka : CLOSED_FOR_NONE;
2216
+ if (f === CLOSED_FOR_ALL) {
2217
+ if (key === j1)
2218
+ w1 = Math.max(w1, c);
2219
+ else if (c > w1) {
2220
+ w2 = w1;
2221
+ w1 = c;
2222
+ j1 = key;
2223
+ }
2224
+ else if (c > w2)
2225
+ w2 = c;
2226
+ }
2227
+ else if (f !== CLOSED_FOR_NONE) {
2228
+ let bk = closeForL.get(f);
2229
+ if (bk === undefined)
2230
+ closeForL.set(f, (bk = { s: [], c: [], f: [] }));
2231
+ const last = bk.c.length - 1;
2232
+ const s = t.starts[r];
2233
+ bk.s.push(s);
2234
+ if (last >= 0 && bk.c[last] >= c) {
2235
+ bk.c.push(bk.c[last]);
2236
+ bk.f.push(bk.f[last]);
2237
+ }
2238
+ else {
2239
+ bk.c.push(c);
2240
+ bk.f.push(s);
2241
+ }
2242
+ }
2243
+ }
2244
+ closedFor[r] = f;
2245
+ c1[r] = w1;
2246
+ ck1[r] = j1;
2247
+ c2[r] = w2;
2248
+ }
2249
+ const closeFor = new Map();
2250
+ for (const [k, b] of closeForL)
2251
+ closeFor.set(k, { starts: Int32Array.from(b.s), maxClose: Int32Array.from(b.c), maxFrom: Int32Array.from(b.f) });
2252
+ const D = dsL.length;
2253
+ const ds = Int32Array.from(dsL);
2254
+ const reach1 = new Int32Array(D);
2255
+ const reach2 = new Int32Array(D);
2256
+ const stS = [];
2257
+ const stR = [];
2258
+ for (let d = D - 1; d >= 0; d--) {
2259
+ let r1 = dE1L[d];
2260
+ let r2 = dE2L[d];
2261
+ for (let q = stS.length - 1; q >= 0 && r2 >= 0 && stS[q] <= r2; q--)
2262
+ r2 = Math.max(r2, stR[q]);
2263
+ while (stS.length > 0 && stS[stS.length - 1] <= r1) {
2264
+ r1 = Math.max(r1, stR[stR.length - 1]);
2265
+ stS.pop();
2266
+ stR.pop();
2267
+ }
2268
+ reach1[d] = r1;
2269
+ reach2[d] = r2;
2270
+ stS.push(ds[d]);
2271
+ stR.push(r1);
2272
+ }
2273
+ const sS = [];
2274
+ const sE = [];
2275
+ for (let r = 0; r < m; r++) {
2276
+ const s = t.starts[r];
2277
+ const e = t.ends[r];
2278
+ const last = sE.length - 1;
2279
+ if (last >= 0 && s <= sE[last])
2280
+ sE[last] = Math.max(sE[last], e);
2281
+ else {
2282
+ sS.push(s);
2283
+ sE.push(e);
2284
+ }
2285
+ }
2286
+ const saved = new Float64Array(sS.length + 1);
2287
+ for (let k = 0; k < sS.length; k++)
2288
+ saved[k + 1] = saved[k] + (sE[k] - sS[k] - 1);
2289
+ const keyStarts = new Map();
2290
+ for (let r = 0; r < m; r++) {
2291
+ const key = readingKey(t.family[r], t.anchor[r]);
2292
+ const at = keyStarts.get(key);
2293
+ if (at === undefined)
2294
+ keyStarts.set(key, [t.starts[r]]);
2295
+ else
2296
+ at.push(t.starts[r]);
2297
+ }
2298
+ const byEndRow = Int32Array.from({ length: m }, (_, r) => r);
2299
+ for (let lo = 0; lo < m;) {
2300
+ let hi = lo + 1;
2301
+ while (hi < m && t.starts[hi] === t.starts[lo])
2302
+ hi += 1;
2303
+ if (hi - lo > 1)
2304
+ byEndRow.subarray(lo, hi).sort((x, y) => t.ends[x] - t.ends[y]);
2305
+ lo = hi;
2306
+ }
2307
+ const byEnd = Int32Array.from(byEndRow, (r) => t.ends[r]);
2308
+ const chainSaved = new Float64Array(D + 1);
2309
+ for (let d = D - 1; d >= 0; d--)
2310
+ chainSaved[d] = Math.max(0, reach1[d] - ds[d] - 1) + chainSaved[upperBound(ds, reach1[d])];
2311
+ const ix = { e1, ek1, e2, closedFor, closeFor, byEndRow, byEnd, chainSaved, bounded: new Map(), c1, ck1, c2, ds, dKey1: Float64Array.from(dK1L), dEnd2: Int32Array.from(dE2L), reach1, reach2, segS: Int32Array.from(sS), segE: Int32Array.from(sE), saved, keyStarts };
2312
+ ROW_INDEX.set(t, ix);
2313
+ return ix;
2314
+ }
2315
+ function readableMaxEnd(t, ix, x, strict, self) {
2316
+ const r = (strict ? lowerBound(t.starts, x) : upperBound(t.starts, x)) - 1;
2317
+ if (r < 0)
2318
+ return -1;
2319
+ return ix.ek1[r] === self ? ix.e2[r] : ix.e1[r];
2320
+ }
2321
+ function readableMaxClose(t, ix, x, strict, self) {
2322
+ const r = (strict ? lowerBound(t.starts, x) : upperBound(t.starts, x)) - 1;
2323
+ if (r < 0)
2324
+ return -1;
2325
+ const all = ix.ck1[r] === self ? ix.c2[r] : ix.c1[r];
2326
+ const own = ix.closeFor.get(self);
2327
+ if (own === undefined)
2328
+ return all;
2329
+ const k = (strict ? lowerBound(own.starts, x) : upperBound(own.starts, x)) - 1;
2330
+ return k < 0 ? all : Math.max(all, own.maxClose[k]);
2331
+ }
2332
+ function reachAt(ix, d, self) {
2333
+ if (ix.dKey1[d] !== self)
2334
+ return ix.reach1[d];
2335
+ return ix.dEnd2[d] < 0 ? -1 : ix.reach2[d];
1892
2336
  }
1893
2337
  const FOOT_NONE = 0x7fffffff;
1894
- function buildForensicScan(table, armCount) {
2338
+ function buildForensicScan(tokens, rows, armCount) {
1895
2339
  return {
1896
- table,
1897
- others: undefined,
1898
- cursor: 0,
2340
+ rows,
2341
+ tokens,
2342
+ tokenCursor: 0,
2343
+ self: NO_SELF,
2344
+ rowCursor: 0,
1899
2345
  dirty: undefined,
1900
2346
  footLo: FOOT_NONE,
1901
2347
  footHi: -1,
@@ -1905,15 +2351,14 @@ function buildForensicScan(table, armCount) {
1905
2351
  headsWrite: { a: NO_URL_SEGMENTS, n: 0 },
1906
2352
  paramsRead: { a: NO_URL_SEGMENTS, n: 0 },
1907
2353
  paramsWrite: { a: NO_URL_SEGMENTS, n: 0 },
2354
+ opsRead: { a: NO_URL_SEGMENTS, n: 0, cut: OPS_UNCUT },
2355
+ opsWrite: { a: NO_URL_SEGMENTS, n: 0, cut: OPS_UNCUT },
1908
2356
  hdr: { search: 0, contentEnd: 0, next: 0, hitLimit: false, jsonEnd: false },
2357
+ delimPos: new Map(),
2358
+ delimIx: new Map(),
2359
+ delimIxRows: rows,
1909
2360
  };
1910
2361
  }
1911
- function useEvidenceView(scan, table, dirty, others) {
1912
- scan.table = table;
1913
- scan.others = others;
1914
- scan.cursor = 0;
1915
- scan.dirty = dirty;
1916
- }
1917
2362
  function noteEvidenceRead(scan, p) {
1918
2363
  if (p < scan.footLo)
1919
2364
  scan.footLo = p;
@@ -1935,55 +2380,416 @@ function footprintClean(dirty, lo, hi) {
1935
2380
  }
1936
2381
  return a >= ends.length || dirty.starts[a] > hi;
1937
2382
  }
1938
- function inOpaqueUnit(scan, p) {
1939
- noteEvidenceRead(scan, p);
1940
- const t = scan.table;
2383
+ function tokenEnd(scan, p) {
2384
+ const t = scan.tokens;
1941
2385
  const st = t.starts;
1942
2386
  if (st.length === 0)
1943
- return false;
1944
- const k = firstCredentialSpanFrom(t, p) - 1;
1945
- return k >= 0 && st[k] < p && t.ends[k] > p;
2387
+ return -1;
2388
+ let k = scan.tokenCursor;
2389
+ while (k < st.length && st[k] < p)
2390
+ k += 1;
2391
+ while (k > 0 && st[k - 1] >= p)
2392
+ k -= 1;
2393
+ scan.tokenCursor = k;
2394
+ return k < st.length && st[k] === p ? t.ends[k] : -1;
1946
2395
  }
1947
- function othersOf(scan) {
1948
- const t = scan.others;
1949
- if (t === undefined)
1950
- throw new Error("redact: a structural-anchor question was asked by a forensic arm that declares no credential family — it has no other-families evidence to read");
1951
- return t;
2396
+ function crossEndAt(scan, p) {
2397
+ const t = scan.rows;
2398
+ if (t.starts.length === 0)
2399
+ return -1;
2400
+ const ix = rowIndexOf(t);
2401
+ const ds = ix.ds;
2402
+ let k = scan.rowCursor;
2403
+ while (k < ds.length && ds[k] < p)
2404
+ k += 1;
2405
+ while (k > 0 && ds[k - 1] >= p)
2406
+ k -= 1;
2407
+ scan.rowCursor = k;
2408
+ if (k >= ds.length || ds[k] !== p)
2409
+ return -1;
2410
+ const r = reachAt(ix, k, scan.self);
2411
+ return r > p ? r : -1;
1952
2412
  }
1953
- function anchorHeld(scan, p, lineEnd) {
1954
- const t = othersOf(scan);
2413
+ function crossEnd(scan, p) {
2414
+ noteEvidenceRead(scan, p);
2415
+ const r = crossEndAt(scan, p);
2416
+ if (r >= 0)
2417
+ noteEvidenceRead(scan, r);
2418
+ return r;
2419
+ }
2420
+ const DELIM_FREE = 0;
2421
+ const DELIM_AMBIGUOUS = 1;
2422
+ const DELIM_HIDDEN = 2;
2423
+ function delimClass(scan, c, self = scan.self) {
2424
+ noteEvidenceRead(scan, c);
2425
+ const t = scan.rows;
2426
+ if (t.starts.length === 0)
2427
+ return DELIM_FREE;
2428
+ const ix = rowIndexOf(t);
2429
+ if (readableMaxClose(t, ix, c, false, self) > c)
2430
+ return DELIM_HIDDEN;
2431
+ return readableMaxEnd(t, ix, c, false, self) > c ? DELIM_AMBIGUOUS : DELIM_FREE;
2432
+ }
2433
+ function lineHeld(scan, p, lineEnd, self) {
1955
2434
  noteEvidenceRead(scan, p);
1956
2435
  noteEvidenceRead(scan, lineEnd - 1);
1957
- const st = t.starts;
1958
- if (st.length === 0)
2436
+ const t = scan.rows;
2437
+ if (t.starts.length === 0)
1959
2438
  return false;
1960
- const k = firstCredentialSpanFrom(t, p) - 1;
1961
- return k >= 0 && st[k] < p && t.ends[k] >= lineEnd;
2439
+ return readableMaxClose(t, rowIndexOf(t), p, true, self) >= lineEnd;
2440
+ }
2441
+ function readableReach(t, ix, p, strict, self) {
2442
+ let q = readableMaxEnd(t, ix, p, strict, self);
2443
+ if (q <= p)
2444
+ return p;
2445
+ const ds = ix.ds;
2446
+ for (let k = strict ? lowerBound(ds, p) : upperBound(ds, p); k < ds.length && ds[k] <= q;) {
2447
+ const r = reachAt(ix, k, self);
2448
+ if (r > q)
2449
+ q = r;
2450
+ k = r > ds[k] ? Math.max(k + 1, upperBound(ds, r)) : k + 1;
2451
+ }
2452
+ return q;
1962
2453
  }
1963
- function opaqueUnitEnd(scan, p) {
2454
+ function readableCover(scan, p, q) {
1964
2455
  noteEvidenceRead(scan, p);
1965
- return opaqueUnitEndAt(scan, p);
2456
+ noteEvidenceRead(scan, q);
2457
+ const t = scan.rows;
2458
+ if (t.starts.length === 0)
2459
+ return false;
2460
+ return readableReach(t, rowIndexOf(t), p, false, scan.self) >= q;
2461
+ }
2462
+ const ROW_ENDS = new WeakMap();
2463
+ function rowEndsOf(t) {
2464
+ const hit = ROW_ENDS.get(t);
2465
+ if (hit !== undefined)
2466
+ return hit;
2467
+ const byEnd = Int32Array.from({ length: t.starts.length }, (_, r) => r).sort((x, y) => t.ends[x] - t.ends[y]);
2468
+ const v = { ends: Int32Array.from(byEnd, (r) => t.ends[r]), keys: Float64Array.from(byEnd, (r) => readingKey(t.family[r], t.anchor[r])) };
2469
+ ROW_ENDS.set(t, v);
2470
+ return v;
1966
2471
  }
1967
- function opaqueUnitEndAt(scan, p) {
1968
- const t = scan.table;
1969
- const st = t.starts;
1970
- if (st.length === 0)
2472
+ function nameBoundAt(scan, lo, hi) {
2473
+ noteEvidenceRead(scan, lo);
2474
+ noteEvidenceRead(scan, hi);
2475
+ const t = scan.rows;
2476
+ if (t.starts.length === 0)
1971
2477
  return -1;
1972
- let k = scan.cursor;
1973
- while (k < st.length && st[k] < p)
1974
- k += 1;
1975
- while (k > 0 && st[k - 1] >= p)
1976
- k -= 1;
1977
- scan.cursor = k;
1978
- return k < st.length && st[k] === p ? t.ends[k] : -1;
2478
+ const { ends, keys } = rowEndsOf(t);
2479
+ for (let k = upperBound(ends, hi) - 1; k >= 0 && ends[k] > lo; k--)
2480
+ if (keys[k] !== scan.self)
2481
+ return ends[k];
2482
+ return -1;
2483
+ }
2484
+ function measureWithin(scan, from, to, max) {
2485
+ if (to - from <= max)
2486
+ return true;
2487
+ noteEvidenceRead(scan, from);
2488
+ noteEvidenceRead(scan, to);
2489
+ const t = scan.rows;
2490
+ if (t.starts.length === 0)
2491
+ return false;
2492
+ const ix = rowIndexOf(t);
2493
+ const self = scan.self;
2494
+ const ds = ix.ds;
2495
+ const g1 = upperBound(ix.segE, to);
2496
+ let saved = 0;
2497
+ for (let k = lowerBound(ds, from); k < ds.length && ds[k] < to;) {
2498
+ const s0 = ds[k];
2499
+ if (s0 - from - saved > max)
2500
+ return false;
2501
+ const g = upperBound(ix.segS, s0) - 1;
2502
+ if (g < g1) {
2503
+ if (ix.segS[g] >= from) {
2504
+ const gEnd = firstOwnSegment(ix, self, g, g1);
2505
+ if (gEnd > g) {
2506
+ saved += ix.saved[gEnd] - ix.saved[g];
2507
+ k = upperBound(ds, ix.segE[gEnd - 1]);
2508
+ continue;
2509
+ }
2510
+ }
2511
+ else if (!ownStartWithin(ix, self, s0, ix.segE[g])) {
2512
+ const kn = upperBound(ds, ix.segE[g]);
2513
+ saved += ix.chainSaved[k] - ix.chainSaved[kn];
2514
+ k = kn;
2515
+ continue;
2516
+ }
2517
+ }
2518
+ saved += boundedSavedFrom(t, ix, k, to, ownStartWithin(ix, self, s0, to) ? self : NO_SELF);
2519
+ break;
2520
+ }
2521
+ return to - from - saved <= max;
2522
+ }
2523
+ function ownStartWithin(ix, self, a, b) {
2524
+ const own = ix.keyStarts.get(self);
2525
+ if (own === undefined)
2526
+ return false;
2527
+ for (const x of own)
2528
+ if (x >= a && x <= b)
2529
+ return true;
2530
+ return false;
2531
+ }
2532
+ function boundedSavedFrom(t, ix, k, to, self) {
2533
+ let byReader = ix.bounded.get(to);
2534
+ if (byReader === undefined)
2535
+ ix.bounded.set(to, (byReader = new Map()));
2536
+ let ch = byReader.get(self);
2537
+ if (ch === undefined) {
2538
+ const hi = lowerBound(ix.ds, to);
2539
+ byReader.set(self, (ch = { lo: hi, hi, saved: [], stS: [], stR: [], stV: [] }));
2540
+ }
2541
+ const { stS, stR, stV } = ch;
2542
+ for (let d = ch.lo - 1; d >= k; d--) {
2543
+ const s = ix.ds[d];
2544
+ const x = maxEndWithin(t, ix, s, to, self);
2545
+ let v = stV.length > 0 ? stV[stV.length - 1] : 0;
2546
+ if (x >= 0) {
2547
+ let r = x;
2548
+ while (stS.length > 0 && stS[stS.length - 1] <= r) {
2549
+ r = Math.max(r, stR[stR.length - 1]);
2550
+ stS.pop();
2551
+ stR.pop();
2552
+ stV.pop();
2553
+ }
2554
+ v = Math.max(0, r - s - 1) + (stV.length > 0 ? stV[stV.length - 1] : 0);
2555
+ stS.push(s);
2556
+ stR.push(r);
2557
+ stV.push(v);
2558
+ }
2559
+ ch.saved.push(v);
2560
+ ch.lo = d;
2561
+ }
2562
+ return k >= ch.hi ? 0 : ch.saved[ch.hi - 1 - k];
2563
+ }
2564
+ function uncoveredWithin(scan, from, to) {
2565
+ noteEvidenceRead(scan, from);
2566
+ noteEvidenceRead(scan, to);
2567
+ const t = scan.rows;
2568
+ if (t.starts.length === 0 || to <= from)
2569
+ return Math.max(0, to - from);
2570
+ const ix = rowIndexOf(t);
2571
+ const lo = upperBound(ix.segE, from);
2572
+ const hi = lowerBound(ix.segS, to);
2573
+ if (lo >= hi)
2574
+ return to - from;
2575
+ let covered = ix.saved[hi] - ix.saved[lo] + (hi - lo);
2576
+ covered -= Math.max(0, from - ix.segS[lo]);
2577
+ covered -= Math.max(0, ix.segE[hi - 1] - to);
2578
+ return to - from - covered;
2579
+ }
2580
+ function firstOwnSegment(ix, self, g, g1) {
2581
+ const own = ix.keyStarts.get(self);
2582
+ if (own === undefined)
2583
+ return g1;
2584
+ let best = g1;
2585
+ for (const a of own) {
2586
+ const ga = upperBound(ix.segS, a) - 1;
2587
+ if (ga >= g && ga < best)
2588
+ best = ga;
2589
+ }
2590
+ return best;
2591
+ }
2592
+ function maxEndWithin(t, ix, s, to, self) {
2593
+ const lo = lowerBound(t.starts, s);
2594
+ let a = lo;
2595
+ let b = upperBound(t.starts, s);
2596
+ while (a < b) {
2597
+ const mid = (a + b) >>> 1;
2598
+ if (ix.byEnd[mid] <= to)
2599
+ a = mid + 1;
2600
+ else
2601
+ b = mid;
2602
+ }
2603
+ for (let j = a - 1; j >= lo; j--) {
2604
+ rowVisits += 1;
2605
+ const r = ix.byEndRow[j];
2606
+ if (readingKey(t.family[r], t.anchor[r]) !== self)
2607
+ return ix.byEnd[j];
2608
+ }
2609
+ return -1;
2610
+ }
2611
+ export function unitCountWithinForTest(table, from, to, max, self) {
2612
+ return unitCountsWithinForTest(table, [[from, to, max, self]])[0];
2613
+ }
2614
+ export function unitCountsWithinForTest(table, queries) {
2615
+ const keyOf = (x) => (x.length > 2 ? x[2] : x[0]);
2616
+ const sorted = [...table].sort((a, b) => a[0] - b[0] || keyOf(a) - keyOf(b));
2617
+ const rows = {
2618
+ starts: Int32Array.from(sorted, (x) => x[0]),
2619
+ ends: Int32Array.from(sorted, (x) => x[1]),
2620
+ family: new Uint8Array(sorted.length).fill(familyCode("prose")),
2621
+ anchor: Int32Array.from(sorted, keyOf),
2622
+ closeAt: new Int32Array(sorted.length).fill(NOT_CLOSED),
2623
+ };
2624
+ const scan = buildForensicScan(NO_TOKEN_UNITS, rows, 1);
2625
+ return queries.map(([from, to, max, self]) => {
2626
+ scan.self = self === undefined ? NO_SELF : readingKey(familyCode("prose"), self);
2627
+ return measureWithin(scan, from, to, max);
2628
+ });
2629
+ }
2630
+ function delimPositions(scan, text, kind) {
2631
+ const hit = scan.delimPos.get(kind);
2632
+ if (hit !== undefined)
2633
+ return hit;
2634
+ const out = [];
2635
+ switch (kind) {
2636
+ case "ssh2-open":
2637
+ case "ssh2-close": {
2638
+ const needle = kind === "ssh2-open" ? SSH2_FENCE_OPEN : SSH2_FENCE_CLOSE;
2639
+ for (let p = text.indexOf(needle); p >= 0; p = text.indexOf(needle, p + 1))
2640
+ out.push(p);
2641
+ ssh2ScanBytes += text.length;
2642
+ break;
2643
+ }
2644
+ case "pem-end":
2645
+ for (let b = text.indexOf(PEM_END_OPEN); b >= 0; b = text.indexOf(PEM_END_OPEN, b + 1))
2646
+ if (pemFenceLineEnd(text, b + PEM_END_OPEN.length) >= 0)
2647
+ out.push(b);
2648
+ break;
2649
+ case "at":
2650
+ case "at-json":
2651
+ for (let p = text.indexOf("@"); p >= 0; p = text.indexOf("@", p + 1)) {
2652
+ if (kind === "at-json") {
2653
+ let b = 0;
2654
+ while (p - b - 1 >= 0 && text.charCodeAt(p - b - 1) === CH_BACKSLASH)
2655
+ b += 1;
2656
+ if (b % 2 === 1)
2657
+ continue;
2658
+ }
2659
+ out.push(p);
2660
+ }
2661
+ break;
2662
+ }
2663
+ const pos = Int32Array.from(out);
2664
+ scan.delimPos.set(kind, pos);
2665
+ return pos;
2666
+ }
2667
+ function delimIndexOf(scan, text, kind) {
2668
+ const t = scan.rows;
2669
+ if (scan.delimIxRows !== t) {
2670
+ scan.delimIx = new Map();
2671
+ scan.delimIxRows = t;
2672
+ }
2673
+ const byKind = scan.delimIx;
2674
+ const hit = byKind.get(kind);
2675
+ if (hit !== undefined)
2676
+ return hit;
2677
+ const pos = delimPositions(scan, text, kind);
2678
+ const n = pos.length;
2679
+ const nextNotHidden = new Int32Array(n + 1);
2680
+ const nextFree = new Int32Array(n + 1);
2681
+ const prevNotHidden = new Int32Array(n);
2682
+ const hid = new Map();
2683
+ const cov = new Map();
2684
+ const hiddenAll = new Uint8Array(n);
2685
+ const coveredAll = new Uint8Array(n);
2686
+ if (t.starts.length > 0) {
2687
+ const ix = rowIndexOf(t);
2688
+ for (let i = 0; i < n; i++) {
2689
+ const c = pos[i];
2690
+ const r = upperBound(t.starts, c) - 1;
2691
+ if (r < 0)
2692
+ continue;
2693
+ if (ix.c1[r] > c) {
2694
+ hiddenAll[i] = 1;
2695
+ if (ix.c2[r] <= c) {
2696
+ const k = ix.ck1[r];
2697
+ let b = hid.get(k);
2698
+ if (b === undefined)
2699
+ hid.set(k, (b = []));
2700
+ b.push(i);
2701
+ }
2702
+ }
2703
+ if (ix.e1[r] > c) {
2704
+ coveredAll[i] = 1;
2705
+ if (ix.e2[r] <= c) {
2706
+ const k = ix.ek1[r];
2707
+ let b = cov.get(k);
2708
+ if (b === undefined)
2709
+ cov.set(k, (b = []));
2710
+ b.push(i);
2711
+ }
2712
+ }
2713
+ }
2714
+ }
2715
+ nextNotHidden[n] = n;
2716
+ nextFree[n] = n;
2717
+ for (let i = n - 1; i >= 0; i--) {
2718
+ nextNotHidden[i] = hiddenAll[i] === 1 ? nextNotHidden[i + 1] : i;
2719
+ nextFree[i] = coveredAll[i] === 1 ? nextFree[i + 1] : i;
2720
+ }
2721
+ for (let i = 0, last = -1; i < n; i++) {
2722
+ if (hiddenAll[i] !== 1)
2723
+ last = i;
2724
+ prevNotHidden[i] = last;
2725
+ }
2726
+ const freeze = (m) => new Map([...m].map(([k, v]) => [k, Int32Array.from(v)]));
2727
+ const di = { pos, nextNotHidden, nextFree, prevNotHidden, onlyHiddenBy: freeze(hid), onlyCoveredBy: freeze(cov) };
2728
+ byKind.set(kind, di);
2729
+ return di;
2730
+ }
2731
+ function bucketFirstFrom(b, i, n) {
2732
+ if (b === undefined)
2733
+ return n;
2734
+ const k = lowerBound(b, i);
2735
+ return k < b.length ? b[k] : n;
2736
+ }
2737
+ function firstDelim(scan, text, kind, from, wantFree) {
2738
+ noteEvidenceRead(scan, from);
2739
+ const di = delimIndexOf(scan, text, kind);
2740
+ const n = di.pos.length;
2741
+ const bucket = (wantFree ? di.onlyCoveredBy : di.onlyHiddenBy).get(scan.self);
2742
+ const firstAt = (i) => (i >= n ? n : Math.min(wantFree ? di.nextFree[i] : di.nextNotHidden[i], bucketFirstFrom(bucket, i, n)));
2743
+ let j = firstAt(lowerBound(di.pos, from));
2744
+ const own = wantFree || scan.rows.starts.length === 0 ? undefined : rowIndexOf(scan.rows).closeFor.get(scan.self);
2745
+ while (own !== undefined && j < n) {
2746
+ const p = di.pos[j];
2747
+ const k = upperBound(own.starts, p) - 1;
2748
+ if (k < 0 || own.maxClose[k] <= p)
2749
+ break;
2750
+ j = firstAt(lowerBound(di.pos, own.maxClose[k]));
2751
+ }
2752
+ const at = j >= n ? -1 : di.pos[j];
2753
+ noteEvidenceRead(scan, at < 0 ? text.length : at);
2754
+ return at;
2755
+ }
2756
+ function lastNotHiddenDelim(scan, text, kind, a, b) {
2757
+ const di = delimIndexOf(scan, text, kind);
2758
+ const bucket = di.onlyHiddenBy.get(scan.self);
2759
+ const lastAt = (j) => {
2760
+ let best = di.prevNotHidden[j];
2761
+ if (bucket !== undefined) {
2762
+ const k = upperBound(bucket, j) - 1;
2763
+ if (k >= 0 && bucket[k] > best)
2764
+ best = bucket[k];
2765
+ }
2766
+ return best;
2767
+ };
2768
+ const j = lowerBound(di.pos, b) - 1;
2769
+ if (j < 0 || di.pos[j] < a)
2770
+ return -1;
2771
+ let best = lastAt(j);
2772
+ const own = scan.rows.starts.length === 0 ? undefined : rowIndexOf(scan.rows).closeFor.get(scan.self);
2773
+ while (own !== undefined && best >= 0 && di.pos[best] >= a) {
2774
+ const p = di.pos[best];
2775
+ const k = upperBound(own.starts, p) - 1;
2776
+ if (k < 0 || own.maxClose[k] <= p)
2777
+ break;
2778
+ const j2 = lowerBound(di.pos, own.maxFrom[k]) - 1;
2779
+ best = j2 < 0 ? -1 : lastAt(j2);
2780
+ }
2781
+ return best >= 0 && di.pos[best] >= a ? di.pos[best] : -1;
1979
2782
  }
1980
2783
  function scanUrlRun(scan, text, i, kind, json) {
1981
2784
  const n = text.length;
1982
2785
  noteEvidenceRead(scan, i);
1983
2786
  scan.lastAt = -1;
1984
2787
  run: while (i < n) {
1985
- const unit = opaqueUnitEndAt(scan, i);
2788
+ const unit = crossEndAt(scan, i);
1986
2789
  if (unit >= 0) {
2790
+ const at = lastNotHiddenDelim(scan, text, json ? "at-json" : "at", i, unit);
2791
+ if (at > scan.lastAt)
2792
+ scan.lastAt = at;
1987
2793
  i = unit;
1988
2794
  continue;
1989
2795
  }
@@ -2024,7 +2830,7 @@ function scanUrlRun(scan, text, i, kind, json) {
2024
2830
  }
2025
2831
  function matchUserinfoAt(scan, text, p, json, isHead) {
2026
2832
  const probeEnd = scanUrlRun(scan, text, p, "plain-user", json);
2027
- const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && credentialSpansCover(scan, p, probeEnd);
2833
+ const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && readableCover(scan, p, probeEnd);
2028
2834
  const runEnd = scanUrlRun(scan, text, p, keyId ? "token-pass" : "plain-pass", json);
2029
2835
  if (isHead && runEnd >= text.length && runEnd > p)
2030
2836
  return runEnd;
@@ -2036,7 +2842,9 @@ function matchUserinfoAt(scan, text, p, json, isHead) {
2036
2842
  const URL_SEG_STRIDE = 4;
2037
2843
  const URL_SEG_MAX_INTS = 3 * 1_048_576;
2038
2844
  const NO_URL_SEGMENTS = new Int32Array(0);
2039
- function growUrlSegments(list) {
2845
+ const ANCHOR_SEG_STRIDE = 4;
2846
+ const OPS_UNCUT = 0x7fffffff;
2847
+ function growSegments(list) {
2040
2848
  const len = list.a.length;
2041
2849
  if (len >= URL_SEG_MAX_INTS)
2042
2850
  return false;
@@ -2047,6 +2855,7 @@ function growUrlSegments(list) {
2047
2855
  }
2048
2856
  function collectUrlCredentialSpans(scan, text, json, span, isHead) {
2049
2857
  const dirty = scan.dirty;
2858
+ const urlFamily = familyCode("url");
2050
2859
  const hr = scan.headsRead;
2051
2860
  const hw = scan.headsWrite;
2052
2861
  const pr = scan.paramsRead;
@@ -2081,14 +2890,17 @@ function collectUrlCredentialSpans(scan, text, json, span, isHead) {
2081
2890
  else {
2082
2891
  scan.footLo = FOOT_NONE;
2083
2892
  scan.footHi = -1;
2893
+ scan.self = readingKey(urlFamily, colon);
2084
2894
  at = matchUserinfoAt(scan, text, start, json, isHead) ?? -1;
2085
2895
  lo = scan.footLo;
2086
2896
  hi = scan.footHi;
2087
2897
  urlSegmentRuns += 1;
2088
- if (at >= 0 && at !== (known ? ph[pc + 1] : -2))
2089
- span(start, at, isHead && at >= text.length);
2898
+ if (at >= 0 && at !== (known ? ph[pc + 1] : -2)) {
2899
+ const undecided = isHead && at >= text.length;
2900
+ span(start, at, colon, undecided ? NOT_CLOSED : at, undecided);
2901
+ }
2090
2902
  }
2091
- if (hw.n + URL_SEG_STRIDE <= hw.a.length || growUrlSegments(hw)) {
2903
+ if (hw.n + URL_SEG_STRIDE <= hw.a.length || growSegments(hw)) {
2092
2904
  const w = hw.a;
2093
2905
  const k = hw.n;
2094
2906
  w[k] = colon;
@@ -2128,14 +2940,15 @@ function collectUrlCredentialSpans(scan, text, json, span, isHead) {
2128
2940
  else {
2129
2941
  scan.footLo = FOOT_NONE;
2130
2942
  scan.footHi = -1;
2943
+ scan.self = readingKey(urlFamily, i);
2131
2944
  valueEnd = scanUrlRun(scan, text, valueStart, "param-value", json);
2132
2945
  lo = scan.footLo;
2133
2946
  hi = scan.footHi;
2134
2947
  urlSegmentRuns += 1;
2135
2948
  if (valueEnd > valueStart && valueEnd !== (known ? pp[qc + 1] : -2))
2136
- span(valueStart, valueEnd);
2949
+ span(valueStart, valueEnd, i, NOT_CLOSED);
2137
2950
  }
2138
- if (pw.n + URL_SEG_STRIDE <= pw.a.length || growUrlSegments(pw)) {
2951
+ if (pw.n + URL_SEG_STRIDE <= pw.a.length || growSegments(pw)) {
2139
2952
  const w = pw.a;
2140
2953
  const k = pw.n;
2141
2954
  w[k] = i;
@@ -2267,7 +3080,7 @@ function collectJsonSecretFieldSpans(text, span, isHead) {
2267
3080
  continue;
2268
3081
  const open = found === STRING_OPEN;
2269
3082
  const end = open ? text.length : found;
2270
- span(q, end, open, JSON_FIELD_VALUE_REPLS[d]);
3083
+ span(q, end, q, open ? NOT_CLOSED : end - form.length, open, JSON_FIELD_VALUE_REPLS[d]);
2271
3084
  from = scanResumeAfterHit(start, end, JSON_FIELD_BACK);
2272
3085
  next = text.indexOf(JSON_FIELD_ANCHOR, from);
2273
3086
  if (next < 0)
@@ -2349,7 +3162,7 @@ function escapedAt(text, i) {
2349
3162
  anchoredScanUnits += i - 1 - k;
2350
3163
  return ((i - 1 - k) & 1) === 1;
2351
3164
  }
2352
- function anchorHeadAt(text, op, json) {
3165
+ function anchorHeadAt(scan, text, op, json) {
2353
3166
  let i = op - 1;
2354
3167
  let spacedLeft = false;
2355
3168
  for (;;) {
@@ -2383,7 +3196,10 @@ function anchorHeadAt(text, op, json) {
2383
3196
  while (j > floor && isIdentCode(text.charCodeAt(j)))
2384
3197
  j -= 1;
2385
3198
  anchoredScanUnits += nameEnd - (j + 1);
2386
- if (j === floor && j >= 0 && isIdentCode(text.charCodeAt(j)))
3199
+ const bound = j + 1 < nameEnd ? nameBoundAt(scan, Math.max(j, 0), nameEnd) : -1;
3200
+ if (bound > j)
3201
+ j = bound - 1;
3202
+ else if (j === floor && j >= 0 && isIdentCode(text.charCodeAt(j)))
2387
3203
  return undefined;
2388
3204
  let nameStart = j + 1;
2389
3205
  let afterEscape = false;
@@ -2451,50 +3267,96 @@ function quotedValueEnd(text, cs, quote, json, isHead) {
2451
3267
  }
2452
3268
  return isHead ? STRING_OPEN : -1;
2453
3269
  }
2454
- function bareValueEnd(text, s, json, listSep) {
3270
+ const NO_MEMO_GROUP = -1;
3271
+ function buildBareValueMemo() {
3272
+ return {
3273
+ from: new Int32Array(4),
3274
+ at: new Int32Array(4).fill(-1),
3275
+ netFrom: new Int32Array(4).fill(NO_MEMO_GROUP),
3276
+ net: new Int32Array(16),
3277
+ runFrom: new Int32Array(4).fill(NO_MEMO_GROUP),
3278
+ closers: Array.from({ length: 16 }, () => []),
3279
+ };
3280
+ }
3281
+ const listSepSlot = (listSep) => (listSep === 0 ? 0 : listSep === CH_AMP ? 1 : listSep === CH_SEMI ? 2 : 3);
3282
+ function bareValueEnd(text, s, json, listSep, memo) {
2455
3283
  const n = text.length;
2456
- let i = s;
2457
- while (i < n) {
2458
- const unit = readAnchorUnit(text, i, json);
2459
- const ch = unitChOf(unit);
2460
- if (ch === JSON_STRING_END || isRegexSpace(ch))
2461
- break;
2462
- if (listSep !== 0 && (ch === listSep || (listSep === CH_AMP && ch === CH_HASH)))
2463
- break;
2464
- const next = i + unitLenOf(unit);
2465
- if (isQuoteUnit(ch)) {
2466
- if (next >= n)
3284
+ const k = listSepSlot(listSep);
3285
+ let i;
3286
+ if (memo.from[k] <= s && s <= memo.at[k])
3287
+ i = memo.at[k];
3288
+ else {
3289
+ i = s;
3290
+ while (i < n) {
3291
+ const unit = readAnchorUnit(text, i, json);
3292
+ const ch = unitChOf(unit);
3293
+ if (ch === JSON_STRING_END || isRegexSpace(ch))
2467
3294
  break;
2468
- if (endsValueUnit(unitChOf(readAnchorUnit(text, next, json))))
3295
+ if (listSep !== 0 && (ch === listSep || (listSep === CH_AMP && ch === CH_HASH)))
2469
3296
  break;
3297
+ const next = i + unitLenOf(unit);
3298
+ if (isQuoteUnit(ch)) {
3299
+ if (next >= n)
3300
+ break;
3301
+ if (endsValueUnit(unitChOf(readAnchorUnit(text, next, json))))
3302
+ break;
3303
+ }
3304
+ i = next;
2470
3305
  }
2471
- i = next;
3306
+ memo.from[k] = s;
3307
+ memo.at[k] = i;
3308
+ memo.netFrom[k] = NO_MEMO_GROUP;
3309
+ memo.runFrom[k] = NO_MEMO_GROUP;
2472
3310
  }
2473
- anchoredScanUnits += i - s;
2474
- const excess = [0, 0, 0, 0];
2475
- for (let k = s; k < i; k += 1) {
2476
- const b = bracketOf(text.charCodeAt(k));
2477
- if (b !== 0)
2478
- excess[Math.abs(b) - 1] = excess[Math.abs(b) - 1] + (b < 0 ? 1 : -1);
3311
+ const nb = 4 * k;
3312
+ const net = memo.net;
3313
+ const netFrom = memo.netFrom[k];
3314
+ if (netFrom === NO_MEMO_GROUP) {
3315
+ net.fill(0, nb, nb + 4);
3316
+ for (let q = s; q < i; q++) {
3317
+ const b = bracketOf(text.charCodeAt(q));
3318
+ if (b !== 0)
3319
+ net[nb + Math.abs(b) - 1] = net[nb + Math.abs(b) - 1] + (b < 0 ? 1 : -1);
3320
+ }
3321
+ anchoredScanUnits += i - s;
2479
3322
  }
2480
- let e = i;
2481
- while (e > s) {
2482
- const c = text.charCodeAt(e - 1);
2483
- if (c === CH_DOT || c === CH_COMMA || c === CH_SEMI || c === CH_COLON) {
2484
- e -= 1;
2485
- continue;
3323
+ else {
3324
+ for (let q = netFrom; q < s; q++) {
3325
+ const b = bracketOf(text.charCodeAt(q));
3326
+ if (b !== 0)
3327
+ net[nb + Math.abs(b) - 1] = net[nb + Math.abs(b) - 1] - (b < 0 ? 1 : -1);
2486
3328
  }
2487
- const b = bracketOf(c);
2488
- if (b < 0 && excess[-b - 1] > 0) {
2489
- excess[-b - 1] = excess[-b - 1] - 1;
2490
- e -= 1;
2491
- continue;
3329
+ anchoredScanUnits += s - netFrom;
3330
+ }
3331
+ memo.netFrom[k] = s;
3332
+ if (memo.runFrom[k] === NO_MEMO_GROUP) {
3333
+ for (let q = 0; q < 4; q++)
3334
+ memo.closers[nb + q].length = 0;
3335
+ const lowest = memo.from[k];
3336
+ let r = i;
3337
+ while (r > lowest) {
3338
+ const c = text.charCodeAt(r - 1);
3339
+ if (c !== CH_DOT && c !== CH_COMMA && c !== CH_SEMI && c !== CH_COLON) {
3340
+ const b = bracketOf(c);
3341
+ if (b >= 0)
3342
+ break;
3343
+ memo.closers[nb - b - 1].push(r - 1);
3344
+ }
3345
+ r -= 1;
2492
3346
  }
2493
- break;
3347
+ anchoredScanUnits += i - r;
3348
+ memo.runFrom[k] = r;
3349
+ }
3350
+ let e = Math.max(s, memo.runFrom[k]);
3351
+ for (let q = 0; q < 4; q++) {
3352
+ const at = memo.closers[nb + q];
3353
+ const stop = Math.max(net[nb + q], 0);
3354
+ if (stop < at.length && at[stop] + 1 > e)
3355
+ e = at[stop] + 1;
2494
3356
  }
2495
3357
  return e;
2496
3358
  }
2497
- function literalValueAt(text, s, json, declaredIfBare, listSep, isHead) {
3359
+ function literalValueAt(text, s, json, declaredIfBare, listSep, isHead, memo) {
2498
3360
  const n = text.length;
2499
3361
  if (s >= n)
2500
3362
  return undefined;
@@ -2526,7 +3388,7 @@ function literalValueAt(text, s, json, declaredIfBare, listSep, isHead) {
2526
3388
  if (after === CH_LPAREN || after === CH_LBRACKET || (after === CH_DOT && k + 1 < n && isProgramIdentStartCode(text.charCodeAt(k + 1))))
2527
3389
  return undefined;
2528
3390
  }
2529
- const e = bareValueEnd(text, vs, json, listSep);
3391
+ const e = bareValueEnd(text, vs, json, listSep, memo);
2530
3392
  if (e <= vs)
2531
3393
  return undefined;
2532
3394
  if (e - vs <= ABSENT_OR_BOOLEAN_MAX && ABSENT_OR_BOOLEAN_WORDS.has(text.slice(vs, e).toLowerCase()))
@@ -2595,18 +3457,60 @@ function schemeValueAt(text, s, json) {
2595
3457
  return undefined;
2596
3458
  return { start: vs, end: e, quoted: false };
2597
3459
  }
2598
- function collectAnchoredValueSpans(text, json, span, isHead) {
3460
+ function anchoredValueAt(scan, text, op, json, span, isHead, memo) {
3461
+ const head = anchorHeadAt(scan, text, op, json);
3462
+ if (head === undefined)
3463
+ return;
2599
3464
  const n = text.length;
2600
- let resume = 0;
3465
+ let s = op + 1;
3466
+ let spacedRight = false;
3467
+ while (s < n) {
3468
+ const unit = readAnchorUnit(text, s, json);
3469
+ if (!isInlineSpaceUnit(unitChOf(unit)))
3470
+ break;
3471
+ s += unitLenOf(unit);
3472
+ spacedRight = true;
3473
+ }
3474
+ let v;
3475
+ switch (head.strength) {
3476
+ case "structure-only":
3477
+ return;
3478
+ case "auth-scheme":
3479
+ v = schemeValueAt(text, s, json);
3480
+ break;
3481
+ case "declarative":
3482
+ v = literalValueAt(text, s, json, head.quotedName || (!head.spacedLeft && !spacedRight), head.listSep, isHead, memo);
3483
+ break;
3484
+ case "any-form":
3485
+ v = literalValueAt(text, s, json, true, head.listSep, isHead, memo);
3486
+ break;
3487
+ default: {
3488
+ const unhandled = head.strength;
3489
+ throw new Error(`redact: unhandled free-text anchor strength ${String(unhandled)}`);
3490
+ }
3491
+ }
3492
+ if (v === undefined)
3493
+ return;
3494
+ span(v.start, v.end, op, v.quoted && v.open !== true ? v.end : NOT_CLOSED, v.open === true);
3495
+ }
3496
+ function collectAnchoredValueSpans(scan, text, json, span, isHead) {
3497
+ const dirty = scan.dirty;
3498
+ const anchorFamily = familyCode("anchor");
3499
+ const rd = scan.opsRead;
3500
+ const wr = scan.opsWrite;
3501
+ wr.n = 0;
3502
+ wr.cut = OPS_UNCUT;
3503
+ const pa = rd.a;
3504
+ const pan = dirty === undefined ? 0 : rd.n;
3505
+ let pc = 0;
3506
+ let armLo = FOOT_NONE;
3507
+ let armHi = -1;
3508
+ const memo = buildBareValueMemo();
2601
3509
  let nextEq = text.indexOf("=");
2602
3510
  let nextColon = text.indexOf(":");
2603
3511
  for (;;) {
2604
- if (nextEq >= 0 && nextEq < resume)
2605
- nextEq = text.indexOf("=", resume);
2606
- if (nextColon >= 0 && nextColon < resume)
2607
- nextColon = text.indexOf(":", resume);
2608
3512
  if (nextEq < 0 && nextColon < 0)
2609
- return;
3513
+ break;
2610
3514
  let op;
2611
3515
  if (nextColon < 0 || (nextEq >= 0 && nextEq < nextColon)) {
2612
3516
  op = nextEq;
@@ -2616,41 +3520,51 @@ function collectAnchoredValueSpans(text, json, span, isHead) {
2616
3520
  op = nextColon;
2617
3521
  nextColon = text.indexOf(":", op + 1);
2618
3522
  }
2619
- const head = anchorHeadAt(text, op, json);
2620
- if (head === undefined)
3523
+ while (pc < pan && pa[pc] < op)
3524
+ pc += ANCHOR_SEG_STRIDE;
3525
+ const known = pc < pan && pa[pc] === op;
3526
+ if (!known && dirty !== undefined && op < rd.cut)
2621
3527
  continue;
2622
- let s = op + 1;
2623
- let spacedRight = false;
2624
- while (s < n) {
2625
- const unit = readAnchorUnit(text, s, json);
2626
- if (!isInlineSpaceUnit(unitChOf(unit)))
2627
- break;
2628
- s += unitLenOf(unit);
2629
- spacedRight = true;
3528
+ scan.self = readingKey(anchorFamily, op);
3529
+ let lo;
3530
+ let hi;
3531
+ let answer;
3532
+ if (known && (answer = nameBoundAt(scan, pa[pc + 1], pa[pc + 2])) === pa[pc + 3]) {
3533
+ lo = pa[pc + 1];
3534
+ hi = pa[pc + 2];
2630
3535
  }
2631
- let v;
2632
- switch (head.strength) {
2633
- case "structure-only":
3536
+ else {
3537
+ scan.footLo = FOOT_NONE;
3538
+ scan.footHi = -1;
3539
+ anchoredValueAt(scan, text, op, json, span, isHead, memo);
3540
+ lo = scan.footLo;
3541
+ hi = scan.footHi;
3542
+ if (lo > hi)
2634
3543
  continue;
2635
- case "auth-scheme":
2636
- v = schemeValueAt(text, s, json);
2637
- break;
2638
- case "declarative":
2639
- v = literalValueAt(text, s, json, head.quotedName || (!head.spacedLeft && !spacedRight), head.listSep, isHead);
2640
- break;
2641
- case "any-form":
2642
- v = literalValueAt(text, s, json, true, head.listSep, isHead);
2643
- break;
2644
- default: {
2645
- const unhandled = head.strength;
2646
- throw new Error(`redact: unhandled free-text anchor strength ${String(unhandled)}`);
3544
+ answer = nameBoundAt(scan, lo, hi);
3545
+ }
3546
+ if (wr.cut === OPS_UNCUT) {
3547
+ if (wr.n + ANCHOR_SEG_STRIDE <= wr.a.length || growSegments(wr)) {
3548
+ const w = wr.a;
3549
+ const k = wr.n;
3550
+ w[k] = op;
3551
+ w[k + 1] = lo;
3552
+ w[k + 2] = hi;
3553
+ w[k + 3] = answer;
3554
+ wr.n = k + ANCHOR_SEG_STRIDE;
2647
3555
  }
3556
+ else
3557
+ wr.cut = op;
2648
3558
  }
2649
- if (v === undefined)
2650
- continue;
2651
- span(v.start, v.end, v.open === true);
2652
- resume = v.quoted ? v.start : v.end;
3559
+ if (lo < armLo)
3560
+ armLo = lo;
3561
+ if (hi > armHi)
3562
+ armHi = hi;
2653
3563
  }
3564
+ scan.opsRead = wr;
3565
+ scan.opsWrite = rd;
3566
+ scan.footLo = armLo;
3567
+ scan.footHi = armHi;
2654
3568
  }
2655
3569
  export function structuralScannerForTest(arm) {
2656
3570
  const collect = (scan) => (text) => {
@@ -2662,17 +3576,18 @@ export function structuralScannerForTest(arm) {
2662
3576
  case "json-secret-field":
2663
3577
  return { declared: new RegExp(JSON_SECRET_FIELD_PATTERN[0].source, JSON_SECRET_FIELD_PATTERN[0].flags), spans: collect(collectJsonSecretFieldSpans) };
2664
3578
  case "pem-fence":
2665
- return { declared: new RegExp(PEM_FENCE_PATTERN[0].source, PEM_FENCE_PATTERN[0].flags), spans: collect((text, span, isHead) => collectPemFenceSpans({ ...buildForensicScan(NO_CREDENTIAL_SPANS, 1), others: NO_CREDENTIAL_SPANS }, text, span, isHead)) };
3579
+ return { declared: new RegExp(PEM_FENCE_PATTERN[0].source, PEM_FENCE_PATTERN[0].flags), spans: collect((text, span, isHead) => collectPemFenceSpans(buildForensicScan(NO_TOKEN_UNITS, NO_ROWS, 1), text, span, isHead)) };
2666
3580
  }
2667
3581
  }
2668
3582
  let ppkScanBytes = 0;
2669
3583
  export function ppkScanBytesForTest() { return ppkScanBytes; }
2670
3584
  export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
2671
3585
  function collectPpkContainerSpans(scan, text, eat, span, isHead) {
2672
- const HEAD = "PuTTY-User-Key-File-";
3586
+ const fam = familyCode("putty-ppk");
3587
+ const HEAD = PPK_FILE_HEAD;
2673
3588
  const MAC = "Private-MAC: ";
2674
3589
  const HASH = "Private-Hash: ";
2675
- const LINES = "Private-Lines: ";
3590
+ const LINES = PPK_LINES_FIELD;
2676
3591
  const cache = new Map();
2677
3592
  const find = (needle, from) => {
2678
3593
  const c = cache.get(needle);
@@ -2686,7 +3601,7 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
2686
3601
  };
2687
3602
  const versionEnd = (at) => {
2688
3603
  let p = at + HEAD.length;
2689
- while (p < text.length && opaqueUnitEnd(scan, p) < 0) {
3604
+ while (p < text.length && tokenEnd(scan, p) < 0) {
2690
3605
  const c = text.charCodeAt(p);
2691
3606
  if (c >= 48 && c <= 57)
2692
3607
  p += 1;
@@ -2707,14 +3622,14 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
2707
3622
  let p = find(HEAD, from);
2708
3623
  for (; p !== Number.POSITIVE_INFINITY; p = find(HEAD, p + HEAD.length)) {
2709
3624
  const lineEnd = versionedHeadEnd(p);
2710
- if (lineEnd >= 0 && !anchorHeld(scan, p, lineEnd))
3625
+ if (lineEnd >= 0 && !lineHeld(scan, p, lineEnd, readingKey(fam, p)))
2711
3626
  break;
2712
3627
  }
2713
3628
  return p;
2714
3629
  };
2715
3630
  const hexRunEnd = (from) => {
2716
3631
  let e = from;
2717
- while (e < text.length && opaqueUnitEnd(scan, e) < 0) {
3632
+ while (e < text.length && tokenEnd(scan, e) < 0) {
2718
3633
  const c = text.charCodeAt(e);
2719
3634
  if ((c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70))
2720
3635
  e += 1;
@@ -2725,7 +3640,9 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
2725
3640
  };
2726
3641
  let at = nextVersionedHead(0);
2727
3642
  while (at !== Number.POSITIVE_INFINITY) {
3643
+ scan.self = readingKey(fam, at);
2728
3644
  let pos = at + HEAD.length + 1;
3645
+ let closeAt = NOT_CLOSED;
2729
3646
  let end = text.length;
2730
3647
  let ateToEnd = !isHead;
2731
3648
  let open = false;
@@ -2752,12 +3669,8 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
2752
3669
  let gapBlank = true;
2753
3670
  if (limit !== Number.POSITIVE_INFINITY) {
2754
3671
  for (let i = hexEnd; i < limit; i++) {
2755
- if (opaqueUnitEnd(scan, i) >= 0) {
2756
- gapBlank = false;
2757
- break;
2758
- }
2759
3672
  const c = text.charCodeAt(i);
2760
- if (c !== 32 && c !== 10 && c !== 13 && c !== 9) {
3673
+ if ((c !== 32 && c !== 10 && c !== 13 && c !== 9) || delimClass(scan, i) !== DELIM_FREE) {
2761
3674
  gapBlank = false;
2762
3675
  break;
2763
3676
  }
@@ -2767,18 +3680,20 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
2767
3680
  ateToEnd = straggler < limit || !gapBlank;
2768
3681
  end = ateToEnd || undecided ? text.length : hexEnd;
2769
3682
  open = undecided;
3683
+ if (end === hexEnd)
3684
+ closeAt = term;
2770
3685
  break;
2771
3686
  }
2772
3687
  if (ateToEnd && eat !== undefined && eat.from < 0) {
2773
3688
  eat.from = at;
2774
3689
  eat.len = text.length;
2775
3690
  }
2776
- span(at, end, open);
3691
+ span(at, end, at, closeAt, open);
2777
3692
  at = end >= text.length ? Number.POSITIVE_INFINITY : nextVersionedHead(end);
2778
3693
  }
2779
3694
  const last = isHead ? text.lastIndexOf(HEAD) : -1;
2780
- if (last >= 0 && isOpenHead(last) && !anchorHeld(scan, last, text.length))
2781
- span(last, text.length, true);
3695
+ if (last >= 0 && isOpenHead(last) && !lineHeld(scan, last, text.length, readingKey(fam, last)))
3696
+ span(last, text.length, last, NOT_CLOSED, true);
2782
3697
  }
2783
3698
  const armPool = (() => {
2784
3699
  const spanRepl = (token) => {
@@ -2816,67 +3731,68 @@ const armPool = (() => {
2816
3731
  if (isHead && keywordTail !== undefined) {
2817
3732
  const at = undecidedKeywordStart(text, keywordTail);
2818
3733
  if (at >= 0)
2819
- out.push({ start: at, end: text.length, repl, rank, order, undecided: true });
3734
+ out.push({ start: at, end: text.length, repl, rank, order, undecided: true, anchor: at, closeAt: NOT_CLOSED });
2820
3735
  }
2821
3736
  },
2822
3737
  retry: boundary === undefined ? undefined : (text, at, out) => collectStickySpans(boundary, text, at, out),
2823
3738
  headSrc: boundary === undefined ? undefined : headSrc,
2824
3739
  headFirst: boundary === undefined ? undefined : headFirst,
2825
- family: undefined,
3740
+ family: "prose",
3741
+ reach: regexDecisionReach(re, LEXICAL_KEYWORD_ARMS.has(re)),
2826
3742
  };
2827
3743
  };
2828
- const scannerMember = (token, order, arm, rank = ARM_RANK.structural, lexical = false) => {
3744
+ const scannerMember = (token, order, arm, reach, family) => {
2829
3745
  const repl = spanRepl(token);
3746
+ const rank = ARM_RANK.structural;
2830
3747
  return {
2831
3748
  token,
2832
3749
  rank,
2833
3750
  order,
2834
3751
  phase: 1,
2835
- lexical,
2836
- collect: (text, json, eat, out, isHead) => arm(text, json, eat, (start, end, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided } : { start, end, repl: r, rank, order }), isHead),
3752
+ lexical: false,
3753
+ collect: (text, json, eat, out, isHead) => arm(text, json, eat, (start, end, anchor, closeAt, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided, anchor, closeAt } : { start, end, repl: r, rank, order, anchor, closeAt }), isHead),
2837
3754
  retry: undefined,
2838
3755
  headSrc: undefined,
2839
3756
  headFirst: undefined,
2840
- family: undefined,
3757
+ family,
3758
+ reach,
2841
3759
  };
2842
3760
  };
2843
- const forensicMember = (token, order, arm, family = undefined, view = "all") => {
2844
- if (view === "other-families" && family === undefined)
2845
- throw new Error(`redact: forensic arm ${token} reads the other-families evidence view but declares no credential family`);
3761
+ const forensicMember = (token, order, arm, reach, family, rank = ARM_RANK.structural, lexical = false) => {
2846
3762
  const repl = spanRepl(token);
2847
- const rank = ARM_RANK.structural;
2848
3763
  return {
2849
3764
  token,
2850
3765
  rank,
2851
3766
  order,
2852
3767
  phase: 2,
2853
- lexical: false,
2854
- collect: (scan, text, json, eat, out, isHead) => arm(scan, text, json, eat, (start, end, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided } : { start, end, repl: r, rank, order }), isHead),
3768
+ lexical,
3769
+ collect: (scan, text, json, eat, out, isHead) => arm(scan, text, json, eat, (start, end, anchor, closeAt, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided, anchor, closeAt } : { start, end, repl: r, rank, order, anchor, closeAt }), isHead),
2855
3770
  retry: undefined,
2856
3771
  headSrc: undefined,
2857
3772
  headFirst: undefined,
2858
3773
  family,
2859
- view,
3774
+ repl,
3775
+ reach,
2860
3776
  };
2861
3777
  };
2862
3778
  const structuralScannerMember = (row, order) => {
2863
3779
  if (row === PEM_FENCE_PATTERN)
2864
- return forensicMember(row[1], order, (scan, text, _json, _eat, span, isHead) => collectPemFenceSpans(scan, text, span, isHead), "pem-armor", "other-families");
3780
+ return forensicMember(row[1], order, (scan, text, _json, _eat, span, isHead) => collectPemFenceSpans(scan, text, span, isHead), PEM_FENCE_OPEN.length, "pem-armor");
2865
3781
  if (row === SSH2_FENCE_PATTERN)
2866
- return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectSsh2FenceSpans(scan, text, json, span, isHead), "ssh2-armor");
3782
+ return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectSsh2FenceSpans(scan, text, json, span, isHead), SSH2_FENCE_OPEN.length, "ssh2-armor");
2867
3783
  if (row === URL_CREDENTIAL_PATTERN)
2868
- return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectUrlCredentialSpans(scan, text, json, span, isHead));
3784
+ return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectUrlCredentialSpans(scan, text, json, span, isHead), "://".length, "url");
2869
3785
  if (row === JSON_SECRET_FIELD_PATTERN)
2870
- return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectJsonSecretFieldSpans(text, span, isHead));
3786
+ return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectJsonSecretFieldSpans(text, span, isHead), JSON_FIELD_BACK, "json-field");
2871
3787
  throw new Error(`redact: structural row /${row[0].source}/ has no hand-written scanner — structural grammars are declared as regexes but never executed as one (S-102 / S-265)`);
2872
3788
  };
2873
3789
  const members = [];
2874
3790
  for (const row of STRUCTURAL_SECRET_PATTERNS)
2875
3791
  members.push(structuralScannerMember(row, members.length));
2876
- members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span, isHead) => collectPpkContainerSpans(scan, text, eat, span, isHead), "putty-ppk"));
2877
- members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectPemFragmentSpans(scan, text, json, span, isHead), "pem-armor"));
2878
- members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span, isHead) => collectHeadlessPpkSpans(scan, text, span, isHead), "putty-ppk"));
2879
- members.push(scannerMember(ANCHORED_VALUE_TOKEN, members.length, (text, json, _eat, span, isHead) => collectAnchoredValueSpans(text, json, span, isHead), ARM_RANK.keyword, true));
3792
+ members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span, isHead) => collectPpkContainerSpans(scan, text, eat, span, isHead), PPK_FILE_HEAD.length, "putty-ppk"));
3793
+ members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectPemFragmentSpans(scan, text, json, span, isHead), PEM_FENCE_OPEN.length, "pem-armor"));
3794
+ members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span, isHead) => collectHeadlessPpkSpans(scan, text, span, isHead), PPK_LINES_FIELD.length, "putty-ppk"));
3795
+ members.push(forensicMember(ANCHORED_VALUE_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectAnchoredValueSpans(scan, text, json, span, isHead), Math.max(BEARER_VALUE_MIN, ABSENT_OR_BOOLEAN_MAX + 1, STRONG_AUTH_SCHEME_MAX + 1), "anchor", ARM_RANK.keyword, true));
2880
3796
  for (const row of KEYWORD_SECRET_PATTERNS)
2881
3797
  members.push(regexMember(row, ARM_RANK.keyword, members.length));
2882
3798
  for (const row of SHAPE_SECRET_PATTERNS)
@@ -2890,13 +3806,7 @@ const armPool = (() => {
2890
3806
  const retryMembers = members.filter((m) => m.retry !== undefined);
2891
3807
  const phase1 = members.filter((m) => m.phase === 1);
2892
3808
  const phase2 = members.filter((m) => m.phase === 2);
2893
- const familyOfOrder = members.map((m) => m.family);
2894
- const viewKeyOf = (m) => {
2895
- switch (m.view) {
2896
- case "all": return undefined;
2897
- case "other-families": return m.family;
2898
- }
2899
- };
3809
+ const familyCodeOfOrder = Int32Array.from(members, (m) => familyCode(m.family));
2900
3810
  const laneOf = (scope) => {
2901
3811
  const inScope = (m) => {
2902
3812
  switch (scope) {
@@ -2904,15 +3814,10 @@ const armPool = (() => {
2904
3814
  case "bounded-unit": return !m.lexical;
2905
3815
  }
2906
3816
  };
2907
- const lanePhase2 = phase2.filter(inScope);
2908
- const viewKeys = [...new Set([...lanePhase2.map(viewKeyOf), ...lanePhase2.flatMap((m) => (m.family === undefined ? [] : [m.family]))])];
2909
3817
  return {
2910
3818
  phase1: phase1.filter(inScope),
2911
- phase2: lanePhase2,
3819
+ phase2: phase2.filter(inScope),
2912
3820
  retry: retryMembers.filter(inScope),
2913
- viewKeys,
2914
- viewOf: lanePhase2.map((m) => viewKeys.indexOf(viewKeyOf(m))),
2915
- othersOf: lanePhase2.map((m) => (m.family === undefined ? -1 : viewKeys.indexOf(m.family))),
2916
3821
  };
2917
3822
  };
2918
3823
  const LANES = {
@@ -2923,21 +3828,39 @@ const armPool = (() => {
2923
3828
  const headFirstUnit = unionMembers(retryMembers.map((m) => m.headFirst)) ?? new Uint32Array(0x10000 >>> 5).fill(0xffffffff);
2924
3829
  poolArmCount = members.length;
2925
3830
  poolBoundaryArmCount = retryMembers.length;
2926
- return (text, wantCount, json, eat, scope, reach, engineMarked) => {
3831
+ const reaches = members.map((m) => ({ token: m.token, order: m.order, reach: m.reach }));
3832
+ const run = (text, wantCount, json, eat, scope, reach, engineMarked) => {
2927
3833
  readings += 1;
2928
3834
  const isHead = reach !== undefined;
2929
3835
  const lane = LANES[scope];
2930
3836
  const spans = [];
2931
3837
  let tried;
3838
+ const untriedFrom = (nu, p) => {
3839
+ let q = p;
3840
+ while (nu[q] !== 0) {
3841
+ coverageVisits += 1;
3842
+ q += nu[q];
3843
+ }
3844
+ for (let x = p; x !== q;) {
3845
+ const n = x + nu[x];
3846
+ nu[x] = q - x;
3847
+ x = n;
3848
+ }
3849
+ return q;
3850
+ };
2932
3851
  const freshCovered = (from) => {
2933
3852
  const at = [];
2934
- const seen = (tried ??= new Uint8Array(text.length + 1));
3853
+ const seen = (tried ??= new Int32Array(text.length + 2));
2935
3854
  for (let k = from; k < spans.length; k++) {
2936
3855
  const s = spans[k];
2937
3856
  for (let p = s.start; p <= s.end; p++) {
2938
- if (seen[p] === 1)
2939
- continue;
3857
+ if (seen[p] !== 0) {
3858
+ p = untriedFrom(seen, p);
3859
+ if (p > s.end)
3860
+ break;
3861
+ }
2940
3862
  seen[p] = 1;
3863
+ coverageVisits += 1;
2941
3864
  coveredPositions += 1;
2942
3865
  if (unitBit(headFirstUnit, text.charCodeAt(p)) !== 1)
2943
3866
  continue;
@@ -2964,26 +3887,23 @@ const armPool = (() => {
2964
3887
  m.collect(text, json, eat, spans, isHead);
2965
3888
  closeCoveragePositions(0);
2966
3889
  if (lane.phase2.length > 0) {
2967
- const tokens = credentialSpans(text);
2968
- const scan = buildForensicScan(tokens, lane.phase2.length);
3890
+ const scan = buildForensicScan(tokenUnits(text), buildEvidenceRows(text), lane.phase2.length);
2969
3891
  const reuse = forensicRescanMode === "incremental";
2970
3892
  const foot = scan.armFoot;
2971
- const views = lane.viewKeys.map((key) => ({ key, table: tokens, dirty: undefined }));
3893
+ let dirty;
2972
3894
  let folded = 0;
2973
3895
  for (let level = 0;; level += 1) {
2974
3896
  const before = spans.length;
2975
3897
  for (let j = 0; j < lane.phase2.length; j++) {
2976
- const view = views[lane.viewOf[j]];
2977
- const oi = lane.othersOf[j];
2978
- const others = oi < 0 ? undefined : views[oi];
2979
- const dirty = view.dirty;
2980
3898
  const lo = foot[2 * j];
2981
3899
  const hi = foot[2 * j + 1];
2982
- if (dirty !== undefined && footprintClean(dirty, lo, hi) && (others === undefined || (others.dirty !== undefined && footprintClean(others.dirty, lo, hi)))) {
3900
+ if (dirty !== undefined && footprintClean(dirty, lo, hi)) {
2983
3901
  forensicSkips += 1;
2984
3902
  continue;
2985
3903
  }
2986
- useEvidenceView(scan, view.table, dirty, others?.table);
3904
+ scan.dirty = dirty;
3905
+ scan.rowCursor = 0;
3906
+ scan.self = NO_SELF;
2987
3907
  scan.footLo = FOOT_NONE;
2988
3908
  scan.footHi = -1;
2989
3909
  lane.phase2[j].collect(scan, text, json, level === 0 ? eat : undefined, spans, isHead);
@@ -2993,18 +3913,26 @@ const armPool = (() => {
2993
3913
  }
2994
3914
  closeCoveragePositions(before);
2995
3915
  forensicLevels += 1;
2996
- let grew = false;
2997
- for (const view of views) {
2998
- const key = view.key;
2999
- const grown = foldCredentialSpans(view.table, spans, folded, key === undefined ? undefined : (sp) => familyOfOrder[sp.order] !== key);
3000
- if (grown.dirty.starts.length > 0)
3001
- grew = true;
3002
- view.table = grown.next;
3003
- view.dirty = reuse ? grown.dirty : undefined;
3004
- }
3916
+ const grown = foldEvidenceRows(scan.rows, spans, folded, (o) => familyCodeOfOrder[o]);
3005
3917
  folded = spans.length;
3006
- if (!grew)
3918
+ if (grown.dirty.starts.length === 0)
3919
+ break;
3920
+ if (level + 1 >= FORENSIC_LEVEL_MAX) {
3921
+ for (let j = 0; j < lane.phase2.length; j++) {
3922
+ const lo = foot[2 * j];
3923
+ if (footprintClean(grown.dirty, lo, foot[2 * j + 1]))
3924
+ continue;
3925
+ const m = lane.phase2[j];
3926
+ spans.push({ start: lo, end: text.length, repl: m.repl, rank: m.rank, order: m.order, anchor: lo, closeAt: NOT_CLOSED });
3927
+ if (eat !== undefined && eat.capFrom < 0) {
3928
+ eat.capFrom = lo;
3929
+ eat.capLen = text.length;
3930
+ }
3931
+ }
3007
3932
  break;
3933
+ }
3934
+ scan.rows = grown.next;
3935
+ dirty = reuse ? grown.dirty : undefined;
3008
3936
  }
3009
3937
  }
3010
3938
  poolSpanCount += spans.length;
@@ -3035,6 +3963,7 @@ const armPool = (() => {
3035
3963
  }
3036
3964
  return { hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })), kept: stamped };
3037
3965
  };
3966
+ return Object.assign(run, { reaches, pendingTailUnits: Math.max(...reaches.map((r) => r.reach)) });
3038
3967
  })();
3039
3968
  let onRedaction;
3040
3969
  export function setRedactionObserver(fn) {
@@ -3132,12 +4061,18 @@ function readHexRunValue(s, from, to) {
3132
4061
  return v;
3133
4062
  }
3134
4063
  function shortestReading(text, layout) {
4064
+ const r = readShortest(text, layout);
4065
+ return { visible: r.visible, partial: r.partial };
4066
+ }
4067
+ function readShortest(text, layout) {
3135
4068
  const raw = collectRawControlCuts(text, invisibleClassesOf(layout));
3136
- const normalized = normalizeForRedaction(deleteInvisibleCharacters(text, raw, raw.present).text);
4069
+ const rfold = deleteInvisibleCharacters(text, raw, raw.present);
4070
+ const normalized = normalizeForRedaction(rfold.text);
3137
4071
  const open = trailingPartialEscapeStart(normalized);
3138
4072
  const spelled = collectSpelledCuts(normalized);
3139
- const length = deleteInvisibleCharacters(normalized, spelled, spelled.present).text.length;
3140
- return open < 0 ? { visible: length, partial: false } : { visible: length - (normalized.length - open), partial: true };
4073
+ const sfold = deleteInvisibleCharacters(normalized, spelled, spelled.present);
4074
+ const layers = [{ fold: sfold }, { input: rfold.text, view: normalized }, { fold: rfold }];
4075
+ return open < 0 ? { visible: sfold.text.length, partial: false, layers } : { visible: sfold.text.length - (normalized.length - open), partial: true, layers };
3141
4076
  }
3142
4077
  export function readingHeadEnd(text, layout, units, from = 0) {
3143
4078
  const rest = text.length - from;
@@ -3161,7 +4096,7 @@ export function readingHeadEnd(text, layout, units, from = 0) {
3161
4096
  }
3162
4097
  function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
3163
4098
  const wantCount = wantHits || (onRedaction !== undefined && observe);
3164
- const eat = observe ? { from: -1, len: 0 } : undefined;
4099
+ const eat = observe ? { from: -1, len: 0, capFrom: -1, capLen: 0 } : undefined;
3165
4100
  let spans = [];
3166
4101
  let hits = NO_HITS;
3167
4102
  let readsSoFar = 0;
@@ -3202,6 +4137,8 @@ function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
3202
4137
  hits = hitsOfSpans(spans);
3203
4138
  if (eat !== undefined && eat.from >= 0)
3204
4139
  recordFailOpen("server.redact.ppk-eat-to-end", `from=${eat.from} len=${eat.len}`);
4140
+ if (eat !== undefined && eat.capFrom >= 0)
4141
+ recordFailOpen("server.redact.forensic-level-cap", `from=${eat.capFrom} len=${eat.capLen}`);
3205
4142
  if (observe)
3206
4143
  reportHits(hits);
3207
4144
  return { text: applySpans(text, spans).out, hits, spans };
@@ -3251,6 +4188,91 @@ export function redactionSpansForTest(text, opts) {
3251
4188
  export function proposeRedactionSpans(text, scope, layout, extent) {
3252
4189
  return runRedaction(text, true, false, false, scope, layout, extent).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
3253
4190
  }
4191
+ export const PENDING_TAIL_UNITS = armPool.pendingTailUnits;
4192
+ export function regexDecisionReachForTest(re, lexical) {
4193
+ return regexDecisionReach(re, lexical);
4194
+ }
4195
+ export function pendingTailReachForTest() {
4196
+ return armPool.reaches;
4197
+ }
4198
+ function pendingTailStart(text, units) {
4199
+ const r = readShortest(text, "multi-line");
4200
+ return r.visible <= units ? 0 : createViewToOriginal(r.layers).start(r.visible - units);
4201
+ }
4202
+ function renderRange(text, spans, from, until) {
4203
+ if (from >= until)
4204
+ return "";
4205
+ const parts = [];
4206
+ let at = from;
4207
+ for (const s of spans) {
4208
+ if (s.end <= from)
4209
+ continue;
4210
+ if (s.start >= until)
4211
+ break;
4212
+ if (s.start > at)
4213
+ parts.push(text.slice(at, s.start));
4214
+ parts.push(s.repl);
4215
+ at = Math.max(at, s.end);
4216
+ }
4217
+ if (at < until)
4218
+ parts.push(text.slice(at, until));
4219
+ return parts.join("");
4220
+ }
4221
+ const STREAM_RESCAN_MIN = 64;
4222
+ const streamRescanStep = (n) => Math.max(STREAM_RESCAN_MIN, n >>> 5);
4223
+ export function createRedactionStream() {
4224
+ return openRedactionStream(PENDING_TAIL_UNITS);
4225
+ }
4226
+ export function createRedactionStreamWithTailForTest(units) {
4227
+ return openRedactionStream(units);
4228
+ }
4229
+ function openRedactionStream(pendingTail) {
4230
+ let buf = "";
4231
+ let released = 0;
4232
+ let deliveredFrom = 0;
4233
+ let sent = "";
4234
+ let scannedAt = 0;
4235
+ return {
4236
+ feed(chunk) {
4237
+ buf += chunk;
4238
+ },
4239
+ takeSettled() {
4240
+ if (buf.length - released <= pendingTail)
4241
+ return "";
4242
+ if (buf.length - scannedAt < streamRescanStep(buf.length))
4243
+ return "";
4244
+ scannedAt = buf.length;
4245
+ const emitEnd = pendingTailStart(buf, pendingTail);
4246
+ if (emitEnd <= released)
4247
+ return "";
4248
+ const spans = runRedaction(buf, false, false, false, "free-text", "multi-line", { kind: "head", emitEnd }).spans;
4249
+ let cut = emitEnd;
4250
+ for (const s of spans)
4251
+ if (s.start < cut && s.end > cut)
4252
+ cut = s.start;
4253
+ if (cut > 0 && cut < buf.length && (buf.charCodeAt(cut - 1) & 0xfc00) === 0xd800 && (buf.charCodeAt(cut) & 0xfc00) === 0xdc00)
4254
+ cut -= 1;
4255
+ if (cut <= released)
4256
+ return "";
4257
+ const out = renderRange(buf, spans, released, cut);
4258
+ released = cut;
4259
+ sent += out;
4260
+ return out;
4261
+ },
4262
+ flush() {
4263
+ const spans = runRedaction(buf, false, false, false, "free-text", "multi-line", WHOLE_EXTENT).spans;
4264
+ const split = released > deliveredFrom && spans.some((s) => s.start < released && s.end > released);
4265
+ return { rest: renderRange(buf, spans, released, buf.length), diverged: split || renderRange(buf, spans, deliveredFrom, released) !== sent };
4266
+ },
4267
+ skip() {
4268
+ released = deliveredFrom = scannedAt = buf.length;
4269
+ sent = "";
4270
+ },
4271
+ get fed() {
4272
+ return buf;
4273
+ },
4274
+ };
4275
+ }
3254
4276
  const REDACT_MAX_DEPTH = 64;
3255
4277
  export function redactDeep(value) {
3256
4278
  return redactDeepWith(value, redactSecrets);