@sema-agent/server 7.141.0 → 7.143.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +657 -0
- package/USAGE.md +102 -30
- package/dist/adoption/plan.js +22 -0
- package/dist/approval-ask-audit-store.d.ts +61 -43
- package/dist/approval-ask-audit-store.js +69 -44
- package/dist/approval-ask-machine.d.ts +15 -7
- package/dist/approval-ask-machine.js +10 -4
- package/dist/approval-card.d.ts +126 -27
- package/dist/approval-card.js +103 -23
- package/dist/approval-crash-convergence.d.ts +17 -7
- package/dist/approval-crash-convergence.js +94 -22
- package/dist/approval-deny-reasons.d.ts +34 -30
- package/dist/approval-deny-reasons.js +21 -6
- package/dist/approval-policy-settlement.d.ts +1 -1
- package/dist/approval-queue-kind.d.ts +27 -0
- package/dist/approval-queue-kind.js +6 -0
- package/dist/approval-reconciler.d.ts +41 -9
- package/dist/approval-reconciler.js +45 -42
- package/dist/audit.js +4 -2
- package/dist/auth-bridge.d.ts +9 -6
- package/dist/auth-bridge.js +9 -3
- package/dist/boot/adoption.js +2 -1
- package/dist/boot/budget-tracing.d.ts +17 -1
- package/dist/boot/budget-tracing.js +10 -6
- package/dist/boot/device-lane.d.ts +1 -1
- package/dist/boot/execution-env.d.ts +24 -4
- package/dist/boot/execution-env.js +26 -23
- package/dist/boot/execution-lane.js +2 -1
- package/dist/boot/exit-boundary.js +5 -4
- package/dist/boot/governance-seams.d.ts +26 -41
- package/dist/boot/governance-seams.js +61 -28
- package/dist/boot/late-binds.d.ts +7 -4
- package/dist/boot/late-binds.js +3 -1
- package/dist/boot/leader.js +2 -4
- package/dist/boot/memory-consolidation.d.ts +0 -8
- package/dist/boot/memory-consolidation.js +5 -4
- package/dist/boot/parked-revive-gate.js +2 -1
- package/dist/boot/permission-rules-audit.d.ts +0 -28
- package/dist/boot/permission-rules-audit.js +2 -1
- package/dist/boot/read-deny-anchor-audit.js +2 -1
- package/dist/boot/reapers.js +9 -7
- package/dist/boot/resolve-spec.d.ts +1 -1
- package/dist/boot/resolve-spec.js +19 -10
- package/dist/boot/retention-lane.js +5 -3
- package/dist/boot/runner-deps.d.ts +10 -7
- package/dist/boot/runner-deps.js +20 -29
- package/dist/boot/runtime-caps.js +3 -2
- package/dist/boot/session-faces.js +4 -3
- package/dist/boot/shutdown.d.ts +9 -0
- package/dist/boot/shutdown.js +19 -4
- package/dist/boot/stage-02-web-admin-pull.js +3 -0
- package/dist/boot/stage-03-store-backend.d.ts +4 -1
- package/dist/boot/stage-03-store-backend.js +1 -2
- package/dist/boot/stage-04-stores.d.ts +5 -2
- package/dist/boot/stage-04-stores.js +6 -7
- package/dist/boot/stage-05-execution-env.d.ts +6 -3
- package/dist/boot/stage-05-execution-env.js +8 -6
- package/dist/boot/stage-06-runners.d.ts +6 -3
- package/dist/boot/stage-06-runners.js +3 -4
- package/dist/boot/stage-07-capability-layer.d.ts +6 -3
- package/dist/boot/stage-07-capability-layer.js +7 -2
- package/dist/boot/stage-08-reapers.d.ts +6 -3
- package/dist/boot/stage-08-reapers.js +3 -2
- package/dist/boot/stage-09-leader.d.ts +6 -3
- package/dist/boot/stage-10-http-server.d.ts +6 -3
- package/dist/boot/stage-10-http-server.js +14 -5
- package/dist/boot/stores.d.ts +5 -2
- package/dist/boot/stores.js +10 -9
- package/dist/boot/web-admin.d.ts +5 -1
- package/dist/boot/web-admin.js +48 -36
- package/dist/boot/webfetch-summarize-lane.js +2 -1
- package/dist/boot/workflow-orchestration.js +3 -2
- package/dist/capabilities/builtin-tools.d.ts +2 -1
- package/dist/capabilities/builtin-tools.js +2 -4
- package/dist/capabilities/composite-members.d.ts +2 -1
- package/dist/capabilities/composite-members.js +2 -1
- package/dist/capabilities/hands-lane.d.ts +9 -39
- package/dist/capabilities/hands-lane.js +8 -13
- package/dist/capabilities/mcp-probe.d.ts +3 -2
- package/dist/capabilities/scenarios.d.ts +11 -1
- package/dist/capabilities/scenarios.js +1 -0
- package/dist/capabilities/skills.d.ts +1 -17
- package/dist/capabilities/skills.js +0 -8
- package/dist/capability-facts.d.ts +31 -7
- package/dist/capability-facts.js +27 -4
- package/dist/card-void.d.ts +6 -9
- package/dist/config-catalog.js +2 -1
- package/dist/config-dir-trust.js +58 -29
- package/dist/config-invariants.d.ts +7 -1
- package/dist/config-invariants.js +7 -0
- package/dist/config-provider.d.ts +10 -0
- package/dist/config-provider.js +15 -4
- package/dist/config-root-migration.d.ts +13 -0
- package/dist/config-root-migration.js +99 -0
- package/dist/config-types.d.ts +19 -8
- package/dist/config.d.ts +10 -3
- package/dist/config.js +18 -4
- package/dist/delegated-session-owner.d.ts +5 -3
- package/dist/delegated-session-owner.js +1 -1
- package/dist/device-ws-protocol.d.ts +1 -0
- package/dist/device-ws-protocol.js +2 -1
- package/dist/env-model-table-gate.d.ts +53 -0
- package/dist/env-model-table-gate.js +53 -0
- package/dist/execution-lane-caps.d.ts +2 -1
- package/dist/fleet/fleet-bus.d.ts +34 -55
- package/dist/fleet/fleet-bus.js +35 -100
- package/dist/fleet/fleet-reconciler.d.ts +4 -4
- package/dist/fleet/fleet-reconciler.js +1 -1
- package/dist/fleet/subagent-tail-bus.d.ts +14 -11
- package/dist/fleet/subagent-tail-bus.js +15 -28
- package/dist/hooks/hook-runner.d.ts +6 -0
- package/dist/hooks/hook-runner.js +58 -8
- package/dist/host-decision.d.ts +54 -20
- package/dist/host-decision.js +42 -14
- package/dist/http/admission.d.ts +1 -1
- package/dist/http/admission.js +7 -3
- package/dist/http/approval-sweeps.js +11 -8
- package/dist/http/caller-trust.d.ts +1 -1
- package/dist/http/dispatch.js +4 -2
- package/dist/http/durable-run-leg.js +2 -2
- package/dist/http/engine-refusal.d.ts +16 -0
- package/dist/http/engine-refusal.js +17 -0
- package/dist/http/execution-context.d.ts +2 -2
- package/dist/http/idempotency.d.ts +1 -1
- package/dist/http/memory-compliance-failure-reply.d.ts +1 -3
- package/dist/http/memory-compliance-failure-reply.js +4 -6
- package/dist/http/owner-access.d.ts +11 -0
- package/dist/http/owner-access.js +13 -0
- package/dist/http/principal-gate.d.ts +3 -1
- package/dist/http/principal-gate.js +17 -8
- package/dist/http/resume-legs.d.ts +21 -7
- package/dist/http/resume-legs.js +76 -58
- package/dist/http/resume-restatement.d.ts +39 -0
- package/dist/http/resume-restatement.js +25 -0
- package/dist/http/route-ctx.d.ts +35 -10
- package/dist/http/route-ctx.js +1 -1
- package/dist/http/route-table.js +2 -0
- package/dist/http/routes/approvals-assistant.d.ts +9 -1
- package/dist/http/routes/approvals-assistant.js +142 -31
- package/dist/http/routes/attachments.js +18 -0
- package/dist/http/routes/capabilities.js +16 -4
- package/dist/http/routes/memory-policy.d.ts +2 -2
- package/dist/http/routes/memory-policy.js +27 -7
- package/dist/http/routes/notify-wake.js +23 -25
- package/dist/http/routes/rules.js +48 -13
- package/dist/http/routes/runs.js +12 -11
- package/dist/http/routes/session-memory-sync.d.ts +91 -0
- package/dist/http/routes/session-memory-sync.js +174 -0
- package/dist/http/routes/session-sync.js +13 -20
- package/dist/http/routes/sessions-list.d.ts +2 -1
- package/dist/http/routes/sessions.d.ts +2 -2
- package/dist/http/routes/sessions.js +22 -23
- package/dist/http/routes/tasks.js +34 -54
- package/dist/http/run-read-face.d.ts +7 -0
- package/dist/http/run-read-face.js +9 -3
- package/dist/http/send.d.ts +5 -0
- package/dist/http/send.js +9 -2
- package/dist/http/server.d.ts +27 -15
- package/dist/http/server.js +4 -6
- package/dist/http/stored-body-replay.js +15 -4
- package/dist/http/submit-replay.d.ts +4 -3
- package/dist/http/submit-replay.js +2 -2
- package/dist/http/sync-submit-body.d.ts +21 -4
- package/dist/http/sync-submit-body.js +13 -0
- package/dist/http/wire-types.d.ts +17 -7
- package/dist/identity-source.d.ts +21 -2
- package/dist/identity-source.js +6 -0
- package/dist/leader/leader.js +5 -2
- package/dist/leader/wire.d.ts +7 -1
- package/dist/leader/wire.js +6 -5
- package/dist/leg-facts.d.ts +37 -0
- package/dist/leg-facts.js +15 -0
- package/dist/memory-bundle-engine.d.ts +1 -4
- package/dist/memory-bundle-engine.js +2 -2
- package/dist/memory-incident-sink.d.ts +19 -6
- package/dist/memory-incident-sink.js +8 -4
- package/dist/memory-operator-faces.d.ts +30 -66
- package/dist/memory-operator-faces.js +17 -88
- package/dist/memory-scope-stores.d.ts +38 -40
- package/dist/memory-scope-stores.js +51 -36
- package/dist/memory-scope.d.ts +0 -9
- package/dist/memory-scope.js +0 -9
- package/dist/memory-sync-client.d.ts +21 -21
- package/dist/memory-sync-client.js +62 -43
- package/dist/memory-sync-identity.d.ts +24 -0
- package/dist/memory-sync-identity.js +57 -0
- package/dist/memory-sync.d.ts +27 -5
- package/dist/memory-sync.js +29 -7
- package/dist/model-plane-face.d.ts +13 -7
- package/dist/model-plane-face.js +2 -1
- package/dist/observability/fail-open.d.ts +47 -14
- package/dist/observability/fail-open.js +37 -11
- package/dist/observability/run-terminal.d.ts +92 -34
- package/dist/observability/run-terminal.js +171 -162
- package/dist/orchestration/workflow-completion-inbox.d.ts +5 -19
- package/dist/orchestration/workflow-completion-inbox.js +0 -12
- package/dist/orchestration/workflow-notify-journal.d.ts +3 -2
- package/dist/orchestration/workflow-park-decision.d.ts +8 -3
- package/dist/orchestration/workflow-park-decision.js +5 -3
- package/dist/org-delivery-gate.d.ts +11 -0
- package/dist/org-delivery-gate.js +19 -1
- package/dist/parked-decide.d.ts +13 -9
- package/dist/parked-decide.js +7 -7
- package/dist/parked-revive-surface.js +1 -1
- package/dist/peer-inbound-face.d.ts +127 -0
- package/dist/peer-inbound-face.js +158 -0
- package/dist/permission-rule-list.d.ts +13 -9
- package/dist/permission-rule-list.js +7 -3
- package/dist/plugins/approval-ask-store-file.d.ts +14 -4
- package/dist/plugins/approval-ask-store-file.js +64 -11
- package/dist/plugins/approval-ask-store-memory.d.ts +2 -1
- package/dist/plugins/approval-ask-store-memory.js +17 -14
- package/dist/plugins/approval-ask-store-sql.d.ts +70 -23
- package/dist/plugins/approval-ask-store-sql.js +79 -30
- package/dist/plugins/checkpoint-store-sql.d.ts +4 -4
- package/dist/plugins/checkpoint-store-sql.js +9 -7
- package/dist/plugins/file-history-store-sql.d.ts +24 -18
- package/dist/plugins/file-history-store-sql.js +24 -19
- package/dist/plugins/file-run-store.d.ts +35 -1
- package/dist/plugins/file-run-store.js +108 -9
- package/dist/plugins/imported-layer-root-retirement.d.ts +82 -0
- package/dist/plugins/imported-layer-root-retirement.js +96 -0
- package/dist/plugins/local-checkpoint-store.d.ts +1 -1
- package/dist/plugins/local-checkpoint-store.js +2 -2
- package/dist/plugins/local-session-store.d.ts +42 -9
- package/dist/plugins/local-session-store.js +83 -33
- package/dist/plugins/local-task-attachment-store.d.ts +12 -2
- package/dist/plugins/local-task-attachment-store.js +67 -43
- package/dist/plugins/mailbox-recipient-lifecycle.d.ts +9 -31
- package/dist/plugins/mailbox-recipient-lifecycle.js +4 -0
- package/dist/plugins/mailbox-store-sql.d.ts +12 -8
- package/dist/plugins/mailbox-store-sql.js +76 -7
- package/dist/plugins/memory-key-guards.d.ts +3 -3
- package/dist/plugins/memory-run-store.d.ts +6 -1
- package/dist/plugins/memory-run-store.js +11 -1
- package/dist/plugins/memory-sync-store-pg.d.ts +15 -0
- package/dist/plugins/memory-sync-store-pg.js +21 -0
- package/dist/plugins/memory-sync-store-tidb.d.ts +2 -0
- package/dist/plugins/memory-sync-store-tidb.js +13 -1
- package/dist/plugins/one-time-migrations.d.ts +58 -2
- package/dist/plugins/one-time-migrations.js +161 -4
- package/dist/plugins/org-delivery-quota-store.d.ts +1 -1
- package/dist/plugins/org-delivery-quota-store.js +2 -12
- package/dist/plugins/park-commit-watch.d.ts +1 -1
- package/dist/plugins/permission-rule-store-file.d.ts +10 -1
- package/dist/plugins/permission-rule-store-file.js +15 -2
- package/dist/plugins/permission-rule-store-sql.d.ts +31 -1
- package/dist/plugins/permission-rule-store-sql.js +35 -2
- package/dist/plugins/remote-env-device.d.ts +8 -11
- package/dist/plugins/remote-env-device.js +6 -3
- package/dist/plugins/remote-env-e2b.d.ts +23 -4
- package/dist/plugins/remote-env-e2b.js +111 -28
- package/dist/plugins/remote-env-k8s.d.ts +61 -22
- package/dist/plugins/remote-env-k8s.js +133 -33
- package/dist/plugins/remote-env-local-docker.d.ts +15 -3
- package/dist/plugins/remote-env-local-docker.js +40 -12
- package/dist/plugins/retention-store-sql.d.ts +2 -2
- package/dist/plugins/run-store-sql.d.ts +9 -1
- package/dist/plugins/run-store-sql.js +9 -1
- package/dist/plugins/sandbox-read-floor.d.ts +5 -7
- package/dist/plugins/sandbox-read-floor.js +2 -7
- package/dist/plugins/sql-errors.d.ts +10 -1
- package/dist/plugins/sql-errors.js +13 -0
- package/dist/plugins/store-backend.d.ts +33 -35
- package/dist/plugins/store-backend.js +10 -1
- package/dist/plugins/store-contracts.d.ts +50 -1
- package/dist/plugins/store-contracts.js +6 -0
- package/dist/plugins/task-attachment-store.d.ts +112 -14
- package/dist/plugins/task-attachment-store.js +207 -29
- package/dist/plugins/tool-result-store-sql.d.ts +9 -8
- package/dist/plugins/tool-result-store-sql.js +23 -2
- package/dist/plugins/wiring-section-codec.d.ts +32 -0
- package/dist/plugins/wiring-section-codec.js +55 -0
- package/dist/plugins/workspace-fill.d.ts +60 -0
- package/dist/plugins/workspace-fill.js +106 -0
- package/dist/project-bundle.d.ts +38 -0
- package/dist/project-bundle.js +609 -0
- package/dist/rule-entry-list.d.ts +3 -2
- package/dist/rule-entry-list.js +5 -1
- package/dist/rules-consent.d.ts +28 -5
- package/dist/rules-consent.js +40 -28
- package/dist/run-local.js +11 -13
- package/dist/runs.d.ts +12 -8
- package/dist/runs.js +6 -8
- package/dist/runtime-governance.d.ts +1 -1
- package/dist/sandbox-egress-policy.d.ts +4 -1
- package/dist/sandbox-egress-policy.js +3 -1
- package/dist/sandbox-git-plane.d.ts +31 -12
- package/dist/sandbox-git-plane.js +24 -4
- package/dist/security.d.ts +9 -0
- package/dist/security.js +8 -0
- package/dist/server-secret-env.d.ts +4 -0
- package/dist/session-error-wire.d.ts +36 -0
- package/dist/session-error-wire.js +13 -0
- package/dist/session-incarnation.d.ts +1 -1
- package/dist/session-notify-door.d.ts +106 -0
- package/dist/session-notify-door.js +93 -0
- package/dist/session-sync.d.ts +9 -14
- package/dist/session-sync.js +5 -5
- package/dist/session-workspace.d.ts +3 -13
- package/dist/session-workspace.js +0 -9
- package/dist/standing-answer-grant.d.ts +5 -0
- package/dist/standing-answer-grant.js +10 -0
- package/dist/stores-sql/session-policy-store-sql.d.ts +4 -2
- package/dist/stores-sql/session-policy-store-sql.js +3 -13
- package/dist/task-mcp.d.ts +1 -1
- package/dist/task-settings.d.ts +21 -10
- package/dist/task-settings.js +46 -13
- package/dist/terminal.d.ts +39 -19
- package/dist/terminal.js +65 -22
- package/dist/tool-approval.d.ts +56 -37
- package/dist/tool-approval.js +78 -71
- package/dist/trace/core-keyset-guard.d.ts +134 -14
- package/dist/trace/engine-notice-wire.d.ts +9 -3
- package/dist/trace/engine-notice-wire.js +7 -3
- package/dist/trace/exit-seat.d.ts +9 -2
- package/dist/trace/exit-seat.js +1 -0
- package/dist/trace/ledger-sink.d.ts +10 -6
- package/dist/trace/ledger-sink.js +2 -2
- package/dist/trace/model-segment.d.ts +55 -25
- package/dist/trace/model-segment.js +101 -33
- package/dist/trace/project.d.ts +25 -5
- package/dist/trace/project.js +22 -0
- package/dist/trace/redact.d.ts +61 -9
- package/dist/trace/redact.js +1490 -468
- package/dist/trace/self-minted-wire.d.ts +745 -24
- package/dist/trace/self-minted-wire.js +239 -9
- package/dist/trace/wire-projection-faces.d.ts +2 -2
- package/dist/trace/wire-projection-faces.js +2 -2
- package/dist/verifier-pause-card.d.ts +33 -5
- package/dist/verifier-pause-card.js +15 -10
- package/dist/web-admin-client/apply-effective.d.ts +12 -16
- package/dist/web-admin-client/apply-effective.js +109 -9
- package/dist/web-admin-client/effective-keys.js +2 -0
- package/dist/web-admin-client/types.d.ts +4 -0
- package/dist/workspace-space.d.ts +19 -3
- package/dist/workspace-space.js +8 -4
- package/docs/DEVICE-EXECUTOR-PROTOCOL.md +3 -1
- package/package.json +3 -2
- package/wire-changes/server@7.142.0.json +639 -0
- package/wire-changes/server@7.143.0.json +1167 -0
package/dist/trace/redact.js
CHANGED
|
@@ -461,9 +461,11 @@ let urlSegmentRuns = 0;
|
|
|
461
461
|
let urlSegmentReplays = 0;
|
|
462
462
|
let ssh2ScanBytes = 0;
|
|
463
463
|
let mappedUnits = 0;
|
|
464
|
+
let coverageVisits = 0;
|
|
465
|
+
let rowVisits = 0;
|
|
464
466
|
let readings = 0;
|
|
465
467
|
export function armPoolBudgetForTest() {
|
|
466
|
-
return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings };
|
|
468
|
+
return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings, coverageVisits, rowVisits };
|
|
467
469
|
}
|
|
468
470
|
export function resetArmPoolBudgetForTest() {
|
|
469
471
|
poolSpanCount = 0;
|
|
@@ -478,6 +480,8 @@ export function resetArmPoolBudgetForTest() {
|
|
|
478
480
|
ssh2ScanBytes = 0;
|
|
479
481
|
mappedUnits = 0;
|
|
480
482
|
readings = 0;
|
|
483
|
+
coverageVisits = 0;
|
|
484
|
+
rowVisits = 0;
|
|
481
485
|
}
|
|
482
486
|
const unitBit = (t, c) => (t[c >>> 5] >>> (c & 31)) & 1;
|
|
483
487
|
let codeUnitDomain;
|
|
@@ -707,6 +711,166 @@ function deriveLiteralHead(re) {
|
|
|
707
711
|
first: unionMembers(per.map((p) => p.first)),
|
|
708
712
|
};
|
|
709
713
|
}
|
|
714
|
+
const repeatLen = (k, len) => (k === 0 ? 0 : k * len);
|
|
715
|
+
function readReachAlternatives(src, from) {
|
|
716
|
+
const alts = [];
|
|
717
|
+
let i = from;
|
|
718
|
+
for (;;) {
|
|
719
|
+
const one = readReachSequence(src, i);
|
|
720
|
+
alts.push(one.reading);
|
|
721
|
+
i = one.next;
|
|
722
|
+
if (src[i] !== "|")
|
|
723
|
+
break;
|
|
724
|
+
i += 1;
|
|
725
|
+
}
|
|
726
|
+
return { alts, next: i };
|
|
727
|
+
}
|
|
728
|
+
function readReachSequence(src, from) {
|
|
729
|
+
const fail = (at, why) => {
|
|
730
|
+
throw new Error(`redact: decision-reach derivation cannot read ${why} at ${at} in /${src}/`);
|
|
731
|
+
};
|
|
732
|
+
let pos = 0;
|
|
733
|
+
let reach = 0;
|
|
734
|
+
let beyond = false;
|
|
735
|
+
let wallReach = 0;
|
|
736
|
+
const goesBeyond = () => {
|
|
737
|
+
beyond = true;
|
|
738
|
+
wallReach = Math.max(wallReach, reach);
|
|
739
|
+
};
|
|
740
|
+
let i = from;
|
|
741
|
+
while (i < src.length) {
|
|
742
|
+
const c = src[i];
|
|
743
|
+
if (c === "|" || c === ")")
|
|
744
|
+
break;
|
|
745
|
+
let atom;
|
|
746
|
+
if (c === "(") {
|
|
747
|
+
const open = /^\((\?:|\?=|\?!|\?<=|\?<!)/.exec(src.slice(i));
|
|
748
|
+
if (open === null)
|
|
749
|
+
return fail(i, "a capturing or named group");
|
|
750
|
+
const mark = open[1];
|
|
751
|
+
const inner = readReachAlternatives(src, i + 1 + mark.length);
|
|
752
|
+
if (src[inner.next] !== ")")
|
|
753
|
+
return fail(i, "an unbalanced group");
|
|
754
|
+
i = inner.next + 1;
|
|
755
|
+
atom = {
|
|
756
|
+
len: Math.max(...inner.alts.map((a) => a.len)),
|
|
757
|
+
reach: Math.max(...inner.alts.map((a) => a.reach)),
|
|
758
|
+
beyond: inner.alts.some((a) => a.beyond),
|
|
759
|
+
wallReach: Math.max(0, ...inner.alts.filter((a) => a.beyond).map((a) => a.wallReach)),
|
|
760
|
+
};
|
|
761
|
+
if (mark === "?<=" || mark === "?<!")
|
|
762
|
+
continue;
|
|
763
|
+
if (mark === "?=" || mark === "?!") {
|
|
764
|
+
if (pos === Infinity)
|
|
765
|
+
goesBeyond();
|
|
766
|
+
else {
|
|
767
|
+
reach = Math.max(reach, pos + atom.reach);
|
|
768
|
+
if (atom.beyond) {
|
|
769
|
+
beyond = true;
|
|
770
|
+
wallReach = Math.max(wallReach, pos + atom.wallReach);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
continue;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
else if (c === "[") {
|
|
777
|
+
let k = i + 1;
|
|
778
|
+
while (k < src.length && src[k] !== "]")
|
|
779
|
+
k += src[k] === "\\" ? 2 : 1;
|
|
780
|
+
if (k >= src.length)
|
|
781
|
+
return fail(i, "an unterminated character class");
|
|
782
|
+
atom = { len: 1, reach: 1, beyond: false, wallReach: 0 };
|
|
783
|
+
i = k + 1;
|
|
784
|
+
}
|
|
785
|
+
else if (c === "\\") {
|
|
786
|
+
const e = src[i + 1];
|
|
787
|
+
if (e === undefined)
|
|
788
|
+
return fail(i, "a dangling escape");
|
|
789
|
+
if (e === "b" || e === "B") {
|
|
790
|
+
if (pos === Infinity)
|
|
791
|
+
goesBeyond();
|
|
792
|
+
else
|
|
793
|
+
reach = Math.max(reach, pos + 1);
|
|
794
|
+
i += 2;
|
|
795
|
+
continue;
|
|
796
|
+
}
|
|
797
|
+
if (/[1-9k]/.test(e))
|
|
798
|
+
return fail(i, "a back-reference");
|
|
799
|
+
const wide = /^\\(?:u[0-9A-Fa-f]{4}|u\{[0-9A-Fa-f]+\}|x[0-9A-Fa-f]{2}|[pP]\{[^}]+\}|c[A-Za-z]|.)/.exec(src.slice(i));
|
|
800
|
+
atom = { len: 1, reach: 1, beyond: false, wallReach: 0 };
|
|
801
|
+
i += wide[0].length;
|
|
802
|
+
}
|
|
803
|
+
else if (c === "^" || c === "$") {
|
|
804
|
+
return fail(i, "an anchor");
|
|
805
|
+
}
|
|
806
|
+
else if (c === "*" || c === "+" || c === "?" || c === "{") {
|
|
807
|
+
return fail(i, "a quantifier without an atom");
|
|
808
|
+
}
|
|
809
|
+
else {
|
|
810
|
+
atom = { len: 1, reach: 1, beyond: false, wallReach: 0 };
|
|
811
|
+
i += 1;
|
|
812
|
+
}
|
|
813
|
+
let lo = 1;
|
|
814
|
+
let hi = 1;
|
|
815
|
+
const q = src[i];
|
|
816
|
+
if (q === "*") {
|
|
817
|
+
lo = 0;
|
|
818
|
+
hi = Infinity;
|
|
819
|
+
i += 1;
|
|
820
|
+
}
|
|
821
|
+
else if (q === "+") {
|
|
822
|
+
lo = 1;
|
|
823
|
+
hi = Infinity;
|
|
824
|
+
i += 1;
|
|
825
|
+
}
|
|
826
|
+
else if (q === "?") {
|
|
827
|
+
lo = 0;
|
|
828
|
+
hi = 1;
|
|
829
|
+
i += 1;
|
|
830
|
+
}
|
|
831
|
+
else if (q === "{") {
|
|
832
|
+
const m = /^\{(\d+)(,(\d*))?\}/.exec(src.slice(i));
|
|
833
|
+
if (m === null)
|
|
834
|
+
return fail(i, "a quantifier");
|
|
835
|
+
lo = Number(m[1]);
|
|
836
|
+
hi = m[2] === undefined ? lo : m[3] === "" ? Infinity : Number(m[3]);
|
|
837
|
+
i += m[0].length;
|
|
838
|
+
}
|
|
839
|
+
if (src[i] === "?")
|
|
840
|
+
return fail(i, "a lazy quantifier");
|
|
841
|
+
if (pos === Infinity) {
|
|
842
|
+
if (lo > 0)
|
|
843
|
+
goesBeyond();
|
|
844
|
+
continue;
|
|
845
|
+
}
|
|
846
|
+
const counted = hi === Infinity ? lo : hi;
|
|
847
|
+
if (counted > 0) {
|
|
848
|
+
reach = Math.max(reach, pos + repeatLen(counted - 1, atom.len) + atom.reach);
|
|
849
|
+
if (atom.beyond) {
|
|
850
|
+
beyond = true;
|
|
851
|
+
wallReach = Math.max(wallReach, pos + repeatLen(counted - 1, atom.len) + atom.wallReach);
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
pos = hi === Infinity || atom.len === Infinity ? Infinity : pos + repeatLen(hi, atom.len);
|
|
855
|
+
}
|
|
856
|
+
return { reading: { len: pos, reach, beyond, wallReach }, next: i };
|
|
857
|
+
}
|
|
858
|
+
function regexDecisionReach(re, lexical) {
|
|
859
|
+
if (/[uv]/.test(re.flags))
|
|
860
|
+
throw new Error(`redact: decision-reach derivation does not support the u/v flags on /${re.source}/`);
|
|
861
|
+
const read = readReachAlternatives(re.source, 0);
|
|
862
|
+
if (read.next !== re.source.length)
|
|
863
|
+
throw new Error(`redact: decision-reach derivation stopped at ${read.next} in /${re.source}/`);
|
|
864
|
+
let reach = 0;
|
|
865
|
+
for (const alt of read.alts) {
|
|
866
|
+
if (alt.beyond && !lexical)
|
|
867
|
+
throw new Error(`redact: arm /${re.source}/ decides beyond an unbounded run but registers no undecided tail`);
|
|
868
|
+
reach = Math.max(reach, alt.reach, alt.beyond ? alt.wallReach : 0);
|
|
869
|
+
}
|
|
870
|
+
if (!Number.isFinite(reach))
|
|
871
|
+
throw new Error(`redact: decision-reach derivation read no finite reach for /${re.source}/`);
|
|
872
|
+
return reach;
|
|
873
|
+
}
|
|
710
874
|
function scanResumeAfterHit(start, end, back) {
|
|
711
875
|
return Math.max(start + 1, end - back);
|
|
712
876
|
}
|
|
@@ -715,7 +879,7 @@ function collectRegexSpans(a, text, out) {
|
|
|
715
879
|
g.lastIndex = 0;
|
|
716
880
|
for (let m = g.exec(text); m !== null; m = g.exec(text)) {
|
|
717
881
|
const end = m.index + m[0].length;
|
|
718
|
-
out.push({ start: m.index, end, repl: a.repl, rank: a.rank, order: a.order });
|
|
882
|
+
out.push({ start: m.index, end, repl: a.repl, rank: a.rank, order: a.order, anchor: m.index, closeAt: NOT_CLOSED });
|
|
719
883
|
g.lastIndex = scanResumeAfterHit(m.index, end, a.back);
|
|
720
884
|
}
|
|
721
885
|
}
|
|
@@ -792,7 +956,7 @@ function collectValueArmSpans(arm, text, out) {
|
|
|
792
956
|
}
|
|
793
957
|
else
|
|
794
958
|
break;
|
|
795
|
-
out.push({ start: idx, end, repl, rank, order });
|
|
959
|
+
out.push({ start: idx, end, repl, rank, order, anchor: idx, closeAt: NOT_CLOSED });
|
|
796
960
|
pos = scanResumeAfterHit(idx, end, back);
|
|
797
961
|
}
|
|
798
962
|
}
|
|
@@ -808,7 +972,7 @@ function collectStickySpans(a, text, at, out) {
|
|
|
808
972
|
if (m === null)
|
|
809
973
|
continue;
|
|
810
974
|
const end = p + m[0].length;
|
|
811
|
-
out.push({ start: p, end, repl, rank, order });
|
|
975
|
+
out.push({ start: p, end, repl, rank, order, anchor: p, closeAt: NOT_CLOSED });
|
|
812
976
|
pos = scanResumeAfterHit(p, end, back);
|
|
813
977
|
}
|
|
814
978
|
}
|
|
@@ -871,13 +1035,13 @@ function undecidedKeywordStart(text, t) {
|
|
|
871
1035
|
return -1;
|
|
872
1036
|
}
|
|
873
1037
|
const CREDENTIAL_NAME_ANCHORS = {
|
|
874
|
-
api_key: "
|
|
875
|
-
token: "structure-only", access_token: "
|
|
1038
|
+
api_key: "declarative", apikey: "declarative", "api-key": "declarative", key: "structure-only",
|
|
1039
|
+
token: "structure-only", access_token: "declarative", refresh_token: "declarative", id_token: "declarative",
|
|
876
1040
|
auth: "structure-only", authorization: "auth-scheme", "proxy-authorization": "auth-scheme",
|
|
877
1041
|
password: "declarative", passwd: "any-form", pwd: "any-form",
|
|
878
|
-
secret: "declarative", client_secret: "declarative", private_key: "
|
|
1042
|
+
secret: "declarative", client_secret: "declarative", private_key: "declarative",
|
|
879
1043
|
signature: "structure-only", sig: "structure-only",
|
|
880
|
-
"x-amz-signature": "
|
|
1044
|
+
"x-amz-signature": "declarative", "x-amz-security-token": "declarative", "x-amz-credential": "declarative",
|
|
881
1045
|
sas: "structure-only",
|
|
882
1046
|
};
|
|
883
1047
|
const CREDENTIAL_NAME_ALT = (() => {
|
|
@@ -921,6 +1085,8 @@ const STRUCTURAL_SECRET_PATTERNS = [
|
|
|
921
1085
|
[/:(?<=[A-Za-z][A-Za-z0-9+.-]*:)\/\//g, "$1«redacted»"],
|
|
922
1086
|
[new RegExp(buildJsonSecretFieldSource(), "gi"), '$1"«redacted»"'],
|
|
923
1087
|
];
|
|
1088
|
+
const PPK_FILE_HEAD = "PuTTY-User-Key-File-";
|
|
1089
|
+
const PPK_LINES_FIELD = "Private-Lines: ";
|
|
924
1090
|
const PPK_CONTAINER_TOKEN = "«redacted:private-key»";
|
|
925
1091
|
const PEM_FRAGMENT_TOKEN = "«redacted:private-key:partial»";
|
|
926
1092
|
const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
|
|
@@ -962,7 +1128,7 @@ function skipPemSeps(scan, s, i, limit) {
|
|
|
962
1128
|
for (;;) {
|
|
963
1129
|
if (i >= limit)
|
|
964
1130
|
return i;
|
|
965
|
-
if (
|
|
1131
|
+
if (tokenEnd(scan, i) >= 0)
|
|
966
1132
|
return i;
|
|
967
1133
|
const c = s.charCodeAt(i);
|
|
968
1134
|
if (isSepCode(c)) {
|
|
@@ -1006,16 +1172,16 @@ function pemInlineSepLen(s, i, limit) {
|
|
|
1006
1172
|
return 0;
|
|
1007
1173
|
}
|
|
1008
1174
|
function probePemHeaderToken(scan, s, p, limit) {
|
|
1009
|
-
if (p >= limit ||
|
|
1175
|
+
if (p >= limit || tokenEnd(scan, p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
|
|
1010
1176
|
return -1;
|
|
1011
1177
|
p += 1;
|
|
1012
1178
|
let identLen = 1;
|
|
1013
|
-
while (identLen < PEM_HEADER_IDENT_MAX && p < limit &&
|
|
1179
|
+
while (identLen < PEM_HEADER_IDENT_MAX && p < limit && tokenEnd(scan, p) < 0 && isIdentCode(s.charCodeAt(p))) {
|
|
1014
1180
|
p += 1;
|
|
1015
1181
|
identLen += 1;
|
|
1016
1182
|
}
|
|
1017
1183
|
for (;;) {
|
|
1018
|
-
if (p >= limit ||
|
|
1184
|
+
if (p >= limit || tokenEnd(scan, p) >= 0)
|
|
1019
1185
|
return -1;
|
|
1020
1186
|
const c = s.charCodeAt(p);
|
|
1021
1187
|
if (c === CH_SPACE || c === CH_TAB) {
|
|
@@ -1047,7 +1213,7 @@ function parsePemHeaderLine(scan, s, i, limit, allowBare, json) {
|
|
|
1047
1213
|
for (;;) {
|
|
1048
1214
|
if (p >= limit)
|
|
1049
1215
|
return setPemHdr(scan, start, p, p, true);
|
|
1050
|
-
const unit =
|
|
1216
|
+
const unit = crossEnd(scan, p);
|
|
1051
1217
|
if (unit >= 0) {
|
|
1052
1218
|
p = Math.min(unit, limit);
|
|
1053
1219
|
continue;
|
|
@@ -1077,7 +1243,7 @@ function parsePemHeaderLine(scan, s, i, limit, allowBare, json) {
|
|
|
1077
1243
|
}
|
|
1078
1244
|
let q = p;
|
|
1079
1245
|
for (;;) {
|
|
1080
|
-
if (
|
|
1246
|
+
if (tokenEnd(scan, q) >= 0)
|
|
1081
1247
|
break;
|
|
1082
1248
|
const b = pemLineBreakLen(s, q, limit);
|
|
1083
1249
|
if (b === 1)
|
|
@@ -1104,7 +1270,7 @@ function parsePemBody(scan, s, i, limit) {
|
|
|
1104
1270
|
for (;;) {
|
|
1105
1271
|
const q = skipPemSeps(scan, s, p, limit);
|
|
1106
1272
|
let r = q;
|
|
1107
|
-
while (r < limit &&
|
|
1273
|
+
while (r < limit && tokenEnd(scan, r) < 0 && isBase64Code(s.charCodeAt(r)))
|
|
1108
1274
|
r += 1;
|
|
1109
1275
|
if (r - q < PEM_BODY_RUN_MIN)
|
|
1110
1276
|
break;
|
|
@@ -1113,20 +1279,34 @@ function parsePemBody(scan, s, i, limit) {
|
|
|
1113
1279
|
}
|
|
1114
1280
|
return end;
|
|
1115
1281
|
}
|
|
1116
|
-
function rightmostPemRunEnd(scan, s, from, to) {
|
|
1282
|
+
function rightmostPemRunEnd(scan, s, from, to, crossEndsRun) {
|
|
1117
1283
|
let best = -1;
|
|
1118
1284
|
let runStart = -1;
|
|
1119
1285
|
let plainBackslashTill = -1;
|
|
1120
1286
|
for (let p = from; p <= to; p++) {
|
|
1121
1287
|
if (p < to) {
|
|
1122
|
-
const
|
|
1123
|
-
if (
|
|
1288
|
+
const tok = tokenEnd(scan, p);
|
|
1289
|
+
if (tok > 0) {
|
|
1124
1290
|
if (runStart >= 0 && p - runStart >= PEM_BODY_RUN_MIN)
|
|
1125
1291
|
best = p;
|
|
1126
1292
|
runStart = -1;
|
|
1127
|
-
if (
|
|
1293
|
+
if (tok > to)
|
|
1128
1294
|
return best;
|
|
1129
|
-
p =
|
|
1295
|
+
p = tok - 1;
|
|
1296
|
+
continue;
|
|
1297
|
+
}
|
|
1298
|
+
const cross = crossEnd(scan, p);
|
|
1299
|
+
if (cross > 0) {
|
|
1300
|
+
if (crossEndsRun) {
|
|
1301
|
+
if (runStart >= 0 && p - runStart >= PEM_BODY_RUN_MIN)
|
|
1302
|
+
best = p;
|
|
1303
|
+
runStart = -1;
|
|
1304
|
+
if (cross >= to)
|
|
1305
|
+
return best;
|
|
1306
|
+
}
|
|
1307
|
+
else if (cross >= to)
|
|
1308
|
+
return runStart >= 0 && to - runStart >= PEM_BODY_RUN_MIN ? to : best;
|
|
1309
|
+
p = cross - 1;
|
|
1130
1310
|
continue;
|
|
1131
1311
|
}
|
|
1132
1312
|
if (p >= plainBackslashTill && s.charCodeAt(p) === CH_BACKSLASH) {
|
|
@@ -1193,7 +1373,7 @@ function findPemFence(scan, text, from) {
|
|
|
1193
1373
|
if (!isPemFenceLabel(text, labelFrom, close))
|
|
1194
1374
|
continue;
|
|
1195
1375
|
const after = close + PEM_FENCE_CLOSE.length;
|
|
1196
|
-
if (
|
|
1376
|
+
if (lineHeld(scan, fence, after, readingKey(familyCode("pem-armor"), fence)))
|
|
1197
1377
|
continue;
|
|
1198
1378
|
return { fence, after };
|
|
1199
1379
|
}
|
|
@@ -1201,131 +1381,112 @@ function findPemFence(scan, text, from) {
|
|
|
1201
1381
|
const SSH2_FENCE_OPEN = "---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
1202
1382
|
const SSH2_FENCE_CLOSE = "---- END SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
1203
1383
|
const SSH2_BODY_MAX = 8192;
|
|
1204
|
-
function
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
const
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
const mid = (a + b) >>> 1;
|
|
1214
|
-
if (t.ends[mid] <= to)
|
|
1215
|
-
a = mid + 1;
|
|
1216
|
-
else
|
|
1217
|
-
b = mid;
|
|
1218
|
-
}
|
|
1219
|
-
if (a === lo)
|
|
1220
|
-
return false;
|
|
1221
|
-
const saved = unitSavings(t);
|
|
1222
|
-
return to - from - (saved[a] - saved[lo]) <= max;
|
|
1384
|
+
function armorCloseAt(scan, text, kind, bodyFrom, max) {
|
|
1385
|
+
const near = firstDelim(scan, text, kind, bodyFrom, false);
|
|
1386
|
+
if (near < 0)
|
|
1387
|
+
return -1;
|
|
1388
|
+
const free = firstDelim(scan, text, kind, bodyFrom, true);
|
|
1389
|
+
const far = free >= 0 ? free : lastNotHiddenDelim(scan, text, kind, near, text.length);
|
|
1390
|
+
if (far > near && blockWithin(scan, text, kind, bodyFrom, far, max))
|
|
1391
|
+
return far;
|
|
1392
|
+
return blockWithin(scan, text, kind, bodyFrom, near, max) ? near : -1;
|
|
1223
1393
|
}
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
let saved = UNIT_SAVINGS.get(t);
|
|
1227
|
-
if (saved === undefined) {
|
|
1228
|
-
const n = t.starts.length;
|
|
1229
|
-
saved = new Float64Array(n + 1);
|
|
1230
|
-
for (let k = 0; k < n; k++)
|
|
1231
|
-
saved[k + 1] = saved[k] + (t.ends[k] - t.starts[k] - 1);
|
|
1232
|
-
UNIT_SAVINGS.set(t, saved);
|
|
1233
|
-
}
|
|
1234
|
-
return saved;
|
|
1394
|
+
function closeLineEnd(text, kind, close) {
|
|
1395
|
+
return kind === "ssh2-close" ? close + SSH2_FENCE_CLOSE.length : pemFenceLineEnd(text, close + PEM_END_OPEN.length);
|
|
1235
1396
|
}
|
|
1236
|
-
|
|
1237
|
-
const
|
|
1238
|
-
return
|
|
1397
|
+
function blockWithin(scan, text, kind, bodyFrom, close, max) {
|
|
1398
|
+
const lineEnd = closeLineEnd(text, kind, close);
|
|
1399
|
+
return measureWithin(scan, bodyFrom, lineEnd, max + (lineEnd - close));
|
|
1239
1400
|
}
|
|
1240
1401
|
function collectSsh2FenceSpans(scan, text, json, span, isHead) {
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
const
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1402
|
+
const fam = familyCode("ssh2-armor");
|
|
1403
|
+
const opens = delimPositions(scan, text, "ssh2-open");
|
|
1404
|
+
const closes = delimPositions(scan, text, "ssh2-close");
|
|
1405
|
+
const blockEnds = [];
|
|
1406
|
+
let firstLive = 0;
|
|
1407
|
+
let freshFrom = -1;
|
|
1408
|
+
let freshEnd = -1;
|
|
1409
|
+
for (let oi = 0, ci = 0, rooted = false, prevEnd = -1; ci < closes.length;) {
|
|
1410
|
+
const c = closes[ci];
|
|
1411
|
+
if (oi < opens.length && opens[oi] < c) {
|
|
1412
|
+
rooted = true;
|
|
1413
|
+
prevEnd = -1;
|
|
1414
|
+
oi += 1;
|
|
1415
|
+
continue;
|
|
1416
|
+
}
|
|
1417
|
+
if (rooted && prevEnd >= 0) {
|
|
1418
|
+
scan.self = readingKey(fam, c);
|
|
1419
|
+
while (firstLive < blockEnds.length && uncoveredWithin(scan, blockEnds[firstLive], c + SSH2_FENCE_CLOSE.length) > SSH2_BODY_MAX + SSH2_FENCE_CLOSE.length)
|
|
1420
|
+
firstLive += 1;
|
|
1421
|
+
for (let j = firstLive; j < blockEnds.length; j++) {
|
|
1422
|
+
const from = blockEnds[j];
|
|
1423
|
+
if (from > c)
|
|
1424
|
+
break;
|
|
1425
|
+
if ((from === freshFrom && blockWithin(scan, text, "ssh2-close", freshEnd, c, SSH2_BODY_MAX - 1)) || blockWithin(scan, text, "ssh2-close", from, c, SSH2_BODY_MAX)) {
|
|
1426
|
+
span(from, c + SSH2_FENCE_CLOSE.length, c, NOT_CLOSED);
|
|
1427
|
+
freshFrom = from;
|
|
1428
|
+
freshEnd = c + SSH2_FENCE_CLOSE.length;
|
|
1429
|
+
break;
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
prevEnd = c + SSH2_FENCE_CLOSE.length;
|
|
1434
|
+
if (rooted)
|
|
1435
|
+
blockEnds.push(prevEnd);
|
|
1436
|
+
ci += 1;
|
|
1437
|
+
}
|
|
1274
1438
|
let openFrom = 0;
|
|
1275
1439
|
let openAt;
|
|
1276
1440
|
const nextFreeOpen = (from) => {
|
|
1277
1441
|
if (openAt !== undefined && openFrom <= from && (openAt < 0 || openAt >= from))
|
|
1278
1442
|
return openAt;
|
|
1279
|
-
let
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
const q = text.indexOf(SSH2_FENCE_OPEN, p + 1);
|
|
1283
|
-
ssh2ScanBytes += (q < 0 ? text.length : q) - (p + 1);
|
|
1284
|
-
p = q;
|
|
1285
|
-
}
|
|
1443
|
+
let k = lowerBound(opens, from);
|
|
1444
|
+
while (k < opens.length && lineHeld(scan, opens[k], opens[k] + SSH2_FENCE_OPEN.length, readingKey(fam, opens[k])))
|
|
1445
|
+
k += 1;
|
|
1286
1446
|
openFrom = from;
|
|
1287
|
-
openAt =
|
|
1288
|
-
return
|
|
1447
|
+
openAt = k < opens.length ? opens[k] : -1;
|
|
1448
|
+
return openAt;
|
|
1289
1449
|
};
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
return;
|
|
1450
|
+
const truncatedEnd = createTruncatedBlockEnd(scan, text, json, (from) => firstDelim(scan, text, "ssh2-close", from, true));
|
|
1451
|
+
for (let oi = 0; oi < opens.length;) {
|
|
1452
|
+
const fence = opens[oi];
|
|
1294
1453
|
const bodyFrom = fence + SSH2_FENCE_OPEN.length;
|
|
1295
|
-
if (
|
|
1296
|
-
|
|
1454
|
+
if (lineHeld(scan, fence, bodyFrom, readingKey(fam, fence))) {
|
|
1455
|
+
oi += 1;
|
|
1297
1456
|
continue;
|
|
1298
1457
|
}
|
|
1299
|
-
|
|
1300
|
-
|
|
1458
|
+
scan.self = readingKey(fam, fence);
|
|
1459
|
+
if (isHead && measureWithin(scan, bodyFrom, text.length, SSH2_BODY_MAX)) {
|
|
1460
|
+
span(fence, text.length, fence, NOT_CLOSED, true);
|
|
1301
1461
|
return;
|
|
1302
1462
|
}
|
|
1303
1463
|
let resume = fence + 1;
|
|
1304
1464
|
if (!isHead) {
|
|
1305
1465
|
const open = nextFreeOpen(bodyFrom);
|
|
1306
1466
|
const limit = open < 0 ? text.length : open;
|
|
1307
|
-
const
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1467
|
+
const readEnd = (crossEndsRun) => {
|
|
1468
|
+
const t = truncatedEnd(bodyFrom, limit, crossEndsRun);
|
|
1469
|
+
const bodyEnd = t < 0 ? -1 : armorBodyEnd(scan, text, json, bodyFrom, limit, false, crossEndsRun);
|
|
1470
|
+
return bodyEnd < 0 ? -1 : Math.max(bodyEnd, t);
|
|
1471
|
+
};
|
|
1472
|
+
const far = readEnd(false);
|
|
1473
|
+
const near = far >= 0 && !measureWithin(scan, bodyFrom, far, SSH2_BODY_MAX) ? readEnd(true) : -1;
|
|
1474
|
+
const end = near >= 0 && near < far ? near : far;
|
|
1475
|
+
if (end >= 0 && measureWithin(scan, bodyFrom, end, SSH2_BODY_MAX)) {
|
|
1476
|
+
span(fence, end, fence, NOT_CLOSED);
|
|
1312
1477
|
resume = end;
|
|
1313
1478
|
}
|
|
1314
1479
|
}
|
|
1315
|
-
const close =
|
|
1480
|
+
const close = armorCloseAt(scan, text, "ssh2-close", bodyFrom, SSH2_BODY_MAX);
|
|
1316
1481
|
if (close < 0) {
|
|
1317
|
-
if (isHead)
|
|
1482
|
+
if (isHead && firstDelim(scan, text, "ssh2-close", bodyFrom, false) < 0)
|
|
1318
1483
|
return;
|
|
1319
|
-
|
|
1320
|
-
continue;
|
|
1321
|
-
}
|
|
1322
|
-
if (!unitCountWithin(scan, bodyFrom, close, SSH2_BODY_MAX)) {
|
|
1323
|
-
from = resume;
|
|
1484
|
+
oi = lowerBound(opens, resume);
|
|
1324
1485
|
continue;
|
|
1325
1486
|
}
|
|
1326
1487
|
const end = close + SSH2_FENCE_CLOSE.length;
|
|
1327
|
-
span(fence, end);
|
|
1328
|
-
|
|
1488
|
+
span(fence, end, fence, close);
|
|
1489
|
+
oi = lowerBound(opens, delimClass(scan, close) === DELIM_FREE ? Math.max(resume, scanResumeAfterHit(fence, end, SSH2_FENCE_OPEN.length)) : resume);
|
|
1329
1490
|
}
|
|
1330
1491
|
}
|
|
1331
1492
|
const PEM_END_OPEN = "-----END";
|
|
@@ -1344,24 +1505,7 @@ function pemFenceLineEnd(text, labelFrom) {
|
|
|
1344
1505
|
return isPemFenceLabel(text, labelFrom, close) ? close + PEM_FENCE_CLOSE.length : PEM_LINE_INVALID;
|
|
1345
1506
|
}
|
|
1346
1507
|
function collectPemFenceSpans(scan, text, span, isHead) {
|
|
1347
|
-
|
|
1348
|
-
let endAt = -1;
|
|
1349
|
-
let endAfter = -1;
|
|
1350
|
-
const visibleEndLine = (b) => {
|
|
1351
|
-
const after = pemFenceLineEnd(text, b + PEM_END_OPEN.length);
|
|
1352
|
-
return after >= 0 && !inOpaqueUnit(scan, b) ? after : -1;
|
|
1353
|
-
};
|
|
1354
|
-
const earliestEndFence = (from) => {
|
|
1355
|
-
if (endFrom >= 0 && from >= endFrom && (endAt < 0 || endAt >= from))
|
|
1356
|
-
return;
|
|
1357
|
-
let b = text.indexOf(PEM_END_OPEN, from);
|
|
1358
|
-
let after = -1;
|
|
1359
|
-
while (b >= 0 && (after = visibleEndLine(b)) < 0)
|
|
1360
|
-
b = text.indexOf(PEM_END_OPEN, b + 1);
|
|
1361
|
-
endFrom = from;
|
|
1362
|
-
endAt = b;
|
|
1363
|
-
endAfter = after;
|
|
1364
|
-
};
|
|
1508
|
+
const fam = familyCode("pem-armor");
|
|
1365
1509
|
for (let from = 0;;) {
|
|
1366
1510
|
const fence = text.indexOf(PEM_FENCE_OPEN, from);
|
|
1367
1511
|
if (fence < 0)
|
|
@@ -1371,23 +1515,25 @@ function collectPemFenceSpans(scan, text, span, isHead) {
|
|
|
1371
1515
|
from = fence + 1;
|
|
1372
1516
|
continue;
|
|
1373
1517
|
}
|
|
1374
|
-
if (
|
|
1518
|
+
if (lineHeld(scan, fence, bodyFrom === PEM_LINE_OPEN ? text.length : bodyFrom, readingKey(fam, fence))) {
|
|
1375
1519
|
from = fence + 1;
|
|
1376
1520
|
continue;
|
|
1377
1521
|
}
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
if (
|
|
1522
|
+
scan.self = readingKey(fam, fence);
|
|
1523
|
+
const near = bodyFrom === PEM_LINE_OPEN ? -1 : firstDelim(scan, text, "pem-end", bodyFrom, false);
|
|
1524
|
+
if (near < 0 || (isHead && firstDelim(scan, text, "pem-end", bodyFrom, true) < 0)) {
|
|
1381
1525
|
if (isHead)
|
|
1382
|
-
span(fence, text.length, true);
|
|
1526
|
+
span(fence, text.length, fence, NOT_CLOSED, true);
|
|
1383
1527
|
return;
|
|
1384
1528
|
}
|
|
1385
|
-
|
|
1529
|
+
const endAt = armorCloseAt(scan, text, "pem-end", bodyFrom, PEM_FENCE_BODY_MAX);
|
|
1530
|
+
if (endAt < 0) {
|
|
1386
1531
|
from = fence + 1;
|
|
1387
1532
|
continue;
|
|
1388
1533
|
}
|
|
1389
|
-
|
|
1390
|
-
|
|
1534
|
+
const endAfter = pemFenceLineEnd(text, endAt + PEM_END_OPEN.length);
|
|
1535
|
+
span(fence, endAfter, fence, endAt);
|
|
1536
|
+
from = delimClass(scan, endAt) === DELIM_FREE ? scanResumeAfterHit(fence, endAfter, PEM_FENCE_OPEN.length) : fence + 1;
|
|
1391
1537
|
}
|
|
1392
1538
|
}
|
|
1393
1539
|
function truncationWindowEnd(text, from, json) {
|
|
@@ -1404,13 +1550,13 @@ function truncationWindowEnd(text, from, json) {
|
|
|
1404
1550
|
}
|
|
1405
1551
|
return text.length;
|
|
1406
1552
|
}
|
|
1407
|
-
function armorBodyEnd(scan, text, json, from, limit, allowBare) {
|
|
1553
|
+
function armorBodyEnd(scan, text, json, from, limit, allowBare, crossEndsRun) {
|
|
1408
1554
|
let pos = from;
|
|
1409
1555
|
let bare = allowBare;
|
|
1410
1556
|
let hitLimit = false;
|
|
1411
1557
|
let rightmost = -1;
|
|
1412
1558
|
while (parsePemHeaderLine(scan, text, pos, limit, bare, json)) {
|
|
1413
|
-
const e = rightmostPemRunEnd(scan, text, scan.hdr.search, scan.hdr.contentEnd);
|
|
1559
|
+
const e = rightmostPemRunEnd(scan, text, scan.hdr.search, scan.hdr.contentEnd, crossEndsRun);
|
|
1414
1560
|
if (e > rightmost)
|
|
1415
1561
|
rightmost = e;
|
|
1416
1562
|
pos = scan.hdr.next;
|
|
@@ -1429,7 +1575,7 @@ function armorBodyEnd(scan, text, json, from, limit, allowBare) {
|
|
|
1429
1575
|
function createTruncatedBlockEnd(scan, text, json, closeAt) {
|
|
1430
1576
|
let winFrom = -1;
|
|
1431
1577
|
let winAt = -1;
|
|
1432
|
-
return (from, limit) => {
|
|
1578
|
+
return (from, limit, crossEndsRun) => {
|
|
1433
1579
|
if (!(winFrom >= 0 && from >= winFrom && from <= winAt)) {
|
|
1434
1580
|
winFrom = from;
|
|
1435
1581
|
winAt = truncationWindowEnd(text, from, json);
|
|
@@ -1437,38 +1583,28 @@ function createTruncatedBlockEnd(scan, text, json, closeAt) {
|
|
|
1437
1583
|
const close = closeAt(from);
|
|
1438
1584
|
if (close >= 0 && close < winAt)
|
|
1439
1585
|
return -1;
|
|
1440
|
-
return rightmostPemRunEnd(scan, text, from, Math.min(limit, winAt));
|
|
1586
|
+
return rightmostPemRunEnd(scan, text, from, Math.min(limit, winAt), crossEndsRun);
|
|
1441
1587
|
};
|
|
1442
1588
|
}
|
|
1443
1589
|
function collectPemFragmentSpans(scan, text, json, span, isHead) {
|
|
1590
|
+
const fam = familyCode("pem-armor");
|
|
1444
1591
|
let cur = findPemFence(scan, text, 0);
|
|
1445
1592
|
let inherited = false;
|
|
1446
|
-
|
|
1447
|
-
let closeAt = -1;
|
|
1448
|
-
const freeClose = (from) => {
|
|
1449
|
-
if (closeFrom >= 0 && from >= closeFrom && (closeAt < 0 || closeAt >= from))
|
|
1450
|
-
return closeAt;
|
|
1451
|
-
let b = text.indexOf(PEM_END_OPEN, from);
|
|
1452
|
-
while (b >= 0 && (pemFenceLineEnd(text, b + PEM_END_OPEN.length) < 0 || anchorHeld(scan, b, b + 1)))
|
|
1453
|
-
b = text.indexOf(PEM_END_OPEN, b + 1);
|
|
1454
|
-
closeFrom = from;
|
|
1455
|
-
closeAt = b;
|
|
1456
|
-
return b;
|
|
1457
|
-
};
|
|
1458
|
-
const truncatedEnd = createTruncatedBlockEnd(scan, text, json, freeClose);
|
|
1593
|
+
const truncatedEnd = createTruncatedBlockEnd(scan, text, json, (from) => firstDelim(scan, text, "pem-end", from, true));
|
|
1459
1594
|
while (cur !== undefined) {
|
|
1460
1595
|
const next = findPemFence(scan, text, cur.after);
|
|
1461
1596
|
const limit = next === undefined ? text.length : next.fence;
|
|
1462
|
-
|
|
1597
|
+
scan.self = readingKey(fam, cur.fence);
|
|
1598
|
+
const bodyEnd = armorBodyEnd(scan, text, json, cur.after, limit, inherited, false);
|
|
1463
1599
|
inherited = scan.hdr.hitLimit;
|
|
1464
|
-
const end = isHead || bodyEnd < 0 ? bodyEnd : Math.max(bodyEnd, truncatedEnd(cur.after, limit));
|
|
1600
|
+
const end = isHead || bodyEnd < 0 ? bodyEnd : Math.max(bodyEnd, truncatedEnd(cur.after, limit, false));
|
|
1465
1601
|
if (end >= 0)
|
|
1466
|
-
span(cur.fence, end);
|
|
1602
|
+
span(cur.fence, end, cur.fence, NOT_CLOSED);
|
|
1467
1603
|
cur = next;
|
|
1468
1604
|
}
|
|
1469
1605
|
}
|
|
1470
1606
|
function collectHeadlessPpkSpans(scan, text, span, isHead) {
|
|
1471
|
-
const KEY =
|
|
1607
|
+
const KEY = PPK_LINES_FIELD;
|
|
1472
1608
|
const n = text.length;
|
|
1473
1609
|
const LINE_NONE = -1;
|
|
1474
1610
|
const LINE_OPEN = -2;
|
|
@@ -1476,19 +1612,20 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
|
|
|
1476
1612
|
if (!isHead)
|
|
1477
1613
|
return false;
|
|
1478
1614
|
let k = i;
|
|
1479
|
-
while (k < n &&
|
|
1615
|
+
while (k < n && tokenEnd(scan, k) < 0 && text.charCodeAt(k) === 92)
|
|
1480
1616
|
k += 1;
|
|
1481
1617
|
return k >= n;
|
|
1482
1618
|
};
|
|
1483
1619
|
const isB64 = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 43 || c === 47 || c === 61;
|
|
1484
|
-
const b64At = (i) =>
|
|
1485
|
-
const skipHs = (i) => { while (i < n &&
|
|
1620
|
+
const b64At = (i) => tokenEnd(scan, i) < 0 && isB64(text.charCodeAt(i));
|
|
1621
|
+
const skipHs = (i) => { while (i < n && tokenEnd(scan, i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
|
|
1486
1622
|
i += 1; return i; };
|
|
1623
|
+
const lineEndHidden = (i) => i < n && delimClass(scan, i) === DELIM_HIDDEN;
|
|
1487
1624
|
const skipNewlines = (i) => {
|
|
1488
1625
|
let q = skipHs(i);
|
|
1489
1626
|
let seen = false;
|
|
1490
1627
|
for (;;) {
|
|
1491
|
-
if (
|
|
1628
|
+
if (lineEndHidden(q))
|
|
1492
1629
|
break;
|
|
1493
1630
|
let any = false;
|
|
1494
1631
|
if (text.charCodeAt(q) === 13) {
|
|
@@ -1523,7 +1660,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
|
|
|
1523
1660
|
return seen ? q : LINE_NONE;
|
|
1524
1661
|
};
|
|
1525
1662
|
const escapedNewlineLen = (i) => {
|
|
1526
|
-
if (
|
|
1663
|
+
if (lineEndHidden(i))
|
|
1527
1664
|
return 0;
|
|
1528
1665
|
let k = i;
|
|
1529
1666
|
while (k < n && text.charCodeAt(k) === 92)
|
|
@@ -1537,7 +1674,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
|
|
|
1537
1674
|
const b = skipHs(i);
|
|
1538
1675
|
if (b >= n || openAt(b))
|
|
1539
1676
|
return true;
|
|
1540
|
-
if (
|
|
1677
|
+
if (tokenEnd(scan, b) >= 0 || lineEndHidden(b))
|
|
1541
1678
|
return false;
|
|
1542
1679
|
const c = text.charCodeAt(b);
|
|
1543
1680
|
return c === 10 || c === 13 || c === 34 || escapedNewlineLen(b) > 0;
|
|
@@ -1547,7 +1684,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
|
|
|
1547
1684
|
if (q0 < 0)
|
|
1548
1685
|
return q0;
|
|
1549
1686
|
let q = q0;
|
|
1550
|
-
const mk =
|
|
1687
|
+
const mk = tokenEnd(scan, q) < 0 ? text.charCodeAt(q) : -1;
|
|
1551
1688
|
if (mk === 43 || mk === 45)
|
|
1552
1689
|
q = skipHs(q + 1);
|
|
1553
1690
|
let r = q;
|
|
@@ -1571,16 +1708,17 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
|
|
|
1571
1708
|
break;
|
|
1572
1709
|
let p = at + KEY.length;
|
|
1573
1710
|
const d0 = p;
|
|
1574
|
-
while (p < n &&
|
|
1711
|
+
while (p < n && tokenEnd(scan, p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
|
|
1575
1712
|
p += 1;
|
|
1576
1713
|
if (p === d0 && p < n) {
|
|
1577
1714
|
search = at + KEY.length;
|
|
1578
1715
|
continue;
|
|
1579
1716
|
}
|
|
1580
|
-
if (
|
|
1717
|
+
if (lineHeld(scan, at, p, readingKey(familyCode("putty-ppk"), at))) {
|
|
1581
1718
|
search = at + KEY.length;
|
|
1582
1719
|
continue;
|
|
1583
1720
|
}
|
|
1721
|
+
scan.self = readingKey(familyCode("putty-ppk"), at);
|
|
1584
1722
|
let end = -1;
|
|
1585
1723
|
let open = false;
|
|
1586
1724
|
for (let from = p;;) {
|
|
@@ -1598,7 +1736,7 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
|
|
|
1598
1736
|
search = p;
|
|
1599
1737
|
continue;
|
|
1600
1738
|
}
|
|
1601
|
-
span(at, end, open);
|
|
1739
|
+
span(at, end, at, NOT_CLOSED, open);
|
|
1602
1740
|
search = end;
|
|
1603
1741
|
}
|
|
1604
1742
|
}
|
|
@@ -1735,8 +1873,8 @@ function tokenUnitEnd(text, p) {
|
|
|
1735
1873
|
TOKEN_UNIT_Y.lastIndex = p;
|
|
1736
1874
|
return TOKEN_UNIT_Y.test(text) ? TOKEN_UNIT_Y.lastIndex : -1;
|
|
1737
1875
|
}
|
|
1738
|
-
const
|
|
1739
|
-
function
|
|
1876
|
+
const NO_TOKEN_UNITS = { starts: new Int32Array(0), ends: new Int32Array(0) };
|
|
1877
|
+
function tokenUnits(text) {
|
|
1740
1878
|
const iv = [];
|
|
1741
1879
|
for (let i = text.indexOf("«"); i >= 0; i = text.indexOf("«", i + 1)) {
|
|
1742
1880
|
const e = tokenUnitEnd(text, i);
|
|
@@ -1746,8 +1884,7 @@ function credentialSpans(text) {
|
|
|
1746
1884
|
}
|
|
1747
1885
|
}
|
|
1748
1886
|
if (iv.length === 0)
|
|
1749
|
-
return
|
|
1750
|
-
iv.sort((a, b) => a[0] - b[0]);
|
|
1887
|
+
return NO_TOKEN_UNITS;
|
|
1751
1888
|
const starts = new Int32Array(iv.length);
|
|
1752
1889
|
const ends = new Int32Array(iv.length);
|
|
1753
1890
|
let k = 0;
|
|
@@ -1769,133 +1906,442 @@ function credentialSpans(text) {
|
|
|
1769
1906
|
k += 1;
|
|
1770
1907
|
return { starts: starts.subarray(0, k), ends: ends.subarray(0, k) };
|
|
1771
1908
|
}
|
|
1909
|
+
function familyCode(f) {
|
|
1910
|
+
switch (f) {
|
|
1911
|
+
case "token": return 0;
|
|
1912
|
+
case "pem-armor": return 1;
|
|
1913
|
+
case "ssh2-armor": return 2;
|
|
1914
|
+
case "putty-ppk": return 3;
|
|
1915
|
+
case "url": return 4;
|
|
1916
|
+
case "json-field": return 5;
|
|
1917
|
+
case "anchor": return 6;
|
|
1918
|
+
case "prose": return 7;
|
|
1919
|
+
}
|
|
1920
|
+
}
|
|
1921
|
+
const FAMILY_SLOTS = 8;
|
|
1922
|
+
const TOKEN_FAMILY = familyCode("token");
|
|
1923
|
+
const readingKey = (family, anchor) => anchor * FAMILY_SLOTS + family;
|
|
1924
|
+
const NOT_CLOSED = -1;
|
|
1925
|
+
const NO_SELF = -1;
|
|
1926
|
+
const NO_ROWS = { starts: new Int32Array(0), ends: new Int32Array(0), family: new Uint8Array(0), anchor: new Int32Array(0), closeAt: new Int32Array(0) };
|
|
1927
|
+
function buildEvidenceRows(text) {
|
|
1928
|
+
const s = [];
|
|
1929
|
+
const e = [];
|
|
1930
|
+
for (let i = text.indexOf("«"); i >= 0; i = text.indexOf("«", i + 1)) {
|
|
1931
|
+
const end = tokenUnitEnd(text, i);
|
|
1932
|
+
if (end > 0) {
|
|
1933
|
+
s.push(i);
|
|
1934
|
+
e.push(end);
|
|
1935
|
+
i = end - 1;
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
if (s.length === 0)
|
|
1939
|
+
return NO_ROWS;
|
|
1940
|
+
const n = s.length;
|
|
1941
|
+
const family = new Uint8Array(n).fill(TOKEN_FAMILY);
|
|
1942
|
+
const ends = Int32Array.from(e);
|
|
1943
|
+
return { starts: Int32Array.from(s), ends, family, anchor: Int32Array.from(s), closeAt: ends.map((x) => x - 1) };
|
|
1944
|
+
}
|
|
1945
|
+
const NO_DIRTY = { starts: new Int32Array(0), ends: new Int32Array(0) };
|
|
1772
1946
|
let forensicRescanMode = "incremental";
|
|
1773
1947
|
export function setForensicRescanModeForTest(mode) {
|
|
1774
1948
|
forensicRescanMode = mode;
|
|
1775
1949
|
}
|
|
1776
|
-
|
|
1950
|
+
const FORENSIC_LEVEL_MAX = 32;
|
|
1951
|
+
function compareRow(s1, f1, a1, c1, s2, f2, a2, c2) {
|
|
1952
|
+
return s1 - s2 || f1 - f2 || a1 - a2 || (c1 === c2 ? 0 : c1 ? 1 : -1);
|
|
1953
|
+
}
|
|
1954
|
+
function foldEvidenceRows(prev, spans, from, familyOf) {
|
|
1777
1955
|
const cap = spans.length - from;
|
|
1778
1956
|
if (cap === 0)
|
|
1779
|
-
return { next: prev, dirty:
|
|
1780
|
-
const
|
|
1781
|
-
const
|
|
1782
|
-
let
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
const
|
|
1797
|
-
const
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
let
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1957
|
+
return { next: prev, dirty: NO_DIRTY };
|
|
1958
|
+
const idx = Array.from({ length: cap }, (_, i) => from + i);
|
|
1959
|
+
const fam = new Uint8Array(spans.length);
|
|
1960
|
+
for (let i = from; i < spans.length; i++)
|
|
1961
|
+
fam[i] = familyOf(spans[i].order);
|
|
1962
|
+
idx.sort((x, y) => {
|
|
1963
|
+
const a = spans[x];
|
|
1964
|
+
const b = spans[y];
|
|
1965
|
+
return compareRow(a.start, fam[x], a.anchor, a.closeAt >= 0, b.start, fam[y], b.anchor, b.closeAt >= 0) || b.end - a.end || b.closeAt - a.closeAt;
|
|
1966
|
+
});
|
|
1967
|
+
const P = prev.starts.length;
|
|
1968
|
+
const total = P + cap;
|
|
1969
|
+
const oS = new Int32Array(total);
|
|
1970
|
+
const oE = new Int32Array(total);
|
|
1971
|
+
const oF = new Uint8Array(total);
|
|
1972
|
+
const oA = new Int32Array(total);
|
|
1973
|
+
const oC = new Int32Array(total);
|
|
1974
|
+
const dS = [];
|
|
1975
|
+
const dE = [];
|
|
1976
|
+
let on = 0;
|
|
1977
|
+
let pi = 0;
|
|
1978
|
+
let ni = 0;
|
|
1979
|
+
const push = (s, e, f, a, c) => {
|
|
1980
|
+
oS[on] = s;
|
|
1981
|
+
oE[on] = e;
|
|
1982
|
+
oF[on] = f;
|
|
1983
|
+
oA[on] = a;
|
|
1984
|
+
oC[on] = c;
|
|
1985
|
+
on += 1;
|
|
1986
|
+
};
|
|
1987
|
+
while (pi < P || ni < cap) {
|
|
1988
|
+
let take;
|
|
1989
|
+
let sp;
|
|
1990
|
+
let nf = 0;
|
|
1991
|
+
if (ni < cap) {
|
|
1992
|
+
sp = spans[idx[ni]];
|
|
1993
|
+
nf = fam[idx[ni]];
|
|
1807
1994
|
}
|
|
1995
|
+
if (pi >= P)
|
|
1996
|
+
take = "new";
|
|
1997
|
+
else if (sp === undefined)
|
|
1998
|
+
take = "prev";
|
|
1808
1999
|
else {
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
2000
|
+
const c = compareRow(prev.starts[pi], prev.family[pi], prev.anchor[pi], prev.closeAt[pi] >= 0, sp.start, nf, sp.anchor, sp.closeAt >= 0);
|
|
2001
|
+
take = c < 0 ? "prev" : c > 0 ? "new" : "both";
|
|
2002
|
+
}
|
|
2003
|
+
if (take === "prev") {
|
|
2004
|
+
push(prev.starts[pi], prev.ends[pi], prev.family[pi], prev.anchor[pi], prev.closeAt[pi]);
|
|
2005
|
+
pi += 1;
|
|
2006
|
+
continue;
|
|
2007
|
+
}
|
|
2008
|
+
const s = sp.start;
|
|
2009
|
+
let e = sp.end;
|
|
2010
|
+
let c = sp.closeAt;
|
|
2011
|
+
const a = sp.anchor;
|
|
2012
|
+
let nj = ni + 1;
|
|
2013
|
+
while (nj < cap) {
|
|
2014
|
+
const t = spans[idx[nj]];
|
|
2015
|
+
if (compareRow(t.start, fam[idx[nj]], t.anchor, t.closeAt >= 0, s, nf, a, c >= 0) !== 0)
|
|
2016
|
+
break;
|
|
2017
|
+
nj += 1;
|
|
2018
|
+
}
|
|
2019
|
+
ni = nj;
|
|
2020
|
+
if (take === "both") {
|
|
2021
|
+
const pe = prev.ends[pi];
|
|
2022
|
+
const pc = prev.closeAt[pi];
|
|
2023
|
+
pi += 1;
|
|
2024
|
+
if (pe > e || (pe === e && pc >= c)) {
|
|
2025
|
+
push(s, pe, nf, a, pc);
|
|
2026
|
+
continue;
|
|
1814
2027
|
}
|
|
1815
|
-
|
|
2028
|
+
dS.push(s);
|
|
2029
|
+
dE.push(Math.max(e, pe));
|
|
1816
2030
|
}
|
|
2031
|
+
else {
|
|
2032
|
+
dS.push(s);
|
|
2033
|
+
dE.push(e);
|
|
2034
|
+
}
|
|
2035
|
+
push(s, e, nf, a, c);
|
|
1817
2036
|
}
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
const
|
|
1821
|
-
const
|
|
1822
|
-
const
|
|
1823
|
-
let
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
let onlyPrev = false;
|
|
1829
|
-
const flush = () => {
|
|
1830
|
-
if (sources === 0)
|
|
1831
|
-
return;
|
|
1832
|
-
oS[on] = curS;
|
|
1833
|
-
oE[on] = curE;
|
|
1834
|
-
on += 1;
|
|
1835
|
-
if (!(sources === 1 && onlyPrev)) {
|
|
1836
|
-
dS[dn] = curS;
|
|
1837
|
-
dE[dn] = curE;
|
|
1838
|
-
dn += 1;
|
|
2037
|
+
if (dS.length === 0)
|
|
2038
|
+
return { next: prev, dirty: NO_DIRTY };
|
|
2039
|
+
const next = { starts: oS.subarray(0, on), ends: oE.subarray(0, on), family: oF.subarray(0, on), anchor: oA.subarray(0, on), closeAt: oC.subarray(0, on) };
|
|
2040
|
+
const ixP = rowIndexOf(prev);
|
|
2041
|
+
const ixN = rowIndexOf(next);
|
|
2042
|
+
for (let a = 0, b = 0; a < P && b < on;) {
|
|
2043
|
+
const c = compareRow(prev.starts[a], prev.family[a], prev.anchor[a], prev.closeAt[a] >= 0, next.starts[b], next.family[b], next.anchor[b], next.closeAt[b] >= 0);
|
|
2044
|
+
if (c > 0) {
|
|
2045
|
+
b += 1;
|
|
2046
|
+
continue;
|
|
1839
2047
|
}
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
const s = takePrev ? prev.starts[pi] : cs[ci];
|
|
1844
|
-
const e = takePrev ? prev.ends[pi] : ce[ci];
|
|
1845
|
-
if (takePrev)
|
|
1846
|
-
pi += 1;
|
|
1847
|
-
else
|
|
1848
|
-
ci += 1;
|
|
1849
|
-
if (sources > 0 && s <= curE) {
|
|
1850
|
-
if (e > curE)
|
|
1851
|
-
curE = e;
|
|
1852
|
-
sources += 1;
|
|
2048
|
+
if (c === 0 && ixP.closedFor[a] !== ixN.closedFor[b]) {
|
|
2049
|
+
dS.push(next.starts[b]);
|
|
2050
|
+
dE.push(Math.max(next.closeAt[b], prev.closeAt[a]));
|
|
1853
2051
|
}
|
|
2052
|
+
a += 1;
|
|
2053
|
+
if (c === 0)
|
|
2054
|
+
b += 1;
|
|
2055
|
+
}
|
|
2056
|
+
return { next, dirty: mergeClosedIntervals(dS, dE) };
|
|
2057
|
+
}
|
|
2058
|
+
function mergeClosedIntervals(s, e) {
|
|
2059
|
+
const order = Array.from(s.keys()).sort((x, y) => s[x] - s[y]);
|
|
2060
|
+
const os = [];
|
|
2061
|
+
const oe = [];
|
|
2062
|
+
for (const k of order) {
|
|
2063
|
+
const a = s[k];
|
|
2064
|
+
const b = e[k];
|
|
2065
|
+
const last = oe.length - 1;
|
|
2066
|
+
if (last >= 0 && a <= oe[last])
|
|
2067
|
+
oe[last] = Math.max(oe[last], b);
|
|
1854
2068
|
else {
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
curE = e;
|
|
1858
|
-
sources = 1;
|
|
1859
|
-
onlyPrev = takePrev;
|
|
2069
|
+
os.push(a);
|
|
2070
|
+
oe.push(b);
|
|
1860
2071
|
}
|
|
1861
2072
|
}
|
|
1862
|
-
|
|
1863
|
-
|
|
2073
|
+
return { starts: Int32Array.from(os), ends: Int32Array.from(oe) };
|
|
2074
|
+
}
|
|
2075
|
+
export function evidenceRowsFoldForTest(prev, add) {
|
|
2076
|
+
const sorted = [...prev].sort((x, y) => compareRow(x[0], x[2], x[3], x[4] >= 0, y[0], y[2], y[3], y[4] >= 0));
|
|
2077
|
+
const table = {
|
|
2078
|
+
starts: Int32Array.from(sorted, (x) => x[0]),
|
|
2079
|
+
ends: Int32Array.from(sorted, (x) => x[1]),
|
|
2080
|
+
family: Uint8Array.from(sorted, (x) => x[2]),
|
|
2081
|
+
anchor: Int32Array.from(sorted, (x) => x[3]),
|
|
2082
|
+
closeAt: Int32Array.from(sorted, (x) => x[4]),
|
|
2083
|
+
};
|
|
2084
|
+
const fams = add.map((x) => x[2]);
|
|
2085
|
+
const spans = add.map(([start, end, , anchor, closeAt], order) => ({ start, end, repl: "", rank: 0, order, anchor, closeAt }));
|
|
2086
|
+
const { next, dirty } = foldEvidenceRows(table, spans, 0, (order) => fams[order]);
|
|
2087
|
+
return {
|
|
2088
|
+
next: Array.from(next.starts, (s, i) => [s, next.ends[i], next.family[i], next.anchor[i], next.closeAt[i]]),
|
|
2089
|
+
dirty: Array.from(dirty.starts, (s, i) => [s, dirty.ends[i]]),
|
|
2090
|
+
};
|
|
1864
2091
|
}
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
2092
|
+
function lowerBound(a, x) {
|
|
2093
|
+
let lo = 0;
|
|
2094
|
+
let hi = a.length;
|
|
2095
|
+
while (lo < hi) {
|
|
2096
|
+
const mid = (lo + hi) >>> 1;
|
|
2097
|
+
if (a[mid] < x)
|
|
2098
|
+
lo = mid + 1;
|
|
2099
|
+
else
|
|
2100
|
+
hi = mid;
|
|
2101
|
+
}
|
|
2102
|
+
return lo;
|
|
1871
2103
|
}
|
|
1872
|
-
function
|
|
2104
|
+
function upperBound(a, x) {
|
|
1873
2105
|
let lo = 0;
|
|
1874
|
-
let hi =
|
|
2106
|
+
let hi = a.length;
|
|
1875
2107
|
while (lo < hi) {
|
|
1876
2108
|
const mid = (lo + hi) >>> 1;
|
|
1877
|
-
if (
|
|
2109
|
+
if (a[mid] <= x)
|
|
1878
2110
|
lo = mid + 1;
|
|
1879
2111
|
else
|
|
1880
2112
|
hi = mid;
|
|
1881
2113
|
}
|
|
1882
2114
|
return lo;
|
|
1883
2115
|
}
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
if (
|
|
1890
|
-
|
|
1891
|
-
|
|
2116
|
+
const CLOSED_FOR_ALL = -1;
|
|
2117
|
+
const CLOSED_FOR_NONE = -2;
|
|
2118
|
+
const ROW_INDEX = new WeakMap();
|
|
2119
|
+
function rowIndexOf(t) {
|
|
2120
|
+
const hit = ROW_INDEX.get(t);
|
|
2121
|
+
if (hit !== undefined)
|
|
2122
|
+
return hit;
|
|
2123
|
+
const m = t.starts.length;
|
|
2124
|
+
const e1 = new Int32Array(m);
|
|
2125
|
+
const ek1 = new Float64Array(m);
|
|
2126
|
+
const e2 = new Int32Array(m);
|
|
2127
|
+
const ek2 = new Float64Array(m);
|
|
2128
|
+
const e3 = new Int32Array(m);
|
|
2129
|
+
const c1 = new Int32Array(m);
|
|
2130
|
+
const ck1 = new Float64Array(m);
|
|
2131
|
+
const c2 = new Int32Array(m);
|
|
2132
|
+
let v1 = -1, k1 = -1, v2 = -1, k2 = -1, v3 = -1, w1 = -1, j1 = -1, w2 = -1;
|
|
2133
|
+
const dsL = [];
|
|
2134
|
+
const dK1L = [];
|
|
2135
|
+
const dE1L = [];
|
|
2136
|
+
const dE2L = [];
|
|
2137
|
+
for (let r = 0; r < m; r++) {
|
|
2138
|
+
const key = readingKey(t.family[r], t.anchor[r]);
|
|
2139
|
+
const e = t.ends[r];
|
|
2140
|
+
if (key === k1)
|
|
2141
|
+
v1 = Math.max(v1, e);
|
|
2142
|
+
else if (key === k2) {
|
|
2143
|
+
if (e > v2)
|
|
2144
|
+
v2 = e;
|
|
2145
|
+
if (v2 > v1) {
|
|
2146
|
+
const tv = v1;
|
|
2147
|
+
v1 = v2;
|
|
2148
|
+
v2 = tv;
|
|
2149
|
+
const tk = k1;
|
|
2150
|
+
k1 = k2;
|
|
2151
|
+
k2 = tk;
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
else if (e > v1) {
|
|
2155
|
+
v3 = v2;
|
|
2156
|
+
v2 = v1;
|
|
2157
|
+
k2 = k1;
|
|
2158
|
+
v1 = e;
|
|
2159
|
+
k1 = key;
|
|
2160
|
+
}
|
|
2161
|
+
else if (e > v2) {
|
|
2162
|
+
v3 = v2;
|
|
2163
|
+
v2 = e;
|
|
2164
|
+
k2 = key;
|
|
2165
|
+
}
|
|
2166
|
+
else if (e > v3)
|
|
2167
|
+
v3 = e;
|
|
2168
|
+
e1[r] = v1;
|
|
2169
|
+
ek1[r] = k1;
|
|
2170
|
+
e2[r] = v2;
|
|
2171
|
+
ek2[r] = k2;
|
|
2172
|
+
e3[r] = v3;
|
|
2173
|
+
const s = t.starts[r];
|
|
2174
|
+
const d = dsL.length - 1;
|
|
2175
|
+
if (d < 0 || dsL[d] !== s) {
|
|
2176
|
+
dsL.push(s);
|
|
2177
|
+
dK1L.push(key);
|
|
2178
|
+
dE1L.push(e);
|
|
2179
|
+
dE2L.push(-1);
|
|
2180
|
+
}
|
|
2181
|
+
else if (key === dK1L[d])
|
|
2182
|
+
dE1L[d] = Math.max(dE1L[d], e);
|
|
2183
|
+
else if (e > dE1L[d]) {
|
|
2184
|
+
dE2L[d] = dE1L[d];
|
|
2185
|
+
dE1L[d] = e;
|
|
2186
|
+
dK1L[d] = key;
|
|
2187
|
+
}
|
|
2188
|
+
else if (e > dE2L[d])
|
|
2189
|
+
dE2L[d] = e;
|
|
2190
|
+
}
|
|
2191
|
+
const closedFor = new Float64Array(m);
|
|
2192
|
+
const closeForL = new Map();
|
|
2193
|
+
for (let r = 0; r < m; r++) {
|
|
2194
|
+
const c = t.closeAt[r];
|
|
2195
|
+
let f = CLOSED_FOR_NONE;
|
|
2196
|
+
if (c >= 0) {
|
|
2197
|
+
const key = readingKey(t.family[r], t.anchor[r]);
|
|
2198
|
+
const q = upperBound(t.starts, c) - 1;
|
|
2199
|
+
let a, ka, b;
|
|
2200
|
+
if (ek1[q] === key) {
|
|
2201
|
+
a = e2[q];
|
|
2202
|
+
ka = ek2[q];
|
|
2203
|
+
b = e3[q];
|
|
2204
|
+
}
|
|
2205
|
+
else if (ek2[q] === key) {
|
|
2206
|
+
a = e1[q];
|
|
2207
|
+
ka = ek1[q];
|
|
2208
|
+
b = e3[q];
|
|
2209
|
+
}
|
|
2210
|
+
else {
|
|
2211
|
+
a = e1[q];
|
|
2212
|
+
ka = ek1[q];
|
|
2213
|
+
b = e2[q];
|
|
2214
|
+
}
|
|
2215
|
+
f = a <= c ? CLOSED_FOR_ALL : b <= c ? ka : CLOSED_FOR_NONE;
|
|
2216
|
+
if (f === CLOSED_FOR_ALL) {
|
|
2217
|
+
if (key === j1)
|
|
2218
|
+
w1 = Math.max(w1, c);
|
|
2219
|
+
else if (c > w1) {
|
|
2220
|
+
w2 = w1;
|
|
2221
|
+
w1 = c;
|
|
2222
|
+
j1 = key;
|
|
2223
|
+
}
|
|
2224
|
+
else if (c > w2)
|
|
2225
|
+
w2 = c;
|
|
2226
|
+
}
|
|
2227
|
+
else if (f !== CLOSED_FOR_NONE) {
|
|
2228
|
+
let bk = closeForL.get(f);
|
|
2229
|
+
if (bk === undefined)
|
|
2230
|
+
closeForL.set(f, (bk = { s: [], c: [], f: [] }));
|
|
2231
|
+
const last = bk.c.length - 1;
|
|
2232
|
+
const s = t.starts[r];
|
|
2233
|
+
bk.s.push(s);
|
|
2234
|
+
if (last >= 0 && bk.c[last] >= c) {
|
|
2235
|
+
bk.c.push(bk.c[last]);
|
|
2236
|
+
bk.f.push(bk.f[last]);
|
|
2237
|
+
}
|
|
2238
|
+
else {
|
|
2239
|
+
bk.c.push(c);
|
|
2240
|
+
bk.f.push(s);
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
closedFor[r] = f;
|
|
2245
|
+
c1[r] = w1;
|
|
2246
|
+
ck1[r] = j1;
|
|
2247
|
+
c2[r] = w2;
|
|
2248
|
+
}
|
|
2249
|
+
const closeFor = new Map();
|
|
2250
|
+
for (const [k, b] of closeForL)
|
|
2251
|
+
closeFor.set(k, { starts: Int32Array.from(b.s), maxClose: Int32Array.from(b.c), maxFrom: Int32Array.from(b.f) });
|
|
2252
|
+
const D = dsL.length;
|
|
2253
|
+
const ds = Int32Array.from(dsL);
|
|
2254
|
+
const reach1 = new Int32Array(D);
|
|
2255
|
+
const reach2 = new Int32Array(D);
|
|
2256
|
+
const stS = [];
|
|
2257
|
+
const stR = [];
|
|
2258
|
+
for (let d = D - 1; d >= 0; d--) {
|
|
2259
|
+
let r1 = dE1L[d];
|
|
2260
|
+
let r2 = dE2L[d];
|
|
2261
|
+
for (let q = stS.length - 1; q >= 0 && r2 >= 0 && stS[q] <= r2; q--)
|
|
2262
|
+
r2 = Math.max(r2, stR[q]);
|
|
2263
|
+
while (stS.length > 0 && stS[stS.length - 1] <= r1) {
|
|
2264
|
+
r1 = Math.max(r1, stR[stR.length - 1]);
|
|
2265
|
+
stS.pop();
|
|
2266
|
+
stR.pop();
|
|
2267
|
+
}
|
|
2268
|
+
reach1[d] = r1;
|
|
2269
|
+
reach2[d] = r2;
|
|
2270
|
+
stS.push(ds[d]);
|
|
2271
|
+
stR.push(r1);
|
|
2272
|
+
}
|
|
2273
|
+
const sS = [];
|
|
2274
|
+
const sE = [];
|
|
2275
|
+
for (let r = 0; r < m; r++) {
|
|
2276
|
+
const s = t.starts[r];
|
|
2277
|
+
const e = t.ends[r];
|
|
2278
|
+
const last = sE.length - 1;
|
|
2279
|
+
if (last >= 0 && s <= sE[last])
|
|
2280
|
+
sE[last] = Math.max(sE[last], e);
|
|
2281
|
+
else {
|
|
2282
|
+
sS.push(s);
|
|
2283
|
+
sE.push(e);
|
|
2284
|
+
}
|
|
2285
|
+
}
|
|
2286
|
+
const saved = new Float64Array(sS.length + 1);
|
|
2287
|
+
for (let k = 0; k < sS.length; k++)
|
|
2288
|
+
saved[k + 1] = saved[k] + (sE[k] - sS[k] - 1);
|
|
2289
|
+
const keyStarts = new Map();
|
|
2290
|
+
for (let r = 0; r < m; r++) {
|
|
2291
|
+
const key = readingKey(t.family[r], t.anchor[r]);
|
|
2292
|
+
const at = keyStarts.get(key);
|
|
2293
|
+
if (at === undefined)
|
|
2294
|
+
keyStarts.set(key, [t.starts[r]]);
|
|
2295
|
+
else
|
|
2296
|
+
at.push(t.starts[r]);
|
|
2297
|
+
}
|
|
2298
|
+
const byEndRow = Int32Array.from({ length: m }, (_, r) => r);
|
|
2299
|
+
for (let lo = 0; lo < m;) {
|
|
2300
|
+
let hi = lo + 1;
|
|
2301
|
+
while (hi < m && t.starts[hi] === t.starts[lo])
|
|
2302
|
+
hi += 1;
|
|
2303
|
+
if (hi - lo > 1)
|
|
2304
|
+
byEndRow.subarray(lo, hi).sort((x, y) => t.ends[x] - t.ends[y]);
|
|
2305
|
+
lo = hi;
|
|
2306
|
+
}
|
|
2307
|
+
const byEnd = Int32Array.from(byEndRow, (r) => t.ends[r]);
|
|
2308
|
+
const chainSaved = new Float64Array(D + 1);
|
|
2309
|
+
for (let d = D - 1; d >= 0; d--)
|
|
2310
|
+
chainSaved[d] = Math.max(0, reach1[d] - ds[d] - 1) + chainSaved[upperBound(ds, reach1[d])];
|
|
2311
|
+
const ix = { e1, ek1, e2, closedFor, closeFor, byEndRow, byEnd, chainSaved, bounded: new Map(), c1, ck1, c2, ds, dKey1: Float64Array.from(dK1L), dEnd2: Int32Array.from(dE2L), reach1, reach2, segS: Int32Array.from(sS), segE: Int32Array.from(sE), saved, keyStarts };
|
|
2312
|
+
ROW_INDEX.set(t, ix);
|
|
2313
|
+
return ix;
|
|
2314
|
+
}
|
|
2315
|
+
function readableMaxEnd(t, ix, x, strict, self) {
|
|
2316
|
+
const r = (strict ? lowerBound(t.starts, x) : upperBound(t.starts, x)) - 1;
|
|
2317
|
+
if (r < 0)
|
|
2318
|
+
return -1;
|
|
2319
|
+
return ix.ek1[r] === self ? ix.e2[r] : ix.e1[r];
|
|
2320
|
+
}
|
|
2321
|
+
function readableMaxClose(t, ix, x, strict, self) {
|
|
2322
|
+
const r = (strict ? lowerBound(t.starts, x) : upperBound(t.starts, x)) - 1;
|
|
2323
|
+
if (r < 0)
|
|
2324
|
+
return -1;
|
|
2325
|
+
const all = ix.ck1[r] === self ? ix.c2[r] : ix.c1[r];
|
|
2326
|
+
const own = ix.closeFor.get(self);
|
|
2327
|
+
if (own === undefined)
|
|
2328
|
+
return all;
|
|
2329
|
+
const k = (strict ? lowerBound(own.starts, x) : upperBound(own.starts, x)) - 1;
|
|
2330
|
+
return k < 0 ? all : Math.max(all, own.maxClose[k]);
|
|
2331
|
+
}
|
|
2332
|
+
function reachAt(ix, d, self) {
|
|
2333
|
+
if (ix.dKey1[d] !== self)
|
|
2334
|
+
return ix.reach1[d];
|
|
2335
|
+
return ix.dEnd2[d] < 0 ? -1 : ix.reach2[d];
|
|
1892
2336
|
}
|
|
1893
2337
|
const FOOT_NONE = 0x7fffffff;
|
|
1894
|
-
function buildForensicScan(
|
|
2338
|
+
function buildForensicScan(tokens, rows, armCount) {
|
|
1895
2339
|
return {
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
2340
|
+
rows,
|
|
2341
|
+
tokens,
|
|
2342
|
+
tokenCursor: 0,
|
|
2343
|
+
self: NO_SELF,
|
|
2344
|
+
rowCursor: 0,
|
|
1899
2345
|
dirty: undefined,
|
|
1900
2346
|
footLo: FOOT_NONE,
|
|
1901
2347
|
footHi: -1,
|
|
@@ -1905,15 +2351,14 @@ function buildForensicScan(table, armCount) {
|
|
|
1905
2351
|
headsWrite: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1906
2352
|
paramsRead: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1907
2353
|
paramsWrite: { a: NO_URL_SEGMENTS, n: 0 },
|
|
2354
|
+
opsRead: { a: NO_URL_SEGMENTS, n: 0, cut: OPS_UNCUT },
|
|
2355
|
+
opsWrite: { a: NO_URL_SEGMENTS, n: 0, cut: OPS_UNCUT },
|
|
1908
2356
|
hdr: { search: 0, contentEnd: 0, next: 0, hitLimit: false, jsonEnd: false },
|
|
2357
|
+
delimPos: new Map(),
|
|
2358
|
+
delimIx: new Map(),
|
|
2359
|
+
delimIxRows: rows,
|
|
1909
2360
|
};
|
|
1910
2361
|
}
|
|
1911
|
-
function useEvidenceView(scan, table, dirty, others) {
|
|
1912
|
-
scan.table = table;
|
|
1913
|
-
scan.others = others;
|
|
1914
|
-
scan.cursor = 0;
|
|
1915
|
-
scan.dirty = dirty;
|
|
1916
|
-
}
|
|
1917
2362
|
function noteEvidenceRead(scan, p) {
|
|
1918
2363
|
if (p < scan.footLo)
|
|
1919
2364
|
scan.footLo = p;
|
|
@@ -1935,55 +2380,416 @@ function footprintClean(dirty, lo, hi) {
|
|
|
1935
2380
|
}
|
|
1936
2381
|
return a >= ends.length || dirty.starts[a] > hi;
|
|
1937
2382
|
}
|
|
1938
|
-
function
|
|
1939
|
-
|
|
1940
|
-
const t = scan.table;
|
|
2383
|
+
function tokenEnd(scan, p) {
|
|
2384
|
+
const t = scan.tokens;
|
|
1941
2385
|
const st = t.starts;
|
|
1942
2386
|
if (st.length === 0)
|
|
1943
|
-
return
|
|
1944
|
-
|
|
1945
|
-
|
|
2387
|
+
return -1;
|
|
2388
|
+
let k = scan.tokenCursor;
|
|
2389
|
+
while (k < st.length && st[k] < p)
|
|
2390
|
+
k += 1;
|
|
2391
|
+
while (k > 0 && st[k - 1] >= p)
|
|
2392
|
+
k -= 1;
|
|
2393
|
+
scan.tokenCursor = k;
|
|
2394
|
+
return k < st.length && st[k] === p ? t.ends[k] : -1;
|
|
1946
2395
|
}
|
|
1947
|
-
function
|
|
1948
|
-
const t = scan.
|
|
1949
|
-
if (t ===
|
|
1950
|
-
|
|
1951
|
-
|
|
2396
|
+
function crossEndAt(scan, p) {
|
|
2397
|
+
const t = scan.rows;
|
|
2398
|
+
if (t.starts.length === 0)
|
|
2399
|
+
return -1;
|
|
2400
|
+
const ix = rowIndexOf(t);
|
|
2401
|
+
const ds = ix.ds;
|
|
2402
|
+
let k = scan.rowCursor;
|
|
2403
|
+
while (k < ds.length && ds[k] < p)
|
|
2404
|
+
k += 1;
|
|
2405
|
+
while (k > 0 && ds[k - 1] >= p)
|
|
2406
|
+
k -= 1;
|
|
2407
|
+
scan.rowCursor = k;
|
|
2408
|
+
if (k >= ds.length || ds[k] !== p)
|
|
2409
|
+
return -1;
|
|
2410
|
+
const r = reachAt(ix, k, scan.self);
|
|
2411
|
+
return r > p ? r : -1;
|
|
1952
2412
|
}
|
|
1953
|
-
function
|
|
1954
|
-
|
|
2413
|
+
function crossEnd(scan, p) {
|
|
2414
|
+
noteEvidenceRead(scan, p);
|
|
2415
|
+
const r = crossEndAt(scan, p);
|
|
2416
|
+
if (r >= 0)
|
|
2417
|
+
noteEvidenceRead(scan, r);
|
|
2418
|
+
return r;
|
|
2419
|
+
}
|
|
2420
|
+
const DELIM_FREE = 0;
|
|
2421
|
+
const DELIM_AMBIGUOUS = 1;
|
|
2422
|
+
const DELIM_HIDDEN = 2;
|
|
2423
|
+
function delimClass(scan, c, self = scan.self) {
|
|
2424
|
+
noteEvidenceRead(scan, c);
|
|
2425
|
+
const t = scan.rows;
|
|
2426
|
+
if (t.starts.length === 0)
|
|
2427
|
+
return DELIM_FREE;
|
|
2428
|
+
const ix = rowIndexOf(t);
|
|
2429
|
+
if (readableMaxClose(t, ix, c, false, self) > c)
|
|
2430
|
+
return DELIM_HIDDEN;
|
|
2431
|
+
return readableMaxEnd(t, ix, c, false, self) > c ? DELIM_AMBIGUOUS : DELIM_FREE;
|
|
2432
|
+
}
|
|
2433
|
+
function lineHeld(scan, p, lineEnd, self) {
|
|
1955
2434
|
noteEvidenceRead(scan, p);
|
|
1956
2435
|
noteEvidenceRead(scan, lineEnd - 1);
|
|
1957
|
-
const
|
|
1958
|
-
if (
|
|
2436
|
+
const t = scan.rows;
|
|
2437
|
+
if (t.starts.length === 0)
|
|
1959
2438
|
return false;
|
|
1960
|
-
|
|
1961
|
-
|
|
2439
|
+
return readableMaxClose(t, rowIndexOf(t), p, true, self) >= lineEnd;
|
|
2440
|
+
}
|
|
2441
|
+
function readableReach(t, ix, p, strict, self) {
|
|
2442
|
+
let q = readableMaxEnd(t, ix, p, strict, self);
|
|
2443
|
+
if (q <= p)
|
|
2444
|
+
return p;
|
|
2445
|
+
const ds = ix.ds;
|
|
2446
|
+
for (let k = strict ? lowerBound(ds, p) : upperBound(ds, p); k < ds.length && ds[k] <= q;) {
|
|
2447
|
+
const r = reachAt(ix, k, self);
|
|
2448
|
+
if (r > q)
|
|
2449
|
+
q = r;
|
|
2450
|
+
k = r > ds[k] ? Math.max(k + 1, upperBound(ds, r)) : k + 1;
|
|
2451
|
+
}
|
|
2452
|
+
return q;
|
|
1962
2453
|
}
|
|
1963
|
-
function
|
|
2454
|
+
function readableCover(scan, p, q) {
|
|
1964
2455
|
noteEvidenceRead(scan, p);
|
|
1965
|
-
|
|
2456
|
+
noteEvidenceRead(scan, q);
|
|
2457
|
+
const t = scan.rows;
|
|
2458
|
+
if (t.starts.length === 0)
|
|
2459
|
+
return false;
|
|
2460
|
+
return readableReach(t, rowIndexOf(t), p, false, scan.self) >= q;
|
|
2461
|
+
}
|
|
2462
|
+
const ROW_ENDS = new WeakMap();
|
|
2463
|
+
function rowEndsOf(t) {
|
|
2464
|
+
const hit = ROW_ENDS.get(t);
|
|
2465
|
+
if (hit !== undefined)
|
|
2466
|
+
return hit;
|
|
2467
|
+
const byEnd = Int32Array.from({ length: t.starts.length }, (_, r) => r).sort((x, y) => t.ends[x] - t.ends[y]);
|
|
2468
|
+
const v = { ends: Int32Array.from(byEnd, (r) => t.ends[r]), keys: Float64Array.from(byEnd, (r) => readingKey(t.family[r], t.anchor[r])) };
|
|
2469
|
+
ROW_ENDS.set(t, v);
|
|
2470
|
+
return v;
|
|
1966
2471
|
}
|
|
1967
|
-
function
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
2472
|
+
function nameBoundAt(scan, lo, hi) {
|
|
2473
|
+
noteEvidenceRead(scan, lo);
|
|
2474
|
+
noteEvidenceRead(scan, hi);
|
|
2475
|
+
const t = scan.rows;
|
|
2476
|
+
if (t.starts.length === 0)
|
|
1971
2477
|
return -1;
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
k
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
2478
|
+
const { ends, keys } = rowEndsOf(t);
|
|
2479
|
+
for (let k = upperBound(ends, hi) - 1; k >= 0 && ends[k] > lo; k--)
|
|
2480
|
+
if (keys[k] !== scan.self)
|
|
2481
|
+
return ends[k];
|
|
2482
|
+
return -1;
|
|
2483
|
+
}
|
|
2484
|
+
function measureWithin(scan, from, to, max) {
|
|
2485
|
+
if (to - from <= max)
|
|
2486
|
+
return true;
|
|
2487
|
+
noteEvidenceRead(scan, from);
|
|
2488
|
+
noteEvidenceRead(scan, to);
|
|
2489
|
+
const t = scan.rows;
|
|
2490
|
+
if (t.starts.length === 0)
|
|
2491
|
+
return false;
|
|
2492
|
+
const ix = rowIndexOf(t);
|
|
2493
|
+
const self = scan.self;
|
|
2494
|
+
const ds = ix.ds;
|
|
2495
|
+
const g1 = upperBound(ix.segE, to);
|
|
2496
|
+
let saved = 0;
|
|
2497
|
+
for (let k = lowerBound(ds, from); k < ds.length && ds[k] < to;) {
|
|
2498
|
+
const s0 = ds[k];
|
|
2499
|
+
if (s0 - from - saved > max)
|
|
2500
|
+
return false;
|
|
2501
|
+
const g = upperBound(ix.segS, s0) - 1;
|
|
2502
|
+
if (g < g1) {
|
|
2503
|
+
if (ix.segS[g] >= from) {
|
|
2504
|
+
const gEnd = firstOwnSegment(ix, self, g, g1);
|
|
2505
|
+
if (gEnd > g) {
|
|
2506
|
+
saved += ix.saved[gEnd] - ix.saved[g];
|
|
2507
|
+
k = upperBound(ds, ix.segE[gEnd - 1]);
|
|
2508
|
+
continue;
|
|
2509
|
+
}
|
|
2510
|
+
}
|
|
2511
|
+
else if (!ownStartWithin(ix, self, s0, ix.segE[g])) {
|
|
2512
|
+
const kn = upperBound(ds, ix.segE[g]);
|
|
2513
|
+
saved += ix.chainSaved[k] - ix.chainSaved[kn];
|
|
2514
|
+
k = kn;
|
|
2515
|
+
continue;
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
saved += boundedSavedFrom(t, ix, k, to, ownStartWithin(ix, self, s0, to) ? self : NO_SELF);
|
|
2519
|
+
break;
|
|
2520
|
+
}
|
|
2521
|
+
return to - from - saved <= max;
|
|
2522
|
+
}
|
|
2523
|
+
function ownStartWithin(ix, self, a, b) {
|
|
2524
|
+
const own = ix.keyStarts.get(self);
|
|
2525
|
+
if (own === undefined)
|
|
2526
|
+
return false;
|
|
2527
|
+
for (const x of own)
|
|
2528
|
+
if (x >= a && x <= b)
|
|
2529
|
+
return true;
|
|
2530
|
+
return false;
|
|
2531
|
+
}
|
|
2532
|
+
function boundedSavedFrom(t, ix, k, to, self) {
|
|
2533
|
+
let byReader = ix.bounded.get(to);
|
|
2534
|
+
if (byReader === undefined)
|
|
2535
|
+
ix.bounded.set(to, (byReader = new Map()));
|
|
2536
|
+
let ch = byReader.get(self);
|
|
2537
|
+
if (ch === undefined) {
|
|
2538
|
+
const hi = lowerBound(ix.ds, to);
|
|
2539
|
+
byReader.set(self, (ch = { lo: hi, hi, saved: [], stS: [], stR: [], stV: [] }));
|
|
2540
|
+
}
|
|
2541
|
+
const { stS, stR, stV } = ch;
|
|
2542
|
+
for (let d = ch.lo - 1; d >= k; d--) {
|
|
2543
|
+
const s = ix.ds[d];
|
|
2544
|
+
const x = maxEndWithin(t, ix, s, to, self);
|
|
2545
|
+
let v = stV.length > 0 ? stV[stV.length - 1] : 0;
|
|
2546
|
+
if (x >= 0) {
|
|
2547
|
+
let r = x;
|
|
2548
|
+
while (stS.length > 0 && stS[stS.length - 1] <= r) {
|
|
2549
|
+
r = Math.max(r, stR[stR.length - 1]);
|
|
2550
|
+
stS.pop();
|
|
2551
|
+
stR.pop();
|
|
2552
|
+
stV.pop();
|
|
2553
|
+
}
|
|
2554
|
+
v = Math.max(0, r - s - 1) + (stV.length > 0 ? stV[stV.length - 1] : 0);
|
|
2555
|
+
stS.push(s);
|
|
2556
|
+
stR.push(r);
|
|
2557
|
+
stV.push(v);
|
|
2558
|
+
}
|
|
2559
|
+
ch.saved.push(v);
|
|
2560
|
+
ch.lo = d;
|
|
2561
|
+
}
|
|
2562
|
+
return k >= ch.hi ? 0 : ch.saved[ch.hi - 1 - k];
|
|
2563
|
+
}
|
|
2564
|
+
function uncoveredWithin(scan, from, to) {
|
|
2565
|
+
noteEvidenceRead(scan, from);
|
|
2566
|
+
noteEvidenceRead(scan, to);
|
|
2567
|
+
const t = scan.rows;
|
|
2568
|
+
if (t.starts.length === 0 || to <= from)
|
|
2569
|
+
return Math.max(0, to - from);
|
|
2570
|
+
const ix = rowIndexOf(t);
|
|
2571
|
+
const lo = upperBound(ix.segE, from);
|
|
2572
|
+
const hi = lowerBound(ix.segS, to);
|
|
2573
|
+
if (lo >= hi)
|
|
2574
|
+
return to - from;
|
|
2575
|
+
let covered = ix.saved[hi] - ix.saved[lo] + (hi - lo);
|
|
2576
|
+
covered -= Math.max(0, from - ix.segS[lo]);
|
|
2577
|
+
covered -= Math.max(0, ix.segE[hi - 1] - to);
|
|
2578
|
+
return to - from - covered;
|
|
2579
|
+
}
|
|
2580
|
+
function firstOwnSegment(ix, self, g, g1) {
|
|
2581
|
+
const own = ix.keyStarts.get(self);
|
|
2582
|
+
if (own === undefined)
|
|
2583
|
+
return g1;
|
|
2584
|
+
let best = g1;
|
|
2585
|
+
for (const a of own) {
|
|
2586
|
+
const ga = upperBound(ix.segS, a) - 1;
|
|
2587
|
+
if (ga >= g && ga < best)
|
|
2588
|
+
best = ga;
|
|
2589
|
+
}
|
|
2590
|
+
return best;
|
|
2591
|
+
}
|
|
2592
|
+
function maxEndWithin(t, ix, s, to, self) {
|
|
2593
|
+
const lo = lowerBound(t.starts, s);
|
|
2594
|
+
let a = lo;
|
|
2595
|
+
let b = upperBound(t.starts, s);
|
|
2596
|
+
while (a < b) {
|
|
2597
|
+
const mid = (a + b) >>> 1;
|
|
2598
|
+
if (ix.byEnd[mid] <= to)
|
|
2599
|
+
a = mid + 1;
|
|
2600
|
+
else
|
|
2601
|
+
b = mid;
|
|
2602
|
+
}
|
|
2603
|
+
for (let j = a - 1; j >= lo; j--) {
|
|
2604
|
+
rowVisits += 1;
|
|
2605
|
+
const r = ix.byEndRow[j];
|
|
2606
|
+
if (readingKey(t.family[r], t.anchor[r]) !== self)
|
|
2607
|
+
return ix.byEnd[j];
|
|
2608
|
+
}
|
|
2609
|
+
return -1;
|
|
2610
|
+
}
|
|
2611
|
+
export function unitCountWithinForTest(table, from, to, max, self) {
|
|
2612
|
+
return unitCountsWithinForTest(table, [[from, to, max, self]])[0];
|
|
2613
|
+
}
|
|
2614
|
+
export function unitCountsWithinForTest(table, queries) {
|
|
2615
|
+
const keyOf = (x) => (x.length > 2 ? x[2] : x[0]);
|
|
2616
|
+
const sorted = [...table].sort((a, b) => a[0] - b[0] || keyOf(a) - keyOf(b));
|
|
2617
|
+
const rows = {
|
|
2618
|
+
starts: Int32Array.from(sorted, (x) => x[0]),
|
|
2619
|
+
ends: Int32Array.from(sorted, (x) => x[1]),
|
|
2620
|
+
family: new Uint8Array(sorted.length).fill(familyCode("prose")),
|
|
2621
|
+
anchor: Int32Array.from(sorted, keyOf),
|
|
2622
|
+
closeAt: new Int32Array(sorted.length).fill(NOT_CLOSED),
|
|
2623
|
+
};
|
|
2624
|
+
const scan = buildForensicScan(NO_TOKEN_UNITS, rows, 1);
|
|
2625
|
+
return queries.map(([from, to, max, self]) => {
|
|
2626
|
+
scan.self = self === undefined ? NO_SELF : readingKey(familyCode("prose"), self);
|
|
2627
|
+
return measureWithin(scan, from, to, max);
|
|
2628
|
+
});
|
|
2629
|
+
}
|
|
2630
|
+
function delimPositions(scan, text, kind) {
|
|
2631
|
+
const hit = scan.delimPos.get(kind);
|
|
2632
|
+
if (hit !== undefined)
|
|
2633
|
+
return hit;
|
|
2634
|
+
const out = [];
|
|
2635
|
+
switch (kind) {
|
|
2636
|
+
case "ssh2-open":
|
|
2637
|
+
case "ssh2-close": {
|
|
2638
|
+
const needle = kind === "ssh2-open" ? SSH2_FENCE_OPEN : SSH2_FENCE_CLOSE;
|
|
2639
|
+
for (let p = text.indexOf(needle); p >= 0; p = text.indexOf(needle, p + 1))
|
|
2640
|
+
out.push(p);
|
|
2641
|
+
ssh2ScanBytes += text.length;
|
|
2642
|
+
break;
|
|
2643
|
+
}
|
|
2644
|
+
case "pem-end":
|
|
2645
|
+
for (let b = text.indexOf(PEM_END_OPEN); b >= 0; b = text.indexOf(PEM_END_OPEN, b + 1))
|
|
2646
|
+
if (pemFenceLineEnd(text, b + PEM_END_OPEN.length) >= 0)
|
|
2647
|
+
out.push(b);
|
|
2648
|
+
break;
|
|
2649
|
+
case "at":
|
|
2650
|
+
case "at-json":
|
|
2651
|
+
for (let p = text.indexOf("@"); p >= 0; p = text.indexOf("@", p + 1)) {
|
|
2652
|
+
if (kind === "at-json") {
|
|
2653
|
+
let b = 0;
|
|
2654
|
+
while (p - b - 1 >= 0 && text.charCodeAt(p - b - 1) === CH_BACKSLASH)
|
|
2655
|
+
b += 1;
|
|
2656
|
+
if (b % 2 === 1)
|
|
2657
|
+
continue;
|
|
2658
|
+
}
|
|
2659
|
+
out.push(p);
|
|
2660
|
+
}
|
|
2661
|
+
break;
|
|
2662
|
+
}
|
|
2663
|
+
const pos = Int32Array.from(out);
|
|
2664
|
+
scan.delimPos.set(kind, pos);
|
|
2665
|
+
return pos;
|
|
2666
|
+
}
|
|
2667
|
+
function delimIndexOf(scan, text, kind) {
|
|
2668
|
+
const t = scan.rows;
|
|
2669
|
+
if (scan.delimIxRows !== t) {
|
|
2670
|
+
scan.delimIx = new Map();
|
|
2671
|
+
scan.delimIxRows = t;
|
|
2672
|
+
}
|
|
2673
|
+
const byKind = scan.delimIx;
|
|
2674
|
+
const hit = byKind.get(kind);
|
|
2675
|
+
if (hit !== undefined)
|
|
2676
|
+
return hit;
|
|
2677
|
+
const pos = delimPositions(scan, text, kind);
|
|
2678
|
+
const n = pos.length;
|
|
2679
|
+
const nextNotHidden = new Int32Array(n + 1);
|
|
2680
|
+
const nextFree = new Int32Array(n + 1);
|
|
2681
|
+
const prevNotHidden = new Int32Array(n);
|
|
2682
|
+
const hid = new Map();
|
|
2683
|
+
const cov = new Map();
|
|
2684
|
+
const hiddenAll = new Uint8Array(n);
|
|
2685
|
+
const coveredAll = new Uint8Array(n);
|
|
2686
|
+
if (t.starts.length > 0) {
|
|
2687
|
+
const ix = rowIndexOf(t);
|
|
2688
|
+
for (let i = 0; i < n; i++) {
|
|
2689
|
+
const c = pos[i];
|
|
2690
|
+
const r = upperBound(t.starts, c) - 1;
|
|
2691
|
+
if (r < 0)
|
|
2692
|
+
continue;
|
|
2693
|
+
if (ix.c1[r] > c) {
|
|
2694
|
+
hiddenAll[i] = 1;
|
|
2695
|
+
if (ix.c2[r] <= c) {
|
|
2696
|
+
const k = ix.ck1[r];
|
|
2697
|
+
let b = hid.get(k);
|
|
2698
|
+
if (b === undefined)
|
|
2699
|
+
hid.set(k, (b = []));
|
|
2700
|
+
b.push(i);
|
|
2701
|
+
}
|
|
2702
|
+
}
|
|
2703
|
+
if (ix.e1[r] > c) {
|
|
2704
|
+
coveredAll[i] = 1;
|
|
2705
|
+
if (ix.e2[r] <= c) {
|
|
2706
|
+
const k = ix.ek1[r];
|
|
2707
|
+
let b = cov.get(k);
|
|
2708
|
+
if (b === undefined)
|
|
2709
|
+
cov.set(k, (b = []));
|
|
2710
|
+
b.push(i);
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
}
|
|
2715
|
+
nextNotHidden[n] = n;
|
|
2716
|
+
nextFree[n] = n;
|
|
2717
|
+
for (let i = n - 1; i >= 0; i--) {
|
|
2718
|
+
nextNotHidden[i] = hiddenAll[i] === 1 ? nextNotHidden[i + 1] : i;
|
|
2719
|
+
nextFree[i] = coveredAll[i] === 1 ? nextFree[i + 1] : i;
|
|
2720
|
+
}
|
|
2721
|
+
for (let i = 0, last = -1; i < n; i++) {
|
|
2722
|
+
if (hiddenAll[i] !== 1)
|
|
2723
|
+
last = i;
|
|
2724
|
+
prevNotHidden[i] = last;
|
|
2725
|
+
}
|
|
2726
|
+
const freeze = (m) => new Map([...m].map(([k, v]) => [k, Int32Array.from(v)]));
|
|
2727
|
+
const di = { pos, nextNotHidden, nextFree, prevNotHidden, onlyHiddenBy: freeze(hid), onlyCoveredBy: freeze(cov) };
|
|
2728
|
+
byKind.set(kind, di);
|
|
2729
|
+
return di;
|
|
2730
|
+
}
|
|
2731
|
+
function bucketFirstFrom(b, i, n) {
|
|
2732
|
+
if (b === undefined)
|
|
2733
|
+
return n;
|
|
2734
|
+
const k = lowerBound(b, i);
|
|
2735
|
+
return k < b.length ? b[k] : n;
|
|
2736
|
+
}
|
|
2737
|
+
function firstDelim(scan, text, kind, from, wantFree) {
|
|
2738
|
+
noteEvidenceRead(scan, from);
|
|
2739
|
+
const di = delimIndexOf(scan, text, kind);
|
|
2740
|
+
const n = di.pos.length;
|
|
2741
|
+
const bucket = (wantFree ? di.onlyCoveredBy : di.onlyHiddenBy).get(scan.self);
|
|
2742
|
+
const firstAt = (i) => (i >= n ? n : Math.min(wantFree ? di.nextFree[i] : di.nextNotHidden[i], bucketFirstFrom(bucket, i, n)));
|
|
2743
|
+
let j = firstAt(lowerBound(di.pos, from));
|
|
2744
|
+
const own = wantFree || scan.rows.starts.length === 0 ? undefined : rowIndexOf(scan.rows).closeFor.get(scan.self);
|
|
2745
|
+
while (own !== undefined && j < n) {
|
|
2746
|
+
const p = di.pos[j];
|
|
2747
|
+
const k = upperBound(own.starts, p) - 1;
|
|
2748
|
+
if (k < 0 || own.maxClose[k] <= p)
|
|
2749
|
+
break;
|
|
2750
|
+
j = firstAt(lowerBound(di.pos, own.maxClose[k]));
|
|
2751
|
+
}
|
|
2752
|
+
const at = j >= n ? -1 : di.pos[j];
|
|
2753
|
+
noteEvidenceRead(scan, at < 0 ? text.length : at);
|
|
2754
|
+
return at;
|
|
2755
|
+
}
|
|
2756
|
+
function lastNotHiddenDelim(scan, text, kind, a, b) {
|
|
2757
|
+
const di = delimIndexOf(scan, text, kind);
|
|
2758
|
+
const bucket = di.onlyHiddenBy.get(scan.self);
|
|
2759
|
+
const lastAt = (j) => {
|
|
2760
|
+
let best = di.prevNotHidden[j];
|
|
2761
|
+
if (bucket !== undefined) {
|
|
2762
|
+
const k = upperBound(bucket, j) - 1;
|
|
2763
|
+
if (k >= 0 && bucket[k] > best)
|
|
2764
|
+
best = bucket[k];
|
|
2765
|
+
}
|
|
2766
|
+
return best;
|
|
2767
|
+
};
|
|
2768
|
+
const j = lowerBound(di.pos, b) - 1;
|
|
2769
|
+
if (j < 0 || di.pos[j] < a)
|
|
2770
|
+
return -1;
|
|
2771
|
+
let best = lastAt(j);
|
|
2772
|
+
const own = scan.rows.starts.length === 0 ? undefined : rowIndexOf(scan.rows).closeFor.get(scan.self);
|
|
2773
|
+
while (own !== undefined && best >= 0 && di.pos[best] >= a) {
|
|
2774
|
+
const p = di.pos[best];
|
|
2775
|
+
const k = upperBound(own.starts, p) - 1;
|
|
2776
|
+
if (k < 0 || own.maxClose[k] <= p)
|
|
2777
|
+
break;
|
|
2778
|
+
const j2 = lowerBound(di.pos, own.maxFrom[k]) - 1;
|
|
2779
|
+
best = j2 < 0 ? -1 : lastAt(j2);
|
|
2780
|
+
}
|
|
2781
|
+
return best >= 0 && di.pos[best] >= a ? di.pos[best] : -1;
|
|
1979
2782
|
}
|
|
1980
2783
|
function scanUrlRun(scan, text, i, kind, json) {
|
|
1981
2784
|
const n = text.length;
|
|
1982
2785
|
noteEvidenceRead(scan, i);
|
|
1983
2786
|
scan.lastAt = -1;
|
|
1984
2787
|
run: while (i < n) {
|
|
1985
|
-
const unit =
|
|
2788
|
+
const unit = crossEndAt(scan, i);
|
|
1986
2789
|
if (unit >= 0) {
|
|
2790
|
+
const at = lastNotHiddenDelim(scan, text, json ? "at-json" : "at", i, unit);
|
|
2791
|
+
if (at > scan.lastAt)
|
|
2792
|
+
scan.lastAt = at;
|
|
1987
2793
|
i = unit;
|
|
1988
2794
|
continue;
|
|
1989
2795
|
}
|
|
@@ -2024,7 +2830,7 @@ function scanUrlRun(scan, text, i, kind, json) {
|
|
|
2024
2830
|
}
|
|
2025
2831
|
function matchUserinfoAt(scan, text, p, json, isHead) {
|
|
2026
2832
|
const probeEnd = scanUrlRun(scan, text, p, "plain-user", json);
|
|
2027
|
-
const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 &&
|
|
2833
|
+
const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && readableCover(scan, p, probeEnd);
|
|
2028
2834
|
const runEnd = scanUrlRun(scan, text, p, keyId ? "token-pass" : "plain-pass", json);
|
|
2029
2835
|
if (isHead && runEnd >= text.length && runEnd > p)
|
|
2030
2836
|
return runEnd;
|
|
@@ -2036,7 +2842,9 @@ function matchUserinfoAt(scan, text, p, json, isHead) {
|
|
|
2036
2842
|
const URL_SEG_STRIDE = 4;
|
|
2037
2843
|
const URL_SEG_MAX_INTS = 3 * 1_048_576;
|
|
2038
2844
|
const NO_URL_SEGMENTS = new Int32Array(0);
|
|
2039
|
-
|
|
2845
|
+
const ANCHOR_SEG_STRIDE = 4;
|
|
2846
|
+
const OPS_UNCUT = 0x7fffffff;
|
|
2847
|
+
function growSegments(list) {
|
|
2040
2848
|
const len = list.a.length;
|
|
2041
2849
|
if (len >= URL_SEG_MAX_INTS)
|
|
2042
2850
|
return false;
|
|
@@ -2047,6 +2855,7 @@ function growUrlSegments(list) {
|
|
|
2047
2855
|
}
|
|
2048
2856
|
function collectUrlCredentialSpans(scan, text, json, span, isHead) {
|
|
2049
2857
|
const dirty = scan.dirty;
|
|
2858
|
+
const urlFamily = familyCode("url");
|
|
2050
2859
|
const hr = scan.headsRead;
|
|
2051
2860
|
const hw = scan.headsWrite;
|
|
2052
2861
|
const pr = scan.paramsRead;
|
|
@@ -2081,14 +2890,17 @@ function collectUrlCredentialSpans(scan, text, json, span, isHead) {
|
|
|
2081
2890
|
else {
|
|
2082
2891
|
scan.footLo = FOOT_NONE;
|
|
2083
2892
|
scan.footHi = -1;
|
|
2893
|
+
scan.self = readingKey(urlFamily, colon);
|
|
2084
2894
|
at = matchUserinfoAt(scan, text, start, json, isHead) ?? -1;
|
|
2085
2895
|
lo = scan.footLo;
|
|
2086
2896
|
hi = scan.footHi;
|
|
2087
2897
|
urlSegmentRuns += 1;
|
|
2088
|
-
if (at >= 0 && at !== (known ? ph[pc + 1] : -2))
|
|
2089
|
-
|
|
2898
|
+
if (at >= 0 && at !== (known ? ph[pc + 1] : -2)) {
|
|
2899
|
+
const undecided = isHead && at >= text.length;
|
|
2900
|
+
span(start, at, colon, undecided ? NOT_CLOSED : at, undecided);
|
|
2901
|
+
}
|
|
2090
2902
|
}
|
|
2091
|
-
if (hw.n + URL_SEG_STRIDE <= hw.a.length ||
|
|
2903
|
+
if (hw.n + URL_SEG_STRIDE <= hw.a.length || growSegments(hw)) {
|
|
2092
2904
|
const w = hw.a;
|
|
2093
2905
|
const k = hw.n;
|
|
2094
2906
|
w[k] = colon;
|
|
@@ -2128,14 +2940,15 @@ function collectUrlCredentialSpans(scan, text, json, span, isHead) {
|
|
|
2128
2940
|
else {
|
|
2129
2941
|
scan.footLo = FOOT_NONE;
|
|
2130
2942
|
scan.footHi = -1;
|
|
2943
|
+
scan.self = readingKey(urlFamily, i);
|
|
2131
2944
|
valueEnd = scanUrlRun(scan, text, valueStart, "param-value", json);
|
|
2132
2945
|
lo = scan.footLo;
|
|
2133
2946
|
hi = scan.footHi;
|
|
2134
2947
|
urlSegmentRuns += 1;
|
|
2135
2948
|
if (valueEnd > valueStart && valueEnd !== (known ? pp[qc + 1] : -2))
|
|
2136
|
-
span(valueStart, valueEnd);
|
|
2949
|
+
span(valueStart, valueEnd, i, NOT_CLOSED);
|
|
2137
2950
|
}
|
|
2138
|
-
if (pw.n + URL_SEG_STRIDE <= pw.a.length ||
|
|
2951
|
+
if (pw.n + URL_SEG_STRIDE <= pw.a.length || growSegments(pw)) {
|
|
2139
2952
|
const w = pw.a;
|
|
2140
2953
|
const k = pw.n;
|
|
2141
2954
|
w[k] = i;
|
|
@@ -2267,7 +3080,7 @@ function collectJsonSecretFieldSpans(text, span, isHead) {
|
|
|
2267
3080
|
continue;
|
|
2268
3081
|
const open = found === STRING_OPEN;
|
|
2269
3082
|
const end = open ? text.length : found;
|
|
2270
|
-
span(q, end, open, JSON_FIELD_VALUE_REPLS[d]);
|
|
3083
|
+
span(q, end, q, open ? NOT_CLOSED : end - form.length, open, JSON_FIELD_VALUE_REPLS[d]);
|
|
2271
3084
|
from = scanResumeAfterHit(start, end, JSON_FIELD_BACK);
|
|
2272
3085
|
next = text.indexOf(JSON_FIELD_ANCHOR, from);
|
|
2273
3086
|
if (next < 0)
|
|
@@ -2349,7 +3162,7 @@ function escapedAt(text, i) {
|
|
|
2349
3162
|
anchoredScanUnits += i - 1 - k;
|
|
2350
3163
|
return ((i - 1 - k) & 1) === 1;
|
|
2351
3164
|
}
|
|
2352
|
-
function anchorHeadAt(text, op, json) {
|
|
3165
|
+
function anchorHeadAt(scan, text, op, json) {
|
|
2353
3166
|
let i = op - 1;
|
|
2354
3167
|
let spacedLeft = false;
|
|
2355
3168
|
for (;;) {
|
|
@@ -2383,7 +3196,10 @@ function anchorHeadAt(text, op, json) {
|
|
|
2383
3196
|
while (j > floor && isIdentCode(text.charCodeAt(j)))
|
|
2384
3197
|
j -= 1;
|
|
2385
3198
|
anchoredScanUnits += nameEnd - (j + 1);
|
|
2386
|
-
|
|
3199
|
+
const bound = j + 1 < nameEnd ? nameBoundAt(scan, Math.max(j, 0), nameEnd) : -1;
|
|
3200
|
+
if (bound > j)
|
|
3201
|
+
j = bound - 1;
|
|
3202
|
+
else if (j === floor && j >= 0 && isIdentCode(text.charCodeAt(j)))
|
|
2387
3203
|
return undefined;
|
|
2388
3204
|
let nameStart = j + 1;
|
|
2389
3205
|
let afterEscape = false;
|
|
@@ -2451,50 +3267,96 @@ function quotedValueEnd(text, cs, quote, json, isHead) {
|
|
|
2451
3267
|
}
|
|
2452
3268
|
return isHead ? STRING_OPEN : -1;
|
|
2453
3269
|
}
|
|
2454
|
-
|
|
3270
|
+
const NO_MEMO_GROUP = -1;
|
|
3271
|
+
function buildBareValueMemo() {
|
|
3272
|
+
return {
|
|
3273
|
+
from: new Int32Array(4),
|
|
3274
|
+
at: new Int32Array(4).fill(-1),
|
|
3275
|
+
netFrom: new Int32Array(4).fill(NO_MEMO_GROUP),
|
|
3276
|
+
net: new Int32Array(16),
|
|
3277
|
+
runFrom: new Int32Array(4).fill(NO_MEMO_GROUP),
|
|
3278
|
+
closers: Array.from({ length: 16 }, () => []),
|
|
3279
|
+
};
|
|
3280
|
+
}
|
|
3281
|
+
const listSepSlot = (listSep) => (listSep === 0 ? 0 : listSep === CH_AMP ? 1 : listSep === CH_SEMI ? 2 : 3);
|
|
3282
|
+
function bareValueEnd(text, s, json, listSep, memo) {
|
|
2455
3283
|
const n = text.length;
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
if (next >= n)
|
|
3284
|
+
const k = listSepSlot(listSep);
|
|
3285
|
+
let i;
|
|
3286
|
+
if (memo.from[k] <= s && s <= memo.at[k])
|
|
3287
|
+
i = memo.at[k];
|
|
3288
|
+
else {
|
|
3289
|
+
i = s;
|
|
3290
|
+
while (i < n) {
|
|
3291
|
+
const unit = readAnchorUnit(text, i, json);
|
|
3292
|
+
const ch = unitChOf(unit);
|
|
3293
|
+
if (ch === JSON_STRING_END || isRegexSpace(ch))
|
|
2467
3294
|
break;
|
|
2468
|
-
if (
|
|
3295
|
+
if (listSep !== 0 && (ch === listSep || (listSep === CH_AMP && ch === CH_HASH)))
|
|
2469
3296
|
break;
|
|
3297
|
+
const next = i + unitLenOf(unit);
|
|
3298
|
+
if (isQuoteUnit(ch)) {
|
|
3299
|
+
if (next >= n)
|
|
3300
|
+
break;
|
|
3301
|
+
if (endsValueUnit(unitChOf(readAnchorUnit(text, next, json))))
|
|
3302
|
+
break;
|
|
3303
|
+
}
|
|
3304
|
+
i = next;
|
|
2470
3305
|
}
|
|
2471
|
-
|
|
3306
|
+
memo.from[k] = s;
|
|
3307
|
+
memo.at[k] = i;
|
|
3308
|
+
memo.netFrom[k] = NO_MEMO_GROUP;
|
|
3309
|
+
memo.runFrom[k] = NO_MEMO_GROUP;
|
|
2472
3310
|
}
|
|
2473
|
-
|
|
2474
|
-
const
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
3311
|
+
const nb = 4 * k;
|
|
3312
|
+
const net = memo.net;
|
|
3313
|
+
const netFrom = memo.netFrom[k];
|
|
3314
|
+
if (netFrom === NO_MEMO_GROUP) {
|
|
3315
|
+
net.fill(0, nb, nb + 4);
|
|
3316
|
+
for (let q = s; q < i; q++) {
|
|
3317
|
+
const b = bracketOf(text.charCodeAt(q));
|
|
3318
|
+
if (b !== 0)
|
|
3319
|
+
net[nb + Math.abs(b) - 1] = net[nb + Math.abs(b) - 1] + (b < 0 ? 1 : -1);
|
|
3320
|
+
}
|
|
3321
|
+
anchoredScanUnits += i - s;
|
|
2479
3322
|
}
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
continue;
|
|
3323
|
+
else {
|
|
3324
|
+
for (let q = netFrom; q < s; q++) {
|
|
3325
|
+
const b = bracketOf(text.charCodeAt(q));
|
|
3326
|
+
if (b !== 0)
|
|
3327
|
+
net[nb + Math.abs(b) - 1] = net[nb + Math.abs(b) - 1] - (b < 0 ? 1 : -1);
|
|
2486
3328
|
}
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
3329
|
+
anchoredScanUnits += s - netFrom;
|
|
3330
|
+
}
|
|
3331
|
+
memo.netFrom[k] = s;
|
|
3332
|
+
if (memo.runFrom[k] === NO_MEMO_GROUP) {
|
|
3333
|
+
for (let q = 0; q < 4; q++)
|
|
3334
|
+
memo.closers[nb + q].length = 0;
|
|
3335
|
+
const lowest = memo.from[k];
|
|
3336
|
+
let r = i;
|
|
3337
|
+
while (r > lowest) {
|
|
3338
|
+
const c = text.charCodeAt(r - 1);
|
|
3339
|
+
if (c !== CH_DOT && c !== CH_COMMA && c !== CH_SEMI && c !== CH_COLON) {
|
|
3340
|
+
const b = bracketOf(c);
|
|
3341
|
+
if (b >= 0)
|
|
3342
|
+
break;
|
|
3343
|
+
memo.closers[nb - b - 1].push(r - 1);
|
|
3344
|
+
}
|
|
3345
|
+
r -= 1;
|
|
2492
3346
|
}
|
|
2493
|
-
|
|
3347
|
+
anchoredScanUnits += i - r;
|
|
3348
|
+
memo.runFrom[k] = r;
|
|
3349
|
+
}
|
|
3350
|
+
let e = Math.max(s, memo.runFrom[k]);
|
|
3351
|
+
for (let q = 0; q < 4; q++) {
|
|
3352
|
+
const at = memo.closers[nb + q];
|
|
3353
|
+
const stop = Math.max(net[nb + q], 0);
|
|
3354
|
+
if (stop < at.length && at[stop] + 1 > e)
|
|
3355
|
+
e = at[stop] + 1;
|
|
2494
3356
|
}
|
|
2495
3357
|
return e;
|
|
2496
3358
|
}
|
|
2497
|
-
function literalValueAt(text, s, json, declaredIfBare, listSep, isHead) {
|
|
3359
|
+
function literalValueAt(text, s, json, declaredIfBare, listSep, isHead, memo) {
|
|
2498
3360
|
const n = text.length;
|
|
2499
3361
|
if (s >= n)
|
|
2500
3362
|
return undefined;
|
|
@@ -2526,7 +3388,7 @@ function literalValueAt(text, s, json, declaredIfBare, listSep, isHead) {
|
|
|
2526
3388
|
if (after === CH_LPAREN || after === CH_LBRACKET || (after === CH_DOT && k + 1 < n && isProgramIdentStartCode(text.charCodeAt(k + 1))))
|
|
2527
3389
|
return undefined;
|
|
2528
3390
|
}
|
|
2529
|
-
const e = bareValueEnd(text, vs, json, listSep);
|
|
3391
|
+
const e = bareValueEnd(text, vs, json, listSep, memo);
|
|
2530
3392
|
if (e <= vs)
|
|
2531
3393
|
return undefined;
|
|
2532
3394
|
if (e - vs <= ABSENT_OR_BOOLEAN_MAX && ABSENT_OR_BOOLEAN_WORDS.has(text.slice(vs, e).toLowerCase()))
|
|
@@ -2595,18 +3457,60 @@ function schemeValueAt(text, s, json) {
|
|
|
2595
3457
|
return undefined;
|
|
2596
3458
|
return { start: vs, end: e, quoted: false };
|
|
2597
3459
|
}
|
|
2598
|
-
function
|
|
3460
|
+
function anchoredValueAt(scan, text, op, json, span, isHead, memo) {
|
|
3461
|
+
const head = anchorHeadAt(scan, text, op, json);
|
|
3462
|
+
if (head === undefined)
|
|
3463
|
+
return;
|
|
2599
3464
|
const n = text.length;
|
|
2600
|
-
let
|
|
3465
|
+
let s = op + 1;
|
|
3466
|
+
let spacedRight = false;
|
|
3467
|
+
while (s < n) {
|
|
3468
|
+
const unit = readAnchorUnit(text, s, json);
|
|
3469
|
+
if (!isInlineSpaceUnit(unitChOf(unit)))
|
|
3470
|
+
break;
|
|
3471
|
+
s += unitLenOf(unit);
|
|
3472
|
+
spacedRight = true;
|
|
3473
|
+
}
|
|
3474
|
+
let v;
|
|
3475
|
+
switch (head.strength) {
|
|
3476
|
+
case "structure-only":
|
|
3477
|
+
return;
|
|
3478
|
+
case "auth-scheme":
|
|
3479
|
+
v = schemeValueAt(text, s, json);
|
|
3480
|
+
break;
|
|
3481
|
+
case "declarative":
|
|
3482
|
+
v = literalValueAt(text, s, json, head.quotedName || (!head.spacedLeft && !spacedRight), head.listSep, isHead, memo);
|
|
3483
|
+
break;
|
|
3484
|
+
case "any-form":
|
|
3485
|
+
v = literalValueAt(text, s, json, true, head.listSep, isHead, memo);
|
|
3486
|
+
break;
|
|
3487
|
+
default: {
|
|
3488
|
+
const unhandled = head.strength;
|
|
3489
|
+
throw new Error(`redact: unhandled free-text anchor strength ${String(unhandled)}`);
|
|
3490
|
+
}
|
|
3491
|
+
}
|
|
3492
|
+
if (v === undefined)
|
|
3493
|
+
return;
|
|
3494
|
+
span(v.start, v.end, op, v.quoted && v.open !== true ? v.end : NOT_CLOSED, v.open === true);
|
|
3495
|
+
}
|
|
3496
|
+
function collectAnchoredValueSpans(scan, text, json, span, isHead) {
|
|
3497
|
+
const dirty = scan.dirty;
|
|
3498
|
+
const anchorFamily = familyCode("anchor");
|
|
3499
|
+
const rd = scan.opsRead;
|
|
3500
|
+
const wr = scan.opsWrite;
|
|
3501
|
+
wr.n = 0;
|
|
3502
|
+
wr.cut = OPS_UNCUT;
|
|
3503
|
+
const pa = rd.a;
|
|
3504
|
+
const pan = dirty === undefined ? 0 : rd.n;
|
|
3505
|
+
let pc = 0;
|
|
3506
|
+
let armLo = FOOT_NONE;
|
|
3507
|
+
let armHi = -1;
|
|
3508
|
+
const memo = buildBareValueMemo();
|
|
2601
3509
|
let nextEq = text.indexOf("=");
|
|
2602
3510
|
let nextColon = text.indexOf(":");
|
|
2603
3511
|
for (;;) {
|
|
2604
|
-
if (nextEq >= 0 && nextEq < resume)
|
|
2605
|
-
nextEq = text.indexOf("=", resume);
|
|
2606
|
-
if (nextColon >= 0 && nextColon < resume)
|
|
2607
|
-
nextColon = text.indexOf(":", resume);
|
|
2608
3512
|
if (nextEq < 0 && nextColon < 0)
|
|
2609
|
-
|
|
3513
|
+
break;
|
|
2610
3514
|
let op;
|
|
2611
3515
|
if (nextColon < 0 || (nextEq >= 0 && nextEq < nextColon)) {
|
|
2612
3516
|
op = nextEq;
|
|
@@ -2616,41 +3520,51 @@ function collectAnchoredValueSpans(text, json, span, isHead) {
|
|
|
2616
3520
|
op = nextColon;
|
|
2617
3521
|
nextColon = text.indexOf(":", op + 1);
|
|
2618
3522
|
}
|
|
2619
|
-
|
|
2620
|
-
|
|
3523
|
+
while (pc < pan && pa[pc] < op)
|
|
3524
|
+
pc += ANCHOR_SEG_STRIDE;
|
|
3525
|
+
const known = pc < pan && pa[pc] === op;
|
|
3526
|
+
if (!known && dirty !== undefined && op < rd.cut)
|
|
2621
3527
|
continue;
|
|
2622
|
-
|
|
2623
|
-
let
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
spacedRight = true;
|
|
3528
|
+
scan.self = readingKey(anchorFamily, op);
|
|
3529
|
+
let lo;
|
|
3530
|
+
let hi;
|
|
3531
|
+
let answer;
|
|
3532
|
+
if (known && (answer = nameBoundAt(scan, pa[pc + 1], pa[pc + 2])) === pa[pc + 3]) {
|
|
3533
|
+
lo = pa[pc + 1];
|
|
3534
|
+
hi = pa[pc + 2];
|
|
2630
3535
|
}
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
3536
|
+
else {
|
|
3537
|
+
scan.footLo = FOOT_NONE;
|
|
3538
|
+
scan.footHi = -1;
|
|
3539
|
+
anchoredValueAt(scan, text, op, json, span, isHead, memo);
|
|
3540
|
+
lo = scan.footLo;
|
|
3541
|
+
hi = scan.footHi;
|
|
3542
|
+
if (lo > hi)
|
|
2634
3543
|
continue;
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
throw new Error(`redact: unhandled free-text anchor strength ${String(unhandled)}`);
|
|
3544
|
+
answer = nameBoundAt(scan, lo, hi);
|
|
3545
|
+
}
|
|
3546
|
+
if (wr.cut === OPS_UNCUT) {
|
|
3547
|
+
if (wr.n + ANCHOR_SEG_STRIDE <= wr.a.length || growSegments(wr)) {
|
|
3548
|
+
const w = wr.a;
|
|
3549
|
+
const k = wr.n;
|
|
3550
|
+
w[k] = op;
|
|
3551
|
+
w[k + 1] = lo;
|
|
3552
|
+
w[k + 2] = hi;
|
|
3553
|
+
w[k + 3] = answer;
|
|
3554
|
+
wr.n = k + ANCHOR_SEG_STRIDE;
|
|
2647
3555
|
}
|
|
3556
|
+
else
|
|
3557
|
+
wr.cut = op;
|
|
2648
3558
|
}
|
|
2649
|
-
if (
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
3559
|
+
if (lo < armLo)
|
|
3560
|
+
armLo = lo;
|
|
3561
|
+
if (hi > armHi)
|
|
3562
|
+
armHi = hi;
|
|
2653
3563
|
}
|
|
3564
|
+
scan.opsRead = wr;
|
|
3565
|
+
scan.opsWrite = rd;
|
|
3566
|
+
scan.footLo = armLo;
|
|
3567
|
+
scan.footHi = armHi;
|
|
2654
3568
|
}
|
|
2655
3569
|
export function structuralScannerForTest(arm) {
|
|
2656
3570
|
const collect = (scan) => (text) => {
|
|
@@ -2662,17 +3576,18 @@ export function structuralScannerForTest(arm) {
|
|
|
2662
3576
|
case "json-secret-field":
|
|
2663
3577
|
return { declared: new RegExp(JSON_SECRET_FIELD_PATTERN[0].source, JSON_SECRET_FIELD_PATTERN[0].flags), spans: collect(collectJsonSecretFieldSpans) };
|
|
2664
3578
|
case "pem-fence":
|
|
2665
|
-
return { declared: new RegExp(PEM_FENCE_PATTERN[0].source, PEM_FENCE_PATTERN[0].flags), spans: collect((text, span, isHead) => collectPemFenceSpans(
|
|
3579
|
+
return { declared: new RegExp(PEM_FENCE_PATTERN[0].source, PEM_FENCE_PATTERN[0].flags), spans: collect((text, span, isHead) => collectPemFenceSpans(buildForensicScan(NO_TOKEN_UNITS, NO_ROWS, 1), text, span, isHead)) };
|
|
2666
3580
|
}
|
|
2667
3581
|
}
|
|
2668
3582
|
let ppkScanBytes = 0;
|
|
2669
3583
|
export function ppkScanBytesForTest() { return ppkScanBytes; }
|
|
2670
3584
|
export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
|
|
2671
3585
|
function collectPpkContainerSpans(scan, text, eat, span, isHead) {
|
|
2672
|
-
const
|
|
3586
|
+
const fam = familyCode("putty-ppk");
|
|
3587
|
+
const HEAD = PPK_FILE_HEAD;
|
|
2673
3588
|
const MAC = "Private-MAC: ";
|
|
2674
3589
|
const HASH = "Private-Hash: ";
|
|
2675
|
-
const LINES =
|
|
3590
|
+
const LINES = PPK_LINES_FIELD;
|
|
2676
3591
|
const cache = new Map();
|
|
2677
3592
|
const find = (needle, from) => {
|
|
2678
3593
|
const c = cache.get(needle);
|
|
@@ -2686,7 +3601,7 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
|
|
|
2686
3601
|
};
|
|
2687
3602
|
const versionEnd = (at) => {
|
|
2688
3603
|
let p = at + HEAD.length;
|
|
2689
|
-
while (p < text.length &&
|
|
3604
|
+
while (p < text.length && tokenEnd(scan, p) < 0) {
|
|
2690
3605
|
const c = text.charCodeAt(p);
|
|
2691
3606
|
if (c >= 48 && c <= 57)
|
|
2692
3607
|
p += 1;
|
|
@@ -2707,14 +3622,14 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
|
|
|
2707
3622
|
let p = find(HEAD, from);
|
|
2708
3623
|
for (; p !== Number.POSITIVE_INFINITY; p = find(HEAD, p + HEAD.length)) {
|
|
2709
3624
|
const lineEnd = versionedHeadEnd(p);
|
|
2710
|
-
if (lineEnd >= 0 && !
|
|
3625
|
+
if (lineEnd >= 0 && !lineHeld(scan, p, lineEnd, readingKey(fam, p)))
|
|
2711
3626
|
break;
|
|
2712
3627
|
}
|
|
2713
3628
|
return p;
|
|
2714
3629
|
};
|
|
2715
3630
|
const hexRunEnd = (from) => {
|
|
2716
3631
|
let e = from;
|
|
2717
|
-
while (e < text.length &&
|
|
3632
|
+
while (e < text.length && tokenEnd(scan, e) < 0) {
|
|
2718
3633
|
const c = text.charCodeAt(e);
|
|
2719
3634
|
if ((c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70))
|
|
2720
3635
|
e += 1;
|
|
@@ -2725,7 +3640,9 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
|
|
|
2725
3640
|
};
|
|
2726
3641
|
let at = nextVersionedHead(0);
|
|
2727
3642
|
while (at !== Number.POSITIVE_INFINITY) {
|
|
3643
|
+
scan.self = readingKey(fam, at);
|
|
2728
3644
|
let pos = at + HEAD.length + 1;
|
|
3645
|
+
let closeAt = NOT_CLOSED;
|
|
2729
3646
|
let end = text.length;
|
|
2730
3647
|
let ateToEnd = !isHead;
|
|
2731
3648
|
let open = false;
|
|
@@ -2752,12 +3669,8 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
|
|
|
2752
3669
|
let gapBlank = true;
|
|
2753
3670
|
if (limit !== Number.POSITIVE_INFINITY) {
|
|
2754
3671
|
for (let i = hexEnd; i < limit; i++) {
|
|
2755
|
-
if (opaqueUnitEnd(scan, i) >= 0) {
|
|
2756
|
-
gapBlank = false;
|
|
2757
|
-
break;
|
|
2758
|
-
}
|
|
2759
3672
|
const c = text.charCodeAt(i);
|
|
2760
|
-
if (c !== 32 && c !== 10 && c !== 13 && c !== 9) {
|
|
3673
|
+
if ((c !== 32 && c !== 10 && c !== 13 && c !== 9) || delimClass(scan, i) !== DELIM_FREE) {
|
|
2761
3674
|
gapBlank = false;
|
|
2762
3675
|
break;
|
|
2763
3676
|
}
|
|
@@ -2767,18 +3680,20 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
|
|
|
2767
3680
|
ateToEnd = straggler < limit || !gapBlank;
|
|
2768
3681
|
end = ateToEnd || undecided ? text.length : hexEnd;
|
|
2769
3682
|
open = undecided;
|
|
3683
|
+
if (end === hexEnd)
|
|
3684
|
+
closeAt = term;
|
|
2770
3685
|
break;
|
|
2771
3686
|
}
|
|
2772
3687
|
if (ateToEnd && eat !== undefined && eat.from < 0) {
|
|
2773
3688
|
eat.from = at;
|
|
2774
3689
|
eat.len = text.length;
|
|
2775
3690
|
}
|
|
2776
|
-
span(at, end, open);
|
|
3691
|
+
span(at, end, at, closeAt, open);
|
|
2777
3692
|
at = end >= text.length ? Number.POSITIVE_INFINITY : nextVersionedHead(end);
|
|
2778
3693
|
}
|
|
2779
3694
|
const last = isHead ? text.lastIndexOf(HEAD) : -1;
|
|
2780
|
-
if (last >= 0 && isOpenHead(last) && !
|
|
2781
|
-
span(last, text.length, true);
|
|
3695
|
+
if (last >= 0 && isOpenHead(last) && !lineHeld(scan, last, text.length, readingKey(fam, last)))
|
|
3696
|
+
span(last, text.length, last, NOT_CLOSED, true);
|
|
2782
3697
|
}
|
|
2783
3698
|
const armPool = (() => {
|
|
2784
3699
|
const spanRepl = (token) => {
|
|
@@ -2816,67 +3731,68 @@ const armPool = (() => {
|
|
|
2816
3731
|
if (isHead && keywordTail !== undefined) {
|
|
2817
3732
|
const at = undecidedKeywordStart(text, keywordTail);
|
|
2818
3733
|
if (at >= 0)
|
|
2819
|
-
out.push({ start: at, end: text.length, repl, rank, order, undecided: true });
|
|
3734
|
+
out.push({ start: at, end: text.length, repl, rank, order, undecided: true, anchor: at, closeAt: NOT_CLOSED });
|
|
2820
3735
|
}
|
|
2821
3736
|
},
|
|
2822
3737
|
retry: boundary === undefined ? undefined : (text, at, out) => collectStickySpans(boundary, text, at, out),
|
|
2823
3738
|
headSrc: boundary === undefined ? undefined : headSrc,
|
|
2824
3739
|
headFirst: boundary === undefined ? undefined : headFirst,
|
|
2825
|
-
family:
|
|
3740
|
+
family: "prose",
|
|
3741
|
+
reach: regexDecisionReach(re, LEXICAL_KEYWORD_ARMS.has(re)),
|
|
2826
3742
|
};
|
|
2827
3743
|
};
|
|
2828
|
-
const scannerMember = (token, order, arm,
|
|
3744
|
+
const scannerMember = (token, order, arm, reach, family) => {
|
|
2829
3745
|
const repl = spanRepl(token);
|
|
3746
|
+
const rank = ARM_RANK.structural;
|
|
2830
3747
|
return {
|
|
2831
3748
|
token,
|
|
2832
3749
|
rank,
|
|
2833
3750
|
order,
|
|
2834
3751
|
phase: 1,
|
|
2835
|
-
lexical,
|
|
2836
|
-
collect: (text, json, eat, out, isHead) => arm(text, json, eat, (start, end, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided } : { start, end, repl: r, rank, order }), isHead),
|
|
3752
|
+
lexical: false,
|
|
3753
|
+
collect: (text, json, eat, out, isHead) => arm(text, json, eat, (start, end, anchor, closeAt, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided, anchor, closeAt } : { start, end, repl: r, rank, order, anchor, closeAt }), isHead),
|
|
2837
3754
|
retry: undefined,
|
|
2838
3755
|
headSrc: undefined,
|
|
2839
3756
|
headFirst: undefined,
|
|
2840
|
-
family
|
|
3757
|
+
family,
|
|
3758
|
+
reach,
|
|
2841
3759
|
};
|
|
2842
3760
|
};
|
|
2843
|
-
const forensicMember = (token, order, arm, family =
|
|
2844
|
-
if (view === "other-families" && family === undefined)
|
|
2845
|
-
throw new Error(`redact: forensic arm ${token} reads the other-families evidence view but declares no credential family`);
|
|
3761
|
+
const forensicMember = (token, order, arm, reach, family, rank = ARM_RANK.structural, lexical = false) => {
|
|
2846
3762
|
const repl = spanRepl(token);
|
|
2847
|
-
const rank = ARM_RANK.structural;
|
|
2848
3763
|
return {
|
|
2849
3764
|
token,
|
|
2850
3765
|
rank,
|
|
2851
3766
|
order,
|
|
2852
3767
|
phase: 2,
|
|
2853
|
-
lexical
|
|
2854
|
-
collect: (scan, text, json, eat, out, isHead) => arm(scan, text, json, eat, (start, end, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided } : { start, end, repl: r, rank, order }), isHead),
|
|
3768
|
+
lexical,
|
|
3769
|
+
collect: (scan, text, json, eat, out, isHead) => arm(scan, text, json, eat, (start, end, anchor, closeAt, undecided, r = repl) => out.push(undecided === true ? { start, end, repl: r, rank, order, undecided, anchor, closeAt } : { start, end, repl: r, rank, order, anchor, closeAt }), isHead),
|
|
2855
3770
|
retry: undefined,
|
|
2856
3771
|
headSrc: undefined,
|
|
2857
3772
|
headFirst: undefined,
|
|
2858
3773
|
family,
|
|
2859
|
-
|
|
3774
|
+
repl,
|
|
3775
|
+
reach,
|
|
2860
3776
|
};
|
|
2861
3777
|
};
|
|
2862
3778
|
const structuralScannerMember = (row, order) => {
|
|
2863
3779
|
if (row === PEM_FENCE_PATTERN)
|
|
2864
|
-
return forensicMember(row[1], order, (scan, text, _json, _eat, span, isHead) => collectPemFenceSpans(scan, text, span, isHead), "pem-armor"
|
|
3780
|
+
return forensicMember(row[1], order, (scan, text, _json, _eat, span, isHead) => collectPemFenceSpans(scan, text, span, isHead), PEM_FENCE_OPEN.length, "pem-armor");
|
|
2865
3781
|
if (row === SSH2_FENCE_PATTERN)
|
|
2866
|
-
return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectSsh2FenceSpans(scan, text, json, span, isHead), "ssh2-armor");
|
|
3782
|
+
return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectSsh2FenceSpans(scan, text, json, span, isHead), SSH2_FENCE_OPEN.length, "ssh2-armor");
|
|
2867
3783
|
if (row === URL_CREDENTIAL_PATTERN)
|
|
2868
|
-
return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectUrlCredentialSpans(scan, text, json, span, isHead));
|
|
3784
|
+
return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectUrlCredentialSpans(scan, text, json, span, isHead), "://".length, "url");
|
|
2869
3785
|
if (row === JSON_SECRET_FIELD_PATTERN)
|
|
2870
|
-
return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectJsonSecretFieldSpans(text, span, isHead));
|
|
3786
|
+
return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectJsonSecretFieldSpans(text, span, isHead), JSON_FIELD_BACK, "json-field");
|
|
2871
3787
|
throw new Error(`redact: structural row /${row[0].source}/ has no hand-written scanner — structural grammars are declared as regexes but never executed as one (S-102 / S-265)`);
|
|
2872
3788
|
};
|
|
2873
3789
|
const members = [];
|
|
2874
3790
|
for (const row of STRUCTURAL_SECRET_PATTERNS)
|
|
2875
3791
|
members.push(structuralScannerMember(row, members.length));
|
|
2876
|
-
members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span, isHead) => collectPpkContainerSpans(scan, text, eat, span, isHead), "putty-ppk"));
|
|
2877
|
-
members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectPemFragmentSpans(scan, text, json, span, isHead), "pem-armor"));
|
|
2878
|
-
members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span, isHead) => collectHeadlessPpkSpans(scan, text, span, isHead), "putty-ppk"));
|
|
2879
|
-
members.push(
|
|
3792
|
+
members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span, isHead) => collectPpkContainerSpans(scan, text, eat, span, isHead), PPK_FILE_HEAD.length, "putty-ppk"));
|
|
3793
|
+
members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectPemFragmentSpans(scan, text, json, span, isHead), PEM_FENCE_OPEN.length, "pem-armor"));
|
|
3794
|
+
members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span, isHead) => collectHeadlessPpkSpans(scan, text, span, isHead), PPK_LINES_FIELD.length, "putty-ppk"));
|
|
3795
|
+
members.push(forensicMember(ANCHORED_VALUE_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectAnchoredValueSpans(scan, text, json, span, isHead), Math.max(BEARER_VALUE_MIN, ABSENT_OR_BOOLEAN_MAX + 1, STRONG_AUTH_SCHEME_MAX + 1), "anchor", ARM_RANK.keyword, true));
|
|
2880
3796
|
for (const row of KEYWORD_SECRET_PATTERNS)
|
|
2881
3797
|
members.push(regexMember(row, ARM_RANK.keyword, members.length));
|
|
2882
3798
|
for (const row of SHAPE_SECRET_PATTERNS)
|
|
@@ -2890,13 +3806,7 @@ const armPool = (() => {
|
|
|
2890
3806
|
const retryMembers = members.filter((m) => m.retry !== undefined);
|
|
2891
3807
|
const phase1 = members.filter((m) => m.phase === 1);
|
|
2892
3808
|
const phase2 = members.filter((m) => m.phase === 2);
|
|
2893
|
-
const
|
|
2894
|
-
const viewKeyOf = (m) => {
|
|
2895
|
-
switch (m.view) {
|
|
2896
|
-
case "all": return undefined;
|
|
2897
|
-
case "other-families": return m.family;
|
|
2898
|
-
}
|
|
2899
|
-
};
|
|
3809
|
+
const familyCodeOfOrder = Int32Array.from(members, (m) => familyCode(m.family));
|
|
2900
3810
|
const laneOf = (scope) => {
|
|
2901
3811
|
const inScope = (m) => {
|
|
2902
3812
|
switch (scope) {
|
|
@@ -2904,15 +3814,10 @@ const armPool = (() => {
|
|
|
2904
3814
|
case "bounded-unit": return !m.lexical;
|
|
2905
3815
|
}
|
|
2906
3816
|
};
|
|
2907
|
-
const lanePhase2 = phase2.filter(inScope);
|
|
2908
|
-
const viewKeys = [...new Set([...lanePhase2.map(viewKeyOf), ...lanePhase2.flatMap((m) => (m.family === undefined ? [] : [m.family]))])];
|
|
2909
3817
|
return {
|
|
2910
3818
|
phase1: phase1.filter(inScope),
|
|
2911
|
-
phase2:
|
|
3819
|
+
phase2: phase2.filter(inScope),
|
|
2912
3820
|
retry: retryMembers.filter(inScope),
|
|
2913
|
-
viewKeys,
|
|
2914
|
-
viewOf: lanePhase2.map((m) => viewKeys.indexOf(viewKeyOf(m))),
|
|
2915
|
-
othersOf: lanePhase2.map((m) => (m.family === undefined ? -1 : viewKeys.indexOf(m.family))),
|
|
2916
3821
|
};
|
|
2917
3822
|
};
|
|
2918
3823
|
const LANES = {
|
|
@@ -2923,21 +3828,39 @@ const armPool = (() => {
|
|
|
2923
3828
|
const headFirstUnit = unionMembers(retryMembers.map((m) => m.headFirst)) ?? new Uint32Array(0x10000 >>> 5).fill(0xffffffff);
|
|
2924
3829
|
poolArmCount = members.length;
|
|
2925
3830
|
poolBoundaryArmCount = retryMembers.length;
|
|
2926
|
-
|
|
3831
|
+
const reaches = members.map((m) => ({ token: m.token, order: m.order, reach: m.reach }));
|
|
3832
|
+
const run = (text, wantCount, json, eat, scope, reach, engineMarked) => {
|
|
2927
3833
|
readings += 1;
|
|
2928
3834
|
const isHead = reach !== undefined;
|
|
2929
3835
|
const lane = LANES[scope];
|
|
2930
3836
|
const spans = [];
|
|
2931
3837
|
let tried;
|
|
3838
|
+
const untriedFrom = (nu, p) => {
|
|
3839
|
+
let q = p;
|
|
3840
|
+
while (nu[q] !== 0) {
|
|
3841
|
+
coverageVisits += 1;
|
|
3842
|
+
q += nu[q];
|
|
3843
|
+
}
|
|
3844
|
+
for (let x = p; x !== q;) {
|
|
3845
|
+
const n = x + nu[x];
|
|
3846
|
+
nu[x] = q - x;
|
|
3847
|
+
x = n;
|
|
3848
|
+
}
|
|
3849
|
+
return q;
|
|
3850
|
+
};
|
|
2932
3851
|
const freshCovered = (from) => {
|
|
2933
3852
|
const at = [];
|
|
2934
|
-
const seen = (tried ??= new
|
|
3853
|
+
const seen = (tried ??= new Int32Array(text.length + 2));
|
|
2935
3854
|
for (let k = from; k < spans.length; k++) {
|
|
2936
3855
|
const s = spans[k];
|
|
2937
3856
|
for (let p = s.start; p <= s.end; p++) {
|
|
2938
|
-
if (seen[p]
|
|
2939
|
-
|
|
3857
|
+
if (seen[p] !== 0) {
|
|
3858
|
+
p = untriedFrom(seen, p);
|
|
3859
|
+
if (p > s.end)
|
|
3860
|
+
break;
|
|
3861
|
+
}
|
|
2940
3862
|
seen[p] = 1;
|
|
3863
|
+
coverageVisits += 1;
|
|
2941
3864
|
coveredPositions += 1;
|
|
2942
3865
|
if (unitBit(headFirstUnit, text.charCodeAt(p)) !== 1)
|
|
2943
3866
|
continue;
|
|
@@ -2964,26 +3887,23 @@ const armPool = (() => {
|
|
|
2964
3887
|
m.collect(text, json, eat, spans, isHead);
|
|
2965
3888
|
closeCoveragePositions(0);
|
|
2966
3889
|
if (lane.phase2.length > 0) {
|
|
2967
|
-
const
|
|
2968
|
-
const scan = buildForensicScan(tokens, lane.phase2.length);
|
|
3890
|
+
const scan = buildForensicScan(tokenUnits(text), buildEvidenceRows(text), lane.phase2.length);
|
|
2969
3891
|
const reuse = forensicRescanMode === "incremental";
|
|
2970
3892
|
const foot = scan.armFoot;
|
|
2971
|
-
|
|
3893
|
+
let dirty;
|
|
2972
3894
|
let folded = 0;
|
|
2973
3895
|
for (let level = 0;; level += 1) {
|
|
2974
3896
|
const before = spans.length;
|
|
2975
3897
|
for (let j = 0; j < lane.phase2.length; j++) {
|
|
2976
|
-
const view = views[lane.viewOf[j]];
|
|
2977
|
-
const oi = lane.othersOf[j];
|
|
2978
|
-
const others = oi < 0 ? undefined : views[oi];
|
|
2979
|
-
const dirty = view.dirty;
|
|
2980
3898
|
const lo = foot[2 * j];
|
|
2981
3899
|
const hi = foot[2 * j + 1];
|
|
2982
|
-
if (dirty !== undefined && footprintClean(dirty, lo, hi)
|
|
3900
|
+
if (dirty !== undefined && footprintClean(dirty, lo, hi)) {
|
|
2983
3901
|
forensicSkips += 1;
|
|
2984
3902
|
continue;
|
|
2985
3903
|
}
|
|
2986
|
-
|
|
3904
|
+
scan.dirty = dirty;
|
|
3905
|
+
scan.rowCursor = 0;
|
|
3906
|
+
scan.self = NO_SELF;
|
|
2987
3907
|
scan.footLo = FOOT_NONE;
|
|
2988
3908
|
scan.footHi = -1;
|
|
2989
3909
|
lane.phase2[j].collect(scan, text, json, level === 0 ? eat : undefined, spans, isHead);
|
|
@@ -2993,18 +3913,26 @@ const armPool = (() => {
|
|
|
2993
3913
|
}
|
|
2994
3914
|
closeCoveragePositions(before);
|
|
2995
3915
|
forensicLevels += 1;
|
|
2996
|
-
|
|
2997
|
-
for (const view of views) {
|
|
2998
|
-
const key = view.key;
|
|
2999
|
-
const grown = foldCredentialSpans(view.table, spans, folded, key === undefined ? undefined : (sp) => familyOfOrder[sp.order] !== key);
|
|
3000
|
-
if (grown.dirty.starts.length > 0)
|
|
3001
|
-
grew = true;
|
|
3002
|
-
view.table = grown.next;
|
|
3003
|
-
view.dirty = reuse ? grown.dirty : undefined;
|
|
3004
|
-
}
|
|
3916
|
+
const grown = foldEvidenceRows(scan.rows, spans, folded, (o) => familyCodeOfOrder[o]);
|
|
3005
3917
|
folded = spans.length;
|
|
3006
|
-
if (
|
|
3918
|
+
if (grown.dirty.starts.length === 0)
|
|
3919
|
+
break;
|
|
3920
|
+
if (level + 1 >= FORENSIC_LEVEL_MAX) {
|
|
3921
|
+
for (let j = 0; j < lane.phase2.length; j++) {
|
|
3922
|
+
const lo = foot[2 * j];
|
|
3923
|
+
if (footprintClean(grown.dirty, lo, foot[2 * j + 1]))
|
|
3924
|
+
continue;
|
|
3925
|
+
const m = lane.phase2[j];
|
|
3926
|
+
spans.push({ start: lo, end: text.length, repl: m.repl, rank: m.rank, order: m.order, anchor: lo, closeAt: NOT_CLOSED });
|
|
3927
|
+
if (eat !== undefined && eat.capFrom < 0) {
|
|
3928
|
+
eat.capFrom = lo;
|
|
3929
|
+
eat.capLen = text.length;
|
|
3930
|
+
}
|
|
3931
|
+
}
|
|
3007
3932
|
break;
|
|
3933
|
+
}
|
|
3934
|
+
scan.rows = grown.next;
|
|
3935
|
+
dirty = reuse ? grown.dirty : undefined;
|
|
3008
3936
|
}
|
|
3009
3937
|
}
|
|
3010
3938
|
poolSpanCount += spans.length;
|
|
@@ -3035,6 +3963,7 @@ const armPool = (() => {
|
|
|
3035
3963
|
}
|
|
3036
3964
|
return { hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })), kept: stamped };
|
|
3037
3965
|
};
|
|
3966
|
+
return Object.assign(run, { reaches, pendingTailUnits: Math.max(...reaches.map((r) => r.reach)) });
|
|
3038
3967
|
})();
|
|
3039
3968
|
let onRedaction;
|
|
3040
3969
|
export function setRedactionObserver(fn) {
|
|
@@ -3132,12 +4061,18 @@ function readHexRunValue(s, from, to) {
|
|
|
3132
4061
|
return v;
|
|
3133
4062
|
}
|
|
3134
4063
|
function shortestReading(text, layout) {
|
|
4064
|
+
const r = readShortest(text, layout);
|
|
4065
|
+
return { visible: r.visible, partial: r.partial };
|
|
4066
|
+
}
|
|
4067
|
+
function readShortest(text, layout) {
|
|
3135
4068
|
const raw = collectRawControlCuts(text, invisibleClassesOf(layout));
|
|
3136
|
-
const
|
|
4069
|
+
const rfold = deleteInvisibleCharacters(text, raw, raw.present);
|
|
4070
|
+
const normalized = normalizeForRedaction(rfold.text);
|
|
3137
4071
|
const open = trailingPartialEscapeStart(normalized);
|
|
3138
4072
|
const spelled = collectSpelledCuts(normalized);
|
|
3139
|
-
const
|
|
3140
|
-
|
|
4073
|
+
const sfold = deleteInvisibleCharacters(normalized, spelled, spelled.present);
|
|
4074
|
+
const layers = [{ fold: sfold }, { input: rfold.text, view: normalized }, { fold: rfold }];
|
|
4075
|
+
return open < 0 ? { visible: sfold.text.length, partial: false, layers } : { visible: sfold.text.length - (normalized.length - open), partial: true, layers };
|
|
3141
4076
|
}
|
|
3142
4077
|
export function readingHeadEnd(text, layout, units, from = 0) {
|
|
3143
4078
|
const rest = text.length - from;
|
|
@@ -3161,7 +4096,7 @@ export function readingHeadEnd(text, layout, units, from = 0) {
|
|
|
3161
4096
|
}
|
|
3162
4097
|
function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
|
|
3163
4098
|
const wantCount = wantHits || (onRedaction !== undefined && observe);
|
|
3164
|
-
const eat = observe ? { from: -1, len: 0 } : undefined;
|
|
4099
|
+
const eat = observe ? { from: -1, len: 0, capFrom: -1, capLen: 0 } : undefined;
|
|
3165
4100
|
let spans = [];
|
|
3166
4101
|
let hits = NO_HITS;
|
|
3167
4102
|
let readsSoFar = 0;
|
|
@@ -3202,6 +4137,8 @@ function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
|
|
|
3202
4137
|
hits = hitsOfSpans(spans);
|
|
3203
4138
|
if (eat !== undefined && eat.from >= 0)
|
|
3204
4139
|
recordFailOpen("server.redact.ppk-eat-to-end", `from=${eat.from} len=${eat.len}`);
|
|
4140
|
+
if (eat !== undefined && eat.capFrom >= 0)
|
|
4141
|
+
recordFailOpen("server.redact.forensic-level-cap", `from=${eat.capFrom} len=${eat.capLen}`);
|
|
3205
4142
|
if (observe)
|
|
3206
4143
|
reportHits(hits);
|
|
3207
4144
|
return { text: applySpans(text, spans).out, hits, spans };
|
|
@@ -3251,6 +4188,91 @@ export function redactionSpansForTest(text, opts) {
|
|
|
3251
4188
|
export function proposeRedactionSpans(text, scope, layout, extent) {
|
|
3252
4189
|
return runRedaction(text, true, false, false, scope, layout, extent).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
|
|
3253
4190
|
}
|
|
4191
|
+
export const PENDING_TAIL_UNITS = armPool.pendingTailUnits;
|
|
4192
|
+
export function regexDecisionReachForTest(re, lexical) {
|
|
4193
|
+
return regexDecisionReach(re, lexical);
|
|
4194
|
+
}
|
|
4195
|
+
export function pendingTailReachForTest() {
|
|
4196
|
+
return armPool.reaches;
|
|
4197
|
+
}
|
|
4198
|
+
function pendingTailStart(text, units) {
|
|
4199
|
+
const r = readShortest(text, "multi-line");
|
|
4200
|
+
return r.visible <= units ? 0 : createViewToOriginal(r.layers).start(r.visible - units);
|
|
4201
|
+
}
|
|
4202
|
+
function renderRange(text, spans, from, until) {
|
|
4203
|
+
if (from >= until)
|
|
4204
|
+
return "";
|
|
4205
|
+
const parts = [];
|
|
4206
|
+
let at = from;
|
|
4207
|
+
for (const s of spans) {
|
|
4208
|
+
if (s.end <= from)
|
|
4209
|
+
continue;
|
|
4210
|
+
if (s.start >= until)
|
|
4211
|
+
break;
|
|
4212
|
+
if (s.start > at)
|
|
4213
|
+
parts.push(text.slice(at, s.start));
|
|
4214
|
+
parts.push(s.repl);
|
|
4215
|
+
at = Math.max(at, s.end);
|
|
4216
|
+
}
|
|
4217
|
+
if (at < until)
|
|
4218
|
+
parts.push(text.slice(at, until));
|
|
4219
|
+
return parts.join("");
|
|
4220
|
+
}
|
|
4221
|
+
const STREAM_RESCAN_MIN = 64;
|
|
4222
|
+
const streamRescanStep = (n) => Math.max(STREAM_RESCAN_MIN, n >>> 5);
|
|
4223
|
+
export function createRedactionStream() {
|
|
4224
|
+
return openRedactionStream(PENDING_TAIL_UNITS);
|
|
4225
|
+
}
|
|
4226
|
+
export function createRedactionStreamWithTailForTest(units) {
|
|
4227
|
+
return openRedactionStream(units);
|
|
4228
|
+
}
|
|
4229
|
+
function openRedactionStream(pendingTail) {
|
|
4230
|
+
let buf = "";
|
|
4231
|
+
let released = 0;
|
|
4232
|
+
let deliveredFrom = 0;
|
|
4233
|
+
let sent = "";
|
|
4234
|
+
let scannedAt = 0;
|
|
4235
|
+
return {
|
|
4236
|
+
feed(chunk) {
|
|
4237
|
+
buf += chunk;
|
|
4238
|
+
},
|
|
4239
|
+
takeSettled() {
|
|
4240
|
+
if (buf.length - released <= pendingTail)
|
|
4241
|
+
return "";
|
|
4242
|
+
if (buf.length - scannedAt < streamRescanStep(buf.length))
|
|
4243
|
+
return "";
|
|
4244
|
+
scannedAt = buf.length;
|
|
4245
|
+
const emitEnd = pendingTailStart(buf, pendingTail);
|
|
4246
|
+
if (emitEnd <= released)
|
|
4247
|
+
return "";
|
|
4248
|
+
const spans = runRedaction(buf, false, false, false, "free-text", "multi-line", { kind: "head", emitEnd }).spans;
|
|
4249
|
+
let cut = emitEnd;
|
|
4250
|
+
for (const s of spans)
|
|
4251
|
+
if (s.start < cut && s.end > cut)
|
|
4252
|
+
cut = s.start;
|
|
4253
|
+
if (cut > 0 && cut < buf.length && (buf.charCodeAt(cut - 1) & 0xfc00) === 0xd800 && (buf.charCodeAt(cut) & 0xfc00) === 0xdc00)
|
|
4254
|
+
cut -= 1;
|
|
4255
|
+
if (cut <= released)
|
|
4256
|
+
return "";
|
|
4257
|
+
const out = renderRange(buf, spans, released, cut);
|
|
4258
|
+
released = cut;
|
|
4259
|
+
sent += out;
|
|
4260
|
+
return out;
|
|
4261
|
+
},
|
|
4262
|
+
flush() {
|
|
4263
|
+
const spans = runRedaction(buf, false, false, false, "free-text", "multi-line", WHOLE_EXTENT).spans;
|
|
4264
|
+
const split = released > deliveredFrom && spans.some((s) => s.start < released && s.end > released);
|
|
4265
|
+
return { rest: renderRange(buf, spans, released, buf.length), diverged: split || renderRange(buf, spans, deliveredFrom, released) !== sent };
|
|
4266
|
+
},
|
|
4267
|
+
skip() {
|
|
4268
|
+
released = deliveredFrom = scannedAt = buf.length;
|
|
4269
|
+
sent = "";
|
|
4270
|
+
},
|
|
4271
|
+
get fed() {
|
|
4272
|
+
return buf;
|
|
4273
|
+
},
|
|
4274
|
+
};
|
|
4275
|
+
}
|
|
3254
4276
|
const REDACT_MAX_DEPTH = 64;
|
|
3255
4277
|
export function redactDeep(value) {
|
|
3256
4278
|
return redactDeepWith(value, redactSecrets);
|