@sema-agent/server 7.132.0 → 7.133.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +266 -0
- package/README.md +8 -9
- package/README.zh-CN.md +7 -6
- package/USAGE.md +174 -55
- package/deploy/sema-up/README.md +19 -1
- package/deploy/sema-up/chart/templates/server-v4pro.yaml +5 -2
- package/deploy/sema-up/chart/templates/server.yaml +5 -2
- package/deploy/sema-up/stack/compose.yaml +3 -0
- package/deploy/sema-up/stack/sema-stack-cluster.yaml +4 -2
- package/dist/adoption/plan.d.ts +3 -2
- package/dist/adoption/plan.js +7 -5
- package/dist/adoption/runner.js +3 -10
- package/dist/adoption/sql.d.ts +3 -2
- package/dist/adoption/sql.js +3 -5
- package/dist/adoption/wire.d.ts +4 -1
- package/dist/adoption/wire.js +1 -0
- package/dist/approval-ask-audit-store.d.ts +31 -13
- package/dist/approval-ask-audit-store.js +272 -74
- package/dist/approval-ask-machine.d.ts +15 -0
- package/dist/approval-ask-machine.js +6 -0
- package/dist/approval-ask-retention.d.ts +136 -0
- package/dist/approval-ask-retention.js +102 -0
- package/dist/approval-card.d.ts +79 -19
- package/dist/approval-card.js +44 -13
- package/dist/approval-deny-reasons.d.ts +1 -1
- package/dist/approval.d.ts +9 -0
- package/dist/approval.js +24 -3
- package/dist/audit.d.ts +27 -10
- package/dist/audit.js +12 -16
- package/dist/bake-runner/main.js +3 -2
- package/dist/boot/deferred-sandbox-path-env.js +1 -1
- package/dist/boot/execution-env.js +9 -7
- package/dist/boot/execution-lane.js +11 -0
- package/dist/boot/leader.js +2 -0
- package/dist/boot/memory-boundary.js +2 -2
- package/dist/boot/memory-consolidation.d.ts +5 -4
- package/dist/boot/memory-consolidation.js +7 -6
- package/dist/boot/org-delivery.d.ts +28 -0
- package/dist/boot/org-delivery.js +20 -0
- package/dist/boot/reapers.d.ts +3 -0
- package/dist/boot/reapers.js +21 -1
- package/dist/boot/resolve-spec.d.ts +3 -0
- package/dist/boot/resolve-spec.js +60 -81
- package/dist/boot/runner-deps.js +1 -0
- package/dist/boot/session-faces.d.ts +1 -0
- package/dist/boot/session-faces.js +4 -2
- package/dist/boot/stage-02-web-admin-pull.js +4 -1
- package/dist/boot/stage-06-runners.d.ts +2 -0
- package/dist/boot/stage-06-runners.js +6 -3
- package/dist/boot/stage-07-capability-layer.d.ts +2 -0
- package/dist/boot/stage-07-capability-layer.js +24 -3
- package/dist/boot/stage-08-reapers.d.ts +2 -0
- package/dist/boot/stage-08-reapers.js +2 -2
- package/dist/boot/stage-09-leader.d.ts +3 -0
- package/dist/boot/stage-10-http-server.d.ts +3 -0
- package/dist/boot/stage-10-http-server.js +5 -6
- package/dist/boot/stores.js +2 -2
- package/dist/boot/web-admin.d.ts +4 -0
- package/dist/boot/web-admin.js +151 -63
- package/dist/boot/workflow-orchestration.d.ts +2 -0
- package/dist/boot/workflow-orchestration.js +2 -1
- package/dist/brain.js +8 -3
- package/dist/breaker-key.d.ts +6 -0
- package/dist/breaker-key.js +29 -0
- package/dist/caller-text-echo.d.ts +9 -6
- package/dist/caller-text-echo.js +8 -11
- package/dist/capabilities/center-plugins.d.ts +7 -0
- package/dist/capabilities/center-plugins.js +1 -1
- package/dist/capabilities/composite-members.d.ts +1 -1
- package/dist/capabilities/composite-members.js +1 -1
- package/dist/capabilities/execution-lane.js +2 -2
- package/dist/capabilities/hands-lane.d.ts +21 -0
- package/dist/capabilities/hands-lane.js +10 -0
- package/dist/capabilities/mcp-probe.d.ts +36 -4
- package/dist/capabilities/mcp-probe.js +27 -4
- package/dist/capabilities/scenarios.d.ts +63 -7
- package/dist/capabilities/scenarios.js +39 -7
- package/dist/capability-facts.d.ts +34 -4
- package/dist/capability-facts.js +28 -2
- package/dist/center-config-tier.d.ts +69 -0
- package/dist/center-config-tier.js +42 -0
- package/dist/checkpoint-identity.d.ts +16 -0
- package/dist/checkpoint-identity.js +19 -0
- package/dist/config-catalog.d.ts +17 -10
- package/dist/config-catalog.js +43 -28
- package/dist/config-d-migration.d.ts +3 -1
- package/dist/config-d-migration.js +2 -2
- package/dist/config-dir-trust.d.ts +114 -0
- package/dist/config-dir-trust.js +388 -0
- package/dist/config-invariants.d.ts +18 -5
- package/dist/config-invariants.js +35 -3
- package/dist/config-provider.d.ts +6 -1
- package/dist/config-provider.js +6 -2
- package/dist/config-types.d.ts +71 -14
- package/dist/config-types.js +8 -0
- package/dist/config.d.ts +13 -0
- package/dist/config.js +121 -59
- package/dist/delegated-session-owner.d.ts +185 -0
- package/dist/delegated-session-owner.js +119 -0
- package/dist/device-protocol.d.ts +5 -3
- package/dist/device-protocol.js +1 -1
- package/dist/device-ws-hub.js +21 -2
- package/dist/device-ws-protocol.d.ts +70 -6
- package/dist/device-ws-protocol.js +50 -23
- package/dist/elicitation.js +3 -2
- package/dist/entry-problems.d.ts +34 -0
- package/dist/entry-problems.js +14 -0
- package/dist/execution-lane-caps.d.ts +49 -10
- package/dist/execution-lane-caps.js +43 -4
- package/dist/field-conflict.d.ts +43 -0
- package/dist/field-conflict.js +20 -0
- package/dist/fleet/fleet-bus.d.ts +80 -6
- package/dist/fleet/fleet-bus.js +41 -9
- package/dist/hooks/classic-stations.d.ts +10 -5
- package/dist/hooks/classic-stations.js +4 -4
- package/dist/hooks/command-spawn.d.ts +57 -0
- package/dist/hooks/command-spawn.js +139 -0
- package/dist/hooks/hook-llm.js +2 -2
- package/dist/hooks/hook-runner.d.ts +127 -25
- package/dist/hooks/hook-runner.js +668 -287
- package/dist/hooks/hook-source.d.ts +44 -0
- package/dist/hooks/hook-source.js +40 -0
- package/dist/http/active-run-conflict.d.ts +5 -13
- package/dist/http/active-run-conflict.js +17 -27
- package/dist/http/admission.js +103 -60
- package/dist/http/approval-sweeps.js +55 -18
- package/dist/http/caller-trust.d.ts +77 -0
- package/dist/http/caller-trust.js +143 -0
- package/dist/http/checkpoint-ctx.d.ts +16 -16
- package/dist/http/checkpoint-ctx.js +13 -10
- package/dist/http/claimed-run.d.ts +4 -3
- package/dist/http/device-placement.d.ts +2 -4
- package/dist/http/device-placement.js +2 -1
- package/dist/http/dispatch.js +10 -4
- package/dist/http/durable-run-leg.d.ts +4 -32
- package/dist/http/durable-run-leg.js +34 -10
- package/dist/http/owner-access.d.ts +56 -0
- package/dist/http/owner-access.js +42 -0
- package/dist/http/principal-gate.d.ts +48 -23
- package/dist/http/principal-gate.js +39 -12
- package/dist/http/readiness.d.ts +11 -5
- package/dist/http/readiness.js +10 -3
- package/dist/http/request-guards.d.ts +16 -23
- package/dist/http/request-guards.js +30 -15
- package/dist/http/resume-ctx-read.d.ts +2 -5
- package/dist/http/resume-ctx-read.js +7 -4
- package/dist/http/resume-legs.d.ts +3 -4
- package/dist/http/resume-legs.js +126 -69
- package/dist/http/retired-service-deps.d.ts +43 -0
- package/dist/http/retired-service-deps.js +18 -0
- package/dist/http/route-ctx.d.ts +44 -28
- package/dist/http/route-ctx.js +4 -0
- package/dist/http/route-table.js +2 -0
- package/dist/http/routes/a2a-serve.js +3 -4
- package/dist/http/routes/admin-config-refresh.js +1 -1
- package/dist/http/routes/admin-drain.js +1 -1
- package/dist/http/routes/admin-memory-optout.js +5 -1
- package/dist/http/routes/adoption.js +2 -9
- package/dist/http/routes/approvals-assistant.js +49 -48
- package/dist/http/routes/background.js +3 -2
- package/dist/http/routes/capabilities.js +20 -4
- package/dist/http/routes/config-catalog.d.ts +3 -3
- package/dist/http/routes/config-catalog.js +1 -1
- package/dist/http/routes/devices.js +3 -17
- package/dist/http/routes/diagnostics.js +5 -1
- package/dist/http/routes/images.js +6 -7
- package/dist/http/routes/leader.js +1 -8
- package/dist/http/routes/mcp-reconnect.d.ts +1 -1
- package/dist/http/routes/mcp-reconnect.js +3 -2
- package/dist/http/routes/memory-bundle.js +1 -1
- package/dist/http/routes/memory-compliance.js +1 -1
- package/dist/http/routes/memory-consolidation.js +1 -1
- package/dist/http/routes/memory-origin.js +1 -1
- package/dist/http/routes/memory-policy.d.ts +1 -1
- package/dist/http/routes/memory-policy.js +3 -2
- package/dist/http/routes/notify-wake.js +8 -21
- package/dist/http/routes/observability.js +2 -2
- package/dist/http/routes/principal-inbox.d.ts +39 -0
- package/dist/http/routes/principal-inbox.js +76 -0
- package/dist/http/routes/retention-ops.js +2 -9
- package/dist/http/routes/rules.js +5 -18
- package/dist/http/routes/runs.d.ts +5 -0
- package/dist/http/routes/runs.js +33 -106
- package/dist/http/routes/session-sync.js +15 -28
- package/dist/http/routes/sessions.js +30 -40
- package/dist/http/routes/side-query.js +6 -4
- package/dist/http/routes/tasks.js +50 -14
- package/dist/http/routes/workflows.d.ts +351 -0
- package/dist/http/routes/workflows.js +153 -119
- package/dist/http/send.d.ts +24 -3
- package/dist/http/send.js +14 -2
- package/dist/http/server.d.ts +17 -18
- package/dist/http/server.js +15 -13
- package/dist/http/stored-body-replay.js +17 -4
- package/dist/http/submit-ctx.js +4 -3
- package/dist/http/submit-replay.d.ts +4 -3
- package/dist/http/submit-replay.js +5 -4
- package/dist/http/wire-types.d.ts +10 -7
- package/dist/index.d.ts +1 -1
- package/dist/index.js +0 -1
- package/dist/lane-home-probe.d.ts +78 -9
- package/dist/lane-home-probe.js +78 -12
- package/dist/leader/endpoint.d.ts +4 -1
- package/dist/leader/endpoint.js +11 -10
- package/dist/limit-refusal.d.ts +17 -5
- package/dist/limit-refusal.js +4 -1
- package/dist/mcp-assertion-header.d.ts +31 -10
- package/dist/mcp-assertion-header.js +10 -3
- package/dist/mcp-tool-faces.js +1 -3
- package/dist/memory-project-layer.d.ts +37 -0
- package/dist/memory-project-layer.js +46 -0
- package/dist/memory-scope.d.ts +23 -16
- package/dist/memory-scope.js +24 -8
- package/dist/memory-sync-client.d.ts +27 -13
- package/dist/memory-sync-client.js +107 -18
- package/dist/model-entry-refusal.d.ts +9 -1
- package/dist/model-entry-refusal.js +6 -1
- package/dist/model-plane-face.d.ts +51 -0
- package/dist/model-plane-face.js +24 -0
- package/dist/model-provider.d.ts +5 -0
- package/dist/model-provider.js +4 -2
- package/dist/model-route-endpoint.d.ts +8 -11
- package/dist/model-route-endpoint.js +4 -7
- package/dist/model-route-env.d.ts +22 -0
- package/dist/model-route-env.js +2 -0
- package/dist/model-route-gap.d.ts +60 -30
- package/dist/model-route-gap.js +21 -15
- package/dist/observability/corrupt-read-seat.d.ts +7 -0
- package/dist/observability/corrupt-read-seat.js +9 -0
- package/dist/observability/err-text.d.ts +6 -0
- package/dist/observability/err-text.js +18 -4
- package/dist/observability/fail-open.d.ts +35 -9
- package/dist/observability/fail-open.js +35 -9
- package/dist/observability/metrics.js +1 -0
- package/dist/observability/prompt-manifest.d.ts +3 -17
- package/dist/observability/prompt-manifest.js +45 -42
- package/dist/observability/tool-trace.js +9 -9
- package/dist/operator-ask.d.ts +10 -0
- package/dist/operator-ask.js +3 -0
- package/dist/orchestration/workflow-agent-session-index.d.ts +12 -12
- package/dist/orchestration/workflow-agent-session-index.js +9 -13
- package/dist/orchestration/workflow-completion-inbox.d.ts +42 -15
- package/dist/orchestration/workflow-completion-inbox.js +13 -0
- package/dist/orchestration/workflow-notify-journal.d.ts +13 -32
- package/dist/orchestration/workflow-notify-journal.js +27 -34
- package/dist/orchestration/workflow-park-decision.d.ts +89 -0
- package/dist/orchestration/workflow-park-decision.js +118 -0
- package/dist/org-delivery-gate.d.ts +81 -0
- package/dist/org-delivery-gate.js +59 -0
- package/dist/panel-tool-names.d.ts +23 -0
- package/dist/panel-tool-names.js +8 -0
- package/dist/parked-decide.d.ts +39 -6
- package/dist/parked-decide.js +98 -51
- package/dist/parked-revive-surface.d.ts +9 -2
- package/dist/parked-revive-surface.js +11 -8
- package/dist/permission-rule-list.d.ts +1 -1
- package/dist/permission-rule-list.js +2 -2
- package/dist/plugins/approval-ask-store-file.d.ts +16 -1
- package/dist/plugins/approval-ask-store-file.js +21 -0
- package/dist/plugins/approval-ask-store-memory.d.ts +18 -1
- package/dist/plugins/approval-ask-store-memory.js +34 -2
- package/dist/plugins/approval-ask-store-sql.d.ts +25 -0
- package/dist/plugins/approval-ask-store-sql.js +37 -1
- package/dist/plugins/blob-backend.d.ts +38 -25
- package/dist/plugins/blob-backend.js +29 -19
- package/dist/plugins/breaker-state-sql.d.ts +2 -2
- package/dist/plugins/breaker-state-sql.js +4 -4
- package/dist/plugins/caching-session-store.d.ts +3 -0
- package/dist/plugins/caching-session-store.js +3 -0
- package/dist/plugins/checkpoint-store-sql.d.ts +14 -0
- package/dist/plugins/checkpoint-store-sql.js +20 -2
- package/dist/plugins/file-history-store-sql.d.ts +5 -2
- package/dist/plugins/file-history-store-sql.js +3 -1
- package/dist/plugins/file-run-store.d.ts +5 -13
- package/dist/plugins/file-run-store.js +9 -0
- package/dist/plugins/fork-routing-session-store.d.ts +9 -1
- package/dist/plugins/fork-routing-session-store.js +12 -1
- package/dist/plugins/host-platform.d.ts +2 -0
- package/dist/plugins/host-platform.js +2 -1
- package/dist/plugins/local-checkpoint-store.d.ts +2 -0
- package/dist/plugins/local-checkpoint-store.js +6 -1
- package/dist/plugins/local-session-store.d.ts +1 -1
- package/dist/plugins/local-session-window.d.ts +39 -0
- package/dist/plugins/local-session-window.js +54 -0
- package/dist/plugins/mailbox-recipient-lifecycle.d.ts +3 -2
- package/dist/plugins/mailbox-store-sql.d.ts +30 -48
- package/dist/plugins/mailbox-store-sql.js +77 -52
- package/dist/plugins/memory-embedder-fingerprint.js +2 -1
- package/dist/plugins/memory-run-store.d.ts +5 -13
- package/dist/plugins/memory-run-store.js +8 -0
- package/dist/plugins/one-time-migrations.d.ts +110 -5
- package/dist/plugins/one-time-migrations.js +320 -8
- package/dist/plugins/org-delivery-quota-store.d.ts +123 -0
- package/dist/plugins/org-delivery-quota-store.js +228 -0
- package/dist/plugins/pg-pool.js +39 -7
- package/dist/plugins/pg-session-storage.d.ts +17 -4
- package/dist/plugins/pg-session-storage.js +29 -6
- package/dist/plugins/remote-env-adb.d.ts +6 -12
- package/dist/plugins/remote-env-adb.js +12 -2
- package/dist/plugins/remote-env-k8s.d.ts +8 -12
- package/dist/plugins/remote-env-k8s.js +23 -5
- package/dist/plugins/remote-env-local-docker.d.ts +10 -15
- package/dist/plugins/remote-env-local-docker.js +12 -9
- package/dist/plugins/remote-env-ssh.d.ts +10 -16
- package/dist/plugins/remote-env-ssh.js +17 -11
- package/dist/plugins/run-store-sql.d.ts +16 -14
- package/dist/plugins/run-store-sql.js +22 -2
- package/dist/plugins/schema-indexes.js +2 -0
- package/dist/plugins/send-user-file.js +2 -2
- package/dist/plugins/session-epoch-anchor.d.ts +32 -6
- package/dist/plugins/session-epoch-anchor.js +2 -3
- package/dist/plugins/session-store.d.ts +18 -8
- package/dist/plugins/session-store.js +15 -4
- package/dist/plugins/sql-driver.d.ts +74 -11
- package/dist/plugins/sql-driver.js +210 -12
- package/dist/plugins/sql-errors.d.ts +23 -0
- package/dist/plugins/sql-errors.js +21 -0
- package/dist/plugins/store-backend.d.ts +36 -1
- package/dist/plugins/store-backend.js +40 -14
- package/dist/plugins/store-contracts.d.ts +42 -0
- package/dist/plugins/store-contracts.js +1 -0
- package/dist/plugins/tidb-pool.d.ts +6 -1
- package/dist/plugins/tidb-pool.js +44 -10
- package/dist/plugins/tidb-session-storage.d.ts +3 -2
- package/dist/plugins/tidb-session-storage.js +2 -2
- package/dist/plugins/tidb-session-store.d.ts +24 -4
- package/dist/plugins/tidb-session-store.js +28 -4
- package/dist/plugins/workflow-journal-store-sql.d.ts +73 -19
- package/dist/plugins/workflow-journal-store-sql.js +19 -6
- package/dist/plugins/workflow-run-store-sql.d.ts +30 -8
- package/dist/plugins/workflow-run-store-sql.js +50 -9
- package/dist/posture-source.d.ts +18 -10
- package/dist/posture-source.js +4 -1
- package/dist/project-identity.d.ts +45 -6
- package/dist/project-identity.js +162 -8
- package/dist/project-layer-key.d.ts +28 -0
- package/dist/project-layer-key.js +29 -0
- package/dist/question.js +3 -2
- package/dist/read-deny-entry.d.ts +25 -0
- package/dist/read-deny-entry.js +12 -0
- package/dist/read-face-posture.d.ts +37 -11
- package/dist/read-face-posture.js +32 -3
- package/dist/request-key-closure.d.ts +7 -0
- package/dist/request-key-closure.js +9 -1
- package/dist/request-value-closure.d.ts +5 -0
- package/dist/request-value-closure.js +11 -3
- package/dist/rule-entry-list.d.ts +26 -2
- package/dist/rule-entry-list.js +72 -12
- package/dist/rules-consent.js +3 -4
- package/dist/run-acceptance.d.ts +19 -3
- package/dist/run-acceptance.js +13 -1
- package/dist/run-local.js +36 -6
- package/dist/runs.d.ts +10 -9
- package/dist/runs.js +21 -3
- package/dist/runtime-caps-resolver.d.ts +16 -6
- package/dist/runtime-caps-resolver.js +18 -5
- package/dist/sandbox-pkg-source.d.ts +19 -1
- package/dist/sandbox-pkg-source.js +29 -7
- package/dist/security.d.ts +87 -9
- package/dist/security.js +52 -9
- package/dist/session-error-wire.d.ts +19 -2
- package/dist/session-error-wire.js +5 -1
- package/dist/session-titled-frame.d.ts +33 -0
- package/dist/session-titled-frame.js +4 -0
- package/dist/session-titler.d.ts +9 -1
- package/dist/session-titler.js +15 -4
- package/dist/spec-fields.js +21 -2
- package/dist/sql-engine-posture.d.ts +3 -1
- package/dist/task-cwd.d.ts +1 -1
- package/dist/task-mcp.d.ts +3 -3
- package/dist/task-mcp.js +7 -66
- package/dist/task-settings.d.ts +89 -88
- package/dist/task-settings.js +71 -63
- package/dist/text-bidi.d.ts +0 -4
- package/dist/text-bidi.js +1 -3
- package/dist/tighten-only-seats.d.ts +150 -0
- package/dist/tighten-only-seats.js +196 -0
- package/dist/tool-approval.d.ts +303 -87
- package/dist/tool-approval.js +623 -239
- package/dist/trace/core-keyset-guard.d.ts +104 -22
- package/dist/trace/exit-seat.d.ts +28 -0
- package/dist/trace/exit-seat.js +76 -0
- package/dist/trace/ledger-events.d.ts +3 -2
- package/dist/trace/project.d.ts +80 -12
- package/dist/trace/project.js +41 -28
- package/dist/trace/projection-table.d.ts +100 -0
- package/dist/trace/projection-table.js +130 -0
- package/dist/trace/redact.d.ts +54 -0
- package/dist/trace/redact.js +778 -152
- package/dist/trace/self-minted-wire.d.ts +345 -1
- package/dist/trace/self-minted-wire.js +117 -1
- package/dist/trace/wire-projection-faces.d.ts +298 -2
- package/dist/trace/wire-projection-faces.js +154 -13
- package/dist/url-refusal.js +2 -0
- package/dist/web-admin-client/apply-effective.d.ts +12 -0
- package/dist/web-admin-client/apply-effective.js +56 -5
- package/dist/web-admin-client/facade.d.ts +1 -1
- package/dist/web-admin-client/facade.js +1 -1
- package/dist/web-admin-client/read-face.js +5 -9
- package/dist/web-admin-client/skills-mcp.js +4 -5
- package/docs/DEVICE-EXECUTOR-PROTOCOL.md +57 -9
- package/package.json +7 -3
package/dist/trace/redact.js
CHANGED
|
@@ -276,10 +276,14 @@ let coveredPositions = 0;
|
|
|
276
276
|
let headScreenHits = 0;
|
|
277
277
|
let fullYCalls = 0;
|
|
278
278
|
let forensicLevels = 0;
|
|
279
|
+
let forensicRuns = 0;
|
|
280
|
+
let forensicSkips = 0;
|
|
281
|
+
let urlSegmentRuns = 0;
|
|
282
|
+
let urlSegmentReplays = 0;
|
|
279
283
|
let ssh2ScanBytes = 0;
|
|
280
284
|
let mappedUnits = 0;
|
|
281
285
|
export function armPoolBudgetForTest() {
|
|
282
|
-
return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, ssh2ScanBytes, mappedUnits };
|
|
286
|
+
return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits };
|
|
283
287
|
}
|
|
284
288
|
export function resetArmPoolBudgetForTest() {
|
|
285
289
|
poolSpanCount = 0;
|
|
@@ -287,6 +291,10 @@ export function resetArmPoolBudgetForTest() {
|
|
|
287
291
|
headScreenHits = 0;
|
|
288
292
|
fullYCalls = 0;
|
|
289
293
|
forensicLevels = 0;
|
|
294
|
+
forensicRuns = 0;
|
|
295
|
+
forensicSkips = 0;
|
|
296
|
+
urlSegmentRuns = 0;
|
|
297
|
+
urlSegmentReplays = 0;
|
|
290
298
|
ssh2ScanBytes = 0;
|
|
291
299
|
mappedUnits = 0;
|
|
292
300
|
}
|
|
@@ -631,16 +639,17 @@ const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") => {
|
|
|
631
639
|
LEXICAL_KEYWORD_ARMS.add(re);
|
|
632
640
|
return re;
|
|
633
641
|
};
|
|
634
|
-
const
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
"auth", "authorization",
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
"x-amz-signature", "x-amz-security-token", "x-amz-credential",
|
|
642
|
-
|
|
643
|
-
|
|
642
|
+
const CREDENTIAL_NAME_ANCHORS = {
|
|
643
|
+
api_key: "structure-only", apikey: "structure-only", "api-key": "structure-only", key: "structure-only",
|
|
644
|
+
token: "structure-only", access_token: "structure-only", refresh_token: "structure-only", id_token: "structure-only",
|
|
645
|
+
auth: "structure-only", authorization: "auth-scheme", "proxy-authorization": "auth-scheme",
|
|
646
|
+
password: "declarative", passwd: "any-form", pwd: "any-form",
|
|
647
|
+
secret: "declarative", client_secret: "declarative", private_key: "structure-only",
|
|
648
|
+
signature: "structure-only", sig: "structure-only",
|
|
649
|
+
"x-amz-signature": "structure-only", "x-amz-security-token": "structure-only", "x-amz-credential": "structure-only",
|
|
650
|
+
sas: "structure-only",
|
|
651
|
+
};
|
|
652
|
+
const CREDENTIAL_NAME_WORDS = Object.keys(CREDENTIAL_NAME_ANCHORS);
|
|
644
653
|
const CREDENTIAL_NAME_ALT = (() => {
|
|
645
654
|
for (const w of CREDENTIAL_NAME_WORDS) {
|
|
646
655
|
if (!/^[a-z][a-z0-9_-]*$/.test(w))
|
|
@@ -657,6 +666,7 @@ const STRUCTURAL_SECRET_PATTERNS = [
|
|
|
657
666
|
const PPK_CONTAINER_TOKEN = "«redacted:private-key»";
|
|
658
667
|
const PEM_FRAGMENT_TOKEN = "«redacted:private-key:partial»";
|
|
659
668
|
const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
|
|
669
|
+
const ANCHORED_VALUE_TOKEN = "«redacted»";
|
|
660
670
|
const KEYWORD_SECRET_PATTERNS = [
|
|
661
671
|
[keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", 20, "=*"), "Bearer «redacted»"],
|
|
662
672
|
[keywordArm("Basic", "[A-Za-z0-9+/]", 16, "={0,2}"), "Basic «redacted»"],
|
|
@@ -689,11 +699,11 @@ function pemEscapeLetterAt(s, i, limit) {
|
|
|
689
699
|
const n = s.charCodeAt(j);
|
|
690
700
|
return n === CH_r || n === CH_n || n === CH_t ? j : -1;
|
|
691
701
|
}
|
|
692
|
-
function skipPemSeps(s, i, limit) {
|
|
702
|
+
function skipPemSeps(scan, s, i, limit) {
|
|
693
703
|
for (;;) {
|
|
694
704
|
if (i >= limit)
|
|
695
705
|
return i;
|
|
696
|
-
if (opaqueUnitEnd(i) >= 0)
|
|
706
|
+
if (opaqueUnitEnd(scan, i) >= 0)
|
|
697
707
|
return i;
|
|
698
708
|
const c = s.charCodeAt(i);
|
|
699
709
|
if (isSepCode(c)) {
|
|
@@ -736,17 +746,17 @@ function pemInlineSepLen(s, i, limit) {
|
|
|
736
746
|
}
|
|
737
747
|
return 0;
|
|
738
748
|
}
|
|
739
|
-
function probePemHeaderToken(s, p, limit) {
|
|
740
|
-
if (p >= limit || opaqueUnitEnd(p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
|
|
749
|
+
function probePemHeaderToken(scan, s, p, limit) {
|
|
750
|
+
if (p >= limit || opaqueUnitEnd(scan, p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
|
|
741
751
|
return -1;
|
|
742
752
|
p += 1;
|
|
743
753
|
let identLen = 1;
|
|
744
|
-
while (identLen < PEM_HEADER_IDENT_MAX && p < limit && opaqueUnitEnd(p) < 0 && isIdentCode(s.charCodeAt(p))) {
|
|
754
|
+
while (identLen < PEM_HEADER_IDENT_MAX && p < limit && opaqueUnitEnd(scan, p) < 0 && isIdentCode(s.charCodeAt(p))) {
|
|
745
755
|
p += 1;
|
|
746
756
|
identLen += 1;
|
|
747
757
|
}
|
|
748
758
|
for (;;) {
|
|
749
|
-
if (p >= limit || opaqueUnitEnd(p) >= 0)
|
|
759
|
+
if (p >= limit || opaqueUnitEnd(scan, p) >= 0)
|
|
750
760
|
return -1;
|
|
751
761
|
const c = s.charCodeAt(p);
|
|
752
762
|
if (c === CH_SPACE || c === CH_TAB) {
|
|
@@ -756,29 +766,29 @@ function probePemHeaderToken(s, p, limit) {
|
|
|
756
766
|
return c === CH_COLON ? p + 1 : -1;
|
|
757
767
|
}
|
|
758
768
|
}
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
769
|
+
function setPemHdr(scan, search, contentEnd, next, hitLimit, jsonEnd = false) {
|
|
770
|
+
const h = scan.hdr;
|
|
771
|
+
h.search = search;
|
|
772
|
+
h.contentEnd = contentEnd;
|
|
773
|
+
h.next = next;
|
|
774
|
+
h.hitLimit = hitLimit;
|
|
775
|
+
h.jsonEnd = jsonEnd;
|
|
766
776
|
return true;
|
|
767
777
|
}
|
|
768
|
-
function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
769
|
-
const start = skipPemSeps(s, i, limit);
|
|
778
|
+
function parsePemHeaderLine(scan, s, i, limit, allowBare, json) {
|
|
779
|
+
const start = skipPemSeps(scan, s, i, limit);
|
|
770
780
|
if (start >= limit)
|
|
771
781
|
return false;
|
|
772
|
-
const afterToken = probePemHeaderToken(s, start, limit);
|
|
782
|
+
const afterToken = probePemHeaderToken(scan, s, start, limit);
|
|
773
783
|
if (afterToken < 0 && !allowBare)
|
|
774
784
|
return false;
|
|
775
785
|
let p = afterToken < 0 ? start : afterToken;
|
|
776
|
-
if (afterToken >= 0 && probePemHeaderToken(s, p, limit) >= 0)
|
|
777
|
-
return setPemHdr(start, p, p, false);
|
|
786
|
+
if (afterToken >= 0 && probePemHeaderToken(scan, s, p, limit) >= 0)
|
|
787
|
+
return setPemHdr(scan, start, p, p, false);
|
|
778
788
|
for (;;) {
|
|
779
789
|
if (p >= limit)
|
|
780
|
-
return setPemHdr(start, p, p, true);
|
|
781
|
-
const unit = opaqueUnitEnd(p);
|
|
790
|
+
return setPemHdr(scan, start, p, p, true);
|
|
791
|
+
const unit = opaqueUnitEnd(scan, p);
|
|
782
792
|
if (unit >= 0) {
|
|
783
793
|
p = Math.min(unit, limit);
|
|
784
794
|
continue;
|
|
@@ -790,12 +800,12 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
790
800
|
}
|
|
791
801
|
if (c === CH_QUOTE) {
|
|
792
802
|
if (json)
|
|
793
|
-
return setPemHdr(start, p, p, false, true);
|
|
803
|
+
return setPemHdr(scan, start, p, p, false, true);
|
|
794
804
|
p += 1;
|
|
795
805
|
continue;
|
|
796
806
|
}
|
|
797
807
|
if (c === CH_CR || c === CH_LF)
|
|
798
|
-
return setPemHdr(start, p, p + 1, false);
|
|
808
|
+
return setPemHdr(scan, start, p, p + 1, false);
|
|
799
809
|
if (c === CH_BACKSLASH) {
|
|
800
810
|
let j = p;
|
|
801
811
|
while (j < limit && s.charCodeAt(j) === CH_BACKSLASH)
|
|
@@ -808,11 +818,11 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
808
818
|
}
|
|
809
819
|
let q = p;
|
|
810
820
|
for (;;) {
|
|
811
|
-
if (opaqueUnitEnd(q) >= 0)
|
|
821
|
+
if (opaqueUnitEnd(scan, q) >= 0)
|
|
812
822
|
break;
|
|
813
823
|
const b = pemLineBreakLen(s, q, limit);
|
|
814
824
|
if (b === 1)
|
|
815
|
-
return setPemHdr(start, p, q + 1, false);
|
|
825
|
+
return setPemHdr(scan, start, p, q + 1, false);
|
|
816
826
|
if (b >= 2) {
|
|
817
827
|
q += b;
|
|
818
828
|
continue;
|
|
@@ -823,19 +833,19 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
823
833
|
q += u;
|
|
824
834
|
}
|
|
825
835
|
if (q >= limit)
|
|
826
|
-
return setPemHdr(start, p, q, true);
|
|
827
|
-
if (probePemHeaderToken(s, q, limit) >= 0)
|
|
828
|
-
return setPemHdr(start, p, q, false);
|
|
836
|
+
return setPemHdr(scan, start, p, q, true);
|
|
837
|
+
if (probePemHeaderToken(scan, s, q, limit) >= 0)
|
|
838
|
+
return setPemHdr(scan, start, p, q, false);
|
|
829
839
|
p = q;
|
|
830
840
|
}
|
|
831
841
|
}
|
|
832
|
-
function parsePemBody(s, i, limit) {
|
|
842
|
+
function parsePemBody(scan, s, i, limit) {
|
|
833
843
|
let end = -1;
|
|
834
844
|
let p = i;
|
|
835
845
|
for (;;) {
|
|
836
|
-
const q = skipPemSeps(s, p, limit);
|
|
846
|
+
const q = skipPemSeps(scan, s, p, limit);
|
|
837
847
|
let r = q;
|
|
838
|
-
while (r < limit && opaqueUnitEnd(r) < 0 && isBase64Code(s.charCodeAt(r)))
|
|
848
|
+
while (r < limit && opaqueUnitEnd(scan, r) < 0 && isBase64Code(s.charCodeAt(r)))
|
|
839
849
|
r += 1;
|
|
840
850
|
if (r - q < PEM_BODY_RUN_MIN)
|
|
841
851
|
break;
|
|
@@ -844,12 +854,12 @@ function parsePemBody(s, i, limit) {
|
|
|
844
854
|
}
|
|
845
855
|
return end;
|
|
846
856
|
}
|
|
847
|
-
function rightmostPemRunEnd(s, from, to) {
|
|
857
|
+
function rightmostPemRunEnd(scan, s, from, to) {
|
|
848
858
|
let best = -1;
|
|
849
859
|
let runStart = -1;
|
|
850
860
|
for (let p = from; p <= to; p++) {
|
|
851
861
|
if (p < to) {
|
|
852
|
-
const unit = opaqueUnitEnd(p);
|
|
862
|
+
const unit = opaqueUnitEnd(scan, p);
|
|
853
863
|
if (unit > 0) {
|
|
854
864
|
if (runStart >= 0 && p - runStart >= PEM_BODY_RUN_MIN)
|
|
855
865
|
best = p;
|
|
@@ -893,7 +903,7 @@ function isPemFenceLabel(s, from, to) {
|
|
|
893
903
|
}
|
|
894
904
|
const PEM_FENCE_OPEN = "-----BEGIN";
|
|
895
905
|
const PEM_FENCE_CLOSE = "-----";
|
|
896
|
-
function findPemFence(text, from) {
|
|
906
|
+
function findPemFence(scan, text, from) {
|
|
897
907
|
for (;;) {
|
|
898
908
|
const fence = text.indexOf(PEM_FENCE_OPEN, from);
|
|
899
909
|
if (fence < 0)
|
|
@@ -903,7 +913,7 @@ function findPemFence(text, from) {
|
|
|
903
913
|
if (close < 0)
|
|
904
914
|
return undefined;
|
|
905
915
|
from = labelFrom;
|
|
906
|
-
if (inOpaqueUnit(fence))
|
|
916
|
+
if (inOpaqueUnit(scan, fence))
|
|
907
917
|
continue;
|
|
908
918
|
if (!isPemFenceLabel(text, labelFrom, close))
|
|
909
919
|
continue;
|
|
@@ -913,26 +923,28 @@ function findPemFence(text, from) {
|
|
|
913
923
|
const SSH2_FENCE_OPEN = "---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
914
924
|
const SSH2_FENCE_CLOSE = "---- END SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
915
925
|
const SSH2_BODY_MAX = 8192;
|
|
916
|
-
function unitCountWithin(from, to, max) {
|
|
926
|
+
function unitCountWithin(scan, from, to, max) {
|
|
917
927
|
if (to - from <= max)
|
|
918
928
|
return true;
|
|
919
|
-
|
|
929
|
+
noteEvidenceRead(scan, from);
|
|
930
|
+
noteEvidenceRead(scan, to);
|
|
931
|
+
const st = scan.table.starts;
|
|
920
932
|
if (st.length === 0)
|
|
921
933
|
return false;
|
|
922
|
-
const k = firstCredentialSpanFrom(
|
|
934
|
+
const k = firstCredentialSpanFrom(scan.table, from);
|
|
923
935
|
if (k >= st.length || st[k] >= to)
|
|
924
936
|
return false;
|
|
925
937
|
let n = 0;
|
|
926
938
|
for (let p = from; p < to;) {
|
|
927
939
|
if (n > max)
|
|
928
940
|
return false;
|
|
929
|
-
const unit = opaqueUnitEnd(p);
|
|
941
|
+
const unit = opaqueUnitEnd(scan, p);
|
|
930
942
|
p = unit > 0 && unit <= to ? unit : p + 1;
|
|
931
943
|
n += 1;
|
|
932
944
|
}
|
|
933
945
|
return n <= max;
|
|
934
946
|
}
|
|
935
|
-
function collectSsh2FenceSpans(text, span) {
|
|
947
|
+
function collectSsh2FenceSpans(scan, text, span) {
|
|
936
948
|
let cachedFrom = 0;
|
|
937
949
|
let cachedAt;
|
|
938
950
|
const nextClose = (from) => {
|
|
@@ -940,7 +952,7 @@ function collectSsh2FenceSpans(text, span) {
|
|
|
940
952
|
return cachedAt;
|
|
941
953
|
let p = text.indexOf(SSH2_FENCE_CLOSE, from);
|
|
942
954
|
ssh2ScanBytes += (p < 0 ? text.length : p) - from;
|
|
943
|
-
while (p >= 0 && inOpaqueUnit(p)) {
|
|
955
|
+
while (p >= 0 && inOpaqueUnit(scan, p)) {
|
|
944
956
|
const q = text.indexOf(SSH2_FENCE_CLOSE, p + 1);
|
|
945
957
|
ssh2ScanBytes += (q < 0 ? text.length : q) - (p + 1);
|
|
946
958
|
p = q;
|
|
@@ -953,7 +965,7 @@ function collectSsh2FenceSpans(text, span) {
|
|
|
953
965
|
const fence = text.indexOf(SSH2_FENCE_OPEN, from);
|
|
954
966
|
if (fence < 0)
|
|
955
967
|
return;
|
|
956
|
-
if (inOpaqueUnit(fence)) {
|
|
968
|
+
if (inOpaqueUnit(scan, fence)) {
|
|
957
969
|
from = fence + 1;
|
|
958
970
|
continue;
|
|
959
971
|
}
|
|
@@ -961,7 +973,7 @@ function collectSsh2FenceSpans(text, span) {
|
|
|
961
973
|
const close = nextClose(bodyFrom);
|
|
962
974
|
if (close < 0)
|
|
963
975
|
return;
|
|
964
|
-
if (!unitCountWithin(bodyFrom, close, SSH2_BODY_MAX)) {
|
|
976
|
+
if (!unitCountWithin(scan, bodyFrom, close, SSH2_BODY_MAX)) {
|
|
965
977
|
from = fence + 1;
|
|
966
978
|
continue;
|
|
967
979
|
}
|
|
@@ -1011,31 +1023,31 @@ function collectPemFenceSpans(text, span) {
|
|
|
1011
1023
|
from = scanResumeAfterHit(fence, endAfter, PEM_FENCE_OPEN.length);
|
|
1012
1024
|
}
|
|
1013
1025
|
}
|
|
1014
|
-
function collectPemFragmentSpans(text, json, span) {
|
|
1015
|
-
let cur = findPemFence(text, 0);
|
|
1026
|
+
function collectPemFragmentSpans(scan, text, json, span) {
|
|
1027
|
+
let cur = findPemFence(scan, text, 0);
|
|
1016
1028
|
let inherited = false;
|
|
1017
1029
|
while (cur !== undefined) {
|
|
1018
|
-
const next = findPemFence(text, cur.after);
|
|
1030
|
+
const next = findPemFence(scan, text, cur.after);
|
|
1019
1031
|
const limit = next === undefined ? text.length : next.fence;
|
|
1020
1032
|
let pos = cur.after;
|
|
1021
1033
|
let allowBare = inherited;
|
|
1022
1034
|
let hitLimit = false;
|
|
1023
1035
|
let rightmost = -1;
|
|
1024
|
-
while (parsePemHeaderLine(text, pos, limit, allowBare, json)) {
|
|
1025
|
-
const e = rightmostPemRunEnd(text,
|
|
1036
|
+
while (parsePemHeaderLine(scan, text, pos, limit, allowBare, json)) {
|
|
1037
|
+
const e = rightmostPemRunEnd(scan, text, scan.hdr.search, scan.hdr.contentEnd);
|
|
1026
1038
|
if (e > rightmost)
|
|
1027
1039
|
rightmost = e;
|
|
1028
|
-
pos =
|
|
1040
|
+
pos = scan.hdr.next;
|
|
1029
1041
|
allowBare = false;
|
|
1030
|
-
if (
|
|
1042
|
+
if (scan.hdr.hitLimit) {
|
|
1031
1043
|
hitLimit = true;
|
|
1032
1044
|
break;
|
|
1033
1045
|
}
|
|
1034
|
-
if (
|
|
1046
|
+
if (scan.hdr.jsonEnd)
|
|
1035
1047
|
break;
|
|
1036
1048
|
}
|
|
1037
1049
|
inherited = hitLimit;
|
|
1038
|
-
let bodyEnd = parsePemBody(text, pos, limit);
|
|
1050
|
+
let bodyEnd = parsePemBody(scan, text, pos, limit);
|
|
1039
1051
|
if (bodyEnd < 0)
|
|
1040
1052
|
bodyEnd = rightmost;
|
|
1041
1053
|
if (bodyEnd >= 0)
|
|
@@ -1043,18 +1055,18 @@ function collectPemFragmentSpans(text, json, span) {
|
|
|
1043
1055
|
cur = next;
|
|
1044
1056
|
}
|
|
1045
1057
|
}
|
|
1046
|
-
function collectHeadlessPpkSpans(text, span) {
|
|
1058
|
+
function collectHeadlessPpkSpans(scan, text, span) {
|
|
1047
1059
|
const KEY = "Private-Lines: ";
|
|
1048
1060
|
const n = text.length;
|
|
1049
1061
|
const isB64 = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 43 || c === 47 || c === 61;
|
|
1050
|
-
const b64At = (i) => opaqueUnitEnd(i) < 0 && isB64(text.charCodeAt(i));
|
|
1051
|
-
const skipHs = (i) => { while (i < n && opaqueUnitEnd(i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
|
|
1062
|
+
const b64At = (i) => opaqueUnitEnd(scan, i) < 0 && isB64(text.charCodeAt(i));
|
|
1063
|
+
const skipHs = (i) => { while (i < n && opaqueUnitEnd(scan, i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
|
|
1052
1064
|
i += 1; return i; };
|
|
1053
1065
|
const skipNewlines = (i) => {
|
|
1054
1066
|
let q = skipHs(i);
|
|
1055
1067
|
let seen = false;
|
|
1056
1068
|
for (;;) {
|
|
1057
|
-
if (opaqueUnitEnd(q) >= 0)
|
|
1069
|
+
if (opaqueUnitEnd(scan, q) >= 0)
|
|
1058
1070
|
break;
|
|
1059
1071
|
let any = false;
|
|
1060
1072
|
if (text.charCodeAt(q) === 13) {
|
|
@@ -1087,7 +1099,7 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1087
1099
|
return seen ? q : -1;
|
|
1088
1100
|
};
|
|
1089
1101
|
const escapedNewlineLen = (i) => {
|
|
1090
|
-
if (opaqueUnitEnd(i) >= 0)
|
|
1102
|
+
if (opaqueUnitEnd(scan, i) >= 0)
|
|
1091
1103
|
return 0;
|
|
1092
1104
|
let k = i;
|
|
1093
1105
|
while (k < n && text.charCodeAt(k) === 92)
|
|
@@ -1101,7 +1113,7 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1101
1113
|
const b = skipHs(i);
|
|
1102
1114
|
if (b >= n)
|
|
1103
1115
|
return true;
|
|
1104
|
-
if (opaqueUnitEnd(b) >= 0)
|
|
1116
|
+
if (opaqueUnitEnd(scan, b) >= 0)
|
|
1105
1117
|
return false;
|
|
1106
1118
|
const c = text.charCodeAt(b);
|
|
1107
1119
|
return c === 10 || c === 13 || c === 34 || escapedNewlineLen(b) > 0;
|
|
@@ -1111,13 +1123,13 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1111
1123
|
const at = text.indexOf(KEY, search);
|
|
1112
1124
|
if (at === -1)
|
|
1113
1125
|
break;
|
|
1114
|
-
if (inOpaqueUnit(at)) {
|
|
1126
|
+
if (inOpaqueUnit(scan, at)) {
|
|
1115
1127
|
search = at + KEY.length;
|
|
1116
1128
|
continue;
|
|
1117
1129
|
}
|
|
1118
1130
|
let p = at + KEY.length;
|
|
1119
1131
|
const d0 = p;
|
|
1120
|
-
while (p < n && opaqueUnitEnd(p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
|
|
1132
|
+
while (p < n && opaqueUnitEnd(scan, p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
|
|
1121
1133
|
p += 1;
|
|
1122
1134
|
if (p === d0) {
|
|
1123
1135
|
search = at + KEY.length;
|
|
@@ -1130,7 +1142,7 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1130
1142
|
if (q0 === -1)
|
|
1131
1143
|
break;
|
|
1132
1144
|
let q = q0;
|
|
1133
|
-
const mk = opaqueUnitEnd(q) < 0 ? text.charCodeAt(q) : -1;
|
|
1145
|
+
const mk = opaqueUnitEnd(scan, q) < 0 ? text.charCodeAt(q) : -1;
|
|
1134
1146
|
if (mk === 43 || mk === 45)
|
|
1135
1147
|
q = skipHs(q + 1);
|
|
1136
1148
|
let r = q;
|
|
@@ -1169,7 +1181,7 @@ const FRAGMENT_SECRET_PATTERNS = [
|
|
|
1169
1181
|
const DISPLAY_SAFE_URL_TOKEN = "«redacted:url»";
|
|
1170
1182
|
export { DISPLAY_SAFE_URL_TOKEN };
|
|
1171
1183
|
const WITHHELD_VALUE_TOKEN = "«redacted»";
|
|
1172
|
-
const REDACTION_TOKENS = new Set([...FULL_ARM_PATTERNS.map(([, r]) => r), PEM_FRAGMENT_TOKEN, DISPLAY_SAFE_URL_TOKEN, ...FRAGMENT_SECRET_PATTERNS.map(([, r]) => r)]
|
|
1184
|
+
const REDACTION_TOKENS = new Set([...FULL_ARM_PATTERNS.map(([, r]) => r), PEM_FRAGMENT_TOKEN, ANCHORED_VALUE_TOKEN, DISPLAY_SAFE_URL_TOKEN, ...FRAGMENT_SECRET_PATTERNS.map(([, r]) => r)]
|
|
1173
1185
|
.flatMap((r) => r.match(/«[^»]*»/g) ?? []));
|
|
1174
1186
|
{
|
|
1175
1187
|
const closed = new RegExp(`^${REDACTION_TOKEN_RE_SRC}$`);
|
|
@@ -1285,8 +1297,8 @@ function tokenUnitEnd(text, p) {
|
|
|
1285
1297
|
return TOKEN_UNIT_Y.test(text) ? TOKEN_UNIT_Y.lastIndex : -1;
|
|
1286
1298
|
}
|
|
1287
1299
|
const NO_CREDENTIAL_SPANS = { starts: new Int32Array(0), ends: new Int32Array(0) };
|
|
1288
|
-
function credentialSpans(text
|
|
1289
|
-
const iv =
|
|
1300
|
+
function credentialSpans(text) {
|
|
1301
|
+
const iv = [];
|
|
1290
1302
|
for (let i = text.indexOf("«"); i >= 0; i = text.indexOf("«", i + 1)) {
|
|
1291
1303
|
const e = tokenUnitEnd(text, i);
|
|
1292
1304
|
if (e > 0) {
|
|
@@ -1318,13 +1330,103 @@ function credentialSpans(text, spans, withSpans) {
|
|
|
1318
1330
|
k += 1;
|
|
1319
1331
|
return { starts: starts.subarray(0, k), ends: ends.subarray(0, k) };
|
|
1320
1332
|
}
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1333
|
+
let forensicRescanMode = "incremental";
|
|
1334
|
+
export function setForensicRescanModeForTest(mode) {
|
|
1335
|
+
forensicRescanMode = mode;
|
|
1336
|
+
}
|
|
1337
|
+
function foldCredentialSpans(prev, spans, from) {
|
|
1338
|
+
const cap = spans.length - from;
|
|
1339
|
+
if (cap === 0)
|
|
1340
|
+
return { next: prev, dirty: NO_CREDENTIAL_SPANS };
|
|
1341
|
+
const Sa = new Int32Array(cap);
|
|
1342
|
+
const Ea = new Int32Array(cap);
|
|
1343
|
+
let m = 0;
|
|
1344
|
+
for (let i = from; i < spans.length; i++) {
|
|
1345
|
+
const sp = spans[i];
|
|
1346
|
+
const k = firstCredentialSpanFrom(prev, sp.start + 1) - 1;
|
|
1347
|
+
if (k >= 0 && prev.ends[k] >= sp.end)
|
|
1348
|
+
continue;
|
|
1349
|
+
Sa[m] = sp.start;
|
|
1350
|
+
Ea[m] = sp.end;
|
|
1351
|
+
m += 1;
|
|
1352
|
+
}
|
|
1353
|
+
if (m === 0)
|
|
1354
|
+
return { next: prev, dirty: NO_CREDENTIAL_SPANS };
|
|
1355
|
+
const S = Sa.subarray(0, m).sort();
|
|
1356
|
+
const E = Ea.subarray(0, m).sort();
|
|
1357
|
+
const cs = new Int32Array(m);
|
|
1358
|
+
const ce = new Int32Array(m);
|
|
1359
|
+
let C = 0;
|
|
1360
|
+
for (let i = 0, j = 0, depth = 0, open = 0; i < m || j < m;) {
|
|
1361
|
+
if (i < m && (j >= m || S[i] <= E[j])) {
|
|
1362
|
+
if (depth === 0)
|
|
1363
|
+
open = S[i];
|
|
1364
|
+
depth += 1;
|
|
1365
|
+
i += 1;
|
|
1366
|
+
}
|
|
1367
|
+
else {
|
|
1368
|
+
depth -= 1;
|
|
1369
|
+
if (depth === 0) {
|
|
1370
|
+
cs[C] = open;
|
|
1371
|
+
ce[C] = E[j];
|
|
1372
|
+
C += 1;
|
|
1373
|
+
}
|
|
1374
|
+
j += 1;
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
const P = prev.starts.length;
|
|
1378
|
+
const oS = new Int32Array(P + C);
|
|
1379
|
+
const oE = new Int32Array(P + C);
|
|
1380
|
+
const dS = new Int32Array(C + P);
|
|
1381
|
+
const dE = new Int32Array(C + P);
|
|
1382
|
+
let on = 0;
|
|
1383
|
+
let dn = 0;
|
|
1384
|
+
let curS = 0;
|
|
1385
|
+
let curE = 0;
|
|
1386
|
+
let sources = 0;
|
|
1387
|
+
let onlyPrev = false;
|
|
1388
|
+
const flush = () => {
|
|
1389
|
+
if (sources === 0)
|
|
1390
|
+
return;
|
|
1391
|
+
oS[on] = curS;
|
|
1392
|
+
oE[on] = curE;
|
|
1393
|
+
on += 1;
|
|
1394
|
+
if (!(sources === 1 && onlyPrev)) {
|
|
1395
|
+
dS[dn] = curS;
|
|
1396
|
+
dE[dn] = curE;
|
|
1397
|
+
dn += 1;
|
|
1398
|
+
}
|
|
1399
|
+
};
|
|
1400
|
+
for (let pi = 0, ci = 0; pi < P || ci < C;) {
|
|
1401
|
+
const takePrev = ci >= C || (pi < P && prev.starts[pi] <= cs[ci]);
|
|
1402
|
+
const s = takePrev ? prev.starts[pi] : cs[ci];
|
|
1403
|
+
const e = takePrev ? prev.ends[pi] : ce[ci];
|
|
1404
|
+
if (takePrev)
|
|
1405
|
+
pi += 1;
|
|
1406
|
+
else
|
|
1407
|
+
ci += 1;
|
|
1408
|
+
if (sources > 0 && s <= curE) {
|
|
1409
|
+
if (e > curE)
|
|
1410
|
+
curE = e;
|
|
1411
|
+
sources += 1;
|
|
1412
|
+
}
|
|
1413
|
+
else {
|
|
1414
|
+
flush();
|
|
1415
|
+
curS = s;
|
|
1416
|
+
curE = e;
|
|
1417
|
+
sources = 1;
|
|
1418
|
+
onlyPrev = takePrev;
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
flush();
|
|
1422
|
+
return { next: { starts: oS.subarray(0, on), ends: oE.subarray(0, on) }, dirty: { starts: dS.subarray(0, dn), ends: dE.subarray(0, dn) } };
|
|
1423
|
+
}
|
|
1424
|
+
export function credentialSpansFoldForTest(prev, add) {
|
|
1425
|
+
const table = (iv) => ({ starts: Int32Array.from(iv, (x) => x[0]), ends: Int32Array.from(iv, (x) => x[1]) });
|
|
1426
|
+
const back = (t) => Array.from(t.starts, (s, i) => [s, t.ends[i]]);
|
|
1427
|
+
const spans = add.map(([start, end]) => ({ start, end, repl: "", rank: 0, order: 0 }));
|
|
1428
|
+
const { next, dirty } = foldCredentialSpans(table(prev), spans, 0);
|
|
1429
|
+
return { next: back(next), dirty: back(dirty) };
|
|
1328
1430
|
}
|
|
1329
1431
|
function firstCredentialSpanFrom(cs, from) {
|
|
1330
1432
|
let lo = 0;
|
|
@@ -1338,43 +1440,90 @@ function firstCredentialSpanFrom(cs, from) {
|
|
|
1338
1440
|
}
|
|
1339
1441
|
return lo;
|
|
1340
1442
|
}
|
|
1341
|
-
function credentialSpansCover(
|
|
1443
|
+
function credentialSpansCover(scan, from, to) {
|
|
1444
|
+
noteEvidenceRead(scan, from);
|
|
1445
|
+
noteEvidenceRead(scan, to);
|
|
1446
|
+
const cs = scan.table;
|
|
1342
1447
|
let k = firstCredentialSpanFrom(cs, from + 1) - 1;
|
|
1343
1448
|
if (k < 0)
|
|
1344
1449
|
k = 0;
|
|
1345
1450
|
return k < cs.starts.length && cs.starts[k] <= from && cs.ends[k] >= to;
|
|
1346
1451
|
}
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1452
|
+
const FOOT_NONE = 0x7fffffff;
|
|
1453
|
+
function buildForensicScan(table, armCount) {
|
|
1454
|
+
return {
|
|
1455
|
+
table,
|
|
1456
|
+
cursor: 0,
|
|
1457
|
+
dirty: undefined,
|
|
1458
|
+
footLo: FOOT_NONE,
|
|
1459
|
+
footHi: -1,
|
|
1460
|
+
armFoot: new Int32Array(2 * armCount),
|
|
1461
|
+
lastAt: -1,
|
|
1462
|
+
headsRead: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1463
|
+
headsWrite: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1464
|
+
paramsRead: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1465
|
+
paramsWrite: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1466
|
+
hdr: { search: 0, contentEnd: 0, next: 0, hitLimit: false, jsonEnd: false },
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
function advanceEvidence(scan, table, dirty) {
|
|
1470
|
+
scan.table = table;
|
|
1471
|
+
scan.cursor = 0;
|
|
1472
|
+
scan.dirty = dirty;
|
|
1473
|
+
}
|
|
1474
|
+
function noteEvidenceRead(scan, p) {
|
|
1475
|
+
if (p < scan.footLo)
|
|
1476
|
+
scan.footLo = p;
|
|
1477
|
+
if (p > scan.footHi)
|
|
1478
|
+
scan.footHi = p;
|
|
1479
|
+
}
|
|
1480
|
+
function footprintClean(dirty, lo, hi) {
|
|
1481
|
+
if (lo > hi)
|
|
1482
|
+
return true;
|
|
1483
|
+
const ends = dirty.ends;
|
|
1484
|
+
let a = 0;
|
|
1485
|
+
let b = ends.length;
|
|
1486
|
+
while (a < b) {
|
|
1487
|
+
const mid = (a + b) >>> 1;
|
|
1488
|
+
if (ends[mid] < lo)
|
|
1489
|
+
a = mid + 1;
|
|
1490
|
+
else
|
|
1491
|
+
b = mid;
|
|
1492
|
+
}
|
|
1493
|
+
return a >= ends.length || dirty.starts[a] > hi;
|
|
1352
1494
|
}
|
|
1353
|
-
function inOpaqueUnit(p) {
|
|
1354
|
-
|
|
1495
|
+
function inOpaqueUnit(scan, p) {
|
|
1496
|
+
noteEvidenceRead(scan, p);
|
|
1497
|
+
const t = scan.table;
|
|
1498
|
+
const st = t.starts;
|
|
1355
1499
|
if (st.length === 0)
|
|
1356
1500
|
return false;
|
|
1357
|
-
const k = firstCredentialSpanFrom(
|
|
1358
|
-
return k >= 0 && st[k] < p &&
|
|
1501
|
+
const k = firstCredentialSpanFrom(t, p) - 1;
|
|
1502
|
+
return k >= 0 && st[k] < p && t.ends[k] > p;
|
|
1503
|
+
}
|
|
1504
|
+
function opaqueUnitEnd(scan, p) {
|
|
1505
|
+
noteEvidenceRead(scan, p);
|
|
1506
|
+
return opaqueUnitEndAt(scan, p);
|
|
1359
1507
|
}
|
|
1360
|
-
function
|
|
1361
|
-
const
|
|
1508
|
+
function opaqueUnitEndAt(scan, p) {
|
|
1509
|
+
const t = scan.table;
|
|
1510
|
+
const st = t.starts;
|
|
1362
1511
|
if (st.length === 0)
|
|
1363
1512
|
return -1;
|
|
1364
|
-
let k =
|
|
1513
|
+
let k = scan.cursor;
|
|
1365
1514
|
while (k < st.length && st[k] < p)
|
|
1366
1515
|
k += 1;
|
|
1367
1516
|
while (k > 0 && st[k - 1] >= p)
|
|
1368
1517
|
k -= 1;
|
|
1369
|
-
|
|
1370
|
-
return k < st.length && st[k] === p ?
|
|
1518
|
+
scan.cursor = k;
|
|
1519
|
+
return k < st.length && st[k] === p ? t.ends[k] : -1;
|
|
1371
1520
|
}
|
|
1372
|
-
|
|
1373
|
-
function scanUrlRun(text, i, kind, json) {
|
|
1521
|
+
function scanUrlRun(scan, text, i, kind, json) {
|
|
1374
1522
|
const n = text.length;
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1523
|
+
noteEvidenceRead(scan, i);
|
|
1524
|
+
scan.lastAt = -1;
|
|
1525
|
+
run: while (i < n) {
|
|
1526
|
+
const unit = opaqueUnitEndAt(scan, i);
|
|
1378
1527
|
if (unit >= 0) {
|
|
1379
1528
|
i = unit;
|
|
1380
1529
|
continue;
|
|
@@ -1385,56 +1534,117 @@ function scanUrlRun(text, i, kind, json) {
|
|
|
1385
1534
|
i += 2;
|
|
1386
1535
|
continue;
|
|
1387
1536
|
}
|
|
1388
|
-
|
|
1537
|
+
break;
|
|
1389
1538
|
}
|
|
1390
1539
|
if (isRegexSpace(c) || (json && c === 34))
|
|
1391
|
-
|
|
1540
|
+
break;
|
|
1392
1541
|
switch (kind) {
|
|
1393
1542
|
case "plain-user":
|
|
1394
1543
|
if (c === CH_AT || c === 47 || c === 58)
|
|
1395
|
-
|
|
1544
|
+
break run;
|
|
1396
1545
|
break;
|
|
1397
1546
|
case "plain-pass":
|
|
1398
1547
|
if (c === 47)
|
|
1399
|
-
|
|
1548
|
+
break run;
|
|
1400
1549
|
break;
|
|
1401
1550
|
case "token-pass":
|
|
1402
1551
|
if (c === 58 && text.charCodeAt(i + 1) === 47 && text.charCodeAt(i + 2) === 47)
|
|
1403
|
-
|
|
1552
|
+
break run;
|
|
1404
1553
|
break;
|
|
1405
1554
|
case "param-value":
|
|
1406
1555
|
if (c === 38 || c === 35 || isControlCode(c))
|
|
1407
|
-
|
|
1556
|
+
break run;
|
|
1408
1557
|
break;
|
|
1409
1558
|
}
|
|
1410
1559
|
if (c === CH_AT)
|
|
1411
|
-
|
|
1560
|
+
scan.lastAt = i;
|
|
1412
1561
|
i += 1;
|
|
1413
1562
|
}
|
|
1563
|
+
noteEvidenceRead(scan, i);
|
|
1414
1564
|
return i;
|
|
1415
1565
|
}
|
|
1416
|
-
function matchUserinfoAt(text, p, json) {
|
|
1417
|
-
const probeEnd = scanUrlRun(text, p, "plain-user", json);
|
|
1418
|
-
const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && credentialSpansCover(
|
|
1419
|
-
scanUrlRun(text, p, keyId ? "token-pass" : "plain-pass", json);
|
|
1420
|
-
|
|
1566
|
+
function matchUserinfoAt(scan, text, p, json) {
|
|
1567
|
+
const probeEnd = scanUrlRun(scan, text, p, "plain-user", json);
|
|
1568
|
+
const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && credentialSpansCover(scan, p, probeEnd);
|
|
1569
|
+
scanUrlRun(scan, text, p, keyId ? "token-pass" : "plain-pass", json);
|
|
1570
|
+
const at = scan.lastAt;
|
|
1571
|
+
if (at <= p)
|
|
1421
1572
|
return null;
|
|
1422
|
-
return
|
|
1573
|
+
return at;
|
|
1574
|
+
}
|
|
1575
|
+
const URL_SEG_STRIDE = 4;
|
|
1576
|
+
const URL_SEG_MAX_INTS = 3 * 1_048_576;
|
|
1577
|
+
const NO_URL_SEGMENTS = new Int32Array(0);
|
|
1578
|
+
function growUrlSegments(list) {
|
|
1579
|
+
const len = list.a.length;
|
|
1580
|
+
if (len >= URL_SEG_MAX_INTS)
|
|
1581
|
+
return false;
|
|
1582
|
+
const bigger = new Int32Array(Math.min(Math.max(len * 2, 4 * URL_SEG_STRIDE), URL_SEG_MAX_INTS));
|
|
1583
|
+
bigger.set(list.a.subarray(0, list.n));
|
|
1584
|
+
list.a = bigger;
|
|
1585
|
+
return true;
|
|
1423
1586
|
}
|
|
1424
|
-
function collectUrlCredentialSpans(text, json, span) {
|
|
1587
|
+
function collectUrlCredentialSpans(scan, text, json, span) {
|
|
1588
|
+
const dirty = scan.dirty;
|
|
1589
|
+
const hr = scan.headsRead;
|
|
1590
|
+
const hw = scan.headsWrite;
|
|
1591
|
+
const pr = scan.paramsRead;
|
|
1592
|
+
const pw = scan.paramsWrite;
|
|
1593
|
+
hw.n = 0;
|
|
1594
|
+
pw.n = 0;
|
|
1595
|
+
let armLo = FOOT_NONE;
|
|
1596
|
+
let armHi = -1;
|
|
1425
1597
|
const head = URL_CREDENTIAL_PATTERN[0];
|
|
1426
|
-
|
|
1427
|
-
|
|
1598
|
+
const ph = hr.a;
|
|
1599
|
+
const phn = dirty === undefined ? 0 : hr.n;
|
|
1600
|
+
let pc = 0;
|
|
1601
|
+
for (let from = 0;;) {
|
|
1602
|
+
head.lastIndex = from;
|
|
1603
|
+
const m = head.exec(text);
|
|
1604
|
+
if (m === null)
|
|
1605
|
+
break;
|
|
1428
1606
|
const colon = m.index;
|
|
1429
1607
|
const start = colon + 3;
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1608
|
+
while (pc < phn && ph[pc] < colon)
|
|
1609
|
+
pc += URL_SEG_STRIDE;
|
|
1610
|
+
const known = pc < phn && ph[pc] === colon;
|
|
1611
|
+
let at;
|
|
1612
|
+
let lo;
|
|
1613
|
+
let hi;
|
|
1614
|
+
if (known && footprintClean(dirty, ph[pc + 2], ph[pc + 3])) {
|
|
1615
|
+
at = ph[pc + 1];
|
|
1616
|
+
lo = ph[pc + 2];
|
|
1617
|
+
hi = ph[pc + 3];
|
|
1618
|
+
urlSegmentReplays += 1;
|
|
1434
1619
|
}
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1620
|
+
else {
|
|
1621
|
+
scan.footLo = FOOT_NONE;
|
|
1622
|
+
scan.footHi = -1;
|
|
1623
|
+
at = matchUserinfoAt(scan, text, start, json) ?? -1;
|
|
1624
|
+
lo = scan.footLo;
|
|
1625
|
+
hi = scan.footHi;
|
|
1626
|
+
urlSegmentRuns += 1;
|
|
1627
|
+
if (at >= 0 && at !== (known ? ph[pc + 1] : -2))
|
|
1628
|
+
span(start, at);
|
|
1629
|
+
}
|
|
1630
|
+
if (hw.n + URL_SEG_STRIDE <= hw.a.length || growUrlSegments(hw)) {
|
|
1631
|
+
const w = hw.a;
|
|
1632
|
+
const k = hw.n;
|
|
1633
|
+
w[k] = colon;
|
|
1634
|
+
w[k + 1] = at;
|
|
1635
|
+
w[k + 2] = lo;
|
|
1636
|
+
w[k + 3] = hi;
|
|
1637
|
+
hw.n = k + URL_SEG_STRIDE;
|
|
1638
|
+
}
|
|
1639
|
+
if (lo < armLo)
|
|
1640
|
+
armLo = lo;
|
|
1641
|
+
if (hi > armHi)
|
|
1642
|
+
armHi = hi;
|
|
1643
|
+
from = at < 0 ? colon + 1 : at + 1;
|
|
1644
|
+
}
|
|
1645
|
+
const pp = pr.a;
|
|
1646
|
+
const ppn = dirty === undefined ? 0 : pr.n;
|
|
1647
|
+
let qc = 0;
|
|
1438
1648
|
for (let i = 0; i < text.length; i++) {
|
|
1439
1649
|
const c = text.charCodeAt(i);
|
|
1440
1650
|
if (c !== 63 && c !== 38 && c !== 35)
|
|
@@ -1442,11 +1652,49 @@ function collectUrlCredentialSpans(text, json, span) {
|
|
|
1442
1652
|
const valueStart = credentialParamValueStart(text, i + 1);
|
|
1443
1653
|
if (valueStart < 0)
|
|
1444
1654
|
continue;
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1655
|
+
while (qc < ppn && pp[qc] < i)
|
|
1656
|
+
qc += URL_SEG_STRIDE;
|
|
1657
|
+
const known = qc < ppn && pp[qc] === i;
|
|
1658
|
+
let valueEnd;
|
|
1659
|
+
let lo;
|
|
1660
|
+
let hi;
|
|
1661
|
+
if (known && footprintClean(dirty, pp[qc + 2], pp[qc + 3])) {
|
|
1662
|
+
valueEnd = pp[qc + 1];
|
|
1663
|
+
lo = pp[qc + 2];
|
|
1664
|
+
hi = pp[qc + 3];
|
|
1665
|
+
urlSegmentReplays += 1;
|
|
1666
|
+
}
|
|
1667
|
+
else {
|
|
1668
|
+
scan.footLo = FOOT_NONE;
|
|
1669
|
+
scan.footHi = -1;
|
|
1670
|
+
valueEnd = scanUrlRun(scan, text, valueStart, "param-value", json);
|
|
1671
|
+
lo = scan.footLo;
|
|
1672
|
+
hi = scan.footHi;
|
|
1673
|
+
urlSegmentRuns += 1;
|
|
1674
|
+
if (valueEnd > valueStart && valueEnd !== (known ? pp[qc + 1] : -2))
|
|
1675
|
+
span(valueStart, valueEnd);
|
|
1676
|
+
}
|
|
1677
|
+
if (pw.n + URL_SEG_STRIDE <= pw.a.length || growUrlSegments(pw)) {
|
|
1678
|
+
const w = pw.a;
|
|
1679
|
+
const k = pw.n;
|
|
1680
|
+
w[k] = i;
|
|
1681
|
+
w[k + 1] = valueEnd;
|
|
1682
|
+
w[k + 2] = lo;
|
|
1683
|
+
w[k + 3] = hi;
|
|
1684
|
+
pw.n = k + URL_SEG_STRIDE;
|
|
1685
|
+
}
|
|
1686
|
+
if (lo < armLo)
|
|
1687
|
+
armLo = lo;
|
|
1688
|
+
if (hi > armHi)
|
|
1689
|
+
armHi = hi;
|
|
1448
1690
|
i = valueEnd - 1;
|
|
1449
1691
|
}
|
|
1692
|
+
scan.headsRead = hw;
|
|
1693
|
+
scan.headsWrite = hr;
|
|
1694
|
+
scan.paramsRead = pw;
|
|
1695
|
+
scan.paramsWrite = pr;
|
|
1696
|
+
scan.footLo = armLo;
|
|
1697
|
+
scan.footHi = armHi;
|
|
1450
1698
|
}
|
|
1451
1699
|
const JSON_FIELD_ANCHOR = '"';
|
|
1452
1700
|
const CREDENTIAL_NAME_MAX = Math.max(...CREDENTIAL_NAME_WORDS.map((w) => w.length));
|
|
@@ -1507,6 +1755,353 @@ function collectJsonSecretFieldSpans(text, span) {
|
|
|
1507
1755
|
p = text.indexOf(JSON_FIELD_ANCHOR, scanResumeAfterHit(p, end, JSON_FIELD_ANCHOR.length));
|
|
1508
1756
|
}
|
|
1509
1757
|
}
|
|
1758
|
+
const STRONG_AUTH_SCHEMES = new Set(["basic", "bearer"]);
|
|
1759
|
+
const STRONG_AUTH_SCHEME_MAX = Math.max(...[...STRONG_AUTH_SCHEMES].map((w) => w.length));
|
|
1760
|
+
const ABSENT_OR_BOOLEAN_WORDS = new Set(["null", "nil", "none", "undefined", "true", "false", "yes", "no"]);
|
|
1761
|
+
const ABSENT_OR_BOOLEAN_MAX = Math.max(...[...ABSENT_OR_BOOLEAN_WORDS].map((w) => w.length));
|
|
1762
|
+
const CREDENTIAL_ANCHOR_OF = new Map(Object.entries(CREDENTIAL_NAME_ANCHORS));
|
|
1763
|
+
const CH_GRAVE = 96, CH_APOS = 39, CH_DOT = 46, CH_COMMA = 44, CH_SEMI = 59, CH_DOLLAR = 36;
|
|
1764
|
+
const CH_LPAREN = 40, CH_RPAREN = 41, CH_RBRACKET = 93, CH_LBRACE = 123, CH_RBRACE = 125, CH_LT = 60, CH_GT = 62;
|
|
1765
|
+
const CH_QMARK = 63, CH_AMP = 38, CH_HASH = 35;
|
|
1766
|
+
const JSON_STRING_END = -1;
|
|
1767
|
+
const isAsciiAlnumCode = (c) => (c >= CH_0 && c <= CH_9) || (c >= CH_A && c <= CH_Z) || (c >= CH_a && c <= CH_z);
|
|
1768
|
+
const isProgramIdentCode = (c) => isAsciiAlnumCode(c) || c === CH_UNDERSCORE || c === CH_DOLLAR;
|
|
1769
|
+
const isProgramIdentStartCode = (c) => isIdentStartCode(c) || c === CH_UNDERSCORE || c === CH_DOLLAR;
|
|
1770
|
+
const isQuoteUnit = (ch) => ch === CH_QUOTE || ch === CH_APOS || ch === CH_GRAVE;
|
|
1771
|
+
const isValueCloser = (ch) => ch === CH_DOT || ch === CH_COMMA || ch === CH_SEMI || ch === CH_COLON || ch === CH_RPAREN || ch === CH_RBRACKET || ch === CH_RBRACE || ch === CH_GT;
|
|
1772
|
+
const isInlineSpaceUnit = (ch) => ch === CH_SPACE || ch === CH_TAB;
|
|
1773
|
+
const isToken68Code = (c) => isAsciiAlnumCode(c) || c === CH_DOT || c === CH_MINUS || c === CH_UNDERSCORE || c === CH_TILDE || c === CH_PLUS || c === CH_SLASH;
|
|
1774
|
+
const endsValueUnit = (ch) => ch === JSON_STRING_END || isRegexSpace(ch) || isQuoteUnit(ch) || isValueCloser(ch);
|
|
1775
|
+
const endsHeaderValueUnit = (ch) => ch === JSON_STRING_END || isLineTerminator(ch) || isQuoteUnit(ch) || ch === CH_COMMA || ch === CH_SEMI || ch === CH_RPAREN || ch === CH_RBRACKET || ch === CH_RBRACE || ch === CH_GT;
|
|
1776
|
+
const bracketOf = (c) => c === CH_LPAREN ? 1 : c === CH_RPAREN ? -1 : c === CH_LBRACKET ? 2 : c === CH_RBRACKET ? -2 : c === CH_LBRACE ? 3 : c === CH_RBRACE ? -3 : c === CH_LT ? 4 : c === CH_GT ? -4 : 0;
|
|
1777
|
+
let anchoredScanUnits = 0;
|
|
1778
|
+
export function anchoredScanUnitsForTest() {
|
|
1779
|
+
return anchoredScanUnits;
|
|
1780
|
+
}
|
|
1781
|
+
export function resetAnchoredScanUnitsForTest() {
|
|
1782
|
+
anchoredScanUnits = 0;
|
|
1783
|
+
}
|
|
1784
|
+
const packUnit = (ch, len) => ((ch + 1) << 3) | len;
|
|
1785
|
+
const unitChOf = (unit) => (unit >> 3) - 1;
|
|
1786
|
+
const unitLenOf = (unit) => unit & 7;
|
|
1787
|
+
function readAnchorUnit(text, i, json) {
|
|
1788
|
+
anchoredScanUnits += 1;
|
|
1789
|
+
const c = text.charCodeAt(i);
|
|
1790
|
+
if (!json)
|
|
1791
|
+
return packUnit(c, 1);
|
|
1792
|
+
if (c === CH_QUOTE)
|
|
1793
|
+
return packUnit(JSON_STRING_END, 1);
|
|
1794
|
+
if (c !== CH_BACKSLASH || i + 1 >= text.length)
|
|
1795
|
+
return packUnit(c, 1);
|
|
1796
|
+
switch (text.charCodeAt(i + 1)) {
|
|
1797
|
+
case CH_n: return packUnit(CH_LF, 2);
|
|
1798
|
+
case CH_r: return packUnit(CH_CR, 2);
|
|
1799
|
+
case CH_t: return packUnit(CH_TAB, 2);
|
|
1800
|
+
case CH_f: return packUnit(12, 2);
|
|
1801
|
+
case CH_b: return packUnit(8, 2);
|
|
1802
|
+
case CH_QUOTE: return packUnit(CH_QUOTE, 2);
|
|
1803
|
+
case CH_BACKSLASH: return packUnit(CH_BACKSLASH, 2);
|
|
1804
|
+
case CH_SLASH: return packUnit(CH_SLASH, 2);
|
|
1805
|
+
case CH_u: {
|
|
1806
|
+
const h0 = hexDigitValue(text.charCodeAt(i + 2));
|
|
1807
|
+
const h1 = hexDigitValue(text.charCodeAt(i + 3));
|
|
1808
|
+
const h2 = hexDigitValue(text.charCodeAt(i + 4));
|
|
1809
|
+
const h3 = hexDigitValue(text.charCodeAt(i + 5));
|
|
1810
|
+
return h0 >= 0 && h1 >= 0 && h2 >= 0 && h3 >= 0 ? packUnit((h0 << 12) | (h1 << 8) | (h2 << 4) | h3, 6) : packUnit(c, 1);
|
|
1811
|
+
}
|
|
1812
|
+
default:
|
|
1813
|
+
return packUnit(c, 1);
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
function escapedAt(text, i) {
|
|
1817
|
+
let k = i - 1;
|
|
1818
|
+
while (k >= 0 && text.charCodeAt(k) === CH_BACKSLASH)
|
|
1819
|
+
k -= 1;
|
|
1820
|
+
anchoredScanUnits += i - 1 - k;
|
|
1821
|
+
return ((i - 1 - k) & 1) === 1;
|
|
1822
|
+
}
|
|
1823
|
+
function anchorHeadAt(text, op, json) {
|
|
1824
|
+
let i = op - 1;
|
|
1825
|
+
let spacedLeft = false;
|
|
1826
|
+
for (;;) {
|
|
1827
|
+
if (i < 0)
|
|
1828
|
+
return undefined;
|
|
1829
|
+
const c = text.charCodeAt(i);
|
|
1830
|
+
if (c === CH_SPACE || c === CH_TAB) {
|
|
1831
|
+
i -= 1;
|
|
1832
|
+
spacedLeft = true;
|
|
1833
|
+
anchoredScanUnits += 1;
|
|
1834
|
+
continue;
|
|
1835
|
+
}
|
|
1836
|
+
if (json && c === CH_t && i >= 1 && text.charCodeAt(i - 1) === CH_BACKSLASH && escapedAt(text, i)) {
|
|
1837
|
+
i -= 2;
|
|
1838
|
+
spacedLeft = true;
|
|
1839
|
+
continue;
|
|
1840
|
+
}
|
|
1841
|
+
break;
|
|
1842
|
+
}
|
|
1843
|
+
let quote = 0;
|
|
1844
|
+
let quoteLen = 0;
|
|
1845
|
+
const q = text.charCodeAt(i);
|
|
1846
|
+
if (q === CH_QUOTE || q === CH_APOS || q === CH_GRAVE) {
|
|
1847
|
+
quote = q;
|
|
1848
|
+
quoteLen = json && q === CH_QUOTE && escapedAt(text, i) ? 2 : 1;
|
|
1849
|
+
i -= quoteLen;
|
|
1850
|
+
}
|
|
1851
|
+
const nameEnd = i + 1;
|
|
1852
|
+
const floor = Math.max(-1, i - (CREDENTIAL_NAME_MAX + 2));
|
|
1853
|
+
let j = i;
|
|
1854
|
+
while (j > floor && isIdentCode(text.charCodeAt(j)))
|
|
1855
|
+
j -= 1;
|
|
1856
|
+
anchoredScanUnits += nameEnd - (j + 1);
|
|
1857
|
+
if (j === floor && j >= 0 && isIdentCode(text.charCodeAt(j)))
|
|
1858
|
+
return undefined;
|
|
1859
|
+
let nameStart = j + 1;
|
|
1860
|
+
let afterEscape = false;
|
|
1861
|
+
if (j >= 0 && text.charCodeAt(j) === CH_BACKSLASH && escapedAt(text, j + 1)) {
|
|
1862
|
+
const e = text.charCodeAt(nameStart);
|
|
1863
|
+
if (e === CH_b || e === CH_f || e === CH_n || e === CH_r || e === CH_t) {
|
|
1864
|
+
nameStart += 1;
|
|
1865
|
+
afterEscape = true;
|
|
1866
|
+
}
|
|
1867
|
+
else if (e === CH_u && nameStart + 5 <= nameEnd && hexDigitValue(text.charCodeAt(nameStart + 1)) >= 0 && hexDigitValue(text.charCodeAt(nameStart + 2)) >= 0
|
|
1868
|
+
&& hexDigitValue(text.charCodeAt(nameStart + 3)) >= 0 && hexDigitValue(text.charCodeAt(nameStart + 4)) >= 0) {
|
|
1869
|
+
nameStart += 5;
|
|
1870
|
+
afterEscape = true;
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
const runStart = nameStart;
|
|
1874
|
+
while (nameStart < nameEnd && text.charCodeAt(nameStart) === CH_MINUS)
|
|
1875
|
+
nameStart += 1;
|
|
1876
|
+
if (nameStart >= nameEnd)
|
|
1877
|
+
return undefined;
|
|
1878
|
+
let before = j;
|
|
1879
|
+
if (quote !== 0) {
|
|
1880
|
+
if (afterEscape || nameStart !== j + 1 || j < 0 || text.charCodeAt(j) !== quote)
|
|
1881
|
+
return undefined;
|
|
1882
|
+
const openLen = json && quote === CH_QUOTE && escapedAt(text, j) ? 2 : 1;
|
|
1883
|
+
if (openLen !== quoteLen)
|
|
1884
|
+
return undefined;
|
|
1885
|
+
before = j - openLen;
|
|
1886
|
+
}
|
|
1887
|
+
const strength = CREDENTIAL_ANCHOR_OF.get(text.slice(nameStart, nameEnd).toLowerCase());
|
|
1888
|
+
if (strength === undefined)
|
|
1889
|
+
return undefined;
|
|
1890
|
+
if (!afterEscape && runStart === j + 1 && quote === 0 && j >= 0 && text.charCodeAt(j) === CH_SLASH)
|
|
1891
|
+
return undefined;
|
|
1892
|
+
let listSep = 0;
|
|
1893
|
+
if (!afterEscape) {
|
|
1894
|
+
let k = before;
|
|
1895
|
+
while (k >= 0 && (text.charCodeAt(k) === CH_SPACE || text.charCodeAt(k) === CH_TAB))
|
|
1896
|
+
k -= 1;
|
|
1897
|
+
anchoredScanUnits += before - k;
|
|
1898
|
+
const left = k >= 0 ? text.charCodeAt(k) : 0;
|
|
1899
|
+
listSep = left === CH_QMARK || left === CH_AMP || left === CH_HASH ? CH_AMP : left === CH_SEMI ? CH_SEMI : left === CH_COMMA || left === CH_LPAREN || left === CH_LBRACKET || left === CH_LBRACE ? CH_COMMA : 0;
|
|
1900
|
+
}
|
|
1901
|
+
return { strength, quotedName: quote !== 0, spacedLeft, listSep };
|
|
1902
|
+
}
|
|
1903
|
+
function quotedValueEnd(text, cs, quote, json) {
|
|
1904
|
+
const n = text.length;
|
|
1905
|
+
let i = cs;
|
|
1906
|
+
while (i < n) {
|
|
1907
|
+
const unit = readAnchorUnit(text, i, json);
|
|
1908
|
+
const ch = unitChOf(unit);
|
|
1909
|
+
if (ch === quote)
|
|
1910
|
+
return i;
|
|
1911
|
+
if (ch === JSON_STRING_END)
|
|
1912
|
+
break;
|
|
1913
|
+
i += unitLenOf(unit);
|
|
1914
|
+
if (ch === CH_BACKSLASH && quote !== JSON_STRING_END) {
|
|
1915
|
+
if (i >= n)
|
|
1916
|
+
break;
|
|
1917
|
+
const escaped = readAnchorUnit(text, i, json);
|
|
1918
|
+
if (unitChOf(escaped) === JSON_STRING_END)
|
|
1919
|
+
break;
|
|
1920
|
+
i += unitLenOf(escaped);
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
return -1;
|
|
1924
|
+
}
|
|
1925
|
+
function bareValueEnd(text, s, json, listSep) {
|
|
1926
|
+
const n = text.length;
|
|
1927
|
+
let i = s;
|
|
1928
|
+
while (i < n) {
|
|
1929
|
+
const unit = readAnchorUnit(text, i, json);
|
|
1930
|
+
const ch = unitChOf(unit);
|
|
1931
|
+
if (ch === JSON_STRING_END || isRegexSpace(ch))
|
|
1932
|
+
break;
|
|
1933
|
+
if (listSep !== 0 && (ch === listSep || (listSep === CH_AMP && ch === CH_HASH)))
|
|
1934
|
+
break;
|
|
1935
|
+
const next = i + unitLenOf(unit);
|
|
1936
|
+
if (isQuoteUnit(ch)) {
|
|
1937
|
+
if (next >= n)
|
|
1938
|
+
break;
|
|
1939
|
+
if (endsValueUnit(unitChOf(readAnchorUnit(text, next, json))))
|
|
1940
|
+
break;
|
|
1941
|
+
}
|
|
1942
|
+
i = next;
|
|
1943
|
+
}
|
|
1944
|
+
anchoredScanUnits += i - s;
|
|
1945
|
+
const excess = [0, 0, 0, 0];
|
|
1946
|
+
for (let k = s; k < i; k += 1) {
|
|
1947
|
+
const b = bracketOf(text.charCodeAt(k));
|
|
1948
|
+
if (b !== 0)
|
|
1949
|
+
excess[Math.abs(b) - 1] = excess[Math.abs(b) - 1] + (b < 0 ? 1 : -1);
|
|
1950
|
+
}
|
|
1951
|
+
let e = i;
|
|
1952
|
+
while (e > s) {
|
|
1953
|
+
const c = text.charCodeAt(e - 1);
|
|
1954
|
+
if (c === CH_DOT || c === CH_COMMA || c === CH_SEMI || c === CH_COLON) {
|
|
1955
|
+
e -= 1;
|
|
1956
|
+
continue;
|
|
1957
|
+
}
|
|
1958
|
+
const b = bracketOf(c);
|
|
1959
|
+
if (b < 0 && excess[-b - 1] > 0) {
|
|
1960
|
+
excess[-b - 1] = excess[-b - 1] - 1;
|
|
1961
|
+
e -= 1;
|
|
1962
|
+
continue;
|
|
1963
|
+
}
|
|
1964
|
+
break;
|
|
1965
|
+
}
|
|
1966
|
+
return e;
|
|
1967
|
+
}
|
|
1968
|
+
function literalValueAt(text, s, json, declaredIfBare, listSep) {
|
|
1969
|
+
const n = text.length;
|
|
1970
|
+
if (s >= n)
|
|
1971
|
+
return undefined;
|
|
1972
|
+
const firstUnit = readAnchorUnit(text, s, json);
|
|
1973
|
+
const first = unitChOf(firstUnit);
|
|
1974
|
+
let vs = s;
|
|
1975
|
+
if (isQuoteUnit(first) || first === JSON_STRING_END) {
|
|
1976
|
+
vs = s + unitLenOf(firstUnit);
|
|
1977
|
+
if (vs >= n)
|
|
1978
|
+
return undefined;
|
|
1979
|
+
if (!isAsciiAlnumCode(unitChOf(readAnchorUnit(text, vs, json))))
|
|
1980
|
+
return undefined;
|
|
1981
|
+
const close = quotedValueEnd(text, vs, first, json);
|
|
1982
|
+
if (close >= 0)
|
|
1983
|
+
return { start: vs, end: close, quoted: true };
|
|
1984
|
+
}
|
|
1985
|
+
else if (!isAsciiAlnumCode(first))
|
|
1986
|
+
return undefined;
|
|
1987
|
+
if (!declaredIfBare)
|
|
1988
|
+
return undefined;
|
|
1989
|
+
if (isProgramIdentStartCode(text.charCodeAt(vs))) {
|
|
1990
|
+
let k = vs;
|
|
1991
|
+
while (k < n && isProgramIdentCode(text.charCodeAt(k)))
|
|
1992
|
+
k += 1;
|
|
1993
|
+
anchoredScanUnits += k - vs;
|
|
1994
|
+
const after = text.charCodeAt(k);
|
|
1995
|
+
if (after === CH_LPAREN || after === CH_LBRACKET || (after === CH_DOT && k + 1 < n && isProgramIdentStartCode(text.charCodeAt(k + 1))))
|
|
1996
|
+
return undefined;
|
|
1997
|
+
}
|
|
1998
|
+
const e = bareValueEnd(text, vs, json, listSep);
|
|
1999
|
+
if (e <= vs)
|
|
2000
|
+
return undefined;
|
|
2001
|
+
if (e - vs <= ABSENT_OR_BOOLEAN_MAX && ABSENT_OR_BOOLEAN_WORDS.has(text.slice(vs, e).toLowerCase()))
|
|
2002
|
+
return undefined;
|
|
2003
|
+
return { start: vs, end: e, quoted: false };
|
|
2004
|
+
}
|
|
2005
|
+
function schemeValueAt(text, s, json) {
|
|
2006
|
+
const n = text.length;
|
|
2007
|
+
let i = s;
|
|
2008
|
+
if (i >= n)
|
|
2009
|
+
return undefined;
|
|
2010
|
+
const opening = readAnchorUnit(text, i, json);
|
|
2011
|
+
if (isQuoteUnit(unitChOf(opening)) || unitChOf(opening) === JSON_STRING_END)
|
|
2012
|
+
i += unitLenOf(opening);
|
|
2013
|
+
const w = i;
|
|
2014
|
+
while (i < n && i - w <= STRONG_AUTH_SCHEME_MAX && (isIdentStartCode(text.charCodeAt(i))))
|
|
2015
|
+
i += 1;
|
|
2016
|
+
anchoredScanUnits += i - w;
|
|
2017
|
+
if (i - w > STRONG_AUTH_SCHEME_MAX || !STRONG_AUTH_SCHEMES.has(text.slice(w, i).toLowerCase()))
|
|
2018
|
+
return undefined;
|
|
2019
|
+
let spaced = false;
|
|
2020
|
+
while (i < n) {
|
|
2021
|
+
const unit = readAnchorUnit(text, i, json);
|
|
2022
|
+
if (!isInlineSpaceUnit(unitChOf(unit)))
|
|
2023
|
+
break;
|
|
2024
|
+
i += unitLenOf(unit);
|
|
2025
|
+
spaced = true;
|
|
2026
|
+
}
|
|
2027
|
+
if (!spaced)
|
|
2028
|
+
return undefined;
|
|
2029
|
+
const vs = i;
|
|
2030
|
+
if (i >= n || !isAsciiAlnumCode(text.charCodeAt(i)))
|
|
2031
|
+
return undefined;
|
|
2032
|
+
while (i < n && isToken68Code(text.charCodeAt(i)))
|
|
2033
|
+
i += 1;
|
|
2034
|
+
while (i < n && text.charCodeAt(i) === CH_EQ)
|
|
2035
|
+
i += 1;
|
|
2036
|
+
anchoredScanUnits += i - vs;
|
|
2037
|
+
const e = i;
|
|
2038
|
+
while (i < n) {
|
|
2039
|
+
const unit = readAnchorUnit(text, i, json);
|
|
2040
|
+
if (!isInlineSpaceUnit(unitChOf(unit)))
|
|
2041
|
+
break;
|
|
2042
|
+
i += unitLenOf(unit);
|
|
2043
|
+
}
|
|
2044
|
+
if (i < n && !endsHeaderValueUnit(unitChOf(readAnchorUnit(text, i, json))))
|
|
2045
|
+
return undefined;
|
|
2046
|
+
return { start: vs, end: e, quoted: false };
|
|
2047
|
+
}
|
|
2048
|
+
function collectAnchoredValueSpans(text, json, span) {
|
|
2049
|
+
const n = text.length;
|
|
2050
|
+
let resume = 0;
|
|
2051
|
+
let nextEq = text.indexOf("=");
|
|
2052
|
+
let nextColon = text.indexOf(":");
|
|
2053
|
+
for (;;) {
|
|
2054
|
+
if (nextEq >= 0 && nextEq < resume)
|
|
2055
|
+
nextEq = text.indexOf("=", resume);
|
|
2056
|
+
if (nextColon >= 0 && nextColon < resume)
|
|
2057
|
+
nextColon = text.indexOf(":", resume);
|
|
2058
|
+
if (nextEq < 0 && nextColon < 0)
|
|
2059
|
+
return;
|
|
2060
|
+
let op;
|
|
2061
|
+
if (nextColon < 0 || (nextEq >= 0 && nextEq < nextColon)) {
|
|
2062
|
+
op = nextEq;
|
|
2063
|
+
nextEq = text.indexOf("=", op + 1);
|
|
2064
|
+
}
|
|
2065
|
+
else {
|
|
2066
|
+
op = nextColon;
|
|
2067
|
+
nextColon = text.indexOf(":", op + 1);
|
|
2068
|
+
}
|
|
2069
|
+
const head = anchorHeadAt(text, op, json);
|
|
2070
|
+
if (head === undefined)
|
|
2071
|
+
continue;
|
|
2072
|
+
let s = op + 1;
|
|
2073
|
+
let spacedRight = false;
|
|
2074
|
+
while (s < n) {
|
|
2075
|
+
const unit = readAnchorUnit(text, s, json);
|
|
2076
|
+
if (!isInlineSpaceUnit(unitChOf(unit)))
|
|
2077
|
+
break;
|
|
2078
|
+
s += unitLenOf(unit);
|
|
2079
|
+
spacedRight = true;
|
|
2080
|
+
}
|
|
2081
|
+
let v;
|
|
2082
|
+
switch (head.strength) {
|
|
2083
|
+
case "structure-only":
|
|
2084
|
+
continue;
|
|
2085
|
+
case "auth-scheme":
|
|
2086
|
+
v = schemeValueAt(text, s, json);
|
|
2087
|
+
break;
|
|
2088
|
+
case "declarative":
|
|
2089
|
+
v = literalValueAt(text, s, json, head.quotedName || (!head.spacedLeft && !spacedRight), head.listSep);
|
|
2090
|
+
break;
|
|
2091
|
+
case "any-form":
|
|
2092
|
+
v = literalValueAt(text, s, json, true, head.listSep);
|
|
2093
|
+
break;
|
|
2094
|
+
default: {
|
|
2095
|
+
const unhandled = head.strength;
|
|
2096
|
+
throw new Error(`redact: unhandled free-text anchor strength ${String(unhandled)}`);
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
if (v === undefined)
|
|
2100
|
+
continue;
|
|
2101
|
+
span(v.start, v.end);
|
|
2102
|
+
resume = v.quoted ? v.start : v.end;
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
1510
2105
|
export function structuralScannerForTest(arm) {
|
|
1511
2106
|
const collect = (scan) => (text) => {
|
|
1512
2107
|
const out = [];
|
|
@@ -1523,7 +2118,7 @@ export function structuralScannerForTest(arm) {
|
|
|
1523
2118
|
let ppkScanBytes = 0;
|
|
1524
2119
|
export function ppkScanBytesForTest() { return ppkScanBytes; }
|
|
1525
2120
|
export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
|
|
1526
|
-
function collectPpkContainerSpans(text, record, span) {
|
|
2121
|
+
function collectPpkContainerSpans(scan, text, record, span) {
|
|
1527
2122
|
const HEAD = "PuTTY-User-Key-File-";
|
|
1528
2123
|
const MAC = "Private-MAC: ";
|
|
1529
2124
|
const HASH = "Private-Hash: ";
|
|
@@ -1542,7 +2137,7 @@ function collectPpkContainerSpans(text, record, span) {
|
|
|
1542
2137
|
const isVersionedHead = (at) => {
|
|
1543
2138
|
let p = at + HEAD.length;
|
|
1544
2139
|
const start = p;
|
|
1545
|
-
while (p < text.length && opaqueUnitEnd(p) < 0) {
|
|
2140
|
+
while (p < text.length && opaqueUnitEnd(scan, p) < 0) {
|
|
1546
2141
|
const c = text.charCodeAt(p);
|
|
1547
2142
|
if (c >= 48 && c <= 57)
|
|
1548
2143
|
p += 1;
|
|
@@ -1553,13 +2148,13 @@ function collectPpkContainerSpans(text, record, span) {
|
|
|
1553
2148
|
};
|
|
1554
2149
|
const nextVersionedHead = (from) => {
|
|
1555
2150
|
let p = find(HEAD, from);
|
|
1556
|
-
while (p !== Number.POSITIVE_INFINITY && (inOpaqueUnit(p) || !isVersionedHead(p)))
|
|
2151
|
+
while (p !== Number.POSITIVE_INFINITY && (inOpaqueUnit(scan, p) || !isVersionedHead(p)))
|
|
1557
2152
|
p = find(HEAD, p + HEAD.length);
|
|
1558
2153
|
return p;
|
|
1559
2154
|
};
|
|
1560
2155
|
const hexRunEnd = (from) => {
|
|
1561
2156
|
let e = from;
|
|
1562
|
-
while (e < text.length && opaqueUnitEnd(e) < 0) {
|
|
2157
|
+
while (e < text.length && opaqueUnitEnd(scan, e) < 0) {
|
|
1563
2158
|
const c = text.charCodeAt(e);
|
|
1564
2159
|
if ((c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70))
|
|
1565
2160
|
e += 1;
|
|
@@ -1594,7 +2189,7 @@ function collectPpkContainerSpans(text, record, span) {
|
|
|
1594
2189
|
let gapBlank = true;
|
|
1595
2190
|
if (limit !== Number.POSITIVE_INFINITY) {
|
|
1596
2191
|
for (let i = hexEnd; i < limit; i++) {
|
|
1597
|
-
if (opaqueUnitEnd(i) >= 0) {
|
|
2192
|
+
if (opaqueUnitEnd(scan, i) >= 0) {
|
|
1598
2193
|
gapBlank = false;
|
|
1599
2194
|
break;
|
|
1600
2195
|
}
|
|
@@ -1650,15 +2245,30 @@ const armPool = (() => {
|
|
|
1650
2245
|
headFirst: boundary === undefined ? undefined : headFirst,
|
|
1651
2246
|
};
|
|
1652
2247
|
};
|
|
1653
|
-
const scannerMember = (token, order,
|
|
2248
|
+
const scannerMember = (token, order, arm, rank = ARM_RANK.structural, lexical = false) => {
|
|
1654
2249
|
const repl = spanRepl(token);
|
|
1655
2250
|
return {
|
|
1656
2251
|
token,
|
|
1657
|
-
rank
|
|
2252
|
+
rank,
|
|
1658
2253
|
order,
|
|
1659
|
-
phase,
|
|
2254
|
+
phase: 1,
|
|
2255
|
+
lexical,
|
|
2256
|
+
collect: (text, json, observe, out) => arm(text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
|
|
2257
|
+
retry: undefined,
|
|
2258
|
+
headSrc: undefined,
|
|
2259
|
+
headFirst: undefined,
|
|
2260
|
+
};
|
|
2261
|
+
};
|
|
2262
|
+
const forensicMember = (token, order, arm) => {
|
|
2263
|
+
const repl = spanRepl(token);
|
|
2264
|
+
const rank = ARM_RANK.structural;
|
|
2265
|
+
return {
|
|
2266
|
+
token,
|
|
2267
|
+
rank,
|
|
2268
|
+
order,
|
|
2269
|
+
phase: 2,
|
|
1660
2270
|
lexical: false,
|
|
1661
|
-
collect: (text, json, observe, out) => scan
|
|
2271
|
+
collect: (scan, text, json, observe, out) => arm(scan, text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
|
|
1662
2272
|
retry: undefined,
|
|
1663
2273
|
headSrc: undefined,
|
|
1664
2274
|
headFirst: undefined,
|
|
@@ -1666,21 +2276,22 @@ const armPool = (() => {
|
|
|
1666
2276
|
};
|
|
1667
2277
|
const structuralScannerMember = (row, order) => {
|
|
1668
2278
|
if (row === PEM_FENCE_PATTERN)
|
|
1669
|
-
return scannerMember(row[1], order,
|
|
2279
|
+
return scannerMember(row[1], order, (text, _json, _observe, span) => collectPemFenceSpans(text, span));
|
|
1670
2280
|
if (row === SSH2_FENCE_PATTERN)
|
|
1671
|
-
return
|
|
2281
|
+
return forensicMember(row[1], order, (scan, text, _json, _observe, span) => collectSsh2FenceSpans(scan, text, span));
|
|
1672
2282
|
if (row === URL_CREDENTIAL_PATTERN)
|
|
1673
|
-
return
|
|
2283
|
+
return forensicMember(row[1], order, (scan, text, json, _observe, span) => collectUrlCredentialSpans(scan, text, json, span));
|
|
1674
2284
|
if (row === JSON_SECRET_FIELD_PATTERN)
|
|
1675
|
-
return scannerMember(row[1], order,
|
|
2285
|
+
return scannerMember(row[1], order, (text, _json, _observe, span) => collectJsonSecretFieldSpans(text, span));
|
|
1676
2286
|
throw new Error(`redact: structural row /${row[0].source}/ has no hand-written scanner — structural grammars are declared as regexes but never executed as one (S-102 / S-265)`);
|
|
1677
2287
|
};
|
|
1678
2288
|
const members = [];
|
|
1679
2289
|
for (const row of STRUCTURAL_SECRET_PATTERNS)
|
|
1680
2290
|
members.push(structuralScannerMember(row, members.length));
|
|
1681
|
-
members.push(
|
|
1682
|
-
members.push(
|
|
1683
|
-
members.push(
|
|
2291
|
+
members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, observe, span) => collectPpkContainerSpans(scan, text, observe, span)));
|
|
2292
|
+
members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _observe, span) => collectPemFragmentSpans(scan, text, json, span)));
|
|
2293
|
+
members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _observe, span) => collectHeadlessPpkSpans(scan, text, span)));
|
|
2294
|
+
members.push(scannerMember(ANCHORED_VALUE_TOKEN, members.length, (text, json, _observe, span) => collectAnchoredValueSpans(text, json, span), ARM_RANK.keyword, true));
|
|
1684
2295
|
for (const row of KEYWORD_SECRET_PATTERNS)
|
|
1685
2296
|
members.push(regexMember(row, ARM_RANK.keyword, members.length));
|
|
1686
2297
|
for (const row of SHAPE_SECRET_PATTERNS)
|
|
@@ -1746,25 +2357,37 @@ const armPool = (() => {
|
|
|
1746
2357
|
at = freshCovered(before);
|
|
1747
2358
|
}
|
|
1748
2359
|
};
|
|
1749
|
-
setScanEvidence(NO_CREDENTIAL_SPANS);
|
|
1750
2360
|
for (const m of lane.phase1)
|
|
1751
2361
|
m.collect(text, json, observe, spans);
|
|
1752
2362
|
closeCoveragePositions(0);
|
|
1753
2363
|
if (lane.phase2.length > 0) {
|
|
1754
|
-
|
|
2364
|
+
const scan = buildForensicScan(credentialSpans(text), lane.phase2.length);
|
|
2365
|
+
const reuse = forensicRescanMode === "incremental";
|
|
2366
|
+
const foot = scan.armFoot;
|
|
2367
|
+
let folded = 0;
|
|
1755
2368
|
for (let level = 0;; level += 1) {
|
|
1756
2369
|
const before = spans.length;
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
2370
|
+
for (let j = 0; j < lane.phase2.length; j++) {
|
|
2371
|
+
const dirty = scan.dirty;
|
|
2372
|
+
if (dirty !== undefined && footprintClean(dirty, foot[2 * j], foot[2 * j + 1])) {
|
|
2373
|
+
forensicSkips += 1;
|
|
2374
|
+
continue;
|
|
2375
|
+
}
|
|
2376
|
+
scan.footLo = FOOT_NONE;
|
|
2377
|
+
scan.footHi = -1;
|
|
2378
|
+
lane.phase2[j].collect(scan, text, json, observe && level === 0, spans);
|
|
2379
|
+
foot[2 * j] = scan.footLo;
|
|
2380
|
+
foot[2 * j + 1] = scan.footHi;
|
|
2381
|
+
forensicRuns += 1;
|
|
2382
|
+
}
|
|
1760
2383
|
closeCoveragePositions(before);
|
|
1761
2384
|
forensicLevels += 1;
|
|
1762
|
-
const
|
|
1763
|
-
|
|
2385
|
+
const grown = foldCredentialSpans(scan.table, spans, folded);
|
|
2386
|
+
folded = spans.length;
|
|
2387
|
+
if (grown.dirty.starts.length === 0)
|
|
1764
2388
|
break;
|
|
1765
|
-
|
|
2389
|
+
advanceEvidence(scan, grown.next, reuse ? grown.dirty : undefined);
|
|
1766
2390
|
}
|
|
1767
|
-
setScanEvidence(NO_CREDENTIAL_SPANS);
|
|
1768
2391
|
}
|
|
1769
2392
|
poolSpanCount += spans.length;
|
|
1770
2393
|
const kept = resolveSpanOverlaps(spans);
|
|
@@ -1837,6 +2460,9 @@ export function redactionSpansForTest(text, opts) {
|
|
|
1837
2460
|
const r = runRedaction(text, false, opts?.json === true, false, "free-text");
|
|
1838
2461
|
return { text: r.text, spans: r.spans.map((sp) => ({ start: sp.start, end: sp.end })) };
|
|
1839
2462
|
}
|
|
2463
|
+
export function proposeRedactionSpans(text, scope) {
|
|
2464
|
+
return runRedaction(text, true, false, false, scope).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
|
|
2465
|
+
}
|
|
1840
2466
|
const REDACT_MAX_DEPTH = 64;
|
|
1841
2467
|
export function redactDeep(value) {
|
|
1842
2468
|
return redactDeepWith(value, redactSecrets);
|