@sema-agent/server 7.132.0 → 7.134.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +353 -0
- package/README.md +8 -9
- package/README.zh-CN.md +7 -6
- package/USAGE.md +301 -64
- package/deploy/sema-up/README.md +35 -1
- package/deploy/sema-up/chart/Chart.yaml +1 -1
- package/deploy/sema-up/chart/templates/NOTES.txt +15 -0
- package/deploy/sema-up/chart/templates/_helpers.tpl +129 -0
- package/deploy/sema-up/chart/templates/sealed-key.yaml +24 -0
- package/deploy/sema-up/chart/templates/server-v4pro.yaml +19 -2
- package/deploy/sema-up/chart/templates/server.yaml +19 -2
- package/deploy/sema-up/chart/values-full-single.yaml +5 -1
- package/deploy/sema-up/chart/values.yaml +34 -0
- package/deploy/sema-up/stack/compose.yaml +3 -0
- package/deploy/sema-up/stack/sema-stack-cluster.yaml +4 -2
- package/dist/adoption/plan.d.ts +3 -2
- package/dist/adoption/plan.js +7 -5
- package/dist/adoption/runner.js +3 -10
- package/dist/adoption/sql.d.ts +3 -2
- package/dist/adoption/sql.js +3 -5
- package/dist/adoption/wire.d.ts +4 -1
- package/dist/adoption/wire.js +1 -0
- package/dist/approval-ask-audit-store.d.ts +32 -14
- package/dist/approval-ask-audit-store.js +272 -74
- package/dist/approval-ask-machine.d.ts +15 -0
- package/dist/approval-ask-machine.js +6 -0
- package/dist/approval-ask-retention.d.ts +136 -0
- package/dist/approval-ask-retention.js +102 -0
- package/dist/approval-card.d.ts +83 -23
- package/dist/approval-card.js +45 -15
- package/dist/approval-content-kind.d.ts +5 -2
- package/dist/approval-content-kind.js +16 -3
- package/dist/approval-deny-reasons.d.ts +1 -1
- package/dist/approval-gate-currency.d.ts +72 -0
- package/dist/approval-gate-currency.js +94 -0
- package/dist/approval.d.ts +15 -0
- package/dist/approval.js +55 -17
- package/dist/audit.d.ts +27 -10
- package/dist/audit.js +12 -16
- package/dist/bake-runner/main.js +3 -2
- package/dist/boot/coordinators.d.ts +6 -1
- package/dist/boot/coordinators.js +6 -3
- package/dist/boot/deferred-sandbox-path-env.js +1 -1
- package/dist/boot/execution-env.js +9 -7
- package/dist/boot/execution-lane.js +11 -0
- package/dist/boot/leader.d.ts +2 -0
- package/dist/boot/leader.js +4 -1
- package/dist/boot/memory-boundary.js +2 -2
- package/dist/boot/memory-consolidation.d.ts +5 -4
- package/dist/boot/memory-consolidation.js +7 -6
- package/dist/boot/org-delivery.d.ts +28 -0
- package/dist/boot/org-delivery.js +20 -0
- package/dist/boot/read-deny-anchor-audit.js +2 -2
- package/dist/boot/reapers.d.ts +3 -0
- package/dist/boot/reapers.js +21 -1
- package/dist/boot/resolve-spec.d.ts +3 -0
- package/dist/boot/resolve-spec.js +60 -81
- package/dist/boot/runner-deps.d.ts +18 -5
- package/dist/boot/runner-deps.js +10 -0
- package/dist/boot/session-faces.d.ts +1 -0
- package/dist/boot/session-faces.js +4 -2
- package/dist/boot/stage-02-web-admin-pull.js +4 -1
- package/dist/boot/stage-06-runners.d.ts +6 -3
- package/dist/boot/stage-06-runners.js +20 -4
- package/dist/boot/stage-07-capability-layer.d.ts +4 -1
- package/dist/boot/stage-07-capability-layer.js +24 -3
- package/dist/boot/stage-08-reapers.d.ts +4 -1
- package/dist/boot/stage-08-reapers.js +2 -2
- package/dist/boot/stage-09-leader.d.ts +5 -1
- package/dist/boot/stage-09-leader.js +2 -2
- package/dist/boot/stage-10-http-server.d.ts +5 -1
- package/dist/boot/stage-10-http-server.js +6 -6
- package/dist/boot/stores.d.ts +8 -0
- package/dist/boot/stores.js +10 -14
- package/dist/boot/web-admin.d.ts +10 -0
- package/dist/boot/web-admin.js +162 -65
- package/dist/boot/workflow-orchestration.d.ts +2 -0
- package/dist/boot/workflow-orchestration.js +2 -1
- package/dist/brain.js +8 -3
- package/dist/breaker-key.d.ts +6 -0
- package/dist/breaker-key.js +29 -0
- package/dist/budget.js +5 -1
- package/dist/caller-text-echo.d.ts +11 -6
- package/dist/caller-text-echo.js +10 -12
- package/dist/canonical-json.d.ts +2 -41
- package/dist/canonical-json.js +2 -13
- package/dist/capabilities/center-plugins.d.ts +7 -0
- package/dist/capabilities/center-plugins.js +1 -1
- package/dist/capabilities/composite-members.d.ts +1 -1
- package/dist/capabilities/composite-members.js +1 -1
- package/dist/capabilities/execution-lane.js +2 -2
- package/dist/capabilities/hands-lane.d.ts +21 -0
- package/dist/capabilities/hands-lane.js +10 -0
- package/dist/capabilities/mcp-probe.d.ts +36 -4
- package/dist/capabilities/mcp-probe.js +29 -6
- package/dist/capabilities/scenarios.d.ts +63 -7
- package/dist/capabilities/scenarios.js +39 -7
- package/dist/capability-facts.d.ts +36 -4
- package/dist/capability-facts.js +30 -4
- package/dist/center-config-tier.d.ts +69 -0
- package/dist/center-config-tier.js +42 -0
- package/dist/checkpoint-ctx-seal-m28.d.ts +42 -0
- package/dist/checkpoint-ctx-seal-m28.js +19 -0
- package/dist/checkpoint-identity.d.ts +16 -0
- package/dist/checkpoint-identity.js +19 -0
- package/dist/config-catalog.d.ts +17 -10
- package/dist/config-catalog.js +43 -28
- package/dist/config-d-migration.d.ts +3 -1
- package/dist/config-d-migration.js +2 -2
- package/dist/config-dir-trust.d.ts +114 -0
- package/dist/config-dir-trust.js +388 -0
- package/dist/config-invariants.d.ts +18 -5
- package/dist/config-invariants.js +35 -3
- package/dist/config-provider.d.ts +6 -1
- package/dist/config-provider.js +6 -2
- package/dist/config-types.d.ts +71 -14
- package/dist/config-types.js +8 -0
- package/dist/config.d.ts +13 -0
- package/dist/config.js +121 -59
- package/dist/delegated-session-owner.d.ts +185 -0
- package/dist/delegated-session-owner.js +119 -0
- package/dist/device-protocol.d.ts +5 -3
- package/dist/device-protocol.js +1 -1
- package/dist/device-ws-hub.js +21 -2
- package/dist/device-ws-protocol.d.ts +70 -6
- package/dist/device-ws-protocol.js +50 -23
- package/dist/elicitation.js +3 -2
- package/dist/entry-problems.d.ts +34 -0
- package/dist/entry-problems.js +14 -0
- package/dist/execution-lane-caps.d.ts +49 -10
- package/dist/execution-lane-caps.js +43 -4
- package/dist/field-conflict.d.ts +44 -0
- package/dist/field-conflict.js +21 -0
- package/dist/file-history-disclosure.d.ts +34 -5
- package/dist/file-history-disclosure.js +2 -2
- package/dist/fleet/fleet-bus.d.ts +84 -6
- package/dist/fleet/fleet-bus.js +42 -9
- package/dist/hooks/classic-stations.d.ts +10 -5
- package/dist/hooks/classic-stations.js +4 -4
- package/dist/hooks/command-spawn.d.ts +57 -0
- package/dist/hooks/command-spawn.js +139 -0
- package/dist/hooks/hook-llm.js +2 -2
- package/dist/hooks/hook-runner.d.ts +127 -25
- package/dist/hooks/hook-runner.js +668 -287
- package/dist/hooks/hook-source.d.ts +44 -0
- package/dist/hooks/hook-source.js +40 -0
- package/dist/http/active-run-conflict.d.ts +5 -13
- package/dist/http/active-run-conflict.js +17 -27
- package/dist/http/admission.js +111 -60
- package/dist/http/approval-sweeps.js +58 -19
- package/dist/http/caller-trust.d.ts +77 -0
- package/dist/http/caller-trust.js +143 -0
- package/dist/http/checkpoint-ctx.d.ts +173 -18
- package/dist/http/checkpoint-ctx.js +171 -12
- package/dist/http/claimed-run.d.ts +4 -3
- package/dist/http/device-placement.d.ts +2 -4
- package/dist/http/device-placement.js +2 -1
- package/dist/http/dispatch.js +13 -7
- package/dist/http/durable-run-leg.d.ts +4 -32
- package/dist/http/durable-run-leg.js +37 -12
- package/dist/http/idempotency.d.ts +2 -2
- package/dist/http/idempotency.js +2 -2
- package/dist/http/owner-access.d.ts +56 -0
- package/dist/http/owner-access.js +42 -0
- package/dist/http/principal-gate.d.ts +78 -27
- package/dist/http/principal-gate.js +64 -14
- package/dist/http/readiness.d.ts +11 -5
- package/dist/http/readiness.js +10 -3
- package/dist/http/refusal-body.d.ts +12 -0
- package/dist/http/refusal-body.js +11 -0
- package/dist/http/request-guards.d.ts +16 -32
- package/dist/http/request-guards.js +31 -30
- package/dist/http/resume-ctx-read.d.ts +50 -6
- package/dist/http/resume-ctx-read.js +33 -7
- package/dist/http/resume-legs.d.ts +5 -5
- package/dist/http/resume-legs.js +163 -91
- package/dist/http/retired-service-deps.d.ts +43 -0
- package/dist/http/retired-service-deps.js +18 -0
- package/dist/http/route-ctx.d.ts +44 -37
- package/dist/http/route-ctx.js +4 -0
- package/dist/http/route-table.js +2 -0
- package/dist/http/routes/a2a-serve.js +3 -4
- package/dist/http/routes/admin-config-refresh.js +1 -1
- package/dist/http/routes/admin-drain.js +1 -1
- package/dist/http/routes/admin-memory-optout.js +5 -1
- package/dist/http/routes/adoption.js +2 -9
- package/dist/http/routes/agents-roster.js +2 -3
- package/dist/http/routes/approvals-assistant.js +91 -56
- package/dist/http/routes/background.js +3 -2
- package/dist/http/routes/capabilities.js +20 -4
- package/dist/http/routes/config-catalog.d.ts +3 -3
- package/dist/http/routes/config-catalog.js +1 -1
- package/dist/http/routes/devices.js +3 -17
- package/dist/http/routes/diagnostics.d.ts +3 -3
- package/dist/http/routes/diagnostics.js +6 -2
- package/dist/http/routes/fleet.js +4 -4
- package/dist/http/routes/images.js +6 -7
- package/dist/http/routes/leader.js +1 -8
- package/dist/http/routes/mcp-reconnect.d.ts +1 -1
- package/dist/http/routes/mcp-reconnect.js +3 -2
- package/dist/http/routes/memory-bundle.js +1 -1
- package/dist/http/routes/memory-compliance.js +1 -1
- package/dist/http/routes/memory-consolidation.js +1 -1
- package/dist/http/routes/memory-origin.d.ts +1 -1
- package/dist/http/routes/memory-origin.js +1 -1
- package/dist/http/routes/memory-policy.d.ts +1 -1
- package/dist/http/routes/memory-policy.js +3 -2
- package/dist/http/routes/notify-wake.js +8 -21
- package/dist/http/routes/observability.js +2 -2
- package/dist/http/routes/principal-inbox.d.ts +39 -0
- package/dist/http/routes/principal-inbox.js +76 -0
- package/dist/http/routes/retention-ops.js +2 -9
- package/dist/http/routes/rules.js +9 -22
- package/dist/http/routes/runs.d.ts +5 -0
- package/dist/http/routes/runs.js +63 -115
- package/dist/http/routes/session-sync.d.ts +3 -2
- package/dist/http/routes/session-sync.js +53 -48
- package/dist/http/routes/sessions-list.d.ts +1 -1
- package/dist/http/routes/sessions-list.js +3 -4
- package/dist/http/routes/sessions.js +33 -43
- package/dist/http/routes/side-query.js +6 -4
- package/dist/http/routes/tasks.js +71 -30
- package/dist/http/routes/trace-usage.js +7 -9
- package/dist/http/routes/workflows.d.ts +351 -0
- package/dist/http/routes/workflows.js +153 -119
- package/dist/http/run-declared-families.d.ts +16 -0
- package/dist/http/run-declared-families.js +12 -0
- package/dist/http/send.d.ts +24 -3
- package/dist/http/send.js +14 -2
- package/dist/http/server.d.ts +23 -18
- package/dist/http/server.js +20 -15
- package/dist/http/stored-body-replay.js +17 -4
- package/dist/http/submit-ctx.d.ts +4 -3
- package/dist/http/submit-ctx.js +23 -6
- package/dist/http/submit-replay.d.ts +4 -3
- package/dist/http/submit-replay.js +5 -4
- package/dist/http/wire-types.d.ts +17 -10
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -1
- package/dist/lane-home-probe.d.ts +78 -9
- package/dist/lane-home-probe.js +78 -12
- package/dist/leader/endpoint.d.ts +4 -1
- package/dist/leader/endpoint.js +11 -10
- package/dist/leader/wire.d.ts +4 -0
- package/dist/leader/wire.js +8 -7
- package/dist/limit-refusal.d.ts +17 -5
- package/dist/limit-refusal.js +4 -1
- package/dist/main.js +1 -1
- package/dist/mcp-assertion-header.d.ts +31 -10
- package/dist/mcp-assertion-header.js +10 -3
- package/dist/memory-origin-fence.js +2 -2
- package/dist/memory-project-layer.d.ts +37 -0
- package/dist/memory-project-layer.js +46 -0
- package/dist/memory-scope.d.ts +23 -16
- package/dist/memory-scope.js +24 -8
- package/dist/memory-sync-client.d.ts +27 -13
- package/dist/memory-sync-client.js +107 -18
- package/dist/model-compat.d.ts +1 -1
- package/dist/model-entry-refusal.d.ts +9 -1
- package/dist/model-entry-refusal.js +6 -1
- package/dist/model-plane-face.d.ts +51 -0
- package/dist/model-plane-face.js +24 -0
- package/dist/model-provider.d.ts +5 -0
- package/dist/model-provider.js +4 -2
- package/dist/model-route-endpoint.d.ts +8 -11
- package/dist/model-route-endpoint.js +4 -7
- package/dist/model-route-env.d.ts +22 -0
- package/dist/model-route-env.js +2 -0
- package/dist/model-route-gap.d.ts +60 -30
- package/dist/model-route-gap.js +21 -15
- package/dist/observability/corrupt-read-seat.d.ts +7 -0
- package/dist/observability/corrupt-read-seat.js +9 -0
- package/dist/observability/err-text.d.ts +6 -0
- package/dist/observability/err-text.js +18 -4
- package/dist/observability/fail-open.d.ts +35 -9
- package/dist/observability/fail-open.js +35 -9
- package/dist/observability/metrics.js +1 -0
- package/dist/observability/prompt-manifest.d.ts +22 -25
- package/dist/observability/prompt-manifest.js +54 -50
- package/dist/observability/tool-trace.js +9 -10
- package/dist/operator-ask.d.ts +10 -0
- package/dist/operator-ask.js +3 -0
- package/dist/orchestration/workflow-agent-session-index.d.ts +12 -12
- package/dist/orchestration/workflow-agent-session-index.js +9 -13
- package/dist/orchestration/workflow-completion-inbox.d.ts +42 -15
- package/dist/orchestration/workflow-completion-inbox.js +13 -0
- package/dist/orchestration/workflow-notify-journal.d.ts +13 -32
- package/dist/orchestration/workflow-notify-journal.js +27 -34
- package/dist/orchestration/workflow-park-decision.d.ts +89 -0
- package/dist/orchestration/workflow-park-decision.js +119 -0
- package/dist/org-delivery-gate.d.ts +81 -0
- package/dist/org-delivery-gate.js +59 -0
- package/dist/panel-tool-names.d.ts +23 -0
- package/dist/panel-tool-names.js +8 -0
- package/dist/parked-decide.d.ts +43 -11
- package/dist/parked-decide.js +100 -51
- package/dist/parked-revive-surface.d.ts +9 -2
- package/dist/parked-revive-surface.js +11 -8
- package/dist/permission-rule-list.d.ts +1 -1
- package/dist/permission-rule-list.js +2 -2
- package/dist/permission-rule-vocab.d.ts +1 -23
- package/dist/permission-rule-vocab.js +0 -13
- package/dist/plugins/approval-ask-store-file.d.ts +16 -1
- package/dist/plugins/approval-ask-store-file.js +21 -0
- package/dist/plugins/approval-ask-store-memory.d.ts +18 -1
- package/dist/plugins/approval-ask-store-memory.js +34 -2
- package/dist/plugins/approval-ask-store-sql.d.ts +25 -0
- package/dist/plugins/approval-ask-store-sql.js +37 -1
- package/dist/plugins/approval-payload-fingerprint.d.ts +14 -6
- package/dist/plugins/approval-payload-fingerprint.js +1 -1
- package/dist/plugins/blob-backend.d.ts +38 -25
- package/dist/plugins/blob-backend.js +29 -19
- package/dist/plugins/breaker-state-sql.d.ts +2 -2
- package/dist/plugins/breaker-state-sql.js +4 -4
- package/dist/plugins/caching-session-store.d.ts +4 -1
- package/dist/plugins/caching-session-store.js +7 -3
- package/dist/plugins/checkpoint-store-sql.d.ts +36 -1
- package/dist/plugins/checkpoint-store-sql.js +65 -3
- package/dist/plugins/file-history-store-sql.d.ts +5 -2
- package/dist/plugins/file-history-store-sql.js +3 -1
- package/dist/plugins/file-run-store.d.ts +5 -13
- package/dist/plugins/file-run-store.js +9 -0
- package/dist/plugins/fork-routing-session-store.d.ts +9 -1
- package/dist/plugins/fork-routing-session-store.js +12 -1
- package/dist/plugins/host-platform.d.ts +2 -0
- package/dist/plugins/host-platform.js +2 -1
- package/dist/plugins/local-checkpoint-store.d.ts +15 -2
- package/dist/plugins/local-checkpoint-store.js +32 -1
- package/dist/plugins/local-session-store.d.ts +13 -10
- package/dist/plugins/local-session-store.js +23 -14
- package/dist/plugins/local-session-window.d.ts +45 -0
- package/dist/plugins/local-session-window.js +38 -0
- package/dist/plugins/mailbox-recipient-lifecycle.d.ts +3 -2
- package/dist/plugins/mailbox-store-sql.d.ts +30 -48
- package/dist/plugins/mailbox-store-sql.js +77 -52
- package/dist/plugins/memory-embedder-fingerprint.js +2 -1
- package/dist/plugins/memory-run-store.d.ts +5 -13
- package/dist/plugins/memory-run-store.js +8 -0
- package/dist/plugins/one-time-migrations.d.ts +121 -5
- package/dist/plugins/one-time-migrations.js +332 -8
- package/dist/plugins/org-delivery-quota-store.d.ts +123 -0
- package/dist/plugins/org-delivery-quota-store.js +228 -0
- package/dist/plugins/permission-rule-store-file.js +10 -6
- package/dist/plugins/permission-rule-store-sql.js +8 -8
- package/dist/plugins/pg-pool.js +39 -7
- package/dist/plugins/pg-session-storage.d.ts +18 -5
- package/dist/plugins/pg-session-storage.js +68 -23
- package/dist/plugins/remote-env-adb.d.ts +6 -12
- package/dist/plugins/remote-env-adb.js +12 -2
- package/dist/plugins/remote-env-device.d.ts +14 -7
- package/dist/plugins/remote-env-device.js +23 -15
- package/dist/plugins/remote-env-k8s.d.ts +8 -12
- package/dist/plugins/remote-env-k8s.js +23 -5
- package/dist/plugins/remote-env-local-docker.d.ts +10 -15
- package/dist/plugins/remote-env-local-docker.js +12 -9
- package/dist/plugins/remote-env-ssh.d.ts +10 -16
- package/dist/plugins/remote-env-ssh.js +17 -11
- package/dist/plugins/run-store-sql.d.ts +16 -14
- package/dist/plugins/run-store-sql.js +22 -2
- package/dist/plugins/schema-indexes.js +2 -0
- package/dist/plugins/send-user-file.js +2 -2
- package/dist/plugins/session-epoch-anchor.d.ts +32 -6
- package/dist/plugins/session-epoch-anchor.js +2 -3
- package/dist/plugins/session-ownership.d.ts +11 -0
- package/dist/plugins/session-ownership.js +11 -0
- package/dist/plugins/session-placement.d.ts +12 -14
- package/dist/plugins/session-placement.js +2 -15
- package/dist/plugins/session-store.d.ts +18 -8
- package/dist/plugins/session-store.js +15 -4
- package/dist/plugins/sql-driver.d.ts +74 -11
- package/dist/plugins/sql-driver.js +210 -12
- package/dist/plugins/sql-errors.d.ts +23 -0
- package/dist/plugins/sql-errors.js +21 -0
- package/dist/plugins/store-backend.d.ts +36 -1
- package/dist/plugins/store-backend.js +40 -14
- package/dist/plugins/store-contracts.d.ts +42 -0
- package/dist/plugins/store-contracts.js +1 -0
- package/dist/plugins/tidb-pool.d.ts +6 -1
- package/dist/plugins/tidb-pool.js +44 -10
- package/dist/plugins/tidb-session-storage.d.ts +3 -2
- package/dist/plugins/tidb-session-storage.js +2 -2
- package/dist/plugins/tidb-session-store.d.ts +25 -5
- package/dist/plugins/tidb-session-store.js +67 -21
- package/dist/plugins/workflow-journal-store-sql.d.ts +73 -19
- package/dist/plugins/workflow-journal-store-sql.js +19 -6
- package/dist/plugins/workflow-run-store-sql.d.ts +30 -8
- package/dist/plugins/workflow-run-store-sql.js +50 -9
- package/dist/posture-source.d.ts +18 -10
- package/dist/posture-source.js +4 -1
- package/dist/project-identity.d.ts +45 -6
- package/dist/project-identity.js +162 -8
- package/dist/project-layer-key.d.ts +28 -0
- package/dist/project-layer-key.js +29 -0
- package/dist/question.js +3 -2
- package/dist/read-deny-entry.d.ts +25 -0
- package/dist/read-deny-entry.js +12 -0
- package/dist/read-face-posture.d.ts +37 -11
- package/dist/read-face-posture.js +32 -3
- package/dist/request-key-closure.d.ts +7 -0
- package/dist/request-key-closure.js +9 -1
- package/dist/request-tool-axes.d.ts +12 -0
- package/dist/request-tool-axes.js +20 -0
- package/dist/request-value-closure.d.ts +5 -0
- package/dist/request-value-closure.js +11 -3
- package/dist/rule-entry-list.d.ts +26 -2
- package/dist/rule-entry-list.js +72 -12
- package/dist/rules-consent.js +3 -4
- package/dist/run-acceptance.d.ts +19 -3
- package/dist/run-acceptance.js +13 -1
- package/dist/run-local.js +37 -6
- package/dist/runs.d.ts +12 -10
- package/dist/runs.js +25 -6
- package/dist/runtime-caps-resolver.d.ts +16 -6
- package/dist/runtime-caps-resolver.js +18 -5
- package/dist/sandbox-pkg-source.d.ts +19 -1
- package/dist/sandbox-pkg-source.js +29 -7
- package/dist/sealed-key.d.ts +32 -0
- package/dist/sealed-key.js +55 -1
- package/dist/security.d.ts +95 -15
- package/dist/security.js +52 -9
- package/dist/session-error-wire.d.ts +31 -7
- package/dist/session-error-wire.js +9 -2
- package/dist/session-sync-kernel.d.ts +37 -10
- package/dist/session-sync-kernel.js +10 -0
- package/dist/session-titled-frame.d.ts +33 -0
- package/dist/session-titled-frame.js +4 -0
- package/dist/session-titler.d.ts +9 -1
- package/dist/session-titler.js +15 -4
- package/dist/spec-fields.js +21 -2
- package/dist/sql-engine-posture.d.ts +3 -1
- package/dist/task-a2a.js +4 -3
- package/dist/task-cwd.d.ts +1 -1
- package/dist/task-mcp.d.ts +5 -5
- package/dist/task-mcp.js +27 -74
- package/dist/task-settings.d.ts +93 -88
- package/dist/task-settings.js +71 -63
- package/dist/text-bidi.d.ts +0 -4
- package/dist/text-bidi.js +1 -3
- package/dist/tighten-only-seats.d.ts +150 -0
- package/dist/tighten-only-seats.js +196 -0
- package/dist/tool-approval.d.ts +451 -136
- package/dist/tool-approval.js +733 -256
- package/dist/tool-rule-names.d.ts +6 -0
- package/dist/trace/core-keyset-guard.d.ts +108 -25
- package/dist/trace/exit-seat.d.ts +40 -0
- package/dist/trace/exit-seat.js +130 -0
- package/dist/trace/ledger-events.d.ts +6 -3
- package/dist/trace/project.d.ts +98 -13
- package/dist/trace/project.js +57 -32
- package/dist/trace/projection-table.d.ts +100 -0
- package/dist/trace/projection-table.js +130 -0
- package/dist/trace/redact.d.ts +57 -0
- package/dist/trace/redact.js +962 -168
- package/dist/trace/self-minted-wire.d.ts +349 -1
- package/dist/trace/self-minted-wire.js +118 -1
- package/dist/trace/wire-projection-faces.d.ts +304 -2
- package/dist/trace/wire-projection-faces.js +160 -13
- package/dist/url-refusal.js +2 -0
- package/dist/web-admin-client/apply-effective.d.ts +12 -0
- package/dist/web-admin-client/apply-effective.js +56 -5
- package/dist/web-admin-client/facade.d.ts +1 -1
- package/dist/web-admin-client/facade.js +1 -1
- package/dist/web-admin-client/read-face.js +5 -9
- package/dist/web-admin-client/skills-mcp.d.ts +1 -1
- package/dist/web-admin-client/skills-mcp.js +6 -8
- package/dist/web-admin-client/types.d.ts +1 -1
- package/docs/DEVICE-EXECUTOR-PROTOCOL.md +57 -9
- package/package.json +8 -4
- package/scripts/drain-check-local.mjs +34 -2
- package/scripts/drain-check.mysql.sql +20 -2
- package/scripts/drain-check.pg.sql +20 -2
- package/dist/mcp-tool-faces.d.ts +0 -69
- package/dist/mcp-tool-faces.js +0 -106
package/dist/trace/redact.js
CHANGED
|
@@ -28,6 +28,137 @@ export function mapNormalizedToOriginal(text, normalized) {
|
|
|
28
28
|
map[normalized.length] = text.length;
|
|
29
29
|
return map;
|
|
30
30
|
}
|
|
31
|
+
function deleteSpelledFormatCharacters(s) {
|
|
32
|
+
let i = s.indexOf("\\");
|
|
33
|
+
if (i < 0)
|
|
34
|
+
return { text: s, at: undefined, cum: undefined };
|
|
35
|
+
const n = s.length;
|
|
36
|
+
const read = { end: 0 };
|
|
37
|
+
let cuts;
|
|
38
|
+
while (i >= 0) {
|
|
39
|
+
let body = i + 1;
|
|
40
|
+
while (body < n && s.charCodeAt(body) === CH_BACKSLASH)
|
|
41
|
+
body += 1;
|
|
42
|
+
const cp = readSpelledCodePoint(s, body, read);
|
|
43
|
+
if (cp >= 0 && isFormatCodePoint(cp)) {
|
|
44
|
+
(cuts ??= []).push(i, read.end);
|
|
45
|
+
body = read.end;
|
|
46
|
+
}
|
|
47
|
+
i = s.indexOf("\\", body);
|
|
48
|
+
}
|
|
49
|
+
if (cuts === undefined)
|
|
50
|
+
return { text: s, at: undefined, cum: undefined };
|
|
51
|
+
const count = cuts.length / 2;
|
|
52
|
+
const at = new Int32Array(count);
|
|
53
|
+
const cum = new Int32Array(count);
|
|
54
|
+
const parts = [];
|
|
55
|
+
let from = 0;
|
|
56
|
+
let removed = 0;
|
|
57
|
+
for (let k = 0; k < count; k++) {
|
|
58
|
+
const a = cuts[2 * k];
|
|
59
|
+
const b = cuts[2 * k + 1];
|
|
60
|
+
parts.push(s.slice(from, a));
|
|
61
|
+
at[k] = a - removed;
|
|
62
|
+
removed += b - a;
|
|
63
|
+
cum[k] = removed;
|
|
64
|
+
from = b;
|
|
65
|
+
}
|
|
66
|
+
parts.push(s.slice(from));
|
|
67
|
+
return { text: parts.join(""), at, cum };
|
|
68
|
+
}
|
|
69
|
+
function viewIndexInInput(fold, v) {
|
|
70
|
+
const at = fold.at;
|
|
71
|
+
let lo = 0;
|
|
72
|
+
let hi = at.length;
|
|
73
|
+
while (lo < hi) {
|
|
74
|
+
const mid = (lo + hi) >>> 1;
|
|
75
|
+
if (at[mid] <= v)
|
|
76
|
+
lo = mid + 1;
|
|
77
|
+
else
|
|
78
|
+
hi = mid;
|
|
79
|
+
}
|
|
80
|
+
return lo === 0 ? v : v + fold.cum[lo - 1];
|
|
81
|
+
}
|
|
82
|
+
function readHexRun(s, at, count) {
|
|
83
|
+
let v = 0;
|
|
84
|
+
for (let k = at; k < at + count; k++) {
|
|
85
|
+
const d = hexDigitValue(s.charCodeAt(k));
|
|
86
|
+
if (d < 0)
|
|
87
|
+
return -1;
|
|
88
|
+
v = v * 16 + d;
|
|
89
|
+
}
|
|
90
|
+
return v;
|
|
91
|
+
}
|
|
92
|
+
function readEscapeElement(s, at, out) {
|
|
93
|
+
const kind = s.charCodeAt(at);
|
|
94
|
+
if (kind === CH_x) {
|
|
95
|
+
out.end = at + 3;
|
|
96
|
+
return readHexRun(s, at + 1, 2);
|
|
97
|
+
}
|
|
98
|
+
if (kind !== CH_u)
|
|
99
|
+
return -1;
|
|
100
|
+
if (s.charCodeAt(at + 1) === CH_LBRACE) {
|
|
101
|
+
let v = 0;
|
|
102
|
+
let k = at + 2;
|
|
103
|
+
for (; k < s.length; k++) {
|
|
104
|
+
const d = hexDigitValue(s.charCodeAt(k));
|
|
105
|
+
if (d < 0)
|
|
106
|
+
break;
|
|
107
|
+
v = v * 16 + d;
|
|
108
|
+
if (v > 0x10ffff)
|
|
109
|
+
return -1;
|
|
110
|
+
}
|
|
111
|
+
if (k === at + 2 || s.charCodeAt(k) !== CH_RBRACE)
|
|
112
|
+
return -1;
|
|
113
|
+
out.end = k + 1;
|
|
114
|
+
return v;
|
|
115
|
+
}
|
|
116
|
+
out.end = at + 5;
|
|
117
|
+
return readHexRun(s, at + 1, 4);
|
|
118
|
+
}
|
|
119
|
+
function readSpelledCodePoint(s, at, out) {
|
|
120
|
+
const v = readEscapeElement(s, at, out);
|
|
121
|
+
if (v < 0xd800 || v > 0xdfff)
|
|
122
|
+
return v;
|
|
123
|
+
if (v > 0xdbff)
|
|
124
|
+
return -1;
|
|
125
|
+
let k = out.end;
|
|
126
|
+
if (s.charCodeAt(k) !== CH_BACKSLASH)
|
|
127
|
+
return -1;
|
|
128
|
+
while (s.charCodeAt(k) === CH_BACKSLASH)
|
|
129
|
+
k += 1;
|
|
130
|
+
const lo = readEscapeElement(s, k, out);
|
|
131
|
+
return lo >= 0xdc00 && lo <= 0xdfff ? 0x10000 + ((v - 0xd800) << 10) + (lo - 0xdc00) : -1;
|
|
132
|
+
}
|
|
133
|
+
const FORMAT_CODEPOINT_TABLE = (() => {
|
|
134
|
+
const table = new Uint8Array(0x10000);
|
|
135
|
+
const units = [];
|
|
136
|
+
const codes = [];
|
|
137
|
+
for (let c = 0; c < 0x10000; c++) {
|
|
138
|
+
if (c >= 0xd800 && c <= 0xdfff)
|
|
139
|
+
continue;
|
|
140
|
+
units.push(String.fromCharCode(c));
|
|
141
|
+
codes.push(c);
|
|
142
|
+
}
|
|
143
|
+
const kept = normalizeForRedaction(units.join(""));
|
|
144
|
+
let j = 0;
|
|
145
|
+
for (let i = 0; i < codes.length; i++) {
|
|
146
|
+
if (j < kept.length && kept.charCodeAt(j) === codes[i])
|
|
147
|
+
j += 1;
|
|
148
|
+
else
|
|
149
|
+
table[codes[i]] = 1;
|
|
150
|
+
}
|
|
151
|
+
if (j !== kept.length)
|
|
152
|
+
throw new Error("redact: format-character stripping is not a code-unit deletion over the BMP — refusing to derive the escape-spelled format-character table");
|
|
153
|
+
return table;
|
|
154
|
+
})();
|
|
155
|
+
function isFormatCodePoint(cp) {
|
|
156
|
+
if (cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))
|
|
157
|
+
return false;
|
|
158
|
+
if (cp >= 0x10000)
|
|
159
|
+
return normalizeForRedaction(String.fromCodePoint(cp)) === "";
|
|
160
|
+
return FORMAT_CODEPOINT_TABLE[cp] === 1;
|
|
161
|
+
}
|
|
31
162
|
export const REDACTION_TOKEN_RE_SRC = String.raw `«redacted(?::[a-z-]{1,16}){0,2}»`;
|
|
32
163
|
const ARM_VALUE_CLASSES = [];
|
|
33
164
|
const atLeast = (cls, min) => {
|
|
@@ -42,7 +173,7 @@ const WORD_INTERIOR_CLASS = "[A-Za-z0-9]";
|
|
|
42
173
|
const CH_TAB = 9, CH_LF = 10, CH_CR = 13, CH_ESC = 27, CH_SPACE = 32, CH_PLUS = 43, CH_MINUS = 45, CH_SLASH = 47;
|
|
43
174
|
const CH_0 = 48, CH_9 = 57, CH_COLON = 58, CH_EQ = 61, CH_AT = 64, CH_A = 65, CH_Z = 90;
|
|
44
175
|
const CH_LBRACKET = 91, CH_BACKSLASH = 92, CH_UNDERSCORE = 95;
|
|
45
|
-
const CH_a = 97, CH_b = 98, CH_f = 102, CH_n = 110, CH_r = 114, CH_t = 116, CH_u = 117, CH_z = 122, CH_TILDE = 126;
|
|
176
|
+
const CH_a = 97, CH_b = 98, CH_f = 102, CH_n = 110, CH_r = 114, CH_t = 116, CH_u = 117, CH_x = 120, CH_z = 122, CH_TILDE = 126;
|
|
46
177
|
const CH_QUOTE = 34;
|
|
47
178
|
const CH_CSI8 = 155;
|
|
48
179
|
const CH_LAQUO = 171;
|
|
@@ -276,10 +407,14 @@ let coveredPositions = 0;
|
|
|
276
407
|
let headScreenHits = 0;
|
|
277
408
|
let fullYCalls = 0;
|
|
278
409
|
let forensicLevels = 0;
|
|
410
|
+
let forensicRuns = 0;
|
|
411
|
+
let forensicSkips = 0;
|
|
412
|
+
let urlSegmentRuns = 0;
|
|
413
|
+
let urlSegmentReplays = 0;
|
|
279
414
|
let ssh2ScanBytes = 0;
|
|
280
415
|
let mappedUnits = 0;
|
|
281
416
|
export function armPoolBudgetForTest() {
|
|
282
|
-
return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, ssh2ScanBytes, mappedUnits };
|
|
417
|
+
return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits };
|
|
283
418
|
}
|
|
284
419
|
export function resetArmPoolBudgetForTest() {
|
|
285
420
|
poolSpanCount = 0;
|
|
@@ -287,6 +422,10 @@ export function resetArmPoolBudgetForTest() {
|
|
|
287
422
|
headScreenHits = 0;
|
|
288
423
|
fullYCalls = 0;
|
|
289
424
|
forensicLevels = 0;
|
|
425
|
+
forensicRuns = 0;
|
|
426
|
+
forensicSkips = 0;
|
|
427
|
+
urlSegmentRuns = 0;
|
|
428
|
+
urlSegmentReplays = 0;
|
|
290
429
|
ssh2ScanBytes = 0;
|
|
291
430
|
mappedUnits = 0;
|
|
292
431
|
}
|
|
@@ -624,23 +763,30 @@ function collectStickySpans(a, text, at, out) {
|
|
|
624
763
|
}
|
|
625
764
|
}
|
|
626
765
|
const WORD_SEGMENT_CLASS = "[A-Za-z0-9._~+/-]";
|
|
766
|
+
const SCHEME_DELIMITER_CHARS = "\\\"'`“”|";
|
|
767
|
+
if (/[\]^-]/.test(SCHEME_DELIMITER_CHARS))
|
|
768
|
+
throw new Error("redact: scheme delimiter table holds a character-class metacharacter — the class source is built without escaping it");
|
|
769
|
+
const SCHEME_DELIMITER_CODES = new Set([...SCHEME_DELIMITER_CHARS].map((c) => c.charCodeAt(0)));
|
|
770
|
+
const isSchemeDelimiterCode = (c) => SCHEME_DELIMITER_CODES.has(c);
|
|
771
|
+
const SCHEME_SEPARATOR_CLASS = `[\\s${SCHEME_DELIMITER_CHARS.replace(/\\/g, "\\\\")}]`;
|
|
627
772
|
const LEXICAL_KEYWORD_ARMS = new Set();
|
|
628
|
-
const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") => {
|
|
629
|
-
const re = new RegExp(`(?:${keywordSrc}
|
|
773
|
+
const keywordArm = (keywordSrc, valueClass, min, tailSrc = "", separatorClass = "\\s") => {
|
|
774
|
+
const re = new RegExp(`(?:${keywordSrc}${separatorClass}+${atLeast(valueClass, min)}${tailSrc}` +
|
|
630
775
|
`|(?<!${WORD_SEGMENT_CLASS})${keywordSrc}(?:${twoStrongAhead(valueClass)}|${noConnectorAhead(valueClass)})${atLeast(valueClass, min)}${tailSrc})`, "g");
|
|
631
776
|
LEXICAL_KEYWORD_ARMS.add(re);
|
|
632
777
|
return re;
|
|
633
778
|
};
|
|
634
|
-
const
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
"auth", "authorization",
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
"x-amz-signature", "x-amz-security-token", "x-amz-credential",
|
|
642
|
-
|
|
643
|
-
|
|
779
|
+
const CREDENTIAL_NAME_ANCHORS = {
|
|
780
|
+
api_key: "structure-only", apikey: "structure-only", "api-key": "structure-only", key: "structure-only",
|
|
781
|
+
token: "structure-only", access_token: "structure-only", refresh_token: "structure-only", id_token: "structure-only",
|
|
782
|
+
auth: "structure-only", authorization: "auth-scheme", "proxy-authorization": "auth-scheme",
|
|
783
|
+
password: "declarative", passwd: "any-form", pwd: "any-form",
|
|
784
|
+
secret: "declarative", client_secret: "declarative", private_key: "structure-only",
|
|
785
|
+
signature: "structure-only", sig: "structure-only",
|
|
786
|
+
"x-amz-signature": "structure-only", "x-amz-security-token": "structure-only", "x-amz-credential": "structure-only",
|
|
787
|
+
sas: "structure-only",
|
|
788
|
+
};
|
|
789
|
+
const CREDENTIAL_NAME_WORDS = Object.keys(CREDENTIAL_NAME_ANCHORS);
|
|
644
790
|
const CREDENTIAL_NAME_ALT = (() => {
|
|
645
791
|
for (const w of CREDENTIAL_NAME_WORDS) {
|
|
646
792
|
if (!/^[a-z][a-z0-9_-]*$/.test(w))
|
|
@@ -657,9 +803,10 @@ const STRUCTURAL_SECRET_PATTERNS = [
|
|
|
657
803
|
const PPK_CONTAINER_TOKEN = "«redacted:private-key»";
|
|
658
804
|
const PEM_FRAGMENT_TOKEN = "«redacted:private-key:partial»";
|
|
659
805
|
const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
|
|
806
|
+
const ANCHORED_VALUE_TOKEN = "«redacted»";
|
|
660
807
|
const KEYWORD_SECRET_PATTERNS = [
|
|
661
|
-
[keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", 20, "=*"), "Bearer «redacted»"],
|
|
662
|
-
[keywordArm("Basic", "[A-Za-z0-9+/]", 16, "={0,2}"), "Basic «redacted»"],
|
|
808
|
+
[keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", 20, "=*", SCHEME_SEPARATOR_CLASS), "Bearer «redacted»"],
|
|
809
|
+
[keywordArm("Basic", "[A-Za-z0-9+/]", 16, "={0,2}", SCHEME_SEPARATOR_CLASS), "Basic «redacted»"],
|
|
663
810
|
[keywordArm("[Tt]oken", "[A-Za-z0-9._~+/-]", 20), "token «redacted»"],
|
|
664
811
|
];
|
|
665
812
|
const SK_FULL_ARM_MIN = 20;
|
|
@@ -689,11 +836,11 @@ function pemEscapeLetterAt(s, i, limit) {
|
|
|
689
836
|
const n = s.charCodeAt(j);
|
|
690
837
|
return n === CH_r || n === CH_n || n === CH_t ? j : -1;
|
|
691
838
|
}
|
|
692
|
-
function skipPemSeps(s, i, limit) {
|
|
839
|
+
function skipPemSeps(scan, s, i, limit) {
|
|
693
840
|
for (;;) {
|
|
694
841
|
if (i >= limit)
|
|
695
842
|
return i;
|
|
696
|
-
if (opaqueUnitEnd(i) >= 0)
|
|
843
|
+
if (opaqueUnitEnd(scan, i) >= 0)
|
|
697
844
|
return i;
|
|
698
845
|
const c = s.charCodeAt(i);
|
|
699
846
|
if (isSepCode(c)) {
|
|
@@ -736,17 +883,17 @@ function pemInlineSepLen(s, i, limit) {
|
|
|
736
883
|
}
|
|
737
884
|
return 0;
|
|
738
885
|
}
|
|
739
|
-
function probePemHeaderToken(s, p, limit) {
|
|
740
|
-
if (p >= limit || opaqueUnitEnd(p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
|
|
886
|
+
function probePemHeaderToken(scan, s, p, limit) {
|
|
887
|
+
if (p >= limit || opaqueUnitEnd(scan, p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
|
|
741
888
|
return -1;
|
|
742
889
|
p += 1;
|
|
743
890
|
let identLen = 1;
|
|
744
|
-
while (identLen < PEM_HEADER_IDENT_MAX && p < limit && opaqueUnitEnd(p) < 0 && isIdentCode(s.charCodeAt(p))) {
|
|
891
|
+
while (identLen < PEM_HEADER_IDENT_MAX && p < limit && opaqueUnitEnd(scan, p) < 0 && isIdentCode(s.charCodeAt(p))) {
|
|
745
892
|
p += 1;
|
|
746
893
|
identLen += 1;
|
|
747
894
|
}
|
|
748
895
|
for (;;) {
|
|
749
|
-
if (p >= limit || opaqueUnitEnd(p) >= 0)
|
|
896
|
+
if (p >= limit || opaqueUnitEnd(scan, p) >= 0)
|
|
750
897
|
return -1;
|
|
751
898
|
const c = s.charCodeAt(p);
|
|
752
899
|
if (c === CH_SPACE || c === CH_TAB) {
|
|
@@ -756,29 +903,29 @@ function probePemHeaderToken(s, p, limit) {
|
|
|
756
903
|
return c === CH_COLON ? p + 1 : -1;
|
|
757
904
|
}
|
|
758
905
|
}
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
906
|
+
function setPemHdr(scan, search, contentEnd, next, hitLimit, jsonEnd = false) {
|
|
907
|
+
const h = scan.hdr;
|
|
908
|
+
h.search = search;
|
|
909
|
+
h.contentEnd = contentEnd;
|
|
910
|
+
h.next = next;
|
|
911
|
+
h.hitLimit = hitLimit;
|
|
912
|
+
h.jsonEnd = jsonEnd;
|
|
766
913
|
return true;
|
|
767
914
|
}
|
|
768
|
-
function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
769
|
-
const start = skipPemSeps(s, i, limit);
|
|
915
|
+
function parsePemHeaderLine(scan, s, i, limit, allowBare, json) {
|
|
916
|
+
const start = skipPemSeps(scan, s, i, limit);
|
|
770
917
|
if (start >= limit)
|
|
771
918
|
return false;
|
|
772
|
-
const afterToken = probePemHeaderToken(s, start, limit);
|
|
919
|
+
const afterToken = probePemHeaderToken(scan, s, start, limit);
|
|
773
920
|
if (afterToken < 0 && !allowBare)
|
|
774
921
|
return false;
|
|
775
922
|
let p = afterToken < 0 ? start : afterToken;
|
|
776
|
-
if (afterToken >= 0 && probePemHeaderToken(s, p, limit) >= 0)
|
|
777
|
-
return setPemHdr(start, p, p, false);
|
|
923
|
+
if (afterToken >= 0 && probePemHeaderToken(scan, s, p, limit) >= 0)
|
|
924
|
+
return setPemHdr(scan, start, p, p, false);
|
|
778
925
|
for (;;) {
|
|
779
926
|
if (p >= limit)
|
|
780
|
-
return setPemHdr(start, p, p, true);
|
|
781
|
-
const unit = opaqueUnitEnd(p);
|
|
927
|
+
return setPemHdr(scan, start, p, p, true);
|
|
928
|
+
const unit = opaqueUnitEnd(scan, p);
|
|
782
929
|
if (unit >= 0) {
|
|
783
930
|
p = Math.min(unit, limit);
|
|
784
931
|
continue;
|
|
@@ -790,12 +937,12 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
790
937
|
}
|
|
791
938
|
if (c === CH_QUOTE) {
|
|
792
939
|
if (json)
|
|
793
|
-
return setPemHdr(start, p, p, false, true);
|
|
940
|
+
return setPemHdr(scan, start, p, p, false, true);
|
|
794
941
|
p += 1;
|
|
795
942
|
continue;
|
|
796
943
|
}
|
|
797
944
|
if (c === CH_CR || c === CH_LF)
|
|
798
|
-
return setPemHdr(start, p, p + 1, false);
|
|
945
|
+
return setPemHdr(scan, start, p, p + 1, false);
|
|
799
946
|
if (c === CH_BACKSLASH) {
|
|
800
947
|
let j = p;
|
|
801
948
|
while (j < limit && s.charCodeAt(j) === CH_BACKSLASH)
|
|
@@ -808,11 +955,11 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
808
955
|
}
|
|
809
956
|
let q = p;
|
|
810
957
|
for (;;) {
|
|
811
|
-
if (opaqueUnitEnd(q) >= 0)
|
|
958
|
+
if (opaqueUnitEnd(scan, q) >= 0)
|
|
812
959
|
break;
|
|
813
960
|
const b = pemLineBreakLen(s, q, limit);
|
|
814
961
|
if (b === 1)
|
|
815
|
-
return setPemHdr(start, p, q + 1, false);
|
|
962
|
+
return setPemHdr(scan, start, p, q + 1, false);
|
|
816
963
|
if (b >= 2) {
|
|
817
964
|
q += b;
|
|
818
965
|
continue;
|
|
@@ -823,19 +970,19 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
823
970
|
q += u;
|
|
824
971
|
}
|
|
825
972
|
if (q >= limit)
|
|
826
|
-
return setPemHdr(start, p, q, true);
|
|
827
|
-
if (probePemHeaderToken(s, q, limit) >= 0)
|
|
828
|
-
return setPemHdr(start, p, q, false);
|
|
973
|
+
return setPemHdr(scan, start, p, q, true);
|
|
974
|
+
if (probePemHeaderToken(scan, s, q, limit) >= 0)
|
|
975
|
+
return setPemHdr(scan, start, p, q, false);
|
|
829
976
|
p = q;
|
|
830
977
|
}
|
|
831
978
|
}
|
|
832
|
-
function parsePemBody(s, i, limit) {
|
|
979
|
+
function parsePemBody(scan, s, i, limit) {
|
|
833
980
|
let end = -1;
|
|
834
981
|
let p = i;
|
|
835
982
|
for (;;) {
|
|
836
|
-
const q = skipPemSeps(s, p, limit);
|
|
983
|
+
const q = skipPemSeps(scan, s, p, limit);
|
|
837
984
|
let r = q;
|
|
838
|
-
while (r < limit && opaqueUnitEnd(r) < 0 && isBase64Code(s.charCodeAt(r)))
|
|
985
|
+
while (r < limit && opaqueUnitEnd(scan, r) < 0 && isBase64Code(s.charCodeAt(r)))
|
|
839
986
|
r += 1;
|
|
840
987
|
if (r - q < PEM_BODY_RUN_MIN)
|
|
841
988
|
break;
|
|
@@ -844,12 +991,12 @@ function parsePemBody(s, i, limit) {
|
|
|
844
991
|
}
|
|
845
992
|
return end;
|
|
846
993
|
}
|
|
847
|
-
function rightmostPemRunEnd(s, from, to) {
|
|
994
|
+
function rightmostPemRunEnd(scan, s, from, to) {
|
|
848
995
|
let best = -1;
|
|
849
996
|
let runStart = -1;
|
|
850
997
|
for (let p = from; p <= to; p++) {
|
|
851
998
|
if (p < to) {
|
|
852
|
-
const unit = opaqueUnitEnd(p);
|
|
999
|
+
const unit = opaqueUnitEnd(scan, p);
|
|
853
1000
|
if (unit > 0) {
|
|
854
1001
|
if (runStart >= 0 && p - runStart >= PEM_BODY_RUN_MIN)
|
|
855
1002
|
best = p;
|
|
@@ -893,7 +1040,7 @@ function isPemFenceLabel(s, from, to) {
|
|
|
893
1040
|
}
|
|
894
1041
|
const PEM_FENCE_OPEN = "-----BEGIN";
|
|
895
1042
|
const PEM_FENCE_CLOSE = "-----";
|
|
896
|
-
function findPemFence(text, from) {
|
|
1043
|
+
function findPemFence(scan, text, from) {
|
|
897
1044
|
for (;;) {
|
|
898
1045
|
const fence = text.indexOf(PEM_FENCE_OPEN, from);
|
|
899
1046
|
if (fence < 0)
|
|
@@ -903,7 +1050,7 @@ function findPemFence(text, from) {
|
|
|
903
1050
|
if (close < 0)
|
|
904
1051
|
return undefined;
|
|
905
1052
|
from = labelFrom;
|
|
906
|
-
if (inOpaqueUnit(fence))
|
|
1053
|
+
if (inOpaqueUnit(scan, fence))
|
|
907
1054
|
continue;
|
|
908
1055
|
if (!isPemFenceLabel(text, labelFrom, close))
|
|
909
1056
|
continue;
|
|
@@ -913,26 +1060,28 @@ function findPemFence(text, from) {
|
|
|
913
1060
|
const SSH2_FENCE_OPEN = "---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
914
1061
|
const SSH2_FENCE_CLOSE = "---- END SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
915
1062
|
const SSH2_BODY_MAX = 8192;
|
|
916
|
-
function unitCountWithin(from, to, max) {
|
|
1063
|
+
function unitCountWithin(scan, from, to, max) {
|
|
917
1064
|
if (to - from <= max)
|
|
918
1065
|
return true;
|
|
919
|
-
|
|
1066
|
+
noteEvidenceRead(scan, from);
|
|
1067
|
+
noteEvidenceRead(scan, to);
|
|
1068
|
+
const st = scan.table.starts;
|
|
920
1069
|
if (st.length === 0)
|
|
921
1070
|
return false;
|
|
922
|
-
const k = firstCredentialSpanFrom(
|
|
1071
|
+
const k = firstCredentialSpanFrom(scan.table, from);
|
|
923
1072
|
if (k >= st.length || st[k] >= to)
|
|
924
1073
|
return false;
|
|
925
1074
|
let n = 0;
|
|
926
1075
|
for (let p = from; p < to;) {
|
|
927
1076
|
if (n > max)
|
|
928
1077
|
return false;
|
|
929
|
-
const unit = opaqueUnitEnd(p);
|
|
1078
|
+
const unit = opaqueUnitEnd(scan, p);
|
|
930
1079
|
p = unit > 0 && unit <= to ? unit : p + 1;
|
|
931
1080
|
n += 1;
|
|
932
1081
|
}
|
|
933
1082
|
return n <= max;
|
|
934
1083
|
}
|
|
935
|
-
function collectSsh2FenceSpans(text, span) {
|
|
1084
|
+
function collectSsh2FenceSpans(scan, text, span) {
|
|
936
1085
|
let cachedFrom = 0;
|
|
937
1086
|
let cachedAt;
|
|
938
1087
|
const nextClose = (from) => {
|
|
@@ -940,7 +1089,7 @@ function collectSsh2FenceSpans(text, span) {
|
|
|
940
1089
|
return cachedAt;
|
|
941
1090
|
let p = text.indexOf(SSH2_FENCE_CLOSE, from);
|
|
942
1091
|
ssh2ScanBytes += (p < 0 ? text.length : p) - from;
|
|
943
|
-
while (p >= 0 && inOpaqueUnit(p)) {
|
|
1092
|
+
while (p >= 0 && inOpaqueUnit(scan, p)) {
|
|
944
1093
|
const q = text.indexOf(SSH2_FENCE_CLOSE, p + 1);
|
|
945
1094
|
ssh2ScanBytes += (q < 0 ? text.length : q) - (p + 1);
|
|
946
1095
|
p = q;
|
|
@@ -953,7 +1102,7 @@ function collectSsh2FenceSpans(text, span) {
|
|
|
953
1102
|
const fence = text.indexOf(SSH2_FENCE_OPEN, from);
|
|
954
1103
|
if (fence < 0)
|
|
955
1104
|
return;
|
|
956
|
-
if (inOpaqueUnit(fence)) {
|
|
1105
|
+
if (inOpaqueUnit(scan, fence)) {
|
|
957
1106
|
from = fence + 1;
|
|
958
1107
|
continue;
|
|
959
1108
|
}
|
|
@@ -961,7 +1110,7 @@ function collectSsh2FenceSpans(text, span) {
|
|
|
961
1110
|
const close = nextClose(bodyFrom);
|
|
962
1111
|
if (close < 0)
|
|
963
1112
|
return;
|
|
964
|
-
if (!unitCountWithin(bodyFrom, close, SSH2_BODY_MAX)) {
|
|
1113
|
+
if (!unitCountWithin(scan, bodyFrom, close, SSH2_BODY_MAX)) {
|
|
965
1114
|
from = fence + 1;
|
|
966
1115
|
continue;
|
|
967
1116
|
}
|
|
@@ -1011,31 +1160,31 @@ function collectPemFenceSpans(text, span) {
|
|
|
1011
1160
|
from = scanResumeAfterHit(fence, endAfter, PEM_FENCE_OPEN.length);
|
|
1012
1161
|
}
|
|
1013
1162
|
}
|
|
1014
|
-
function collectPemFragmentSpans(text, json, span) {
|
|
1015
|
-
let cur = findPemFence(text, 0);
|
|
1163
|
+
function collectPemFragmentSpans(scan, text, json, span) {
|
|
1164
|
+
let cur = findPemFence(scan, text, 0);
|
|
1016
1165
|
let inherited = false;
|
|
1017
1166
|
while (cur !== undefined) {
|
|
1018
|
-
const next = findPemFence(text, cur.after);
|
|
1167
|
+
const next = findPemFence(scan, text, cur.after);
|
|
1019
1168
|
const limit = next === undefined ? text.length : next.fence;
|
|
1020
1169
|
let pos = cur.after;
|
|
1021
1170
|
let allowBare = inherited;
|
|
1022
1171
|
let hitLimit = false;
|
|
1023
1172
|
let rightmost = -1;
|
|
1024
|
-
while (parsePemHeaderLine(text, pos, limit, allowBare, json)) {
|
|
1025
|
-
const e = rightmostPemRunEnd(text,
|
|
1173
|
+
while (parsePemHeaderLine(scan, text, pos, limit, allowBare, json)) {
|
|
1174
|
+
const e = rightmostPemRunEnd(scan, text, scan.hdr.search, scan.hdr.contentEnd);
|
|
1026
1175
|
if (e > rightmost)
|
|
1027
1176
|
rightmost = e;
|
|
1028
|
-
pos =
|
|
1177
|
+
pos = scan.hdr.next;
|
|
1029
1178
|
allowBare = false;
|
|
1030
|
-
if (
|
|
1179
|
+
if (scan.hdr.hitLimit) {
|
|
1031
1180
|
hitLimit = true;
|
|
1032
1181
|
break;
|
|
1033
1182
|
}
|
|
1034
|
-
if (
|
|
1183
|
+
if (scan.hdr.jsonEnd)
|
|
1035
1184
|
break;
|
|
1036
1185
|
}
|
|
1037
1186
|
inherited = hitLimit;
|
|
1038
|
-
let bodyEnd = parsePemBody(text, pos, limit);
|
|
1187
|
+
let bodyEnd = parsePemBody(scan, text, pos, limit);
|
|
1039
1188
|
if (bodyEnd < 0)
|
|
1040
1189
|
bodyEnd = rightmost;
|
|
1041
1190
|
if (bodyEnd >= 0)
|
|
@@ -1043,18 +1192,18 @@ function collectPemFragmentSpans(text, json, span) {
|
|
|
1043
1192
|
cur = next;
|
|
1044
1193
|
}
|
|
1045
1194
|
}
|
|
1046
|
-
function collectHeadlessPpkSpans(text, span) {
|
|
1195
|
+
function collectHeadlessPpkSpans(scan, text, span) {
|
|
1047
1196
|
const KEY = "Private-Lines: ";
|
|
1048
1197
|
const n = text.length;
|
|
1049
1198
|
const isB64 = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 43 || c === 47 || c === 61;
|
|
1050
|
-
const b64At = (i) => opaqueUnitEnd(i) < 0 && isB64(text.charCodeAt(i));
|
|
1051
|
-
const skipHs = (i) => { while (i < n && opaqueUnitEnd(i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
|
|
1199
|
+
const b64At = (i) => opaqueUnitEnd(scan, i) < 0 && isB64(text.charCodeAt(i));
|
|
1200
|
+
const skipHs = (i) => { while (i < n && opaqueUnitEnd(scan, i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
|
|
1052
1201
|
i += 1; return i; };
|
|
1053
1202
|
const skipNewlines = (i) => {
|
|
1054
1203
|
let q = skipHs(i);
|
|
1055
1204
|
let seen = false;
|
|
1056
1205
|
for (;;) {
|
|
1057
|
-
if (opaqueUnitEnd(q) >= 0)
|
|
1206
|
+
if (opaqueUnitEnd(scan, q) >= 0)
|
|
1058
1207
|
break;
|
|
1059
1208
|
let any = false;
|
|
1060
1209
|
if (text.charCodeAt(q) === 13) {
|
|
@@ -1087,7 +1236,7 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1087
1236
|
return seen ? q : -1;
|
|
1088
1237
|
};
|
|
1089
1238
|
const escapedNewlineLen = (i) => {
|
|
1090
|
-
if (opaqueUnitEnd(i) >= 0)
|
|
1239
|
+
if (opaqueUnitEnd(scan, i) >= 0)
|
|
1091
1240
|
return 0;
|
|
1092
1241
|
let k = i;
|
|
1093
1242
|
while (k < n && text.charCodeAt(k) === 92)
|
|
@@ -1101,7 +1250,7 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1101
1250
|
const b = skipHs(i);
|
|
1102
1251
|
if (b >= n)
|
|
1103
1252
|
return true;
|
|
1104
|
-
if (opaqueUnitEnd(b) >= 0)
|
|
1253
|
+
if (opaqueUnitEnd(scan, b) >= 0)
|
|
1105
1254
|
return false;
|
|
1106
1255
|
const c = text.charCodeAt(b);
|
|
1107
1256
|
return c === 10 || c === 13 || c === 34 || escapedNewlineLen(b) > 0;
|
|
@@ -1111,13 +1260,13 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1111
1260
|
const at = text.indexOf(KEY, search);
|
|
1112
1261
|
if (at === -1)
|
|
1113
1262
|
break;
|
|
1114
|
-
if (inOpaqueUnit(at)) {
|
|
1263
|
+
if (inOpaqueUnit(scan, at)) {
|
|
1115
1264
|
search = at + KEY.length;
|
|
1116
1265
|
continue;
|
|
1117
1266
|
}
|
|
1118
1267
|
let p = at + KEY.length;
|
|
1119
1268
|
const d0 = p;
|
|
1120
|
-
while (p < n && opaqueUnitEnd(p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
|
|
1269
|
+
while (p < n && opaqueUnitEnd(scan, p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
|
|
1121
1270
|
p += 1;
|
|
1122
1271
|
if (p === d0) {
|
|
1123
1272
|
search = at + KEY.length;
|
|
@@ -1130,7 +1279,7 @@ function collectHeadlessPpkSpans(text, span) {
|
|
|
1130
1279
|
if (q0 === -1)
|
|
1131
1280
|
break;
|
|
1132
1281
|
let q = q0;
|
|
1133
|
-
const mk = opaqueUnitEnd(q) < 0 ? text.charCodeAt(q) : -1;
|
|
1282
|
+
const mk = opaqueUnitEnd(scan, q) < 0 ? text.charCodeAt(q) : -1;
|
|
1134
1283
|
if (mk === 43 || mk === 45)
|
|
1135
1284
|
q = skipHs(q + 1);
|
|
1136
1285
|
let r = q;
|
|
@@ -1169,7 +1318,7 @@ const FRAGMENT_SECRET_PATTERNS = [
|
|
|
1169
1318
|
const DISPLAY_SAFE_URL_TOKEN = "«redacted:url»";
|
|
1170
1319
|
export { DISPLAY_SAFE_URL_TOKEN };
|
|
1171
1320
|
const WITHHELD_VALUE_TOKEN = "«redacted»";
|
|
1172
|
-
const REDACTION_TOKENS = new Set([...FULL_ARM_PATTERNS.map(([, r]) => r), PEM_FRAGMENT_TOKEN, DISPLAY_SAFE_URL_TOKEN, ...FRAGMENT_SECRET_PATTERNS.map(([, r]) => r)]
|
|
1321
|
+
const REDACTION_TOKENS = new Set([...FULL_ARM_PATTERNS.map(([, r]) => r), PEM_FRAGMENT_TOKEN, ANCHORED_VALUE_TOKEN, DISPLAY_SAFE_URL_TOKEN, ...FRAGMENT_SECRET_PATTERNS.map(([, r]) => r)]
|
|
1173
1322
|
.flatMap((r) => r.match(/«[^»]*»/g) ?? []));
|
|
1174
1323
|
{
|
|
1175
1324
|
const closed = new RegExp(`^${REDACTION_TOKEN_RE_SRC}$`);
|
|
@@ -1285,8 +1434,8 @@ function tokenUnitEnd(text, p) {
|
|
|
1285
1434
|
return TOKEN_UNIT_Y.test(text) ? TOKEN_UNIT_Y.lastIndex : -1;
|
|
1286
1435
|
}
|
|
1287
1436
|
const NO_CREDENTIAL_SPANS = { starts: new Int32Array(0), ends: new Int32Array(0) };
|
|
1288
|
-
function credentialSpans(text
|
|
1289
|
-
const iv =
|
|
1437
|
+
function credentialSpans(text) {
|
|
1438
|
+
const iv = [];
|
|
1290
1439
|
for (let i = text.indexOf("«"); i >= 0; i = text.indexOf("«", i + 1)) {
|
|
1291
1440
|
const e = tokenUnitEnd(text, i);
|
|
1292
1441
|
if (e > 0) {
|
|
@@ -1318,13 +1467,103 @@ function credentialSpans(text, spans, withSpans) {
|
|
|
1318
1467
|
k += 1;
|
|
1319
1468
|
return { starts: starts.subarray(0, k), ends: ends.subarray(0, k) };
|
|
1320
1469
|
}
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1470
|
+
let forensicRescanMode = "incremental";
|
|
1471
|
+
export function setForensicRescanModeForTest(mode) {
|
|
1472
|
+
forensicRescanMode = mode;
|
|
1473
|
+
}
|
|
1474
|
+
function foldCredentialSpans(prev, spans, from) {
|
|
1475
|
+
const cap = spans.length - from;
|
|
1476
|
+
if (cap === 0)
|
|
1477
|
+
return { next: prev, dirty: NO_CREDENTIAL_SPANS };
|
|
1478
|
+
const Sa = new Int32Array(cap);
|
|
1479
|
+
const Ea = new Int32Array(cap);
|
|
1480
|
+
let m = 0;
|
|
1481
|
+
for (let i = from; i < spans.length; i++) {
|
|
1482
|
+
const sp = spans[i];
|
|
1483
|
+
const k = firstCredentialSpanFrom(prev, sp.start + 1) - 1;
|
|
1484
|
+
if (k >= 0 && prev.ends[k] >= sp.end)
|
|
1485
|
+
continue;
|
|
1486
|
+
Sa[m] = sp.start;
|
|
1487
|
+
Ea[m] = sp.end;
|
|
1488
|
+
m += 1;
|
|
1489
|
+
}
|
|
1490
|
+
if (m === 0)
|
|
1491
|
+
return { next: prev, dirty: NO_CREDENTIAL_SPANS };
|
|
1492
|
+
const S = Sa.subarray(0, m).sort();
|
|
1493
|
+
const E = Ea.subarray(0, m).sort();
|
|
1494
|
+
const cs = new Int32Array(m);
|
|
1495
|
+
const ce = new Int32Array(m);
|
|
1496
|
+
let C = 0;
|
|
1497
|
+
for (let i = 0, j = 0, depth = 0, open = 0; i < m || j < m;) {
|
|
1498
|
+
if (i < m && (j >= m || S[i] <= E[j])) {
|
|
1499
|
+
if (depth === 0)
|
|
1500
|
+
open = S[i];
|
|
1501
|
+
depth += 1;
|
|
1502
|
+
i += 1;
|
|
1503
|
+
}
|
|
1504
|
+
else {
|
|
1505
|
+
depth -= 1;
|
|
1506
|
+
if (depth === 0) {
|
|
1507
|
+
cs[C] = open;
|
|
1508
|
+
ce[C] = E[j];
|
|
1509
|
+
C += 1;
|
|
1510
|
+
}
|
|
1511
|
+
j += 1;
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
const P = prev.starts.length;
|
|
1515
|
+
const oS = new Int32Array(P + C);
|
|
1516
|
+
const oE = new Int32Array(P + C);
|
|
1517
|
+
const dS = new Int32Array(C + P);
|
|
1518
|
+
const dE = new Int32Array(C + P);
|
|
1519
|
+
let on = 0;
|
|
1520
|
+
let dn = 0;
|
|
1521
|
+
let curS = 0;
|
|
1522
|
+
let curE = 0;
|
|
1523
|
+
let sources = 0;
|
|
1524
|
+
let onlyPrev = false;
|
|
1525
|
+
const flush = () => {
|
|
1526
|
+
if (sources === 0)
|
|
1527
|
+
return;
|
|
1528
|
+
oS[on] = curS;
|
|
1529
|
+
oE[on] = curE;
|
|
1530
|
+
on += 1;
|
|
1531
|
+
if (!(sources === 1 && onlyPrev)) {
|
|
1532
|
+
dS[dn] = curS;
|
|
1533
|
+
dE[dn] = curE;
|
|
1534
|
+
dn += 1;
|
|
1535
|
+
}
|
|
1536
|
+
};
|
|
1537
|
+
for (let pi = 0, ci = 0; pi < P || ci < C;) {
|
|
1538
|
+
const takePrev = ci >= C || (pi < P && prev.starts[pi] <= cs[ci]);
|
|
1539
|
+
const s = takePrev ? prev.starts[pi] : cs[ci];
|
|
1540
|
+
const e = takePrev ? prev.ends[pi] : ce[ci];
|
|
1541
|
+
if (takePrev)
|
|
1542
|
+
pi += 1;
|
|
1543
|
+
else
|
|
1544
|
+
ci += 1;
|
|
1545
|
+
if (sources > 0 && s <= curE) {
|
|
1546
|
+
if (e > curE)
|
|
1547
|
+
curE = e;
|
|
1548
|
+
sources += 1;
|
|
1549
|
+
}
|
|
1550
|
+
else {
|
|
1551
|
+
flush();
|
|
1552
|
+
curS = s;
|
|
1553
|
+
curE = e;
|
|
1554
|
+
sources = 1;
|
|
1555
|
+
onlyPrev = takePrev;
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
flush();
|
|
1559
|
+
return { next: { starts: oS.subarray(0, on), ends: oE.subarray(0, on) }, dirty: { starts: dS.subarray(0, dn), ends: dE.subarray(0, dn) } };
|
|
1560
|
+
}
|
|
1561
|
+
export function credentialSpansFoldForTest(prev, add) {
|
|
1562
|
+
const table = (iv) => ({ starts: Int32Array.from(iv, (x) => x[0]), ends: Int32Array.from(iv, (x) => x[1]) });
|
|
1563
|
+
const back = (t) => Array.from(t.starts, (s, i) => [s, t.ends[i]]);
|
|
1564
|
+
const spans = add.map(([start, end]) => ({ start, end, repl: "", rank: 0, order: 0 }));
|
|
1565
|
+
const { next, dirty } = foldCredentialSpans(table(prev), spans, 0);
|
|
1566
|
+
return { next: back(next), dirty: back(dirty) };
|
|
1328
1567
|
}
|
|
1329
1568
|
function firstCredentialSpanFrom(cs, from) {
|
|
1330
1569
|
let lo = 0;
|
|
@@ -1338,43 +1577,90 @@ function firstCredentialSpanFrom(cs, from) {
|
|
|
1338
1577
|
}
|
|
1339
1578
|
return lo;
|
|
1340
1579
|
}
|
|
1341
|
-
function credentialSpansCover(
|
|
1580
|
+
function credentialSpansCover(scan, from, to) {
|
|
1581
|
+
noteEvidenceRead(scan, from);
|
|
1582
|
+
noteEvidenceRead(scan, to);
|
|
1583
|
+
const cs = scan.table;
|
|
1342
1584
|
let k = firstCredentialSpanFrom(cs, from + 1) - 1;
|
|
1343
1585
|
if (k < 0)
|
|
1344
1586
|
k = 0;
|
|
1345
1587
|
return k < cs.starts.length && cs.starts[k] <= from && cs.ends[k] >= to;
|
|
1346
1588
|
}
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1589
|
+
const FOOT_NONE = 0x7fffffff;
|
|
1590
|
+
function buildForensicScan(table, armCount) {
|
|
1591
|
+
return {
|
|
1592
|
+
table,
|
|
1593
|
+
cursor: 0,
|
|
1594
|
+
dirty: undefined,
|
|
1595
|
+
footLo: FOOT_NONE,
|
|
1596
|
+
footHi: -1,
|
|
1597
|
+
armFoot: new Int32Array(2 * armCount),
|
|
1598
|
+
lastAt: -1,
|
|
1599
|
+
headsRead: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1600
|
+
headsWrite: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1601
|
+
paramsRead: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1602
|
+
paramsWrite: { a: NO_URL_SEGMENTS, n: 0 },
|
|
1603
|
+
hdr: { search: 0, contentEnd: 0, next: 0, hitLimit: false, jsonEnd: false },
|
|
1604
|
+
};
|
|
1605
|
+
}
|
|
1606
|
+
function advanceEvidence(scan, table, dirty) {
|
|
1607
|
+
scan.table = table;
|
|
1608
|
+
scan.cursor = 0;
|
|
1609
|
+
scan.dirty = dirty;
|
|
1610
|
+
}
|
|
1611
|
+
function noteEvidenceRead(scan, p) {
|
|
1612
|
+
if (p < scan.footLo)
|
|
1613
|
+
scan.footLo = p;
|
|
1614
|
+
if (p > scan.footHi)
|
|
1615
|
+
scan.footHi = p;
|
|
1352
1616
|
}
|
|
1353
|
-
function
|
|
1354
|
-
|
|
1617
|
+
function footprintClean(dirty, lo, hi) {
|
|
1618
|
+
if (lo > hi)
|
|
1619
|
+
return true;
|
|
1620
|
+
const ends = dirty.ends;
|
|
1621
|
+
let a = 0;
|
|
1622
|
+
let b = ends.length;
|
|
1623
|
+
while (a < b) {
|
|
1624
|
+
const mid = (a + b) >>> 1;
|
|
1625
|
+
if (ends[mid] < lo)
|
|
1626
|
+
a = mid + 1;
|
|
1627
|
+
else
|
|
1628
|
+
b = mid;
|
|
1629
|
+
}
|
|
1630
|
+
return a >= ends.length || dirty.starts[a] > hi;
|
|
1631
|
+
}
|
|
1632
|
+
function inOpaqueUnit(scan, p) {
|
|
1633
|
+
noteEvidenceRead(scan, p);
|
|
1634
|
+
const t = scan.table;
|
|
1635
|
+
const st = t.starts;
|
|
1355
1636
|
if (st.length === 0)
|
|
1356
1637
|
return false;
|
|
1357
|
-
const k = firstCredentialSpanFrom(
|
|
1358
|
-
return k >= 0 && st[k] < p &&
|
|
1638
|
+
const k = firstCredentialSpanFrom(t, p) - 1;
|
|
1639
|
+
return k >= 0 && st[k] < p && t.ends[k] > p;
|
|
1640
|
+
}
|
|
1641
|
+
function opaqueUnitEnd(scan, p) {
|
|
1642
|
+
noteEvidenceRead(scan, p);
|
|
1643
|
+
return opaqueUnitEndAt(scan, p);
|
|
1359
1644
|
}
|
|
1360
|
-
function
|
|
1361
|
-
const
|
|
1645
|
+
function opaqueUnitEndAt(scan, p) {
|
|
1646
|
+
const t = scan.table;
|
|
1647
|
+
const st = t.starts;
|
|
1362
1648
|
if (st.length === 0)
|
|
1363
1649
|
return -1;
|
|
1364
|
-
let k =
|
|
1650
|
+
let k = scan.cursor;
|
|
1365
1651
|
while (k < st.length && st[k] < p)
|
|
1366
1652
|
k += 1;
|
|
1367
1653
|
while (k > 0 && st[k - 1] >= p)
|
|
1368
1654
|
k -= 1;
|
|
1369
|
-
|
|
1370
|
-
return k < st.length && st[k] === p ?
|
|
1655
|
+
scan.cursor = k;
|
|
1656
|
+
return k < st.length && st[k] === p ? t.ends[k] : -1;
|
|
1371
1657
|
}
|
|
1372
|
-
|
|
1373
|
-
function scanUrlRun(text, i, kind, json) {
|
|
1658
|
+
function scanUrlRun(scan, text, i, kind, json) {
|
|
1374
1659
|
const n = text.length;
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1660
|
+
noteEvidenceRead(scan, i);
|
|
1661
|
+
scan.lastAt = -1;
|
|
1662
|
+
run: while (i < n) {
|
|
1663
|
+
const unit = opaqueUnitEndAt(scan, i);
|
|
1378
1664
|
if (unit >= 0) {
|
|
1379
1665
|
i = unit;
|
|
1380
1666
|
continue;
|
|
@@ -1385,56 +1671,117 @@ function scanUrlRun(text, i, kind, json) {
|
|
|
1385
1671
|
i += 2;
|
|
1386
1672
|
continue;
|
|
1387
1673
|
}
|
|
1388
|
-
|
|
1674
|
+
break;
|
|
1389
1675
|
}
|
|
1390
1676
|
if (isRegexSpace(c) || (json && c === 34))
|
|
1391
|
-
|
|
1677
|
+
break;
|
|
1392
1678
|
switch (kind) {
|
|
1393
1679
|
case "plain-user":
|
|
1394
1680
|
if (c === CH_AT || c === 47 || c === 58)
|
|
1395
|
-
|
|
1681
|
+
break run;
|
|
1396
1682
|
break;
|
|
1397
1683
|
case "plain-pass":
|
|
1398
1684
|
if (c === 47)
|
|
1399
|
-
|
|
1685
|
+
break run;
|
|
1400
1686
|
break;
|
|
1401
1687
|
case "token-pass":
|
|
1402
1688
|
if (c === 58 && text.charCodeAt(i + 1) === 47 && text.charCodeAt(i + 2) === 47)
|
|
1403
|
-
|
|
1689
|
+
break run;
|
|
1404
1690
|
break;
|
|
1405
1691
|
case "param-value":
|
|
1406
1692
|
if (c === 38 || c === 35 || isControlCode(c))
|
|
1407
|
-
|
|
1693
|
+
break run;
|
|
1408
1694
|
break;
|
|
1409
1695
|
}
|
|
1410
1696
|
if (c === CH_AT)
|
|
1411
|
-
|
|
1697
|
+
scan.lastAt = i;
|
|
1412
1698
|
i += 1;
|
|
1413
1699
|
}
|
|
1700
|
+
noteEvidenceRead(scan, i);
|
|
1414
1701
|
return i;
|
|
1415
1702
|
}
|
|
1416
|
-
function matchUserinfoAt(text, p, json) {
|
|
1417
|
-
const probeEnd = scanUrlRun(text, p, "plain-user", json);
|
|
1418
|
-
const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && credentialSpansCover(
|
|
1419
|
-
scanUrlRun(text, p, keyId ? "token-pass" : "plain-pass", json);
|
|
1420
|
-
|
|
1703
|
+
function matchUserinfoAt(scan, text, p, json) {
|
|
1704
|
+
const probeEnd = scanUrlRun(scan, text, p, "plain-user", json);
|
|
1705
|
+
const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && credentialSpansCover(scan, p, probeEnd);
|
|
1706
|
+
scanUrlRun(scan, text, p, keyId ? "token-pass" : "plain-pass", json);
|
|
1707
|
+
const at = scan.lastAt;
|
|
1708
|
+
if (at <= p)
|
|
1421
1709
|
return null;
|
|
1422
|
-
return
|
|
1710
|
+
return at;
|
|
1711
|
+
}
|
|
1712
|
+
const URL_SEG_STRIDE = 4;
|
|
1713
|
+
const URL_SEG_MAX_INTS = 3 * 1_048_576;
|
|
1714
|
+
const NO_URL_SEGMENTS = new Int32Array(0);
|
|
1715
|
+
function growUrlSegments(list) {
|
|
1716
|
+
const len = list.a.length;
|
|
1717
|
+
if (len >= URL_SEG_MAX_INTS)
|
|
1718
|
+
return false;
|
|
1719
|
+
const bigger = new Int32Array(Math.min(Math.max(len * 2, 4 * URL_SEG_STRIDE), URL_SEG_MAX_INTS));
|
|
1720
|
+
bigger.set(list.a.subarray(0, list.n));
|
|
1721
|
+
list.a = bigger;
|
|
1722
|
+
return true;
|
|
1423
1723
|
}
|
|
1424
|
-
function collectUrlCredentialSpans(text, json, span) {
|
|
1724
|
+
function collectUrlCredentialSpans(scan, text, json, span) {
|
|
1725
|
+
const dirty = scan.dirty;
|
|
1726
|
+
const hr = scan.headsRead;
|
|
1727
|
+
const hw = scan.headsWrite;
|
|
1728
|
+
const pr = scan.paramsRead;
|
|
1729
|
+
const pw = scan.paramsWrite;
|
|
1730
|
+
hw.n = 0;
|
|
1731
|
+
pw.n = 0;
|
|
1732
|
+
let armLo = FOOT_NONE;
|
|
1733
|
+
let armHi = -1;
|
|
1425
1734
|
const head = URL_CREDENTIAL_PATTERN[0];
|
|
1426
|
-
|
|
1427
|
-
|
|
1735
|
+
const ph = hr.a;
|
|
1736
|
+
const phn = dirty === undefined ? 0 : hr.n;
|
|
1737
|
+
let pc = 0;
|
|
1738
|
+
for (let from = 0;;) {
|
|
1739
|
+
head.lastIndex = from;
|
|
1740
|
+
const m = head.exec(text);
|
|
1741
|
+
if (m === null)
|
|
1742
|
+
break;
|
|
1428
1743
|
const colon = m.index;
|
|
1429
1744
|
const start = colon + 3;
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1745
|
+
while (pc < phn && ph[pc] < colon)
|
|
1746
|
+
pc += URL_SEG_STRIDE;
|
|
1747
|
+
const known = pc < phn && ph[pc] === colon;
|
|
1748
|
+
let at;
|
|
1749
|
+
let lo;
|
|
1750
|
+
let hi;
|
|
1751
|
+
if (known && footprintClean(dirty, ph[pc + 2], ph[pc + 3])) {
|
|
1752
|
+
at = ph[pc + 1];
|
|
1753
|
+
lo = ph[pc + 2];
|
|
1754
|
+
hi = ph[pc + 3];
|
|
1755
|
+
urlSegmentReplays += 1;
|
|
1434
1756
|
}
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1757
|
+
else {
|
|
1758
|
+
scan.footLo = FOOT_NONE;
|
|
1759
|
+
scan.footHi = -1;
|
|
1760
|
+
at = matchUserinfoAt(scan, text, start, json) ?? -1;
|
|
1761
|
+
lo = scan.footLo;
|
|
1762
|
+
hi = scan.footHi;
|
|
1763
|
+
urlSegmentRuns += 1;
|
|
1764
|
+
if (at >= 0 && at !== (known ? ph[pc + 1] : -2))
|
|
1765
|
+
span(start, at);
|
|
1766
|
+
}
|
|
1767
|
+
if (hw.n + URL_SEG_STRIDE <= hw.a.length || growUrlSegments(hw)) {
|
|
1768
|
+
const w = hw.a;
|
|
1769
|
+
const k = hw.n;
|
|
1770
|
+
w[k] = colon;
|
|
1771
|
+
w[k + 1] = at;
|
|
1772
|
+
w[k + 2] = lo;
|
|
1773
|
+
w[k + 3] = hi;
|
|
1774
|
+
hw.n = k + URL_SEG_STRIDE;
|
|
1775
|
+
}
|
|
1776
|
+
if (lo < armLo)
|
|
1777
|
+
armLo = lo;
|
|
1778
|
+
if (hi > armHi)
|
|
1779
|
+
armHi = hi;
|
|
1780
|
+
from = at < 0 ? colon + 1 : at + 1;
|
|
1781
|
+
}
|
|
1782
|
+
const pp = pr.a;
|
|
1783
|
+
const ppn = dirty === undefined ? 0 : pr.n;
|
|
1784
|
+
let qc = 0;
|
|
1438
1785
|
for (let i = 0; i < text.length; i++) {
|
|
1439
1786
|
const c = text.charCodeAt(i);
|
|
1440
1787
|
if (c !== 63 && c !== 38 && c !== 35)
|
|
@@ -1442,11 +1789,49 @@ function collectUrlCredentialSpans(text, json, span) {
|
|
|
1442
1789
|
const valueStart = credentialParamValueStart(text, i + 1);
|
|
1443
1790
|
if (valueStart < 0)
|
|
1444
1791
|
continue;
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1792
|
+
while (qc < ppn && pp[qc] < i)
|
|
1793
|
+
qc += URL_SEG_STRIDE;
|
|
1794
|
+
const known = qc < ppn && pp[qc] === i;
|
|
1795
|
+
let valueEnd;
|
|
1796
|
+
let lo;
|
|
1797
|
+
let hi;
|
|
1798
|
+
if (known && footprintClean(dirty, pp[qc + 2], pp[qc + 3])) {
|
|
1799
|
+
valueEnd = pp[qc + 1];
|
|
1800
|
+
lo = pp[qc + 2];
|
|
1801
|
+
hi = pp[qc + 3];
|
|
1802
|
+
urlSegmentReplays += 1;
|
|
1803
|
+
}
|
|
1804
|
+
else {
|
|
1805
|
+
scan.footLo = FOOT_NONE;
|
|
1806
|
+
scan.footHi = -1;
|
|
1807
|
+
valueEnd = scanUrlRun(scan, text, valueStart, "param-value", json);
|
|
1808
|
+
lo = scan.footLo;
|
|
1809
|
+
hi = scan.footHi;
|
|
1810
|
+
urlSegmentRuns += 1;
|
|
1811
|
+
if (valueEnd > valueStart && valueEnd !== (known ? pp[qc + 1] : -2))
|
|
1812
|
+
span(valueStart, valueEnd);
|
|
1813
|
+
}
|
|
1814
|
+
if (pw.n + URL_SEG_STRIDE <= pw.a.length || growUrlSegments(pw)) {
|
|
1815
|
+
const w = pw.a;
|
|
1816
|
+
const k = pw.n;
|
|
1817
|
+
w[k] = i;
|
|
1818
|
+
w[k + 1] = valueEnd;
|
|
1819
|
+
w[k + 2] = lo;
|
|
1820
|
+
w[k + 3] = hi;
|
|
1821
|
+
pw.n = k + URL_SEG_STRIDE;
|
|
1822
|
+
}
|
|
1823
|
+
if (lo < armLo)
|
|
1824
|
+
armLo = lo;
|
|
1825
|
+
if (hi > armHi)
|
|
1826
|
+
armHi = hi;
|
|
1448
1827
|
i = valueEnd - 1;
|
|
1449
1828
|
}
|
|
1829
|
+
scan.headsRead = hw;
|
|
1830
|
+
scan.headsWrite = hr;
|
|
1831
|
+
scan.paramsRead = pw;
|
|
1832
|
+
scan.paramsWrite = pr;
|
|
1833
|
+
scan.footLo = armLo;
|
|
1834
|
+
scan.footHi = armHi;
|
|
1450
1835
|
}
|
|
1451
1836
|
const JSON_FIELD_ANCHOR = '"';
|
|
1452
1837
|
const CREDENTIAL_NAME_MAX = Math.max(...CREDENTIAL_NAME_WORDS.map((w) => w.length));
|
|
@@ -1507,6 +1892,354 @@ function collectJsonSecretFieldSpans(text, span) {
|
|
|
1507
1892
|
p = text.indexOf(JSON_FIELD_ANCHOR, scanResumeAfterHit(p, end, JSON_FIELD_ANCHOR.length));
|
|
1508
1893
|
}
|
|
1509
1894
|
}
|
|
1895
|
+
const STRONG_AUTH_SCHEMES = new Set(["basic", "bearer"]);
|
|
1896
|
+
const STRONG_AUTH_SCHEME_MAX = Math.max(...[...STRONG_AUTH_SCHEMES].map((w) => w.length));
|
|
1897
|
+
const ABSENT_OR_BOOLEAN_WORDS = new Set(["null", "nil", "none", "undefined", "true", "false", "yes", "no"]);
|
|
1898
|
+
const ABSENT_OR_BOOLEAN_MAX = Math.max(...[...ABSENT_OR_BOOLEAN_WORDS].map((w) => w.length));
|
|
1899
|
+
const CREDENTIAL_ANCHOR_OF = new Map(Object.entries(CREDENTIAL_NAME_ANCHORS));
|
|
1900
|
+
const CH_GRAVE = 96, CH_APOS = 39, CH_DOT = 46, CH_COMMA = 44, CH_SEMI = 59, CH_DOLLAR = 36;
|
|
1901
|
+
const CH_LPAREN = 40, CH_RPAREN = 41, CH_RBRACKET = 93, CH_LBRACE = 123, CH_RBRACE = 125, CH_LT = 60, CH_GT = 62;
|
|
1902
|
+
const CH_QMARK = 63, CH_AMP = 38, CH_HASH = 35;
|
|
1903
|
+
const JSON_STRING_END = -1;
|
|
1904
|
+
const isAsciiAlnumCode = (c) => (c >= CH_0 && c <= CH_9) || (c >= CH_A && c <= CH_Z) || (c >= CH_a && c <= CH_z);
|
|
1905
|
+
const isProgramIdentCode = (c) => isAsciiAlnumCode(c) || c === CH_UNDERSCORE || c === CH_DOLLAR;
|
|
1906
|
+
const isProgramIdentStartCode = (c) => isIdentStartCode(c) || c === CH_UNDERSCORE || c === CH_DOLLAR;
|
|
1907
|
+
const isQuoteUnit = (ch) => ch === CH_QUOTE || ch === CH_APOS || ch === CH_GRAVE;
|
|
1908
|
+
const isValueCloser = (ch) => ch === CH_DOT || ch === CH_COMMA || ch === CH_SEMI || ch === CH_COLON || ch === CH_RPAREN || ch === CH_RBRACKET || ch === CH_RBRACE || ch === CH_GT;
|
|
1909
|
+
const isInlineSpaceUnit = (ch) => ch === CH_SPACE || ch === CH_TAB;
|
|
1910
|
+
const isToken68Code = (c) => isAsciiAlnumCode(c) || c === CH_DOT || c === CH_MINUS || c === CH_UNDERSCORE || c === CH_TILDE || c === CH_PLUS || c === CH_SLASH;
|
|
1911
|
+
const endsValueUnit = (ch) => ch === JSON_STRING_END || isRegexSpace(ch) || isQuoteUnit(ch) || isValueCloser(ch);
|
|
1912
|
+
const endsHeaderValueUnit = (ch) => ch === JSON_STRING_END || isLineTerminator(ch) || isSchemeDelimiterCode(ch) || ch === CH_COMMA || ch === CH_SEMI || ch === CH_RPAREN || ch === CH_RBRACKET || ch === CH_RBRACE || ch === CH_GT;
|
|
1913
|
+
const bracketOf = (c) => c === CH_LPAREN ? 1 : c === CH_RPAREN ? -1 : c === CH_LBRACKET ? 2 : c === CH_RBRACKET ? -2 : c === CH_LBRACE ? 3 : c === CH_RBRACE ? -3 : c === CH_LT ? 4 : c === CH_GT ? -4 : 0;
|
|
1914
|
+
let anchoredScanUnits = 0;
|
|
1915
|
+
export function anchoredScanUnitsForTest() {
|
|
1916
|
+
return anchoredScanUnits;
|
|
1917
|
+
}
|
|
1918
|
+
export function resetAnchoredScanUnitsForTest() {
|
|
1919
|
+
anchoredScanUnits = 0;
|
|
1920
|
+
}
|
|
1921
|
+
const packUnit = (ch, len) => ((ch + 1) << 3) | len;
|
|
1922
|
+
const unitChOf = (unit) => (unit >> 3) - 1;
|
|
1923
|
+
const unitLenOf = (unit) => unit & 7;
|
|
1924
|
+
function readAnchorUnit(text, i, json) {
|
|
1925
|
+
anchoredScanUnits += 1;
|
|
1926
|
+
const c = text.charCodeAt(i);
|
|
1927
|
+
if (!json)
|
|
1928
|
+
return packUnit(c, 1);
|
|
1929
|
+
if (c === CH_QUOTE)
|
|
1930
|
+
return packUnit(JSON_STRING_END, 1);
|
|
1931
|
+
if (c !== CH_BACKSLASH || i + 1 >= text.length)
|
|
1932
|
+
return packUnit(c, 1);
|
|
1933
|
+
switch (text.charCodeAt(i + 1)) {
|
|
1934
|
+
case CH_n: return packUnit(CH_LF, 2);
|
|
1935
|
+
case CH_r: return packUnit(CH_CR, 2);
|
|
1936
|
+
case CH_t: return packUnit(CH_TAB, 2);
|
|
1937
|
+
case CH_f: return packUnit(12, 2);
|
|
1938
|
+
case CH_b: return packUnit(8, 2);
|
|
1939
|
+
case CH_QUOTE: return packUnit(CH_QUOTE, 2);
|
|
1940
|
+
case CH_BACKSLASH: return packUnit(CH_BACKSLASH, 2);
|
|
1941
|
+
case CH_SLASH: return packUnit(CH_SLASH, 2);
|
|
1942
|
+
case CH_u: {
|
|
1943
|
+
const h0 = hexDigitValue(text.charCodeAt(i + 2));
|
|
1944
|
+
const h1 = hexDigitValue(text.charCodeAt(i + 3));
|
|
1945
|
+
const h2 = hexDigitValue(text.charCodeAt(i + 4));
|
|
1946
|
+
const h3 = hexDigitValue(text.charCodeAt(i + 5));
|
|
1947
|
+
return h0 >= 0 && h1 >= 0 && h2 >= 0 && h3 >= 0 ? packUnit((h0 << 12) | (h1 << 8) | (h2 << 4) | h3, 6) : packUnit(c, 1);
|
|
1948
|
+
}
|
|
1949
|
+
default:
|
|
1950
|
+
return packUnit(c, 1);
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
function escapedAt(text, i) {
|
|
1954
|
+
let k = i - 1;
|
|
1955
|
+
while (k >= 0 && text.charCodeAt(k) === CH_BACKSLASH)
|
|
1956
|
+
k -= 1;
|
|
1957
|
+
anchoredScanUnits += i - 1 - k;
|
|
1958
|
+
return ((i - 1 - k) & 1) === 1;
|
|
1959
|
+
}
|
|
1960
|
+
function anchorHeadAt(text, op, json) {
|
|
1961
|
+
let i = op - 1;
|
|
1962
|
+
let spacedLeft = false;
|
|
1963
|
+
for (;;) {
|
|
1964
|
+
if (i < 0)
|
|
1965
|
+
return undefined;
|
|
1966
|
+
const c = text.charCodeAt(i);
|
|
1967
|
+
if (c === CH_SPACE || c === CH_TAB) {
|
|
1968
|
+
i -= 1;
|
|
1969
|
+
spacedLeft = true;
|
|
1970
|
+
anchoredScanUnits += 1;
|
|
1971
|
+
continue;
|
|
1972
|
+
}
|
|
1973
|
+
if (json && c === CH_t && i >= 1 && text.charCodeAt(i - 1) === CH_BACKSLASH && escapedAt(text, i)) {
|
|
1974
|
+
i -= 2;
|
|
1975
|
+
spacedLeft = true;
|
|
1976
|
+
continue;
|
|
1977
|
+
}
|
|
1978
|
+
break;
|
|
1979
|
+
}
|
|
1980
|
+
let quote = 0;
|
|
1981
|
+
let quoteLen = 0;
|
|
1982
|
+
const q = text.charCodeAt(i);
|
|
1983
|
+
if (q === CH_QUOTE || q === CH_APOS || q === CH_GRAVE) {
|
|
1984
|
+
quote = q;
|
|
1985
|
+
quoteLen = json && q === CH_QUOTE && escapedAt(text, i) ? 2 : 1;
|
|
1986
|
+
i -= quoteLen;
|
|
1987
|
+
}
|
|
1988
|
+
const nameEnd = i + 1;
|
|
1989
|
+
const floor = Math.max(-1, i - (CREDENTIAL_NAME_MAX + 2));
|
|
1990
|
+
let j = i;
|
|
1991
|
+
while (j > floor && isIdentCode(text.charCodeAt(j)))
|
|
1992
|
+
j -= 1;
|
|
1993
|
+
anchoredScanUnits += nameEnd - (j + 1);
|
|
1994
|
+
if (j === floor && j >= 0 && isIdentCode(text.charCodeAt(j)))
|
|
1995
|
+
return undefined;
|
|
1996
|
+
let nameStart = j + 1;
|
|
1997
|
+
let afterEscape = false;
|
|
1998
|
+
if (j >= 0 && text.charCodeAt(j) === CH_BACKSLASH && escapedAt(text, j + 1)) {
|
|
1999
|
+
const e = text.charCodeAt(nameStart);
|
|
2000
|
+
if (e === CH_b || e === CH_f || e === CH_n || e === CH_r || e === CH_t) {
|
|
2001
|
+
nameStart += 1;
|
|
2002
|
+
afterEscape = true;
|
|
2003
|
+
}
|
|
2004
|
+
else if (e === CH_u && nameStart + 5 <= nameEnd && hexDigitValue(text.charCodeAt(nameStart + 1)) >= 0 && hexDigitValue(text.charCodeAt(nameStart + 2)) >= 0
|
|
2005
|
+
&& hexDigitValue(text.charCodeAt(nameStart + 3)) >= 0 && hexDigitValue(text.charCodeAt(nameStart + 4)) >= 0) {
|
|
2006
|
+
nameStart += 5;
|
|
2007
|
+
afterEscape = true;
|
|
2008
|
+
}
|
|
2009
|
+
}
|
|
2010
|
+
const runStart = nameStart;
|
|
2011
|
+
while (nameStart < nameEnd && text.charCodeAt(nameStart) === CH_MINUS)
|
|
2012
|
+
nameStart += 1;
|
|
2013
|
+
if (nameStart >= nameEnd)
|
|
2014
|
+
return undefined;
|
|
2015
|
+
let before = j;
|
|
2016
|
+
if (quote !== 0) {
|
|
2017
|
+
if (afterEscape || nameStart !== j + 1 || j < 0 || text.charCodeAt(j) !== quote)
|
|
2018
|
+
return undefined;
|
|
2019
|
+
const openLen = json && quote === CH_QUOTE && escapedAt(text, j) ? 2 : 1;
|
|
2020
|
+
if (openLen !== quoteLen)
|
|
2021
|
+
return undefined;
|
|
2022
|
+
before = j - openLen;
|
|
2023
|
+
}
|
|
2024
|
+
const strength = CREDENTIAL_ANCHOR_OF.get(text.slice(nameStart, nameEnd).toLowerCase());
|
|
2025
|
+
if (strength === undefined)
|
|
2026
|
+
return undefined;
|
|
2027
|
+
if (!afterEscape && runStart === j + 1 && quote === 0 && j >= 0 && text.charCodeAt(j) === CH_SLASH)
|
|
2028
|
+
return undefined;
|
|
2029
|
+
let listSep = 0;
|
|
2030
|
+
if (!afterEscape) {
|
|
2031
|
+
let k = before;
|
|
2032
|
+
while (k >= 0 && (text.charCodeAt(k) === CH_SPACE || text.charCodeAt(k) === CH_TAB))
|
|
2033
|
+
k -= 1;
|
|
2034
|
+
anchoredScanUnits += before - k;
|
|
2035
|
+
const left = k >= 0 ? text.charCodeAt(k) : 0;
|
|
2036
|
+
listSep = left === CH_QMARK || left === CH_AMP || left === CH_HASH ? CH_AMP : left === CH_SEMI ? CH_SEMI : left === CH_COMMA || left === CH_LPAREN || left === CH_LBRACKET || left === CH_LBRACE ? CH_COMMA : 0;
|
|
2037
|
+
}
|
|
2038
|
+
return { strength, quotedName: quote !== 0, spacedLeft, listSep };
|
|
2039
|
+
}
|
|
2040
|
+
function quotedValueEnd(text, cs, quote, json) {
|
|
2041
|
+
const n = text.length;
|
|
2042
|
+
let i = cs;
|
|
2043
|
+
while (i < n) {
|
|
2044
|
+
const unit = readAnchorUnit(text, i, json);
|
|
2045
|
+
const ch = unitChOf(unit);
|
|
2046
|
+
if (ch === quote)
|
|
2047
|
+
return i;
|
|
2048
|
+
if (ch === JSON_STRING_END)
|
|
2049
|
+
break;
|
|
2050
|
+
i += unitLenOf(unit);
|
|
2051
|
+
if (ch === CH_BACKSLASH && quote !== JSON_STRING_END) {
|
|
2052
|
+
if (i >= n)
|
|
2053
|
+
break;
|
|
2054
|
+
const escaped = readAnchorUnit(text, i, json);
|
|
2055
|
+
if (unitChOf(escaped) === JSON_STRING_END)
|
|
2056
|
+
break;
|
|
2057
|
+
i += unitLenOf(escaped);
|
|
2058
|
+
}
|
|
2059
|
+
}
|
|
2060
|
+
return -1;
|
|
2061
|
+
}
|
|
2062
|
+
function bareValueEnd(text, s, json, listSep) {
|
|
2063
|
+
const n = text.length;
|
|
2064
|
+
let i = s;
|
|
2065
|
+
while (i < n) {
|
|
2066
|
+
const unit = readAnchorUnit(text, i, json);
|
|
2067
|
+
const ch = unitChOf(unit);
|
|
2068
|
+
if (ch === JSON_STRING_END || isRegexSpace(ch))
|
|
2069
|
+
break;
|
|
2070
|
+
if (listSep !== 0 && (ch === listSep || (listSep === CH_AMP && ch === CH_HASH)))
|
|
2071
|
+
break;
|
|
2072
|
+
const next = i + unitLenOf(unit);
|
|
2073
|
+
if (isQuoteUnit(ch)) {
|
|
2074
|
+
if (next >= n)
|
|
2075
|
+
break;
|
|
2076
|
+
if (endsValueUnit(unitChOf(readAnchorUnit(text, next, json))))
|
|
2077
|
+
break;
|
|
2078
|
+
}
|
|
2079
|
+
i = next;
|
|
2080
|
+
}
|
|
2081
|
+
anchoredScanUnits += i - s;
|
|
2082
|
+
const excess = [0, 0, 0, 0];
|
|
2083
|
+
for (let k = s; k < i; k += 1) {
|
|
2084
|
+
const b = bracketOf(text.charCodeAt(k));
|
|
2085
|
+
if (b !== 0)
|
|
2086
|
+
excess[Math.abs(b) - 1] = excess[Math.abs(b) - 1] + (b < 0 ? 1 : -1);
|
|
2087
|
+
}
|
|
2088
|
+
let e = i;
|
|
2089
|
+
while (e > s) {
|
|
2090
|
+
const c = text.charCodeAt(e - 1);
|
|
2091
|
+
if (c === CH_DOT || c === CH_COMMA || c === CH_SEMI || c === CH_COLON) {
|
|
2092
|
+
e -= 1;
|
|
2093
|
+
continue;
|
|
2094
|
+
}
|
|
2095
|
+
const b = bracketOf(c);
|
|
2096
|
+
if (b < 0 && excess[-b - 1] > 0) {
|
|
2097
|
+
excess[-b - 1] = excess[-b - 1] - 1;
|
|
2098
|
+
e -= 1;
|
|
2099
|
+
continue;
|
|
2100
|
+
}
|
|
2101
|
+
break;
|
|
2102
|
+
}
|
|
2103
|
+
return e;
|
|
2104
|
+
}
|
|
2105
|
+
function literalValueAt(text, s, json, declaredIfBare, listSep) {
|
|
2106
|
+
const n = text.length;
|
|
2107
|
+
if (s >= n)
|
|
2108
|
+
return undefined;
|
|
2109
|
+
const firstUnit = readAnchorUnit(text, s, json);
|
|
2110
|
+
const first = unitChOf(firstUnit);
|
|
2111
|
+
let vs = s;
|
|
2112
|
+
if (isQuoteUnit(first) || first === JSON_STRING_END) {
|
|
2113
|
+
vs = s + unitLenOf(firstUnit);
|
|
2114
|
+
if (vs >= n)
|
|
2115
|
+
return undefined;
|
|
2116
|
+
if (!isAsciiAlnumCode(unitChOf(readAnchorUnit(text, vs, json))))
|
|
2117
|
+
return undefined;
|
|
2118
|
+
const close = quotedValueEnd(text, vs, first, json);
|
|
2119
|
+
if (close >= 0)
|
|
2120
|
+
return { start: vs, end: close, quoted: true };
|
|
2121
|
+
}
|
|
2122
|
+
else if (!isAsciiAlnumCode(first))
|
|
2123
|
+
return undefined;
|
|
2124
|
+
if (!declaredIfBare)
|
|
2125
|
+
return undefined;
|
|
2126
|
+
if (isProgramIdentStartCode(text.charCodeAt(vs))) {
|
|
2127
|
+
let k = vs;
|
|
2128
|
+
while (k < n && isProgramIdentCode(text.charCodeAt(k)))
|
|
2129
|
+
k += 1;
|
|
2130
|
+
anchoredScanUnits += k - vs;
|
|
2131
|
+
const after = text.charCodeAt(k);
|
|
2132
|
+
if (after === CH_LPAREN || after === CH_LBRACKET || (after === CH_DOT && k + 1 < n && isProgramIdentStartCode(text.charCodeAt(k + 1))))
|
|
2133
|
+
return undefined;
|
|
2134
|
+
}
|
|
2135
|
+
const e = bareValueEnd(text, vs, json, listSep);
|
|
2136
|
+
if (e <= vs)
|
|
2137
|
+
return undefined;
|
|
2138
|
+
if (e - vs <= ABSENT_OR_BOOLEAN_MAX && ABSENT_OR_BOOLEAN_WORDS.has(text.slice(vs, e).toLowerCase()))
|
|
2139
|
+
return undefined;
|
|
2140
|
+
return { start: vs, end: e, quoted: false };
|
|
2141
|
+
}
|
|
2142
|
+
function schemeValueAt(text, s, json) {
|
|
2143
|
+
const n = text.length;
|
|
2144
|
+
let i = s;
|
|
2145
|
+
if (i >= n)
|
|
2146
|
+
return undefined;
|
|
2147
|
+
const opening = readAnchorUnit(text, i, json);
|
|
2148
|
+
if (isQuoteUnit(unitChOf(opening)) || unitChOf(opening) === JSON_STRING_END)
|
|
2149
|
+
i += unitLenOf(opening);
|
|
2150
|
+
const w = i;
|
|
2151
|
+
while (i < n && i - w <= STRONG_AUTH_SCHEME_MAX && (isIdentStartCode(text.charCodeAt(i))))
|
|
2152
|
+
i += 1;
|
|
2153
|
+
anchoredScanUnits += i - w;
|
|
2154
|
+
if (i - w > STRONG_AUTH_SCHEME_MAX || !STRONG_AUTH_SCHEMES.has(text.slice(w, i).toLowerCase()))
|
|
2155
|
+
return undefined;
|
|
2156
|
+
let separated = false;
|
|
2157
|
+
while (i < n) {
|
|
2158
|
+
const unit = readAnchorUnit(text, i, json);
|
|
2159
|
+
const ch = unitChOf(unit);
|
|
2160
|
+
if (!isInlineSpaceUnit(ch) && !isSchemeDelimiterCode(ch))
|
|
2161
|
+
break;
|
|
2162
|
+
i += unitLenOf(unit);
|
|
2163
|
+
separated = true;
|
|
2164
|
+
}
|
|
2165
|
+
if (!separated)
|
|
2166
|
+
return undefined;
|
|
2167
|
+
const vs = i;
|
|
2168
|
+
if (i >= n || !isAsciiAlnumCode(text.charCodeAt(i)))
|
|
2169
|
+
return undefined;
|
|
2170
|
+
while (i < n && isToken68Code(text.charCodeAt(i)))
|
|
2171
|
+
i += 1;
|
|
2172
|
+
while (i < n && text.charCodeAt(i) === CH_EQ)
|
|
2173
|
+
i += 1;
|
|
2174
|
+
anchoredScanUnits += i - vs;
|
|
2175
|
+
const e = i;
|
|
2176
|
+
while (i < n) {
|
|
2177
|
+
const unit = readAnchorUnit(text, i, json);
|
|
2178
|
+
if (!isInlineSpaceUnit(unitChOf(unit)))
|
|
2179
|
+
break;
|
|
2180
|
+
i += unitLenOf(unit);
|
|
2181
|
+
}
|
|
2182
|
+
if (i < n && !endsHeaderValueUnit(unitChOf(readAnchorUnit(text, i, json))))
|
|
2183
|
+
return undefined;
|
|
2184
|
+
return { start: vs, end: e, quoted: false };
|
|
2185
|
+
}
|
|
2186
|
+
function collectAnchoredValueSpans(text, json, span) {
|
|
2187
|
+
const n = text.length;
|
|
2188
|
+
let resume = 0;
|
|
2189
|
+
let nextEq = text.indexOf("=");
|
|
2190
|
+
let nextColon = text.indexOf(":");
|
|
2191
|
+
for (;;) {
|
|
2192
|
+
if (nextEq >= 0 && nextEq < resume)
|
|
2193
|
+
nextEq = text.indexOf("=", resume);
|
|
2194
|
+
if (nextColon >= 0 && nextColon < resume)
|
|
2195
|
+
nextColon = text.indexOf(":", resume);
|
|
2196
|
+
if (nextEq < 0 && nextColon < 0)
|
|
2197
|
+
return;
|
|
2198
|
+
let op;
|
|
2199
|
+
if (nextColon < 0 || (nextEq >= 0 && nextEq < nextColon)) {
|
|
2200
|
+
op = nextEq;
|
|
2201
|
+
nextEq = text.indexOf("=", op + 1);
|
|
2202
|
+
}
|
|
2203
|
+
else {
|
|
2204
|
+
op = nextColon;
|
|
2205
|
+
nextColon = text.indexOf(":", op + 1);
|
|
2206
|
+
}
|
|
2207
|
+
const head = anchorHeadAt(text, op, json);
|
|
2208
|
+
if (head === undefined)
|
|
2209
|
+
continue;
|
|
2210
|
+
let s = op + 1;
|
|
2211
|
+
let spacedRight = false;
|
|
2212
|
+
while (s < n) {
|
|
2213
|
+
const unit = readAnchorUnit(text, s, json);
|
|
2214
|
+
if (!isInlineSpaceUnit(unitChOf(unit)))
|
|
2215
|
+
break;
|
|
2216
|
+
s += unitLenOf(unit);
|
|
2217
|
+
spacedRight = true;
|
|
2218
|
+
}
|
|
2219
|
+
let v;
|
|
2220
|
+
switch (head.strength) {
|
|
2221
|
+
case "structure-only":
|
|
2222
|
+
continue;
|
|
2223
|
+
case "auth-scheme":
|
|
2224
|
+
v = schemeValueAt(text, s, json);
|
|
2225
|
+
break;
|
|
2226
|
+
case "declarative":
|
|
2227
|
+
v = literalValueAt(text, s, json, head.quotedName || (!head.spacedLeft && !spacedRight), head.listSep);
|
|
2228
|
+
break;
|
|
2229
|
+
case "any-form":
|
|
2230
|
+
v = literalValueAt(text, s, json, true, head.listSep);
|
|
2231
|
+
break;
|
|
2232
|
+
default: {
|
|
2233
|
+
const unhandled = head.strength;
|
|
2234
|
+
throw new Error(`redact: unhandled free-text anchor strength ${String(unhandled)}`);
|
|
2235
|
+
}
|
|
2236
|
+
}
|
|
2237
|
+
if (v === undefined)
|
|
2238
|
+
continue;
|
|
2239
|
+
span(v.start, v.end);
|
|
2240
|
+
resume = v.quoted ? v.start : v.end;
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
1510
2243
|
export function structuralScannerForTest(arm) {
|
|
1511
2244
|
const collect = (scan) => (text) => {
|
|
1512
2245
|
const out = [];
|
|
@@ -1523,7 +2256,7 @@ export function structuralScannerForTest(arm) {
|
|
|
1523
2256
|
let ppkScanBytes = 0;
|
|
1524
2257
|
export function ppkScanBytesForTest() { return ppkScanBytes; }
|
|
1525
2258
|
export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
|
|
1526
|
-
function collectPpkContainerSpans(text, record, span) {
|
|
2259
|
+
function collectPpkContainerSpans(scan, text, record, span) {
|
|
1527
2260
|
const HEAD = "PuTTY-User-Key-File-";
|
|
1528
2261
|
const MAC = "Private-MAC: ";
|
|
1529
2262
|
const HASH = "Private-Hash: ";
|
|
@@ -1542,7 +2275,7 @@ function collectPpkContainerSpans(text, record, span) {
|
|
|
1542
2275
|
const isVersionedHead = (at) => {
|
|
1543
2276
|
let p = at + HEAD.length;
|
|
1544
2277
|
const start = p;
|
|
1545
|
-
while (p < text.length && opaqueUnitEnd(p) < 0) {
|
|
2278
|
+
while (p < text.length && opaqueUnitEnd(scan, p) < 0) {
|
|
1546
2279
|
const c = text.charCodeAt(p);
|
|
1547
2280
|
if (c >= 48 && c <= 57)
|
|
1548
2281
|
p += 1;
|
|
@@ -1553,13 +2286,13 @@ function collectPpkContainerSpans(text, record, span) {
|
|
|
1553
2286
|
};
|
|
1554
2287
|
const nextVersionedHead = (from) => {
|
|
1555
2288
|
let p = find(HEAD, from);
|
|
1556
|
-
while (p !== Number.POSITIVE_INFINITY && (inOpaqueUnit(p) || !isVersionedHead(p)))
|
|
2289
|
+
while (p !== Number.POSITIVE_INFINITY && (inOpaqueUnit(scan, p) || !isVersionedHead(p)))
|
|
1557
2290
|
p = find(HEAD, p + HEAD.length);
|
|
1558
2291
|
return p;
|
|
1559
2292
|
};
|
|
1560
2293
|
const hexRunEnd = (from) => {
|
|
1561
2294
|
let e = from;
|
|
1562
|
-
while (e < text.length && opaqueUnitEnd(e) < 0) {
|
|
2295
|
+
while (e < text.length && opaqueUnitEnd(scan, e) < 0) {
|
|
1563
2296
|
const c = text.charCodeAt(e);
|
|
1564
2297
|
if ((c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70))
|
|
1565
2298
|
e += 1;
|
|
@@ -1594,7 +2327,7 @@ function collectPpkContainerSpans(text, record, span) {
|
|
|
1594
2327
|
let gapBlank = true;
|
|
1595
2328
|
if (limit !== Number.POSITIVE_INFINITY) {
|
|
1596
2329
|
for (let i = hexEnd; i < limit; i++) {
|
|
1597
|
-
if (opaqueUnitEnd(i) >= 0) {
|
|
2330
|
+
if (opaqueUnitEnd(scan, i) >= 0) {
|
|
1598
2331
|
gapBlank = false;
|
|
1599
2332
|
break;
|
|
1600
2333
|
}
|
|
@@ -1650,15 +2383,30 @@ const armPool = (() => {
|
|
|
1650
2383
|
headFirst: boundary === undefined ? undefined : headFirst,
|
|
1651
2384
|
};
|
|
1652
2385
|
};
|
|
1653
|
-
const scannerMember = (token, order,
|
|
2386
|
+
const scannerMember = (token, order, arm, rank = ARM_RANK.structural, lexical = false) => {
|
|
2387
|
+
const repl = spanRepl(token);
|
|
2388
|
+
return {
|
|
2389
|
+
token,
|
|
2390
|
+
rank,
|
|
2391
|
+
order,
|
|
2392
|
+
phase: 1,
|
|
2393
|
+
lexical,
|
|
2394
|
+
collect: (text, json, observe, out) => arm(text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
|
|
2395
|
+
retry: undefined,
|
|
2396
|
+
headSrc: undefined,
|
|
2397
|
+
headFirst: undefined,
|
|
2398
|
+
};
|
|
2399
|
+
};
|
|
2400
|
+
const forensicMember = (token, order, arm) => {
|
|
1654
2401
|
const repl = spanRepl(token);
|
|
2402
|
+
const rank = ARM_RANK.structural;
|
|
1655
2403
|
return {
|
|
1656
2404
|
token,
|
|
1657
|
-
rank
|
|
2405
|
+
rank,
|
|
1658
2406
|
order,
|
|
1659
|
-
phase,
|
|
2407
|
+
phase: 2,
|
|
1660
2408
|
lexical: false,
|
|
1661
|
-
collect: (text, json, observe, out) => scan
|
|
2409
|
+
collect: (scan, text, json, observe, out) => arm(scan, text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
|
|
1662
2410
|
retry: undefined,
|
|
1663
2411
|
headSrc: undefined,
|
|
1664
2412
|
headFirst: undefined,
|
|
@@ -1666,21 +2414,22 @@ const armPool = (() => {
|
|
|
1666
2414
|
};
|
|
1667
2415
|
const structuralScannerMember = (row, order) => {
|
|
1668
2416
|
if (row === PEM_FENCE_PATTERN)
|
|
1669
|
-
return scannerMember(row[1], order,
|
|
2417
|
+
return scannerMember(row[1], order, (text, _json, _observe, span) => collectPemFenceSpans(text, span));
|
|
1670
2418
|
if (row === SSH2_FENCE_PATTERN)
|
|
1671
|
-
return
|
|
2419
|
+
return forensicMember(row[1], order, (scan, text, _json, _observe, span) => collectSsh2FenceSpans(scan, text, span));
|
|
1672
2420
|
if (row === URL_CREDENTIAL_PATTERN)
|
|
1673
|
-
return
|
|
2421
|
+
return forensicMember(row[1], order, (scan, text, json, _observe, span) => collectUrlCredentialSpans(scan, text, json, span));
|
|
1674
2422
|
if (row === JSON_SECRET_FIELD_PATTERN)
|
|
1675
|
-
return scannerMember(row[1], order,
|
|
2423
|
+
return scannerMember(row[1], order, (text, _json, _observe, span) => collectJsonSecretFieldSpans(text, span));
|
|
1676
2424
|
throw new Error(`redact: structural row /${row[0].source}/ has no hand-written scanner — structural grammars are declared as regexes but never executed as one (S-102 / S-265)`);
|
|
1677
2425
|
};
|
|
1678
2426
|
const members = [];
|
|
1679
2427
|
for (const row of STRUCTURAL_SECRET_PATTERNS)
|
|
1680
2428
|
members.push(structuralScannerMember(row, members.length));
|
|
1681
|
-
members.push(
|
|
1682
|
-
members.push(
|
|
1683
|
-
members.push(
|
|
2429
|
+
members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, observe, span) => collectPpkContainerSpans(scan, text, observe, span)));
|
|
2430
|
+
members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _observe, span) => collectPemFragmentSpans(scan, text, json, span)));
|
|
2431
|
+
members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _observe, span) => collectHeadlessPpkSpans(scan, text, span)));
|
|
2432
|
+
members.push(scannerMember(ANCHORED_VALUE_TOKEN, members.length, (text, json, _observe, span) => collectAnchoredValueSpans(text, json, span), ARM_RANK.keyword, true));
|
|
1684
2433
|
for (const row of KEYWORD_SECRET_PATTERNS)
|
|
1685
2434
|
members.push(regexMember(row, ARM_RANK.keyword, members.length));
|
|
1686
2435
|
for (const row of SHAPE_SECRET_PATTERNS)
|
|
@@ -1746,25 +2495,37 @@ const armPool = (() => {
|
|
|
1746
2495
|
at = freshCovered(before);
|
|
1747
2496
|
}
|
|
1748
2497
|
};
|
|
1749
|
-
setScanEvidence(NO_CREDENTIAL_SPANS);
|
|
1750
2498
|
for (const m of lane.phase1)
|
|
1751
2499
|
m.collect(text, json, observe, spans);
|
|
1752
2500
|
closeCoveragePositions(0);
|
|
1753
2501
|
if (lane.phase2.length > 0) {
|
|
1754
|
-
|
|
2502
|
+
const scan = buildForensicScan(credentialSpans(text), lane.phase2.length);
|
|
2503
|
+
const reuse = forensicRescanMode === "incremental";
|
|
2504
|
+
const foot = scan.armFoot;
|
|
2505
|
+
let folded = 0;
|
|
1755
2506
|
for (let level = 0;; level += 1) {
|
|
1756
2507
|
const before = spans.length;
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
2508
|
+
for (let j = 0; j < lane.phase2.length; j++) {
|
|
2509
|
+
const dirty = scan.dirty;
|
|
2510
|
+
if (dirty !== undefined && footprintClean(dirty, foot[2 * j], foot[2 * j + 1])) {
|
|
2511
|
+
forensicSkips += 1;
|
|
2512
|
+
continue;
|
|
2513
|
+
}
|
|
2514
|
+
scan.footLo = FOOT_NONE;
|
|
2515
|
+
scan.footHi = -1;
|
|
2516
|
+
lane.phase2[j].collect(scan, text, json, observe && level === 0, spans);
|
|
2517
|
+
foot[2 * j] = scan.footLo;
|
|
2518
|
+
foot[2 * j + 1] = scan.footHi;
|
|
2519
|
+
forensicRuns += 1;
|
|
2520
|
+
}
|
|
1760
2521
|
closeCoveragePositions(before);
|
|
1761
2522
|
forensicLevels += 1;
|
|
1762
|
-
const
|
|
1763
|
-
|
|
2523
|
+
const grown = foldCredentialSpans(scan.table, spans, folded);
|
|
2524
|
+
folded = spans.length;
|
|
2525
|
+
if (grown.dirty.starts.length === 0)
|
|
1764
2526
|
break;
|
|
1765
|
-
|
|
2527
|
+
advanceEvidence(scan, grown.next, reuse ? grown.dirty : undefined);
|
|
1766
2528
|
}
|
|
1767
|
-
setScanEvidence(NO_CREDENTIAL_SPANS);
|
|
1768
2529
|
}
|
|
1769
2530
|
poolSpanCount += spans.length;
|
|
1770
2531
|
const kept = resolveSpanOverlaps(spans);
|
|
@@ -1773,6 +2534,7 @@ const armPool = (() => {
|
|
|
1773
2534
|
const tokenOf = (sp) => members[sp.order].token;
|
|
1774
2535
|
const strong = spans.filter((sp) => armOfToken(tokenOf(sp)) === "full").sort((a, b) => a.start - b.start);
|
|
1775
2536
|
const byArmedToken = new Map();
|
|
2537
|
+
const stamped = [];
|
|
1776
2538
|
let si = 0;
|
|
1777
2539
|
let strongMaxEnd = -1;
|
|
1778
2540
|
for (const sp of kept) {
|
|
@@ -1788,8 +2550,9 @@ const armPool = (() => {
|
|
|
1788
2550
|
byArmedToken.set(key, { token, arm, count: 1 });
|
|
1789
2551
|
else
|
|
1790
2552
|
cur.count += 1;
|
|
2553
|
+
stamped.push({ ...sp, hit: { token, arm } });
|
|
1791
2554
|
}
|
|
1792
|
-
return { hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })), kept };
|
|
2555
|
+
return { hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })), kept: stamped };
|
|
1793
2556
|
};
|
|
1794
2557
|
})();
|
|
1795
2558
|
let onRedaction;
|
|
@@ -1810,21 +2573,49 @@ function reportHits(hits) {
|
|
|
1810
2573
|
}
|
|
1811
2574
|
}
|
|
1812
2575
|
}
|
|
1813
|
-
function runRedaction(text, observe, json, wantHits, scope) {
|
|
1814
|
-
const wantCount = wantHits || (onRedaction !== undefined && observe);
|
|
2576
|
+
function runRedaction(text, observe, json, wantHits, scope, muteCount = false) {
|
|
2577
|
+
const wantCount = wantHits || (onRedaction !== undefined && observe && !muteCount);
|
|
1815
2578
|
const normalized = normalizeForRedaction(text);
|
|
1816
|
-
const
|
|
2579
|
+
const first = applyRules(normalized, wantCount, json, observe, scope);
|
|
1817
2580
|
forgetEscapedSeparatorTails();
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
const
|
|
1821
|
-
|
|
2581
|
+
let spans = spansInOriginal(text, normalized, undefined, first.kept);
|
|
2582
|
+
let hits = first.hits;
|
|
2583
|
+
const second = deleteSpelledFormatCharacters(normalized);
|
|
2584
|
+
if (second.at !== undefined) {
|
|
2585
|
+
const r2 = applyRules(second.text, wantCount, json, false, scope);
|
|
2586
|
+
forgetEscapedSeparatorTails();
|
|
2587
|
+
const extra = spansInOriginal(text, normalized, second, r2.kept);
|
|
2588
|
+
if (extra.length > 0) {
|
|
2589
|
+
spans = resolveSpanOverlaps([...spans, ...extra]);
|
|
2590
|
+
hits = wantCount ? hitsOfSpans(spans) : NO_HITS;
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2593
|
+
if (observe && !muteCount)
|
|
2594
|
+
reportHits(hits);
|
|
2595
|
+
return { text: applySpans(text, spans).out, hits, spans };
|
|
2596
|
+
}
|
|
2597
|
+
function hitsOfSpans(spans) {
|
|
2598
|
+
const by = new Map();
|
|
2599
|
+
for (const sp of spans) {
|
|
2600
|
+
if (sp.hit === undefined)
|
|
2601
|
+
continue;
|
|
2602
|
+
const key = `${sp.hit.token}\u0000${sp.hit.arm}`;
|
|
2603
|
+
const cur = by.get(key);
|
|
2604
|
+
if (cur === undefined)
|
|
2605
|
+
by.set(key, { token: sp.hit.token, arm: sp.hit.arm, count: 1 });
|
|
2606
|
+
else
|
|
2607
|
+
cur.count += 1;
|
|
2608
|
+
}
|
|
2609
|
+
return [...by.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm }));
|
|
1822
2610
|
}
|
|
1823
|
-
function spansInOriginal(text, normalized, kept) {
|
|
1824
|
-
if (kept.length === 0
|
|
2611
|
+
function spansInOriginal(text, normalized, fold, kept) {
|
|
2612
|
+
if (kept.length === 0)
|
|
1825
2613
|
return kept;
|
|
2614
|
+
const inNormalized = fold?.at === undefined ? kept : kept.map((s) => ({ ...s, start: viewIndexInInput(fold, s.start), end: viewIndexInInput(fold, s.end - 1) + 1 }));
|
|
2615
|
+
if (normalized === text)
|
|
2616
|
+
return inNormalized;
|
|
1826
2617
|
const map = mapNormalizedToOriginal(text, normalized);
|
|
1827
|
-
return
|
|
2618
|
+
return inNormalized.map((s) => ({ ...s, start: map[s.start], end: map[s.end - 1] + 1 }));
|
|
1828
2619
|
}
|
|
1829
2620
|
export function redactSecrets(text, opts) {
|
|
1830
2621
|
return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text").text;
|
|
@@ -1837,6 +2628,9 @@ export function redactionSpansForTest(text, opts) {
|
|
|
1837
2628
|
const r = runRedaction(text, false, opts?.json === true, false, "free-text");
|
|
1838
2629
|
return { text: r.text, spans: r.spans.map((sp) => ({ start: sp.start, end: sp.end })) };
|
|
1839
2630
|
}
|
|
2631
|
+
export function proposeRedactionSpans(text, scope, opts) {
|
|
2632
|
+
return runRedaction(text, true, false, false, scope, opts?.countHits === false).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
|
|
2633
|
+
}
|
|
1840
2634
|
const REDACT_MAX_DEPTH = 64;
|
|
1841
2635
|
export function redactDeep(value) {
|
|
1842
2636
|
return redactDeepWith(value, redactSecrets);
|