@herbertgao/pi-extensions 2026.8.14 → 2026.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -13
- package/THIRD_PARTY_NOTICES.md +26 -1
- package/node_modules/@czottmann/pi-automode/CHANGELOG.md +8 -1
- package/node_modules/@czottmann/pi-automode/README.md +4 -0
- package/node_modules/@czottmann/pi-automode/extensions/auto-mode/classifier.ts +71 -9
- package/node_modules/@czottmann/pi-automode/package.json +1 -1
- package/node_modules/@herbertgao/pi-bark/LICENSE +21 -0
- package/node_modules/@herbertgao/pi-bark/README.md +44 -0
- package/node_modules/@herbertgao/pi-bark/assets/pi-icon.png +0 -0
- package/node_modules/@herbertgao/pi-bark/package.json +53 -0
- package/node_modules/@herbertgao/pi-bark/src/index.ts +178 -0
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/config/config.ts +20 -0
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/config/panel.ts +46 -3
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/mouse/interaction.ts +55 -17
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/mouse/packets.ts +6 -0
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/mouse/scroll.ts +5 -0
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/tool/result.ts +34 -30
- package/node_modules/@herbertgao/pi-cc-extensions/package.json +3 -3
- package/node_modules/@juicesharp/rpiv-ask-user-question/package.json +2 -2
- package/node_modules/@narumitw/pi-btw/README.md +129 -88
- package/node_modules/@narumitw/pi-btw/dist/index.ts +283 -59
- package/node_modules/@narumitw/pi-btw/dist/index.ts.map +3 -3
- package/node_modules/@narumitw/pi-btw/package.json +5 -8
- package/node_modules/@narumitw/pi-btw/src/btw.ts +31 -26
- package/node_modules/@narumitw/pi-btw/src/fullscreen-ui.ts +264 -24
- package/node_modules/@narumitw/pi-btw/src/menu.ts +34 -2
- package/node_modules/@narumitw/pi-btw/src/settings.ts +16 -0
- package/node_modules/@narumitw/pi-btw/src/text.ts +18 -0
- package/node_modules/@narumitw/pi-btw/src/transcript-pager.ts +5 -16
- package/node_modules/@narumitw/pi-caffeinate/LICENSE +21 -0
- package/node_modules/@narumitw/pi-caffeinate/README.md +224 -0
- package/node_modules/@narumitw/pi-caffeinate/dist/index.ts +1112 -0
- package/node_modules/@narumitw/pi-caffeinate/dist/index.ts.map +7 -0
- package/node_modules/@narumitw/pi-caffeinate/package.json +53 -0
- package/node_modules/@narumitw/pi-caffeinate/src/caffeinate.ts +811 -0
- package/node_modules/@narumitw/pi-caffeinate/src/dbus-inhibit.ts +153 -0
- package/node_modules/@narumitw/pi-caffeinate/src/index.ts +1 -0
- package/node_modules/@narumitw/pi-caffeinate/src/inhibitor-process.ts +43 -0
- package/node_modules/@narumitw/pi-caffeinate/src/inhibitors.ts +167 -0
- package/node_modules/@narumitw/pi-caffeinate/src/settings.ts +202 -0
- package/node_modules/@tifan/pi-handoff/README.md +39 -13
- package/node_modules/@tifan/pi-handoff/package.json +2 -2
- package/node_modules/@tifan/pi-handoff/src/index.ts +166 -5
- package/node_modules/@tifan/pi-recap/README.md +1 -1
- package/node_modules/@tifan/pi-recap/package.json +2 -2
- package/node_modules/@tifan/pi-recap/src/index.ts +15 -0
- package/node_modules/@tifan/pi-rename/README.md +42 -21
- package/node_modules/@tifan/pi-rename/package.json +1 -1
- package/node_modules/@tifan/pi-rename/src/index.ts +62 -11
- package/node_modules/@tifan/pi-rename/src/language.ts +18 -0
- package/node_modules/@tifan/pi-rename/src/models.ts +69 -13
- package/node_modules/@tifan/pi-rename/src/naming.ts +36 -7
- package/node_modules/@tifan/pi-rename/src/sanitize.ts +22 -14
- package/node_modules/pi-lens/CHANGELOG.md +320 -0
- package/node_modules/pi-lens/config/dependency-cruiser-eager-allowlist.json +79 -0
- package/node_modules/pi-lens/dist/clients/actionable-warnings-logger.js +16 -1
- package/node_modules/pi-lens/dist/clients/ast-grep-client.js +13 -6
- package/node_modules/pi-lens/dist/clients/availability-probe-flight.js +12 -0
- package/node_modules/pi-lens/dist/clients/biome-client.js +7 -0
- package/node_modules/pi-lens/dist/clients/blocker-freshness.js +20 -0
- package/node_modules/pi-lens/dist/clients/bounded-telemetry.js +56 -19
- package/node_modules/pi-lens/dist/clients/build-identity.js +95 -0
- package/node_modules/pi-lens/dist/clients/bus-publish.js +3 -0
- package/node_modules/pi-lens/dist/clients/cache/rule-cache.js +2 -1
- package/node_modules/pi-lens/dist/clients/cache-observability.js +2 -1
- package/node_modules/pi-lens/dist/clients/cascade-logger.js +15 -1
- package/node_modules/pi-lens/dist/clients/dead-code-client.js +14 -11
- package/node_modules/pi-lens/dist/clients/degradation-ledger.js +56 -4
- package/node_modules/pi-lens/dist/clients/demoted-finding-render.js +14 -0
- package/node_modules/pi-lens/dist/clients/dependency-checker.js +20 -3
- package/node_modules/pi-lens/dist/clients/deps/minimatch.js +1 -1
- package/node_modules/pi-lens/dist/clients/dispatch/collect-later-tier.js +39 -0
- package/node_modules/pi-lens/dist/clients/dispatch/dispatcher.js +150 -9
- package/node_modules/pi-lens/dist/clients/dispatch/fact-store.js +231 -9
- package/node_modules/pi-lens/dist/clients/dispatch/integration.js +205 -100
- package/node_modules/pi-lens/dist/clients/dispatch/pending-runner-findings.js +87 -0
- package/node_modules/pi-lens/dist/clients/dispatch/plan.js +21 -5
- package/node_modules/pi-lens/dist/clients/dispatch/runners/ast-grep-napi.js +174 -55
- package/node_modules/pi-lens/dist/clients/dispatch/runners/eslint.js +5 -1
- package/node_modules/pi-lens/dist/clients/dispatch/runners/helm-lint.js +16 -3
- package/node_modules/pi-lens/dist/clients/dispatch/runners/helm-render.js +54 -9
- package/node_modules/pi-lens/dist/clients/dispatch/runners/index.js +1 -3
- package/node_modules/pi-lens/dist/clients/dispatch/runners/lsp.js +37 -1
- package/node_modules/pi-lens/dist/clients/dispatch/runners/markdownlint.js +4 -1
- package/node_modules/pi-lens/dist/clients/dispatch/runners/oxlint.js +11 -4
- package/node_modules/pi-lens/dist/clients/dispatch/runners/psscriptanalyzer.js +4 -0
- package/node_modules/pi-lens/dist/clients/dispatch/runners/rust-clippy.js +8 -3
- package/node_modules/pi-lens/dist/clients/dispatch/runners/utils/runner-helpers.js +76 -25
- package/node_modules/pi-lens/dist/clients/dispatch/runners/utils/spawn-outcome.js +4 -0
- package/node_modules/pi-lens/dist/clients/dispatch/runners/utils/tool-failure.js +19 -7
- package/node_modules/pi-lens/dist/clients/dispatch/runners/utils/toolchain-availability.js +7 -13
- package/node_modules/pi-lens/dist/clients/dispatch/runners/yaml-rule-parser.js +51 -17
- package/node_modules/pi-lens/dist/clients/event-loop-monitor.js +66 -1
- package/node_modules/pi-lens/dist/clients/file-utils.js +157 -7
- package/node_modules/pi-lens/dist/clients/finding-delivery-gate.js +5 -0
- package/node_modules/pi-lens/dist/clients/formatters.js +28 -3
- package/node_modules/pi-lens/dist/clients/git-guard.js +197 -47
- package/node_modules/pi-lens/dist/clients/git-tracked-ignore.js +32 -1
- package/node_modules/pi-lens/dist/clients/govulncheck-client.js +5 -1
- package/node_modules/pi-lens/dist/clients/gzip-stage-write.js +7 -0
- package/node_modules/pi-lens/dist/clients/installer/index.js +131 -34
- package/node_modules/pi-lens/dist/clients/installer/managed-tool-refresh.js +3 -1
- package/node_modules/pi-lens/dist/clients/instance-reaper.js +4 -13
- package/node_modules/pi-lens/dist/clients/instance-registry-lock.js +145 -0
- package/node_modules/pi-lens/dist/clients/instance-registry.js +432 -122
- package/node_modules/pi-lens/dist/clients/jscpd-client.js +6 -1
- package/node_modules/pi-lens/dist/clients/language-profile.js +4 -2
- package/node_modules/pi-lens/dist/clients/latency-logger.js +19 -1
- package/node_modules/pi-lens/dist/clients/lens-flag-registry.js +8 -0
- package/node_modules/pi-lens/dist/clients/lens-map.js +3 -1
- package/node_modules/pi-lens/dist/clients/lsp/cascade-tier.js +4 -0
- package/node_modules/pi-lens/dist/clients/lsp/client.js +516 -31
- package/node_modules/pi-lens/dist/clients/lsp/config.js +25 -2
- package/node_modules/pi-lens/dist/clients/lsp/diagnostic-binding.js +9 -4
- package/node_modules/pi-lens/dist/clients/lsp/index.js +509 -182
- package/node_modules/pi-lens/dist/clients/lsp/jvm-runtime.js +5 -0
- package/node_modules/pi-lens/dist/clients/lsp/launch.js +36 -8
- package/node_modules/pi-lens/dist/clients/lsp/pending-aux-coverage.js +116 -4
- package/node_modules/pi-lens/dist/clients/lsp/server.js +142 -13
- package/node_modules/pi-lens/dist/clients/lsp/session-roots.js +99 -0
- package/node_modules/pi-lens/dist/clients/lsp/workspace-diagnostics-cache.js +16 -6
- package/node_modules/pi-lens/dist/clients/lsp/workspace-sweep-hold.js +32 -23
- package/node_modules/pi-lens/dist/clients/lsp-mutation.js +16 -2
- package/node_modules/pi-lens/dist/clients/mcp/analyze.js +12 -6
- package/node_modules/pi-lens/dist/clients/mcp/session.js +14 -1
- package/node_modules/pi-lens/dist/clients/memory-sampler.js +23 -1
- package/node_modules/pi-lens/dist/clients/message-end-attribution.js +30 -0
- package/node_modules/pi-lens/dist/clients/metrics-history.js +21 -4
- package/node_modules/pi-lens/dist/clients/opaque-mutation-scan.js +232 -18
- package/node_modules/pi-lens/dist/clients/package-manager.js +6 -18
- package/node_modules/pi-lens/dist/clients/path-keyed-map.js +14 -1
- package/node_modules/pi-lens/dist/clients/path-utils.js +32 -0
- package/node_modules/pi-lens/dist/clients/pipeline.js +0 -8
- package/node_modules/pi-lens/dist/clients/process-singletons.js +210 -0
- package/node_modules/pi-lens/dist/clients/project-diagnostics/runner-adapters/madge.js +2 -1
- package/node_modules/pi-lens/dist/clients/project-diagnostics/scanner.js +10 -4
- package/node_modules/pi-lens/dist/clients/project-snapshot.js +33 -11
- package/node_modules/pi-lens/dist/clients/read-guard-logger.js +24 -2
- package/node_modules/pi-lens/dist/clients/read-guard.js +74 -7
- package/node_modules/pi-lens/dist/clients/recent-touches.js +17 -5
- package/node_modules/pi-lens/dist/clients/review-graph/builder.js +639 -62
- package/node_modules/pi-lens/dist/clients/review-graph/tsconfig-paths.js +5 -1
- package/node_modules/pi-lens/dist/clients/review-graph-logger.js +21 -1
- package/node_modules/pi-lens/dist/clients/runtime-coordinator.js +68 -0
- package/node_modules/pi-lens/dist/clients/runtime-session.js +30 -12
- package/node_modules/pi-lens/dist/clients/runtime-tool-call.js +15 -0
- package/node_modules/pi-lens/dist/clients/runtime-tool-result.js +106 -7
- package/node_modules/pi-lens/dist/clients/runtime-turn.js +242 -27
- package/node_modules/pi-lens/dist/clients/safe-spawn.js +158 -18
- package/node_modules/pi-lens/dist/clients/security-scan-client.js +48 -17
- package/node_modules/pi-lens/dist/clients/session-lifecycle.js +238 -42
- package/node_modules/pi-lens/dist/clients/sg-runner.js +27 -15
- package/node_modules/pi-lens/dist/clients/shared-checkout-guard.js +338 -0
- package/node_modules/pi-lens/dist/clients/spawn-output-cap.js +37 -0
- package/node_modules/pi-lens/dist/clients/spawn-timeout-cooldown.js +15 -1
- package/node_modules/pi-lens/dist/clients/startup-scan.js +6 -7
- package/node_modules/pi-lens/dist/clients/startup-timing.js +69 -4
- package/node_modules/pi-lens/dist/clients/string-utils.js +20 -0
- package/node_modules/pi-lens/dist/clients/test-runner-client.js +419 -88
- package/node_modules/pi-lens/dist/clients/tool-policy.js +1 -1
- package/node_modules/pi-lens/dist/clients/tree-sitter-client.js +11 -5
- package/node_modules/pi-lens/dist/clients/tree-sitter-logger.js +17 -1
- package/node_modules/pi-lens/dist/clients/warm-attach.js +8 -10
- package/node_modules/pi-lens/dist/clients/word-index-logger.js +13 -1
- package/node_modules/pi-lens/dist/clients/word-index-store.js +448 -0
- package/node_modules/pi-lens/dist/clients/word-index.js +582 -104
- package/node_modules/pi-lens/dist/clients/workspace-topology.js +17 -0
- package/node_modules/pi-lens/dist/clients/zizmor-config.js +26 -6
- package/node_modules/pi-lens/dist/index.js +11243 -6651
- package/node_modules/pi-lens/dist/mcp/build-staleness.js +8 -7
- package/node_modules/pi-lens/dist/mcp/cli.js +3 -1
- package/node_modules/pi-lens/dist/mcp/server.js +6 -1
- package/node_modules/pi-lens/dist/tools/lens-diagnostics.js +4 -0
- package/node_modules/pi-lens/dist/tools/lsp-diagnostics.js +14 -5
- package/node_modules/pi-lens/dist/tools/lsp-navigation.js +3 -1
- package/node_modules/pi-lens/docs/agent-guide.md +4 -2
- package/node_modules/pi-lens/docs/ast-grep_rules_catalog.md +41 -10
- package/node_modules/pi-lens/docs/astplayground.md +26 -17
- package/node_modules/pi-lens/package.json +5 -1
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/c-no-malloc-free-test.yml +20 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/cpp-no-malloc-free-test.yml +25 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/cpp-no-printf-test.yml +19 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/go-no-fmt-println-test.yml +19 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/go-no-panic-in-lib-test.yml +17 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/go-no-underscore-func-name-test.yml +18 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/go-prefer-errors-is-test.yml +15 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/go-prefer-string-builder-test.yml +21 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/hyphenated-svg-attribute-test.yml +11 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-no-lateinit-test.yml +17 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-no-nullable-boolean-test.yml +15 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-no-println-test.yml +19 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-no-unnecessary-let-test.yml +7 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-no-unsafe-cast-test.yml +7 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-no-var-test.yml +7 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-prefer-data-class-test.yml +25 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/kotlin-prefer-is-empty-test.yml +11 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/no-important-test.yml +21 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/no-raw-types-test.yml +30 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/no-string-concat-in-loop-test.yml +100 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/no-system-out-println-test.yml +7 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/prefer-string-is-empty-test.yml +7 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/python-optional-type-test.yml +9 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/reducible-list-comprehension-test.yml +11 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/ruby-symbol-to-proc-candidate-test.yml +10 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rule-tests/rust-no-chars-enumerate-test.yml +7 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/c-no-malloc-free.yml +39 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/cpp-no-malloc-free.yml +45 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/cpp-no-printf.yml +41 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/go-no-fmt-println.yml +43 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/go-no-panic-in-lib.yml +41 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/go-no-underscore-func-name.yml +35 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/go-prefer-errors-is.yml +40 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/go-prefer-string-builder.yml +43 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/hyphenated-svg-attribute.yml +32 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-no-lateinit.yml +37 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-no-nullable-boolean.yml +33 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-no-println.yml +42 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-no-unnecessary-let.yml +28 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-no-unsafe-cast.yml +28 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-no-var.yml +35 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-prefer-data-class.yml +66 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/kotlin-prefer-is-empty.yml +35 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/no-important.yml +21 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/no-raw-types.yml +33 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/no-string-concat-in-loop.yml +90 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/no-system-out-println.yml +16 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/prefer-string-is-empty.yml +11 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/python-optional-type.yml +22 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/reducible-list-comprehension.yml +29 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/ruby-symbol-to-proc-candidate.yml +25 -0
- package/node_modules/pi-lens/rules/ast-grep-rules/rules/rust-no-chars-enumerate.yml +19 -0
- package/node_modules/pi-mcp-adapter/CHANGELOG.md +42 -0
- package/node_modules/pi-mcp-adapter/README.md +15 -15
- package/node_modules/pi-mcp-adapter/commands.ts +59 -22
- package/node_modules/pi-mcp-adapter/config.ts +18 -4
- package/node_modules/pi-mcp-adapter/direct-tools.ts +26 -6
- package/node_modules/pi-mcp-adapter/dist/config.d.ts +4 -0
- package/node_modules/pi-mcp-adapter/dist/config.js +13 -4
- package/node_modules/pi-mcp-adapter/dist/config.js.map +1 -1
- package/node_modules/pi-mcp-adapter/dist/types.d.ts +12 -4
- package/node_modules/pi-mcp-adapter/dist/types.js.map +1 -1
- package/node_modules/pi-mcp-adapter/error-signal.ts +15 -4
- package/node_modules/pi-mcp-adapter/errors.ts +141 -0
- package/node_modules/pi-mcp-adapter/host-html-template.ts +58 -5
- package/node_modules/pi-mcp-adapter/index.ts +2 -3
- package/node_modules/pi-mcp-adapter/init.ts +5 -0
- package/node_modules/pi-mcp-adapter/mcp-auth-flow.ts +105 -32
- package/node_modules/pi-mcp-adapter/mcp-panel.ts +30 -9
- package/node_modules/pi-mcp-adapter/mcp-setup-panel.ts +66 -28
- package/node_modules/pi-mcp-adapter/mcp-status.ts +2 -0
- package/node_modules/pi-mcp-adapter/package.json +2 -1
- package/node_modules/pi-mcp-adapter/proxy-modes.ts +88 -25
- package/node_modules/pi-mcp-adapter/sandbox-proxy-template.ts +217 -0
- package/node_modules/pi-mcp-adapter/server-manager.ts +343 -6
- package/node_modules/pi-mcp-adapter/skills/mcp-scripting/SKILL.md +1 -0
- package/node_modules/pi-mcp-adapter/types.ts +19 -4
- package/node_modules/pi-mcp-adapter/ui-resource-handler.ts +18 -2
- package/node_modules/pi-mcp-adapter/ui-server.ts +179 -54
- package/node_modules/pi-mcp-adapter/ui-session.ts +28 -8
- package/node_modules/pi-web-access/CHANGELOG.md +20 -0
- package/node_modules/pi-web-access/README.md +15 -2
- package/node_modules/pi-web-access/extract.ts +57 -12
- package/node_modules/pi-web-access/github-extract.ts +47 -3
- package/node_modules/pi-web-access/index.ts +1 -1
- package/node_modules/pi-web-access/package.json +1 -1
- package/node_modules/pi-web-access/page-query.ts +61 -8
- package/package.json +21 -17
- package/node_modules/@tifan/pi-titlebar-spinner/README.md +0 -24
- package/node_modules/@tifan/pi-titlebar-spinner/package.json +0 -39
- package/node_modules/@tifan/pi-titlebar-spinner/src/index.ts +0 -49
|
@@ -318,7 +318,13 @@ export class TreeSitterClient {
|
|
|
318
318
|
activeMeasurements = 0;
|
|
319
319
|
onWasmAbort;
|
|
320
320
|
wasmAborted = false;
|
|
321
|
-
|
|
321
|
+
grammarDirResolutionDeps;
|
|
322
|
+
constructor(verbose = false, onWasmAbort, grammarDirResolutionDeps) {
|
|
323
|
+
this.grammarDirResolutionDeps = grammarDirResolutionDeps ?? {
|
|
324
|
+
resolveAsset: (asset) => this.resolveWebTreeSitterAsset(asset),
|
|
325
|
+
resolvePackage: (specifier) => _require.resolve(specifier),
|
|
326
|
+
cwd: () => process.cwd(),
|
|
327
|
+
};
|
|
322
328
|
this.grammarsDir = this.findGrammarsDir();
|
|
323
329
|
this.verbose = verbose;
|
|
324
330
|
this.onWasmAbort = onWasmAbort;
|
|
@@ -708,8 +714,8 @@ export class TreeSitterClient {
|
|
|
708
714
|
});
|
|
709
715
|
}
|
|
710
716
|
/** Find tree-sitter grammar directory */
|
|
711
|
-
findGrammarsDir() {
|
|
712
|
-
const grammarsDir =
|
|
717
|
+
findGrammarsDir(deps = this.grammarDirResolutionDeps) {
|
|
718
|
+
const grammarsDir = deps.resolveAsset("grammars");
|
|
713
719
|
if (grammarsDir &&
|
|
714
720
|
fs.existsSync(path.join(grammarsDir, "tree-sitter-typescript.wasm"))) {
|
|
715
721
|
return grammarsDir;
|
|
@@ -717,14 +723,14 @@ export class TreeSitterClient {
|
|
|
717
723
|
// Fallback: a real `tree-sitter-wasms` package, if the user installed one
|
|
718
724
|
// (it is not a pi-lens dependency — grammars ship bundled / lazy-fetched).
|
|
719
725
|
try {
|
|
720
|
-
const wasmsOut = path.join(path.dirname(
|
|
726
|
+
const wasmsOut = path.join(path.dirname(deps.resolvePackage("tree-sitter-wasms/package.json")), "out");
|
|
721
727
|
if (fs.existsSync(wasmsOut))
|
|
722
728
|
return wasmsOut;
|
|
723
729
|
}
|
|
724
730
|
catch {
|
|
725
731
|
/* fall through */
|
|
726
732
|
}
|
|
727
|
-
const cwdWasms = path.join(
|
|
733
|
+
const cwdWasms = path.join(deps.cwd(), "node_modules", "tree-sitter-wasms", "out");
|
|
728
734
|
if (fs.existsSync(cwdWasms))
|
|
729
735
|
return cwdWasms;
|
|
730
736
|
return "";
|
|
@@ -3,6 +3,7 @@ import { isTestMode } from "./env-utils.js";
|
|
|
3
3
|
import { getGlobalPiLensDir } from "./file-utils.js";
|
|
4
4
|
import { createNdjsonLogger } from "./ndjson-logger.js";
|
|
5
5
|
import { getMaxLogSizeMB } from "./log-cleanup.js";
|
|
6
|
+
import { normalizeLoggedPath } from "./path-utils.js";
|
|
6
7
|
const TREE_SITTER_LOG_DIR = getGlobalPiLensDir();
|
|
7
8
|
const TREE_SITTER_LOG_FILE = path.join(TREE_SITTER_LOG_DIR, "tree-sitter.log");
|
|
8
9
|
const writer = createNdjsonLogger({
|
|
@@ -27,11 +28,26 @@ const CACHE_COUNTER_KEYS = [
|
|
|
27
28
|
"parserDurationMs",
|
|
28
29
|
"parserFailures",
|
|
29
30
|
];
|
|
31
|
+
/**
|
|
32
|
+
* #2219 (the #2141 class): `filePath` mixes raw `path.resolve()`/`cwd`
|
|
33
|
+
* values (`project-diagnostics/scanner.ts`, `review-graph/builder.ts`,
|
|
34
|
+
* `tree-sitter-shared.ts`) with already-normalized `ctx.filePath` (the
|
|
35
|
+
* `dispatch/runners/tree-sitter.ts` call sites) and the `"<tree-sitter>"`
|
|
36
|
+
* sentinel `logTreeSitterDiagnostic` falls back to below. This is the single
|
|
37
|
+
* seam every one of those paths funnels through (`logTreeSitterCacheStats`
|
|
38
|
+
* and `logTreeSitterDiagnostic` both call this function), so normalize once
|
|
39
|
+
* here — guarded via `normalizeLoggedPath` so the sentinel passes through
|
|
40
|
+
* unchanged instead of being resolved against the process cwd.
|
|
41
|
+
*/
|
|
30
42
|
export function logTreeSitter(entry) {
|
|
31
43
|
if (isTestMode()) {
|
|
32
44
|
return;
|
|
33
45
|
}
|
|
34
|
-
writer.log({
|
|
46
|
+
writer.log({
|
|
47
|
+
ts: new Date().toISOString(),
|
|
48
|
+
...entry,
|
|
49
|
+
filePath: normalizeLoggedPath(entry.filePath),
|
|
50
|
+
});
|
|
35
51
|
}
|
|
36
52
|
export function logTreeSitterCacheStats(options) {
|
|
37
53
|
const delta = Object.fromEntries(CACHE_COUNTER_KEYS.map((key) => [key, options.stats[key]]));
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as net from "node:net";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
-
import { readInstanceRegistry, } from "./instance-registry.js";
|
|
5
|
-
import { realIsPidAlive
|
|
4
|
+
import { readInstanceRegistry, selectLivePeerInstances, } from "./instance-registry.js";
|
|
5
|
+
import { realIsPidAlive } from "./instance-reaper.js";
|
|
6
6
|
import { logLatency } from "./latency-logger.js";
|
|
7
7
|
import { touchCoverageGap } from "./lsp/diagnostic-binding.js";
|
|
8
8
|
import { loadLspService } from "./lsp-lazy.js";
|
|
@@ -18,15 +18,13 @@ function record(event, cwd, reason, pid, source) {
|
|
|
18
18
|
metadata: { event, reason, incumbentPid: pid, source },
|
|
19
19
|
});
|
|
20
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* The oldest live peer on this root, if any. The liveness rule itself lives
|
|
23
|
+
* in `selectLivePeerInstances` (#2007) so warm-attach and the shared-checkout
|
|
24
|
+
* guard cannot drift apart about what "another session is here" means.
|
|
25
|
+
*/
|
|
21
26
|
export function selectWarmAttachIncumbent(entries, cwd, now = Date.now(), isPidAlive = realIsPidAlive) {
|
|
22
|
-
|
|
23
|
-
return entries
|
|
24
|
-
.filter((entry) => entry.pid !== process.pid &&
|
|
25
|
-
entry.projectRoot === root &&
|
|
26
|
-
isPidAlive(entry.pid) &&
|
|
27
|
-
Number.isFinite(Date.parse(entry.heartbeatAt)) &&
|
|
28
|
-
now - Date.parse(entry.heartbeatAt) <= STALE_HEARTBEAT_MS)
|
|
29
|
-
.sort((a, b) => Date.parse(a.startedAt) - Date.parse(b.startedAt))[0];
|
|
27
|
+
return selectLivePeerInstances(entries, cwd, now, isPidAlive)[0];
|
|
30
28
|
}
|
|
31
29
|
async function serveRequest(req) {
|
|
32
30
|
if (req.route === "code-actions") {
|
|
@@ -24,16 +24,28 @@ import { isTestMode } from "./env-utils.js";
|
|
|
24
24
|
import { getGlobalPiLensDir } from "./file-utils.js";
|
|
25
25
|
import { getMaxLogSizeMB } from "./log-cleanup.js";
|
|
26
26
|
import { createNdjsonLogger } from "./ndjson-logger.js";
|
|
27
|
+
import { normalizeFilePath } from "./path-utils.js";
|
|
27
28
|
const WORD_INDEX_LOG_FILE = path.join(getGlobalPiLensDir(), "word-index.log");
|
|
28
29
|
const writer = createNdjsonLogger({
|
|
29
30
|
filePath: WORD_INDEX_LOG_FILE,
|
|
30
31
|
maxBytes: getMaxLogSizeMB() * 1024 * 1024,
|
|
31
32
|
});
|
|
33
|
+
/**
|
|
34
|
+
* #2141 class sweep: mirrors review-graph-logger.ts's `cwd` normalization.
|
|
35
|
+
* Call sites here feed both raw roots (`snapshotRoot`, `path.resolve(cwd)`)
|
|
36
|
+
* and pre-normalized cache keys (`key`), so the same root could show up in
|
|
37
|
+
* two path forms. Normalizing once at this shared emit seam closes that for
|
|
38
|
+
* every call site, present and future.
|
|
39
|
+
*/
|
|
32
40
|
export function logWordIndex(entry) {
|
|
33
41
|
if (isTestMode()) {
|
|
34
42
|
return;
|
|
35
43
|
}
|
|
36
|
-
writer.log({
|
|
44
|
+
writer.log({
|
|
45
|
+
ts: new Date().toISOString(),
|
|
46
|
+
...entry,
|
|
47
|
+
cwd: normalizeFilePath(entry.cwd),
|
|
48
|
+
});
|
|
37
49
|
}
|
|
38
50
|
export function getWordIndexLogPath() {
|
|
39
51
|
return WORD_INDEX_LOG_FILE;
|
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
import { createDeadline, yieldIfOverBudget } from "./cooperative-budget.js";
|
|
2
|
+
/**
|
|
3
|
+
* Packed backing store for the word index's inverted postings (#2069).
|
|
4
|
+
*
|
|
5
|
+
* The index used to hold one boxed `{ file: string; line: number }` object per
|
|
6
|
+
* (token, file, line) posting. On this repository's own 2,622-document corpus
|
|
7
|
+
* that is 2.22 million objects for 17.9 MB of source: a V8 object header, a
|
|
8
|
+
* pointer into the file table, and a boxed number, to carry eight bytes of
|
|
9
|
+
* information. Measured resident cost was 186.6 MB, or 88.1 bytes per entry.
|
|
10
|
+
*
|
|
11
|
+
* The information content is a pair of small integers, and the serializer has
|
|
12
|
+
* always known it: `SerializedWordIndex.postings` is
|
|
13
|
+
* `[token, [fileIdx, line, fileIdx, line, …]]`. This module makes the in-memory
|
|
14
|
+
* form match the wire form. {@link WordPostingList} stores those pairs in an
|
|
15
|
+
* `Int32Array` — two 32-bit lanes, {@link WORD_POSTING_ENTRY_BYTES} bytes per
|
|
16
|
+
* entry, no per-entry object — and {@link WordIndexFileTable} owns the dense
|
|
17
|
+
* file-id space the `fileIdx` lane refers to.
|
|
18
|
+
*
|
|
19
|
+
* The file table is deliberately ONE structure rather than a key→path map plus
|
|
20
|
+
* a parallel id list: `ids` and `paths` are two views of a single interning
|
|
21
|
+
* decision, so there is nothing to keep in sync by hand. It supersedes the
|
|
22
|
+
* `Map<string, string>` interning table added in #2082 and keeps that fix's
|
|
23
|
+
* invariant in a stronger form — posting removal now compares an integer id
|
|
24
|
+
* instead of a shared string identity, so a divergent spelling cannot survive a
|
|
25
|
+
* removal even in principle.
|
|
26
|
+
*
|
|
27
|
+
* Kept in its own module so the representation is unit-testable without the
|
|
28
|
+
* index's tokenizer, refresh, and persistence machinery around it.
|
|
29
|
+
*/
|
|
30
|
+
/**
|
|
31
|
+
* Bytes of backing store per posting entry: one `fileId` lane and one `line`
|
|
32
|
+
* lane, both `Int32`. This is the constant #2069's acceptance criterion is
|
|
33
|
+
* written against, and {@link estimatePostingListBytes} is the arithmetic the
|
|
34
|
+
* memory-sample record and the regression test both read.
|
|
35
|
+
*/
|
|
36
|
+
export const WORD_POSTING_ENTRY_BYTES = 8;
|
|
37
|
+
/**
|
|
38
|
+
* Fixed bookkeeping charged to every posting list in the byte estimate: the
|
|
39
|
+
* `WordPostingList` wrapper object, its entry in the postings `Map`, and the
|
|
40
|
+
* canonical token string it holds. V8 does not expose these, so this is a
|
|
41
|
+
* deliberate over-estimate of the ~136 bytes they occupy on a 64-bit build with
|
|
42
|
+
* pointer compression.
|
|
43
|
+
*
|
|
44
|
+
* Over-estimating this constant does NOT make the whole figure an upper bound,
|
|
45
|
+
* and {@link estimateWordIndexStoreBytes} is explicit that it is a floor. The
|
|
46
|
+
* blind spot is arena slack: a list that outgrows its slice after an edit stops
|
|
47
|
+
* referring to the arena, but the arena keeps the vacated lanes alive while any
|
|
48
|
+
* sibling still points into it, and nothing charges them to anyone. After a
|
|
49
|
+
* full-corpus rewrite pass the estimate reads 0.58 to 0.66 of the real cost
|
|
50
|
+
* (two independent measurements). Treat this constant as tightening a floor,
|
|
51
|
+
* never as buying headroom.
|
|
52
|
+
*/
|
|
53
|
+
export const WORD_POSTING_LIST_OVERHEAD_BYTES = 160;
|
|
54
|
+
/** Entry capacity a freshly created list reserves when the caller gives no hint. */
|
|
55
|
+
const DEFAULT_POSTING_CAPACITY_ENTRIES = 2;
|
|
56
|
+
/**
|
|
57
|
+
* A token's postings as packed `[fileId, line]` pairs.
|
|
58
|
+
*
|
|
59
|
+
* `length` counts ENTRIES, not lanes — the buffer holds `2 * length` `Int32`s
|
|
60
|
+
* and may hold spare capacity beyond that.
|
|
61
|
+
* {@link compactPostingsIntoArena} releases the spare and shares one backing
|
|
62
|
+
* store; bulk build paths run it once at the end, so a cold index carries no
|
|
63
|
+
* growth slack and one `ArrayBuffer` in total.
|
|
64
|
+
*/
|
|
65
|
+
export class WordPostingList {
|
|
66
|
+
/**
|
|
67
|
+
* The `Int32Array` these postings live in. Either private to this list, or —
|
|
68
|
+
* after {@link compactPostingsIntoArena} — a SHARED arena in which this list
|
|
69
|
+
* owns the half-open lane range `[laneStart, laneStart + capacityEntries*2)`.
|
|
70
|
+
* Held as an offset rather than a `subarray` view on purpose: 37,500 views
|
|
71
|
+
* cost 2.5 MB of `JSTypedArray` headers on this repository's corpus, and two
|
|
72
|
+
* integer fields are free by comparison.
|
|
73
|
+
*/
|
|
74
|
+
lanes;
|
|
75
|
+
laneStart = 0;
|
|
76
|
+
capacityEntries;
|
|
77
|
+
entryCount = 0;
|
|
78
|
+
/**
|
|
79
|
+
* The canonical instance of this list’s token.
|
|
80
|
+
*
|
|
81
|
+
* The tokenizer allocates a FRESH string for every occurrence
|
|
82
|
+
* (`toLowerCase()`, `split()`), and a `Map` key is not addressable, so
|
|
83
|
+
* every structure that stores a token name would otherwise hold its own
|
|
84
|
+
* copy. The forward index does exactly that, once per (document, token):
|
|
85
|
+
* 536,978 separate strings, measured at 17 MB on this repository’s corpus.
|
|
86
|
+
* Holding the first-seen instance here gives every later writer one
|
|
87
|
+
* addressable canonical string to point at, with no second interning map
|
|
88
|
+
* to keep in step with the postings map.
|
|
89
|
+
*/
|
|
90
|
+
token;
|
|
91
|
+
constructor(token, capacityEntries = DEFAULT_POSTING_CAPACITY_ENTRIES) {
|
|
92
|
+
this.token = token;
|
|
93
|
+
this.capacityEntries = Math.max(1, capacityEntries);
|
|
94
|
+
this.lanes = new Int32Array(this.capacityEntries * 2);
|
|
95
|
+
}
|
|
96
|
+
/** Number of postings held. */
|
|
97
|
+
get length() {
|
|
98
|
+
return this.entryCount;
|
|
99
|
+
}
|
|
100
|
+
/** This list's share of backing store, its own spare capacity included. */
|
|
101
|
+
get byteLength() {
|
|
102
|
+
return this.capacityEntries * WORD_POSTING_ENTRY_BYTES;
|
|
103
|
+
}
|
|
104
|
+
fileIdAt(entry) {
|
|
105
|
+
return this.lanes[this.laneStart + entry * 2];
|
|
106
|
+
}
|
|
107
|
+
lineAt(entry) {
|
|
108
|
+
return this.lanes[this.laneStart + entry * 2 + 1];
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Grow so at least `entries` postings fit without another reallocation. The
|
|
112
|
+
* grown store is always PRIVATE: a list that outgrows its arena slice must
|
|
113
|
+
* not write past it into the next token's postings.
|
|
114
|
+
*
|
|
115
|
+
* Returns `true` when it allocated a fresh private store, so the index can
|
|
116
|
+
* keep an O(1) running tally of distinct backing stores instead of rebuilding
|
|
117
|
+
* a `Set` over the whole vocabulary on every edit (#2117).
|
|
118
|
+
*/
|
|
119
|
+
reserve(entries) {
|
|
120
|
+
if (entries <= this.capacityEntries)
|
|
121
|
+
return false;
|
|
122
|
+
const grown = new Int32Array(entries * 2);
|
|
123
|
+
grown.set(this.lanes.subarray(this.laneStart, this.laneStart + this.entryCount * 2));
|
|
124
|
+
this.lanes = grown;
|
|
125
|
+
this.laneStart = 0;
|
|
126
|
+
this.capacityEntries = entries;
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
push(fileId, line) {
|
|
130
|
+
if (this.entryCount === this.capacityEntries) {
|
|
131
|
+
this.reserve(Math.max(DEFAULT_POSTING_CAPACITY_ENTRIES, this.capacityEntries * 2));
|
|
132
|
+
}
|
|
133
|
+
const at = this.laneStart + this.entryCount * 2;
|
|
134
|
+
this.lanes[at] = fileId;
|
|
135
|
+
this.lanes[at + 1] = line;
|
|
136
|
+
this.entryCount += 1;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Re-home this list's lanes into `arena` at lane index `offset`, returning
|
|
140
|
+
* the offset past the range it now owns. Used only by
|
|
141
|
+
* {@link compactPostingsIntoArena}, which sizes the arena from the same
|
|
142
|
+
* entry counts, so the range always fits.
|
|
143
|
+
*/
|
|
144
|
+
adoptArena(arena, offset) {
|
|
145
|
+
const width = this.entryCount * 2;
|
|
146
|
+
for (let i = 0; i < width; i += 1) {
|
|
147
|
+
arena[offset + i] = this.lanes[this.laneStart + i];
|
|
148
|
+
}
|
|
149
|
+
this.lanes = arena;
|
|
150
|
+
this.laneStart = offset;
|
|
151
|
+
this.capacityEntries = this.entryCount;
|
|
152
|
+
return offset + width;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* A new list with every posting for `fileId` removed, exactly sized. Returns
|
|
156
|
+
* a fresh list rather than mutating in place because the incremental refresh
|
|
157
|
+
* stages removals off to the side and publishes them without an await, and
|
|
158
|
+
* because an untouched token's list must keep its reference identity.
|
|
159
|
+
*/
|
|
160
|
+
withoutFile(fileId) {
|
|
161
|
+
let surviving = 0;
|
|
162
|
+
for (let i = 0; i < this.entryCount; i += 1) {
|
|
163
|
+
if (this.fileIdAt(i) !== fileId)
|
|
164
|
+
surviving += 1;
|
|
165
|
+
}
|
|
166
|
+
const next = new WordPostingList(this.token, Math.max(1, surviving));
|
|
167
|
+
for (let i = 0; i < this.entryCount; i += 1) {
|
|
168
|
+
if (this.fileIdAt(i) === fileId)
|
|
169
|
+
continue;
|
|
170
|
+
next.push(this.fileIdAt(i), this.lineAt(i));
|
|
171
|
+
}
|
|
172
|
+
return next;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* The backing store these lanes live in. Exposed so a memory guard can prove
|
|
176
|
+
* the arena is shared rather than inferring it from a heap delta; nothing in
|
|
177
|
+
* the index reads it.
|
|
178
|
+
*/
|
|
179
|
+
get backingStore() {
|
|
180
|
+
return this.lanes;
|
|
181
|
+
}
|
|
182
|
+
/** Build a list from flat `[fileId, line, …]` lanes, exactly sized. */
|
|
183
|
+
static fromLanes(token, lanes) {
|
|
184
|
+
const entries = Math.floor(lanes.length / 2);
|
|
185
|
+
const list = new WordPostingList(token, Math.max(1, entries));
|
|
186
|
+
for (let i = 0; i < entries; i += 1) {
|
|
187
|
+
list.push(lanes[i * 2], lanes[i * 2 + 1]);
|
|
188
|
+
}
|
|
189
|
+
return list;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
/** Estimated bytes one posting list holds, backing store plus fixed headers. */
|
|
193
|
+
export function estimatePostingListBytes(list) {
|
|
194
|
+
return list.byteLength + WORD_POSTING_LIST_OVERHEAD_BYTES;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Re-home every posting list into ONE shared `ArrayBuffer`, exactly sized.
|
|
198
|
+
*
|
|
199
|
+
* A bulk build ends with ~37,500 independently allocated `Int32Array`s on this
|
|
200
|
+
* repository's corpus. Each carries its own `ArrayBuffer` header, and measured
|
|
201
|
+
* across the whole store that bookkeeping was 10.1 MB against 18.1 MB of actual
|
|
202
|
+
* lane data — more than a third of the posting store spent on allocation
|
|
203
|
+
* headers for the long tail of rare tokens. One arena plus per-token
|
|
204
|
+
* `subarray` views keeps the exact same per-list API and `byteLength`, and
|
|
205
|
+
* drops the header to a view without a buffer of its own.
|
|
206
|
+
*
|
|
207
|
+
* A later per-edit `push` that outgrows a list allocates a fresh private array
|
|
208
|
+
* for that token and stops referring to the arena. The vacated slice is not
|
|
209
|
+
* reclaimed while any sibling still points into the arena, so incremental
|
|
210
|
+
* churn can re-fragment what a build compacted. `refreshWordIndexIncrementally`
|
|
211
|
+
* recompacts after a bounded store-count threshold, using the cooperative
|
|
212
|
+
* variant below so a refresh does not monopolize the event loop.
|
|
213
|
+
*
|
|
214
|
+
* Measured on this repository's own tree, 2,699 documents and 2,272,686
|
|
215
|
+
* postings: a fresh build is 32.8 MB in ONE backing store; one full-corpus
|
|
216
|
+
* rewrite pass takes it to 39.9 MB across 37,687 stores (+22%); a second pass
|
|
217
|
+
* reaches 41.7 MB and the store count does not move, so the cost levels off
|
|
218
|
+
* rather than compounding. Even fully churned it stays well under the 151.6 MB
|
|
219
|
+
* a boxed build cost. Recompaction on a churn threshold is tracked in the
|
|
220
|
+
* follow-up issue; this comment is the sizing that decision needs.
|
|
221
|
+
*/
|
|
222
|
+
export function compactPostingsIntoArena(postings) {
|
|
223
|
+
let lanes = 0;
|
|
224
|
+
for (const list of postings.values())
|
|
225
|
+
lanes += list.length * 2;
|
|
226
|
+
if (lanes === 0)
|
|
227
|
+
return;
|
|
228
|
+
const arena = new Int32Array(lanes);
|
|
229
|
+
let offset = 0;
|
|
230
|
+
for (const list of postings.values())
|
|
231
|
+
offset = list.adoptArena(arena, offset);
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Cooperative counterpart for incremental refresh. It yields every work budget
|
|
235
|
+
* while copying the lists, so a large corpus does not monopolize the event
|
|
236
|
+
* loop, and it is safe against a concurrent synchronous edit landing during one
|
|
237
|
+
* of those yields.
|
|
238
|
+
*
|
|
239
|
+
* The arena is sized from a snapshot taken up front. A synchronous
|
|
240
|
+
* `updateWordIndexDocument` that runs during a yield can grow a not-yet-adopted
|
|
241
|
+
* list or replace it wholesale. The publish loop therefore adopts a list ONLY
|
|
242
|
+
* when it is still the map's current entry AND still the size the snapshot
|
|
243
|
+
* charged AND still fits the space that entry reserved. A list that changed is
|
|
244
|
+
* left on its own private store; the next recompaction packs it. This makes an
|
|
245
|
+
* out-of-bounds `adoptArena` write structurally impossible — the earlier code
|
|
246
|
+
* sized the arena once, then wrote each list's CURRENT width, so a grown list
|
|
247
|
+
* overran the arena and V8 silently dropped the tail postings (#2117 review F2).
|
|
248
|
+
*
|
|
249
|
+
* Each `adoptArena` copies and re-homes one list within a single synchronous
|
|
250
|
+
* step, so readers never observe a partially written list.
|
|
251
|
+
*/
|
|
252
|
+
export async function compactPostingsIntoArenaCooperatively(postings) {
|
|
253
|
+
// Snapshot (token, list, width) in one synchronous pass; the arena is sized
|
|
254
|
+
// from these widths and only unchanged lists are adopted, so the sum of the
|
|
255
|
+
// widths actually written can never exceed the arena length.
|
|
256
|
+
const planned = [];
|
|
257
|
+
let lanes = 0;
|
|
258
|
+
for (const [token, list] of postings) {
|
|
259
|
+
const width = list.length * 2;
|
|
260
|
+
planned.push({ token, list, width });
|
|
261
|
+
lanes += width;
|
|
262
|
+
}
|
|
263
|
+
if (lanes === 0)
|
|
264
|
+
return;
|
|
265
|
+
const arena = new Int32Array(lanes);
|
|
266
|
+
let offset = 0;
|
|
267
|
+
const deadline = createDeadline(8);
|
|
268
|
+
for (const plan of planned) {
|
|
269
|
+
if (postings.get(plan.token) === plan.list &&
|
|
270
|
+
plan.list.length * 2 === plan.width &&
|
|
271
|
+
offset + plan.width <= arena.length) {
|
|
272
|
+
offset = plan.list.adoptArena(arena, offset);
|
|
273
|
+
}
|
|
274
|
+
if (deadline.expired())
|
|
275
|
+
await yieldIfOverBudget(deadline);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* How many distinct backing stores the posting lists are spread across. One
|
|
280
|
+
* after a bulk build; it climbs by one for each token that has outgrown its
|
|
281
|
+
* arena slice since. Exists for the memory guard in
|
|
282
|
+
* `tests/clients/word-index-posting-memory.test.ts`.
|
|
283
|
+
*/
|
|
284
|
+
export function countPostingBackingStores(postings) {
|
|
285
|
+
const stores = new Set();
|
|
286
|
+
for (const list of postings.values())
|
|
287
|
+
stores.add(list.backingStore);
|
|
288
|
+
return stores.size;
|
|
289
|
+
}
|
|
290
|
+
/** Total posting entries across every token. O(distinct tokens). */
|
|
291
|
+
export function countPostingEntries(store) {
|
|
292
|
+
let count = 0;
|
|
293
|
+
for (const list of store.postings.values())
|
|
294
|
+
count += list.length;
|
|
295
|
+
return count;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Estimated resident bytes of the two packed stores — postings and the forward
|
|
299
|
+
* index — backing stores plus fixed per-list headers. O(distinct tokens +
|
|
300
|
+
* documents): no posting or forward ELEMENT is read, so this is safe on the
|
|
301
|
+
* memory-sample hot path.
|
|
302
|
+
*
|
|
303
|
+
* It deliberately excludes the token strings, the path-keyed metadata maps, and
|
|
304
|
+
* the `Map` spines, which together measured 2.5 MB against 63 MB of packed
|
|
305
|
+
* store on this repository's own corpus. Distinct backing stores are charged
|
|
306
|
+
* once, including abandoned arena slack. The figure remains a floor on the
|
|
307
|
+
* index's cost because object and map overhead stay unrepresented.
|
|
308
|
+
*/
|
|
309
|
+
export function estimateWordIndexStoreBytes(store) {
|
|
310
|
+
let bytes = 0;
|
|
311
|
+
const backingStores = new Set();
|
|
312
|
+
for (const list of store.postings.values()) {
|
|
313
|
+
backingStores.add(list.backingStore);
|
|
314
|
+
bytes += WORD_POSTING_LIST_OVERHEAD_BYTES;
|
|
315
|
+
}
|
|
316
|
+
for (const backingStore of backingStores)
|
|
317
|
+
bytes += backingStore.byteLength;
|
|
318
|
+
for (const entry of store.forward?.values() ?? []) {
|
|
319
|
+
bytes += entry.estimatedBytes;
|
|
320
|
+
}
|
|
321
|
+
return bytes;
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* One document's forward entry: token → distinct-line count, packed.
|
|
325
|
+
*
|
|
326
|
+
* The forward index mirrors what the postings hold for a file, so it has the
|
|
327
|
+
* same per-element shape and the same cost profile. On this repository's corpus
|
|
328
|
+
* it is 536,600 entries across 2,677 `Map<string, number>` instances, measured
|
|
329
|
+
* at 35.1 MB — larger than the packed posting store it mirrors, and enough on
|
|
330
|
+
* its own to keep the index above #2069's 40 MB ceiling. Packing it into a
|
|
331
|
+
* shared token-name array plus an `Int32Array` of counts costs the same eight
|
|
332
|
+
* bytes per entry as a posting, and the token strings are the SAME objects the
|
|
333
|
+
* postings map already keys on, so nothing is duplicated.
|
|
334
|
+
*
|
|
335
|
+
* Read-only by construction: a document's forward entry is built once from the
|
|
336
|
+
* tokenizer's tally and replaced wholesale on the next edit, which is also what
|
|
337
|
+
* lets an untouched document's entry keep its reference identity across an
|
|
338
|
+
* unrelated update.
|
|
339
|
+
*/
|
|
340
|
+
export class WordForwardEntry {
|
|
341
|
+
tokenNames;
|
|
342
|
+
lineCounts;
|
|
343
|
+
constructor(tokenNames, lineCounts) {
|
|
344
|
+
this.tokenNames = tokenNames;
|
|
345
|
+
this.lineCounts = lineCounts;
|
|
346
|
+
}
|
|
347
|
+
/** Distinct tokens recorded for this document. */
|
|
348
|
+
get size() {
|
|
349
|
+
return this.tokenNames.length;
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Distinct-line count for `token`, or `undefined`. Linear in the document's
|
|
353
|
+
* token count: the hot paths iterate this entry, they do not probe it, so a
|
|
354
|
+
* hash table would cost 40 bytes an entry to serve lookups nobody makes.
|
|
355
|
+
*/
|
|
356
|
+
get(token) {
|
|
357
|
+
const at = this.tokenNames.indexOf(token);
|
|
358
|
+
return at === -1 ? undefined : this.lineCounts[at];
|
|
359
|
+
}
|
|
360
|
+
keys() {
|
|
361
|
+
return this.tokenNames[Symbol.iterator]();
|
|
362
|
+
}
|
|
363
|
+
*entries() {
|
|
364
|
+
for (let i = 0; i < this.tokenNames.length; i += 1) {
|
|
365
|
+
yield [this.tokenNames[i], this.lineCounts[i]];
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
[Symbol.iterator]() {
|
|
369
|
+
return this.entries();
|
|
370
|
+
}
|
|
371
|
+
/** Estimated resident bytes: packed lanes plus fixed headers. */
|
|
372
|
+
get estimatedBytes() {
|
|
373
|
+
return (this.tokenNames.length * WORD_FORWARD_ENTRY_BYTES +
|
|
374
|
+
WORD_POSTING_LIST_OVERHEAD_BYTES);
|
|
375
|
+
}
|
|
376
|
+
/** Pack a tokenizer tally. Insertion order is preserved. */
|
|
377
|
+
static fromTally(tally) {
|
|
378
|
+
const tokenNames = new Array(tally.size);
|
|
379
|
+
const lineCounts = new Int32Array(tally.size);
|
|
380
|
+
let i = 0;
|
|
381
|
+
for (const [token, count] of tally) {
|
|
382
|
+
tokenNames[i] = token;
|
|
383
|
+
lineCounts[i] = count;
|
|
384
|
+
i += 1;
|
|
385
|
+
}
|
|
386
|
+
return new WordForwardEntry(tokenNames, lineCounts);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* Bytes per forward entry: one pointer-compressed slot for the shared token
|
|
391
|
+
* string and one `Int32` count lane.
|
|
392
|
+
*/
|
|
393
|
+
export const WORD_FORWARD_ENTRY_BYTES = 8;
|
|
394
|
+
/**
|
|
395
|
+
* The word index's dense file-id space: canonical path key → integer id →
|
|
396
|
+
* display path.
|
|
397
|
+
*
|
|
398
|
+
* Ids are recycled. A per-edit document replacement removes the document and
|
|
399
|
+
* re-adds it, so a table that only ever appended would grow one string slot per
|
|
400
|
+
* edit — bounded on entry count while growing on the id axis, the shape
|
|
401
|
+
* AGENTS.md catalogs at #9. {@link release} is only ever called after every
|
|
402
|
+
* posting naming that id has been removed (the forward index enumerates them),
|
|
403
|
+
* so a recycled id can never alias a surviving posting.
|
|
404
|
+
*/
|
|
405
|
+
export class WordIndexFileTable {
|
|
406
|
+
idByKey = new Map();
|
|
407
|
+
pathById = [];
|
|
408
|
+
freeIds = [];
|
|
409
|
+
/** Live file count. */
|
|
410
|
+
get size() {
|
|
411
|
+
return this.idByKey.size;
|
|
412
|
+
}
|
|
413
|
+
/** Width of the id space, live plus recycled. Bounded growth is the point. */
|
|
414
|
+
get idSpaceWidth() {
|
|
415
|
+
return this.pathById.length;
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Id for `key`, allocating one on first sight. First writer wins on the
|
|
419
|
+
* display path, matching the interning the boxed representation had: a
|
|
420
|
+
* second walk spelling of the same file reuses the first spelling's id and
|
|
421
|
+
* does not re-write the stored display path.
|
|
422
|
+
*/
|
|
423
|
+
intern(key, displayPath) {
|
|
424
|
+
const existing = this.idByKey.get(key);
|
|
425
|
+
if (existing !== undefined)
|
|
426
|
+
return existing;
|
|
427
|
+
const id = this.freeIds.pop() ?? this.pathById.length;
|
|
428
|
+
this.pathById[id] = displayPath;
|
|
429
|
+
this.idByKey.set(key, id);
|
|
430
|
+
return id;
|
|
431
|
+
}
|
|
432
|
+
idFor(key) {
|
|
433
|
+
return this.idByKey.get(key);
|
|
434
|
+
}
|
|
435
|
+
pathFor(id) {
|
|
436
|
+
return this.pathById[id];
|
|
437
|
+
}
|
|
438
|
+
/** Drop `key` and recycle its id. Returns the freed id, or `undefined`. */
|
|
439
|
+
release(key) {
|
|
440
|
+
const id = this.idByKey.get(key);
|
|
441
|
+
if (id === undefined)
|
|
442
|
+
return undefined;
|
|
443
|
+
this.idByKey.delete(key);
|
|
444
|
+
this.pathById[id] = undefined;
|
|
445
|
+
this.freeIds.push(id);
|
|
446
|
+
return id;
|
|
447
|
+
}
|
|
448
|
+
}
|