cgraphx 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/code/engine/cli/analyze-config.js +320 -0
- package/dist/core/code/engine/cli/analyze.js +1086 -0
- package/dist/core/code/engine/cli/cli-message.js +83 -0
- package/dist/core/code/engine/cli/detect-changes-format.js +58 -0
- package/dist/core/code/engine/cli/embedding-dims.js +43 -0
- package/dist/core/code/engine/cli/format-elapsed.js +12 -0
- package/dist/core/code/engine/cli/help-i18n.js +144 -0
- package/dist/core/code/engine/cli/i18n/en.js +110 -0
- package/dist/core/code/engine/cli/i18n/index.js +47 -0
- package/dist/core/code/engine/cli/i18n/resources.js +9 -0
- package/dist/core/code/engine/cli/i18n/zh-CN.js +110 -0
- package/dist/core/code/engine/cli/lazy-action.js +67 -0
- package/dist/core/code/engine/cli/optional-grammars.js +136 -0
- package/dist/core/code/engine/cli/resolve-invocation.js +76 -0
- package/dist/core/code/engine/cli/status.js +156 -0
- package/dist/core/code/engine/cli/tool.js +384 -0
- package/dist/core/code/engine/config/ignore-service.js +511 -0
- package/dist/core/code/engine/config/supported-languages.js +17 -0
- package/dist/core/code/engine/core/analysis-features.js +64 -0
- package/dist/core/code/engine/core/analyzer-identity.js +2171 -0
- package/dist/core/code/engine/core/git-staleness.js +180 -0
- package/dist/core/code/engine/core/graph/graph.js +180 -0
- package/dist/core/code/engine/core/graph/import-cycles.js +106 -0
- package/dist/core/code/engine/core/graph/types.js +2 -0
- package/dist/core/code/engine/core/index-freshness.js +12 -0
- package/dist/core/code/engine/core/ingestion/binding-accumulator.js +341 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/c-cpp.js +168 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/csharp.js +9 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/dart.js +8 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/go.js +8 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/jvm.js +54 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/php.js +8 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/python.js +8 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/ruby.js +8 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/rust.js +8 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/swift.js +8 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/configs/typescript-javascript.js +11 -0
- package/dist/core/code/engine/core/ingestion/call-extractors/generic.js +62 -0
- package/dist/core/code/engine/core/ingestion/call-processor.js +503 -0
- package/dist/core/code/engine/core/ingestion/call-routing.js +98 -0
- package/dist/core/code/engine/core/ingestion/call-types.js +3 -0
- package/dist/core/code/engine/core/ingestion/cfg/callee-cell-format.js +45 -0
- package/dist/core/code/engine/core/ingestion/cfg/cfg-builder.js +202 -0
- package/dist/core/code/engine/core/ingestion/cfg/collect.js +81 -0
- package/dist/core/code/engine/core/ingestion/cfg/control-dependence.js +185 -0
- package/dist/core/code/engine/core/ingestion/cfg/control-flow-context.js +130 -0
- package/dist/core/code/engine/core/ingestion/cfg/emit.js +646 -0
- package/dist/core/code/engine/core/ingestion/cfg/post-dominators.js +182 -0
- package/dist/core/code/engine/core/ingestion/cfg/reaching-def-reason-codec.js +139 -0
- package/dist/core/code/engine/core/ingestion/cfg/reaching-defs-graph.js +322 -0
- package/dist/core/code/engine/core/ingestion/cfg/reaching-defs.js +792 -0
- package/dist/core/code/engine/core/ingestion/cfg/synthetic-escape.js +305 -0
- package/dist/core/code/engine/core/ingestion/cfg/traversal-result.js +6 -0
- package/dist/core/code/engine/core/ingestion/cfg/types.js +14 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/c-cpp-harvest.js +545 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/c-cpp.js +590 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/call-site-harvest.js +356 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/csharp-harvest.js +593 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/csharp.js +871 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/dart-harvest.js +874 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/dart.js +840 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/go-harvest.js +625 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/go.js +642 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/java-harvest.js +517 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/java.js +816 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/kotlin-harvest.js +723 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/kotlin.js +813 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/php-harvest.js +630 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/php.js +725 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/python-harvest.js +776 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/python.js +562 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/ruby-harvest.js +591 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/ruby.js +760 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/rust-harvest.js +877 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/rust.js +562 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/scope-tree-harvest.js +120 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/swift-harvest.js +683 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/swift.js +791 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/typescript-harvest.js +1060 -0
- package/dist/core/code/engine/core/ingestion/cfg/visitors/typescript.js +587 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/c-cpp.js +77 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/csharp.js +24 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/dart.js +10 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/go.js +28 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/jvm.js +67 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/php.js +10 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/python.js +10 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/ruby.js +13 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/rust.js +10 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/swift.js +21 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/configs/typescript-javascript.js +31 -0
- package/dist/core/code/engine/core/ingestion/class-extractors/generic.js +144 -0
- package/dist/core/code/engine/core/ingestion/class-types.js +2 -0
- package/dist/core/code/engine/core/ingestion/cluster-enricher.js +174 -0
- package/dist/core/code/engine/core/ingestion/community-processor.js +604 -0
- package/dist/core/code/engine/core/ingestion/constants.js +26 -0
- package/dist/core/code/engine/core/ingestion/cpp-ue-preprocessor.js +263 -0
- package/dist/core/code/engine/core/ingestion/csharp-namespace-gate.js +133 -0
- package/dist/core/code/engine/core/ingestion/di-extractors/index.js +38 -0
- package/dist/core/code/engine/core/ingestion/di-extractors/spring.js +310 -0
- package/dist/core/code/engine/core/ingestion/emit-references.js +244 -0
- package/dist/core/code/engine/core/ingestion/entry-point-scoring.js +201 -0
- package/dist/core/code/engine/core/ingestion/export-detection.js +244 -0
- package/dist/core/code/engine/core/ingestion/field-extractor.js +29 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/c-cpp.js +107 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/csharp.js +124 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/dart.js +99 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/go.js +102 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/helpers.js +198 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/jvm.js +172 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/php.js +67 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/python.js +94 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/ruby.js +79 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/rust.js +55 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/swift.js +93 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/configs/typescript-javascript.js +59 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/generic.js +147 -0
- package/dist/core/code/engine/core/ingestion/field-extractors/typescript.js +266 -0
- package/dist/core/code/engine/core/ingestion/field-types.js +3 -0
- package/dist/core/code/engine/core/ingestion/filesystem-walker.js +136 -0
- package/dist/core/code/engine/core/ingestion/finalize-orchestrator.js +159 -0
- package/dist/core/code/engine/core/ingestion/framework-detection.js +432 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/analysis-features.js +38 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/annotation-arguments.js +234 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/aop-candidates.js +88 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/aop.js +487 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/auto-configuration.js +21 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/bean-candidates.js +189 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/bean-catalog.js +32 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/bean-factories.js +73 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/conditionals.js +323 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/config-bindings.js +119 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/di-metadata.js +385 -0
- package/dist/core/code/engine/core/ingestion/frameworks/spring/resource-injection.js +96 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/c-cpp.js +17 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/csharp.js +46 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/dart.js +59 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/go.js +30 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/jvm.js +73 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/php.js +19 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/python.js +45 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/ruby.js +20 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/rust.js +58 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/swift.js +94 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/configs/typescript-javascript.js +26 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/csharp.js +128 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/go.js +50 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/jvm.js +112 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/php.js +80 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/python.js +75 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/resolver-factory.js +36 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/ruby.js +20 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/rust.js +79 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/standard.js +180 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/types.js +7 -0
- package/dist/core/code/engine/core/ingestion/import-resolvers/utils.js +153 -0
- package/dist/core/code/engine/core/ingestion/import-target-adapter.js +99 -0
- package/dist/core/code/engine/core/ingestion/language-config.js +391 -0
- package/dist/core/code/engine/core/ingestion/language-provider.js +25 -0
- package/dist/core/code/engine/core/ingestion/languages/c/arity-metadata.js +98 -0
- package/dist/core/code/engine/core/ingestion/languages/c/arity.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/c/capture-side-channel.js +69 -0
- package/dist/core/code/engine/core/ingestion/languages/c/captures.js +189 -0
- package/dist/core/code/engine/core/ingestion/languages/c/header-scan.js +58 -0
- package/dist/core/code/engine/core/ingestion/languages/c/import-decomposer.js +68 -0
- package/dist/core/code/engine/core/ingestion/languages/c/import-target.js +103 -0
- package/dist/core/code/engine/core/ingestion/languages/c/index.js +33 -0
- package/dist/core/code/engine/core/ingestion/languages/c/interpret.js +53 -0
- package/dist/core/code/engine/core/ingestion/languages/c/merge-bindings.js +26 -0
- package/dist/core/code/engine/core/ingestion/languages/c/query.js +210 -0
- package/dist/core/code/engine/core/ingestion/languages/c/scope-resolver.js +112 -0
- package/dist/core/code/engine/core/ingestion/languages/c/simple-hooks.js +24 -0
- package/dist/core/code/engine/core/ingestion/languages/c/static-linkage.js +109 -0
- package/dist/core/code/engine/core/ingestion/languages/c-cpp.js +506 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/adl.js +804 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/arity-metadata.js +258 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/arity.js +37 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/capture-side-channel.js +89 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/captures.js +1975 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/constraint-extractor.js +311 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/constraint-filter.js +210 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/conversion-rank.js +163 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/file-local-linkage.js +327 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/header-scan.js +53 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/import-decomposer.js +134 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/import-target.js +16 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/index.js +33 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/inline-namespaces.js +379 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/interpret.js +239 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/member-lookup.js +470 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/merge-bindings.js +32 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/query.js +763 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/range-bindings.js +230 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/scope-resolver.js +354 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/simple-hooks.js +67 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/two-phase-lookup.js +348 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/type-classifier.js +56 -0
- package/dist/core/code/engine/core/ingestion/languages/cpp/user-defined-conversions.js +128 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/accessor-unwrap.js +67 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/arity-metadata.js +49 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/arity.js +40 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/cache-stats.js +32 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/captures.js +557 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/import-decomposer.js +96 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/import-target.js +176 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/index.js +95 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/interpret.js +150 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/merge-bindings.js +58 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/namespace-siblings.js +708 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/qualified-type-names.js +62 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/query.js +578 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/receiver-binding.js +142 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/resolution-config.js +18 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/scope-resolver.js +84 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp/simple-hooks.js +81 -0
- package/dist/core/code/engine/core/ingestion/languages/csharp.js +204 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/arity-metadata.js +38 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/arity.js +34 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/built-ins.js +37 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/cache-stats.js +30 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/captures.js +1096 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/expand-wildcards.js +34 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/extension-type-preprocess.js +33 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/import-target.js +68 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/index.js +45 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/interpret.js +101 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/merge-bindings.js +42 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/query.js +246 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/receiver-binding.js +90 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/scope-resolver.js +197 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/signature-bindings.js +54 -0
- package/dist/core/code/engine/core/ingestion/languages/dart/simple-hooks.js +61 -0
- package/dist/core/code/engine/core/ingestion/languages/dart.js +138 -0
- package/dist/core/code/engine/core/ingestion/languages/go/arity-metadata.js +71 -0
- package/dist/core/code/engine/core/ingestion/languages/go/arity.js +17 -0
- package/dist/core/code/engine/core/ingestion/languages/go/cache-stats.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/go/captures.js +492 -0
- package/dist/core/code/engine/core/ingestion/languages/go/expand-wildcards.js +97 -0
- package/dist/core/code/engine/core/ingestion/languages/go/generic-type-parameters.js +146 -0
- package/dist/core/code/engine/core/ingestion/languages/go/import-decomposer.js +47 -0
- package/dist/core/code/engine/core/ingestion/languages/go/import-target.js +70 -0
- package/dist/core/code/engine/core/ingestion/languages/go/index.js +39 -0
- package/dist/core/code/engine/core/ingestion/languages/go/interface-impls.js +955 -0
- package/dist/core/code/engine/core/ingestion/languages/go/interpret.js +177 -0
- package/dist/core/code/engine/core/ingestion/languages/go/merge-bindings.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/go/method-owners.js +131 -0
- package/dist/core/code/engine/core/ingestion/languages/go/namespace-mirror.js +56 -0
- package/dist/core/code/engine/core/ingestion/languages/go/package-clause.js +79 -0
- package/dist/core/code/engine/core/ingestion/languages/go/package-siblings.js +83 -0
- package/dist/core/code/engine/core/ingestion/languages/go/query.js +298 -0
- package/dist/core/code/engine/core/ingestion/languages/go/range-binding.js +127 -0
- package/dist/core/code/engine/core/ingestion/languages/go/receiver-binding.js +24 -0
- package/dist/core/code/engine/core/ingestion/languages/go/scope-resolver.js +75 -0
- package/dist/core/code/engine/core/ingestion/languages/go/simple-hooks.js +31 -0
- package/dist/core/code/engine/core/ingestion/languages/go/type-binding.js +279 -0
- package/dist/core/code/engine/core/ingestion/languages/go.js +160 -0
- package/dist/core/code/engine/core/ingestion/languages/index.js +66 -0
- package/dist/core/code/engine/core/ingestion/languages/java/analysis-features.js +16 -0
- package/dist/core/code/engine/core/ingestion/languages/java/arity-metadata.js +43 -0
- package/dist/core/code/engine/core/ingestion/languages/java/arity.js +27 -0
- package/dist/core/code/engine/core/ingestion/languages/java/cache-stats.js +32 -0
- package/dist/core/code/engine/core/ingestion/languages/java/capture-side-channel.js +123 -0
- package/dist/core/code/engine/core/ingestion/languages/java/captures.js +791 -0
- package/dist/core/code/engine/core/ingestion/languages/java/import-decomposer.js +88 -0
- package/dist/core/code/engine/core/ingestion/languages/java/import-target.js +103 -0
- package/dist/core/code/engine/core/ingestion/languages/java/index.js +43 -0
- package/dist/core/code/engine/core/ingestion/languages/java/interpret.js +146 -0
- package/dist/core/code/engine/core/ingestion/languages/java/merge-bindings.js +43 -0
- package/dist/core/code/engine/core/ingestion/languages/java/package-facts.js +16 -0
- package/dist/core/code/engine/core/ingestion/languages/java/package-siblings.js +11 -0
- package/dist/core/code/engine/core/ingestion/languages/java/query.js +319 -0
- package/dist/core/code/engine/core/ingestion/languages/java/receiver-binding.js +98 -0
- package/dist/core/code/engine/core/ingestion/languages/java/scope-resolver.js +213 -0
- package/dist/core/code/engine/core/ingestion/languages/java/simple-hooks.js +39 -0
- package/dist/core/code/engine/core/ingestion/languages/java/spring-aop.js +53 -0
- package/dist/core/code/engine/core/ingestion/languages/java/spring-bean-metadata.js +11 -0
- package/dist/core/code/engine/core/ingestion/languages/java/spring-conditionals.js +52 -0
- package/dist/core/code/engine/core/ingestion/languages/java/spring-config-bindings.js +222 -0
- package/dist/core/code/engine/core/ingestion/languages/java/spring-di.js +165 -0
- package/dist/core/code/engine/core/ingestion/languages/java.js +192 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/arity.js +15 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/captures.js +1122 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/import-target.js +56 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/index.js +109 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/interpret.js +45 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/merge-bindings.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/query.js +659 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/scope-resolver.js +78 -0
- package/dist/core/code/engine/core/ingestion/languages/javascript/simple-hooks.js +44 -0
- package/dist/core/code/engine/core/ingestion/languages/jvm/package-facts.js +46 -0
- package/dist/core/code/engine/core/ingestion/languages/jvm/package-siblings.js +200 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/arity-metadata.js +23 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/arity.js +18 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/cache-stats.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/capture-side-channel.js +160 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/captures.js +1262 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/companion-scopes.js +72 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/import-decomposer.js +40 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/import-target.js +135 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/index.js +26 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/interpret.js +75 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/merge-bindings.js +28 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/owners.js +134 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/package-facts.js +16 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/package-siblings.js +11 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/query.js +243 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/receiver-binding.js +103 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/scope-resolver.js +207 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/simple-hooks.js +42 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/spring-aop.js +68 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/spring-bean-metadata.js +11 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/spring-conditionals.js +53 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin/spring-di.js +304 -0
- package/dist/core/code/engine/core/ingestion/languages/kotlin.js +189 -0
- package/dist/core/code/engine/core/ingestion/languages/php/arity-metadata.js +66 -0
- package/dist/core/code/engine/core/ingestion/languages/php/arity.js +43 -0
- package/dist/core/code/engine/core/ingestion/languages/php/cache-stats.js +32 -0
- package/dist/core/code/engine/core/ingestion/languages/php/captures.js +1166 -0
- package/dist/core/code/engine/core/ingestion/languages/php/import-decomposer.js +238 -0
- package/dist/core/code/engine/core/ingestion/languages/php/import-target.js +213 -0
- package/dist/core/code/engine/core/ingestion/languages/php/index.js +85 -0
- package/dist/core/code/engine/core/ingestion/languages/php/interpret.js +256 -0
- package/dist/core/code/engine/core/ingestion/languages/php/merge-bindings.js +50 -0
- package/dist/core/code/engine/core/ingestion/languages/php/namespace-siblings.js +353 -0
- package/dist/core/code/engine/core/ingestion/languages/php/query.js +391 -0
- package/dist/core/code/engine/core/ingestion/languages/php/receiver-binding.js +135 -0
- package/dist/core/code/engine/core/ingestion/languages/php/scope-resolver.js +361 -0
- package/dist/core/code/engine/core/ingestion/languages/php/simple-hooks.js +116 -0
- package/dist/core/code/engine/core/ingestion/languages/php.js +302 -0
- package/dist/core/code/engine/core/ingestion/languages/python/arity-metadata.js +49 -0
- package/dist/core/code/engine/core/ingestion/languages/python/arity.js +41 -0
- package/dist/core/code/engine/core/ingestion/languages/python/cache-stats.js +34 -0
- package/dist/core/code/engine/core/ingestion/languages/python/captures.js +299 -0
- package/dist/core/code/engine/core/ingestion/languages/python/depends-references.js +68 -0
- package/dist/core/code/engine/core/ingestion/languages/python/import-decomposer.js +115 -0
- package/dist/core/code/engine/core/ingestion/languages/python/import-target.js +440 -0
- package/dist/core/code/engine/core/ingestion/languages/python/index-stats.js +30 -0
- package/dist/core/code/engine/core/ingestion/languages/python/index.js +96 -0
- package/dist/core/code/engine/core/ingestion/languages/python/interpret.js +430 -0
- package/dist/core/code/engine/core/ingestion/languages/python/merge-bindings.js +47 -0
- package/dist/core/code/engine/core/ingestion/languages/python/query.js +323 -0
- package/dist/core/code/engine/core/ingestion/languages/python/receiver-binding.js +310 -0
- package/dist/core/code/engine/core/ingestion/languages/python/scope-resolver.js +80 -0
- package/dist/core/code/engine/core/ingestion/languages/python/simple-hooks.js +53 -0
- package/dist/core/code/engine/core/ingestion/languages/python.js +140 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/arity.js +41 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/cache-stats.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/captures.js +864 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/import-target.js +88 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/index.js +29 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/interpret.js +115 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/merge-bindings.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/query.js +349 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/receiver-binding.js +70 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/scope-resolver.js +263 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby/simple-hooks.js +68 -0
- package/dist/core/code/engine/core/ingestion/languages/ruby.js +215 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/arity.js +16 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/cache-stats.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/captures.js +304 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/import-decomposer.js +167 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/import-target.js +108 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/index.js +29 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/interpret.js +201 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/merge-bindings.js +21 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/method-owners.js +76 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/module-path.js +222 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/qualified-call.js +482 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/query.js +280 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/range-binding.js +687 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/receiver-binding.js +148 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/scope-resolver.js +151 -0
- package/dist/core/code/engine/core/ingestion/languages/rust/simple-hooks.js +32 -0
- package/dist/core/code/engine/core/ingestion/languages/rust.js +183 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/arity-metadata.js +44 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/arity.js +45 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/base-type.js +30 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/cache-stats.js +32 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/captures.js +594 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/conditional-directive-preprocess.js +256 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/implicit-imports.js +60 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/import-decomposer.js +87 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/import-target.js +84 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/index.js +56 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/interpret.js +93 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/merge-bindings.js +51 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/query.js +226 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/receiver-binding.js +169 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/scope-resolver.js +192 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/sibling-type-bindings.js +68 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/signature-bindings.js +69 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/simple-hooks.js +65 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/target-grouping.js +97 -0
- package/dist/core/code/engine/core/ingestion/languages/swift/target-siblings.js +74 -0
- package/dist/core/code/engine/core/ingestion/languages/swift.js +246 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/arity-metadata.js +106 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/arity.js +57 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/array-callback.js +58 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/cache-stats.js +34 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/captures.js +956 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/cjs-export-assignment.js +535 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/cjs-module-exports.js +196 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/import-decomposer.js +374 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/import-target.js +65 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/index.js +108 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/interpret.js +344 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/merge-bindings.js +161 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/nuxt-auto-imports.js +325 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/query.js +1328 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/receiver-binding.js +201 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/scope-resolver.js +293 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript/simple-hooks.js +139 -0
- package/dist/core/code/engine/core/ingestion/languages/typescript.js +455 -0
- package/dist/core/code/engine/core/ingestion/languages/vue/captures.js +70 -0
- package/dist/core/code/engine/core/ingestion/languages/vue/import-target.js +61 -0
- package/dist/core/code/engine/core/ingestion/languages/vue/index.js +55 -0
- package/dist/core/code/engine/core/ingestion/languages/vue/scope-resolver.js +295 -0
- package/dist/core/code/engine/core/ingestion/languages/vue.js +96 -0
- package/dist/core/code/engine/core/ingestion/local-symbol-pruner.js +68 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/c-cpp.js +387 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/csharp.js +290 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/dart.js +392 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/go.js +179 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/jvm.js +350 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/php.js +306 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/python.js +312 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/ruby.js +289 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/rust.js +198 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/swift.js +286 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/configs/typescript-javascript.js +341 -0
- package/dist/core/code/engine/core/ingestion/method-extractors/generic.js +209 -0
- package/dist/core/code/engine/core/ingestion/method-types.js +3 -0
- package/dist/core/code/engine/core/ingestion/model/field-registry.js +41 -0
- package/dist/core/code/engine/core/ingestion/model/index.js +52 -0
- package/dist/core/code/engine/core/ingestion/model/method-registry.js +138 -0
- package/dist/core/code/engine/core/ingestion/model/owned-members-lookup.js +46 -0
- package/dist/core/code/engine/core/ingestion/model/registration-table.js +234 -0
- package/dist/core/code/engine/core/ingestion/model/resolve.js +183 -0
- package/dist/core/code/engine/core/ingestion/model/scope-resolution-indexes.js +43 -0
- package/dist/core/code/engine/core/ingestion/model/semantic-model.js +179 -0
- package/dist/core/code/engine/core/ingestion/model/symbol-table.js +216 -0
- package/dist/core/code/engine/core/ingestion/model/type-registry.js +84 -0
- package/dist/core/code/engine/core/ingestion/mro-processor.js +709 -0
- package/dist/core/code/engine/core/ingestion/parsing-processor.js +312 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/communities.js +69 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/cross-file.js +71 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/di.js +338 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/http-api-calls.js +312 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/index.js +54 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/mro.js +40 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/orm.js +78 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/parse-impl.js +1286 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/parse.js +41 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/processes.js +193 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/prune-local-symbols.js +29 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/registry.js +52 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/routes.js +409 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/rpc-edges.js +301 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/runner.js +207 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/scan.js +49 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/spring-aop.js +442 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/spring-auto-configuration.js +264 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/spring-config.js +440 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/structure.js +38 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/tools.js +89 -0
- package/dist/core/code/engine/core/ingestion/pipeline-phases/types.js +40 -0
- package/dist/core/code/engine/core/ingestion/pipeline.js +152 -0
- package/dist/core/code/engine/core/ingestion/process-processor.js +325 -0
- package/dist/core/code/engine/core/ingestion/resolve-references.js +205 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/constant-resolver.js +135 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/django-root-discovery.js +221 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/django.js +428 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/expo.js +39 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/fastapi-router-bindings.js +264 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/laravel.js +501 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/middleware.js +175 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/nextjs.js +81 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/php.js +25 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/python-const-resolver.js +307 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/response-shapes.js +299 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/route-path.js +71 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/spring-shared.js +310 -0
- package/dist/core/code/engine/core/ingestion/route-extractors/spring.js +441 -0
- package/dist/core/code/engine/core/ingestion/scope-extractor-bridge.js +60 -0
- package/dist/core/code/engine/core/ingestion/scope-extractor.js +1373 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/contract/scope-resolver.js +282 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/graph-bridge/callee-id-sink.js +72 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/graph-bridge/edges.js +194 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/graph-bridge/ids.js +472 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/graph-bridge/imports-to-edges.js +49 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/graph-bridge/method-dispatch.js +43 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/graph-bridge/node-lookup.js +274 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/graph-bridge/references-to-edges.js +93 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/callable-value-flow.js +1240 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/compound-receiver.js +1174 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/free-call-fallback.js +873 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/imported-return-types.js +226 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/mro.js +107 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/overload-narrowing.js +441 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/property-dispatch.js +122 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/passes/receiver-bound-calls.js +1720 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/pipeline/phase.js +395 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/pipeline/reconcile-ownership.js +208 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/pipeline/registry.js +49 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/pipeline/run.js +632 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/pipeline/validate-bindings-immutability.js +112 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/resolution-outcome.js +41 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/scope/namespace-targets.js +81 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/scope/walkers.js +1835 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/unresolved-receivers.js +242 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/utils/definition-id.js +22 -0
- package/dist/core/code/engine/core/ingestion/scope-resolution/workspace-index.js +151 -0
- package/dist/core/code/engine/core/ingestion/structure-processor.js +40 -0
- package/dist/core/code/engine/core/ingestion/tree-sitter-queries.js +2244 -0
- package/dist/core/code/engine/core/ingestion/ts-js-hoc-utils.js +115 -0
- package/dist/core/code/engine/core/ingestion/type-env.js +1136 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/c-cpp.js +555 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/csharp.js +570 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/dart.js +372 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/go.js +508 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/jvm.js +875 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/php.js +537 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/python.js +477 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/ruby.js +380 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/rust.js +502 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/shared.js +843 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/swift.js +490 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/types.js +2 -0
- package/dist/core/code/engine/core/ingestion/type-extractors/typescript.js +690 -0
- package/dist/core/code/engine/core/ingestion/utils/ast-helpers.js +1693 -0
- package/dist/core/code/engine/core/ingestion/utils/call-analysis.js +779 -0
- package/dist/core/code/engine/core/ingestion/utils/callable-flow-captures.js +932 -0
- package/dist/core/code/engine/core/ingestion/utils/callable-labels.js +49 -0
- package/dist/core/code/engine/core/ingestion/utils/deferred-resolution-profile.js +151 -0
- package/dist/core/code/engine/core/ingestion/utils/effective-ram.js +67 -0
- package/dist/core/code/engine/core/ingestion/utils/env.js +60 -0
- package/dist/core/code/engine/core/ingestion/utils/event-loop.js +9 -0
- package/dist/core/code/engine/core/ingestion/utils/graph-sort.js +103 -0
- package/dist/core/code/engine/core/ingestion/utils/heap-probe.js +45 -0
- package/dist/core/code/engine/core/ingestion/utils/heritage-marker.js +47 -0
- package/dist/core/code/engine/core/ingestion/utils/line-base.js +24 -0
- package/dist/core/code/engine/core/ingestion/utils/max-file-size.js +59 -0
- package/dist/core/code/engine/core/ingestion/utils/method-props.js +198 -0
- package/dist/core/code/engine/core/ingestion/utils/qualified-name.js +73 -0
- package/dist/core/code/engine/core/ingestion/utils/receiver-chain-captures.js +60 -0
- package/dist/core/code/engine/core/ingestion/utils/receiver-chain-codec.js +188 -0
- package/dist/core/code/engine/core/ingestion/utils/scope-tree-walk.js +36 -0
- package/dist/core/code/engine/core/ingestion/utils/symbol-labels.js +48 -0
- package/dist/core/code/engine/core/ingestion/utils/template-arguments.js +187 -0
- package/dist/core/code/engine/core/ingestion/utils/type-parameters.js +209 -0
- package/dist/core/code/engine/core/ingestion/utils/verbose.js +6 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/c-cpp.js +133 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/csharp.js +66 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/dart.js +111 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/go.js +153 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/jvm.js +145 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/php.js +61 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/python.js +104 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/ruby.js +55 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/rust.js +79 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/swift.js +91 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/configs/typescript-javascript.js +86 -0
- package/dist/core/code/engine/core/ingestion/variable-extractors/generic.js +111 -0
- package/dist/core/code/engine/core/ingestion/variable-types.js +3 -0
- package/dist/core/code/engine/core/ingestion/vue-sfc-extractor.js +544 -0
- package/dist/core/code/engine/core/ingestion/workers/callable-id.js +122 -0
- package/dist/core/code/engine/core/ingestion/workers/clone-safety.js +470 -0
- package/dist/core/code/engine/core/ingestion/workers/parse-worker.js +2456 -0
- package/dist/core/code/engine/core/ingestion/workers/post-result.js +90 -0
- package/dist/core/code/engine/core/ingestion/workers/quarantine.js +41 -0
- package/dist/core/code/engine/core/ingestion/workers/result-merge.js +59 -0
- package/dist/core/code/engine/core/ingestion/workers/worker-pool.js +1725 -0
- package/dist/core/code/engine/core/ingestion/workspace-config.js +139 -0
- package/dist/core/code/engine/core/lbug/conn-lock.js +72 -0
- package/dist/core/code/engine/core/lbug/csv-generator.js +710 -0
- package/dist/core/code/engine/core/lbug/cypher-escape.js +24 -0
- package/dist/core/code/engine/core/lbug/extension-load-error.js +339 -0
- package/dist/core/code/engine/core/lbug/extension-loader.js +261 -0
- package/dist/core/code/engine/core/lbug/graph-emit-sink.js +584 -0
- package/dist/core/code/engine/core/lbug/lbug-adapter.js +2703 -0
- package/dist/core/code/engine/core/lbug/lbug-config.js +1029 -0
- package/dist/core/code/engine/core/lbug/native-check.js +435 -0
- package/dist/core/code/engine/core/lbug/pool-adapter.js +973 -0
- package/dist/core/code/engine/core/lbug/query-params.js +25 -0
- package/dist/core/code/engine/core/lbug/query-result-utils.js +31 -0
- package/dist/core/code/engine/core/lbug/rel-pair-routing.js +373 -0
- package/dist/core/code/engine/core/lbug/schema.js +771 -0
- package/dist/core/code/engine/core/lbug/shutdown-helpers.js +40 -0
- package/dist/core/code/engine/core/lbug/sidecar-recovery.js +716 -0
- package/dist/core/code/engine/core/lbug/stdio-capture.js +49 -0
- package/dist/core/code/engine/core/lbug/sync-csv-writer.js +115 -0
- package/dist/core/code/engine/core/lbug/wal-checkpoint-driver.js +215 -0
- package/dist/core/code/engine/core/lbug/wal-driver-state.js +28 -0
- package/dist/core/code/engine/core/logger.js +339 -0
- package/dist/core/code/engine/core/platform/capabilities.js +91 -0
- package/dist/core/code/engine/core/run-analyze.js +1098 -0
- package/dist/core/code/engine/core/tree-sitter/parser-loader.js +281 -0
- package/dist/core/code/engine/core/tree-sitter/safe-parse.js +258 -0
- package/dist/core/code/engine/core/tree-sitter/vendored-grammars.js +64 -0
- package/dist/core/code/engine/lib/utils.js +121 -0
- package/dist/core/code/engine/mcp/core/lbug-adapter.js +27 -0
- package/dist/core/code/engine/mcp/local/aop-metadata.js +230 -0
- package/dist/core/code/engine/mcp/local/bean-metadata.js +49 -0
- package/dist/core/code/engine/mcp/local/limits.js +15 -0
- package/dist/core/code/engine/mcp/local/line-display.js +6 -0
- package/dist/core/code/engine/mcp/local/local-backend.js +4142 -0
- package/dist/core/code/engine/storage/branch-index.js +72 -0
- package/dist/core/code/engine/storage/file-hash.js +95 -0
- package/dist/core/code/engine/storage/fs-atomic.js +34 -0
- package/dist/core/code/engine/storage/git.js +555 -0
- package/dist/core/code/engine/storage/index-lock.js +664 -0
- package/dist/core/code/engine/storage/parse-cache.js +650 -0
- package/dist/core/code/engine/storage/parsedfile-store.js +620 -0
- package/dist/core/code/engine/storage/repo-manager.js +1061 -0
- package/dist/core/code/engine/storage/scope-index-store.js +247 -0
- package/dist/core/code/engine/types/pipeline.js +2 -0
- package/dist/core/code/scripts/install-duckdb-extension.mjs +125 -0
- package/dist/core/code/scripts/resolve-analyze-cmd.cjs +346 -0
- package/dist/core/code/shared/graph/types.js +8 -0
- package/dist/core/code/shared/index.js +105 -0
- package/dist/core/code/shared/integrations/circuit-breaker.js +242 -0
- package/dist/core/code/shared/integrations/resilient-fetch.js +224 -0
- package/dist/core/code/shared/integrations/retry.js +70 -0
- package/dist/core/code/shared/integrations/understand-quickly.js +145 -0
- package/dist/core/code/shared/language-detection.js +162 -0
- package/dist/core/code/shared/languages.js +27 -0
- package/dist/core/code/shared/lbug/schema-constants.js +98 -0
- package/dist/core/code/shared/mro-strategy.js +2 -0
- package/dist/core/code/shared/pipeline.js +5 -0
- package/dist/core/code/shared/scope-resolution/callable-flow-site.js +11 -0
- package/dist/core/code/shared/scope-resolution/def-index.js +53 -0
- package/dist/core/code/shared/scope-resolution/evidence-weights.js +87 -0
- package/dist/core/code/shared/scope-resolution/finalize-algorithm.js +807 -0
- package/dist/core/code/shared/scope-resolution/language-classification.js +46 -0
- package/dist/core/code/shared/scope-resolution/method-dispatch-index.js +100 -0
- package/dist/core/code/shared/scope-resolution/module-scope-index.js +59 -0
- package/dist/core/code/shared/scope-resolution/origin-priority.js +23 -0
- package/dist/core/code/shared/scope-resolution/parsed-file.js +54 -0
- package/dist/core/code/shared/scope-resolution/position-index.js +136 -0
- package/dist/core/code/shared/scope-resolution/qualified-name-index.js +77 -0
- package/dist/core/code/shared/scope-resolution/reference-site.js +24 -0
- package/dist/core/code/shared/scope-resolution/registries/class-registry.js +32 -0
- package/dist/core/code/shared/scope-resolution/registries/context.js +52 -0
- package/dist/core/code/shared/scope-resolution/registries/evidence.js +152 -0
- package/dist/core/code/shared/scope-resolution/registries/field-registry.js +33 -0
- package/dist/core/code/shared/scope-resolution/registries/lookup-core.js +392 -0
- package/dist/core/code/shared/scope-resolution/registries/lookup-qualified.js +58 -0
- package/dist/core/code/shared/scope-resolution/registries/macro-registry.js +34 -0
- package/dist/core/code/shared/scope-resolution/registries/method-registry.js +34 -0
- package/dist/core/code/shared/scope-resolution/registries/tie-breaks.js +63 -0
- package/dist/core/code/shared/scope-resolution/resolve-type-ref.js +128 -0
- package/dist/core/code/shared/scope-resolution/scope-id.js +49 -0
- package/dist/core/code/shared/scope-resolution/scope-tree.js +225 -0
- package/dist/core/code/shared/scope-resolution/symbol-definition.js +12 -0
- package/dist/core/code/shared/scope-resolution/types.js +25 -0
- package/dist/core/code/shared/test-helpers.js +17 -0
- package/dist/core/code/vendor/leiden/index.cjs +355 -0
- package/dist/core/code/vendor/leiden/utils.cjs +419 -0
- package/dist/core/timeline/cli.d.ts.map +1 -1
- package/dist/core/timeline/cli.js +13 -6
- package/dist/core/timeline/cli.js.map +1 -1
- package/dist/core/timeline/debris.d.ts +20 -0
- package/dist/core/timeline/debris.d.ts.map +1 -0
- package/dist/core/timeline/debris.js +124 -0
- package/dist/core/timeline/debris.js.map +1 -0
- package/dist/core/timeline/hook-runner.d.ts.map +1 -1
- package/dist/core/timeline/hook-runner.js +3 -1
- package/dist/core/timeline/hook-runner.js.map +1 -1
- package/dist/core/timeline/hooks.d.ts.map +1 -1
- package/dist/core/timeline/hooks.js +13 -5
- package/dist/core/timeline/hooks.js.map +1 -1
- package/dist/core/timeline/installer.d.ts +2 -0
- package/dist/core/timeline/installer.d.ts.map +1 -1
- package/dist/core/timeline/installer.js +12 -0
- package/dist/core/timeline/installer.js.map +1 -1
- package/dist/core/timeline/project-root.d.ts +21 -0
- package/dist/core/timeline/project-root.d.ts.map +1 -0
- package/dist/core/timeline/project-root.js +83 -0
- package/dist/core/timeline/project-root.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,1725 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.createWorkerPool = exports.WorkerPoolDisabledError = exports.WorkerPoolInitializationError = exports.WorkerPoolDispatchError = void 0;
|
|
7
|
+
exports.buildDispatchMessage = buildDispatchMessage;
|
|
8
|
+
exports.crashSignature = crashSignature;
|
|
9
|
+
exports.resolveWorkerPoolOptions = resolveWorkerPoolOptions;
|
|
10
|
+
exports.workerPoolDisabledByEnv = workerPoolDisabledByEnv;
|
|
11
|
+
exports.resolveAutoPoolSize = resolveAutoPoolSize;
|
|
12
|
+
exports.startHeartbeatStallTracker = startHeartbeatStallTracker;
|
|
13
|
+
exports.resolveWorkerHeapCapMb = resolveWorkerHeapCapMb;
|
|
14
|
+
const node_worker_threads_1 = require("node:worker_threads");
|
|
15
|
+
const node_os_1 = __importDefault(require("node:os"));
|
|
16
|
+
const effective_ram_js_1 = require("../utils/effective-ram.js");
|
|
17
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
18
|
+
const node_url_1 = require("node:url");
|
|
19
|
+
const logger_js_1 = require("../../logger.js");
|
|
20
|
+
const quarantine_js_1 = require("./quarantine.js");
|
|
21
|
+
/**
|
|
22
|
+
* Type guard: every element of `items` has the parse-worker shape
|
|
23
|
+
* (`{path: string, content: string}`). Used to narrow the generic input
|
|
24
|
+
* inside `buildDispatchMessage` so a future rename of
|
|
25
|
+
* `ParseWorkerInput.content` would fail to compile inside the narrowed
|
|
26
|
+
* branch instead of silently mismatching at runtime.
|
|
27
|
+
*/
|
|
28
|
+
function isParseWorkerItemArray(items) {
|
|
29
|
+
if (items.length === 0)
|
|
30
|
+
return false;
|
|
31
|
+
for (const it of items) {
|
|
32
|
+
if (it == null || typeof it !== 'object')
|
|
33
|
+
return false;
|
|
34
|
+
if (typeof it.path !== 'string')
|
|
35
|
+
return false;
|
|
36
|
+
if (typeof it.content !== 'string')
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Build the sub-batch dispatch payload + transferList.
|
|
43
|
+
*
|
|
44
|
+
* For the parse-worker shape `{path, content: string}[]`, encodes each
|
|
45
|
+
* file's content as a `Uint8Array` via `TextEncoder` so the underlying
|
|
46
|
+
* `ArrayBuffer` can be transferred zero-copy. For any other input
|
|
47
|
+
* shape, the items array is passed through verbatim (no transfer).
|
|
48
|
+
*
|
|
49
|
+
* @internal Exported for the unit test suite
|
|
50
|
+
* (`test/unit/worker-pool-transferlist.test.ts`) so the
|
|
51
|
+
* Uint8Array-per-content allocation contract can be pinned without
|
|
52
|
+
* spinning up a real worker_threads.
|
|
53
|
+
*/
|
|
54
|
+
function buildDispatchMessage(items) {
|
|
55
|
+
if (!isParseWorkerItemArray(items)) {
|
|
56
|
+
return { message: { type: 'sub-batch', files: items } };
|
|
57
|
+
}
|
|
58
|
+
// After the type guard, `items` is narrowed to `readonly ParseWorkerItem[]`.
|
|
59
|
+
const encoder = new TextEncoder();
|
|
60
|
+
const files = [];
|
|
61
|
+
const transferList = [];
|
|
62
|
+
for (const item of items) {
|
|
63
|
+
const u8 = encoder.encode(item.content);
|
|
64
|
+
files.push({ path: item.path, content: u8 });
|
|
65
|
+
transferList.push(u8.buffer);
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
message: { type: 'sub-batch', files },
|
|
69
|
+
transferList,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
class WorkerPoolDispatchError extends Error {
|
|
73
|
+
/**
|
|
74
|
+
* Snapshot of the pool's session-scoped quarantine at the moment the
|
|
75
|
+
* dispatch error was raised. Surfaced for operator diagnostics: when
|
|
76
|
+
* the circuit breaker trips, this lists the files the pool had
|
|
77
|
+
* already decided were unsafe before the trip. Read-only at the
|
|
78
|
+
* caller boundary; no in-pool consumer rewires it post-construction.
|
|
79
|
+
*
|
|
80
|
+
* Previously named `fallbackExcludePaths` because the (since-
|
|
81
|
+
* removed) sequential-parser fallback in `processParsing` consumed
|
|
82
|
+
* it to filter the fallback file list. After U20's design pivot
|
|
83
|
+
* (worker pool's resilience layers are the sole failure contract;
|
|
84
|
+
* no sequential rescue), the field is informational only. The
|
|
85
|
+
* rename clarifies semantics without changing wire behavior.
|
|
86
|
+
*/
|
|
87
|
+
quarantinedPaths;
|
|
88
|
+
constructor(message, quarantinedPaths = []) {
|
|
89
|
+
super(message);
|
|
90
|
+
this.name = 'WorkerPoolDispatchError';
|
|
91
|
+
this.quarantinedPaths = quarantinedPaths;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
exports.WorkerPoolDispatchError = WorkerPoolDispatchError;
|
|
95
|
+
class WorkerPoolInitializationError extends WorkerPoolDispatchError {
|
|
96
|
+
readinessFailures;
|
|
97
|
+
/** Pool's automatic classification of the startup crash (#1741). */
|
|
98
|
+
crashClass;
|
|
99
|
+
constructor(message, quarantinedPaths = [], readinessFailures = [], crashClass = 'transient-exhausted') {
|
|
100
|
+
super(message, quarantinedPaths);
|
|
101
|
+
this.name = 'WorkerPoolInitializationError';
|
|
102
|
+
this.readinessFailures = readinessFailures;
|
|
103
|
+
this.crashClass = crashClass;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
exports.WorkerPoolInitializationError = WorkerPoolInitializationError;
|
|
107
|
+
/**
|
|
108
|
+
* Thrown when a caller asks GitNexus to parse without the worker pool —
|
|
109
|
+
* `--workers 0`, `CGRAPH_WORKER_POOL_SIZE=0`, or `skipWorkers: true`.
|
|
110
|
+
*
|
|
111
|
+
* GitNexus no longer has a sequential parser: the worker pool (with its
|
|
112
|
+
* quarantine + respawn/recycle + circuit-breaker resilience) is the SOLE
|
|
113
|
+
* parse path. These channels used to select an in-process fallback; they are
|
|
114
|
+
* now hard configuration errors so the operator gets an actionable message
|
|
115
|
+
* instead of silently parsing through a (deleted) slower path.
|
|
116
|
+
*/
|
|
117
|
+
class WorkerPoolDisabledError extends Error {
|
|
118
|
+
constructor(message) {
|
|
119
|
+
super(message);
|
|
120
|
+
this.name = 'WorkerPoolDisabledError';
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
exports.WorkerPoolDisabledError = WorkerPoolDisabledError;
|
|
124
|
+
/**
|
|
125
|
+
* Max files to send to a worker in a single postMessage.
|
|
126
|
+
* Keeps structured-clone memory bounded per sub-batch.
|
|
127
|
+
*/
|
|
128
|
+
const SUB_BATCH_SIZE = 1500;
|
|
129
|
+
const SUB_BATCH_MAX_BYTES = 8 * 1024 * 1024;
|
|
130
|
+
const DEFAULT_SUB_BATCH_IDLE_TIMEOUT_MS = 30_000;
|
|
131
|
+
const DEFAULT_TIMEOUT_RETRIES = 1;
|
|
132
|
+
const DEFAULT_TIMEOUT_BACKOFF_FACTOR = 2;
|
|
133
|
+
const DEFAULT_MAX_RESPAWNS_PER_SLOT = 3;
|
|
134
|
+
const DEFAULT_MAX_CUMULATIVE_TIMEOUT_FACTOR = 5;
|
|
135
|
+
const DEFAULT_CONSECUTIVE_FAILURE_THRESHOLD_FLOOR = 3;
|
|
136
|
+
const DEFAULT_WORKER_READY_TIMEOUT_MS = 5_000;
|
|
137
|
+
/**
|
|
138
|
+
* Default upper bound on auto-resolved pool size. Past 16 workers the
|
|
139
|
+
* dominant cost shifts from worker-side parsing to main-thread merge /
|
|
140
|
+
* extraction / structured-clone overhead, and the marginal worker adds
|
|
141
|
+
* memory pressure (tree-sitter state + sub-batch buffer) without much
|
|
142
|
+
* throughput gain. Operators on bigger machines override via
|
|
143
|
+
* `CGRAPH_WORKER_POOL_SIZE` or `--workers <N>`.
|
|
144
|
+
*/
|
|
145
|
+
const DEFAULT_POOL_SIZE_CAP = 16;
|
|
146
|
+
// ── Self-healing startup restart policy (#1741) ──────────────────────────────
|
|
147
|
+
// A worker that crashes during top-of-script init (broken native binding, bad
|
|
148
|
+
// import) is retried a BOUNDED number of times with jittered backoff before
|
|
149
|
+
// its slot is dropped, so a transient blip self-heals with no operator
|
|
150
|
+
// intervention. The bound is the whole point of #1741: recovery must never
|
|
151
|
+
// become a silent, unbounded "stuck" run. When the budget is exhausted (or a
|
|
152
|
+
// deterministic crash-loop is detected), the slot is dropped; if every slot is
|
|
153
|
+
// dropped the first dispatch fails fast with the captured cause.
|
|
154
|
+
/** Retries beyond the first attempt, per slot, to bring a startup worker ready. */
|
|
155
|
+
const STARTUP_RESTART_BUDGET = 2;
|
|
156
|
+
const RESTART_BACKOFF_BASE_MS = 250;
|
|
157
|
+
const RESTART_BACKOFF_CAP_MS = 2_000;
|
|
158
|
+
/**
|
|
159
|
+
* When this many freshly-spawned workers crash with the SAME crash signature
|
|
160
|
+
* before ANY worker reaches the `{type:'ready'}` handshake, the failure is
|
|
161
|
+
* deterministic (the #1741 missing-binding case: every worker prints a
|
|
162
|
+
* byte-identical native-binding stack). The pool stops retrying immediately
|
|
163
|
+
* instead of burning every slot's budget, and fails fast with the cause.
|
|
164
|
+
*/
|
|
165
|
+
const DETERMINISTIC_STARTUP_FINGERPRINT_THRESHOLD = 2;
|
|
166
|
+
/**
|
|
167
|
+
* Capped exponential backoff with FULL jitter (AWS "Exponential Backoff And
|
|
168
|
+
* Jitter"): random(0, min(CAP, BASE·2^attempt)). Full jitter de-synchronizes
|
|
169
|
+
* the N workers that crash near-simultaneously on a shared startup fault so
|
|
170
|
+
* their respawns don't re-storm in lockstep (Google SRE thundering herd).
|
|
171
|
+
*/
|
|
172
|
+
function startupBackoffMs(attempt) {
|
|
173
|
+
const ceil = Math.min(RESTART_BACKOFF_CAP_MS, RESTART_BACKOFF_BASE_MS * 2 ** attempt);
|
|
174
|
+
return Math.floor(Math.random() * (ceil + 1));
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Sleep used between startup self-heal retries. The timer is intentionally NOT
|
|
178
|
+
* `unref`'d: a pending retry is necessary work, so it must keep the event loop
|
|
179
|
+
* alive long enough to actually respawn — otherwise a pool whose only live work
|
|
180
|
+
* is a startup backoff could let the process exit mid-recovery (#1741). To
|
|
181
|
+
* avoid wedging shutdown, the timer registers a cancel function in `pending`;
|
|
182
|
+
* `terminate()` invokes those cancels to `clearTimeout` and resolve early, and
|
|
183
|
+
* a normally-fired timer removes its own cancel. `aborted()` is checked once up
|
|
184
|
+
* front; the CALLER re-checks after wake (it owns the terminated/deterministic
|
|
185
|
+
* decision) — this function does not itself re-evaluate abort on wake.
|
|
186
|
+
*/
|
|
187
|
+
function abortableSleep(ms, aborted, pending) {
|
|
188
|
+
return new Promise((resolve) => {
|
|
189
|
+
if (ms <= 0 || aborted()) {
|
|
190
|
+
resolve();
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
// `cancel` is registered so terminate() can clear a pending backoff; it is
|
|
194
|
+
// also the timer's own callback, so a normally-fired sleep self-deregisters.
|
|
195
|
+
const cancel = () => {
|
|
196
|
+
clearTimeout(timer);
|
|
197
|
+
pending.delete(cancel);
|
|
198
|
+
resolve();
|
|
199
|
+
};
|
|
200
|
+
const timer = setTimeout(cancel, ms);
|
|
201
|
+
pending.add(cancel);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Normalize a worker crash message into a stable signature so two instances of
|
|
206
|
+
* the SAME deterministic crash compare equal while unrelated crashes don't.
|
|
207
|
+
* Strips hex addresses, digit runs (pids / line numbers / timestamps) and
|
|
208
|
+
* absolute paths. Best-effort by design: the deterministic classification's
|
|
209
|
+
* correctness rests on the STRUCTURAL signal (zero workers ever ready + startup
|
|
210
|
+
* budget exhausted), so an imperfect signature only changes how fast the
|
|
211
|
+
* short-circuit fires, never whether the pool ultimately fails fast. Even a
|
|
212
|
+
* stderr-less crash normalizes its "exited with code N" message to a stable
|
|
213
|
+
* key, so the empty-stderr timing case still groups.
|
|
214
|
+
*
|
|
215
|
+
* @internal Exported for unit tests; production callers are in this module.
|
|
216
|
+
*/
|
|
217
|
+
function crashSignature(message) {
|
|
218
|
+
return (message
|
|
219
|
+
.replace(/0x[0-9a-fA-F]+/g, '0xADDR') // 0x-prefixed addresses
|
|
220
|
+
// Windows backslash paths (optional drive letter), e.g. C:\Users\ci\Temp\w-7f3a.js
|
|
221
|
+
.replace(/(?:[A-Za-z]:)?(?:\\[^\s\\'"]+)+/g, '\\PATH')
|
|
222
|
+
.replace(/(?:\/[^\s:'"]+)+/g, '/PATH') // POSIX paths
|
|
223
|
+
.replace(/\b[0-9a-fA-F]{6,}\b/g, 'HEX') // bare hex runs (ASLR addrs / backtrace tokens)
|
|
224
|
+
.replace(/[0-9]+/g, 'N') // pids / line numbers / exit codes / timestamps
|
|
225
|
+
.replace(/\s+/g, ' ')
|
|
226
|
+
.trim()
|
|
227
|
+
.slice(0, 300));
|
|
228
|
+
}
|
|
229
|
+
function positiveInteger(value) {
|
|
230
|
+
const parsed = typeof value === 'string' ? Number(value) : value;
|
|
231
|
+
return typeof parsed === 'number' && Number.isFinite(parsed) && parsed > 0
|
|
232
|
+
? Math.floor(parsed)
|
|
233
|
+
: undefined;
|
|
234
|
+
}
|
|
235
|
+
function nonNegativeInteger(value) {
|
|
236
|
+
const parsed = typeof value === 'string' ? Number(value) : value;
|
|
237
|
+
return typeof parsed === 'number' && Number.isFinite(parsed) && parsed >= 0
|
|
238
|
+
? Math.floor(parsed)
|
|
239
|
+
: undefined;
|
|
240
|
+
}
|
|
241
|
+
/** See {@link WorkerPoolOptions.shutdownDrainMs}. */
|
|
242
|
+
const DEFAULT_SHUTDOWN_DRAIN_MS = 30_000;
|
|
243
|
+
function resolveWorkerPoolOptions(options = {}, poolSize) {
|
|
244
|
+
const subBatchIdleTimeoutMs = positiveInteger(options.subBatchIdleTimeoutMs) ??
|
|
245
|
+
positiveInteger(process.env.CGRAPH_WORKER_SUB_BATCH_TIMEOUT_MS) ??
|
|
246
|
+
DEFAULT_SUB_BATCH_IDLE_TIMEOUT_MS;
|
|
247
|
+
return {
|
|
248
|
+
subBatchSize: positiveInteger(options.subBatchSize) ?? SUB_BATCH_SIZE,
|
|
249
|
+
subBatchMaxBytes: positiveInteger(options.subBatchMaxBytes) ??
|
|
250
|
+
positiveInteger(process.env.CGRAPH_WORKER_SUB_BATCH_MAX_BYTES) ??
|
|
251
|
+
SUB_BATCH_MAX_BYTES,
|
|
252
|
+
subBatchIdleTimeoutMs,
|
|
253
|
+
maxTimeoutRetries: nonNegativeInteger(options.maxTimeoutRetries) ?? DEFAULT_TIMEOUT_RETRIES,
|
|
254
|
+
timeoutBackoffFactor: positiveInteger(options.timeoutBackoffFactor) ?? DEFAULT_TIMEOUT_BACKOFF_FACTOR,
|
|
255
|
+
maxRespawnsPerSlot: nonNegativeInteger(options.maxRespawnsPerSlot) ??
|
|
256
|
+
nonNegativeInteger(process.env.CGRAPH_WORKER_MAX_RESPAWNS_PER_SLOT) ??
|
|
257
|
+
DEFAULT_MAX_RESPAWNS_PER_SLOT,
|
|
258
|
+
maxCumulativeTimeoutMs: positiveInteger(options.maxCumulativeTimeoutMs) ??
|
|
259
|
+
positiveInteger(process.env.CGRAPH_WORKER_MAX_CUMULATIVE_TIMEOUT_MS) ??
|
|
260
|
+
subBatchIdleTimeoutMs * DEFAULT_MAX_CUMULATIVE_TIMEOUT_FACTOR,
|
|
261
|
+
consecutiveFailureThreshold: positiveInteger(options.consecutiveFailureThreshold) ??
|
|
262
|
+
positiveInteger(process.env.CGRAPH_WORKER_CONSECUTIVE_FAILURE_THRESHOLD) ??
|
|
263
|
+
Math.max(DEFAULT_CONSECUTIVE_FAILURE_THRESHOLD_FLOOR, poolSize ?? 0),
|
|
264
|
+
shutdownDrainMs: nonNegativeInteger(options.shutdownDrainMs) ??
|
|
265
|
+
nonNegativeInteger(process.env.CGRAPH_WORKER_SHUTDOWN_DRAIN_MS) ??
|
|
266
|
+
DEFAULT_SHUTDOWN_DRAIN_MS,
|
|
267
|
+
workerReadyTimeoutMs: positiveInteger(options.workerReadyTimeoutMs) ??
|
|
268
|
+
positiveInteger(process.env.CGRAPH_WORKER_READY_TIMEOUT_MS) ??
|
|
269
|
+
DEFAULT_WORKER_READY_TIMEOUT_MS,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* The pool size requested via the `CGRAPH_WORKER_POOL_SIZE` env var, or
|
|
274
|
+
* `undefined` when unset, empty/whitespace, or invalid. Module-internal sizing
|
|
275
|
+
* reader consumed by {@link resolveAutoPoolSize} (the env override) and
|
|
276
|
+
* {@link workerPoolDisabledByEnv} (the disabled-channel check). Reads only —
|
|
277
|
+
* never mutates `process.env`. Empty/whitespace is treated as *unset* (falls
|
|
278
|
+
* through to the auto formula), not as 0 — an empty assignment (`export
|
|
279
|
+
* CGRAPH_WORKER_POOL_SIZE=`) is an accident, not a request for zero workers;
|
|
280
|
+
* only a literal `0` disables the pool.
|
|
281
|
+
*/
|
|
282
|
+
function envWorkerPoolSize() {
|
|
283
|
+
const raw = process.env.CGRAPH_WORKER_POOL_SIZE;
|
|
284
|
+
if (raw === undefined || raw.trim() === '')
|
|
285
|
+
return undefined;
|
|
286
|
+
return nonNegativeInteger(raw);
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* True when the operator set `CGRAPH_WORKER_POOL_SIZE=0` — the env-channel
|
|
290
|
+
* equivalent of `--workers 0`. The parse phase consults this (only when no
|
|
291
|
+
* explicit `--workers <N>` was passed) and HARD-ERRORS: sequential parsing was
|
|
292
|
+
* removed, so a disabled pool is an actionable configuration error, not a
|
|
293
|
+
* silent fallback. An explicit positive `--workers N` always wins.
|
|
294
|
+
*/
|
|
295
|
+
function workerPoolDisabledByEnv() {
|
|
296
|
+
return envWorkerPoolSize() === 0;
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Resolve the auto-default worker pool size when no explicit `poolSize`
|
|
300
|
+
* arg is passed to `createWorkerPool`. Precedence:
|
|
301
|
+
*
|
|
302
|
+
* 1. `CGRAPH_WORKER_POOL_SIZE` env var (operator override).
|
|
303
|
+
* 2. `os.cpus().length - 1`, clamped to `[1, DEFAULT_POOL_SIZE_CAP]`.
|
|
304
|
+
*
|
|
305
|
+
* The cap exists because past ~16 workers the main-thread merge /
|
|
306
|
+
* extraction work and structured-clone overhead dominate; adding more
|
|
307
|
+
* worker threads costs memory without much throughput gain. Operators
|
|
308
|
+
* who want to push past the cap set the env var explicitly.
|
|
309
|
+
*
|
|
310
|
+
* Exported for unit tests; production code should not call this
|
|
311
|
+
* directly — pass an explicit `poolSize` to `createWorkerPool` or rely
|
|
312
|
+
* on the env / default.
|
|
313
|
+
*/
|
|
314
|
+
function resolveAutoPoolSize() {
|
|
315
|
+
const envOverride = envWorkerPoolSize();
|
|
316
|
+
if (envOverride !== undefined)
|
|
317
|
+
return envOverride;
|
|
318
|
+
// Prefer os.availableParallelism (Node 18.14+) so cgroup CPU limits
|
|
319
|
+
// (containers, taskset-restricted runtimes, CI runners with explicit
|
|
320
|
+
// CPU quotas) are honored — os.cpus().length returns the host count,
|
|
321
|
+
// which over-sizes the pool on constrained shapes and can reintroduce
|
|
322
|
+
// the very "main-thread saturated by oversubscription" symptom the
|
|
323
|
+
// pool cap exists to prevent. Falls back to os.cpus().length on
|
|
324
|
+
// older Node versions. Mirrors `capabilities.ts:85`
|
|
325
|
+
// (`defaultEmbeddingThreads`).
|
|
326
|
+
const cores = typeof node_os_1.default.availableParallelism === 'function' ? node_os_1.default.availableParallelism() : node_os_1.default.cpus().length;
|
|
327
|
+
return Math.min(DEFAULT_POOL_SIZE_CAP, Math.max(1, cores - 1));
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Max characters of a worker's stderr retained for crash diagnostics. A
|
|
331
|
+
* native-binding load failure or a top-of-script throw prints a stack to
|
|
332
|
+
* stderr; we keep the tail so `waitForWorkerReady` can attach the real
|
|
333
|
+
* reason to its rejection instead of the generic "did not report ready".
|
|
334
|
+
*/
|
|
335
|
+
const WORKER_STDERR_TAIL_LIMIT = 4000;
|
|
336
|
+
/**
|
|
337
|
+
* Per-worker captured stderr tail. Populated only for workers spawned with
|
|
338
|
+
* `{ stderr: true }` (the production factory below). Test-injected workers
|
|
339
|
+
* via `workerFactory` typically inherit the parent's stderr and have no
|
|
340
|
+
* `worker.stderr` stream — those are simply skipped (empty tail). A WeakMap
|
|
341
|
+
* so the buffer is released when the worker is GC'd.
|
|
342
|
+
*/
|
|
343
|
+
const workerStderrTails = new WeakMap();
|
|
344
|
+
/**
|
|
345
|
+
* Tee a worker's stderr into a bounded in-memory tail (for surfacing the
|
|
346
|
+
* real crash on a startup failure) while still mirroring it to the parent
|
|
347
|
+
* process's stderr — preserving the live-diagnostics behavior workers had
|
|
348
|
+
* when they inherited stderr, before `{ stderr: true }` redirected it to a
|
|
349
|
+
* stream. No-op when the worker has no `stderr` stream (test factories).
|
|
350
|
+
*/
|
|
351
|
+
function captureWorkerStderr(worker) {
|
|
352
|
+
const stream = worker.stderr;
|
|
353
|
+
if (!stream)
|
|
354
|
+
return;
|
|
355
|
+
const buf = { text: '' };
|
|
356
|
+
workerStderrTails.set(worker, buf);
|
|
357
|
+
stream.on('data', (chunk) => {
|
|
358
|
+
const s = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
|
359
|
+
process.stderr.write(s);
|
|
360
|
+
buf.text = (buf.text + s).slice(-WORKER_STDERR_TAIL_LIMIT);
|
|
361
|
+
});
|
|
362
|
+
// A stderr stream error must never crash the pool.
|
|
363
|
+
stream.on('error', () => undefined);
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Forward a worker's piped stdout to the parent process's stdout, so worker
|
|
367
|
+
* logs stay visible now that the production factory spawns with
|
|
368
|
+
* `{ stdout: true }`. Workers with INHERITED stdout have been observed to
|
|
369
|
+
* crash silently during top-of-script init (exit code 1, nothing on stderr,
|
|
370
|
+
* roughly half of a concurrently spawned pool) on macOS 26.5 under both
|
|
371
|
+
* Node 22 and 26; piping stdout eliminates the crash entirely. Piping also
|
|
372
|
+
* matches the existing stderr handling, so worker output no longer races the
|
|
373
|
+
* parent's raw fd. No-op when the worker has no `stdout` stream (test
|
|
374
|
+
* factories).
|
|
375
|
+
*/
|
|
376
|
+
function forwardWorkerStdout(worker) {
|
|
377
|
+
const stream = worker.stdout;
|
|
378
|
+
if (!stream)
|
|
379
|
+
return;
|
|
380
|
+
stream.on('data', (chunk) => {
|
|
381
|
+
process.stdout.write(chunk);
|
|
382
|
+
});
|
|
383
|
+
// A stdout stream error must never crash the pool.
|
|
384
|
+
stream.on('error', () => undefined);
|
|
385
|
+
}
|
|
386
|
+
/** Captured stderr tail for a worker, trimmed; '' when nothing was captured. */
|
|
387
|
+
function workerStderrTail(worker) {
|
|
388
|
+
return workerStderrTails.get(worker)?.text.trim() ?? '';
|
|
389
|
+
}
|
|
390
|
+
/** Append the worker's captured stderr to a readiness-failure message. */
|
|
391
|
+
function withStderr(worker, message) {
|
|
392
|
+
const tail = workerStderrTail(worker);
|
|
393
|
+
return tail ? `${message}. Worker stderr:\n${tail}` : message;
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Build a worker-death reason string that carries the worker-side stack when one
|
|
397
|
+
* is available (#2068). The stack is appended AFTER the `Worker N error: <msg>`
|
|
398
|
+
* prefix so every prefix/substring consumer downstream — recoverAndResume →
|
|
399
|
+
* handleWorkerDeath → the circuit-breaker `WorkerPoolDispatchError` message, and
|
|
400
|
+
* the tests that regex-match those — keeps working unchanged, while the operator
|
|
401
|
+
* now gets the real frame instead of a bare one-liner. The stack's first line is
|
|
402
|
+
* normally the message itself; keeping both is harmless and the indented block
|
|
403
|
+
* scans cleanly in a log. The stack is capped at WORKER_STDERR_TAIL_LIMIT,
|
|
404
|
+
* mirroring the sibling stderr-tail bound, so a pathological error type (or a
|
|
405
|
+
* raised `Error.stackTraceLimit`) can't bloat the death reason. `stack` is
|
|
406
|
+
* `undefined` for an older worker build (or a thrown non-Error), in which case
|
|
407
|
+
* the reason is exactly the prior message-only form.
|
|
408
|
+
*/
|
|
409
|
+
function workerErrorReason(workerIndex, message, stack) {
|
|
410
|
+
const base = `Worker ${workerIndex} error: ${message}`;
|
|
411
|
+
return stack ? `${base}\n worker stack:\n${stack.slice(0, WORKER_STDERR_TAIL_LIMIT)}` : base;
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Wait for a freshly-spawned replacement worker to emit the
|
|
415
|
+
* `{type:'ready'}` handshake from `parse-worker.ts` before treating its
|
|
416
|
+
* slot as dispatch-ready. Trusting Node's `online` event alone (which
|
|
417
|
+
* fires when the worker thread starts, BEFORE the worker script's
|
|
418
|
+
* top-of-script body runs) let a worker that crashes during init
|
|
419
|
+
* (parser/grammar import failure, missing native binding) slip past
|
|
420
|
+
* pool startup. The pool then only noticed the dead replacement on the
|
|
421
|
+
* first dispatch's idle timeout (default 30s) — a long stall masking
|
|
422
|
+
* an actual crash. This handshake bounds the wait at `readyTimeoutMs`
|
|
423
|
+
* (see {@link WorkerPoolOptions.workerReadyTimeoutMs}) and surfaces init
|
|
424
|
+
* failures as `error` / `exit` / `messageerror` events directly.
|
|
425
|
+
* `messageerror` is wired the same way: a V8 deserialization failure
|
|
426
|
+
* during startup is treated as worker death and rejects the readiness
|
|
427
|
+
* promise.
|
|
428
|
+
*/
|
|
429
|
+
function waitForWorkerReady(worker, readyTimeoutMs) {
|
|
430
|
+
return new Promise((resolve, reject) => {
|
|
431
|
+
const cleanup = () => {
|
|
432
|
+
clearTimeout(timer);
|
|
433
|
+
worker.removeListener('message', onMessage);
|
|
434
|
+
worker.removeListener('error', onError);
|
|
435
|
+
worker.removeListener('exit', onExit);
|
|
436
|
+
worker.removeListener('messageerror', onMessageError);
|
|
437
|
+
};
|
|
438
|
+
const onMessage = (msg) => {
|
|
439
|
+
// Native postMessage delivers POJO directly via Node's structured
|
|
440
|
+
// clone. The ready handshake is `{type:'ready'}`; any other early
|
|
441
|
+
// message during the startup window is ignored — the eventual
|
|
442
|
+
// timeout / exit / error handlers catch a genuinely-broken worker.
|
|
443
|
+
if (typeof msg === 'object' && msg !== null && msg.type === 'ready') {
|
|
444
|
+
cleanup();
|
|
445
|
+
resolve();
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
const onError = (err) => {
|
|
449
|
+
cleanup();
|
|
450
|
+
// The 'error' event carries the real top-of-script exception; enrich it
|
|
451
|
+
// with the worker's stderr tail (native-binding stacks land there).
|
|
452
|
+
reject(new Error(withStderr(worker, err.message)));
|
|
453
|
+
};
|
|
454
|
+
const onExit = (code) => {
|
|
455
|
+
cleanup();
|
|
456
|
+
reject(new Error(withStderr(worker, `Replacement worker exited with code ${code} before reporting ready`)));
|
|
457
|
+
};
|
|
458
|
+
const onMessageError = (err) => {
|
|
459
|
+
cleanup();
|
|
460
|
+
reject(new Error(withStderr(worker, `Replacement worker emitted messageerror before reporting ready: ${err.message}`)));
|
|
461
|
+
};
|
|
462
|
+
// `timer` is declared after `cleanup` so the cleanup closure can reference
|
|
463
|
+
// it. The const is reached before any handler attaches below, so no TDZ
|
|
464
|
+
// access can fire from the listeners.
|
|
465
|
+
const timer = setTimeout(() => {
|
|
466
|
+
cleanup();
|
|
467
|
+
reject(new Error(withStderr(worker, `Replacement worker did not report ready within ${readyTimeoutMs}ms — likely crashed during top-of-script init (slow host? raise CGRAPH_WORKER_READY_TIMEOUT_MS; repeated on a large repo? likely main-thread memory pressure — see the "Analysis runs out of memory" README section, #2649)`)));
|
|
468
|
+
}, readyTimeoutMs);
|
|
469
|
+
worker.on('message', onMessage);
|
|
470
|
+
worker.once('error', onError);
|
|
471
|
+
worker.once('exit', onExit);
|
|
472
|
+
worker.once('messageerror', onMessageError);
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
function estimateItemBytes(item) {
|
|
476
|
+
if (typeof item !== 'object' || item === null)
|
|
477
|
+
return 0;
|
|
478
|
+
const content = item.content;
|
|
479
|
+
return typeof content === 'string' ? Buffer.byteLength(content, 'utf8') : 0;
|
|
480
|
+
}
|
|
481
|
+
function itemPath(item) {
|
|
482
|
+
if (typeof item !== 'object' || item === null)
|
|
483
|
+
return undefined;
|
|
484
|
+
const path = item.path;
|
|
485
|
+
return typeof path === 'string' ? path : undefined;
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Best-guess path of the file in flight when a worker dies mid-job — used as
|
|
489
|
+
* the fallback when the authoritative `starting-file` message hasn't been
|
|
490
|
+
* observed yet (very early job-startup crash, or older worker build that
|
|
491
|
+
* doesn't emit the signal).
|
|
492
|
+
*
|
|
493
|
+
* `lastProgress` is the number of files the worker has acknowledged via
|
|
494
|
+
* `progress` messages, so `items[lastProgress]` is the next file it was
|
|
495
|
+
* about to process — the most likely culprit when the worker crashes
|
|
496
|
+
* (OOM, native addon SIGSEGV) or reports an error.
|
|
497
|
+
*
|
|
498
|
+
* Returns `[]` when no path is determinable so the caller retries the whole
|
|
499
|
+
* job.
|
|
500
|
+
*/
|
|
501
|
+
function inFlightExcludePath(job, lastProgress) {
|
|
502
|
+
if (lastProgress >= job.items.length)
|
|
503
|
+
return [];
|
|
504
|
+
const path = itemPath(job.items[lastProgress]);
|
|
505
|
+
return path ? [path] : [];
|
|
506
|
+
}
|
|
507
|
+
function createJobs(items, maxItems, maxBytes, timeoutMs, chunkHash) {
|
|
508
|
+
const jobs = [];
|
|
509
|
+
let startIndex = 0;
|
|
510
|
+
let batch = [];
|
|
511
|
+
let batchBytes = 0;
|
|
512
|
+
const flush = () => {
|
|
513
|
+
if (batch.length === 0)
|
|
514
|
+
return;
|
|
515
|
+
jobs.push({
|
|
516
|
+
startIndex,
|
|
517
|
+
items: batch,
|
|
518
|
+
estimatedBytes: batchBytes,
|
|
519
|
+
attempt: 0,
|
|
520
|
+
splitDepth: 0,
|
|
521
|
+
chunkHash,
|
|
522
|
+
timeoutMs,
|
|
523
|
+
cumulativeTimeoutMs: timeoutMs,
|
|
524
|
+
});
|
|
525
|
+
startIndex += batch.length;
|
|
526
|
+
batch = [];
|
|
527
|
+
batchBytes = 0;
|
|
528
|
+
};
|
|
529
|
+
for (const item of items) {
|
|
530
|
+
const itemBytes = estimateItemBytes(item);
|
|
531
|
+
const wouldExceedItems = batch.length >= maxItems;
|
|
532
|
+
const wouldExceedBytes = batch.length > 0 && batchBytes + itemBytes > maxBytes;
|
|
533
|
+
if (wouldExceedItems || wouldExceedBytes)
|
|
534
|
+
flush();
|
|
535
|
+
batch.push(item);
|
|
536
|
+
batchBytes += itemBytes;
|
|
537
|
+
}
|
|
538
|
+
flush();
|
|
539
|
+
return jobs;
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Create a pool of worker threads.
|
|
543
|
+
*
|
|
544
|
+
* Resilience model (PR #1693 / 1694):
|
|
545
|
+
* - Layer 1 (auto-respawn): a worker `error`/`exit` triggers a replacement on
|
|
546
|
+
* the same slot, bounded by {@link WorkerPoolOptions.maxRespawnsPerSlot}.
|
|
547
|
+
* The slot is dropped from the rotation when its budget is exhausted.
|
|
548
|
+
* - Layer 2 (circuit breaker): `consecutiveFailureThreshold` consecutive
|
|
549
|
+
* worker deaths (no successful job between) — OR all slots exhausting their
|
|
550
|
+
* respawn budget — trip the breaker. Every subsequent dispatch rejects
|
|
551
|
+
* with `WorkerPoolDispatchError` and the caller must build a new pool.
|
|
552
|
+
* - Layer 3 (quarantine): a path identified as the in-flight file at the
|
|
553
|
+
* time of a worker death is added to `quarantined` and filtered out of
|
|
554
|
+
* future dispatches. Snapshot via {@link WorkerPool.getQuarantinedPaths}.
|
|
555
|
+
* - Layer 4 (authoritative in-flight): the worker emits a `starting-file`
|
|
556
|
+
* message before each parse attempt; the pool prefers this for crash
|
|
557
|
+
* attribution and falls back to {@link inFlightExcludePath} only when no
|
|
558
|
+
* signal has been observed yet.
|
|
559
|
+
* - Layer 5 (cumulative timeout budget): each job tracks the total wall
|
|
560
|
+
* time spent across all attempts/splits/retries. When the budget is
|
|
561
|
+
* exhausted, the pool surfaces the in-flight path via `WorkerPoolDispatchError`
|
|
562
|
+
* instead of letting timeouts compound indefinitely.
|
|
563
|
+
*
|
|
564
|
+
* Upstream of these layers, the parse worker self-sanitizes a result that the
|
|
565
|
+
* structured-clone algorithm can't serialize (#2112) — stripping or dropping
|
|
566
|
+
* the offending value and reporting the affected paths on the result — so a
|
|
567
|
+
* single non-cloneable value can't masquerade as a worker death and exhaust a
|
|
568
|
+
* slot's respawn budget here.
|
|
569
|
+
*/
|
|
570
|
+
/**
|
|
571
|
+
* Main-thread stall tracking (#2649). Near the V8 heap limit, multi-second
|
|
572
|
+
* mark-compact pauses freeze the main thread's message processing, so a
|
|
573
|
+
* healthy worker's `progress` messages sit unread and the worker LOOKS idle —
|
|
574
|
+
* the idle-timeout path then splits/retires it, and the respawn storm ends in
|
|
575
|
+
* "Replacement worker did not report ready". A 250ms unref'd heartbeat
|
|
576
|
+
* accumulates observed event-loop drift; the idle-timeout handler credits
|
|
577
|
+
* that stall once per job instead of retiring a worker the main thread
|
|
578
|
+
* starved. The floor filters scheduler jitter from real stalls.
|
|
579
|
+
*/
|
|
580
|
+
const HEARTBEAT_INTERVAL_MS = 250;
|
|
581
|
+
const HEARTBEAT_STALL_FLOOR_MS = 100;
|
|
582
|
+
/** Fraction of the idle-timeout budget that must be main-thread stall before
|
|
583
|
+
* the timeout is credited and re-armed instead of acted on. */
|
|
584
|
+
const STALL_CREDIT_FRACTION = 0.5;
|
|
585
|
+
function startHeartbeatStallTracker() {
|
|
586
|
+
let totalStallMs = 0;
|
|
587
|
+
let last = Date.now();
|
|
588
|
+
const handle = setInterval(() => {
|
|
589
|
+
const now = Date.now();
|
|
590
|
+
const drift = now - last - HEARTBEAT_INTERVAL_MS;
|
|
591
|
+
if (drift > HEARTBEAT_STALL_FLOOR_MS)
|
|
592
|
+
totalStallMs += drift;
|
|
593
|
+
last = now;
|
|
594
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
595
|
+
handle.unref?.();
|
|
596
|
+
return { read: () => totalStallMs, stop: () => clearInterval(handle) };
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Per-worker V8 old-generation heap cap in MB (#2649). Without one, worker
|
|
600
|
+
* isolates inherit an unbounded default and a full pool can inflate process
|
|
601
|
+
* RSS past physical RAM on large repos. Half of RAM split across the pool,
|
|
602
|
+
* clamped to [512, 4096] MB — generous for the per-sub-batch working set
|
|
603
|
+
* (jobs are byte-budgeted), and a worker that does exceed it dies with a
|
|
604
|
+
* real heap error surfaced by the stderr-tail machinery + the
|
|
605
|
+
* quarantine/respawn path, instead of silently dragging the host into swap.
|
|
606
|
+
* `CGRAPH_WORKER_HEAP_MB` overrides the formula. Exported for unit tests.
|
|
607
|
+
*/
|
|
608
|
+
function resolveWorkerHeapCapMb(poolSize) {
|
|
609
|
+
return (positiveInteger(process.env.CGRAPH_WORKER_HEAP_MB) ??
|
|
610
|
+
Math.min(4096, Math.max(512, Math.floor((0, effective_ram_js_1.effectiveRamBytes)() / (1024 * 1024) / 2 / poolSize))));
|
|
611
|
+
}
|
|
612
|
+
const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
613
|
+
// Validate worker script exists before spawning to prevent uncaught
|
|
614
|
+
// MODULE_NOT_FOUND crashes in worker threads (e.g. when running from src/ via vitest)
|
|
615
|
+
const workerPath = (0, node_url_1.fileURLToPath)(workerUrl);
|
|
616
|
+
if (!node_fs_1.default.existsSync(workerPath)) {
|
|
617
|
+
throw new Error(`Worker script not found: ${workerPath}`);
|
|
618
|
+
}
|
|
619
|
+
const size = poolSize ?? resolveAutoPoolSize();
|
|
620
|
+
const poolOptions = resolveWorkerPoolOptions(options, size);
|
|
621
|
+
// Production factory spawns with `{ stderr: true }` so a worker's crash
|
|
622
|
+
// output is redirected to a `worker.stderr` stream we can tee + capture
|
|
623
|
+
// (see captureWorkerStderr) and attach to readiness-failure messages —
|
|
624
|
+
// instead of the generic "did not report ready" that hid the real cause
|
|
625
|
+
// in #1741. Test factories (workerFactory) are used verbatim.
|
|
626
|
+
// Bake the (immutable) ParsedFile store path into the factory closure so it
|
|
627
|
+
// reaches EVERY spawned worker — including respawns, which reuse this same
|
|
628
|
+
// factory — via `workerData`, read once at worker init. The `(url) => Worker`
|
|
629
|
+
// signature is unchanged so the zero-arg test factories keep working.
|
|
630
|
+
const parsedFileStoreStoragePath = options?.parsedFileStoreStoragePath;
|
|
631
|
+
const durableParsedFileStoragePath = options?.durableParsedFileStoragePath;
|
|
632
|
+
// CFG/PDG opt-in (#2081 M1) — carried in workerData alongside the store paths.
|
|
633
|
+
const pdg = options?.pdg === true;
|
|
634
|
+
const pdgMaxFunctionLines = options?.pdgMaxFunctionLines;
|
|
635
|
+
const workerStoreData = parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg
|
|
636
|
+
? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines }
|
|
637
|
+
: undefined;
|
|
638
|
+
const workerHeapCapMb = resolveWorkerHeapCapMb(size);
|
|
639
|
+
// The 512MB per-worker floor exists so a worker can parse anything real,
|
|
640
|
+
// but on a very small container a large pool of floored workers can still
|
|
641
|
+
// overcommit total memory (#2649 review). Behavior is unchanged — deaths
|
|
642
|
+
// are attributed and quarantine converges — but say so up front, with the
|
|
643
|
+
// two levers, instead of letting the operator discover it from worker OOMs.
|
|
644
|
+
const poolCommitMb = workerHeapCapMb * size;
|
|
645
|
+
const effectiveMb = Math.floor((0, effective_ram_js_1.effectiveRamBytes)() / (1024 * 1024));
|
|
646
|
+
if (poolCommitMb > 0.6 * effectiveMb) {
|
|
647
|
+
logger_js_1.logger.warn({ poolSize: size, workerHeapCapMb, effectiveMb }, `Worker pool may overcommit memory: ${size} workers × ${workerHeapCapMb}MB heap cap exceeds 60% of the ${effectiveMb}MB available to this process. Reduce CGRAPH_WORKER_POOL_SIZE or set CGRAPH_WORKER_HEAP_MB.`);
|
|
648
|
+
}
|
|
649
|
+
// #2649 stall probe: test seam wins; production uses the heartbeat tracker.
|
|
650
|
+
const stallTracker = options?.stallMsProbe
|
|
651
|
+
? { read: options.stallMsProbe, stop: () => undefined }
|
|
652
|
+
: startHeartbeatStallTracker();
|
|
653
|
+
const spawnWorker = options?.workerFactory ??
|
|
654
|
+
((url) => new node_worker_threads_1.Worker(url, {
|
|
655
|
+
// Piped (not inherited) stdio: stderr for crash capture (#1741),
|
|
656
|
+
// stdout because inherited stdout triggers silent startup crashes on
|
|
657
|
+
// some hosts (see forwardWorkerStdout).
|
|
658
|
+
stdout: true,
|
|
659
|
+
stderr: true,
|
|
660
|
+
workerData: workerStoreData,
|
|
661
|
+
// The CFG visitors build per-function control-flow graphs by RECURSIVE
|
|
662
|
+
// descent over the tree-sitter AST, so deeply-nested source overflows
|
|
663
|
+
// the worker thread's call stack. A worker's stack is governed by
|
|
664
|
+
// `resourceLimits.stackSizeMb` (Node default 4 MB) — the main process's
|
|
665
|
+
// `--stack-size` flag does NOT propagate to worker threads — so raise it
|
|
666
|
+
// here. This pushes the overflow threshold from ~1.5k to several-k
|
|
667
|
+
// nesting levels (far beyond any hand-written code); a deeper machine-
|
|
668
|
+
// generated nest is still caught per-function (buildFunctionCfg's R4
|
|
669
|
+
// try/catch) and only that function's PDG is skipped, never a crash.
|
|
670
|
+
resourceLimits: { stackSizeMb: 16, maxOldGenerationSizeMb: workerHeapCapMb },
|
|
671
|
+
}));
|
|
672
|
+
/** Spawn + wire stdio capture/forwarding in one step (used by all spawn sites). */
|
|
673
|
+
const spawnAndCapture = (url) => {
|
|
674
|
+
const worker = spawnWorker(url);
|
|
675
|
+
captureWorkerStderr(worker);
|
|
676
|
+
forwardWorkerStdout(worker);
|
|
677
|
+
return worker;
|
|
678
|
+
};
|
|
679
|
+
const workers = new Array(size);
|
|
680
|
+
const retiredWorkers = new Set();
|
|
681
|
+
const respawnCount = new Array(size).fill(0);
|
|
682
|
+
const activeSlots = new Set();
|
|
683
|
+
// Layer 3 (quarantine): tracked via the dedicated `quarantine.ts`
|
|
684
|
+
// module so the resilience layer is addressable as a unit (named
|
|
685
|
+
// interface, isolated tests) rather than an inline Set tangled into
|
|
686
|
+
// 1100+ LOC of pool plumbing. Public worker-pool API is unchanged —
|
|
687
|
+
// `getQuarantinedPaths()` still returns the same defensive copy.
|
|
688
|
+
const quarantine = (0, quarantine_js_1.createQuarantine)();
|
|
689
|
+
const initialReadinessFailures = [];
|
|
690
|
+
// Per-slot consecutive-failure counter (F6): replaces the prior pool-wide
|
|
691
|
+
// scalar so a chronically-failing slot trips the breaker on its own
|
|
692
|
+
// failure streak instead of being masked by another slot's successes.
|
|
693
|
+
// Reset to 0 on that slot's next successful job.
|
|
694
|
+
const consecutiveFailuresPerSlot = new Array(size).fill(0);
|
|
695
|
+
// Per-slot generation counter (U12). Incremented on every successful
|
|
696
|
+
// worker replacement (see replaceWorker below). Handlers in the
|
|
697
|
+
// dispatch loop capture the slot's generation at attach time and
|
|
698
|
+
// short-circuit when they fire on a stale generation. Defensive layer
|
|
699
|
+
// on top of the existing `settled` flag + listener removal — protects
|
|
700
|
+
// against any future refactor that loosens cleanup() ordering or
|
|
701
|
+
// re-attaches handlers without resetting the per-job state. Exposed
|
|
702
|
+
// via getStats so operators (and tests) can verify a slot was
|
|
703
|
+
// actually replaced and not just the same worker recycled.
|
|
704
|
+
const slotGenerations = new Array(size).fill(0);
|
|
705
|
+
let poolBroken = false;
|
|
706
|
+
let poolFailure;
|
|
707
|
+
// Set by `terminate()` (below). Also read by the self-healing startup loop so
|
|
708
|
+
// a terminate during startup aborts pending backoff/retries (#1741).
|
|
709
|
+
let terminated = false;
|
|
710
|
+
/** Resolves `true` when `promise` settles within `ms`, else `false`. The
|
|
711
|
+
* timer is unref'd so an expiring drain never holds the process open. */
|
|
712
|
+
const settledWithin = (promise, ms) => {
|
|
713
|
+
if (ms <= 0)
|
|
714
|
+
return Promise.resolve(false);
|
|
715
|
+
return new Promise((resolve) => {
|
|
716
|
+
const timer = setTimeout(() => resolve(false), ms);
|
|
717
|
+
timer.unref?.();
|
|
718
|
+
void promise.then(() => {
|
|
719
|
+
clearTimeout(timer);
|
|
720
|
+
resolve(true);
|
|
721
|
+
});
|
|
722
|
+
});
|
|
723
|
+
};
|
|
724
|
+
const terminateTrackedWorkers = async (liveWorkers) => {
|
|
725
|
+
const retired = Array.from(retiredWorkers);
|
|
726
|
+
await Promise.all([
|
|
727
|
+
...liveWorkers.map((worker) => worker?.terminate().catch(() => undefined)),
|
|
728
|
+
...retired.map(async (record) => {
|
|
729
|
+
// #2432: a retired worker that has not reached a JS-visible safe
|
|
730
|
+
// point may be inside an N-API call — terminating it aborts the
|
|
731
|
+
// WHOLE process (`Napi::Error` → std::terminate → SIGABRT). Drain:
|
|
732
|
+
// wait (bounded) for its safe point; on expiry leave it running —
|
|
733
|
+
// it is unref'd and its at-safe-point terminate listener stays
|
|
734
|
+
// armed — and log which file wedged it.
|
|
735
|
+
if (!record.safeToTerminate) {
|
|
736
|
+
const drained = await settledWithin(record.safePoint, poolOptions.shutdownDrainMs);
|
|
737
|
+
if (!drained) {
|
|
738
|
+
logger_js_1.logger.warn({
|
|
739
|
+
workerIndex: record.workerIndex,
|
|
740
|
+
reason: record.reason,
|
|
741
|
+
drainMs: poolOptions.shutdownDrainMs,
|
|
742
|
+
}, `Worker ${record.workerIndex} is still inside native code after the ` +
|
|
743
|
+
`${poolOptions.shutdownDrainMs}ms shutdown drain; leaving it un-terminated ` +
|
|
744
|
+
`to avoid a native abort (#2432). It will be terminated at its next safe point.`);
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
await record.terminate();
|
|
749
|
+
}),
|
|
750
|
+
]);
|
|
751
|
+
// Undrained records stay tracked so a repeated shutdown call can retry
|
|
752
|
+
// their (now possibly safe) terminate; record.terminate() removes each
|
|
753
|
+
// drained record via its cleanup.
|
|
754
|
+
};
|
|
755
|
+
for (let i = 0; i < size; i++) {
|
|
756
|
+
workers[i] = spawnAndCapture(workerUrl);
|
|
757
|
+
activeSlots.add(i);
|
|
758
|
+
}
|
|
759
|
+
// ── Self-healing startup readiness (#1741) ────────────────────────────────
|
|
760
|
+
// Bring every initial slot to readiness with a BOUNDED, jittered retry loop
|
|
761
|
+
// instead of dropping it on the first crash. This symmetrizes the gate with
|
|
762
|
+
// the runtime `replaceWorker` path (which already respawns a crashed slot),
|
|
763
|
+
// and adds genuine self-healing at startup:
|
|
764
|
+
//
|
|
765
|
+
// - TRANSIENT crash (a one-off OS hiccup / fork throttle): the slot is
|
|
766
|
+
// respawned after jittered backoff and retried, up to STARTUP_RESTART_BUDGET
|
|
767
|
+
// — so a blip heals itself with no operator intervention.
|
|
768
|
+
// - DETERMINISTIC crash-loop (every worker dies with the SAME signature
|
|
769
|
+
// before any reaches ready — the #1741 missing-binding case): detected via
|
|
770
|
+
// `crashSignature` and short-circuited immediately, so the pool gives up in
|
|
771
|
+
// ~1s rather than burning every slot's budget.
|
|
772
|
+
//
|
|
773
|
+
// When the loop exhausts, the slot is dropped from `activeSlots`. If EVERY
|
|
774
|
+
// slot is dropped, the first dispatch throws WorkerPoolInitializationError
|
|
775
|
+
// carrying the captured crash cause + classification — never a silent hang.
|
|
776
|
+
// Correctness of the deterministic short-circuit rests on the STRUCTURAL
|
|
777
|
+
// signal (zero workers ever ready + budget exhausted), not on signature
|
|
778
|
+
// matching alone: a missed match only costs a few seconds of extra retrying.
|
|
779
|
+
// Deterministic crash-loop detection (#1741). A crash counts toward
|
|
780
|
+
// "deterministic" ONLY after its signature reproduces across a respawn on the
|
|
781
|
+
// same slot — so every slot is guaranteed at least one self-heal attempt and
|
|
782
|
+
// a simultaneous attempt-0 crash storm (e.g. transient `spawn EAGAIN` under
|
|
783
|
+
// fork pressure) cannot be misclassified as deterministic. We short-circuit
|
|
784
|
+
// once enough DISTINCT slots have each reproduced: ≥2 normally, or 1 for a
|
|
785
|
+
// size-1 pool. Until then the structural floor (every slot exhausts its
|
|
786
|
+
// budget) still bounds the worst case, so a missed match only costs retries.
|
|
787
|
+
const lastStartupSignature = new Map();
|
|
788
|
+
const reproducedStartupSlots = new Set();
|
|
789
|
+
const deterministicSlotThreshold = Math.min(DETERMINISTIC_STARTUP_FINGERPRINT_THRESHOLD, size);
|
|
790
|
+
let deterministicStartupDetected = false;
|
|
791
|
+
let anyWorkerReachedReady = false;
|
|
792
|
+
// Cancel functions for in-flight startup backoffs (see abortableSleep). The
|
|
793
|
+
// backoff timer is ref'd so a retry actually runs; terminate() invokes these
|
|
794
|
+
// to clear pending backoffs and resolve their sleeps so the slot loops wake,
|
|
795
|
+
// see `terminated`, and give up — instead of the process staying pinned for
|
|
796
|
+
// the backoff cap after terminate (#1741).
|
|
797
|
+
const pendingStartupTimers = new Set();
|
|
798
|
+
const bringSlotReady = async (i) => {
|
|
799
|
+
for (let attempt = 0;; attempt++) {
|
|
800
|
+
const worker = workers[i];
|
|
801
|
+
if (!worker)
|
|
802
|
+
return; // terminated mid-startup
|
|
803
|
+
try {
|
|
804
|
+
await waitForWorkerReady(worker, poolOptions.workerReadyTimeoutMs);
|
|
805
|
+
anyWorkerReachedReady = true;
|
|
806
|
+
return; // ready — slot stays in activeSlots
|
|
807
|
+
}
|
|
808
|
+
catch (err) {
|
|
809
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
810
|
+
const sig = crashSignature(msg);
|
|
811
|
+
// Same signature as this slot's previous attempt => it survived a
|
|
812
|
+
// respawn, so retrying this slot is futile. (First crash has no prior
|
|
813
|
+
// signature, so attempt 0 never counts — every slot self-heals once.)
|
|
814
|
+
if (lastStartupSignature.get(i) === sig)
|
|
815
|
+
reproducedStartupSlots.add(i);
|
|
816
|
+
lastStartupSignature.set(i, sig);
|
|
817
|
+
if (!anyWorkerReachedReady && reproducedStartupSlots.size >= deterministicSlotThreshold) {
|
|
818
|
+
deterministicStartupDetected = true;
|
|
819
|
+
}
|
|
820
|
+
await worker.terminate().catch(() => undefined);
|
|
821
|
+
workers[i] = undefined;
|
|
822
|
+
const giveUp = terminated || deterministicStartupDetected || attempt >= STARTUP_RESTART_BUDGET;
|
|
823
|
+
if (giveUp) {
|
|
824
|
+
initialReadinessFailures.push(msg);
|
|
825
|
+
activeSlots.delete(i);
|
|
826
|
+
logger_js_1.logger.warn({ workerIndex: i, attempt, err: msg, deterministic: deterministicStartupDetected }, deterministicStartupDetected
|
|
827
|
+
? `Worker ${i} hit a deterministic startup crash-loop; dropping slot without further retries.`
|
|
828
|
+
: `Worker ${i} did not report ready after ${attempt + 1} attempt(s); dropping slot.`);
|
|
829
|
+
return;
|
|
830
|
+
}
|
|
831
|
+
// Transient: jittered backoff, then respawn the slot and retry.
|
|
832
|
+
await abortableSleep(startupBackoffMs(attempt), () => terminated || deterministicStartupDetected, pendingStartupTimers);
|
|
833
|
+
if (terminated || deterministicStartupDetected) {
|
|
834
|
+
initialReadinessFailures.push(msg);
|
|
835
|
+
activeSlots.delete(i);
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
logger_js_1.logger.warn({ workerIndex: i, attempt: attempt + 1 }, `Worker ${i} crashed during startup; respawning slot (self-heal attempt ${attempt + 1}/${STARTUP_RESTART_BUDGET}).`);
|
|
839
|
+
workers[i] = spawnAndCapture(workerUrl);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
};
|
|
843
|
+
// First dispatch awaits this; it settles every slot's bounded retry loop in
|
|
844
|
+
// parallel and drops the unrecoverable ones before any dispatch can fire.
|
|
845
|
+
const initialReadyGate = Promise.allSettled(workers.map((_, i) => bringSlotReady(i))).then(() => undefined);
|
|
846
|
+
const dispatch = async (items, onProgress, chunkHash) => {
|
|
847
|
+
// Await the initial-spawn readiness gate (F13). On first dispatch
|
|
848
|
+
// this blocks for up to poolOptions.workerReadyTimeoutMs while every initial
|
|
849
|
+
// worker's `{type:'ready'}` handshake is checked; on subsequent
|
|
850
|
+
// dispatches the promise is already settled and resolves
|
|
851
|
+
// synchronously. Slots whose initial worker crashed in top-of-
|
|
852
|
+
// script init have been dropped from `activeSlots` by the gate
|
|
853
|
+
// before this point — they don't surface here as "no active
|
|
854
|
+
// workers" until *all* initial slots fail.
|
|
855
|
+
await initialReadyGate;
|
|
856
|
+
if (poolBroken) {
|
|
857
|
+
const reason = poolFailure ? `: ${poolFailure.message}` : '';
|
|
858
|
+
throw new WorkerPoolDispatchError(`Worker pool circuit breaker tripped${reason}. ` +
|
|
859
|
+
`Subsequent dispatches require a fresh pool instance.`, []);
|
|
860
|
+
}
|
|
861
|
+
if (items.length === 0)
|
|
862
|
+
return [];
|
|
863
|
+
if (activeSlots.size === 0) {
|
|
864
|
+
const detail = initialReadinessFailures.length > 0
|
|
865
|
+
? ` after initial ready handshake: ${initialReadinessFailures.join('; ')}`
|
|
866
|
+
: '';
|
|
867
|
+
// The bounded self-heal exhausted (or short-circuited a deterministic
|
|
868
|
+
// crash-loop). Classify automatically so the caller renders the real
|
|
869
|
+
// cause without consulting any operator flag (#1741).
|
|
870
|
+
const crashClass = deterministicStartupDetected
|
|
871
|
+
? 'deterministic-startup'
|
|
872
|
+
: 'transient-exhausted';
|
|
873
|
+
throw new WorkerPoolInitializationError(`Worker pool has no active workers${detail}`, [], initialReadinessFailures, crashClass);
|
|
874
|
+
}
|
|
875
|
+
// Layer 3: filter out quarantined paths so a known-bad file never reaches
|
|
876
|
+
// a worker again this pool lifetime. The caller queries
|
|
877
|
+
// `getQuarantinedPaths` after dispatch to route filtered items.
|
|
878
|
+
const dispatchableItems = [];
|
|
879
|
+
for (const item of items) {
|
|
880
|
+
const path = itemPath(item);
|
|
881
|
+
if (path !== undefined && quarantine.has(path))
|
|
882
|
+
continue;
|
|
883
|
+
dispatchableItems.push(item);
|
|
884
|
+
}
|
|
885
|
+
if (dispatchableItems.length === 0)
|
|
886
|
+
return [];
|
|
887
|
+
const jobs = createJobs(dispatchableItems, poolOptions.subBatchSize, poolOptions.subBatchMaxBytes, poolOptions.subBatchIdleTimeoutMs, chunkHash);
|
|
888
|
+
return new Promise((resolve, reject) => {
|
|
889
|
+
const results = [];
|
|
890
|
+
const inFlightProgress = new Array(size).fill(0);
|
|
891
|
+
// Tracks which slots are currently mid-job so the "wake idle slots"
|
|
892
|
+
// pass after a requeue doesn't double-dispatch to a busy slot.
|
|
893
|
+
const busySlots = new Set();
|
|
894
|
+
// Per-conceptual-job (identified by startIndex) death count for the
|
|
895
|
+
// unattributable-crash path (F5). On the 2nd time a job dies with
|
|
896
|
+
// no exclusion attribution, requeueRemainder quarantines items[0]
|
|
897
|
+
// as a best-guess culprit to break the death loop.
|
|
898
|
+
const unattributedJobDeaths = new Map();
|
|
899
|
+
let completedFiles = 0;
|
|
900
|
+
let activeWorkers = 0;
|
|
901
|
+
let stopped = false;
|
|
902
|
+
let maxReported = 0;
|
|
903
|
+
const wakeIdleSlots = () => {
|
|
904
|
+
if (stopped || jobs.length === 0)
|
|
905
|
+
return;
|
|
906
|
+
for (const slot of activeSlots) {
|
|
907
|
+
if (busySlots.has(slot))
|
|
908
|
+
continue;
|
|
909
|
+
if (jobs.length === 0)
|
|
910
|
+
break;
|
|
911
|
+
runWorker(slot);
|
|
912
|
+
}
|
|
913
|
+
};
|
|
914
|
+
const reportProgress = () => {
|
|
915
|
+
if (!onProgress)
|
|
916
|
+
return;
|
|
917
|
+
const inFlight = inFlightProgress.reduce((sum, value) => sum + value, 0);
|
|
918
|
+
const next = Math.min(dispatchableItems.length, Math.max(maxReported, completedFiles + inFlight));
|
|
919
|
+
if (next === maxReported)
|
|
920
|
+
return;
|
|
921
|
+
maxReported = next;
|
|
922
|
+
onProgress(next);
|
|
923
|
+
};
|
|
924
|
+
const retireWorkerAfterTimeout = (worker, workerIndex, reason) => {
|
|
925
|
+
let cleaned = false;
|
|
926
|
+
let terminateStarted = false;
|
|
927
|
+
let resolveSafePoint;
|
|
928
|
+
const safePoint = new Promise((resolve) => {
|
|
929
|
+
resolveSafePoint = resolve;
|
|
930
|
+
});
|
|
931
|
+
// A message/messageerror proves the worker is executing JS again; an
|
|
932
|
+
// exit/error means the thread is gone. Either way `worker.terminate()`
|
|
933
|
+
// can no longer land mid-N-API call (#2432), so shutdown's drain may
|
|
934
|
+
// stop waiting.
|
|
935
|
+
function markSafeToTerminate() {
|
|
936
|
+
record.safeToTerminate = true;
|
|
937
|
+
resolveSafePoint();
|
|
938
|
+
}
|
|
939
|
+
function cleanupRetired() {
|
|
940
|
+
if (cleaned)
|
|
941
|
+
return;
|
|
942
|
+
cleaned = true;
|
|
943
|
+
worker.removeListener('message', onRetiredMessage);
|
|
944
|
+
worker.removeListener('error', onRetiredError);
|
|
945
|
+
worker.removeListener('exit', onRetiredExit);
|
|
946
|
+
worker.removeListener('messageerror', onRetiredMessageError);
|
|
947
|
+
retiredWorkers.delete(record);
|
|
948
|
+
}
|
|
949
|
+
async function terminateRetired() {
|
|
950
|
+
if (terminateStarted)
|
|
951
|
+
return;
|
|
952
|
+
terminateStarted = true;
|
|
953
|
+
cleanupRetired();
|
|
954
|
+
await worker.terminate().catch(() => undefined);
|
|
955
|
+
}
|
|
956
|
+
function terminateWhenBackInJs() {
|
|
957
|
+
markSafeToTerminate();
|
|
958
|
+
void terminateRetired();
|
|
959
|
+
}
|
|
960
|
+
function onRetiredMessage(raw) {
|
|
961
|
+
if (raw === null || typeof raw !== 'object')
|
|
962
|
+
return;
|
|
963
|
+
const type = raw.type;
|
|
964
|
+
if (type === 'sub-batch-done' || type === 'result' || type === 'error') {
|
|
965
|
+
terminateWhenBackInJs();
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
const onRetiredError = () => {
|
|
969
|
+
markSafeToTerminate();
|
|
970
|
+
cleanupRetired();
|
|
971
|
+
};
|
|
972
|
+
const onRetiredExit = () => {
|
|
973
|
+
markSafeToTerminate();
|
|
974
|
+
cleanupRetired();
|
|
975
|
+
};
|
|
976
|
+
const onRetiredMessageError = () => terminateWhenBackInJs();
|
|
977
|
+
const record = {
|
|
978
|
+
worker,
|
|
979
|
+
workerIndex,
|
|
980
|
+
reason,
|
|
981
|
+
cleanup: cleanupRetired,
|
|
982
|
+
terminate: terminateRetired,
|
|
983
|
+
safeToTerminate: false,
|
|
984
|
+
safePoint,
|
|
985
|
+
};
|
|
986
|
+
retiredWorkers.add(record);
|
|
987
|
+
worker.on('message', onRetiredMessage);
|
|
988
|
+
worker.once('error', onRetiredError);
|
|
989
|
+
worker.once('exit', onRetiredExit);
|
|
990
|
+
worker.once('messageerror', onRetiredMessageError);
|
|
991
|
+
worker.unref?.();
|
|
992
|
+
logger_js_1.logger.warn({ workerIndex, reason }, `Worker ${workerIndex} timed out; retiring without immediate terminate to avoid aborting native parser state.`);
|
|
993
|
+
};
|
|
994
|
+
const removeWorkerFromSlot = async (workerIndex, mode, reason) => {
|
|
995
|
+
const existing = workers[workerIndex];
|
|
996
|
+
workers[workerIndex] = undefined;
|
|
997
|
+
if (!existing)
|
|
998
|
+
return;
|
|
999
|
+
if (mode === 'retire') {
|
|
1000
|
+
retireWorkerAfterTimeout(existing, workerIndex, reason);
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
await existing.terminate().catch(() => undefined);
|
|
1004
|
+
};
|
|
1005
|
+
const replaceWorker = async (workerIndex, mode = 'terminate', reason = 'replacing worker') => {
|
|
1006
|
+
await removeWorkerFromSlot(workerIndex, mode, reason);
|
|
1007
|
+
if (stopped)
|
|
1008
|
+
return false;
|
|
1009
|
+
const replacement = spawnAndCapture(workerUrl);
|
|
1010
|
+
try {
|
|
1011
|
+
await waitForWorkerReady(replacement, poolOptions.workerReadyTimeoutMs);
|
|
1012
|
+
}
|
|
1013
|
+
catch (err) {
|
|
1014
|
+
await replacement.terminate().catch(() => undefined);
|
|
1015
|
+
logger_js_1.logger.warn({ workerIndex, error: err instanceof Error ? err.message : String(err) }, `Worker ${workerIndex} replacement failed to come online; dropping slot.`);
|
|
1016
|
+
return false;
|
|
1017
|
+
}
|
|
1018
|
+
if (stopped) {
|
|
1019
|
+
await replacement.terminate().catch(() => undefined);
|
|
1020
|
+
return false;
|
|
1021
|
+
}
|
|
1022
|
+
workers[workerIndex] = replacement;
|
|
1023
|
+
// U12: bump the slot generation atomically with the worker swap so
|
|
1024
|
+
// any late event from the OLD worker that somehow slipped past
|
|
1025
|
+
// cleanup() carries a stale generation and short-circuits in the
|
|
1026
|
+
// handler guard below. Increment AFTER `workers[workerIndex]` is
|
|
1027
|
+
// updated so observers (getStats) see the new pair consistently.
|
|
1028
|
+
slotGenerations[workerIndex]++;
|
|
1029
|
+
return true;
|
|
1030
|
+
};
|
|
1031
|
+
// Terminal failure path: trip the pool circuit breaker and reject the
|
|
1032
|
+
// outer dispatch promise with the cumulative exclude paths. This is the
|
|
1033
|
+
// ONLY place that sets `poolBroken = true` — recoverable single-worker
|
|
1034
|
+
// failures stay local to `handleWorkerDeath`.
|
|
1035
|
+
//
|
|
1036
|
+
// Reject the caller's promise BEFORE awaiting `worker.terminate()` so a
|
|
1037
|
+
// stuck terminate (OOM-killed thread, hung native addon) can't block
|
|
1038
|
+
// the caller indefinitely. Worker cleanup runs in the background; the
|
|
1039
|
+
// next `dispatch` call sees `poolBroken=true` and rejects up front.
|
|
1040
|
+
const tripBreaker = (err) => {
|
|
1041
|
+
poolBroken = true;
|
|
1042
|
+
poolFailure = err;
|
|
1043
|
+
if (stopped)
|
|
1044
|
+
return;
|
|
1045
|
+
stopped = true;
|
|
1046
|
+
reject(err);
|
|
1047
|
+
const liveWorkers = workers.slice();
|
|
1048
|
+
for (let i = 0; i < workers.length; i++)
|
|
1049
|
+
workers[i] = undefined;
|
|
1050
|
+
// #2432: a live worker with a job in flight may be inside an N-API
|
|
1051
|
+
// call — direct terminate risks the same native abort as the retired
|
|
1052
|
+
// case. Route busy workers through the retire path (terminate at
|
|
1053
|
+
// their next JS-visible safe point); idle workers are parked in the
|
|
1054
|
+
// JS event loop and terminate safely right away.
|
|
1055
|
+
const idleWorkers = [];
|
|
1056
|
+
for (let i = 0; i < liveWorkers.length; i++) {
|
|
1057
|
+
const worker = liveWorkers[i];
|
|
1058
|
+
if (worker === undefined)
|
|
1059
|
+
continue;
|
|
1060
|
+
if (busySlots.has(i)) {
|
|
1061
|
+
retireWorkerAfterTimeout(worker, i, 'circuit breaker tripped with job in flight');
|
|
1062
|
+
}
|
|
1063
|
+
else {
|
|
1064
|
+
idleWorkers.push(worker);
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
activeSlots.clear();
|
|
1068
|
+
void terminateTrackedWorkers(idleWorkers);
|
|
1069
|
+
};
|
|
1070
|
+
const maybeDone = () => {
|
|
1071
|
+
if (stopped)
|
|
1072
|
+
return;
|
|
1073
|
+
if (jobs.length === 0 && activeWorkers === 0) {
|
|
1074
|
+
stopped = true;
|
|
1075
|
+
results.sort((a, b) => a.startIndex - b.startIndex);
|
|
1076
|
+
if (onProgress && maxReported < dispatchableItems.length)
|
|
1077
|
+
onProgress(dispatchableItems.length);
|
|
1078
|
+
resolve(results.map((result) => result.data));
|
|
1079
|
+
}
|
|
1080
|
+
};
|
|
1081
|
+
// Re-queue the non-quarantined remainder of a dead worker's job so a
|
|
1082
|
+
// healthy worker can finish the work. Earlier items in the dead job
|
|
1083
|
+
// were never flushed back to the main thread, so they must be
|
|
1084
|
+
// re-processed. The new job carries the existing job's startIndex so
|
|
1085
|
+
// result ordering is preserved. `cumulativeTimeoutMs` is carried
|
|
1086
|
+
// forward unchanged — the death itself consumed no timeout budget,
|
|
1087
|
+
// so charging another timeoutMs here would double-bill the next
|
|
1088
|
+
// `requeueAfterTimeout` call's accumulation.
|
|
1089
|
+
//
|
|
1090
|
+
// Unattributed-death tracking (F5): when called with `excluded=[]`
|
|
1091
|
+
// the worker died without identifying a culprit (no `starting-file`
|
|
1092
|
+
// observed, `lastProgress=0`, `items[lastProgress]` heuristic empty).
|
|
1093
|
+
// The first time, re-queue the job intact and hope another worker
|
|
1094
|
+
// succeeds. On the second such death of the SAME conceptual job
|
|
1095
|
+
// (same `startIndex`), quarantine `items[0]` as a best-guess
|
|
1096
|
+
// culprit so the next attempt isn't condemned to the same death.
|
|
1097
|
+
// This bounds the unattributable-crash death loop and ensures the
|
|
1098
|
+
// pool's final `quarantinedPaths` snapshot carries SOME signal
|
|
1099
|
+
// for downstream diagnostics instead of silently re-hitting the
|
|
1100
|
+
// bad file.
|
|
1101
|
+
const requeueRemainder = (job, excluded) => {
|
|
1102
|
+
let effectiveExcluded = excluded;
|
|
1103
|
+
if (excluded.length === 0) {
|
|
1104
|
+
const deaths = (unattributedJobDeaths.get(job.startIndex) ?? 0) + 1;
|
|
1105
|
+
unattributedJobDeaths.set(job.startIndex, deaths);
|
|
1106
|
+
if (deaths < 2) {
|
|
1107
|
+
jobs.unshift(job);
|
|
1108
|
+
return;
|
|
1109
|
+
}
|
|
1110
|
+
const firstPath = itemPath(job.items[0]);
|
|
1111
|
+
if (firstPath !== undefined) {
|
|
1112
|
+
quarantine.add(firstPath);
|
|
1113
|
+
logger_js_1.logger.warn({ startIndex: job.startIndex, firstPath, deaths }, `Conceptual job ${job.startIndex} died ${deaths} times unattributably; ` +
|
|
1114
|
+
`quarantining items[0] (${firstPath}) as best-guess culprit.`);
|
|
1115
|
+
effectiveExcluded = [firstPath];
|
|
1116
|
+
}
|
|
1117
|
+
else {
|
|
1118
|
+
// No identifiable file on items[0] either — drop the job to
|
|
1119
|
+
// break the loop. The breaker counter still increments via
|
|
1120
|
+
// handleWorkerDeath, so consecutive unattributable deaths
|
|
1121
|
+
// eventually trip it even without quarantine signal.
|
|
1122
|
+
logger_js_1.logger.warn({ startIndex: job.startIndex, deaths }, `Conceptual job ${job.startIndex} died ${deaths} times unattributably with ` +
|
|
1123
|
+
`no identifiable file; dropping job to break the death loop.`);
|
|
1124
|
+
return;
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
const excludeSet = new Set(effectiveExcluded);
|
|
1128
|
+
const filtered = job.items.filter((item) => {
|
|
1129
|
+
const p = itemPath(item);
|
|
1130
|
+
return p === undefined || !excludeSet.has(p);
|
|
1131
|
+
});
|
|
1132
|
+
if (filtered.length === 0)
|
|
1133
|
+
return;
|
|
1134
|
+
jobs.unshift({
|
|
1135
|
+
startIndex: job.startIndex,
|
|
1136
|
+
items: filtered,
|
|
1137
|
+
estimatedBytes: filtered.reduce((sum, item) => sum + estimateItemBytes(item), 0),
|
|
1138
|
+
attempt: job.attempt,
|
|
1139
|
+
splitDepth: job.splitDepth,
|
|
1140
|
+
chunkHash: job.chunkHash,
|
|
1141
|
+
timeoutMs: job.timeoutMs,
|
|
1142
|
+
cumulativeTimeoutMs: job.cumulativeTimeoutMs,
|
|
1143
|
+
});
|
|
1144
|
+
};
|
|
1145
|
+
// Recoverable worker death — quarantine the in-flight path, attempt
|
|
1146
|
+
// to respawn the slot, re-queue the rest of the job, and continue.
|
|
1147
|
+
// Trips the circuit breaker only when consecutiveFailures crosses the
|
|
1148
|
+
// threshold OR all slots have exhausted their respawn budget.
|
|
1149
|
+
const handleWorkerDeath = async (workerIndex, reason, excludePaths, removalMode = 'terminate') => {
|
|
1150
|
+
if (stopped)
|
|
1151
|
+
return;
|
|
1152
|
+
consecutiveFailuresPerSlot[workerIndex]++;
|
|
1153
|
+
for (const p of excludePaths) {
|
|
1154
|
+
if (p)
|
|
1155
|
+
quarantine.add(p);
|
|
1156
|
+
}
|
|
1157
|
+
if (consecutiveFailuresPerSlot[workerIndex] >= poolOptions.consecutiveFailureThreshold) {
|
|
1158
|
+
tripBreaker(new WorkerPoolDispatchError(`${reason}. Pool circuit breaker tripped: slot ${workerIndex} hit ` +
|
|
1159
|
+
`${consecutiveFailuresPerSlot[workerIndex]} consecutive failures ` +
|
|
1160
|
+
`(threshold: ${poolOptions.consecutiveFailureThreshold}).`, quarantine.snapshot()));
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
1163
|
+
respawnCount[workerIndex]++;
|
|
1164
|
+
if (respawnCount[workerIndex] > poolOptions.maxRespawnsPerSlot) {
|
|
1165
|
+
logger_js_1.logger.warn({
|
|
1166
|
+
workerIndex,
|
|
1167
|
+
respawnCount: respawnCount[workerIndex],
|
|
1168
|
+
maxRespawns: poolOptions.maxRespawnsPerSlot,
|
|
1169
|
+
reason,
|
|
1170
|
+
}, `Worker ${workerIndex} exceeded respawn budget; dropping slot.`);
|
|
1171
|
+
await removeWorkerFromSlot(workerIndex, removalMode, reason);
|
|
1172
|
+
activeSlots.delete(workerIndex);
|
|
1173
|
+
if (activeSlots.size === 0) {
|
|
1174
|
+
tripBreaker(new WorkerPoolDispatchError(`${reason}. All ${size} worker slot(s) exhausted their respawn budget.`, quarantine.snapshot()));
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
logger_js_1.logger.warn({
|
|
1180
|
+
workerIndex,
|
|
1181
|
+
respawnCount: respawnCount[workerIndex],
|
|
1182
|
+
reason,
|
|
1183
|
+
excludePaths,
|
|
1184
|
+
}, `Worker ${workerIndex} died; respawning slot (attempt ${respawnCount[workerIndex]}/${poolOptions.maxRespawnsPerSlot}).`);
|
|
1185
|
+
const respawned = await replaceWorker(workerIndex, removalMode, reason);
|
|
1186
|
+
if (!respawned) {
|
|
1187
|
+
activeSlots.delete(workerIndex);
|
|
1188
|
+
if (activeSlots.size === 0) {
|
|
1189
|
+
tripBreaker(new WorkerPoolDispatchError(`${reason}. Replacement worker startup failed and no slots remain.`, quarantine.snapshot()));
|
|
1190
|
+
}
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
};
|
|
1194
|
+
const requeueAfterTimeout = (workerIndex, job, lastProgress, inFlightPath) => {
|
|
1195
|
+
const nextTimeout = Math.ceil(job.timeoutMs * poolOptions.timeoutBackoffFactor);
|
|
1196
|
+
const nextCumulative = job.cumulativeTimeoutMs + nextTimeout;
|
|
1197
|
+
// Layer 5: respect the per-job cumulative timeout budget. Once
|
|
1198
|
+
// exhausted, surface the in-flight file via WorkerPoolDispatchError
|
|
1199
|
+
// instead of letting exponential backoff stall further.
|
|
1200
|
+
if (nextCumulative > poolOptions.maxCumulativeTimeoutMs) {
|
|
1201
|
+
const firstPath = itemPath(job.items[0]);
|
|
1202
|
+
const exhausted = inFlightPath !== undefined
|
|
1203
|
+
? [inFlightPath]
|
|
1204
|
+
: firstPath !== undefined
|
|
1205
|
+
? [firstPath]
|
|
1206
|
+
: [];
|
|
1207
|
+
logger_js_1.logger.warn({
|
|
1208
|
+
workerIndex,
|
|
1209
|
+
cumulativeMs: job.cumulativeTimeoutMs,
|
|
1210
|
+
nextCumulativeMs: nextCumulative,
|
|
1211
|
+
maxCumulativeMs: poolOptions.maxCumulativeTimeoutMs,
|
|
1212
|
+
exhausted,
|
|
1213
|
+
}, `Worker ${workerIndex} parse job exhausted cumulative timeout budget. Surfacing in-flight file(s).`);
|
|
1214
|
+
return {
|
|
1215
|
+
kind: 'give-up',
|
|
1216
|
+
reason: `Worker ${workerIndex} parse job exhausted cumulative timeout budget ` +
|
|
1217
|
+
`(${(nextCumulative / 1000).toFixed(0)}s > ${(poolOptions.maxCumulativeTimeoutMs / 1000).toFixed(0)}s cap)`,
|
|
1218
|
+
excludePaths: exhausted,
|
|
1219
|
+
};
|
|
1220
|
+
}
|
|
1221
|
+
if (job.items.length > 1) {
|
|
1222
|
+
const midpoint = Math.ceil(job.items.length / 2);
|
|
1223
|
+
const firstItems = job.items.slice(0, midpoint);
|
|
1224
|
+
const secondItems = job.items.slice(midpoint);
|
|
1225
|
+
const first = {
|
|
1226
|
+
startIndex: job.startIndex,
|
|
1227
|
+
items: firstItems,
|
|
1228
|
+
estimatedBytes: firstItems.reduce((sum, item) => sum + estimateItemBytes(item), 0),
|
|
1229
|
+
attempt: job.attempt,
|
|
1230
|
+
splitDepth: job.splitDepth + 1,
|
|
1231
|
+
chunkHash: job.chunkHash,
|
|
1232
|
+
timeoutMs: nextTimeout,
|
|
1233
|
+
cumulativeTimeoutMs: nextCumulative,
|
|
1234
|
+
};
|
|
1235
|
+
const second = {
|
|
1236
|
+
startIndex: job.startIndex + midpoint,
|
|
1237
|
+
items: secondItems,
|
|
1238
|
+
estimatedBytes: secondItems.reduce((sum, item) => sum + estimateItemBytes(item), 0),
|
|
1239
|
+
attempt: job.attempt,
|
|
1240
|
+
splitDepth: job.splitDepth + 1,
|
|
1241
|
+
chunkHash: job.chunkHash,
|
|
1242
|
+
timeoutMs: nextTimeout,
|
|
1243
|
+
cumulativeTimeoutMs: nextCumulative,
|
|
1244
|
+
};
|
|
1245
|
+
logger_js_1.logger.warn({
|
|
1246
|
+
workerIndex,
|
|
1247
|
+
timeoutSec: job.timeoutMs / 1000,
|
|
1248
|
+
items: job.items.length,
|
|
1249
|
+
estimatedBytes: job.estimatedBytes,
|
|
1250
|
+
lastProgress,
|
|
1251
|
+
firstSplitItems: first.items.length,
|
|
1252
|
+
secondSplitItems: second.items.length,
|
|
1253
|
+
nextTimeoutSec: nextTimeout / 1000,
|
|
1254
|
+
}, `Worker ${workerIndex} parse job idle timeout. Splitting into ${first.items.length}/${second.items.length} item jobs.`);
|
|
1255
|
+
// Preserve intuitive retry order; final result order is still enforced by startIndex sort.
|
|
1256
|
+
jobs.unshift(first, second);
|
|
1257
|
+
return { kind: 'retry' };
|
|
1258
|
+
}
|
|
1259
|
+
const nextAttempt = job.attempt + 1;
|
|
1260
|
+
if (nextAttempt <= poolOptions.maxTimeoutRetries) {
|
|
1261
|
+
logger_js_1.logger.warn({
|
|
1262
|
+
workerIndex,
|
|
1263
|
+
timeoutSec: job.timeoutMs / 1000,
|
|
1264
|
+
attempt: nextAttempt,
|
|
1265
|
+
maxAttempts: poolOptions.maxTimeoutRetries + 1,
|
|
1266
|
+
nextTimeoutSec: nextTimeout / 1000,
|
|
1267
|
+
}, `Worker ${workerIndex} parse job idle timeout (single item). Retrying with ${nextTimeout / 1000}s timeout.`);
|
|
1268
|
+
jobs.unshift({
|
|
1269
|
+
...job,
|
|
1270
|
+
attempt: nextAttempt,
|
|
1271
|
+
timeoutMs: nextTimeout,
|
|
1272
|
+
cumulativeTimeoutMs: nextCumulative,
|
|
1273
|
+
});
|
|
1274
|
+
return { kind: 'retry' };
|
|
1275
|
+
}
|
|
1276
|
+
const stalledPath = inFlightPath ?? itemPath(job.items[0]);
|
|
1277
|
+
const excludes = stalledPath ? [stalledPath] : [];
|
|
1278
|
+
logger_js_1.logger.warn({
|
|
1279
|
+
workerIndex,
|
|
1280
|
+
timeoutSec: job.timeoutMs / 1000,
|
|
1281
|
+
stalledPath,
|
|
1282
|
+
cumulativeMs: job.cumulativeTimeoutMs,
|
|
1283
|
+
}, `Worker ${workerIndex} parse job idle timeout exhausted retries; quarantining file and respawning slot.`);
|
|
1284
|
+
return {
|
|
1285
|
+
kind: 'give-up',
|
|
1286
|
+
reason: `Worker ${workerIndex} parse job idle timeout after ${job.timeoutMs / 1000}s ` +
|
|
1287
|
+
`(single item${stalledPath ? `: ${stalledPath}` : ''}, ` +
|
|
1288
|
+
`${job.estimatedBytes} bytes, last progress: ${lastProgress})`,
|
|
1289
|
+
excludePaths: excludes,
|
|
1290
|
+
};
|
|
1291
|
+
};
|
|
1292
|
+
const runWorker = (workerIndex) => {
|
|
1293
|
+
if (stopped)
|
|
1294
|
+
return;
|
|
1295
|
+
if (!activeSlots.has(workerIndex))
|
|
1296
|
+
return;
|
|
1297
|
+
// Drop quarantined items that may have been re-queued before a death
|
|
1298
|
+
// added them to quarantine — keeps the worker from ever seeing a
|
|
1299
|
+
// known-bad file. Loops until we find a job with dispatchable items
|
|
1300
|
+
// or exhaust the queue (avoids recursion depth growth when many
|
|
1301
|
+
// queued jobs are fully quarantined back-to-back).
|
|
1302
|
+
let job;
|
|
1303
|
+
while ((job = jobs.shift()) !== undefined) {
|
|
1304
|
+
if (quarantine.size === 0)
|
|
1305
|
+
break;
|
|
1306
|
+
const dispatchable = job.items.filter((item) => {
|
|
1307
|
+
const p = itemPath(item);
|
|
1308
|
+
return p === undefined || !quarantine.has(p);
|
|
1309
|
+
});
|
|
1310
|
+
if (dispatchable.length === 0)
|
|
1311
|
+
continue;
|
|
1312
|
+
if (dispatchable.length !== job.items.length) {
|
|
1313
|
+
job.items = dispatchable;
|
|
1314
|
+
job.estimatedBytes = dispatchable.reduce((sum, item) => sum + estimateItemBytes(item), 0);
|
|
1315
|
+
}
|
|
1316
|
+
break;
|
|
1317
|
+
}
|
|
1318
|
+
if (!job) {
|
|
1319
|
+
maybeDone();
|
|
1320
|
+
return;
|
|
1321
|
+
}
|
|
1322
|
+
activeWorkers++;
|
|
1323
|
+
busySlots.add(workerIndex);
|
|
1324
|
+
inFlightProgress[workerIndex] = 0;
|
|
1325
|
+
const worker = workers[workerIndex];
|
|
1326
|
+
if (!worker) {
|
|
1327
|
+
// Slot's worker is undefined — typically mid-respawn (replaceWorker
|
|
1328
|
+
// clears `workers[i]` before awaiting `waitForWorkerOnline`). The
|
|
1329
|
+
// respawn IIFE / handleWorkerDeath that started the respawn owns
|
|
1330
|
+
// calling runWorker when the new worker is online; we just
|
|
1331
|
+
// unshift the job and bail.
|
|
1332
|
+
//
|
|
1333
|
+
// Do NOT call wakeIdleSlots from here: it would iterate
|
|
1334
|
+
// `activeSlots` and re-enter `runWorker` for this same slot
|
|
1335
|
+
// (now non-busy), find `workers[i]` still undefined, and
|
|
1336
|
+
// recurse until the call stack overflows.
|
|
1337
|
+
activeWorkers--;
|
|
1338
|
+
busySlots.delete(workerIndex);
|
|
1339
|
+
jobs.unshift(job);
|
|
1340
|
+
maybeDone();
|
|
1341
|
+
return;
|
|
1342
|
+
}
|
|
1343
|
+
let settled = false;
|
|
1344
|
+
let waitingForFlush = false;
|
|
1345
|
+
let idleTimer = null;
|
|
1346
|
+
let lastProgress = 0;
|
|
1347
|
+
// Authoritative in-flight file from the worker's `starting-file`
|
|
1348
|
+
// message. Cleared on `progress` so a between-files crash falls
|
|
1349
|
+
// back to the `items[lastProgress]` heuristic, which then points
|
|
1350
|
+
// at the next file (the one about to start) — the right guess.
|
|
1351
|
+
let inFlightPath;
|
|
1352
|
+
const resolveExcludePaths = () => {
|
|
1353
|
+
if (inFlightPath !== undefined)
|
|
1354
|
+
return [inFlightPath];
|
|
1355
|
+
return inFlightExcludePath(job, lastProgress);
|
|
1356
|
+
};
|
|
1357
|
+
const cleanup = () => {
|
|
1358
|
+
if (idleTimer)
|
|
1359
|
+
clearTimeout(idleTimer);
|
|
1360
|
+
worker.removeListener('message', handler);
|
|
1361
|
+
worker.removeListener('error', errorHandler);
|
|
1362
|
+
worker.removeListener('exit', exitHandler);
|
|
1363
|
+
worker.removeListener('messageerror', messageErrorHandler);
|
|
1364
|
+
};
|
|
1365
|
+
const finishJob = () => {
|
|
1366
|
+
activeWorkers--;
|
|
1367
|
+
busySlots.delete(workerIndex);
|
|
1368
|
+
inFlightProgress[workerIndex] = 0;
|
|
1369
|
+
runWorker(workerIndex);
|
|
1370
|
+
maybeDone();
|
|
1371
|
+
};
|
|
1372
|
+
// Recover-and-resume flow shared by all in-pool worker death sites
|
|
1373
|
+
// (`error`, `exit`, msg-channel error). Bridges the per-job teardown
|
|
1374
|
+
// into the pool-level handleWorkerDeath recovery + breaker logic.
|
|
1375
|
+
const recoverAndResume = async (reason, excludePaths) => {
|
|
1376
|
+
activeWorkers--;
|
|
1377
|
+
busySlots.delete(workerIndex);
|
|
1378
|
+
inFlightProgress[workerIndex] = 0;
|
|
1379
|
+
requeueRemainder(job, excludePaths);
|
|
1380
|
+
await handleWorkerDeath(workerIndex, reason, excludePaths);
|
|
1381
|
+
if (stopped)
|
|
1382
|
+
return;
|
|
1383
|
+
// Slot may have been dropped or respawned. Kick the current slot
|
|
1384
|
+
// if still active, then wake any other idle live slots so the
|
|
1385
|
+
// requeued remainder can be picked up immediately (without this,
|
|
1386
|
+
// dropped-slot scenarios can deadlock when no other slot is
|
|
1387
|
+
// currently busy and the next finishJob never fires).
|
|
1388
|
+
if (activeSlots.has(workerIndex)) {
|
|
1389
|
+
runWorker(workerIndex);
|
|
1390
|
+
}
|
|
1391
|
+
wakeIdleSlots();
|
|
1392
|
+
maybeDone();
|
|
1393
|
+
};
|
|
1394
|
+
let stallCreditUsed = false;
|
|
1395
|
+
let stallAtArm = 0;
|
|
1396
|
+
const resetIdleTimer = () => {
|
|
1397
|
+
if (idleTimer)
|
|
1398
|
+
clearTimeout(idleTimer);
|
|
1399
|
+
stallAtArm = stallTracker.read();
|
|
1400
|
+
idleTimer = setTimeout(() => {
|
|
1401
|
+
if (!settled) {
|
|
1402
|
+
// #2649: when at least STALL_CREDIT_FRACTION of the timeout
|
|
1403
|
+
// window was main-thread stall (GC pressure near the heap
|
|
1404
|
+
// limit), the worker's progress messages were starved, not
|
|
1405
|
+
// absent — credit the stall once per job and re-arm instead
|
|
1406
|
+
// of splitting/retiring a healthy worker.
|
|
1407
|
+
const stallMs = stallTracker.read() - stallAtArm;
|
|
1408
|
+
if (!stallCreditUsed && stallMs >= job.timeoutMs * STALL_CREDIT_FRACTION) {
|
|
1409
|
+
stallCreditUsed = true;
|
|
1410
|
+
logger_js_1.logger.warn({ workerIndex, stallMs: Math.round(stallMs), timeoutMs: job.timeoutMs }, `Worker ${workerIndex} idle timeout overlapped a main-thread stall (GC pressure); re-arming once instead of retiring.`);
|
|
1411
|
+
resetIdleTimer();
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
settled = true;
|
|
1415
|
+
cleanup();
|
|
1416
|
+
inFlightProgress[workerIndex] = 0;
|
|
1417
|
+
const stalledPath = inFlightPath;
|
|
1418
|
+
const decision = requeueAfterTimeout(workerIndex, job, lastProgress, stalledPath);
|
|
1419
|
+
if (decision.kind === 'give-up') {
|
|
1420
|
+
// Give-up path: re-queue the non-quarantined remainder,
|
|
1421
|
+
// then await handleWorkerDeath so we know when the slot
|
|
1422
|
+
// is respawned (or dropped) and can dispatch the next
|
|
1423
|
+
// job deterministically.
|
|
1424
|
+
void (async () => {
|
|
1425
|
+
activeWorkers--;
|
|
1426
|
+
busySlots.delete(workerIndex);
|
|
1427
|
+
requeueRemainder(job, decision.excludePaths);
|
|
1428
|
+
await handleWorkerDeath(workerIndex, decision.reason, decision.excludePaths, 'retire');
|
|
1429
|
+
if (stopped)
|
|
1430
|
+
return;
|
|
1431
|
+
if (activeSlots.has(workerIndex))
|
|
1432
|
+
runWorker(workerIndex);
|
|
1433
|
+
wakeIdleSlots();
|
|
1434
|
+
maybeDone();
|
|
1435
|
+
})();
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
1438
|
+
// Timeout-retry path: enforce the per-slot respawn budget
|
|
1439
|
+
// BEFORE spawning a fresh worker. The previous version
|
|
1440
|
+
// called `replaceWorker` unconditionally, letting a
|
|
1441
|
+
// chronically-timing-out slot respawn forever.
|
|
1442
|
+
//
|
|
1443
|
+
// Also increment `consecutiveFailuresPerSlot` here so the
|
|
1444
|
+
// per-slot circuit breaker sees pure-timeout death loops
|
|
1445
|
+
// (not just crashes). Without it, a slot that consistently
|
|
1446
|
+
// times out will consume its full respawn budget without
|
|
1447
|
+
// the breaker ever firing — chronic timeouts are
|
|
1448
|
+
// structurally the same kind of failure as crashes from
|
|
1449
|
+
// the breaker's perspective.
|
|
1450
|
+
void (async () => {
|
|
1451
|
+
try {
|
|
1452
|
+
respawnCount[workerIndex]++;
|
|
1453
|
+
consecutiveFailuresPerSlot[workerIndex]++;
|
|
1454
|
+
// Complete the per-slot breaker contract on the
|
|
1455
|
+
// timeout-retry path. Without this check, chronic
|
|
1456
|
+
// pure-timeout deaths accumulated `consecutive-
|
|
1457
|
+
// FailuresPerSlot` increments that never tripped the
|
|
1458
|
+
// breaker — only the `respawnCount > maxRespawnsPerSlot`
|
|
1459
|
+
// slot-drop path was active. Now timeouts trip the
|
|
1460
|
+
// breaker on the same threshold as crashes, which is
|
|
1461
|
+
// what the increment was meant to enable.
|
|
1462
|
+
if (consecutiveFailuresPerSlot[workerIndex] >=
|
|
1463
|
+
poolOptions.consecutiveFailureThreshold) {
|
|
1464
|
+
logger_js_1.logger.warn({
|
|
1465
|
+
workerIndex,
|
|
1466
|
+
consecutiveFailures: consecutiveFailuresPerSlot[workerIndex],
|
|
1467
|
+
threshold: poolOptions.consecutiveFailureThreshold,
|
|
1468
|
+
}, `Worker ${workerIndex} hit consecutive-failure threshold on idle-timeout retry; tripping circuit breaker.`);
|
|
1469
|
+
await removeWorkerFromSlot(workerIndex, 'retire', 'idle-timeout retry consecutive-failure threshold');
|
|
1470
|
+
activeSlots.delete(workerIndex);
|
|
1471
|
+
tripBreaker(new WorkerPoolDispatchError(`Worker pool tripped circuit breaker: slot ${workerIndex} hit ` +
|
|
1472
|
+
`${consecutiveFailuresPerSlot[workerIndex]} consecutive failures ` +
|
|
1473
|
+
`(threshold: ${poolOptions.consecutiveFailureThreshold}).`, quarantine.snapshot()));
|
|
1474
|
+
return;
|
|
1475
|
+
}
|
|
1476
|
+
if (respawnCount[workerIndex] > poolOptions.maxRespawnsPerSlot) {
|
|
1477
|
+
logger_js_1.logger.warn({
|
|
1478
|
+
workerIndex,
|
|
1479
|
+
respawnCount: respawnCount[workerIndex],
|
|
1480
|
+
maxRespawns: poolOptions.maxRespawnsPerSlot,
|
|
1481
|
+
}, `Worker ${workerIndex} exceeded respawn budget during idle-timeout retry; dropping slot.`);
|
|
1482
|
+
await removeWorkerFromSlot(workerIndex, 'retire', 'idle-timeout retry respawn budget exhausted');
|
|
1483
|
+
activeSlots.delete(workerIndex);
|
|
1484
|
+
}
|
|
1485
|
+
else {
|
|
1486
|
+
const respawned = await replaceWorker(workerIndex, 'retire', 'idle-timeout retry');
|
|
1487
|
+
if (!respawned) {
|
|
1488
|
+
activeSlots.delete(workerIndex);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
finally {
|
|
1493
|
+
activeWorkers--;
|
|
1494
|
+
busySlots.delete(workerIndex);
|
|
1495
|
+
}
|
|
1496
|
+
if (stopped)
|
|
1497
|
+
return;
|
|
1498
|
+
if (activeSlots.size === 0) {
|
|
1499
|
+
tripBreaker(new WorkerPoolDispatchError(`Worker pool exhausted all slots during idle-timeout retry.`, quarantine.snapshot()));
|
|
1500
|
+
return;
|
|
1501
|
+
}
|
|
1502
|
+
reportProgress();
|
|
1503
|
+
if (activeSlots.has(workerIndex))
|
|
1504
|
+
runWorker(workerIndex);
|
|
1505
|
+
wakeIdleSlots();
|
|
1506
|
+
maybeDone();
|
|
1507
|
+
})();
|
|
1508
|
+
}
|
|
1509
|
+
}, job.timeoutMs);
|
|
1510
|
+
};
|
|
1511
|
+
// U12: capture the slot's generation at handler-attach time so any
|
|
1512
|
+
// late event from a previous worker on this slot (which would carry
|
|
1513
|
+
// an older generation) short-circuits below. Defensive — cleanup()
|
|
1514
|
+
// already removes listeners synchronously when a death is observed,
|
|
1515
|
+
// so under the current control flow no listener should fire on a
|
|
1516
|
+
// stale generation. The guard catches future-refactor mistakes.
|
|
1517
|
+
const slotGen = slotGenerations[workerIndex];
|
|
1518
|
+
const handler = (raw) => {
|
|
1519
|
+
if (slotGenerations[workerIndex] !== slotGen)
|
|
1520
|
+
return;
|
|
1521
|
+
if (settled || stopped)
|
|
1522
|
+
return;
|
|
1523
|
+
// Native postMessage delivers POJO directly via Node's
|
|
1524
|
+
// structured clone. Two distinct clone failure modes exist,
|
|
1525
|
+
// and NEITHER reaches this handler: (1) a SENDER-side
|
|
1526
|
+
// non-cloneable value (a function/symbol that leaked into the
|
|
1527
|
+
// result) throws a synchronous `DataCloneError` on the
|
|
1528
|
+
// worker's own postMessage — the parse worker self-sanitizes
|
|
1529
|
+
// such results before delivery (#2112) and falls back to a
|
|
1530
|
+
// primitive-only `{type:'error'}` if it still can't serialize;
|
|
1531
|
+
// (2) a RECEIVER-side deserialization failure surfaces as a
|
|
1532
|
+
// `messageerror` event handled below. The only thing THIS
|
|
1533
|
+
// handler guards is a worker that sends a message without a
|
|
1534
|
+
// `type` discriminant (a worker bug, not a wire-format issue):
|
|
1535
|
+
// without the guard `null.type` would throw a TypeError out of
|
|
1536
|
+
// the EventEmitter listener → uncaughtException on the main
|
|
1537
|
+
// thread.
|
|
1538
|
+
const msg = raw;
|
|
1539
|
+
if (msg === null || typeof msg !== 'object' || typeof msg.type !== 'string') {
|
|
1540
|
+
settled = true;
|
|
1541
|
+
cleanup();
|
|
1542
|
+
void recoverAndResume(`Worker ${workerIndex} sent a malformed message (no type discriminant)`, resolveExcludePaths());
|
|
1543
|
+
return;
|
|
1544
|
+
}
|
|
1545
|
+
if (msg.type === 'starting-file') {
|
|
1546
|
+
inFlightPath = msg.path;
|
|
1547
|
+
resetIdleTimer();
|
|
1548
|
+
}
|
|
1549
|
+
else if (msg.type === 'progress') {
|
|
1550
|
+
const bounded = Math.min(job.items.length, Math.max(0, msg.filesProcessed));
|
|
1551
|
+
inFlightProgress[workerIndex] = bounded;
|
|
1552
|
+
lastProgress = bounded;
|
|
1553
|
+
inFlightPath = undefined;
|
|
1554
|
+
resetIdleTimer();
|
|
1555
|
+
reportProgress();
|
|
1556
|
+
}
|
|
1557
|
+
else if (msg.type === 'warning') {
|
|
1558
|
+
resetIdleTimer();
|
|
1559
|
+
logger_js_1.logger.warn(msg.message);
|
|
1560
|
+
}
|
|
1561
|
+
else if (msg.type === 'sub-batch-done') {
|
|
1562
|
+
waitingForFlush = true;
|
|
1563
|
+
resetIdleTimer();
|
|
1564
|
+
// Carry the chunk hash on the flush so the worker can write a
|
|
1565
|
+
// durable, content-addressed ParsedFile shard (warm-cache reuse)
|
|
1566
|
+
// at the flush boundary where `accumulated.parsedFiles` is complete.
|
|
1567
|
+
worker.postMessage({ type: 'flush', chunkHash: job.chunkHash });
|
|
1568
|
+
}
|
|
1569
|
+
else if (msg.type === 'error') {
|
|
1570
|
+
settled = true;
|
|
1571
|
+
cleanup();
|
|
1572
|
+
void recoverAndResume(workerErrorReason(workerIndex, msg.error, msg.errorStack), resolveExcludePaths());
|
|
1573
|
+
}
|
|
1574
|
+
else if (msg.type === 'result') {
|
|
1575
|
+
if (!waitingForFlush) {
|
|
1576
|
+
settled = true;
|
|
1577
|
+
cleanup();
|
|
1578
|
+
tripBreaker(new WorkerPoolDispatchError(`Worker ${workerIndex} protocol error: result before flush`, quarantine.snapshot()));
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
settled = true;
|
|
1582
|
+
cleanup();
|
|
1583
|
+
results.push({ startIndex: job.startIndex, data: msg.data });
|
|
1584
|
+
completedFiles += job.items.length;
|
|
1585
|
+
// Layer 2 (F6): a successful job resets THIS slot's
|
|
1586
|
+
// consecutive-failure counter so the breaker only trips
|
|
1587
|
+
// when a specific slot is chronically failing — another
|
|
1588
|
+
// slot's successes can't mask a single bad slot.
|
|
1589
|
+
consecutiveFailuresPerSlot[workerIndex] = 0;
|
|
1590
|
+
reportProgress();
|
|
1591
|
+
finishJob();
|
|
1592
|
+
}
|
|
1593
|
+
else if (msg.type === 'ready') {
|
|
1594
|
+
// No-op: the ready handshake is consumed by `waitForWorkerReady`
|
|
1595
|
+
// before dispatch handlers are attached. A stray `ready` here
|
|
1596
|
+
// (e.g., a future worker build re-emitting after an internal
|
|
1597
|
+
// recovery) is benign — ignore so the exhaustiveness check
|
|
1598
|
+
// below keeps catching genuinely-unknown variants.
|
|
1599
|
+
}
|
|
1600
|
+
else {
|
|
1601
|
+
// F7: exhaustiveness check — drift-catcher when a future
|
|
1602
|
+
// WorkerOutgoingMessage variant is added without a handler.
|
|
1603
|
+
const _exhaustive = msg;
|
|
1604
|
+
void _exhaustive;
|
|
1605
|
+
}
|
|
1606
|
+
};
|
|
1607
|
+
const errorHandler = (err) => {
|
|
1608
|
+
if (slotGenerations[workerIndex] !== slotGen)
|
|
1609
|
+
return;
|
|
1610
|
+
if (!settled) {
|
|
1611
|
+
settled = true;
|
|
1612
|
+
cleanup();
|
|
1613
|
+
// The Node 'error' event fires on an UNCAUGHT worker throw (one that
|
|
1614
|
+
// escaped the worker's own try/catch, or an async rejection). Unlike
|
|
1615
|
+
// the `{type:'error'}` message, the event delivers a real Error whose
|
|
1616
|
+
// `.stack` is the worker-side frame — carry it so the surfaced reason
|
|
1617
|
+
// points at the actual failure site, not just `err.message` (#2068).
|
|
1618
|
+
// A worker dying on ITS OWN heap cap (#2649) must be attributable to
|
|
1619
|
+
// that cap, not read as generic quarantine noise — name the cap and
|
|
1620
|
+
// its override so an oversized-but-legitimate file (e.g. under a
|
|
1621
|
+
// raised CGRAPH_MAX_FILE_SIZE) is a one-env-var fix.
|
|
1622
|
+
// The 'error' event does not guarantee a well-formed Error: the
|
|
1623
|
+
// structured-clone failure path can deliver a value with no
|
|
1624
|
+
// `message` — guard every property access or the handler itself
|
|
1625
|
+
// throws and the pool hangs instead of recovering.
|
|
1626
|
+
const isWorkerHeapOom = err?.code === 'ERR_WORKER_OUT_OF_MEMORY' ||
|
|
1627
|
+
(typeof err?.message === 'string' &&
|
|
1628
|
+
err.message.includes('ERR_WORKER_OUT_OF_MEMORY'));
|
|
1629
|
+
const reason = isWorkerHeapOom
|
|
1630
|
+
? `${workerErrorReason(workerIndex, err.message, err.stack)} (worker hit its ${workerHeapCapMb}MB heap cap — raise with CGRAPH_WORKER_HEAP_MB)`
|
|
1631
|
+
: workerErrorReason(workerIndex, err.message, err.stack);
|
|
1632
|
+
void recoverAndResume(reason, resolveExcludePaths());
|
|
1633
|
+
}
|
|
1634
|
+
};
|
|
1635
|
+
const exitHandler = (code) => {
|
|
1636
|
+
if (slotGenerations[workerIndex] !== slotGen)
|
|
1637
|
+
return;
|
|
1638
|
+
if (!settled) {
|
|
1639
|
+
settled = true;
|
|
1640
|
+
cleanup();
|
|
1641
|
+
const excludes = resolveExcludePaths();
|
|
1642
|
+
const inFlightSuffix = excludes.length > 0 ? ` (in-flight: ${excludes[0]})` : '';
|
|
1643
|
+
void recoverAndResume(`Worker ${workerIndex} exited with code ${code}. ` +
|
|
1644
|
+
`Likely OOM or native addon failure${inFlightSuffix}.`, excludes);
|
|
1645
|
+
}
|
|
1646
|
+
};
|
|
1647
|
+
// `messageerror` fires when V8 fails to DESERIALIZE a postMessage
|
|
1648
|
+
// payload on THIS (receiver) side — a value that serialized on the
|
|
1649
|
+
// worker but can't be reconstructed here. (A non-cloneable value on
|
|
1650
|
+
// the SENDER side instead throws a synchronous DataCloneError on the
|
|
1651
|
+
// worker's own postMessage; that path is caught and sanitized
|
|
1652
|
+
// worker-side (#2112) and never arrives here.) The worker stays ALIVE
|
|
1653
|
+
// but the message is lost — without this handler the pool would sit on
|
|
1654
|
+
// the dropped message until the idle timeout expires. Treat it as
|
|
1655
|
+
// worker death so the resilience layers fire:
|
|
1656
|
+
// requeue the remainder via `recoverAndResume`, attribute the
|
|
1657
|
+
// in-flight file from the `starting-file` signal (if observed),
|
|
1658
|
+
// and let the per-slot respawn budget and circuit breaker decide
|
|
1659
|
+
// whether to keep this slot in rotation.
|
|
1660
|
+
const messageErrorHandler = (err) => {
|
|
1661
|
+
if (slotGenerations[workerIndex] !== slotGen)
|
|
1662
|
+
return;
|
|
1663
|
+
if (!settled) {
|
|
1664
|
+
settled = true;
|
|
1665
|
+
cleanup();
|
|
1666
|
+
void recoverAndResume(`Worker ${workerIndex} messageerror (postMessage deserialization failure): ${err.message}`, resolveExcludePaths());
|
|
1667
|
+
}
|
|
1668
|
+
};
|
|
1669
|
+
worker.on('message', handler);
|
|
1670
|
+
worker.once('error', errorHandler);
|
|
1671
|
+
worker.once('exit', exitHandler);
|
|
1672
|
+
worker.once('messageerror', messageErrorHandler);
|
|
1673
|
+
resetIdleTimer();
|
|
1674
|
+
if (stopped) {
|
|
1675
|
+
cleanup();
|
|
1676
|
+
return;
|
|
1677
|
+
}
|
|
1678
|
+
const { message, transferList } = buildDispatchMessage(job.items);
|
|
1679
|
+
if (transferList) {
|
|
1680
|
+
worker.postMessage(message, transferList);
|
|
1681
|
+
}
|
|
1682
|
+
else {
|
|
1683
|
+
worker.postMessage(message);
|
|
1684
|
+
}
|
|
1685
|
+
};
|
|
1686
|
+
for (const slotIndex of activeSlots)
|
|
1687
|
+
runWorker(slotIndex);
|
|
1688
|
+
});
|
|
1689
|
+
};
|
|
1690
|
+
const terminate = async () => {
|
|
1691
|
+
terminated = true;
|
|
1692
|
+
stallTracker.stop();
|
|
1693
|
+
// Cancel any in-flight startup backoff so its ref'd timer doesn't keep the
|
|
1694
|
+
// event loop alive after terminate; each cancel resolves the awaiting sleep
|
|
1695
|
+
// and the slot loop then sees `terminated` and gives up (#1741).
|
|
1696
|
+
for (const cancel of [...pendingStartupTimers])
|
|
1697
|
+
cancel();
|
|
1698
|
+
// `.catch(() => undefined)` per-worker matches every other terminate
|
|
1699
|
+
// site in this file. Without it, a hung/OOM-killed worker's terminate
|
|
1700
|
+
// rejection escapes `Promise.all` and replaces the original pipeline
|
|
1701
|
+
// exception when this is called from `runChunkedParseAndResolve`'s
|
|
1702
|
+
// finally block — masking the real failure and leaving `workers[]`
|
|
1703
|
+
// populated with dead references because the lines below never run.
|
|
1704
|
+
await terminateTrackedWorkers(workers);
|
|
1705
|
+
workers.length = 0;
|
|
1706
|
+
activeSlots.clear();
|
|
1707
|
+
};
|
|
1708
|
+
return {
|
|
1709
|
+
dispatch,
|
|
1710
|
+
terminate,
|
|
1711
|
+
size,
|
|
1712
|
+
getQuarantinedPaths: () => quarantine.snapshot(),
|
|
1713
|
+
getStats: () => ({
|
|
1714
|
+
size,
|
|
1715
|
+
activeSlots: activeSlots.size,
|
|
1716
|
+
droppedSlots: size - activeSlots.size,
|
|
1717
|
+
quarantined: quarantine.size,
|
|
1718
|
+
poolBroken,
|
|
1719
|
+
terminated,
|
|
1720
|
+
pendingStartupTimers: pendingStartupTimers.size,
|
|
1721
|
+
slotGenerations: slotGenerations.slice(),
|
|
1722
|
+
}),
|
|
1723
|
+
};
|
|
1724
|
+
};
|
|
1725
|
+
exports.createWorkerPool = createWorkerPool;
|