gitnexus 1.6.8-rc.2 → 1.6.8-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/dist/_shared/scope-resolution/parsed-file.d.ts +21 -0
- package/dist/_shared/scope-resolution/parsed-file.d.ts.map +1 -1
- package/dist/_shared/scope-resolution/symbol-definition.d.ts +4 -0
- package/dist/_shared/scope-resolution/symbol-definition.d.ts.map +1 -1
- package/dist/cli/ai-context.js +1 -0
- package/dist/cli/analyze-config.js +40 -0
- package/dist/cli/analyze.d.ts +22 -0
- package/dist/cli/analyze.js +36 -6
- package/dist/cli/clean.d.ts +1 -0
- package/dist/cli/clean.js +43 -1
- package/dist/cli/eval-server.js +43 -0
- package/dist/cli/help-i18n.js +11 -0
- package/dist/cli/i18n/en.d.ts +13 -0
- package/dist/cli/i18n/en.js +13 -0
- package/dist/cli/i18n/resources.d.ts +26 -0
- package/dist/cli/i18n/zh-CN.d.ts +13 -0
- package/dist/cli/i18n/zh-CN.js +13 -0
- package/dist/cli/index.js +19 -0
- package/dist/cli/list.js +12 -0
- package/dist/cli/optional-grammars.d.ts +6 -8
- package/dist/cli/optional-grammars.js +8 -11
- package/dist/cli/skill-gen.js +1 -0
- package/dist/cli/status.js +26 -5
- package/dist/cli/tool.d.ts +12 -1
- package/dist/cli/tool.js +41 -1
- package/dist/core/embeddings/embedder.js +4 -0
- package/dist/core/embeddings/embedding-pipeline.js +27 -16
- package/dist/core/embeddings/onnxruntime-common-resolver.d.ts +6 -0
- package/dist/core/embeddings/onnxruntime-common-resolver.js +130 -0
- package/dist/core/graph/import-cycles.d.ts +10 -0
- package/dist/core/graph/import-cycles.js +103 -0
- package/dist/core/group/extractors/grpc-patterns/proto.js +10 -6
- package/dist/core/group/extractors/http-patterns/java.js +3 -50
- package/dist/core/group/extractors/http-patterns/kotlin.js +7 -7
- package/dist/core/group/extractors/include-extractor.js +7 -7
- package/dist/core/ingestion/cfg/cfg-builder.d.ts +64 -0
- package/dist/core/ingestion/cfg/cfg-builder.js +129 -0
- package/dist/core/ingestion/cfg/collect.d.ts +30 -0
- package/dist/core/ingestion/cfg/collect.js +34 -0
- package/dist/core/ingestion/cfg/control-flow-context.d.ts +97 -0
- package/dist/core/ingestion/cfg/control-flow-context.js +113 -0
- package/dist/core/ingestion/cfg/emit.d.ts +144 -0
- package/dist/core/ingestion/cfg/emit.js +315 -0
- package/dist/core/ingestion/cfg/reaching-defs.d.ts +90 -0
- package/dist/core/ingestion/cfg/reaching-defs.js +364 -0
- package/dist/core/ingestion/cfg/traversal-result.d.ts +20 -0
- package/dist/core/ingestion/cfg/traversal-result.js +2 -0
- package/dist/core/ingestion/cfg/types.d.ts +231 -0
- package/dist/core/ingestion/cfg/types.js +13 -0
- package/dist/core/ingestion/cfg/visitors/typescript-harvest.d.ts +144 -0
- package/dist/core/ingestion/cfg/visitors/typescript-harvest.js +1032 -0
- package/dist/core/ingestion/cfg/visitors/typescript.d.ts +66 -0
- package/dist/core/ingestion/cfg/visitors/typescript.js +584 -0
- package/dist/core/ingestion/language-provider.d.ts +36 -2
- package/dist/core/ingestion/languages/c-cpp.js +11 -4
- package/dist/core/ingestion/languages/cpp/arity-metadata.js +6 -2
- package/dist/core/ingestion/languages/cpp/captures.js +24 -1
- package/dist/core/ingestion/languages/cpp/query.js +23 -0
- package/dist/core/ingestion/languages/java.js +3 -0
- package/dist/core/ingestion/languages/kotlin/query.js +3 -2
- package/dist/core/ingestion/languages/kotlin.js +5 -1
- package/dist/core/ingestion/languages/typescript.js +5 -0
- package/dist/core/ingestion/method-extractors/configs/c-cpp.js +6 -13
- package/dist/core/ingestion/method-extractors/generic.js +1 -0
- package/dist/core/ingestion/method-types.d.ts +2 -0
- package/dist/core/ingestion/model/symbol-table.d.ts +1 -0
- package/dist/core/ingestion/model/symbol-table.js +1 -0
- package/dist/core/ingestion/parsing-processor.js +22 -0
- package/dist/core/ingestion/pipeline-phases/parse-impl.js +18 -1
- package/dist/core/ingestion/pipeline-phases/routes.js +64 -14
- package/dist/core/ingestion/pipeline.d.ts +57 -0
- package/dist/core/ingestion/route-extractors/spring-shared.d.ts +50 -0
- package/dist/core/ingestion/route-extractors/spring-shared.js +80 -0
- package/dist/core/ingestion/route-extractors/spring.d.ts +35 -0
- package/dist/core/ingestion/route-extractors/spring.js +136 -0
- package/dist/core/ingestion/scope-extractor.js +3 -0
- package/dist/core/ingestion/scope-resolution/passes/free-call-fallback.js +12 -0
- package/dist/core/ingestion/scope-resolution/passes/receiver-bound-calls.js +145 -24
- package/dist/core/ingestion/scope-resolution/pipeline/phase.js +6 -0
- package/dist/core/ingestion/scope-resolution/pipeline/reconcile-ownership.js +43 -3
- package/dist/core/ingestion/scope-resolution/pipeline/run.d.ts +21 -0
- package/dist/core/ingestion/scope-resolution/pipeline/run.js +207 -0
- package/dist/core/ingestion/scope-resolution/resolution-outcome.d.ts +1 -1
- package/dist/core/ingestion/taint/emit.d.ts +124 -0
- package/dist/core/ingestion/taint/emit.js +204 -0
- package/dist/core/ingestion/taint/match.d.ts +153 -0
- package/dist/core/ingestion/taint/match.js +278 -0
- package/dist/core/ingestion/taint/path-codec.d.ts +134 -0
- package/dist/core/ingestion/taint/path-codec.js +190 -0
- package/dist/core/ingestion/taint/propagate.d.ts +216 -0
- package/dist/core/ingestion/taint/propagate.js +664 -0
- package/dist/core/ingestion/taint/site-safety.d.ts +29 -0
- package/dist/core/ingestion/taint/site-safety.js +98 -0
- package/dist/core/ingestion/taint/source-sink-config.d.ts +94 -23
- package/dist/core/ingestion/taint/source-sink-config.js +11 -11
- package/dist/core/ingestion/taint/source-sink-registry.d.ts +6 -4
- package/dist/core/ingestion/taint/source-sink-registry.js +6 -4
- package/dist/core/ingestion/taint/typescript-model.d.ts +38 -0
- package/dist/core/ingestion/taint/typescript-model.js +102 -0
- package/dist/core/ingestion/utils/method-props.js +1 -0
- package/dist/core/ingestion/workers/clone-safety.d.ts +109 -0
- package/dist/core/ingestion/workers/clone-safety.js +465 -0
- package/dist/core/ingestion/workers/parse-worker.d.ts +11 -0
- package/dist/core/ingestion/workers/parse-worker.js +70 -53
- package/dist/core/ingestion/workers/post-result.d.ts +22 -0
- package/dist/core/ingestion/workers/post-result.js +87 -0
- package/dist/core/ingestion/workers/result-merge.d.ts +20 -0
- package/dist/core/ingestion/workers/result-merge.js +43 -0
- package/dist/core/ingestion/workers/worker-pool.d.ts +15 -0
- package/dist/core/ingestion/workers/worker-pool.js +33 -16
- package/dist/core/lbug/lbug-adapter.d.ts +19 -0
- package/dist/core/lbug/lbug-adapter.js +56 -1
- package/dist/core/run-analyze.d.ts +87 -0
- package/dist/core/run-analyze.js +280 -25
- package/dist/core/tree-sitter/parser-loader.js +5 -4
- package/dist/core/tree-sitter/vendored-grammars.d.ts +39 -0
- package/dist/core/tree-sitter/vendored-grammars.js +57 -0
- package/dist/mcp/core/embedder.js +4 -0
- package/dist/mcp/local/local-backend.d.ts +86 -2
- package/dist/mcp/local/local-backend.js +657 -27
- package/dist/mcp/resources.js +1 -0
- package/dist/mcp/tools.d.ts +9 -0
- package/dist/mcp/tools.js +109 -0
- package/dist/server/analyze-launch.d.ts +29 -0
- package/dist/server/analyze-launch.js +137 -0
- package/dist/server/analyze-upload.d.ts +33 -0
- package/dist/server/analyze-upload.js +123 -0
- package/dist/server/analyze-worker-ipc.d.ts +58 -0
- package/dist/server/analyze-worker-ipc.js +16 -0
- package/dist/server/analyze-worker.d.ts +17 -1
- package/dist/server/analyze-worker.js +7 -1
- package/dist/server/api.js +46 -140
- package/dist/server/git-clone.d.ts +1 -0
- package/dist/server/git-clone.js +1 -1
- package/dist/server/middleware.d.ts +11 -0
- package/dist/server/middleware.js +27 -0
- package/dist/server/upload-ingest.d.ts +56 -0
- package/dist/server/upload-ingest.js +276 -0
- package/dist/server/upload-paths.d.ts +31 -0
- package/dist/server/upload-paths.js +51 -0
- package/dist/server/upload-sweep.d.ts +20 -0
- package/dist/server/upload-sweep.js +57 -0
- package/dist/storage/branch-index.d.ts +52 -0
- package/dist/storage/branch-index.js +65 -0
- package/dist/storage/git.d.ts +11 -0
- package/dist/storage/git.js +28 -0
- package/dist/storage/parse-cache.d.ts +22 -1
- package/dist/storage/parse-cache.js +32 -10
- package/dist/storage/repo-manager.d.ts +122 -10
- package/dist/storage/repo-manager.js +162 -21
- package/hooks/antigravity/gitnexus-antigravity-hook.cjs +42 -8
- package/hooks/claude/gitnexus-hook.cjs +36 -8
- package/hooks/claude/hook-db-lock-probe.cjs +123 -2
- package/package.json +4 -2
- package/scripts/assert-publish-grammar-coverage.cjs +31 -0
- package/scripts/build-tree-sitter-grammars.cjs +16 -10
- package/skills/gitnexus-guide.md +11 -0
- package/vendor/tree-sitter-c/package.json +1 -1
- package/vendor/tree-sitter-dart/package.json +1 -1
- package/vendor/tree-sitter-kotlin/package.json +1 -1
- package/vendor/tree-sitter-proto/package.json +1 -1
- package/vendor/tree-sitter-swift/package.json +1 -1
- package/web/assets/{agent-CKPMqImC.js → agent-ay4LD70X.js} +1 -1
- package/web/assets/{architectureDiagram-UL44E2DR-D1EXI0zA.js → architectureDiagram-UL44E2DR-Dc-viYhd.js} +1 -1
- package/web/assets/{chunk-LCXTWHL2-Dfmux4m1.js → chunk-LCXTWHL2-4rpojOyj.js} +1 -1
- package/web/assets/{chunk-RG4AUYOV-CAkzcoRj.js → chunk-RG4AUYOV-BnOy944n.js} +1 -1
- package/web/assets/{classDiagram-KGZ6W3CR-Bv93af_b.js → classDiagram-KGZ6W3CR-Bsgpy98Q.js} +1 -1
- package/web/assets/{classDiagram-v2-72OJOZXJ-CLfEqHUa.js → classDiagram-v2-72OJOZXJ-D5atDGjc.js} +1 -1
- package/web/assets/{diagram-3NCE3AQN-BqAtKUpW.js → diagram-3NCE3AQN-Cz1OEMVi.js} +1 -1
- package/web/assets/{diagram-GF46GFSD-CaBG6n6o.js → diagram-GF46GFSD-CWYwzfP8.js} +1 -1
- package/web/assets/{diagram-QXG6HAR7-CZ-O3rcV.js → diagram-QXG6HAR7-DtBObr8L.js} +1 -1
- package/web/assets/{diagram-WEQXMOUZ-P4lSL4GH.js → diagram-WEQXMOUZ-BUV44Ov_.js} +1 -1
- package/web/assets/{erDiagram-L5TCEMPS-QN2eEP1e.js → erDiagram-L5TCEMPS-Ds9s-sRF.js} +1 -1
- package/web/assets/{flowDiagram-H6V6AXG4-CXbXImlN.js → flowDiagram-H6V6AXG4-CupStHQb.js} +1 -1
- package/web/assets/index-COMMmbxW.css +2 -0
- package/web/assets/{index-CG6q8eTs.js → index-Do8AE5yF.js} +85 -85
- package/web/assets/{infoDiagram-3YFTVSEB-DmLICZx1.js → infoDiagram-3YFTVSEB-DPTXSbgu.js} +1 -1
- package/web/assets/{ishikawaDiagram-BNXS4ZKH-6LKOvBfp.js → ishikawaDiagram-BNXS4ZKH-CeK-Qhet.js} +1 -1
- package/web/assets/{kanban-definition-75IXJCU3-DgDi9oJT.js → kanban-definition-75IXJCU3-D4tx6QQ-.js} +1 -1
- package/web/assets/{mindmap-definition-2TDM6QVE-CwR5sBB-.js → mindmap-definition-2TDM6QVE-Bcto_kGa.js} +1 -1
- package/web/assets/{pieDiagram-CU6KROY3-By8g6f6B.js → pieDiagram-CU6KROY3-DPIeBTG6.js} +1 -1
- package/web/assets/{requirementDiagram-JXO7QTGE-7oDcJ1_J.js → requirementDiagram-JXO7QTGE-DDGXjVKO.js} +1 -1
- package/web/assets/{sequenceDiagram-VS2MUI6T-FySeKCUy.js → sequenceDiagram-VS2MUI6T-BzDqpcVW.js} +1 -1
- package/web/assets/{stateDiagram-7D4R322I-CB2nABwH.js → stateDiagram-7D4R322I-Bx__als3.js} +1 -1
- package/web/assets/{stateDiagram-v2-36443NZ5-COBGd2RL.js → stateDiagram-v2-36443NZ5-QqOh2yO-.js} +1 -1
- package/web/assets/{timeline-definition-O6YCAMPW-Ds2CnVZK.js → timeline-definition-O6YCAMPW-C1eogTOG.js} +1 -1
- package/web/assets/{vennDiagram-MWXL3ELB-DUIEwXWp.js → vennDiagram-MWXL3ELB-D20F4rSW.js} +1 -1
- package/web/assets/{wardleyDiagram-CUQ6CDDI-DEiFPQih.js → wardleyDiagram-CUQ6CDDI-BeBHg7ST.js} +1 -1
- package/web/assets/{xychartDiagram-N2JHSOCM-BUbayhST.js → xychartDiagram-N2JHSOCM-CdOWeoNI.js} +1 -1
- package/web/index.html +2 -2
- package/scripts/materialize-vendor-grammars.cjs +0 -97
- package/web/assets/index-BKWA-m7o.css +0 -2
|
@@ -11,15 +11,17 @@ import Go from 'tree-sitter-go';
|
|
|
11
11
|
import Rust from 'tree-sitter-rust';
|
|
12
12
|
import PHP from 'tree-sitter-php';
|
|
13
13
|
import Ruby from 'tree-sitter-ruby';
|
|
14
|
-
import {
|
|
14
|
+
import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js';
|
|
15
15
|
import { SupportedLanguages } from '../../../_shared/index.js';
|
|
16
16
|
import { getProvider } from '../languages/index.js';
|
|
17
17
|
import { getTreeSitterBufferSize, getTreeSitterContentByteLength, TREE_SITTER_MAX_BUFFER, } from '../constants.js';
|
|
18
18
|
import { ARRAY_METHOD_HOC_BLOCKLIST_SET, DEFAULT_EXPORT_IDENTIFIER_BLOCKLIST_SET, deriveDefaultExportHocName, } from '../ts-js-hoc-utils.js';
|
|
19
19
|
import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
|
|
20
|
+
import { postResultCloneSafe } from './post-result.js';
|
|
21
|
+
import { mergeResult } from './result-merge.js';
|
|
20
22
|
// ── Worker grammar loading — enforcement boundary (#2091/#2093, #2101) ───────
|
|
21
|
-
// The worker maintains its own grammar table (the guarded
|
|
22
|
-
// `languageMap`) and intentionally does NOT consult the runtime
|
|
23
|
+
// The worker maintains its own grammar table (the guarded vendored-grammar
|
|
24
|
+
// loads below + `languageMap`) and intentionally does NOT consult the runtime
|
|
23
25
|
// `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` opt-out. It does not need to: the MAIN
|
|
24
26
|
// THREAD's `parseableScanned` filter (pipeline-phases/parse-impl.ts, gated on
|
|
25
27
|
// `parser-loader.isLanguageAvailable`, which honors the runtime opt-out and a
|
|
@@ -30,33 +32,29 @@ import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
|
|
|
30
32
|
// `isLanguageAvailable` must re-introduce the gate here. (The cleaner end-state
|
|
31
33
|
// — routing this table through `parser-loader.getLanguageGrammar` so there is
|
|
32
34
|
// one loader — is the deferred Tier-1 consolidation.)
|
|
33
|
-
//
|
|
34
|
-
|
|
35
|
+
// Swift/Dart/Kotlin/C are vendored grammars loaded from `vendor/` by absolute
|
|
36
|
+
// path (NEVER copied into node_modules — see vendored-grammars.ts / #2111). Each
|
|
37
|
+
// may be absent on a platform without a prebuild or a toolchain-less /
|
|
38
|
+
// `--ignore-scripts` install, so every load is guarded so a missing binding
|
|
39
|
+
// cannot crash the worker at module-load (#2091/#2093, #2116).
|
|
35
40
|
let Swift = null;
|
|
36
41
|
try {
|
|
37
|
-
Swift =
|
|
42
|
+
Swift = requireVendoredGrammar('tree-sitter-swift');
|
|
38
43
|
}
|
|
39
44
|
catch { }
|
|
40
|
-
// tree-sitter-dart is an optionalDependency — may not be installed
|
|
41
45
|
let Dart = null;
|
|
42
46
|
try {
|
|
43
|
-
Dart =
|
|
47
|
+
Dart = requireVendoredGrammar('tree-sitter-dart');
|
|
44
48
|
}
|
|
45
49
|
catch { }
|
|
46
|
-
// tree-sitter-kotlin is an optionalDependency — may not be installed
|
|
47
50
|
let Kotlin = null;
|
|
48
51
|
try {
|
|
49
|
-
Kotlin =
|
|
52
|
+
Kotlin = requireVendoredGrammar('tree-sitter-kotlin');
|
|
50
53
|
}
|
|
51
54
|
catch { }
|
|
52
|
-
// tree-sitter-c is now vendored prebuild-only (#2116) and may be absent on a
|
|
53
|
-
// toolchain-less / `--ignore-scripts` install. Guard it like Swift/Dart/Kotlin so
|
|
54
|
-
// a missing binding cannot crash the worker at module-load (#2091/#2093); the
|
|
55
|
-
// main-thread `isLanguageAvailable` filter keeps C files from being dispatched
|
|
56
|
-
// here when the entry is absent.
|
|
57
55
|
let C = null;
|
|
58
56
|
try {
|
|
59
|
-
C =
|
|
57
|
+
C = requireVendoredGrammar('tree-sitter-c');
|
|
60
58
|
}
|
|
61
59
|
catch { }
|
|
62
60
|
import { getLanguageFromFilename } from '../../../_shared/index.js';
|
|
@@ -71,6 +69,7 @@ import { extractTemplateArguments, templateArgumentsIdTag, templateConstraintsId
|
|
|
71
69
|
import { extractParsedFile } from '../scope-extractor-bridge.js';
|
|
72
70
|
import { persistParsedFileShardSync, persistDurableParsedFileShardSync, } from '../../../storage/parsedfile-store.js';
|
|
73
71
|
import { extractLaravelRoutes } from '../route-extractors/laravel.js';
|
|
72
|
+
import { collectFunctionCfgs, DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js';
|
|
74
73
|
import { logger } from '../../logger.js';
|
|
75
74
|
// ── ParsedFile store (#1983 parallel serialization) ─────────────────────────
|
|
76
75
|
// Read ONCE at worker init from `workerData` (immutable for the run, inherited
|
|
@@ -88,6 +87,17 @@ const PARSED_FILE_STORE_STORAGE_PATH = workerData?.parsedFileStoreStoragePath;
|
|
|
88
87
|
// restores them without re-parsing. `undefined` ⇒ no durable write.
|
|
89
88
|
const DURABLE_PARSED_FILE_STORAGE_PATH = workerData?.durableParsedFileStoragePath;
|
|
90
89
|
let shardSeq = 0;
|
|
90
|
+
// ── PDG/CFG opt-in (#2081 M1) ───────────────────────────────────────────────
|
|
91
|
+
// Read ONCE at worker init from `workerData` (the worker never sees
|
|
92
|
+
// PipelineOptions — config arrives via the pool factory's `workerData`, see
|
|
93
|
+
// KTD7 / U5). When `pdg` is set, the worker builds a per-function control-flow
|
|
94
|
+
// graph from the tree-sitter AST (where it lives) and serializes it onto
|
|
95
|
+
// `ParsedFile.cfgSideChannel`. Off ⇒ no CFG work and no field — the default for
|
|
96
|
+
// every run today. `pdgMaxFunctionLines` bounds per-function CFG cost
|
|
97
|
+
// (0/undefined ⇒ no cap; see collectFunctionCfgs).
|
|
98
|
+
const PDG_ENABLED = workerData?.pdg === true;
|
|
99
|
+
const PDG_MAX_FUNCTION_LINES = workerData?.pdgMaxFunctionLines ??
|
|
100
|
+
DEFAULT_PDG_MAX_FUNCTION_LINES;
|
|
91
101
|
// ── Bootstrap-stage diagnostics (#1741) ────────────────────────────────────
|
|
92
102
|
// When GITNEXUS_WORKER_BOOTSTRAP=1 (or --verbose sets GITNEXUS_VERBOSE), each
|
|
93
103
|
// worker reports its startup stage timings to stderr — which the pool tees
|
|
@@ -678,6 +688,7 @@ const ROUTE_DECORATOR_NAMES = new Set([
|
|
|
678
688
|
'PostMapping',
|
|
679
689
|
'PutMapping',
|
|
680
690
|
'DeleteMapping',
|
|
691
|
+
'PatchMapping',
|
|
681
692
|
]);
|
|
682
693
|
// ============================================================================
|
|
683
694
|
// ORM Query Detection (Prisma + Supabase)
|
|
@@ -834,7 +845,34 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
834
845
|
// copy — scopes/defs are carried by reference) to attach the field rather
|
|
835
846
|
// than mutate the frozen object.
|
|
836
847
|
const sideChannel = provider.collectCaptureSideChannel?.(file.path);
|
|
837
|
-
|
|
848
|
+
let withChannels = sideChannel !== undefined ? { ...parsedFile, captureSideChannel: sideChannel } : parsedFile;
|
|
849
|
+
// CFG side-channel (#2081 M1): build the per-function control-flow graph
|
|
850
|
+
// here, where the tree-sitter AST is still in hand, and attach it as plain
|
|
851
|
+
// serializable data. Only on a --pdg run and only for languages with a
|
|
852
|
+
// cfgVisitor (TS/JS in M1). The same disk-store/warm-cache machinery that
|
|
853
|
+
// carries captureSideChannel carries this — its coherence rests on the
|
|
854
|
+
// SCHEMA_BUMP + the pdg-folded chunk-hash key (see parse-cache.ts).
|
|
855
|
+
if (PDG_ENABLED && provider.cfgVisitor) {
|
|
856
|
+
// Isolate the CFG build per file: a throw here (an unexpected tree-sitter
|
|
857
|
+
// node shape, a deep-nesting stack overflow) must NOT propagate — it
|
|
858
|
+
// would escape processFileGroup to the language-group catch, which treats
|
|
859
|
+
// any throw as "parser unavailable" and silently drops EVERY remaining
|
|
860
|
+
// file in the group. Skip CFG for this one file; parsing + scope
|
|
861
|
+
// resolution proceed unaffected (CFG is a strictly-additive opt-in).
|
|
862
|
+
try {
|
|
863
|
+
const { cfgs } = collectFunctionCfgs(tree.rootNode, provider.cfgVisitor, file.path, PDG_MAX_FUNCTION_LINES);
|
|
864
|
+
if (cfgs.length)
|
|
865
|
+
withChannels = { ...withChannels, cfgSideChannel: cfgs };
|
|
866
|
+
}
|
|
867
|
+
catch (err) {
|
|
868
|
+
const message = `CFG build failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`;
|
|
869
|
+
if (parentPort)
|
|
870
|
+
parentPort.postMessage({ type: 'warning', message });
|
|
871
|
+
else
|
|
872
|
+
logger.warn(message);
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
result.parsedFiles.push(withChannels);
|
|
838
876
|
}
|
|
839
877
|
// Build per-file type environment + constructor bindings in a single AST walk.
|
|
840
878
|
// The legacy heritage pre-pass that seeded a file-local parentMap for
|
|
@@ -1674,6 +1712,9 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
1674
1712
|
isReadonly: methodProps.isReadonly,
|
|
1675
1713
|
isAbstract: methodProps.isAbstract,
|
|
1676
1714
|
isFinal: methodProps.isFinal,
|
|
1715
|
+
...(methodProps.isDeleted !== undefined
|
|
1716
|
+
? { isDeleted: methodProps.isDeleted }
|
|
1717
|
+
: {}),
|
|
1677
1718
|
...(methodProps.isVirtual !== undefined
|
|
1678
1719
|
? { isVirtual: methodProps.isVirtual }
|
|
1679
1720
|
: {}),
|
|
@@ -1743,6 +1784,15 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
1743
1784
|
if (language === SupportedLanguages.Python) {
|
|
1744
1785
|
extractFastAPIRouterBindings(file.path, parseContent, result.routerIncludes, result.routerImports, (result.routerModuleAliases ??= []));
|
|
1745
1786
|
}
|
|
1787
|
+
// Language-specific decorator route extraction via provider hook.
|
|
1788
|
+
// The provider's extractDecoratorRoutes walks the AST for framework-specific
|
|
1789
|
+
// route patterns (e.g., Java Spring class-level prefix joining). Routes are
|
|
1790
|
+
// appended to decoratorRoutes for the routes phase to emit as Route nodes.
|
|
1791
|
+
if (provider.extractDecoratorRoutes) {
|
|
1792
|
+
const frameworkRoutes = provider.extractDecoratorRoutes(tree, file.path, lineOffset);
|
|
1793
|
+
for (const r of frameworkRoutes)
|
|
1794
|
+
result.decoratorRoutes.push(r);
|
|
1795
|
+
}
|
|
1746
1796
|
// Vue: emit CALLS edges for components used in <template>
|
|
1747
1797
|
if (language === SupportedLanguages.Vue) {
|
|
1748
1798
|
const templateComponents = extractTemplateComponents(file.content);
|
|
@@ -1783,41 +1833,8 @@ let accumulated = {
|
|
|
1783
1833
|
fileCount: 0,
|
|
1784
1834
|
};
|
|
1785
1835
|
let cumulativeProcessed = 0;
|
|
1786
|
-
//
|
|
1787
|
-
//
|
|
1788
|
-
// and blows the stack when arr has >~65k elements).
|
|
1789
|
-
const appendAll = (target, src) => {
|
|
1790
|
-
for (let i = 0; i < src.length; i++)
|
|
1791
|
-
target.push(src[i]);
|
|
1792
|
-
};
|
|
1793
|
-
const mergeResult = (target, src) => {
|
|
1794
|
-
appendAll(target.nodes, src.nodes);
|
|
1795
|
-
appendAll(target.relationships, src.relationships);
|
|
1796
|
-
appendAll(target.symbols, src.symbols);
|
|
1797
|
-
appendAll(target.calls, src.calls);
|
|
1798
|
-
appendAll(target.assignments, src.assignments);
|
|
1799
|
-
appendAll(target.routes, src.routes);
|
|
1800
|
-
appendAll(target.fetchCalls, src.fetchCalls);
|
|
1801
|
-
appendAll(target.fetchWrapperDefs, src.fetchWrapperDefs);
|
|
1802
|
-
appendAll(target.decoratorRoutes, src.decoratorRoutes);
|
|
1803
|
-
if (src.routerIncludes)
|
|
1804
|
-
appendAll(target.routerIncludes, src.routerIncludes);
|
|
1805
|
-
if (src.routerImports)
|
|
1806
|
-
appendAll(target.routerImports, src.routerImports);
|
|
1807
|
-
if (src.routerModuleAliases) {
|
|
1808
|
-
target.routerModuleAliases ??= [];
|
|
1809
|
-
appendAll(target.routerModuleAliases, src.routerModuleAliases);
|
|
1810
|
-
}
|
|
1811
|
-
appendAll(target.toolDefs, src.toolDefs);
|
|
1812
|
-
appendAll(target.ormQueries, src.ormQueries);
|
|
1813
|
-
appendAll(target.constructorBindings, src.constructorBindings);
|
|
1814
|
-
appendAll(target.fileScopeBindings, src.fileScopeBindings);
|
|
1815
|
-
appendAll(target.parsedFiles, src.parsedFiles);
|
|
1816
|
-
for (const [lang, count] of Object.entries(src.skippedLanguages)) {
|
|
1817
|
-
target.skippedLanguages[lang] = (target.skippedLanguages[lang] || 0) + count;
|
|
1818
|
-
}
|
|
1819
|
-
target.fileCount += src.fileCount;
|
|
1820
|
-
};
|
|
1836
|
+
// `mergeResult` (+ its `appendAll`) lives in ./result-merge.ts (extracted so it
|
|
1837
|
+
// can be unit-tested without importing this entry module).
|
|
1821
1838
|
// Signal the pool that worker-side initialization (parser imports, language
|
|
1822
1839
|
// grammars, type-env setup, all helper modules) is complete and the message
|
|
1823
1840
|
// handler below is about to be attached. The pool's `waitForWorkerReady`
|
|
@@ -1910,7 +1927,7 @@ parentPort.on('message', (msg) => {
|
|
|
1910
1927
|
accumulated.parsedFiles = [];
|
|
1911
1928
|
}
|
|
1912
1929
|
}
|
|
1913
|
-
|
|
1930
|
+
postResultCloneSafe(accumulated);
|
|
1914
1931
|
// Reset for potential reuse
|
|
1915
1932
|
accumulated = {
|
|
1916
1933
|
nodes: [],
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { ParseWorkerResult } from './parse-worker.js';
|
|
2
|
+
/**
|
|
3
|
+
* Deliver the accumulated result to the pool, surviving a non-cloneable value
|
|
4
|
+
* (#2112). Fast path: post as-is — on a healthy result this is the only thing
|
|
5
|
+
* that runs, so clone-safety adds zero overhead to normal runs. If structured
|
|
6
|
+
* clone rejects the payload (a function/symbol leaked into an extraction
|
|
7
|
+
* record — the reporter's case was a node `properties` value pointing at a
|
|
8
|
+
* native `toString`), rewrite the boundary-crossing arrays so the result is
|
|
9
|
+
* cloneable, record the affected paths on `result.skippedPaths`, warn the
|
|
10
|
+
* operator naming the offending field + file (so the still-unpinned leak is
|
|
11
|
+
* diagnosable from logs and fixable at source), and re-post.
|
|
12
|
+
*
|
|
13
|
+
* Recovery is attempted for ANY first-post failure, not only a `DataCloneError`.
|
|
14
|
+
* structuredClone invokes getters, and a getter that THROWS surfaces its own
|
|
15
|
+
* error (a `RangeError`, etc.) — NOT a `DataCloneError` (confirmed against a
|
|
16
|
+
* real MessageChannel). Gating recovery on `DataCloneError` let such a throw
|
|
17
|
+
* re-throw past the sanitizer and re-arm, under `POOL_SIZE=1`, the worker-death
|
|
18
|
+
* cascade this net prevents. The recovery path is wrapped in its own try/catch
|
|
19
|
+
* so a still-uncloneable re-post fails closed to a primitive-only
|
|
20
|
+
* `{type:'error'}` DELIBERATELY rather than escaping the worker.
|
|
21
|
+
*/
|
|
22
|
+
export declare function postResultCloneSafe(result: ParseWorkerResult): void;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker → main result delivery with clone-safety (#2112).
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `parse-worker.ts` into its own side-effect-free module so it
|
|
5
|
+
* can be imported and exercised directly (the parse worker is an entry module:
|
|
6
|
+
* importing it would construct the parser, post `ready`, and attach the real
|
|
7
|
+
* message handler). The integration test imports `postResultCloneSafe` from
|
|
8
|
+
* here to cover the production wiring end to end rather than re-implementing it.
|
|
9
|
+
*/
|
|
10
|
+
import { parentPort } from 'node:worker_threads';
|
|
11
|
+
import { makeWorkerResultCloneSafe } from './clone-safety.js';
|
|
12
|
+
/**
|
|
13
|
+
* Strict mode (opt-in via `GITNEXUS_STRICT_CLONE=1`, inherited by workers). When
|
|
14
|
+
* on, a clone failure THROWS with the offending key path instead of silently
|
|
15
|
+
* sanitizing + delivering — so a leak introduced by a future provider/extractor
|
|
16
|
+
* change fails LOUDLY (in CI / dev) at its origin rather than being quietly
|
|
17
|
+
* stripped in production. The silent-recovery behavior is exactly what hid the
|
|
18
|
+
* original #2112 leak; strict mode removes the silence where we want loudness.
|
|
19
|
+
* Off in production, where the net's job is to keep the run alive.
|
|
20
|
+
*/
|
|
21
|
+
const STRICT_CLONE = process.env.GITNEXUS_STRICT_CLONE === '1';
|
|
22
|
+
/**
|
|
23
|
+
* Deliver the accumulated result to the pool, surviving a non-cloneable value
|
|
24
|
+
* (#2112). Fast path: post as-is — on a healthy result this is the only thing
|
|
25
|
+
* that runs, so clone-safety adds zero overhead to normal runs. If structured
|
|
26
|
+
* clone rejects the payload (a function/symbol leaked into an extraction
|
|
27
|
+
* record — the reporter's case was a node `properties` value pointing at a
|
|
28
|
+
* native `toString`), rewrite the boundary-crossing arrays so the result is
|
|
29
|
+
* cloneable, record the affected paths on `result.skippedPaths`, warn the
|
|
30
|
+
* operator naming the offending field + file (so the still-unpinned leak is
|
|
31
|
+
* diagnosable from logs and fixable at source), and re-post.
|
|
32
|
+
*
|
|
33
|
+
* Recovery is attempted for ANY first-post failure, not only a `DataCloneError`.
|
|
34
|
+
* structuredClone invokes getters, and a getter that THROWS surfaces its own
|
|
35
|
+
* error (a `RangeError`, etc.) — NOT a `DataCloneError` (confirmed against a
|
|
36
|
+
* real MessageChannel). Gating recovery on `DataCloneError` let such a throw
|
|
37
|
+
* re-throw past the sanitizer and re-arm, under `POOL_SIZE=1`, the worker-death
|
|
38
|
+
* cascade this net prevents. The recovery path is wrapped in its own try/catch
|
|
39
|
+
* so a still-uncloneable re-post fails closed to a primitive-only
|
|
40
|
+
* `{type:'error'}` DELIBERATELY rather than escaping the worker.
|
|
41
|
+
*/
|
|
42
|
+
export function postResultCloneSafe(result) {
|
|
43
|
+
try {
|
|
44
|
+
parentPort.postMessage({ type: 'result', data: result });
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// Fall through to recovery on ANY failure (DataCloneError OR a throwing
|
|
49
|
+
// getter's own error). A healthy post returned above and never reaches here.
|
|
50
|
+
}
|
|
51
|
+
try {
|
|
52
|
+
// `as unknown as Record<string, unknown>` is the standard widening for a
|
|
53
|
+
// no-index-signature interface (TS rejects a single-step `as`). The field
|
|
54
|
+
// sets are typed to `keyof ParseWorkerResult` so renaming a field is a
|
|
55
|
+
// compile error here, not a silent loss of the drop-whole / skip protection.
|
|
56
|
+
const { skipped } = makeWorkerResultCloneSafe(result, {
|
|
57
|
+
dropWholeElement: new Set(['parsedFiles']),
|
|
58
|
+
skipFields: new Set(['skippedPaths']),
|
|
59
|
+
});
|
|
60
|
+
if (skipped.length > 0) {
|
|
61
|
+
if (STRICT_CLONE) {
|
|
62
|
+
// Surface the leak loudly with its exact key path(s) instead of
|
|
63
|
+
// delivering a sanitized result. Routes to the catch below → a
|
|
64
|
+
// primitive-only {type:'error'} the pool reports, failing CI.
|
|
65
|
+
const detail = skipped.map((s) => `${s.path}: ${s.reason}`).join('; ');
|
|
66
|
+
throw new Error(`GITNEXUS_STRICT_CLONE: worker result was not structured-cloneable — ${detail}`);
|
|
67
|
+
}
|
|
68
|
+
result.skippedPaths = [...(result.skippedPaths ?? []), ...skipped];
|
|
69
|
+
const sample = skipped
|
|
70
|
+
.slice(0, 5)
|
|
71
|
+
.map((s) => `${s.path} (${s.reason})`)
|
|
72
|
+
.join('; ');
|
|
73
|
+
const more = skipped.length > 5 ? ` …and ${skipped.length - 5} more` : '';
|
|
74
|
+
if (parentPort) {
|
|
75
|
+
parentPort.postMessage({
|
|
76
|
+
type: 'warning',
|
|
77
|
+
message: `Sanitized ${skipped.length} file(s) with non-serializable parse output before delivery: ${sample}${more}`,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
parentPort.postMessage({ type: 'result', data: result });
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
const e = err instanceof Error ? err : new Error(String(err));
|
|
85
|
+
parentPort.postMessage({ type: 'error', error: e.message, errorStack: e.stack });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Merge of accumulated parse-worker results (sub-batch result → the conceptual
|
|
3
|
+
* job's running accumulator).
|
|
4
|
+
*
|
|
5
|
+
* Extracted from `parse-worker.ts` into this side-effect-free module so the
|
|
6
|
+
* merge can be imported and unit-tested directly — the parse worker is an entry
|
|
7
|
+
* module (importing it constructs the parser, posts `ready`, and attaches the
|
|
8
|
+
* real MessagePort handler), so a main-thread test cannot import a helper out of
|
|
9
|
+
* it. Mirrors the `post-result.ts` extraction.
|
|
10
|
+
*
|
|
11
|
+
* `import type` of `ParseWorkerResult` is erased at runtime, so there is no
|
|
12
|
+
* import cycle with `parse-worker.ts` (which imports this module's runtime).
|
|
13
|
+
*/
|
|
14
|
+
import type { ParseWorkerResult } from './parse-worker.js';
|
|
15
|
+
/**
|
|
16
|
+
* Merge `src` into `target` in place: append every boundary-crossing array,
|
|
17
|
+
* sum the per-language skip counts, union the clone-safety `skippedPaths`, and
|
|
18
|
+
* add the file count.
|
|
19
|
+
*/
|
|
20
|
+
export declare const mergeResult: (target: ParseWorkerResult, src: ParseWorkerResult) => void;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// Use a loop instead of push(...spread) to avoid hitting V8's argument limit
|
|
2
|
+
// when merging large result sets (push(...arr) calls apply() under the hood
|
|
3
|
+
// and blows the stack when arr has >~65k elements).
|
|
4
|
+
const appendAll = (target, src) => {
|
|
5
|
+
for (let i = 0; i < src.length; i++)
|
|
6
|
+
target.push(src[i]);
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Merge `src` into `target` in place: append every boundary-crossing array,
|
|
10
|
+
* sum the per-language skip counts, union the clone-safety `skippedPaths`, and
|
|
11
|
+
* add the file count.
|
|
12
|
+
*/
|
|
13
|
+
export const mergeResult = (target, src) => {
|
|
14
|
+
appendAll(target.nodes, src.nodes);
|
|
15
|
+
appendAll(target.relationships, src.relationships);
|
|
16
|
+
appendAll(target.symbols, src.symbols);
|
|
17
|
+
appendAll(target.calls, src.calls);
|
|
18
|
+
appendAll(target.assignments, src.assignments);
|
|
19
|
+
appendAll(target.routes, src.routes);
|
|
20
|
+
appendAll(target.fetchCalls, src.fetchCalls);
|
|
21
|
+
appendAll(target.fetchWrapperDefs, src.fetchWrapperDefs);
|
|
22
|
+
appendAll(target.decoratorRoutes, src.decoratorRoutes);
|
|
23
|
+
if (src.routerIncludes)
|
|
24
|
+
appendAll(target.routerIncludes, src.routerIncludes);
|
|
25
|
+
if (src.routerImports)
|
|
26
|
+
appendAll(target.routerImports, src.routerImports);
|
|
27
|
+
if (src.routerModuleAliases) {
|
|
28
|
+
target.routerModuleAliases ??= [];
|
|
29
|
+
appendAll(target.routerModuleAliases, src.routerModuleAliases);
|
|
30
|
+
}
|
|
31
|
+
appendAll(target.toolDefs, src.toolDefs);
|
|
32
|
+
appendAll(target.ormQueries, src.ormQueries);
|
|
33
|
+
appendAll(target.constructorBindings, src.constructorBindings);
|
|
34
|
+
appendAll(target.fileScopeBindings, src.fileScopeBindings);
|
|
35
|
+
appendAll(target.parsedFiles, src.parsedFiles);
|
|
36
|
+
for (const [lang, count] of Object.entries(src.skippedLanguages)) {
|
|
37
|
+
target.skippedLanguages[lang] = (target.skippedLanguages[lang] || 0) + count;
|
|
38
|
+
}
|
|
39
|
+
if (src.skippedPaths && src.skippedPaths.length > 0) {
|
|
40
|
+
(target.skippedPaths ??= []).push(...src.skippedPaths);
|
|
41
|
+
}
|
|
42
|
+
target.fileCount += src.fileCount;
|
|
43
|
+
};
|
|
@@ -152,6 +152,15 @@ export interface WorkerPoolOptions {
|
|
|
152
152
|
* `undefined` ⇒ no durable write.
|
|
153
153
|
*/
|
|
154
154
|
durableParsedFileStoragePath?: string;
|
|
155
|
+
/**
|
|
156
|
+
* CFG/PDG opt-in (#2081 M1). Baked into every spawned worker's `workerData`
|
|
157
|
+
* (like the store paths above); when `true`, workers build a per-function
|
|
158
|
+
* control-flow graph from the tree-sitter AST and attach it to
|
|
159
|
+
* `ParsedFile.cfgSideChannel`. `undefined`/`false` ⇒ no CFG work.
|
|
160
|
+
*/
|
|
161
|
+
pdg?: boolean;
|
|
162
|
+
/** Per-function source-line cap for worker-side CFG construction (0 ⇒ no cap). */
|
|
163
|
+
pdgMaxFunctionLines?: number;
|
|
155
164
|
}
|
|
156
165
|
export declare class WorkerPoolDispatchError extends Error {
|
|
157
166
|
/**
|
|
@@ -273,6 +282,12 @@ export declare function resolveAutoPoolSize(): number;
|
|
|
273
282
|
* time spent across all attempts/splits/retries. When the budget is
|
|
274
283
|
* exhausted, the pool surfaces the in-flight path via `WorkerPoolDispatchError`
|
|
275
284
|
* instead of letting timeouts compound indefinitely.
|
|
285
|
+
*
|
|
286
|
+
* Upstream of these layers, the parse worker self-sanitizes a result that the
|
|
287
|
+
* structured-clone algorithm can't serialize (#2112) — stripping or dropping
|
|
288
|
+
* the offending value and reporting the affected paths on the result — so a
|
|
289
|
+
* single non-cloneable value can't masquerade as a worker death and exhaust a
|
|
290
|
+
* slot's respawn budget here.
|
|
276
291
|
*/
|
|
277
292
|
export declare const createWorkerPool: (workerUrl: URL, poolSize?: number, options?: WorkerPoolOptions) => WorkerPool;
|
|
278
293
|
export {};
|
|
@@ -523,6 +523,12 @@ function createJobs(items, maxItems, maxBytes, timeoutMs, chunkHash) {
|
|
|
523
523
|
* time spent across all attempts/splits/retries. When the budget is
|
|
524
524
|
* exhausted, the pool surfaces the in-flight path via `WorkerPoolDispatchError`
|
|
525
525
|
* instead of letting timeouts compound indefinitely.
|
|
526
|
+
*
|
|
527
|
+
* Upstream of these layers, the parse worker self-sanitizes a result that the
|
|
528
|
+
* structured-clone algorithm can't serialize (#2112) — stripping or dropping
|
|
529
|
+
* the offending value and reporting the affected paths on the result — so a
|
|
530
|
+
* single non-cloneable value can't masquerade as a worker death and exhaust a
|
|
531
|
+
* slot's respawn budget here.
|
|
526
532
|
*/
|
|
527
533
|
export const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
528
534
|
// Validate worker script exists before spawning to prevent uncaught
|
|
@@ -544,8 +550,11 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
|
544
550
|
// signature is unchanged so the zero-arg test factories keep working.
|
|
545
551
|
const parsedFileStoreStoragePath = options?.parsedFileStoreStoragePath;
|
|
546
552
|
const durableParsedFileStoragePath = options?.durableParsedFileStoragePath;
|
|
547
|
-
|
|
548
|
-
|
|
553
|
+
// CFG/PDG opt-in (#2081 M1) — carried in workerData alongside the store paths.
|
|
554
|
+
const pdg = options?.pdg === true;
|
|
555
|
+
const pdgMaxFunctionLines = options?.pdgMaxFunctionLines;
|
|
556
|
+
const workerStoreData = parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg
|
|
557
|
+
? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines }
|
|
549
558
|
: undefined;
|
|
550
559
|
const spawnWorker = options?.workerFactory ??
|
|
551
560
|
((url) => new Worker(url, {
|
|
@@ -1313,14 +1322,19 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
|
1313
1322
|
if (settled || stopped)
|
|
1314
1323
|
return;
|
|
1315
1324
|
// Native postMessage delivers POJO directly via Node's
|
|
1316
|
-
// structured clone.
|
|
1317
|
-
//
|
|
1318
|
-
//
|
|
1319
|
-
//
|
|
1320
|
-
//
|
|
1321
|
-
//
|
|
1322
|
-
// `
|
|
1323
|
-
//
|
|
1325
|
+
// structured clone. Two distinct clone failure modes exist,
|
|
1326
|
+
// and NEITHER reaches this handler: (1) a SENDER-side
|
|
1327
|
+
// non-cloneable value (a function/symbol that leaked into the
|
|
1328
|
+
// result) throws a synchronous `DataCloneError` on the
|
|
1329
|
+
// worker's own postMessage — the parse worker self-sanitizes
|
|
1330
|
+
// such results before delivery (#2112) and falls back to a
|
|
1331
|
+
// primitive-only `{type:'error'}` if it still can't serialize;
|
|
1332
|
+
// (2) a RECEIVER-side deserialization failure surfaces as a
|
|
1333
|
+
// `messageerror` event handled below. The only thing THIS
|
|
1334
|
+
// handler guards is a worker that sends a message without a
|
|
1335
|
+
// `type` discriminant (a worker bug, not a wire-format issue):
|
|
1336
|
+
// without the guard `null.type` would throw a TypeError out of
|
|
1337
|
+
// the EventEmitter listener → uncaughtException on the main
|
|
1324
1338
|
// thread.
|
|
1325
1339
|
const msg = raw;
|
|
1326
1340
|
if (msg === null || typeof msg !== 'object' || typeof msg.type !== 'string') {
|
|
@@ -1417,12 +1431,15 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
|
1417
1431
|
`Likely OOM or native addon failure${inFlightSuffix}.`, excludes);
|
|
1418
1432
|
}
|
|
1419
1433
|
};
|
|
1420
|
-
// `messageerror` fires when V8 fails to
|
|
1421
|
-
// payload (
|
|
1422
|
-
//
|
|
1423
|
-
//
|
|
1424
|
-
//
|
|
1425
|
-
//
|
|
1434
|
+
// `messageerror` fires when V8 fails to DESERIALIZE a postMessage
|
|
1435
|
+
// payload on THIS (receiver) side — a value that serialized on the
|
|
1436
|
+
// worker but can't be reconstructed here. (A non-cloneable value on
|
|
1437
|
+
// the SENDER side instead throws a synchronous DataCloneError on the
|
|
1438
|
+
// worker's own postMessage; that path is caught and sanitized
|
|
1439
|
+
// worker-side (#2112) and never arrives here.) The worker stays ALIVE
|
|
1440
|
+
// but the message is lost — without this handler the pool would sit on
|
|
1441
|
+
// the dropped message until the idle timeout expires. Treat it as
|
|
1442
|
+
// worker death so the resilience layers fire:
|
|
1426
1443
|
// requeue the remainder via `recoverAndResume`, attribute the
|
|
1427
1444
|
// in-flight file from the `starting-file` signal (if observed),
|
|
1428
1445
|
// and let the per-slot respawn budget and circuit breaker decide
|
|
@@ -233,6 +233,25 @@ export declare const loadVectorExtension: (targetConn?: lbug.Connection, opts?:
|
|
|
233
233
|
* @param stemmer - Stemming algorithm (default: 'porter')
|
|
234
234
|
*/
|
|
235
235
|
export declare const createFTSIndex: (tableName: string, indexName: string, properties: string[], stemmer?: string) => Promise<void>;
|
|
236
|
+
/**
|
|
237
|
+
* Create the HNSW vector index on the CodeEmbedding table.
|
|
238
|
+
*
|
|
239
|
+
* MUST run via `conn.query()` (here through `queryAndDrain`), NOT through the
|
|
240
|
+
* prepared `executeQuery`/`conn.prepare()` path: `CALL CREATE_VECTOR_INDEX(...)`
|
|
241
|
+
* compiles to multiple statements, which LadybugDB cannot prepare — it fails
|
|
242
|
+
* with "Connection Exception: We do not support prepare multiple statements."
|
|
243
|
+
* Routing index creation through `executeQuery` (prepared) is exactly what
|
|
244
|
+
* broke vector-index creation during `analyze` (#2114; the singleton
|
|
245
|
+
* `executeQuery` was switched to the prepared path in #1655 while FTS index
|
|
246
|
+
* creation kept using `conn.query()`, which is why FTS survived and VECTOR did
|
|
247
|
+
* not). Mirrors `createFTSIndex` above.
|
|
248
|
+
*
|
|
249
|
+
* Returns `true` on success (or when the index already exists — idempotent so
|
|
250
|
+
* incremental re-runs don't spuriously downgrade to exact scan), `false` when
|
|
251
|
+
* the VECTOR extension is unavailable or the connection is read-only. Any other
|
|
252
|
+
* failure propagates so the caller can log it.
|
|
253
|
+
*/
|
|
254
|
+
export declare const createVectorIndex: () => Promise<boolean>;
|
|
236
255
|
/**
|
|
237
256
|
* Lazy-create an FTS index, caching the fact in-process.
|
|
238
257
|
*
|
|
@@ -8,7 +8,7 @@ import os from 'os';
|
|
|
8
8
|
import crypto from 'crypto';
|
|
9
9
|
import lbug from '@ladybugdb/core';
|
|
10
10
|
import { closeQueryResults } from './query-result-utils.js';
|
|
11
|
-
import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, STALE_HASH_SENTINEL, } from './schema.js';
|
|
11
|
+
import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, CREATE_VECTOR_INDEX_QUERY, STALE_HASH_SENTINEL, } from './schema.js';
|
|
12
12
|
import { streamAllCSVsToDisk } from './csv-generator.js';
|
|
13
13
|
import { extensionManager } from './extension-loader.js';
|
|
14
14
|
import { closeLbugConnection, isDbBusyError, isOpenRetryExhausted, isWalCorruptionError, openLbugConnection, toNativeSafePath, WAL_RECOVERY_SUGGESTION, waitForWindowsHandleRelease, } from './lbug-config.js';
|
|
@@ -119,6 +119,11 @@ let currentDbPath = null;
|
|
|
119
119
|
let currentDbReadOnly = false;
|
|
120
120
|
let ftsLoaded = false;
|
|
121
121
|
let vectorExtensionLoaded = false;
|
|
122
|
+
// In-process guard so a repeated createVectorIndex() within one connection
|
|
123
|
+
// lifetime skips the DB round-trip (mirrors ensuredFTSIndexes). Reset wherever
|
|
124
|
+
// vectorExtensionLoaded resets, so it can never stay true against a swapped or
|
|
125
|
+
// closed connection.
|
|
126
|
+
let vectorIndexEnsured = false;
|
|
122
127
|
/**
|
|
123
128
|
* In-process cache of FTS indexes observed against the current singleton
|
|
124
129
|
* connection. Avoids repeated `CALL CREATE_FTS_INDEX` calls, which can trip
|
|
@@ -494,6 +499,7 @@ const resetOpenConnectionState = () => {
|
|
|
494
499
|
currentDbPath = null;
|
|
495
500
|
ftsLoaded = false;
|
|
496
501
|
vectorExtensionLoaded = false;
|
|
502
|
+
vectorIndexEnsured = false;
|
|
497
503
|
ensuredFTSIndexes.clear();
|
|
498
504
|
};
|
|
499
505
|
const runSchemaCreationQueries = async (dbPath) => {
|
|
@@ -572,6 +578,7 @@ export const withLbugDb = async (dbPath, operation, options = {}) => {
|
|
|
572
578
|
currentDbPath = null;
|
|
573
579
|
ftsLoaded = false;
|
|
574
580
|
vectorExtensionLoaded = false;
|
|
581
|
+
vectorIndexEnsured = false;
|
|
575
582
|
ensuredFTSIndexes.clear();
|
|
576
583
|
});
|
|
577
584
|
// Sleep outside the lock — no need to block others while waiting
|
|
@@ -596,6 +603,7 @@ const doInitLbug = async (dbPath, readOnly = false) => {
|
|
|
596
603
|
currentDbPath = null;
|
|
597
604
|
ftsLoaded = false;
|
|
598
605
|
vectorExtensionLoaded = false;
|
|
606
|
+
vectorIndexEnsured = false;
|
|
599
607
|
ensuredFTSIndexes.clear();
|
|
600
608
|
}
|
|
601
609
|
// ---------------------------------------------------------------------------
|
|
@@ -1476,6 +1484,7 @@ export const closeLbug = async () => {
|
|
|
1476
1484
|
currentDbPath = null;
|
|
1477
1485
|
ftsLoaded = false;
|
|
1478
1486
|
vectorExtensionLoaded = false;
|
|
1487
|
+
vectorIndexEnsured = false;
|
|
1479
1488
|
ensuredFTSIndexes.clear();
|
|
1480
1489
|
};
|
|
1481
1490
|
export const isLbugReady = () => conn !== null && db !== null;
|
|
@@ -1708,6 +1717,52 @@ export const createFTSIndex = async (tableName, indexName, properties, stemmer =
|
|
|
1708
1717
|
throw e;
|
|
1709
1718
|
}
|
|
1710
1719
|
};
|
|
1720
|
+
/**
|
|
1721
|
+
* Create the HNSW vector index on the CodeEmbedding table.
|
|
1722
|
+
*
|
|
1723
|
+
* MUST run via `conn.query()` (here through `queryAndDrain`), NOT through the
|
|
1724
|
+
* prepared `executeQuery`/`conn.prepare()` path: `CALL CREATE_VECTOR_INDEX(...)`
|
|
1725
|
+
* compiles to multiple statements, which LadybugDB cannot prepare — it fails
|
|
1726
|
+
* with "Connection Exception: We do not support prepare multiple statements."
|
|
1727
|
+
* Routing index creation through `executeQuery` (prepared) is exactly what
|
|
1728
|
+
* broke vector-index creation during `analyze` (#2114; the singleton
|
|
1729
|
+
* `executeQuery` was switched to the prepared path in #1655 while FTS index
|
|
1730
|
+
* creation kept using `conn.query()`, which is why FTS survived and VECTOR did
|
|
1731
|
+
* not). Mirrors `createFTSIndex` above.
|
|
1732
|
+
*
|
|
1733
|
+
* Returns `true` on success (or when the index already exists — idempotent so
|
|
1734
|
+
* incremental re-runs don't spuriously downgrade to exact scan), `false` when
|
|
1735
|
+
* the VECTOR extension is unavailable or the connection is read-only. Any other
|
|
1736
|
+
* failure propagates so the caller can log it.
|
|
1737
|
+
*/
|
|
1738
|
+
export const createVectorIndex = async () => {
|
|
1739
|
+
if (!conn) {
|
|
1740
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1741
|
+
}
|
|
1742
|
+
// Already built on this connection — skip the round-trip (mirrors createFTSIndex).
|
|
1743
|
+
if (vectorIndexEnsured)
|
|
1744
|
+
return true;
|
|
1745
|
+
if (!(await loadVectorExtension())) {
|
|
1746
|
+
return false;
|
|
1747
|
+
}
|
|
1748
|
+
try {
|
|
1749
|
+
await queryAndDrain(conn, CREATE_VECTOR_INDEX_QUERY);
|
|
1750
|
+
vectorIndexEnsured = true;
|
|
1751
|
+
return true;
|
|
1752
|
+
}
|
|
1753
|
+
catch (e) {
|
|
1754
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
1755
|
+
// Idempotent: a prior analyze already built the HNSW index.
|
|
1756
|
+
if (msg.includes('already exists')) {
|
|
1757
|
+
vectorIndexEnsured = true;
|
|
1758
|
+
return true;
|
|
1759
|
+
}
|
|
1760
|
+
// Read-only DB (e.g. the MCP query pool): writable analyze owns creation.
|
|
1761
|
+
if (isReadOnlyDbError(e))
|
|
1762
|
+
return false;
|
|
1763
|
+
throw e;
|
|
1764
|
+
}
|
|
1765
|
+
};
|
|
1711
1766
|
/**
|
|
1712
1767
|
* Lazy-create an FTS index, caching the fact in-process.
|
|
1713
1768
|
*
|