gitnexus 1.6.10-rc.3 → 1.6.10-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/cli/doctor.js +10 -2
- package/dist/core/group/extractors/manifest-extractor.js +4 -0
- package/dist/core/ingestion/cobol/jcl-processor.js +9 -8
- package/dist/core/ingestion/cobol-processor.js +27 -26
- package/dist/core/ingestion/emit-references.js +3 -2
- package/dist/core/ingestion/markdown-processor.js +3 -2
- package/dist/core/ingestion/utils/line-base.d.ts +20 -0
- package/dist/core/ingestion/utils/line-base.js +20 -0
- package/dist/core/ingestion/utils/symbol-labels.d.ts +21 -0
- package/dist/core/ingestion/utils/symbol-labels.js +45 -0
- package/dist/core/lbug/csv-generator.js +8 -2
- package/dist/core/lbug/extension-loader.d.ts +10 -2
- package/dist/core/lbug/extension-loader.js +29 -12
- package/dist/core/lbug/native-check.d.ts +22 -0
- package/dist/core/lbug/native-check.js +66 -0
- package/dist/core/platform/capabilities.js +27 -1
- package/dist/core/run-analyze.js +11 -4
- package/dist/core/search/fts-indexes.d.ts +7 -0
- package/dist/core/search/fts-indexes.js +27 -0
- package/dist/mcp/local/line-display.d.ts +22 -0
- package/dist/mcp/local/line-display.js +3 -0
- package/dist/mcp/local/local-backend.d.ts +8 -0
- package/dist/mcp/local/local-backend.js +42 -17
- package/dist/mcp/local/pdg-impact.d.ts +5 -3
- package/dist/mcp/local/pdg-impact.js +5 -2
- package/dist/mcp/resources.js +1 -0
- package/dist/mcp/tools.js +1 -1
- package/dist/server/api.js +2 -2
- package/dist/storage/repo-manager.d.ts +6 -1
- package/dist/storage/repo-manager.js +6 -1
- package/package.json +1 -1
- package/scripts/cross-platform-tests.ts +4 -0
- package/scripts/install-duckdb-extension.mjs +83 -24
package/README.md
CHANGED
|
@@ -444,14 +444,14 @@ The prefix defaults to `~/.gitnexus/embedding-runtime`; set `GITNEXUS_EMBEDDING_
|
|
|
444
444
|
|
|
445
445
|
### Analyze warns about unavailable FTS or VECTOR extensions
|
|
446
446
|
|
|
447
|
-
GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnexus serve` and MCP read paths only ever try to `LOAD` the extensions — they never block on a network install. The `analyze` command, by default, attempts one bounded out-of-process `INSTALL`
|
|
447
|
+
GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnexus serve` and MCP read paths only ever try to `LOAD` the extensions — they never block on a network install. The `analyze` command, by default, attempts one bounded out-of-process install if `LOAD` fails (a plain `INSTALL` to download a missing extension, escalating to `FORCE INSTALL` only when the `LOAD` error shows the existing file is broken or truncated, so a permanent non-file failure does not re-download on every run) and proceeds even when that install times out, so the index is always written to disk; BM25/vector search degrade gracefully until the extensions become available.
|
|
448
448
|
|
|
449
449
|
Configure the behavior with these environment variables:
|
|
450
450
|
|
|
451
451
|
| Variable | Values | Default | Effect |
|
|
452
452
|
| -------------------------------------------- | ---------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
453
|
-
| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded
|
|
454
|
-
| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process
|
|
453
|
+
| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. |
|
|
454
|
+
| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. |
|
|
455
455
|
| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. |
|
|
456
456
|
| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. |
|
|
457
457
|
| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. |
|
package/dist/cli/doctor.js
CHANGED
|
@@ -4,7 +4,7 @@ import { isHttpMode } from '../core/embeddings/http-client.js';
|
|
|
4
4
|
import { getLocalEmbeddingRuntimeBlocker, localEmbeddingPrefixUnloadableMessage, localEmbeddingStackMissingMessage, } from '../core/embeddings/runtime-support.js';
|
|
5
5
|
import { isPrefixRuntimeLoadable, resolveEmbeddingRuntime, } from '../core/embeddings/runtime-install.js';
|
|
6
6
|
import { cudaRedirectDoctorStatus } from '../core/embeddings/onnxruntime-node-resolver.js';
|
|
7
|
-
import { checkLbugNative } from '../core/lbug/native-check.js';
|
|
7
|
+
import { checkLbugNative, probeFtsExtensionLoad } from '../core/lbug/native-check.js';
|
|
8
8
|
import { getExtensionInstallPolicy } from '../core/lbug/extension-loader.js';
|
|
9
9
|
import { t } from './i18n/index.js';
|
|
10
10
|
function isCombiningMark(codePoint) {
|
|
@@ -108,7 +108,15 @@ export const doctorCommand = async () => {
|
|
|
108
108
|
console.log('');
|
|
109
109
|
console.log(t('doctor.capabilities'));
|
|
110
110
|
console.log(` ${label('doctor.labels.graphStore', 18)}${capabilities.graph}`);
|
|
111
|
-
|
|
111
|
+
// Live LOAD probe, not the static platform capability — the static value
|
|
112
|
+
// said "available" while analyze failed to load the extension (#2374).
|
|
113
|
+
const ftsProbe = nativeCheck.ok
|
|
114
|
+
? await probeFtsExtensionLoad()
|
|
115
|
+
: { loaded: false, reason: 'LadybugDB native module (lbugjs.node) failed to load' };
|
|
116
|
+
console.log(` ${label('doctor.labels.fullTextSearch', 18)}${ftsProbe.loaded ? 'available' : 'unavailable'}`);
|
|
117
|
+
if (!ftsProbe.loaded && ftsProbe.reason) {
|
|
118
|
+
console.log(` ${padDisplayEnd('', 18)}${ftsProbe.reason}`);
|
|
119
|
+
}
|
|
112
120
|
console.log(` ${label('doctor.labels.vectorIndex', 18)}${capabilities.vector}`);
|
|
113
121
|
console.log(` ${label('doctor.labels.semanticMode', 18)}${capabilities.semanticMode}`);
|
|
114
122
|
// Surface the optional-extension install policy so offline users can see
|
|
@@ -7,6 +7,10 @@ import { logger } from '../../logger.js';
|
|
|
7
7
|
// reserved-keyword labels `Macro` and `Union`, and LadybugDB's parser rejects
|
|
8
8
|
// a disjunction that names a reserved keyword (#2325) — which the resolver's
|
|
9
9
|
// try/catch then swallowed. `labels(n) IN` has no such collision.
|
|
10
|
+
// This list overlaps `ingestion/utils/symbol-labels.ts` (SYMBOL_NODE_LABELS) but
|
|
11
|
+
// is a deliberate SUBSET — it omits `Namespace`/`Variable`/`Module`. Unifying the
|
|
12
|
+
// two would widen which nodes resolve as contract symbols and must update the
|
|
13
|
+
// #2325 test, so they are intentionally kept separate for now.
|
|
10
14
|
export const CUSTOM_CONTRACT_RESOLVE_QUERY = `MATCH (n)
|
|
11
15
|
WHERE labels(n) IN ['Function','Method','Class','Interface','Struct','Enum','Trait','Constructor','TypeAlias','Impl','Macro','Union','Typedef','Property','Record','Delegate','Annotation','Template','Const','Static','CodeElement']
|
|
12
16
|
AND n.name = $symbolName
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import { parseJcl } from './jcl-parser.js';
|
|
19
19
|
import { generateId } from '../../../lib/utils.js';
|
|
20
|
+
import { toZeroBasedLine } from '../utils/line-base.js';
|
|
20
21
|
/**
|
|
21
22
|
* Process JCL files and integrate into the knowledge graph.
|
|
22
23
|
*
|
|
@@ -71,8 +72,8 @@ function integrateJclResults(graph, parsed, filePath, moduleNames) {
|
|
|
71
72
|
properties: {
|
|
72
73
|
name: job.name,
|
|
73
74
|
filePath,
|
|
74
|
-
startLine: job.line,
|
|
75
|
-
endLine: job.line,
|
|
75
|
+
startLine: toZeroBasedLine(job.line),
|
|
76
|
+
endLine: toZeroBasedLine(job.line),
|
|
76
77
|
description: `jcl-job${classPart}${msgPart}`,
|
|
77
78
|
},
|
|
78
79
|
});
|
|
@@ -105,8 +106,8 @@ function integrateJclResults(graph, parsed, filePath, moduleNames) {
|
|
|
105
106
|
properties: {
|
|
106
107
|
name: step.name,
|
|
107
108
|
filePath,
|
|
108
|
-
startLine: step.line,
|
|
109
|
-
endLine: step.line,
|
|
109
|
+
startLine: toZeroBasedLine(step.line),
|
|
110
|
+
endLine: toZeroBasedLine(step.line),
|
|
110
111
|
description: `jcl-step${pgmPart}${procPart}`,
|
|
111
112
|
},
|
|
112
113
|
});
|
|
@@ -170,8 +171,8 @@ function integrateJclResults(graph, parsed, filePath, moduleNames) {
|
|
|
170
171
|
properties: {
|
|
171
172
|
name: dd.dataset,
|
|
172
173
|
filePath,
|
|
173
|
-
startLine: dd.line,
|
|
174
|
-
endLine: dd.line,
|
|
174
|
+
startLine: toZeroBasedLine(dd.line),
|
|
175
|
+
endLine: toZeroBasedLine(dd.line),
|
|
175
176
|
description: `jcl-dataset${dispPart}`,
|
|
176
177
|
},
|
|
177
178
|
});
|
|
@@ -202,8 +203,8 @@ function integrateJclResults(graph, parsed, filePath, moduleNames) {
|
|
|
202
203
|
properties: {
|
|
203
204
|
name: proc.name,
|
|
204
205
|
filePath,
|
|
205
|
-
startLine: proc.line,
|
|
206
|
-
endLine: proc.line,
|
|
206
|
+
startLine: toZeroBasedLine(proc.line),
|
|
207
|
+
endLine: toZeroBasedLine(proc.line),
|
|
207
208
|
description: 'jcl-proc-instream',
|
|
208
209
|
},
|
|
209
210
|
});
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import path from 'node:path';
|
|
16
16
|
import { generateId } from '../../lib/utils.js';
|
|
17
|
+
import { toZeroBasedLine } from './utils/line-base.js';
|
|
17
18
|
import { SupportedLanguages } from '../../_shared/index.js';
|
|
18
19
|
import { preprocessCobolSource, extractCobolSymbolsWithRegex, } from './cobol/cobol-preprocessor.js';
|
|
19
20
|
import { expandCopies } from './cobol/cobol-copy-expander.js';
|
|
@@ -275,8 +276,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
275
276
|
properties: {
|
|
276
277
|
name: extracted.programName,
|
|
277
278
|
filePath,
|
|
278
|
-
startLine: 1,
|
|
279
|
-
endLine: lines.length,
|
|
279
|
+
startLine: toZeroBasedLine(1),
|
|
280
|
+
endLine: toZeroBasedLine(lines.length),
|
|
280
281
|
language: SupportedLanguages.Cobol,
|
|
281
282
|
isExported: true,
|
|
282
283
|
description: metaDesc || undefined,
|
|
@@ -310,8 +311,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
310
311
|
properties: {
|
|
311
312
|
name: prog.name,
|
|
312
313
|
filePath,
|
|
313
|
-
startLine: prog.startLine,
|
|
314
|
-
endLine: prog.endLine,
|
|
314
|
+
startLine: toZeroBasedLine(prog.startLine),
|
|
315
|
+
endLine: toZeroBasedLine(prog.endLine),
|
|
315
316
|
language: SupportedLanguages.Cobol,
|
|
316
317
|
isExported: true,
|
|
317
318
|
description: `nested-program${prog.isCommon ? ' common' : ''}`,
|
|
@@ -349,8 +350,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
349
350
|
properties: {
|
|
350
351
|
name: sec.name,
|
|
351
352
|
filePath,
|
|
352
|
-
startLine: sec.line,
|
|
353
|
-
endLine: nextLine,
|
|
353
|
+
startLine: toZeroBasedLine(sec.line),
|
|
354
|
+
endLine: toZeroBasedLine(nextLine),
|
|
354
355
|
language: SupportedLanguages.Cobol,
|
|
355
356
|
isExported: true,
|
|
356
357
|
},
|
|
@@ -379,8 +380,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
379
380
|
properties: {
|
|
380
381
|
name: para.name,
|
|
381
382
|
filePath,
|
|
382
|
-
startLine: para.line,
|
|
383
|
-
endLine: nextLine,
|
|
383
|
+
startLine: toZeroBasedLine(para.line),
|
|
384
|
+
endLine: toZeroBasedLine(nextLine),
|
|
384
385
|
language: SupportedLanguages.Cobol,
|
|
385
386
|
isExported: true,
|
|
386
387
|
},
|
|
@@ -412,8 +413,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
412
413
|
properties: {
|
|
413
414
|
name: item.name,
|
|
414
415
|
filePath,
|
|
415
|
-
startLine: item.line,
|
|
416
|
-
endLine: item.line,
|
|
416
|
+
startLine: toZeroBasedLine(item.line),
|
|
417
|
+
endLine: toZeroBasedLine(item.line),
|
|
417
418
|
language: SupportedLanguages.Cobol,
|
|
418
419
|
description: `level:${item.level} section:${item.section}${item.pic ? ` pic:${item.pic}` : ''}`,
|
|
419
420
|
},
|
|
@@ -506,8 +507,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
506
507
|
properties: {
|
|
507
508
|
name: `CALL ${call.target}`,
|
|
508
509
|
filePath,
|
|
509
|
-
startLine: call.line,
|
|
510
|
-
endLine: call.line,
|
|
510
|
+
startLine: toZeroBasedLine(call.line),
|
|
511
|
+
endLine: toZeroBasedLine(call.line),
|
|
511
512
|
language: SupportedLanguages.Cobol,
|
|
512
513
|
description: 'dynamic-call (target is a data item, not resolvable statically)',
|
|
513
514
|
},
|
|
@@ -619,8 +620,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
619
620
|
properties: {
|
|
620
621
|
name: `EXEC SQL ${sql.operation}`,
|
|
621
622
|
filePath,
|
|
622
|
-
startLine: sql.line,
|
|
623
|
-
endLine: sql.line,
|
|
623
|
+
startLine: toZeroBasedLine(sql.line),
|
|
624
|
+
endLine: toZeroBasedLine(sql.line),
|
|
624
625
|
language: SupportedLanguages.Cobol,
|
|
625
626
|
description: `tables:[${sql.tables.join(',')}] cursors:[${sql.cursors.join(',')}]`,
|
|
626
627
|
},
|
|
@@ -691,8 +692,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
691
692
|
properties: {
|
|
692
693
|
name: `EXEC CICS ${cics.command}`,
|
|
693
694
|
filePath,
|
|
694
|
-
startLine: cics.line,
|
|
695
|
-
endLine: cics.line,
|
|
695
|
+
startLine: toZeroBasedLine(cics.line),
|
|
696
|
+
endLine: toZeroBasedLine(cics.line),
|
|
696
697
|
language: SupportedLanguages.Cobol,
|
|
697
698
|
description: [
|
|
698
699
|
cics.mapName && `map:${cics.mapName}`,
|
|
@@ -726,8 +727,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
726
727
|
properties: {
|
|
727
728
|
name: `CICS ${cics.command} ${cics.programName}`,
|
|
728
729
|
filePath,
|
|
729
|
-
startLine: cics.line,
|
|
730
|
-
endLine: cics.line,
|
|
730
|
+
startLine: toZeroBasedLine(cics.line),
|
|
731
|
+
endLine: toZeroBasedLine(cics.line),
|
|
731
732
|
language: SupportedLanguages.Cobol,
|
|
732
733
|
description: `cics-dynamic-program (target is data item ${cics.programName})`,
|
|
733
734
|
},
|
|
@@ -879,8 +880,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
879
880
|
properties: {
|
|
880
881
|
name: entry.name,
|
|
881
882
|
filePath,
|
|
882
|
-
startLine: entry.line,
|
|
883
|
-
endLine: entry.line,
|
|
883
|
+
startLine: toZeroBasedLine(entry.line),
|
|
884
|
+
endLine: toZeroBasedLine(entry.line),
|
|
884
885
|
language: SupportedLanguages.Cobol,
|
|
885
886
|
isExported: true,
|
|
886
887
|
description: entry.parameters.length > 0 ? `using:${entry.parameters.join(',')}` : undefined,
|
|
@@ -1014,8 +1015,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
1014
1015
|
properties: {
|
|
1015
1016
|
name: `EXEC DLI ${dli.verb}`,
|
|
1016
1017
|
filePath,
|
|
1017
|
-
startLine: dli.line,
|
|
1018
|
-
endLine: dli.line,
|
|
1018
|
+
startLine: toZeroBasedLine(dli.line),
|
|
1019
|
+
endLine: toZeroBasedLine(dli.line),
|
|
1019
1020
|
language: SupportedLanguages.Cobol,
|
|
1020
1021
|
description: [
|
|
1021
1022
|
dli.segmentName && `segment:${dli.segmentName}`,
|
|
@@ -1148,8 +1149,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
1148
1149
|
properties: {
|
|
1149
1150
|
name: fd.selectName,
|
|
1150
1151
|
filePath,
|
|
1151
|
-
startLine: fd.line,
|
|
1152
|
-
endLine: fd.line,
|
|
1152
|
+
startLine: toZeroBasedLine(fd.line),
|
|
1153
|
+
endLine: toZeroBasedLine(fd.line),
|
|
1153
1154
|
language: SupportedLanguages.Cobol,
|
|
1154
1155
|
description: `assign:${fd.assignTo}${fd.isOptional ? ' optional' : ''}${fd.organization ? ` org:${fd.organization}` : ''}${fd.access ? ` access:${fd.access}` : ''}`,
|
|
1155
1156
|
},
|
|
@@ -1231,8 +1232,8 @@ function mapToGraph(graph, extracted, file, copyResolutions, moduleNodeIds) {
|
|
|
1231
1232
|
properties: {
|
|
1232
1233
|
name: `CANCEL ${cancel.target}`,
|
|
1233
1234
|
filePath,
|
|
1234
|
-
startLine: cancel.line,
|
|
1235
|
-
endLine: cancel.line,
|
|
1235
|
+
startLine: toZeroBasedLine(cancel.line),
|
|
1236
|
+
endLine: toZeroBasedLine(cancel.line),
|
|
1236
1237
|
language: SupportedLanguages.Cobol,
|
|
1237
1238
|
description: 'dynamic-cancel (target is a data item, not resolvable statically)',
|
|
1238
1239
|
},
|
|
@@ -44,6 +44,7 @@
|
|
|
44
44
|
* ancestor scope, and if THAT produces nothing either the edge is
|
|
45
45
|
* skipped (with a count returned in `EmitStats.skippedNoCaller`).
|
|
46
46
|
*/
|
|
47
|
+
import { toZeroBasedLine } from './utils/line-base.js';
|
|
47
48
|
/**
|
|
48
49
|
* Drain `referenceIndex.bySourceScope` into graph edges.
|
|
49
50
|
*
|
|
@@ -95,8 +96,8 @@ export function emitScopeGraph(input) {
|
|
|
95
96
|
properties: {
|
|
96
97
|
name: scope.kind,
|
|
97
98
|
filePath: scope.filePath,
|
|
98
|
-
startLine: scope.range.startLine,
|
|
99
|
-
endLine: scope.range.endLine,
|
|
99
|
+
startLine: toZeroBasedLine(scope.range.startLine),
|
|
100
|
+
endLine: toZeroBasedLine(scope.range.endLine),
|
|
100
101
|
description: `Scope: ${scope.kind}`,
|
|
101
102
|
},
|
|
102
103
|
});
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import path from 'node:path';
|
|
9
9
|
import { generateId } from '../../lib/utils.js';
|
|
10
|
+
import { toZeroBasedLine } from './utils/line-base.js';
|
|
10
11
|
const HEADING_RE = /^(#{1,6})\s+(.+)$/;
|
|
11
12
|
const LINK_RE = /\[([^\]]*)\]\(([^)]+)\)/g;
|
|
12
13
|
const MD_EXTENSIONS = new Set(['.md', '.mdx']);
|
|
@@ -59,8 +60,8 @@ export const processMarkdown = (graph, files, allPathSet) => {
|
|
|
59
60
|
properties: {
|
|
60
61
|
name: heading,
|
|
61
62
|
filePath: file.path,
|
|
62
|
-
startLine: lineNum,
|
|
63
|
-
endLine,
|
|
63
|
+
startLine: toZeroBasedLine(lineNum),
|
|
64
|
+
endLine: toZeroBasedLine(endLine),
|
|
64
65
|
level,
|
|
65
66
|
description: `h${level}`,
|
|
66
67
|
},
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert a 1-based source line number to the 0-based convention used by
|
|
3
|
+
* GraphNode `startLine`/`endLine`.
|
|
4
|
+
*
|
|
5
|
+
* The graph layer stores line numbers 0-based (tree-sitter `startPosition.row`),
|
|
6
|
+
* and this is load-bearing: the taint/PDG/CFG join and the MCP consumers all add
|
|
7
|
+
* `+ 1` to recover 1-based (see `summary-harvest-driver.ts` — "Function/Method
|
|
8
|
+
* node startLine is 0-based"). Most emitters get 0-based for free from
|
|
9
|
+
* tree-sitter. The exceptions are the regex-based COBOL/JCL processors (their
|
|
10
|
+
* parsers use `lineNum = i + 1`) and the scope-capture path (`Capture` ranges
|
|
11
|
+
* are 1-based per RFC §2.1). Those must convert to 0-based when they build a
|
|
12
|
+
* graph node, or the exact-content slice in `csv-generator.ts` drops the
|
|
13
|
+
* symbol's declaration line (#2379) and reported line numbers are off (#2377).
|
|
14
|
+
*
|
|
15
|
+
* Apply this ONLY at the graph-node `startLine:`/`endLine:` assignment. The
|
|
16
|
+
* parser-internal 1-based values (`.line`, `prog.startLine`) stay 1-based —
|
|
17
|
+
* they feed `L${line}` node/edge IDs and line-range containment checks that
|
|
18
|
+
* must not shift. The clamp guards degenerate inputs (line 0 / empty files).
|
|
19
|
+
*/
|
|
20
|
+
export declare const toZeroBasedLine: (oneBasedLine: number) => number;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert a 1-based source line number to the 0-based convention used by
|
|
3
|
+
* GraphNode `startLine`/`endLine`.
|
|
4
|
+
*
|
|
5
|
+
* The graph layer stores line numbers 0-based (tree-sitter `startPosition.row`),
|
|
6
|
+
* and this is load-bearing: the taint/PDG/CFG join and the MCP consumers all add
|
|
7
|
+
* `+ 1` to recover 1-based (see `summary-harvest-driver.ts` — "Function/Method
|
|
8
|
+
* node startLine is 0-based"). Most emitters get 0-based for free from
|
|
9
|
+
* tree-sitter. The exceptions are the regex-based COBOL/JCL processors (their
|
|
10
|
+
* parsers use `lineNum = i + 1`) and the scope-capture path (`Capture` ranges
|
|
11
|
+
* are 1-based per RFC §2.1). Those must convert to 0-based when they build a
|
|
12
|
+
* graph node, or the exact-content slice in `csv-generator.ts` drops the
|
|
13
|
+
* symbol's declaration line (#2379) and reported line numbers are off (#2377).
|
|
14
|
+
*
|
|
15
|
+
* Apply this ONLY at the graph-node `startLine:`/`endLine:` assignment. The
|
|
16
|
+
* parser-internal 1-based values (`.line`, `prog.startLine`) stay 1-based —
|
|
17
|
+
* they feed `L${line}` node/edge IDs and line-range containment checks that
|
|
18
|
+
* must not shift. The clamp guards degenerate inputs (line 0 / empty files).
|
|
19
|
+
*/
|
|
20
|
+
export const toZeroBasedLine = (oneBasedLine) => Math.max(0, oneBasedLine - 1);
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { NodeLabel } from '../../../_shared/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Graph-node labels that represent a resolvable code symbol — a definition with
|
|
4
|
+
* its own source span (function, type, member, module-like container).
|
|
5
|
+
*
|
|
6
|
+
* These get EXACT source-span content in the FTS index: `csv-generator.ts`
|
|
7
|
+
* slices exactly `[startLine, endLine]` for them (no ±2 padding), while every
|
|
8
|
+
* other label keeps the context window. That exactness depends on the 0-based
|
|
9
|
+
* `startLine`/`endLine` invariant enforced by `line-base.ts` — the slice is only
|
|
10
|
+
* correct because all emitters store 0-based lines. Keep the two together.
|
|
11
|
+
*
|
|
12
|
+
* Single source of truth so the set can't silently drift the way the inline copy
|
|
13
|
+
* did in #2379.
|
|
14
|
+
*
|
|
15
|
+
* NOTE: `group/extractors/manifest-extractor.ts`'s `CUSTOM_CONTRACT_RESOLVE_QUERY`
|
|
16
|
+
* carries a near-identical hand-list that is intentionally a SUBSET — it excludes
|
|
17
|
+
* `Namespace`, `Variable`, `Module`. Unifying the two needs a contract-resolution
|
|
18
|
+
* behavior check (would widen which nodes resolve as contract symbols), so it is
|
|
19
|
+
* deliberately left separate for now.
|
|
20
|
+
*/
|
|
21
|
+
export declare const SYMBOL_NODE_LABELS: ReadonlySet<NodeLabel>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Graph-node labels that represent a resolvable code symbol — a definition with
|
|
3
|
+
* its own source span (function, type, member, module-like container).
|
|
4
|
+
*
|
|
5
|
+
* These get EXACT source-span content in the FTS index: `csv-generator.ts`
|
|
6
|
+
* slices exactly `[startLine, endLine]` for them (no ±2 padding), while every
|
|
7
|
+
* other label keeps the context window. That exactness depends on the 0-based
|
|
8
|
+
* `startLine`/`endLine` invariant enforced by `line-base.ts` — the slice is only
|
|
9
|
+
* correct because all emitters store 0-based lines. Keep the two together.
|
|
10
|
+
*
|
|
11
|
+
* Single source of truth so the set can't silently drift the way the inline copy
|
|
12
|
+
* did in #2379.
|
|
13
|
+
*
|
|
14
|
+
* NOTE: `group/extractors/manifest-extractor.ts`'s `CUSTOM_CONTRACT_RESOLVE_QUERY`
|
|
15
|
+
* carries a near-identical hand-list that is intentionally a SUBSET — it excludes
|
|
16
|
+
* `Namespace`, `Variable`, `Module`. Unifying the two needs a contract-resolution
|
|
17
|
+
* behavior check (would widen which nodes resolve as contract symbols), so it is
|
|
18
|
+
* deliberately left separate for now.
|
|
19
|
+
*/
|
|
20
|
+
export const SYMBOL_NODE_LABELS = new Set([
|
|
21
|
+
'Function',
|
|
22
|
+
'Method',
|
|
23
|
+
'Class',
|
|
24
|
+
'Interface',
|
|
25
|
+
'CodeElement',
|
|
26
|
+
'Struct',
|
|
27
|
+
'Enum',
|
|
28
|
+
'Macro',
|
|
29
|
+
'Typedef',
|
|
30
|
+
'Union',
|
|
31
|
+
'Namespace',
|
|
32
|
+
'Trait',
|
|
33
|
+
'Impl',
|
|
34
|
+
'TypeAlias',
|
|
35
|
+
'Const',
|
|
36
|
+
'Static',
|
|
37
|
+
'Variable',
|
|
38
|
+
'Property',
|
|
39
|
+
'Record',
|
|
40
|
+
'Delegate',
|
|
41
|
+
'Annotation',
|
|
42
|
+
'Constructor',
|
|
43
|
+
'Template',
|
|
44
|
+
'Module',
|
|
45
|
+
]);
|
|
@@ -17,6 +17,7 @@ import path from 'path';
|
|
|
17
17
|
import { NODE_TABLES } from './schema.js';
|
|
18
18
|
import { RelPairRouter } from './rel-pair-routing.js';
|
|
19
19
|
import { parseTruthyEnv } from '../ingestion/utils/env.js';
|
|
20
|
+
import { SYMBOL_NODE_LABELS } from '../ingestion/utils/symbol-labels.js';
|
|
20
21
|
import { applyCjkSegmentationIfEnabled } from '../search/cjk-segmentation.js';
|
|
21
22
|
/**
|
|
22
23
|
* Deterministic output ordering — optional (out-of-core / windowed-resolve
|
|
@@ -187,6 +188,10 @@ class FileContentCache {
|
|
|
187
188
|
export const normalizeFtsText = (text) => text.replace(/[\r\n\t]+/g, ' ');
|
|
188
189
|
/** Composes both FTS-text transforms for the `description` column — one place for the six emission sites below to call, instead of repeating the composition. */
|
|
189
190
|
const formatFtsDescription = (description) => normalizeFtsText(applyCjkSegmentationIfEnabled(description));
|
|
191
|
+
// Labels that get exact source-span content (no ±2 window). Single source of
|
|
192
|
+
// truth in `symbol-labels.ts` — see there for why the exactness depends on the
|
|
193
|
+
// 0-based line invariant. Kept as a named alias to read intent at the use site.
|
|
194
|
+
const EXACT_SYMBOL_CONTENT_LABELS = SYMBOL_NODE_LABELS;
|
|
190
195
|
const extractContent = async (node, contentCache) => {
|
|
191
196
|
const filePath = node.properties.filePath;
|
|
192
197
|
const content = await contentCache.get(filePath);
|
|
@@ -209,8 +214,9 @@ const extractContent = async (node, contentCache) => {
|
|
|
209
214
|
if (startLine === undefined || endLine === undefined)
|
|
210
215
|
return '';
|
|
211
216
|
const lines = content.split('\n');
|
|
212
|
-
const
|
|
213
|
-
const
|
|
217
|
+
const exactSymbolContent = EXACT_SYMBOL_CONTENT_LABELS.has(node.label);
|
|
218
|
+
const start = Math.max(0, exactSymbolContent ? startLine : startLine - 2);
|
|
219
|
+
const end = Math.min(lines.length - 1, exactSymbolContent ? endLine : endLine + 2);
|
|
214
220
|
const snippet = lines.slice(start, end + 1).join('\n');
|
|
215
221
|
const MAX_SNIPPET = 5000;
|
|
216
222
|
const capped = snippet.length > MAX_SNIPPET ? snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]' : snippet;
|
|
@@ -29,7 +29,7 @@ export interface ExtensionEnsureOptions {
|
|
|
29
29
|
export interface ExtensionManagerOptions {
|
|
30
30
|
policy?: ExtensionInstallPolicy;
|
|
31
31
|
installTimeoutMs?: number;
|
|
32
|
-
installExtension?: (extensionName: string, timeoutMs: number) => Promise<ExtensionInstallResult>;
|
|
32
|
+
installExtension?: (extensionName: string, timeoutMs: number, loadError?: string) => Promise<ExtensionInstallResult>;
|
|
33
33
|
warn?: (message: string) => void;
|
|
34
34
|
}
|
|
35
35
|
export declare const getExtensionInstallPolicy: () => ExtensionInstallPolicy;
|
|
@@ -58,7 +58,7 @@ export declare const getExtensionInstallChildProcessArgs: (extensionName: string
|
|
|
58
58
|
* If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
|
|
59
59
|
* resolves with `timedOut: true`.
|
|
60
60
|
*/
|
|
61
|
-
export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number) => Promise<ExtensionInstallResult>;
|
|
61
|
+
export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number, loadError?: string) => Promise<ExtensionInstallResult>;
|
|
62
62
|
/**
|
|
63
63
|
* Centralized lifecycle manager for optional LadybugDB extensions.
|
|
64
64
|
*
|
|
@@ -90,6 +90,14 @@ export declare class ExtensionManager {
|
|
|
90
90
|
* paths are expected to degrade gracefully.
|
|
91
91
|
*/
|
|
92
92
|
ensure(query: (sql: string) => Promise<unknown>, name: string, label: string, opts?: ExtensionEnsureOptions): Promise<boolean>;
|
|
93
|
+
/**
|
|
94
|
+
* Attempt `LOAD EXTENSION <name>`; returns `null` on success and the
|
|
95
|
+
* collapsed error message on failure. The message is the load-side ground
|
|
96
|
+
* truth — LadybugDB distinguishes a missing extension file from a present
|
|
97
|
+
* but unloadable one (wrong platform, truncated download, version mismatch),
|
|
98
|
+
* and discarding it left users staring at "not pre-installed" when the file
|
|
99
|
+
* existed all along (#2374).
|
|
100
|
+
*/
|
|
93
101
|
private tryLoad;
|
|
94
102
|
private markLoaded;
|
|
95
103
|
private markUnavailable;
|
|
@@ -7,6 +7,8 @@ const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
|
7
7
|
const alreadyAvailable = (message) => message.includes('already loaded') ||
|
|
8
8
|
message.includes('already installed') ||
|
|
9
9
|
message.includes('already exists');
|
|
10
|
+
/** LadybugDB errors are multi-line; collapse for single-line warn/reason strings. */
|
|
11
|
+
const oneLine = (value) => value.replace(/\s+/g, ' ').trim();
|
|
10
12
|
const resolvePolicyFromEnv = () => {
|
|
11
13
|
const raw = process.env.GITNEXUS_LBUG_EXTENSION_INSTALL;
|
|
12
14
|
if (raw === 'load-only' || raw === 'never' || raw === 'auto')
|
|
@@ -51,7 +53,7 @@ export const getExtensionInstallChildProcessArgs = (extensionName, maxDbSize = L
|
|
|
51
53
|
* If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
|
|
52
54
|
* resolves with `timedOut: true`.
|
|
53
55
|
*/
|
|
54
|
-
export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutMs = getExtensionInstallTimeoutMs()) => {
|
|
56
|
+
export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutMs = getExtensionInstallTimeoutMs(), loadError) => {
|
|
55
57
|
if (!EXTENSION_NAME_PATTERN.test(extensionName)) {
|
|
56
58
|
throw new Error(`Invalid DuckDB extension name: ${extensionName}`);
|
|
57
59
|
}
|
|
@@ -60,6 +62,9 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
|
|
|
60
62
|
env: {
|
|
61
63
|
...process.env,
|
|
62
64
|
GITNEXUS_LBUG_EXTENSION_NAME: extensionName,
|
|
65
|
+
// The child picks INSTALL vs FORCE INSTALL from this LOAD error so it
|
|
66
|
+
// only re-downloads when the on-disk extension file is actually broken.
|
|
67
|
+
...(loadError ? { GITNEXUS_LBUG_EXTENSION_LOAD_ERROR: loadError } : {}),
|
|
63
68
|
},
|
|
64
69
|
stdio: ['ignore', 'ignore', 'pipe'],
|
|
65
70
|
windowsHide: true,
|
|
@@ -78,7 +83,7 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
|
|
|
78
83
|
resolve({
|
|
79
84
|
success: false,
|
|
80
85
|
timedOut: true,
|
|
81
|
-
message: `
|
|
86
|
+
message: `extension install for ${extensionName} timed out after ${timeoutMs}ms`,
|
|
82
87
|
});
|
|
83
88
|
}, timeoutMs);
|
|
84
89
|
child.on('error', (err) => {
|
|
@@ -97,8 +102,8 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
|
|
|
97
102
|
success: code === 0,
|
|
98
103
|
timedOut: false,
|
|
99
104
|
message: code === 0
|
|
100
|
-
? `
|
|
101
|
-
: `
|
|
105
|
+
? `extension install for ${extensionName} completed`
|
|
106
|
+
: `extension install for ${extensionName} failed with ${signal ?? `exit code ${code}`}${stderr ? `: ${stderr.trim()}` : ''}`,
|
|
102
107
|
});
|
|
103
108
|
});
|
|
104
109
|
});
|
|
@@ -152,39 +157,51 @@ export class ExtensionManager {
|
|
|
152
157
|
this.markUnavailable(name, label, 'extension install policy is "never"', warn);
|
|
153
158
|
return false;
|
|
154
159
|
}
|
|
155
|
-
|
|
160
|
+
const loadError = await this.tryLoad(query, name);
|
|
161
|
+
if (loadError === null) {
|
|
156
162
|
this.markLoaded(name);
|
|
157
163
|
return true;
|
|
158
164
|
}
|
|
159
165
|
if (policy === 'load-only') {
|
|
160
|
-
this.markUnavailable(name, label,
|
|
166
|
+
this.markUnavailable(name, label, `load-only policy (no install attempted); LOAD ${name} failed: ${loadError}`, warn);
|
|
161
167
|
return false;
|
|
162
168
|
}
|
|
163
169
|
let install = this.installAttempted.get(name);
|
|
164
170
|
if (!install) {
|
|
165
171
|
const installFn = this.options.installExtension ?? installDuckDbExtensionOutOfProcess;
|
|
166
|
-
|
|
172
|
+
// Hand the child the LOAD error so it re-downloads (FORCE) only when the
|
|
173
|
+
// present extension file is provably broken, not on every LOAD failure.
|
|
174
|
+
install = await installFn(name, timeoutMs, loadError);
|
|
167
175
|
this.installAttempted.set(name, install);
|
|
168
176
|
}
|
|
169
177
|
if (!install.success) {
|
|
170
|
-
this.markUnavailable(name, label, install.message
|
|
178
|
+
this.markUnavailable(name, label, `${install.message}; LOAD ${name} had failed: ${loadError}`, warn);
|
|
171
179
|
return false;
|
|
172
180
|
}
|
|
173
|
-
|
|
181
|
+
const retryError = await this.tryLoad(query, name);
|
|
182
|
+
if (retryError === null) {
|
|
174
183
|
this.markLoaded(name);
|
|
175
184
|
return true;
|
|
176
185
|
}
|
|
177
|
-
this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL`, warn);
|
|
186
|
+
this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL: ${retryError}`, warn);
|
|
178
187
|
return false;
|
|
179
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Attempt `LOAD EXTENSION <name>`; returns `null` on success and the
|
|
191
|
+
* collapsed error message on failure. The message is the load-side ground
|
|
192
|
+
* truth — LadybugDB distinguishes a missing extension file from a present
|
|
193
|
+
* but unloadable one (wrong platform, truncated download, version mismatch),
|
|
194
|
+
* and discarding it left users staring at "not pre-installed" when the file
|
|
195
|
+
* existed all along (#2374).
|
|
196
|
+
*/
|
|
180
197
|
async tryLoad(query, name) {
|
|
181
198
|
try {
|
|
182
199
|
await query(`LOAD EXTENSION ${name}`);
|
|
183
|
-
return
|
|
200
|
+
return null;
|
|
184
201
|
}
|
|
185
202
|
catch (err) {
|
|
186
203
|
const msg = err instanceof Error ? err.message : String(err);
|
|
187
|
-
return alreadyAvailable(msg);
|
|
204
|
+
return alreadyAvailable(msg) ? null : oneLine(msg);
|
|
188
205
|
}
|
|
189
206
|
}
|
|
190
207
|
markLoaded(name) {
|
|
@@ -4,3 +4,25 @@ export interface NativeCheckResult {
|
|
|
4
4
|
message?: string;
|
|
5
5
|
}
|
|
6
6
|
export declare function checkLbugNative(overridePkgDir?: string): NativeCheckResult;
|
|
7
|
+
export interface FtsProbeResult {
|
|
8
|
+
loaded: boolean;
|
|
9
|
+
/** Collapsed LadybugDB error when `loaded` is false. */
|
|
10
|
+
reason?: string;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Live-probe `LOAD EXTENSION fts` on a throwaway in-memory database.
|
|
14
|
+
*
|
|
15
|
+
* `doctor` used to print the static platform capability, which contradicted
|
|
16
|
+
* analyze whenever the extension file was missing or unloadable (#2374).
|
|
17
|
+
* LOAD never touches the network, so the probe is safe offline, and it
|
|
18
|
+
* surfaces LadybugDB's real error — which distinguishes a missing extension
|
|
19
|
+
* file from a present-but-broken one (wrong platform, truncated download).
|
|
20
|
+
* Dynamic import so doctor still runs when the native module itself is broken.
|
|
21
|
+
*
|
|
22
|
+
* Bounded by `timeoutMs`: an unresponsive extension file (e.g. on a hung
|
|
23
|
+
* network home dir) must never freeze `doctor` — the tool the degradation
|
|
24
|
+
* warnings send users to. `Promise.race` lets doctor report and move on; it
|
|
25
|
+
* cannot cancel an in-flight native call, so a future thread-blocking case
|
|
26
|
+
* would need an out-of-process probe.
|
|
27
|
+
*/
|
|
28
|
+
export declare function probeFtsExtensionLoad(timeoutMs?: number): Promise<FtsProbeResult>;
|