gitnexus 1.6.9-rc.34 → 1.6.9-rc.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Streams CSV rows directly to disk files in a single pass over graph nodes.
|
|
5
5
|
* File contents are lazy-read from disk per-node to avoid holding the entire
|
|
6
|
-
* repo in RAM. Rows are buffered (
|
|
6
|
+
* repo in RAM. Rows are buffered (FLUSH_BYTES) before writing to minimize
|
|
7
7
|
* per-row Promise overhead.
|
|
8
8
|
*
|
|
9
9
|
* RFC 4180 Compliant:
|
|
@@ -14,6 +14,32 @@
|
|
|
14
14
|
import type { GraphNode, GraphRelationship } from '../../_shared/index.js';
|
|
15
15
|
import { KnowledgeGraph } from '../graph/types.js';
|
|
16
16
|
import { NodeTableName } from './schema.js';
|
|
17
|
+
/**
|
|
18
|
+
* Flush buffered rows to disk once the buffered chunk reaches this many bytes.
|
|
19
|
+
* Byte-bounded rather than row-count-bounded: row size ranges from a few dozen
|
|
20
|
+
* bytes (typical symbol/relationship rows) up to a full File's content
|
|
21
|
+
* (#2317/#2323), so a row-count-only cap lets a handful of huge rows build an
|
|
22
|
+
* unbounded `buffer.join('\n')` string before ever tripping it.
|
|
23
|
+
*
|
|
24
|
+
* Not an env knob — fixed by a safety margin, not a preference. The one worst
|
|
25
|
+
* case that matters: one more oversized row lands right after the buffer was
|
|
26
|
+
* just under this threshold, before the flush fires. That row is capped at
|
|
27
|
+
* TREE_SITTER_MAX_BUFFER (32MB, hard-clamped — GITNEXUS_MAX_FILE_SIZE cannot
|
|
28
|
+
* raise it), and escapeCSVField's quote-doubling can at most double it. So the
|
|
29
|
+
* peak joined-string size is bounded by
|
|
30
|
+
* FLUSH_BYTES + 2 * TREE_SITTER_MAX_BUFFER ≈ 8MB + 64MB = 72MB,
|
|
31
|
+
* versus Node's `buffer.constants.MAX_STRING_LENGTH` (~512MB) that throws
|
|
32
|
+
* `RangeError: Invalid string length` past it — a >7x margin (see the
|
|
33
|
+
* `shouldFlushCSVBuffer stays within the V8 string-length ceiling` test,
|
|
34
|
+
* which fails loudly if either constant ever moves this margin the wrong
|
|
35
|
+
* way). Raising FLUSH_BYTES trades fewer/larger flushes for less margin;
|
|
36
|
+
* lowering it trades the reverse for lower peak transient memory. Change the
|
|
37
|
+
* constant directly if a real workload needs a different point on that
|
|
38
|
+
* curve — a per-host env var would let the margin get silently reintroduced
|
|
39
|
+
* by an operator with no way to know why 512MB is dangerous.
|
|
40
|
+
*/
|
|
41
|
+
export declare const FLUSH_BYTES: number;
|
|
42
|
+
export declare const shouldFlushCSVBuffer: (byteCount: number) => boolean;
|
|
17
43
|
export declare const sanitizeUTF8: (str: string) => string;
|
|
18
44
|
export declare const escapeCSVField: (value: string | number | undefined | null) => string;
|
|
19
45
|
export declare const escapeCSVNumber: (value: number | undefined | null, defaultValue?: number) => string;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Streams CSV rows directly to disk files in a single pass over graph nodes.
|
|
5
5
|
* File contents are lazy-read from disk per-node to avoid holding the entire
|
|
6
|
-
* repo in RAM. Rows are buffered (
|
|
6
|
+
* repo in RAM. Rows are buffered (FLUSH_BYTES) before writing to minimize
|
|
7
7
|
* per-row Promise overhead.
|
|
8
8
|
*
|
|
9
9
|
* RFC 4180 Compliant:
|
|
@@ -31,8 +31,32 @@ import { parseTruthyEnv } from '../ingestion/utils/env.js';
|
|
|
31
31
|
const byGraphId = (a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
32
32
|
const orderedNodes = (graph, sorted) => sorted ? [...graph.iterNodes()].sort(byGraphId) : graph.iterNodes();
|
|
33
33
|
const orderedRelationships = (graph, sorted) => sorted ? [...graph.iterRelationships()].sort(byGraphId) : graph.iterRelationships();
|
|
34
|
-
/**
|
|
35
|
-
|
|
34
|
+
/**
|
|
35
|
+
* Flush buffered rows to disk once the buffered chunk reaches this many bytes.
|
|
36
|
+
* Byte-bounded rather than row-count-bounded: row size ranges from a few dozen
|
|
37
|
+
* bytes (typical symbol/relationship rows) up to a full File's content
|
|
38
|
+
* (#2317/#2323), so a row-count-only cap lets a handful of huge rows build an
|
|
39
|
+
* unbounded `buffer.join('\n')` string before ever tripping it.
|
|
40
|
+
*
|
|
41
|
+
* Not an env knob — fixed by a safety margin, not a preference. The one worst
|
|
42
|
+
* case that matters: one more oversized row lands right after the buffer was
|
|
43
|
+
* just under this threshold, before the flush fires. That row is capped at
|
|
44
|
+
* TREE_SITTER_MAX_BUFFER (32MB, hard-clamped — GITNEXUS_MAX_FILE_SIZE cannot
|
|
45
|
+
* raise it), and escapeCSVField's quote-doubling can at most double it. So the
|
|
46
|
+
* peak joined-string size is bounded by
|
|
47
|
+
* FLUSH_BYTES + 2 * TREE_SITTER_MAX_BUFFER ≈ 8MB + 64MB = 72MB,
|
|
48
|
+
* versus Node's `buffer.constants.MAX_STRING_LENGTH` (~512MB) that throws
|
|
49
|
+
* `RangeError: Invalid string length` past it — a >7x margin (see the
|
|
50
|
+
* `shouldFlushCSVBuffer stays within the V8 string-length ceiling` test,
|
|
51
|
+
* which fails loudly if either constant ever moves this margin the wrong
|
|
52
|
+
* way). Raising FLUSH_BYTES trades fewer/larger flushes for less margin;
|
|
53
|
+
* lowering it trades the reverse for lower peak transient memory. Change the
|
|
54
|
+
* constant directly if a real workload needs a different point on that
|
|
55
|
+
* curve — a per-host env var would let the margin get silently reintroduced
|
|
56
|
+
* by an operator with no way to know why 512MB is dangerous.
|
|
57
|
+
*/
|
|
58
|
+
export const FLUSH_BYTES = 8 * 1024 * 1024;
|
|
59
|
+
export const shouldFlushCSVBuffer = (byteCount) => byteCount >= FLUSH_BYTES;
|
|
36
60
|
/**
|
|
37
61
|
* Yield the event loop every N relationship rows during the emit pass (#2226 F4)
|
|
38
62
|
* so a concurrent node COPY (the overlap in loadGraphToLbug) and write-stream
|
|
@@ -127,6 +151,23 @@ class FileContentCache {
|
|
|
127
151
|
this.accessOrder.push(key);
|
|
128
152
|
}
|
|
129
153
|
}
|
|
154
|
+
/**
|
|
155
|
+
* Flatten newlines and tabs to single spaces for FTS-indexed text columns
|
|
156
|
+
* (`content`, `description`) — the real fix for #2317.
|
|
157
|
+
*
|
|
158
|
+
* Ladybug's full-text-search tokenizer splits ONLY on the space character —
|
|
159
|
+
* `\n`, `\r`, and `\t` are NOT token delimiters. So multiline text indexes as
|
|
160
|
+
* a handful of giant tokens (each whole line, joined across lines), and a
|
|
161
|
+
* word query matches none of them: `searchFTSFromLbug('foo')` misses a file
|
|
162
|
+
* whose content is `... \nfoo\n ...`. Removing the 10KB cap (#2333/#2317)
|
|
163
|
+
* stores the full body but leaves it unsearchable; collapsing intra-text
|
|
164
|
+
* whitespace to spaces is what actually makes every word searchable.
|
|
165
|
+
*
|
|
166
|
+
* This rewrites the STORED column too (the same value is COPYed in), so File
|
|
167
|
+
* content returned via the graph API is space-flattened — an accepted trade
|
|
168
|
+
* for making file/symbol text searchable. Leading/trailing/empty are no-ops.
|
|
169
|
+
*/
|
|
170
|
+
const normalizeFtsText = (text) => text.replace(/[\r\n\t]+/g, ' ');
|
|
130
171
|
const extractContent = async (node, contentCache) => {
|
|
131
172
|
const filePath = node.properties.filePath;
|
|
132
173
|
const content = await contentCache.get(filePath);
|
|
@@ -136,11 +177,13 @@ const extractContent = async (node, contentCache) => {
|
|
|
136
177
|
return '';
|
|
137
178
|
if (isBinaryContent(content))
|
|
138
179
|
return '[Binary file - content not stored]';
|
|
180
|
+
// File content is stored in full — intentionally NOT length-capped here, so
|
|
181
|
+
// text past the old 10KB cutoff stays FTS-searchable (#2317). It is already
|
|
182
|
+
// bounded upstream by the walker's max-file-size cap (512KB default / 32MB),
|
|
183
|
+
// and only whitespace-normalized for the tokenizer. The symbol snippet path
|
|
184
|
+
// below, by contrast, deliberately stays capped at MAX_SNIPPET.
|
|
139
185
|
if (node.label === 'File') {
|
|
140
|
-
|
|
141
|
-
return content.length > MAX_FILE_CONTENT
|
|
142
|
-
? content.slice(0, MAX_FILE_CONTENT) + '\n... [truncated]'
|
|
143
|
-
: content;
|
|
186
|
+
return normalizeFtsText(content);
|
|
144
187
|
}
|
|
145
188
|
const startLine = node.properties.startLine;
|
|
146
189
|
const endLine = node.properties.endLine;
|
|
@@ -151,9 +194,8 @@ const extractContent = async (node, contentCache) => {
|
|
|
151
194
|
const end = Math.min(lines.length - 1, endLine + 2);
|
|
152
195
|
const snippet = lines.slice(start, end + 1).join('\n');
|
|
153
196
|
const MAX_SNIPPET = 5000;
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
: snippet;
|
|
197
|
+
const capped = snippet.length > MAX_SNIPPET ? snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]' : snippet;
|
|
198
|
+
return normalizeFtsText(capped);
|
|
157
199
|
};
|
|
158
200
|
// ============================================================================
|
|
159
201
|
// BUFFERED CSV WRITER
|
|
@@ -161,15 +203,17 @@ const extractContent = async (node, contentCache) => {
|
|
|
161
203
|
class BufferedCSVWriter {
|
|
162
204
|
ws;
|
|
163
205
|
buffer = [];
|
|
206
|
+
bufferedBytes = 0;
|
|
164
207
|
rows = 0;
|
|
165
208
|
constructor(filePath, header) {
|
|
166
209
|
this.ws = createWriteStream(filePath, 'utf-8');
|
|
167
210
|
// Large repos flush many times — raise listener cap to avoid MaxListenersExceededWarning
|
|
168
211
|
this.ws.setMaxListeners(50);
|
|
169
212
|
this.buffer.push(header);
|
|
213
|
+
this.bufferedBytes = Buffer.byteLength(header) + 1;
|
|
170
214
|
}
|
|
171
215
|
/**
|
|
172
|
-
* Buffer a row. Returns a promise ONLY when the buffer crossed
|
|
216
|
+
* Buffer a row. Returns a promise ONLY when the buffer crossed FLUSH_BYTES
|
|
173
217
|
* and a disk write was issued; otherwise returns `undefined` so the caller
|
|
174
218
|
* can skip awaiting (#2203 U3) — avoiding a microtask tick on every buffered
|
|
175
219
|
* row (millions at scale). The flush promise still resolves on drain, so
|
|
@@ -177,8 +221,9 @@ class BufferedCSVWriter {
|
|
|
177
221
|
*/
|
|
178
222
|
addRow(row) {
|
|
179
223
|
this.buffer.push(row);
|
|
224
|
+
this.bufferedBytes += Buffer.byteLength(row) + 1;
|
|
180
225
|
this.rows++;
|
|
181
|
-
if (this.
|
|
226
|
+
if (shouldFlushCSVBuffer(this.bufferedBytes)) {
|
|
182
227
|
return this.flush();
|
|
183
228
|
}
|
|
184
229
|
return undefined;
|
|
@@ -188,6 +233,7 @@ class BufferedCSVWriter {
|
|
|
188
233
|
return Promise.resolve();
|
|
189
234
|
const chunk = this.buffer.join('\n') + '\n';
|
|
190
235
|
this.buffer.length = 0;
|
|
236
|
+
this.bufferedBytes = 0;
|
|
191
237
|
return new Promise((resolve, reject) => {
|
|
192
238
|
this.ws.once('error', reject);
|
|
193
239
|
const ok = this.ws.write(chunk);
|
|
@@ -355,7 +401,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir, onNodePhaseCo
|
|
|
355
401
|
seenNodeIds.add(node.id);
|
|
356
402
|
// addRow returns a promise only when it flushes; awaiting it once after the
|
|
357
403
|
// switch (instead of `await`-ing every addRow) skips a per-row microtask
|
|
358
|
-
// tick on the
|
|
404
|
+
// tick on the rows buffered between byte-bounded flushes (#2203 U3).
|
|
359
405
|
let pending;
|
|
360
406
|
switch (node.label) {
|
|
361
407
|
case 'File': {
|
|
@@ -383,7 +429,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir, onNodePhaseCo
|
|
|
383
429
|
escapeCSVField(node.properties.name || ''),
|
|
384
430
|
escapeCSVField(node.properties.heuristicLabel || ''),
|
|
385
431
|
keywordsStr,
|
|
386
|
-
escapeCSVField(node.properties.description || ''),
|
|
432
|
+
escapeCSVField(normalizeFtsText(node.properties.description || '')),
|
|
387
433
|
escapeCSVField(node.properties.enrichedBy || 'heuristic'),
|
|
388
434
|
escapeCSVNumber(node.properties.cohesion, 0),
|
|
389
435
|
escapeCSVNumber(node.properties.symbolCount, 0),
|
|
@@ -415,7 +461,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir, onNodePhaseCo
|
|
|
415
461
|
escapeCSVNumber(node.properties.endLine, -1),
|
|
416
462
|
node.properties.isExported ? 'true' : 'false',
|
|
417
463
|
escapeCSVField(content),
|
|
418
|
-
escapeCSVField(node.properties.description || ''),
|
|
464
|
+
escapeCSVField(normalizeFtsText(node.properties.description || '')),
|
|
419
465
|
escapeCSVNumber(node.properties.parameterCount, 0),
|
|
420
466
|
escapeCSVField(node.properties.returnType || ''),
|
|
421
467
|
].join(','));
|
|
@@ -431,7 +477,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir, onNodePhaseCo
|
|
|
431
477
|
escapeCSVNumber(node.properties.endLine, -1),
|
|
432
478
|
escapeCSVNumber(node.properties.level, 1),
|
|
433
479
|
escapeCSVField(content),
|
|
434
|
-
escapeCSVField(node.properties.description || ''),
|
|
480
|
+
escapeCSVField(normalizeFtsText(node.properties.description || '')),
|
|
435
481
|
].join(','));
|
|
436
482
|
break;
|
|
437
483
|
}
|
|
@@ -461,7 +507,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir, onNodePhaseCo
|
|
|
461
507
|
escapeCSVField(node.id),
|
|
462
508
|
escapeCSVField(node.properties.name || ''),
|
|
463
509
|
escapeCSVField(node.properties.filePath || ''),
|
|
464
|
-
escapeCSVField(node.properties.description || ''),
|
|
510
|
+
escapeCSVField(normalizeFtsText(node.properties.description || '')),
|
|
465
511
|
].join(','));
|
|
466
512
|
break;
|
|
467
513
|
case 'BasicBlock':
|
|
@@ -480,7 +526,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir, onNodePhaseCo
|
|
|
480
526
|
escapeCSVNumber(node.properties.endLine, -1),
|
|
481
527
|
node.properties.isExported ? 'true' : 'false',
|
|
482
528
|
escapeCSVField(content),
|
|
483
|
-
escapeCSVField(node.properties.description || ''),
|
|
529
|
+
escapeCSVField(normalizeFtsText(node.properties.description || '')),
|
|
484
530
|
].join(','));
|
|
485
531
|
}
|
|
486
532
|
else {
|
|
@@ -495,7 +541,7 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir, onNodePhaseCo
|
|
|
495
541
|
escapeCSVNumber(node.properties.startLine, -1),
|
|
496
542
|
escapeCSVNumber(node.properties.endLine, -1),
|
|
497
543
|
escapeCSVField(content),
|
|
498
|
-
escapeCSVField(node.properties.description || ''),
|
|
544
|
+
escapeCSVField(normalizeFtsText(node.properties.description || '')),
|
|
499
545
|
...(node.label === 'Property'
|
|
500
546
|
? [escapeCSVField(node.properties.declaredType || '')]
|
|
501
547
|
: []),
|
package/package.json
CHANGED