@cleocode/nexus 2026.10.12 → 2026.10.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/pipeline/call-processor.d.ts +9 -5
- package/dist/src/pipeline/call-processor.d.ts.map +1 -1
- package/dist/src/pipeline/call-processor.js +76 -78
- package/dist/src/pipeline/call-processor.js.map +1 -1
- package/dist/src/pipeline/index.d.ts +8 -3
- package/dist/src/pipeline/index.d.ts.map +1 -1
- package/dist/src/pipeline/index.js +50 -39
- package/dist/src/pipeline/index.js.map +1 -1
- package/dist/src/pipeline/knowledge-graph.d.ts +14 -2
- package/dist/src/pipeline/knowledge-graph.d.ts.map +1 -1
- package/dist/src/pipeline/knowledge-graph.js +79 -45
- package/dist/src/pipeline/knowledge-graph.js.map +1 -1
- package/dist/src/pipeline/parse-cache.d.ts +72 -1
- package/dist/src/pipeline/parse-cache.d.ts.map +1 -1
- package/dist/src/pipeline/parse-cache.js +138 -28
- package/dist/src/pipeline/parse-cache.js.map +1 -1
- package/dist/src/pipeline/parse-loop.d.ts +19 -13
- package/dist/src/pipeline/parse-loop.d.ts.map +1 -1
- package/dist/src/pipeline/parse-loop.js +50 -55
- package/dist/src/pipeline/parse-loop.js.map +1 -1
- package/dist/src/pipeline/processors/access-processor.d.ts +7 -2
- package/dist/src/pipeline/processors/access-processor.d.ts.map +1 -1
- package/dist/src/pipeline/processors/access-processor.js +41 -14
- package/dist/src/pipeline/processors/access-processor.js.map +1 -1
- package/dist/src/pipeline/reference-spool.d.ts +43 -0
- package/dist/src/pipeline/reference-spool.d.ts.map +1 -0
- package/dist/src/pipeline/reference-spool.js +69 -0
- package/dist/src/pipeline/reference-spool.js.map +1 -0
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +6 -2
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reference spool — encodes retained references into their stored form as
|
|
3
|
+
* resolution produces them (T13656).
|
|
4
|
+
*
|
|
5
|
+
* A generation retains one reference per unresolved or unmodeled static site:
|
|
6
|
+
* 951 146 of them on a 7 000-file repository, ~270 MiB of heap as objects.
|
|
7
|
+
* Holding them until publication made peak heap grow with the repository, so
|
|
8
|
+
* each reference is serialized the moment it is reported and only its
|
|
9
|
+
* compressed bytes (outside the V8 heap) and a count are kept.
|
|
10
|
+
*
|
|
11
|
+
* The bytes are exactly the `graph_assessment_references` value: the JSON
|
|
12
|
+
* array `[\n<ref>,\n<ref>\n]`, one reference per line, written as concatenated
|
|
13
|
+
* gzip members so no string longer than one member's text is ever built
|
|
14
|
+
* (T13326). The publisher stores them verbatim.
|
|
15
|
+
*
|
|
16
|
+
* @task T13656
|
|
17
|
+
* @module pipeline/reference-spool
|
|
18
|
+
*/
|
|
19
|
+
import { gzipSync } from 'node:zlib';
|
|
20
|
+
/** gzip level of the stored reference list: level 1 is ~16x smaller at ~1 s per 450 MB. */
|
|
21
|
+
const REFERENCES_GZIP_LEVEL = 1;
|
|
22
|
+
/**
|
|
23
|
+
* References compressed per gzip member of the stored list.
|
|
24
|
+
*
|
|
25
|
+
* Bounds the largest string the encoder builds to one member's worth (a few
|
|
26
|
+
* MB) whatever the repository size, while keeping members large enough that
|
|
27
|
+
* per-member gzip framing does not cost compression.
|
|
28
|
+
*/
|
|
29
|
+
const REFERENCES_PER_MEMBER = 4096;
|
|
30
|
+
/**
|
|
31
|
+
* Create a reference spool.
|
|
32
|
+
*
|
|
33
|
+
* @param publicationGeneration - When given, a reference carrying a different
|
|
34
|
+
* publication generation is refused, as the publisher refuses mixed generations.
|
|
35
|
+
* @returns An empty spool.
|
|
36
|
+
* @example
|
|
37
|
+
* ```ts
|
|
38
|
+
* const spool = createReferenceSpool(generation);
|
|
39
|
+
* spool.append(report);
|
|
40
|
+
* const stored = spool.finish(); // { count: 1, payload }
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
43
|
+
export function createReferenceSpool(publicationGeneration) {
|
|
44
|
+
const members = [];
|
|
45
|
+
let text = '[';
|
|
46
|
+
let count = 0;
|
|
47
|
+
return {
|
|
48
|
+
get count() {
|
|
49
|
+
return count;
|
|
50
|
+
},
|
|
51
|
+
append(reference) {
|
|
52
|
+
if (publicationGeneration !== undefined &&
|
|
53
|
+
reference.publicationGeneration !== undefined &&
|
|
54
|
+
reference.publicationGeneration !== publicationGeneration)
|
|
55
|
+
throw new Error('Staged reference publication generation does not match rows');
|
|
56
|
+
text += `${count === 0 ? '\n' : ',\n'}${JSON.stringify(reference)}`;
|
|
57
|
+
if (++count % REFERENCES_PER_MEMBER === 0) {
|
|
58
|
+
members.push(gzipSync(text, { level: REFERENCES_GZIP_LEVEL }));
|
|
59
|
+
text = '';
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
finish() {
|
|
63
|
+
members.push(gzipSync(`${text}\n]`, { level: REFERENCES_GZIP_LEVEL }));
|
|
64
|
+
text = '';
|
|
65
|
+
return { count, payload: Buffer.concat(members) };
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=reference-spool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reference-spool.js","sourceRoot":"","sources":["../../../src/pipeline/reference-spool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,2FAA2F;AAC3F,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEhC;;;;;;GAMG;AACH,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAYnC;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,oBAAoB,CAAC,qBAA8B;IACjE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,IAAI,GAAG,GAAG,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO;QACL,IAAI,KAAK;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,CAAC,SAAS;YACd,IACE,qBAAqB,KAAK,SAAS;gBACnC,SAAS,CAAC,qBAAqB,KAAK,SAAS;gBAC7C,SAAS,CAAC,qBAAqB,KAAK,qBAAqB;gBAEzD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,IAAI,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;YACpE,IAAI,EAAE,KAAK,GAAG,qBAAqB,KAAK,CAAC,EAAE,CAAC;gBAC1C,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAAC;gBAC/D,IAAI,GAAG,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,MAAM;YACJ,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,KAAK,EAAE,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,GAAG,EAAE,CAAC;YACV,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,CAAC;KACF,CAAC;AACJ,CAAC"}
|