@untangle-code/chunker 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +90 -0
- package/README.md +206 -2
- package/THIRD_PARTY_LICENSES.md +24 -0
- package/dist/byte-text.d.ts +39 -0
- package/dist/byte-text.d.ts.map +1 -0
- package/dist/byte-text.js +59 -0
- package/dist/byte-text.js.map +1 -0
- package/dist/chunker.d.ts +11 -0
- package/dist/chunker.d.ts.map +1 -0
- package/dist/chunker.js +112 -0
- package/dist/chunker.js.map +1 -0
- package/dist/config.d.ts +17 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +28 -0
- package/dist/config.js.map +1 -0
- package/dist/context.d.ts +48 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +0 -0
- package/dist/context.js.map +1 -0
- package/dist/coverage.d.ts +44 -0
- package/dist/coverage.d.ts.map +1 -0
- package/dist/coverage.js +95 -0
- package/dist/coverage.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/minified-detection.d.ts +27 -0
- package/dist/minified-detection.d.ts.map +1 -0
- package/dist/minified-detection.js +46 -0
- package/dist/minified-detection.js.map +1 -0
- package/dist/prepared-text.d.ts +48 -0
- package/dist/prepared-text.d.ts.map +1 -0
- package/dist/prepared-text.js +57 -0
- package/dist/prepared-text.js.map +1 -0
- package/dist/split.d.ts +36 -0
- package/dist/split.d.ts.map +1 -0
- package/dist/split.js +126 -0
- package/dist/split.js.map +1 -0
- package/dist/strategies/ast-chunk-strategy.d.ts +52 -0
- package/dist/strategies/ast-chunk-strategy.d.ts.map +1 -0
- package/dist/strategies/ast-chunk-strategy.js +0 -0
- package/dist/strategies/ast-chunk-strategy.js.map +1 -0
- package/dist/strategies/line-fallback-strategy.d.ts +4 -0
- package/dist/strategies/line-fallback-strategy.d.ts.map +1 -0
- package/dist/strategies/line-fallback-strategy.js +79 -0
- package/dist/strategies/line-fallback-strategy.js.map +1 -0
- package/dist/strategies/minified-strategy.d.ts +4 -0
- package/dist/strategies/minified-strategy.d.ts.map +1 -0
- package/dist/strategies/minified-strategy.js +42 -0
- package/dist/strategies/minified-strategy.js.map +1 -0
- package/dist/token-estimator.d.ts +28 -0
- package/dist/token-estimator.d.ts.map +1 -0
- package/dist/token-estimator.js +37 -0
- package/dist/token-estimator.js.map +1 -0
- package/dist/types.d.ts +114 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +20 -0
- package/dist/types.js.map +1 -0
- package/package.json +61 -3
- package/src/byte-text.ts +85 -0
- package/src/chunker.ts +142 -0
- package/src/config.ts +32 -0
- package/src/context.ts +0 -0
- package/src/coverage.ts +143 -0
- package/src/index.ts +63 -0
- package/src/minified-detection.ts +66 -0
- package/src/prepared-text.ts +118 -0
- package/src/split.ts +164 -0
- package/src/strategies/ast-chunk-strategy.ts +0 -0
- package/src/strategies/line-fallback-strategy.ts +108 -0
- package/src/strategies/minified-strategy.ts +49 -0
- package/src/token-estimator.ts +48 -0
- package/src/types.ts +138 -0
package/dist/split.js
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { rangeForByteSpan, safeUtf8Boundary, sliceBytes } from "./byte-text.js";
|
|
2
|
+
import { contentOf, estimateTokensWithHeader, } from "./context.js";
|
|
3
|
+
function fits(ctx, range, header) {
|
|
4
|
+
return (estimateTokensWithHeader(ctx, contentOf(ctx, range), header) <= ctx.config.maxTokensPerChunk);
|
|
5
|
+
}
|
|
6
|
+
function merge(a, b) {
|
|
7
|
+
return { startLine: a.startLine, endLine: b.endLine, startByte: a.startByte, endByte: b.endByte };
|
|
8
|
+
}
|
|
9
|
+
/** Greedily merges adjacent (contiguous, ordered) atoms while staying under budget. */
|
|
10
|
+
export function groupAtomsByBudget(ctx, atoms, header) {
|
|
11
|
+
if (atoms.length === 0) {
|
|
12
|
+
return [];
|
|
13
|
+
}
|
|
14
|
+
const result = [];
|
|
15
|
+
let current = atoms[0];
|
|
16
|
+
for (let i = 1; i < atoms.length; i += 1) {
|
|
17
|
+
const next = atoms[i];
|
|
18
|
+
const merged = merge(current, next);
|
|
19
|
+
if (fits(ctx, merged, header)) {
|
|
20
|
+
current = merged;
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
result.push(current);
|
|
24
|
+
current = next;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
result.push(current);
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Recursively splits `node` into an ordered, gap-free list of ranges that
|
|
32
|
+
* together cover `node.range` exactly. A node that already fits the budget
|
|
33
|
+
* is returned whole. A node with no further AST structure that still does
|
|
34
|
+
* not fit is returned whole too — {@link splitByTokenBoundary} handles it.
|
|
35
|
+
*/
|
|
36
|
+
export function flattenToAtoms(ctx, node, header) {
|
|
37
|
+
if (fits(ctx, node.range, header)) {
|
|
38
|
+
return [node.range];
|
|
39
|
+
}
|
|
40
|
+
if (node.splittable && node.children && node.children.length > 0) {
|
|
41
|
+
const children = [...node.children].sort((a, b) => a.range.startByte - b.range.startByte);
|
|
42
|
+
const atoms = [];
|
|
43
|
+
let cursor = node.range.startByte;
|
|
44
|
+
for (const child of children) {
|
|
45
|
+
if (child.range.startByte > cursor) {
|
|
46
|
+
atoms.push(rangeForByteSpan(ctx.lineIndex, cursor, child.range.startByte));
|
|
47
|
+
}
|
|
48
|
+
atoms.push(...flattenToAtoms(ctx, child, header));
|
|
49
|
+
cursor = Math.max(cursor, child.range.endByte);
|
|
50
|
+
}
|
|
51
|
+
if (cursor < node.range.endByte) {
|
|
52
|
+
atoms.push(rangeForByteSpan(ctx.lineIndex, cursor, node.range.endByte));
|
|
53
|
+
}
|
|
54
|
+
return groupAtomsByBudget(ctx, atoms, header);
|
|
55
|
+
}
|
|
56
|
+
return [node.range];
|
|
57
|
+
}
|
|
58
|
+
/** How far back (bytes) to look for a newline to split on instead of a hard cut. */
|
|
59
|
+
const NEWLINE_LOOKBACK_BYTES = 120;
|
|
60
|
+
function findBudgetEnd(ctx, start, maxEnd, header) {
|
|
61
|
+
const wholeRange = rangeForByteSpan(ctx.lineIndex, start, maxEnd);
|
|
62
|
+
if (fits(ctx, wholeRange, header)) {
|
|
63
|
+
return maxEnd;
|
|
64
|
+
}
|
|
65
|
+
let lo = start + 1;
|
|
66
|
+
let hi = maxEnd;
|
|
67
|
+
// Guard: even one byte does not fit under an absurdly small budget — make
|
|
68
|
+
// forced minimal progress rather than looping forever.
|
|
69
|
+
if (!fits(ctx, rangeForByteSpan(ctx.lineIndex, start, lo), header)) {
|
|
70
|
+
return lo;
|
|
71
|
+
}
|
|
72
|
+
while (lo < hi) {
|
|
73
|
+
const mid = lo + Math.ceil((hi - lo) / 2);
|
|
74
|
+
if (fits(ctx, rangeForByteSpan(ctx.lineIndex, start, mid), header)) {
|
|
75
|
+
lo = mid;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
hi = mid - 1;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return lo;
|
|
82
|
+
}
|
|
83
|
+
function preferNewlineBoundary(buffer, start, end) {
|
|
84
|
+
const lookbackFloor = Math.max(start, end - NEWLINE_LOOKBACK_BYTES);
|
|
85
|
+
// Do not shrink the piece by more than half just for a nicer boundary.
|
|
86
|
+
const minAcceptable = Math.max(start + 1, start + Math.floor((end - start) / 2));
|
|
87
|
+
for (let i = end - 1; i >= lookbackFloor; i -= 1) {
|
|
88
|
+
if (buffer[i] === 0x0a && i + 1 >= minAcceptable) {
|
|
89
|
+
return i + 1;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return end;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Splits `range` into an ordered, gap-free, non-overlapping list of ranges,
|
|
96
|
+
* each within the token budget, at safe UTF-8 boundaries (architecture.md
|
|
97
|
+
* §14.4). Last-resort splitter: used only on atoms `flattenToAtoms` could
|
|
98
|
+
* not shrink through AST structure.
|
|
99
|
+
*/
|
|
100
|
+
export function splitByTokenBoundary(ctx, range, header) {
|
|
101
|
+
const pieces = [];
|
|
102
|
+
let cursor = range.startByte;
|
|
103
|
+
while (cursor < range.endByte) {
|
|
104
|
+
let end = findBudgetEnd(ctx, cursor, range.endByte, header);
|
|
105
|
+
if (end > cursor + 1) {
|
|
106
|
+
end = preferNewlineBoundary(ctx.lineIndex.buffer, cursor, end);
|
|
107
|
+
}
|
|
108
|
+
end = safeUtf8Boundary(ctx.lineIndex.buffer, end);
|
|
109
|
+
if (end <= cursor) {
|
|
110
|
+
// Never emit an empty piece or fail to progress: force one byte, then
|
|
111
|
+
// walk forward off any continuation byte it landed on.
|
|
112
|
+
end = cursor + 1;
|
|
113
|
+
while (end < range.endByte && (ctx.lineIndex.buffer[end] & 0b1100_0000) === 0b1000_0000) {
|
|
114
|
+
end += 1;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
pieces.push(rangeForByteSpan(ctx.lineIndex, cursor, end));
|
|
118
|
+
cursor = end;
|
|
119
|
+
}
|
|
120
|
+
return pieces;
|
|
121
|
+
}
|
|
122
|
+
/** Re-exported for strategies that need a raw byte slice without going through an `AstRegion`. */
|
|
123
|
+
export function sliceRange(ctx, range) {
|
|
124
|
+
return sliceBytes(ctx.lineIndex, range.startByte, range.endByte);
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=split.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"split.js","sourceRoot":"","sources":["../src/split.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,EAEL,SAAS,EACT,wBAAwB,GAEzB,MAAM,cAAc,CAAC;AAItB,SAAS,IAAI,CAAC,GAAiB,EAAE,KAAkB,EAAE,MAAc;IACjE,OAAO,CACL,wBAAwB,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAC7F,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,CAAc,EAAE,CAAc;IAC3C,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;AACpG,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,kBAAkB,CAChC,GAAiB,EACjB,KAA6B,EAC7B,MAAc;IAEd,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACpC,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,GAAiB,EAAE,IAAe,EAAE,MAAc;IAC/E,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjE,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;YAC7E,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;YAClD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,SAAS,aAAa,CAAC,GAAiB,EAAE,KAAa,EAAE,MAAc,EAAE,MAAc;IACrF,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,CAAC;IACnB,IAAI,EAAE,GAAG,MAAM,CAAC;IAChB,0EAA0E;IAC1E,uDAAuD;IACvD,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;QACnE,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,IAAI,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;YACnE,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAc,EAAE,KAAa,EAAE,GAAW;IACvE,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,sBAAsB,CAAC,CAAC;IACpE,uEAAuE;IACvE,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,KAAK,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC,IAAI,aAAa,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,aAAa,EAAE,CAAC;YACjD,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAClC,GAAiB,EACjB,KAAkB,EAClB,MAAc;IAEd,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;IAC7B,OAAO,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAC9B,IAAI,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,GAAG,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,GAAG,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QACjE,CAAC;QACD,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAClD,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;YAClB,sEAAsE;YACtE,uDAAuD;YACvD,GAAG,GAAG,MAAM,GAAG,CAAC,CAAC;YACjB,OAAO,GAAG,GAAG,KAAK,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAE,GAAG,WAAW,CAAC,KAAK,WAAW,EAAE,CAAC;gBACzF,GAAG,IAAI,CAAC,CAAC;YACX,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,UAAU,CAAC,GAAiB,EAAE,KAAkB;IAC9D,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;AACnE,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AST chunk strategy (spec.md §8.3 steps 1-8; architecture.md §14.2-§14.3).
|
|
3
|
+
*
|
|
4
|
+
* Consumes the `AstRegion` tree and `ExtractedSymbol` list a language adapter
|
|
5
|
+
* already produced (this package never looks inside `ParsedAst.root`), per
|
|
6
|
+
* the region contract documented on `AstRegion` in domain `analysis.ts`:
|
|
7
|
+
* adapters deliver ONE root region of kind "module" spanning the file, and
|
|
8
|
+
* this strategy dispatches on `chunkKind` at EVERY depth:
|
|
9
|
+
*
|
|
10
|
+
* - `module` regions are unpacked: each child is dispatched by kind, and
|
|
11
|
+
* interstitial module-level bytes no child covers (top-level statements,
|
|
12
|
+
* stray comments) become real `module` chunks — or "excluded" ledger
|
|
13
|
+
* entries when they are pure whitespace;
|
|
14
|
+
* - `class_overview` regions get a synthesized overview chunk (header + doc
|
|
15
|
+
* + member summary, no method bodies; `synthesized: true`, coverage
|
|
16
|
+
* claimed ONLY for byte spans whose text the overview content verifiably
|
|
17
|
+
* carries verbatim) plus one chunk per method/constructor, grouped chunks
|
|
18
|
+
* for small properties, and real content chunks for class-body bytes no
|
|
19
|
+
* member covers AND for header-span bytes the overview does not carry
|
|
20
|
+
* (comments, static blocks, modifiers a normalized signature drops —
|
|
21
|
+
* architecture.md §14.2; spec.md §8.4: a synthesized chunk must never
|
|
22
|
+
* mask a gap);
|
|
23
|
+
* - every other region (function/interface/type/enum/file_preamble/
|
|
24
|
+
* comment/configuration) is emitted whole if it fits the token budget, or
|
|
25
|
+
* split into `symbol_fragment`s otherwise (spec.md §8.5, architecture.md
|
|
26
|
+
* §14.3) — every fragment inherits the symbol's `symbolId`/`symbolFqn`.
|
|
27
|
+
*
|
|
28
|
+
* Fragment invariant (spec.md §8.5 MUSS): a `symbol_fragment` is only ever
|
|
29
|
+
* emitted WITH a `symbolId`. Wherever no symbol is resolvable (a region
|
|
30
|
+
* without an extracted symbol, a class whose fqn matches no symbol), the
|
|
31
|
+
* over-budget pieces degrade to file-scoped `fallback_lines` instead —
|
|
32
|
+
* still lossless — and `emitChunk` (context.ts) throws
|
|
33
|
+
* `UntangleError("ANALYSIS_INCOMPLETE")` on any violation rather than
|
|
34
|
+
* silently emitting an id-less fragment.
|
|
35
|
+
*/
|
|
36
|
+
import { type AstRegion, type CodeChunkDraft, type CoverageEntry, type ExtractedSymbol } from "@untangle-code/domain";
|
|
37
|
+
import { type BuildContext } from "../context.js";
|
|
38
|
+
export interface StrategyResult {
|
|
39
|
+
drafts: CodeChunkDraft[];
|
|
40
|
+
coverage: CoverageEntry[];
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Runs the AST chunk strategy over one file's regions — normally the single
|
|
44
|
+
* root "module" region an adapter returns (domain `AstRegion` contract),
|
|
45
|
+
* but any list of regions is accepted and dispatched kind-aware. Returns
|
|
46
|
+
* chunks plus their coverage entries ("chunked" and documented "excluded");
|
|
47
|
+
* residual gaps between top-level LIST entries (a legacy/test shape — the
|
|
48
|
+
* adapter contract has no such gaps) are the caller's job — see
|
|
49
|
+
* `chunker.ts`, which fills them with `fallback_lines` after this returns.
|
|
50
|
+
*/
|
|
51
|
+
export declare function runAstChunkStrategy(ctx: BuildContext, regions: readonly AstRegion[], symbols: readonly ExtractedSymbol[]): StrategyResult;
|
|
52
|
+
//# sourceMappingURL=ast-chunk-strategy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ast-chunk-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/ast-chunk-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,eAAe,EAErB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,KAAK,YAAY,EAKlB,MAAM,eAAe,CAAC;AAIvB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B;AAglBD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,YAAY,EACjB,OAAO,EAAE,SAAS,SAAS,EAAE,EAC7B,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,cAAc,CAWhB"}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ast-chunk-strategy.js","sourceRoot":"","sources":["../../src/strategies/ast-chunk-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,OAAO,EACL,aAAa,GAMd,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAEL,SAAS,EACT,SAAS,EACT,wBAAwB,GAEzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAOvF,SAAS,YAAY;IACnB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AACtC,CAAC;AAED,SAAS,IAAI,CAAC,MAAsB,EAAE,KAAqB;IACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAE1C,MAAM,aAAa,GAAG,IAAI,OAAO,EAAqC,CAAC;AAEvE,SAAS,eAAe,CAAC,GAAiB,EAAE,MAAuB;IACjE,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QAClB,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;IACxF,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,EAAE,GAAG,aAAa,CAAC;QACvB,WAAW,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW;QACrC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;QAC3B,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK;QACzB,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ;QACnC,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;KAChD,CAAC,CAAC;IACH,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACxB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,yBAAyB,CAChC,GAAiB,EACjB,MAAiB,EACjB,MAAmC;IAEnC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,MAAM,WAAW,GAAG,MAAM;QACxB,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,aAAa,EAAE;QAC1C,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,MAAM,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,MAAM,EAAE,aAAa,KAAK,SAAS;YACrC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,aAAa,EAAE;YACrC,CAAC,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS;gBAC9B,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE;gBACjC,CAAC,CAAC,EAAE,CAAC;QACT,GAAG,CAAC,MAAM,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,MAAM,CAAC,UAAU,IAAI,WAAW;QAChE,GAAG,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,YAAY,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM;QACxD,cAAc,EAAE,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM;KAChD,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,sBAAsB,CAC7B,GAAiB,EACjB,MAAsB,EACtB,MAAiB,EACjB,OAAe,EACf,MAAkB;IAElB,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,wBAAwB,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QACxF,IAAI,CACF,MAAM,EACN,SAAS,CAAC,GAAG,EAAE;YACb,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,YAAY;YACrB,OAAO;YACP,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM;SACP,CAAC,CACH,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,WAAW,GAAkB,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACzC,IAAI,wBAAwB,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACvF,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC;IAClD,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACnC,IAAI,CACF,MAAM,EACN,SAAS,CAAC,GAAG,EAAE;YACb,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC;YAC9B,OAAO;YACP,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,gBAAgB;YAC7D,MAAM;YACN,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpF,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,yBAAyB,CAChC,GAAiB,EACjB,WAAwC,EACxC,UAAkB,EAClB,OAA6B,EAC7B,YAAsE,EACtE,MAAkB;IAElB,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,IAAI,WAAW,EAAE,QAAQ,EAAE,CAAC;QAC1B,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE1C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACzC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,MAAM,EAAE,SAAS,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC;QACzF,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,IAAI,MAAM,CAAC,SAAS,CAAC;QAC9C,OAAO,OAAO,IAAI,IAAI,KAAK,EAAE,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAU,EAAE;QACvC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC1C,MAAM,MAAM,GACV,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,WAAW,CAAC,MAAM,GAAG,KAAK,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,OAAO,CAAC,QAAQ,EAAE,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,wBAAwB,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAChF,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC;IAC5B,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,IAAI,wBAAwB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACvF,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,SAAS,kBAAkB,CACzB,GAAiB,EACjB,MAAsB,EACtB,GAAyB,EACzB,YAAoB,EACpB,aAAiC,EACjC,WAAuB;IAEvB,MAAM,MAAM,GAAe;QACzB,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,GAAG,CAAC,WAAW,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,GAAG,CAAC,WAAW,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,SAAS,EAAE,GAAG,WAAW,CAAC,SAAS,IAAI,OAAO,aAAa;QAC3D,YAAY,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM;QAC7D,cAAc,EAAE,YAAY;KAC7B,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,KAAK,CAAC,OAAO,CAAC;YAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,OAAO,EAAE,CAAC;gBACrC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACvD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtC,IAAI,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACnF,IAAI,CACF,MAAM,EACN,SAAS,CAAC,GAAG,EAAE;gBACb,KAAK,EAAE,KAAK;gBACZ,OAAO;gBACP,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,gBAAgB;gBAC3B,MAAM;aACP,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,oBAAoB,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;YACxD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;gBAC1B,gEAAgE;gBAChE,kEAAkE;gBAClE,kEAAkE;gBAClE,gDAAgD;gBAChD,IAAI,CACF,MAAM,EACN,SAAS,CAAC,GAAG,EAAE;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC;oBAC1B,OAAO,EAAE,YAAY;oBACrB,SAAS,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,gBAAgB;oBAC7E,MAAM;oBACN,GAAG,CAAC,aAAa,KAAK,SAAS;wBAC7B,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE;wBACxD,CAAC,CAAC,EAAE,CAAC;iBACR,CAAC,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,gBAAgB,CACvB,GAAiB,EACjB,MAAsB,EACtB,KAAkB,EAClB,MAAkB,EAClB,eAAuB;IAEvB,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACrC,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACtC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC;QAC5D,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,MAAM,GAAG,oBAAoB,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACxD,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YAC/C,iEAAiE;YACjE,+DAA+D;YAC/D,gCAAgC;YAChC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,YAAY,CACnB,GAAiB,EACjB,MAAsB,EACtB,YAAuB,EACvB,YAAkD;IAElD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CACtD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAChD,CAAC;IACF,MAAM,YAAY,GAAG,YAAY,CAAC,SAAS;QACzC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC;QAC1C,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,MAAM,GAAG,yBAAyB,CAAC,GAAG,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IAE1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,sEAAsE;QACtE,2CAA2C;QAC3C,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE;YACxF,GAAG,MAAM;YACT,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,UAAU,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE;SACjE,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAe;QAC5B,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,UAAU,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE;QAChE,YAAY,EAAE,MAAM;QACpB,cAAc,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;KACpC,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,SAAiB,EAAE,OAAe,EAAQ,EAAE;QAC3D,IAAI,OAAO,IAAI,SAAS;YAAE,OAAO;QACjC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,gBAAgB,CAC7B,GAAG,EACH,MAAM,EACN,QAAQ,EACR,SAAS,EACT,sDAAsD,CACvD,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CACF,MAAM,EACN,SAAS,CAAC,GAAG,EAAE;gBACb,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC;gBAC9B,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;gBAC5B,SAAS,EAAE,QAAQ;gBACnB,MAAM,EAAE,SAAS;aAClB,CAAC,CACH,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACvC,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QACjD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,WAAW,CAClB,GAAiB,EACjB,MAAsB,EACtB,WAAsB,EACtB,YAAkD;IAElD,MAAM,WAAW,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChG,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClF,MAAM,YAAY,GAAG,aAAa,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC1D,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CACpD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAChD,CAAC;IAEF,MAAM,cAAc,GAAe;QACjC,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,WAAW,EAAE,aAAa,KAAK,SAAS;YAC1C,CAAC,CAAC,EAAE,SAAS,EAAE,WAAW,CAAC,aAAa,EAAE;YAC1C,CAAC,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS;gBACnC,CAAC,CAAC,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE;gBACtC,CAAC,CAAC,EAAE,CAAC;QACT,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,SAAS,EACP,WAAW,EAAE,SAAS,IAAI,WAAW,CAAC,UAAU,IAAI,WAAW,EAAE,aAAa,IAAI,OAAO;QAC3F,GAAG,CAAC,WAAW,EAAE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,YAAY,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM;QAC7D,cAAc,EAAE,YAAY;KAC7B,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,oEAAoE;QACpE,wDAAwD;QACxD,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,CAAC,CAAC;QAC/E,OAAO;IACT,CAAC;IAED,MAAM,UAAU,GAAG,CACjB,WAAW,EAAE,SAAS;QACtB,WAAW,CAAC,UAAU;QACtB,WAAW,EAAE,aAAa;QAC1B,OAAO,CACR,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,eAAe,GAAG,yBAAyB,CAC/C,GAAG,EACH,WAAW,EACX,UAAU,EACV,OAAO,EACP,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,EAClD,cAAc,CACf,CAAC;IACF,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,EAAE;QACnC,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,OAAO,EAAE,eAAe;QACxB,OAAO,EAAE,YAAY;QACrB,SAAS,EAAE,gBAAgB;QAC3B,MAAM,EAAE,cAAc;QACtB,WAAW,EAAE,IAAI;KAClB,CAAC,CAAC;IACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAElC,yEAAyE;IACzE,2EAA2E;IAC3E,uEAAuE;IACvE,yEAAyE;IACzE,2EAA2E;IAC3E,uEAAuE;IACvE,wEAAwE;IACxE,oEAAoE;IACpE,mEAAmE;IACnE,uCAAuC;IACvC,MAAM,aAAa,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,SAAS,CAAC;IAClD,MAAM,iBAAiB,GAA4C,EAAE,CAAC;IACtE,MAAM,gBAAgB,GACpB,wGAAwG,CAAC;IAC3G,IAAI,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9C,MAAM,cAAc,GAAG,CAAC,SAAiB,EAAE,OAAe,EAAQ,EAAE;QAClE,IAAI,SAAS,GAAG,WAAW,IAAI,OAAO,GAAG,aAAa,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;YAC/E,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/D,OAAO;QACT,CAAC;QACD,6DAA6D;QAC7D,yEAAyE;QACzE,sEAAsE;QACtE,gCAAgC;QAChC,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,SAAS,GAAG,WAAW,EAAE,CAAC;YAC5B,iBAAiB,CAAC,IAAI,CAAC;gBACrB,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,SAAS,CAAC;gBAC7D,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC;QACL,CAAC;QACD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAClB,YAAY,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CACzF,CAAC;QACF,WAAW,GAAG,OAAO,CAAC;IACxB,CAAC,CAAC;IAEF,IAAI,WAAW,EAAE,QAAQ,EAAE,CAAC;QAC1B,cAAc,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACxD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;QACzE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,cAAc,CAAC,KAAK,EAAE,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,IAAI,WAAW,GAAG,aAAa,EAAE,CAAC;QAChC,iBAAiB,CAAC,IAAI,CAAC;YACrB,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC;YACjE,MAAM,EAAE,gBAAgB;SACzB,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,wEAAwE;IACxE,sEAAsE;IACtE,sDAAsD;IACtD,MAAM,cAAc,GAClB,8EAA8E,CAAC;IACjF,IAAI,MAAM,GAAG,aAAa,CAAC;IAE3B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QAC3B,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,EAAE,CAAC;YACpC,iBAAiB,CAAC,IAAI,CAAC;gBACrB,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;gBACrE,MAAM,EAAE,cAAc;aACvB,CAAC,CAAC;QACL,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,KAAK,gBAAgB,EAAE,CAAC;YAC1C,MAAM,GAAG,GAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAE,CAAC,SAAS,KAAK,gBAAgB,EAAE,CAAC;gBACxE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;gBACtB,CAAC,IAAI,CAAC,CAAC;YACT,CAAC;YACD,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;YAClF,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;YAClD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAChD,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;IACH,CAAC;IACD,IAAI,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACvC,iBAAiB,CAAC,IAAI,CAAC;YACrB,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC;YACxE,MAAM,EAAE,cAAc;SACvB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,kBAAkB,GAAe;QACrC,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,GAAG,CAAC,cAAc,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,GAAG,CAAC,cAAc,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,CAAC,cAAc,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,GAAG,CAAC,cAAc,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,YAAY,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM;QAC7D,cAAc,EAAE,YAAY;KAC7B,CAAC;IACF,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,yEAAyE;QACzE,wEAAwE;QACxE,qEAAqE;QACrE,IAAI,CACF,MAAM,EACN,SAAS,CAAC,GAAG,EAAE;YACb,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC;YAC9B,OAAO,EAAE,YAAY;YACrB,SAAS,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,gBAAgB;YAC7E,MAAM,EAAE,kBAAkB;YAC1B,GAAG,CAAC,aAAa,KAAK,SAAS;gBAC7B,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE;gBACxD,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CACrB,GAAiB,EACjB,MAAsB,EACtB,MAAiB,EACjB,YAAkD;IAElD,IAAI,MAAM,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAChD,OAAO;IACT,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,KAAK,gBAAgB,EAAE,CAAC;QAC1C,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAC/C,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,MAAM,MAAM,GAAG,yBAAyB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;IACvD,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CACjC,GAAiB,EACjB,OAA6B,EAC7B,OAAmC;IAEnC,MAAM,YAAY,GAAG,IAAI,GAAG,EAA2B,CAAC;IACxD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAClF,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"line-fallback-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/line-fallback-strategy.ts"],"names":[],"mappings":"AAWA,OAAO,EACL,KAAK,YAAY,EAKlB,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AA8B9D,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,YAAY,GAAG,cAAc,CAwDzE"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { rangeForByteSpan } from "../byte-text.js";
|
|
2
|
+
import { contentOf, emitChunk, estimateTokensWithHeader, } from "../context.js";
|
|
3
|
+
import { chunkedEntry } from "../coverage.js";
|
|
4
|
+
import { splitByTokenBoundary } from "../split.js";
|
|
5
|
+
const NO_HEADER = { signature: "fallback_lines" };
|
|
6
|
+
/**
|
|
7
|
+
* Computes the ordered, overlapping line windows covering `[1, totalLines]`.
|
|
8
|
+
* `step = linesPerChunk - overlapLines` (clamped to at least 1 to guarantee
|
|
9
|
+
* termination); the last window is extended to the final line.
|
|
10
|
+
*/
|
|
11
|
+
function computeLineWindows(totalLines, linesPerChunk, overlapLines) {
|
|
12
|
+
const chunkSize = Math.max(1, linesPerChunk);
|
|
13
|
+
const overlap = Math.min(Math.max(0, overlapLines), chunkSize - 1);
|
|
14
|
+
const step = Math.max(1, chunkSize - overlap);
|
|
15
|
+
const windows = [];
|
|
16
|
+
let start = 1;
|
|
17
|
+
while (start <= totalLines) {
|
|
18
|
+
const end = Math.min(totalLines, start + chunkSize - 1);
|
|
19
|
+
windows.push([start, end]);
|
|
20
|
+
if (end >= totalLines) {
|
|
21
|
+
break;
|
|
22
|
+
}
|
|
23
|
+
start += step;
|
|
24
|
+
}
|
|
25
|
+
return windows;
|
|
26
|
+
}
|
|
27
|
+
export function runLineFallbackStrategy(ctx) {
|
|
28
|
+
const result = { drafts: [], coverage: [] };
|
|
29
|
+
const totalLines = ctx.lineIndex.lineStarts.length;
|
|
30
|
+
const windows = computeLineWindows(totalLines, ctx.config.lineFallback.linesPerChunk, ctx.config.lineFallback.overlapLines);
|
|
31
|
+
for (const [startLine, endLine] of windows) {
|
|
32
|
+
const startByte = ctx.lineIndex.lineStarts[startLine - 1];
|
|
33
|
+
const endByte = endLine < totalLines ? ctx.lineIndex.lineStarts[endLine] : ctx.lineIndex.buffer.length;
|
|
34
|
+
if (endByte <= startByte) {
|
|
35
|
+
// Degenerate trailing "line" produced when the file ends in a newline
|
|
36
|
+
// (an empty line index entry with nothing after it) — zero bytes, so
|
|
37
|
+
// there is nothing to cover and nothing to chunk.
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
const range = { startLine, endLine, startByte, endByte };
|
|
41
|
+
const content = contentOf(ctx, range);
|
|
42
|
+
if (estimateTokensWithHeader(ctx, content, NO_HEADER) <= ctx.config.maxTokensPerChunk) {
|
|
43
|
+
const draft = emitChunk(ctx, {
|
|
44
|
+
range,
|
|
45
|
+
content,
|
|
46
|
+
scopeId: ctx.identity.fileId,
|
|
47
|
+
chunkKind: "fallback_lines",
|
|
48
|
+
header: {
|
|
49
|
+
signature: "fallback_lines",
|
|
50
|
+
metricsScope: "chunk",
|
|
51
|
+
metricsScopeId: ctx.identity.fileId,
|
|
52
|
+
},
|
|
53
|
+
});
|
|
54
|
+
result.drafts.push(draft);
|
|
55
|
+
result.coverage.push(chunkedEntry(draft.range, draft.chunkId));
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
for (const piece of splitByTokenBoundary(ctx, range, NO_HEADER)) {
|
|
59
|
+
const pieceContent = contentOf(ctx, piece);
|
|
60
|
+
const rangeWithLines = rangeForByteSpan(ctx.lineIndex, piece.startByte, piece.endByte);
|
|
61
|
+
const draft = emitChunk(ctx, {
|
|
62
|
+
range: rangeWithLines,
|
|
63
|
+
content: pieceContent,
|
|
64
|
+
scopeId: ctx.identity.fileId,
|
|
65
|
+
chunkKind: "fallback_lines",
|
|
66
|
+
header: {
|
|
67
|
+
signature: "fallback_lines",
|
|
68
|
+
metricsScope: "chunk",
|
|
69
|
+
metricsScopeId: ctx.identity.fileId,
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
result.drafts.push(draft);
|
|
73
|
+
result.coverage.push(chunkedEntry(draft.range, draft.chunkId));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=line-fallback-strategy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"line-fallback-strategy.js","sourceRoot":"","sources":["../../src/strategies/line-fallback-strategy.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAEL,SAAS,EACT,SAAS,EACT,wBAAwB,GAEzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAInD,MAAM,SAAS,GAAgD,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC;AAE/F;;;;GAIG;AACH,SAAS,kBAAkB,CACzB,UAAkB,EAClB,aAAqB,EACrB,YAAoB;IAEpB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,MAAM;QACR,CAAC;QACD,KAAK,IAAI,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,GAAiB;IACvD,MAAM,MAAM,GAAmB,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC5D,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;IACnD,MAAM,OAAO,GAAG,kBAAkB,CAChC,UAAU,EACV,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,aAAa,EACrC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CACrC,CAAC;IAEF,KAAK,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAE,CAAC;QAC3D,MAAM,OAAO,GACX,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;QAC1F,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;YACzB,sEAAsE;YACtE,qEAAqE;YACrE,kDAAkD;YAClD,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;QACtE,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtC,IAAI,wBAAwB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACtF,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE;gBAC3B,KAAK;gBACL,OAAO;gBACP,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;gBAC5B,SAAS,EAAE,gBAAgB;gBAC3B,MAAM,EAAE;oBACN,SAAS,EAAE,gBAAgB;oBAC3B,YAAY,EAAE,OAAO;oBACrB,cAAc,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;iBACpC;aACF,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,KAAK,IAAI,oBAAoB,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;gBAChE,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC3C,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;gBACvF,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE;oBAC3B,KAAK,EAAE,cAAc;oBACrB,OAAO,EAAE,YAAY;oBACrB,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;oBAC5B,SAAS,EAAE,gBAAgB;oBAC3B,MAAM,EAAE;wBACN,SAAS,EAAE,gBAAgB;wBAC3B,YAAY,EAAE,OAAO;wBACrB,cAAc,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;qBACpC;iBACF,CAAC,CAAC;gBACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minified-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/minified-strategy.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,YAAY,EAAyC,MAAM,eAAe,CAAC;AAIzF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAI9D,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,YAAY,GAAG,cAAc,CAkCrE"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minified strategy (spec.md §8.3 step 10): the file is treated as one
|
|
3
|
+
* opaque blob and split purely at safe token/byte boundaries — there is no
|
|
4
|
+
* useful AST structure to lean on, by definition of "minified" here.
|
|
5
|
+
*/
|
|
6
|
+
import { rangeForByteSpan, safeUtf8Boundary } from "../byte-text.js";
|
|
7
|
+
import { contentOf, emitChunk } from "../context.js";
|
|
8
|
+
import { chunkedEntry } from "../coverage.js";
|
|
9
|
+
import { splitByTokenBoundary } from "../split.js";
|
|
10
|
+
const HEADER = { signature: "minified_fragment" };
|
|
11
|
+
export function runMinifiedStrategy(ctx) {
|
|
12
|
+
const result = { drafts: [], coverage: [] };
|
|
13
|
+
const pieces = splitByTokenBoundary(ctx, ctx.fileRange, HEADER);
|
|
14
|
+
const overlapBytes = Math.max(0, ctx.config.minifiedStrategy.overlapBytes);
|
|
15
|
+
pieces.forEach((piece, index) => {
|
|
16
|
+
// Configurable trailing overlap: pull this piece's start back into the
|
|
17
|
+
// previous piece's territory for retrieval context. Never before the
|
|
18
|
+
// file start, and never a UTF-8-unsafe cut (architecture.md §14.4).
|
|
19
|
+
// Coverage stays gap-free either way — overlaps are allowed.
|
|
20
|
+
let startByte = piece.startByte;
|
|
21
|
+
if (index > 0 && overlapBytes > 0) {
|
|
22
|
+
startByte = safeUtf8Boundary(ctx.lineIndex.buffer, Math.max(ctx.fileRange.startByte, piece.startByte - overlapBytes));
|
|
23
|
+
}
|
|
24
|
+
const range = rangeForByteSpan(ctx.lineIndex, startByte, piece.endByte);
|
|
25
|
+
const content = contentOf(ctx, range);
|
|
26
|
+
const draft = emitChunk(ctx, {
|
|
27
|
+
range,
|
|
28
|
+
content,
|
|
29
|
+
scopeId: ctx.identity.fileId,
|
|
30
|
+
chunkKind: "minified_fragment",
|
|
31
|
+
header: {
|
|
32
|
+
signature: "minified_fragment",
|
|
33
|
+
metricsScope: "chunk",
|
|
34
|
+
metricsScopeId: ctx.identity.fileId,
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
result.drafts.push(draft);
|
|
38
|
+
result.coverage.push(chunkedEntry(draft.range, draft.chunkId));
|
|
39
|
+
});
|
|
40
|
+
return result;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=minified-strategy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minified-strategy.js","sourceRoot":"","sources":["../../src/strategies/minified-strategy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAAqB,SAAS,EAAE,SAAS,EAAmB,MAAM,eAAe,CAAC;AACzF,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAInD,MAAM,MAAM,GAAgD,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;AAE/F,MAAM,UAAU,mBAAmB,CAAC,GAAiB;IACnD,MAAM,MAAM,GAAmB,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC5D,MAAM,MAAM,GAAG,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAE3E,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,uEAAuE;QACvE,qEAAqE;QACrE,oEAAoE;QACpE,6DAA6D;QAC7D,IAAI,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YAClC,SAAS,GAAG,gBAAgB,CAC1B,GAAG,CAAC,SAAS,CAAC,MAAM,EACpB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,YAAY,CAAC,CAClE,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE;YAC3B,KAAK;YACL,OAAO;YACP,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;YAC5B,SAAS,EAAE,mBAAmB;YAC9B,MAAM,EAAE;gBACN,SAAS,EAAE,mBAAmB;gBAC9B,YAAY,EAAE,OAAO;gBACrB,cAAc,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;aACpC;SACF,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conservative, offline token estimator (architecture.md §14.4).
|
|
3
|
+
*
|
|
4
|
+
* There is no network tokenizer call here on purpose (spec.md §24 privacy,
|
|
5
|
+
* and the chunker must run fully offline). The estimate is deliberately
|
|
6
|
+
* biased to OVER-estimate real BPE-style tokenizer counts rather than
|
|
7
|
+
* under-estimate: real code tokenizers average roughly 3.3-4.5 characters
|
|
8
|
+
* per token (identifiers and keywords tokenize efficiently; whitespace and
|
|
9
|
+
* punctuation less so). Dividing by a lower `charsPerToken` than that
|
|
10
|
+
* average guarantees the estimate is usually >= the true count, so a chunk
|
|
11
|
+
* that "fits the budget" here also fits the embedding model's real budget —
|
|
12
|
+
* the failure mode of under-estimating (silently truncated embeddings) is
|
|
13
|
+
* far worse than over-fragmenting slightly.
|
|
14
|
+
*/
|
|
15
|
+
export interface TokenEstimatorConfig {
|
|
16
|
+
/** Conservative average characters per token; lower over-estimates. */
|
|
17
|
+
charsPerToken: number;
|
|
18
|
+
/** Per-line overhead (in tokens) added for newline/indentation noise. */
|
|
19
|
+
perLineOverheadTokens: number;
|
|
20
|
+
}
|
|
21
|
+
/** Untangle decision: 3 chars/token (vs. the ~4 real average) plus a small per-line overhead. */
|
|
22
|
+
export declare const DEFAULT_TOKEN_ESTIMATOR_CONFIG: TokenEstimatorConfig;
|
|
23
|
+
/**
|
|
24
|
+
* Estimates the token count of `text`. Pure function of its inputs; never
|
|
25
|
+
* touches the network or a real tokenizer vocabulary.
|
|
26
|
+
*/
|
|
27
|
+
export declare function estimateTokens(text: string, config?: TokenEstimatorConfig): number;
|
|
28
|
+
//# sourceMappingURL=token-estimator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-estimator.d.ts","sourceRoot":"","sources":["../src/token-estimator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,MAAM,WAAW,oBAAoB;IACnC,uEAAuE;IACvE,aAAa,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,iGAAiG;AACjG,eAAO,MAAM,8BAA8B,EAAE,oBAG5C,CAAC;AAEF;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,oBAAqD,GAC5D,MAAM,CAYR"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conservative, offline token estimator (architecture.md §14.4).
|
|
3
|
+
*
|
|
4
|
+
* There is no network tokenizer call here on purpose (spec.md §24 privacy,
|
|
5
|
+
* and the chunker must run fully offline). The estimate is deliberately
|
|
6
|
+
* biased to OVER-estimate real BPE-style tokenizer counts rather than
|
|
7
|
+
* under-estimate: real code tokenizers average roughly 3.3-4.5 characters
|
|
8
|
+
* per token (identifiers and keywords tokenize efficiently; whitespace and
|
|
9
|
+
* punctuation less so). Dividing by a lower `charsPerToken` than that
|
|
10
|
+
* average guarantees the estimate is usually >= the true count, so a chunk
|
|
11
|
+
* that "fits the budget" here also fits the embedding model's real budget —
|
|
12
|
+
* the failure mode of under-estimating (silently truncated embeddings) is
|
|
13
|
+
* far worse than over-fragmenting slightly.
|
|
14
|
+
*/
|
|
15
|
+
/** Untangle decision: 3 chars/token (vs. the ~4 real average) plus a small per-line overhead. */
|
|
16
|
+
export const DEFAULT_TOKEN_ESTIMATOR_CONFIG = {
|
|
17
|
+
charsPerToken: 3,
|
|
18
|
+
perLineOverheadTokens: 0.25,
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Estimates the token count of `text`. Pure function of its inputs; never
|
|
22
|
+
* touches the network or a real tokenizer vocabulary.
|
|
23
|
+
*/
|
|
24
|
+
export function estimateTokens(text, config = DEFAULT_TOKEN_ESTIMATOR_CONFIG) {
|
|
25
|
+
if (text.length === 0) {
|
|
26
|
+
return 0;
|
|
27
|
+
}
|
|
28
|
+
let lineCount = 1;
|
|
29
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
30
|
+
if (text.charCodeAt(i) === 10) {
|
|
31
|
+
lineCount += 1;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const raw = text.length / config.charsPerToken + lineCount * config.perLineOverheadTokens;
|
|
35
|
+
return Math.max(1, Math.ceil(raw));
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=token-estimator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-estimator.js","sourceRoot":"","sources":["../src/token-estimator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AASH,iGAAiG;AACjG,MAAM,CAAC,MAAM,8BAA8B,GAAyB;IAClE,aAAa,EAAE,CAAC;IAChB,qBAAqB,EAAE,IAAI;CAC5B,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,SAA+B,8BAA8B;IAE7D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAC9B,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,aAAa,GAAG,SAAS,GAAG,MAAM,CAAC,qBAAqB,CAAC;IAC1F,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACrC,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chunker-local configuration and input/output contracts.
|
|
3
|
+
*
|
|
4
|
+
* These types are additive to `@untangle-code/domain` — they configure how
|
|
5
|
+
* this package turns `AstRegion`/`ExtractedSymbol`/`ParseResult` into
|
|
6
|
+
* `CodeChunkDraft`s, but never redefine a domain contract.
|
|
7
|
+
*/
|
|
8
|
+
import type { AstRegion, CodeChunkDraft, CoverageEntry, ExtractedSymbol, ParseQuality, ParseResult } from "@untangle-code/domain";
|
|
9
|
+
import type { MinifiedDetectionConfig } from "./minified-detection.js";
|
|
10
|
+
import type { PreparedTextContext } from "./prepared-text.js";
|
|
11
|
+
import type { TokenEstimatorConfig } from "./token-estimator.js";
|
|
12
|
+
/** Line-window fallback config for parse failures (spec.md §8.3 step 9). */
|
|
13
|
+
export interface LineFallbackConfig {
|
|
14
|
+
/** Target number of source lines per fallback chunk before overlap. */
|
|
15
|
+
linesPerChunk: number;
|
|
16
|
+
/** Number of trailing lines repeated at the start of the next chunk. */
|
|
17
|
+
overlapLines: number;
|
|
18
|
+
}
|
|
19
|
+
/** Configurable overlap for the minified strategy (spec.md §8.3 step 10). */
|
|
20
|
+
export interface MinifiedStrategyConfig {
|
|
21
|
+
/** Bytes of trailing context repeated at the start of the next fragment. */
|
|
22
|
+
overlapBytes: number;
|
|
23
|
+
}
|
|
24
|
+
/** Full chunker configuration; every field has a documented default. */
|
|
25
|
+
export interface ChunkerConfig {
|
|
26
|
+
/** Chunk identity version input (spec.md §5.6). Defaults to `CHUNKER_VERSION`. */
|
|
27
|
+
chunkerVersion: string;
|
|
28
|
+
/** Stamped onto every draft as provenance (architecture.md §2.3). */
|
|
29
|
+
parserVersion: string;
|
|
30
|
+
/** Conservative token budget per chunk, header included (architecture.md §14.4). */
|
|
31
|
+
maxTokensPerChunk: number;
|
|
32
|
+
tokenEstimator: TokenEstimatorConfig;
|
|
33
|
+
preparedText: PreparedTextContext;
|
|
34
|
+
minifiedDetection: MinifiedDetectionConfig;
|
|
35
|
+
lineFallback: LineFallbackConfig;
|
|
36
|
+
minifiedStrategy: MinifiedStrategyConfig;
|
|
37
|
+
}
|
|
38
|
+
/** User-supplied overrides; every field optional except the prepared-text header. */
|
|
39
|
+
export type ChunkerConfigInput = Partial<Omit<ChunkerConfig, "preparedText">> & {
|
|
40
|
+
preparedText: PreparedTextContext;
|
|
41
|
+
};
|
|
42
|
+
/** Minimal identity of the file being chunked (a slice of domain `SourceFile`). */
|
|
43
|
+
export interface ChunkFileIdentity {
|
|
44
|
+
workspaceId: string;
|
|
45
|
+
repoId: string;
|
|
46
|
+
refId: string;
|
|
47
|
+
fileId: string;
|
|
48
|
+
/** "/"-normalized path relative to the repository root. */
|
|
49
|
+
filePath: string;
|
|
50
|
+
languageId: string;
|
|
51
|
+
}
|
|
52
|
+
/** Input to {@link chunkFile} (see `src/chunker.ts`). */
|
|
53
|
+
export interface ChunkFileParams {
|
|
54
|
+
identity: ChunkFileIdentity;
|
|
55
|
+
/** Full UTF-8 decoded file content (spec.md §7). */
|
|
56
|
+
content: string;
|
|
57
|
+
parseResult: ParseResult;
|
|
58
|
+
/** `adapter.extractChunks(ast)` output; empty when `parseResult.ast` is null. */
|
|
59
|
+
regions: AstRegion[];
|
|
60
|
+
/** `adapter.extractSymbols(ast)` output; empty when `parseResult.ast` is null. */
|
|
61
|
+
symbols: ExtractedSymbol[];
|
|
62
|
+
/**
|
|
63
|
+
* Overrides minified auto-detection when the caller already knows (e.g. a
|
|
64
|
+
* `.min.js` scanner classification). Leave unset to auto-detect.
|
|
65
|
+
*/
|
|
66
|
+
isMinified?: boolean;
|
|
67
|
+
config: ChunkerConfigInput;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Warum der Chunker unterhalb der AST-Strategie geblieben ist (E4, WP-165).
|
|
71
|
+
*
|
|
72
|
+
* Geschlossene Menge, damit die Gruende zaehlbar bleiben; die freie
|
|
73
|
+
* Erlaeuterung steht in `detail`, nicht im Grund. Bewusst PAKETLOKAL und nicht
|
|
74
|
+
* als `SkipReason` aus `@untangle-code/domain`: dessen geschlossener Katalog
|
|
75
|
+
* kennt keinen Grund fuer "der Parser war erfolgreich, lieferte aber keine
|
|
76
|
+
* Regionen", und `parse_failed` waere dafuer schlicht falsch. Dieselbe
|
|
77
|
+
* Entscheidung hat `@untangle-code/context-artifacts` fuer seinen
|
|
78
|
+
* `ArtifactSkipReason` getroffen.
|
|
79
|
+
*/
|
|
80
|
+
export declare const CHUNK_DEGRADATION_REASONS: readonly ["parse_failed", "ast_regions_missing", "minified_content"];
|
|
81
|
+
export type ChunkDegradationReason = (typeof CHUNK_DEGRADATION_REASONS)[number];
|
|
82
|
+
/**
|
|
83
|
+
* Eine Abstufung: der Chunker hat die Datei zwar lueckenlos erfasst, aber OHNE
|
|
84
|
+
* die Symbolstruktur, die die AST-Strategie geliefert haette.
|
|
85
|
+
*
|
|
86
|
+
* Ohne diesen Eintrag ist genau das unsichtbar. Die Byte-Abdeckung ist
|
|
87
|
+
* vollstaendig (`assertLosslessCoverage` ist zufrieden), `parseQuality` kann
|
|
88
|
+
* `"complete"` lauten -- und trotzdem hat kein einziger Chunk eine
|
|
89
|
+
* Symbolzuordnung. Ein Aufrufer, der `parseQuality` liest, haelt das Ergebnis
|
|
90
|
+
* fuer strukturell analysiert. Das ist die Fehlerklasse "Vollstaendigkeit
|
|
91
|
+
* behauptet, Teilbaum verworfen" in ihrer unauffaelligsten Form, deshalb steht
|
|
92
|
+
* sie hier als eigener, benannter Befund im Ergebnis.
|
|
93
|
+
*/
|
|
94
|
+
export interface ChunkDegradation {
|
|
95
|
+
reason: ChunkDegradationReason;
|
|
96
|
+
/** Welche Strategie stattdessen lief. */
|
|
97
|
+
strategy: "line_fallback" | "minified";
|
|
98
|
+
/** Menschenlesbare Begruendung; MUSS nicht leer sein (E4). */
|
|
99
|
+
detail: string;
|
|
100
|
+
}
|
|
101
|
+
/** Output of {@link chunkFile}. */
|
|
102
|
+
export interface ChunkFileResult {
|
|
103
|
+
drafts: CodeChunkDraft[];
|
|
104
|
+
coverage: CoverageEntry[];
|
|
105
|
+
/** Which of the three strategies (spec.md §8.3) produced `drafts`. */
|
|
106
|
+
strategy: "ast" | "line_fallback" | "minified";
|
|
107
|
+
parseQuality: ParseQuality;
|
|
108
|
+
/**
|
|
109
|
+
* Leer, wenn die AST-Strategie lief. Sonst genau ein Eintrag mit dem Grund
|
|
110
|
+
* (E4: keine stillschweigende Abstufung) -- siehe {@link ChunkDegradation}.
|
|
111
|
+
*/
|
|
112
|
+
degradations: ChunkDegradation[];
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EACV,SAAS,EACT,cAAc,EACd,aAAa,EACb,eAAe,EACf,YAAY,EACZ,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,4EAA4E;AAC5E,MAAM,WAAW,kBAAkB;IACjC,uEAAuE;IACvE,aAAa,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,6EAA6E;AAC7E,MAAM,WAAW,sBAAsB;IACrC,4EAA4E;IAC5E,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,kFAAkF;IAClF,cAAc,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,aAAa,EAAE,MAAM,CAAC;IACtB,oFAAoF;IACpF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,oBAAoB,CAAC;IACrC,YAAY,EAAE,mBAAmB,CAAC;IAClC,iBAAiB,EAAE,uBAAuB,CAAC;IAC3C,YAAY,EAAE,kBAAkB,CAAC;IACjC,gBAAgB,EAAE,sBAAsB,CAAC;CAC1C;AAED,qFAAqF;AACrF,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,GAAG;IAC9E,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC;AAEF,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,yDAAyD;AACzD,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,WAAW,CAAC;IACzB,iFAAiF;IACjF,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,kFAAkF;IAClF,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,kBAAkB,CAAC;CAC5B;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,sEAO5B,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEhF;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,sBAAsB,CAAC;IAC/B,yCAAyC;IACzC,QAAQ,EAAE,eAAe,GAAG,UAAU,CAAC;IACvC,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,mCAAmC;AACnC,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,sEAAsE;IACtE,QAAQ,EAAE,KAAK,GAAG,eAAe,GAAG,UAAU,CAAC;IAC/C,YAAY,EAAE,YAAY,CAAC;IAC3B;;;OAGG;IACH,YAAY,EAAE,gBAAgB,EAAE,CAAC;CAClC"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Warum der Chunker unterhalb der AST-Strategie geblieben ist (E4, WP-165).
|
|
3
|
+
*
|
|
4
|
+
* Geschlossene Menge, damit die Gruende zaehlbar bleiben; die freie
|
|
5
|
+
* Erlaeuterung steht in `detail`, nicht im Grund. Bewusst PAKETLOKAL und nicht
|
|
6
|
+
* als `SkipReason` aus `@untangle-code/domain`: dessen geschlossener Katalog
|
|
7
|
+
* kennt keinen Grund fuer "der Parser war erfolgreich, lieferte aber keine
|
|
8
|
+
* Regionen", und `parse_failed` waere dafuer schlicht falsch. Dieselbe
|
|
9
|
+
* Entscheidung hat `@untangle-code/context-artifacts` fuer seinen
|
|
10
|
+
* `ArtifactSkipReason` getroffen.
|
|
11
|
+
*/
|
|
12
|
+
export const CHUNK_DEGRADATION_REASONS = [
|
|
13
|
+
/** Der Parser ist gescheitert; es gab keinen Baum, an dem entlang zu schneiden waere. */
|
|
14
|
+
"parse_failed",
|
|
15
|
+
/** Der Parser meldete Erfolg, der Sprachadapter lieferte aber keine Regionen. */
|
|
16
|
+
"ast_regions_missing",
|
|
17
|
+
/** Der Inhalt ist minifiziert; Byte-Fenster statt Symbolstruktur (spec.md §8.3 Schritt 10). */
|
|
18
|
+
"minified_content",
|
|
19
|
+
];
|
|
20
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAmFA;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,yFAAyF;IACzF,cAAc;IACd,iFAAiF;IACjF,qBAAqB;IACrB,+FAA+F;IAC/F,kBAAkB;CACV,CAAC"}
|