@idfkit/core 0.2.0-rc.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/introspect/describe.d.ts +1 -12
- package/dist/introspect/describe.d.ts.map +1 -1
- package/dist/introspect/describe.js +16 -0
- package/dist/introspect/describe.js.map +1 -1
- package/dist/parse/idf.d.ts.map +1 -1
- package/dist/parse/idf.js +26 -46
- package/dist/parse/idf.js.map +1 -1
- package/dist/parse/lexer.d.ts +6 -13
- package/dist/parse/lexer.d.ts.map +1 -1
- package/dist/parse/lexer.js +44 -109
- package/dist/parse/lexer.js.map +1 -1
- package/dist/parse/scan.d.ts +34 -0
- package/dist/parse/scan.d.ts.map +1 -0
- package/dist/parse/scan.js +177 -0
- package/dist/parse/scan.js.map +1 -0
- package/dist/syntax/classify.d.ts +25 -0
- package/dist/syntax/classify.d.ts.map +1 -0
- package/dist/syntax/classify.js +59 -0
- package/dist/syntax/classify.js.map +1 -0
- package/dist/syntax/layer.d.ts +67 -0
- package/dist/syntax/layer.d.ts.map +1 -0
- package/dist/syntax/layer.js +134 -0
- package/dist/syntax/layer.js.map +1 -0
- package/dist/syntax/region.d.ts +73 -0
- package/dist/syntax/region.d.ts.map +1 -0
- package/dist/syntax/region.js +105 -0
- package/dist/syntax/region.js.map +1 -0
- package/dist/syntax/tokens.d.ts +80 -0
- package/dist/syntax/tokens.d.ts.map +1 -0
- package/dist/syntax/tokens.js +140 -0
- package/dist/syntax/tokens.js.map +1 -0
- package/package.json +3 -3
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Line index per source, built once and reused.
|
|
3
|
+
*
|
|
4
|
+
* Keyed by the source object rather than by its text, because a `SyntaxLayer` is immutable and
|
|
5
|
+
* lives as long as the document does, while keying on the string itself would hold megabytes of
|
|
6
|
+
* text alive for the sake of a few kilobytes of offsets. A caller that passes a fresh object
|
|
7
|
+
* literal each time gets a fresh index each time, which is the documented cost of doing that.
|
|
8
|
+
*/
|
|
9
|
+
const lineIndexes = new WeakMap();
|
|
10
|
+
/**
|
|
11
|
+
* Offsets at which each line begins, one entry per line, ascending.
|
|
12
|
+
*
|
|
13
|
+
* A line break is a line feed, and nothing else. A carriage return before it belongs to the line
|
|
14
|
+
* it ends, so a column at the end of a CRLF line is one greater than the same column in a file
|
|
15
|
+
* using line feeds alone, which is what every editor reports. A lone carriage return is *not* a
|
|
16
|
+
* break: the existing lexer does not treat it as one and Python's `_line_and_column` counts line
|
|
17
|
+
* feeds, and the conformance corpus compares a finding on its line, so a third opinion here would
|
|
18
|
+
* put the two libraries one line apart on a file no gate would explain.
|
|
19
|
+
*
|
|
20
|
+
* @internal
|
|
21
|
+
*/
|
|
22
|
+
export function lineStartsOf(source) {
|
|
23
|
+
const cached = lineIndexes.get(source);
|
|
24
|
+
if (cached !== undefined)
|
|
25
|
+
return cached;
|
|
26
|
+
const starts = buildLineStarts(source.text);
|
|
27
|
+
lineIndexes.set(source, starts);
|
|
28
|
+
return starts;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The line and column an offset falls on.
|
|
32
|
+
*
|
|
33
|
+
* Binary search over the line index, so a query costs the logarithm of the line count rather than
|
|
34
|
+
* a scan of the text. An offset outside `[0, text.length]` is clamped into range rather than
|
|
35
|
+
* throwing: a cursor arrives from an editor that may be a keystroke ahead of the text this layer
|
|
36
|
+
* was built from, and refusing to answer is worse than answering about the nearest character.
|
|
37
|
+
*
|
|
38
|
+
* The column of a *statement* is the column of its first non-blank character, not of the
|
|
39
|
+
* whitespace indenting it. This function does not enforce that rule, because a statement's region
|
|
40
|
+
* already starts at that character; it is stated here because every column this feature reports
|
|
41
|
+
* obeys it, and because the conformance corpus compares findings on their line and type name and
|
|
42
|
+
* would notice the number moving.
|
|
43
|
+
*/
|
|
44
|
+
export function lineColumnAt(source, offset) {
|
|
45
|
+
const clamped = clampOffset(offset, source.text.length);
|
|
46
|
+
const starts = lineStartsOf(source);
|
|
47
|
+
const line = lineIndexAt(starts, clamped);
|
|
48
|
+
return { line: line + 1, column: clamped - starts[line] + 1 };
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The offset a line and column names.
|
|
52
|
+
*
|
|
53
|
+
* The inverse of {@link lineColumnAt} for every position that exists, and clamped for every one
|
|
54
|
+
* that does not: a line past the end resolves on the last line, a column past the end of its line
|
|
55
|
+
* resolves at the line's last position, which is the offset of the break that ends it. Round
|
|
56
|
+
* tripping an out-of-range position therefore returns the clamped position rather than the one
|
|
57
|
+
* asked for, which is the only honest answer available.
|
|
58
|
+
*/
|
|
59
|
+
export function offsetAt(source, position) {
|
|
60
|
+
const text = source.text;
|
|
61
|
+
const starts = lineStartsOf(source);
|
|
62
|
+
const line = clampOffset(position.line - 1, starts.length - 1);
|
|
63
|
+
const lineStart = starts[line];
|
|
64
|
+
// One past the last position on this line is where the next line starts, so the last position on
|
|
65
|
+
// it is one before that: the offset of the line feed, at which a cursor is still on this line.
|
|
66
|
+
const lineEnd = line + 1 < starts.length ? starts[line + 1] - 1 : text.length;
|
|
67
|
+
const offset = lineStart + clampOffset(position.column - 1, text.length);
|
|
68
|
+
return offset > lineEnd ? lineEnd : offset;
|
|
69
|
+
}
|
|
70
|
+
/** Two passes so the array is allocated once at its exact size; `indexOf` does the scanning. */
|
|
71
|
+
function buildLineStarts(text) {
|
|
72
|
+
let count = 1;
|
|
73
|
+
for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1))
|
|
74
|
+
count += 1;
|
|
75
|
+
const starts = new Int32Array(count);
|
|
76
|
+
let line = 1;
|
|
77
|
+
for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1)) {
|
|
78
|
+
starts[line] = at + 1;
|
|
79
|
+
line += 1;
|
|
80
|
+
}
|
|
81
|
+
return starts;
|
|
82
|
+
}
|
|
83
|
+
/** Index of the greatest line start that is at or before `offset`. */
|
|
84
|
+
function lineIndexAt(starts, offset) {
|
|
85
|
+
let low = 0;
|
|
86
|
+
let high = starts.length - 1;
|
|
87
|
+
while (low < high) {
|
|
88
|
+
const middle = (low + high + 1) >> 1;
|
|
89
|
+
if (starts[middle] <= offset)
|
|
90
|
+
low = middle;
|
|
91
|
+
else
|
|
92
|
+
high = middle - 1;
|
|
93
|
+
}
|
|
94
|
+
return low;
|
|
95
|
+
}
|
|
96
|
+
/** Into `[0, max]`, whole. `NaN` lands at 0, since no position is nearer than another. */
|
|
97
|
+
function clampOffset(value, max) {
|
|
98
|
+
if (Number.isNaN(value))
|
|
99
|
+
return 0;
|
|
100
|
+
const whole = Math.trunc(value);
|
|
101
|
+
if (whole < 0)
|
|
102
|
+
return 0;
|
|
103
|
+
return whole > max ? max : whole;
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=region.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"region.js","sourceRoot":"","sources":["../../src/syntax/region.ts"],"names":[],"mappings":"AAkDA;;;;;;;GAOG;AACH,MAAM,WAAW,GAAG,IAAI,OAAO,EAA0B,CAAC;AAE1D;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5C,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB,EAAE,MAAc;IAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAE,GAAG,CAAC,EAAE,CAAC;AACjE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAkB,EAAE,QAAoB;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC;IAChC,iGAAiG;IACjG,+FAA+F;IAC/F,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAC/E,MAAM,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AAC7C,CAAC;AAED,gGAAgG;AAChG,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;QAC7E,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC;IACZ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,sEAAsE;AACtE,SAAS,WAAW,CAAC,MAAkB,EAAE,MAAc;IACrD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC7B,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,CAAE,IAAI,MAAM;YAAE,GAAG,GAAG,MAAM,CAAC;;YACvC,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,0FAA0F;AAC1F,SAAS,WAAW,CAAC,KAAa,EAAE,GAAW;IAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AACnC,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { Region } from './region.js';
|
|
2
|
+
/**
|
|
3
|
+
* What a span of text is, grammatically. Nothing here needs a schema: the layer records what the
|
|
4
|
+
* text says and never what it means, so a `value` is a value whether or not the field exists.
|
|
5
|
+
*/
|
|
6
|
+
export type TokenKind = 'typeName' | 'value' | 'separator' | 'terminator' | 'comment' | 'trivia';
|
|
7
|
+
/**
|
|
8
|
+
* One token, materialised.
|
|
9
|
+
*
|
|
10
|
+
* A `Token` *is* a `Region` rather than carrying one, which keeps a classified stream to one
|
|
11
|
+
* object per token instead of two. On a file of forty thousand lines that difference is the
|
|
12
|
+
* difference between an editor that repaints and one that stutters, and a token still passes
|
|
13
|
+
* anywhere a region is expected.
|
|
14
|
+
*
|
|
15
|
+
* A materialised token never crosses a line boundary; a stored `value` region may, because the
|
|
16
|
+
* format lets a field be written across two lines, and the classification view splits it.
|
|
17
|
+
*/
|
|
18
|
+
export interface Token extends Region {
|
|
19
|
+
readonly kind: TokenKind;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Every token of one text, packed into three parallel arrays.
|
|
23
|
+
*
|
|
24
|
+
* Nine bytes per token, against upwards of forty for an object: a start, an end, and a kind code,
|
|
25
|
+
* held in `Int32Array`, `Int32Array` and `Uint8Array`. On a reference model of ten thousand
|
|
26
|
+
* statements that is roughly 3.6 MB rather than upwards of 16 MB.
|
|
27
|
+
*
|
|
28
|
+
* This is a contract rather than an implementation note. The intended use is an editor holding
|
|
29
|
+
* several large files open at once, and a layer that allocated an object per token would pass
|
|
30
|
+
* every correctness test while being unusable there. So `Token` objects are materialised only when
|
|
31
|
+
* a caller asks for one, which for classification means only for the tokens actually drawn.
|
|
32
|
+
*
|
|
33
|
+
* The store knows nothing of the scanner that fills it or of the layer that holds it, and nothing
|
|
34
|
+
* of the text either: it holds positions, and the text they index stays with the layer.
|
|
35
|
+
*/
|
|
36
|
+
export declare class TokenStore implements Iterable<Token> {
|
|
37
|
+
#private;
|
|
38
|
+
constructor(capacity?: number);
|
|
39
|
+
/** How many tokens are stored. The arrays below are longer than this; they carry slack. */
|
|
40
|
+
get length(): number;
|
|
41
|
+
/**
|
|
42
|
+
* Start offsets, in source order.
|
|
43
|
+
*
|
|
44
|
+
* A view over the live buffer, trimmed to `length`, for a consumer that wants to walk the
|
|
45
|
+
* positions without materialising anything. It is invalidated by the next {@link push}, which in
|
|
46
|
+
* practice means it is safe for as long as the layer is: nothing appends to a built layer.
|
|
47
|
+
*/
|
|
48
|
+
get starts(): Int32Array;
|
|
49
|
+
/** End offsets, exclusive, in source order. A live view, like {@link starts}. */
|
|
50
|
+
get ends(): Int32Array;
|
|
51
|
+
/** Kind codes, indices into the `TokenKind` union. A live view, like {@link starts}. */
|
|
52
|
+
get kinds(): Uint8Array;
|
|
53
|
+
/** Append one token and return its index. Grows geometrically, so a scan stays linear. */
|
|
54
|
+
push(start: number, end: number, kind: TokenKind): number;
|
|
55
|
+
/** The start offset of one token. */
|
|
56
|
+
startAt(index: number): number;
|
|
57
|
+
/** The end offset of one token, exclusive. */
|
|
58
|
+
endAt(index: number): number;
|
|
59
|
+
/** The kind of one token. */
|
|
60
|
+
kindAt(index: number): TokenKind;
|
|
61
|
+
/** The stored kind code of one token, for a consumer encoding kinds numerically. */
|
|
62
|
+
kindCodeAt(index: number): number;
|
|
63
|
+
/**
|
|
64
|
+
* Build a `Token` object for one stored token.
|
|
65
|
+
*
|
|
66
|
+
* The only place a token becomes an object. Called for the tokens a consumer actually reads, and
|
|
67
|
+
* never on the way in.
|
|
68
|
+
*/
|
|
69
|
+
materialise(index: number): Token;
|
|
70
|
+
/**
|
|
71
|
+
* Every stored token, materialised one at a time.
|
|
72
|
+
*
|
|
73
|
+
* Convenient rather than cheap: it allocates per token, so a consumer colouring a viewport reads
|
|
74
|
+
* the arrays or goes through `classify`, which yields only what it is asked for. Note that this
|
|
75
|
+
* yields stored tokens alone, so the gaps between them are not covered; `trivia` is never stored
|
|
76
|
+
* and is computed as the complement.
|
|
77
|
+
*/
|
|
78
|
+
[Symbol.iterator](): IterableIterator<Token>;
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=tokens.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/syntax/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,SAAS,GACjB,UAAU,GACV,OAAO,GACP,WAAW,GACX,YAAY,GACZ,SAAS,GACT,QAAQ,CAAC;AAEb;;;;;;;;;;GAUG;AACH,MAAM,WAAW,KAAM,SAAQ,MAAM;IACnC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;CAC1B;AAwBD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,UAAW,YAAW,QAAQ,CAAC,KAAK,CAAC;;gBAMpC,QAAQ,GAAE,MAAyB;IAO/C,2FAA2F;IAC3F,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;;;;OAMG;IACH,IAAI,MAAM,IAAI,UAAU,CAEvB;IAED,iFAAiF;IACjF,IAAI,IAAI,IAAI,UAAU,CAErB;IAED,wFAAwF;IACxF,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,0FAA0F;IAC1F,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,MAAM;IAUzD,qCAAqC;IACrC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI9B,8CAA8C;IAC9C,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI5B,6BAA6B;IAC7B,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAIhC,oFAAoF;IACpF,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIjC;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK;IASjC;;;;;;;OAOG;IACF,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC;CAuB9C"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/** Kind codes, in the order the union declares them. The index is what `kinds` stores. */
|
|
2
|
+
const KIND_BY_CODE = [
|
|
3
|
+
'typeName',
|
|
4
|
+
'value',
|
|
5
|
+
'separator',
|
|
6
|
+
'terminator',
|
|
7
|
+
'comment',
|
|
8
|
+
'trivia',
|
|
9
|
+
];
|
|
10
|
+
const CODE_BY_KIND = {
|
|
11
|
+
typeName: 0,
|
|
12
|
+
value: 1,
|
|
13
|
+
separator: 2,
|
|
14
|
+
terminator: 3,
|
|
15
|
+
comment: 4,
|
|
16
|
+
trivia: 5,
|
|
17
|
+
};
|
|
18
|
+
/** Where growth starts. Small enough for a one-statement file, large enough to skip early copies. */
|
|
19
|
+
const INITIAL_CAPACITY = 64;
|
|
20
|
+
/**
|
|
21
|
+
* Every token of one text, packed into three parallel arrays.
|
|
22
|
+
*
|
|
23
|
+
* Nine bytes per token, against upwards of forty for an object: a start, an end, and a kind code,
|
|
24
|
+
* held in `Int32Array`, `Int32Array` and `Uint8Array`. On a reference model of ten thousand
|
|
25
|
+
* statements that is roughly 3.6 MB rather than upwards of 16 MB.
|
|
26
|
+
*
|
|
27
|
+
* This is a contract rather than an implementation note. The intended use is an editor holding
|
|
28
|
+
* several large files open at once, and a layer that allocated an object per token would pass
|
|
29
|
+
* every correctness test while being unusable there. So `Token` objects are materialised only when
|
|
30
|
+
* a caller asks for one, which for classification means only for the tokens actually drawn.
|
|
31
|
+
*
|
|
32
|
+
* The store knows nothing of the scanner that fills it or of the layer that holds it, and nothing
|
|
33
|
+
* of the text either: it holds positions, and the text they index stays with the layer.
|
|
34
|
+
*/
|
|
35
|
+
export class TokenStore {
|
|
36
|
+
#starts;
|
|
37
|
+
#ends;
|
|
38
|
+
#kinds;
|
|
39
|
+
#length = 0;
|
|
40
|
+
constructor(capacity = INITIAL_CAPACITY) {
|
|
41
|
+
const initial = Math.max(0, Math.trunc(capacity));
|
|
42
|
+
this.#starts = new Int32Array(initial);
|
|
43
|
+
this.#ends = new Int32Array(initial);
|
|
44
|
+
this.#kinds = new Uint8Array(initial);
|
|
45
|
+
}
|
|
46
|
+
/** How many tokens are stored. The arrays below are longer than this; they carry slack. */
|
|
47
|
+
get length() {
|
|
48
|
+
return this.#length;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Start offsets, in source order.
|
|
52
|
+
*
|
|
53
|
+
* A view over the live buffer, trimmed to `length`, for a consumer that wants to walk the
|
|
54
|
+
* positions without materialising anything. It is invalidated by the next {@link push}, which in
|
|
55
|
+
* practice means it is safe for as long as the layer is: nothing appends to a built layer.
|
|
56
|
+
*/
|
|
57
|
+
get starts() {
|
|
58
|
+
return this.#starts.subarray(0, this.#length);
|
|
59
|
+
}
|
|
60
|
+
/** End offsets, exclusive, in source order. A live view, like {@link starts}. */
|
|
61
|
+
get ends() {
|
|
62
|
+
return this.#ends.subarray(0, this.#length);
|
|
63
|
+
}
|
|
64
|
+
/** Kind codes, indices into the `TokenKind` union. A live view, like {@link starts}. */
|
|
65
|
+
get kinds() {
|
|
66
|
+
return this.#kinds.subarray(0, this.#length);
|
|
67
|
+
}
|
|
68
|
+
/** Append one token and return its index. Grows geometrically, so a scan stays linear. */
|
|
69
|
+
push(start, end, kind) {
|
|
70
|
+
const index = this.#length;
|
|
71
|
+
if (index === this.#starts.length)
|
|
72
|
+
this.#grow();
|
|
73
|
+
this.#starts[index] = start;
|
|
74
|
+
this.#ends[index] = end;
|
|
75
|
+
this.#kinds[index] = CODE_BY_KIND[kind];
|
|
76
|
+
this.#length = index + 1;
|
|
77
|
+
return index;
|
|
78
|
+
}
|
|
79
|
+
/** The start offset of one token. */
|
|
80
|
+
startAt(index) {
|
|
81
|
+
return this.#starts[this.#checked(index)];
|
|
82
|
+
}
|
|
83
|
+
/** The end offset of one token, exclusive. */
|
|
84
|
+
endAt(index) {
|
|
85
|
+
return this.#ends[this.#checked(index)];
|
|
86
|
+
}
|
|
87
|
+
/** The kind of one token. */
|
|
88
|
+
kindAt(index) {
|
|
89
|
+
return KIND_BY_CODE[this.#kinds[this.#checked(index)]];
|
|
90
|
+
}
|
|
91
|
+
/** The stored kind code of one token, for a consumer encoding kinds numerically. */
|
|
92
|
+
kindCodeAt(index) {
|
|
93
|
+
return this.#kinds[this.#checked(index)];
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Build a `Token` object for one stored token.
|
|
97
|
+
*
|
|
98
|
+
* The only place a token becomes an object. Called for the tokens a consumer actually reads, and
|
|
99
|
+
* never on the way in.
|
|
100
|
+
*/
|
|
101
|
+
materialise(index) {
|
|
102
|
+
const at = this.#checked(index);
|
|
103
|
+
return {
|
|
104
|
+
start: this.#starts[at],
|
|
105
|
+
end: this.#ends[at],
|
|
106
|
+
kind: KIND_BY_CODE[this.#kinds[at]],
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Every stored token, materialised one at a time.
|
|
111
|
+
*
|
|
112
|
+
* Convenient rather than cheap: it allocates per token, so a consumer colouring a viewport reads
|
|
113
|
+
* the arrays or goes through `classify`, which yields only what it is asked for. Note that this
|
|
114
|
+
* yields stored tokens alone, so the gaps between them are not covered; `trivia` is never stored
|
|
115
|
+
* and is computed as the complement.
|
|
116
|
+
*/
|
|
117
|
+
*[Symbol.iterator]() {
|
|
118
|
+
for (let index = 0; index < this.#length; index += 1)
|
|
119
|
+
yield this.materialise(index);
|
|
120
|
+
}
|
|
121
|
+
#checked(index) {
|
|
122
|
+
if (!Number.isInteger(index) || index < 0 || index >= this.#length) {
|
|
123
|
+
throw new RangeError(`No token at index ${index} (${this.#length} stored)`);
|
|
124
|
+
}
|
|
125
|
+
return index;
|
|
126
|
+
}
|
|
127
|
+
#grow() {
|
|
128
|
+
const capacity = this.#starts.length === 0 ? INITIAL_CAPACITY : this.#starts.length * 2;
|
|
129
|
+
const starts = new Int32Array(capacity);
|
|
130
|
+
const ends = new Int32Array(capacity);
|
|
131
|
+
const kinds = new Uint8Array(capacity);
|
|
132
|
+
starts.set(this.#starts);
|
|
133
|
+
ends.set(this.#ends);
|
|
134
|
+
kinds.set(this.#kinds);
|
|
135
|
+
this.#starts = starts;
|
|
136
|
+
this.#ends = ends;
|
|
137
|
+
this.#kinds = kinds;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=tokens.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/syntax/tokens.ts"],"names":[],"mappings":"AA6BA,0FAA0F;AAC1F,MAAM,YAAY,GAAyB;IACzC,UAAU;IACV,OAAO;IACP,WAAW;IACX,YAAY;IACZ,SAAS;IACT,QAAQ;CACT,CAAC;AAEF,MAAM,YAAY,GAAwC;IACxD,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;CACV,CAAC;AAEF,qGAAqG;AACrG,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,UAAU;IACrB,OAAO,CAAa;IACpB,KAAK,CAAa;IAClB,MAAM,CAAa;IACnB,OAAO,GAAG,CAAC,CAAC;IAEZ,YAAY,WAAmB,gBAAgB;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,2FAA2F;IAC3F,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,iFAAiF;IACjF,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,wFAAwF;IACxF,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,0FAA0F;IAC1F,IAAI,CAAC,KAAa,EAAE,GAAW,EAAE,IAAe;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3B,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAChD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC;QACzB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,qCAAqC;IACrC,OAAO,CAAC,KAAa;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC7C,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,KAAa;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC3C,CAAC;IAED,6BAA6B;IAC7B,MAAM,CAAC,KAAa;QAClB,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAE,CAAC;IAC3D,CAAC;IAED,oFAAoF;IACpF,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC5C,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,KAAa;QACvB,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE;YACxB,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAE;YACpB,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAE,CAAE;SACtC,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IACtF,CAAC;IAED,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,IAAI,UAAU,CAAC,qBAAqB,KAAK,KAAK,IAAI,CAAC,OAAO,UAAU,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@idfkit/core",
|
|
3
|
-
"version": "0.2.0
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Fast EnergyPlus IDF and epJSON parsing and manipulation for JavaScript",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -43,10 +43,10 @@
|
|
|
43
43
|
},
|
|
44
44
|
"idfkit": {
|
|
45
45
|
"conformance": "conformance-2026.8",
|
|
46
|
-
"governance": "governance-2026.
|
|
46
|
+
"governance": "governance-2026.12"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@idfkit/schemas": "0.2.0
|
|
49
|
+
"@idfkit/schemas": "0.2.0"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@idfkit/types-v26-1": "*",
|