@idfkit/core 0.2.0-rc.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conformance.d.ts +1 -1
- package/dist/conformance.js +1 -1
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/introspect/describe.d.ts +2 -2
- package/dist/introspect/describe.d.ts.map +1 -1
- package/dist/introspect/describe.js +62 -8
- package/dist/introspect/describe.js.map +1 -1
- package/dist/node.d.ts.map +1 -1
- package/dist/node.js +22 -2
- package/dist/node.js.map +1 -1
- package/dist/parse/idf.d.ts +21 -3
- package/dist/parse/idf.d.ts.map +1 -1
- package/dist/parse/idf.js +158 -5
- package/dist/parse/idf.js.map +1 -1
- package/dist/parse/lexer.d.ts +42 -13
- package/dist/parse/lexer.d.ts.map +1 -1
- package/dist/parse/lexer.js +65 -83
- package/dist/parse/lexer.js.map +1 -1
- package/dist/parse/scan.d.ts +34 -0
- package/dist/parse/scan.d.ts.map +1 -0
- package/dist/parse/scan.js +177 -0
- package/dist/parse/scan.js.map +1 -0
- package/dist/syntax/classify.d.ts +25 -0
- package/dist/syntax/classify.d.ts.map +1 -0
- package/dist/syntax/classify.js +59 -0
- package/dist/syntax/classify.js.map +1 -0
- package/dist/syntax/layer.d.ts +67 -0
- package/dist/syntax/layer.d.ts.map +1 -0
- package/dist/syntax/layer.js +134 -0
- package/dist/syntax/layer.js.map +1 -0
- package/dist/syntax/region.d.ts +73 -0
- package/dist/syntax/region.d.ts.map +1 -0
- package/dist/syntax/region.js +105 -0
- package/dist/syntax/region.js.map +1 -0
- package/dist/syntax/tokens.d.ts +80 -0
- package/dist/syntax/tokens.d.ts.map +1 -0
- package/dist/syntax/tokens.js +140 -0
- package/dist/syntax/tokens.js.map +1 -0
- package/dist/write/idf.d.ts +32 -0
- package/dist/write/idf.d.ts.map +1 -1
- package/dist/write/idf.js +24 -5
- package/dist/write/idf.js.map +1 -1
- package/package.json +4 -4
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { Region } from './region.js';
|
|
2
|
+
import { TokenStore } from './tokens.js';
|
|
3
|
+
/**
|
|
4
|
+
* One statement as the text writes it, terminated or not.
|
|
5
|
+
*
|
|
6
|
+
* Not an object in the model and carrying no schema meaning (FR-006): a statement whose type the
|
|
7
|
+
* schema never heard of, or which carries twice the fields its type defines, is still a statement.
|
|
8
|
+
* Deciding what any of it means is the reader's job, and the layer is what a reader positions
|
|
9
|
+
* against.
|
|
10
|
+
*/
|
|
11
|
+
export interface Statement {
|
|
12
|
+
/** The whole statement, from its first non-blank character through its terminator. */
|
|
13
|
+
readonly region: Region;
|
|
14
|
+
/** Just the type name. Empty, at the offset one would have begun, when none was written. */
|
|
15
|
+
readonly typeName: Region;
|
|
16
|
+
/** Type name text, exactly as written and not case-folded. */
|
|
17
|
+
readonly typeNameText: string;
|
|
18
|
+
/**
|
|
19
|
+
* Fields after the type name, in positional order, one region each.
|
|
20
|
+
*
|
|
21
|
+
* Index 0 is the first field after the type name, matching `RawObject.values`, which has already
|
|
22
|
+
* had the type name shifted off. A field written empty still gets a region, an empty one
|
|
23
|
+
* positioned where its value would have begun, so that positional indexing never shifts and a
|
|
24
|
+
* blank slot in the middle of an extensible group can still be pointed at.
|
|
25
|
+
*
|
|
26
|
+
* The count is what was written rather than what the type defines. A statement carrying more or
|
|
27
|
+
* fewer fields than its type is a finding, not a representation problem.
|
|
28
|
+
*/
|
|
29
|
+
readonly fields: readonly Region[];
|
|
30
|
+
/** True when no terminator was found, meaning the statement runs to end of input. */
|
|
31
|
+
readonly unterminated: boolean;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Everything the text contains and where it was.
|
|
35
|
+
*
|
|
36
|
+
* The layer holds the text it was built from, which is what makes byte-identical reconstruction a
|
|
37
|
+
* consequence rather than a feature: a reconstruction defined as concatenating slices of that text
|
|
38
|
+
* returns the text by construction. What can actually break is the tiling, so that is what the
|
|
39
|
+
* corpus test asserts and what {@link classify} makes observable.
|
|
40
|
+
*
|
|
41
|
+
* Nothing builds a layer implicitly (FR-005). `lex` and `parseIdf` read the same characters
|
|
42
|
+
* through the same scan and construct none of this, so a caller who never names `scanIdf` pays
|
|
43
|
+
* neither its time nor its memory.
|
|
44
|
+
*/
|
|
45
|
+
export interface SyntaxLayer {
|
|
46
|
+
/** The text this layer was built from. Held so regions can be resolved. */
|
|
47
|
+
readonly text: string;
|
|
48
|
+
/** Statements in source order. */
|
|
49
|
+
readonly statements: readonly Statement[];
|
|
50
|
+
/** Every meaningful token in source order, packed. Whitespace is not among them. */
|
|
51
|
+
readonly tokens: TokenStore;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Scan IDF text into a syntax layer.
|
|
55
|
+
*
|
|
56
|
+
* Text and nothing else: no schema, because the layer records what the text says and never what it
|
|
57
|
+
* means (FR-006). It never throws, for any input, and text that violates the grammar is
|
|
58
|
+
* represented rather than stopped at (FR-004): an unterminated final statement runs to end of
|
|
59
|
+
* input and says so, a statement written without a type name still tiles, and empty text produces
|
|
60
|
+
* a layer with no statements and no tokens, which satisfies the tiling invariant vacuously.
|
|
61
|
+
*
|
|
62
|
+
* One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a
|
|
63
|
+
* quarter over a plain read of the same text; the work above the scan is a push per token and a
|
|
64
|
+
* record per statement.
|
|
65
|
+
*/
|
|
66
|
+
export declare function scanIdf(text: string): SyntaxLayer;
|
|
67
|
+
//# sourceMappingURL=layer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"layer.d.ts","sourceRoot":"","sources":["../../src/syntax/layer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,sFAAsF;IACtF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,4FAA4F;IAC5F,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,qFAAqF;IACrF,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C,oFAAoF;IACpF,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC7B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAiHjD"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { scan } from '../parse/scan.js';
|
|
2
|
+
import { TokenStore } from './tokens.js';
|
|
3
|
+
/**
|
|
4
|
+
* Scan IDF text into a syntax layer.
|
|
5
|
+
*
|
|
6
|
+
* Text and nothing else: no schema, because the layer records what the text says and never what it
|
|
7
|
+
* means (FR-006). It never throws, for any input, and text that violates the grammar is
|
|
8
|
+
* represented rather than stopped at (FR-004): an unterminated final statement runs to end of
|
|
9
|
+
* input and says so, a statement written without a type name still tiles, and empty text produces
|
|
10
|
+
* a layer with no statements and no tokens, which satisfies the tiling invariant vacuously.
|
|
11
|
+
*
|
|
12
|
+
* One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a
|
|
13
|
+
* quarter over a plain read of the same text; the work above the scan is a push per token and a
|
|
14
|
+
* record per statement.
|
|
15
|
+
*/
|
|
16
|
+
export function scanIdf(text) {
|
|
17
|
+
const tokens = new TokenStore(initialCapacity(text.length));
|
|
18
|
+
const statements = [];
|
|
19
|
+
/**
|
|
20
|
+
* Comments seen since the last field closed, as flat `[start, end]` pairs in source order.
|
|
21
|
+
*
|
|
22
|
+
* They are buffered rather than pushed on sight because a comment can interrupt a field, in
|
|
23
|
+
* which case the scan reports it before the field it sits inside: the field's own bounds are
|
|
24
|
+
* only known once the field closes. Emitting them at that point, ordered against the value,
|
|
25
|
+
* is what keeps the store in source order.
|
|
26
|
+
*/
|
|
27
|
+
const comments = [];
|
|
28
|
+
/** Index into {@link comments} of the first pair not yet pushed. */
|
|
29
|
+
let nextComment = 0;
|
|
30
|
+
/** Raw text runs of the field being read, as flat `[start, end]` pairs in source order. */
|
|
31
|
+
const runs = [];
|
|
32
|
+
/** Offset the current statement opened at. */
|
|
33
|
+
let openedAt = 0;
|
|
34
|
+
/** The current statement's type name, replaced when its field closes. */
|
|
35
|
+
let typeName = EMPTY_REGION;
|
|
36
|
+
/** The current statement's fields. Reassigned per statement, so a pushed record keeps its own. */
|
|
37
|
+
let fields = [];
|
|
38
|
+
/** Push every buffered comment that begins before `offset`, keeping the store in source order. */
|
|
39
|
+
const flushComments = (offset) => {
|
|
40
|
+
while (nextComment < comments.length && comments[nextComment] < offset) {
|
|
41
|
+
tokens.push(comments[nextComment], comments[nextComment + 1], 'comment');
|
|
42
|
+
nextComment += 2;
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
scan(text, {
|
|
46
|
+
statementStart(offset) {
|
|
47
|
+
openedAt = offset;
|
|
48
|
+
fields = [];
|
|
49
|
+
},
|
|
50
|
+
/**
|
|
51
|
+
* The runs are recorded only for the sake of a field written on both sides of a comment that
|
|
52
|
+
* interrupts it. Every other reader of this scan wants the field's value, which arrives whole
|
|
53
|
+
* below.
|
|
54
|
+
*/
|
|
55
|
+
fieldText(start, end) {
|
|
56
|
+
runs.push(start, end);
|
|
57
|
+
},
|
|
58
|
+
fieldEnd(index, start, end) {
|
|
59
|
+
const region = { start, end };
|
|
60
|
+
if (index === 0)
|
|
61
|
+
typeName = region;
|
|
62
|
+
else
|
|
63
|
+
fields.push(region);
|
|
64
|
+
const kind = index === 0 ? 'typeName' : 'value';
|
|
65
|
+
flushComments(start);
|
|
66
|
+
// A field written empty has an empty region, which is a position rather than a span, so it
|
|
67
|
+
// is recorded above and is no token.
|
|
68
|
+
if (end > start)
|
|
69
|
+
tokens.push(start, end, kind);
|
|
70
|
+
// Only a comment can separate two runs of one field, so a run beginning at or after the
|
|
71
|
+
// value ends is text written after an interrupting comment: `A !- why\n B,` writes both `A`
|
|
72
|
+
// and `B` into one field. The scan bounds the value at `A` deliberately, so that a value
|
|
73
|
+
// region and a comment region can never overlap, which leaves `B` reached by no region at
|
|
74
|
+
// all, and a gap holding something other than whitespace is what the tiling invariant
|
|
75
|
+
// forbids. Every other run holds the value itself or the whitespace before it, and both
|
|
76
|
+
// begin before it ends, so a well-formed field never enters the body and never allocates.
|
|
77
|
+
for (let at = 0; at < runs.length; at += 2) {
|
|
78
|
+
const runStart = runs[at];
|
|
79
|
+
if (runStart < end)
|
|
80
|
+
continue;
|
|
81
|
+
const runEnd = runs[at + 1];
|
|
82
|
+
const raw = text.slice(runStart, runEnd);
|
|
83
|
+
const from = runStart + (raw.length - raw.trimStart().length);
|
|
84
|
+
const to = runEnd - (raw.length - raw.trimEnd().length);
|
|
85
|
+
if (from >= to)
|
|
86
|
+
continue;
|
|
87
|
+
flushComments(from);
|
|
88
|
+
tokens.push(from, to, kind);
|
|
89
|
+
}
|
|
90
|
+
flushComments(Infinity);
|
|
91
|
+
comments.length = 0;
|
|
92
|
+
nextComment = 0;
|
|
93
|
+
runs.length = 0;
|
|
94
|
+
},
|
|
95
|
+
separator(offset) {
|
|
96
|
+
tokens.push(offset, offset + 1, 'separator');
|
|
97
|
+
},
|
|
98
|
+
terminator(offset) {
|
|
99
|
+
tokens.push(offset, offset + 1, 'terminator');
|
|
100
|
+
},
|
|
101
|
+
comment(start, end) {
|
|
102
|
+
comments.push(start, end);
|
|
103
|
+
},
|
|
104
|
+
statementEnd(end, unterminated) {
|
|
105
|
+
statements.push({
|
|
106
|
+
region: { start: openedAt, end },
|
|
107
|
+
typeName,
|
|
108
|
+
typeNameText: text.slice(typeName.start, typeName.end),
|
|
109
|
+
fields,
|
|
110
|
+
unterminated,
|
|
111
|
+
});
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
// Comments after the last terminator close no field, so nothing has flushed them. Text that is
|
|
115
|
+
// only comments reaches here having reported no statement at all.
|
|
116
|
+
flushComments(Infinity);
|
|
117
|
+
return { text, statements, tokens };
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Stands in until a statement's first field closes, which it always does before the statement
|
|
121
|
+
* ends, so no statement is ever recorded holding this.
|
|
122
|
+
*/
|
|
123
|
+
const EMPTY_REGION = { start: 0, end: 0 };
|
|
124
|
+
/**
|
|
125
|
+
* A starting size for the token store, so a large file does not grow it a dozen times.
|
|
126
|
+
*
|
|
127
|
+
* A commented IDF line runs to about a dozen characters per token, so one per sixteen undershoots
|
|
128
|
+
* slightly and costs at most one doubling, which is the safer side to be wrong on: a file that is
|
|
129
|
+
* mostly whitespace or mostly comment would otherwise pay for capacity it never fills.
|
|
130
|
+
*/
|
|
131
|
+
function initialCapacity(length) {
|
|
132
|
+
return length < 1024 ? 64 : length >> 4;
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=layer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"layer.js","sourceRoot":"","sources":["../../src/syntax/layer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,OAAO,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAC;AAsDzD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAgB,EAAE,CAAC;IAEnC;;;;;;;OAOG;IACH,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,oEAAoE;IACpE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,2FAA2F;IAC3F,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,8CAA8C;IAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,yEAAyE;IACzE,IAAI,QAAQ,GAAW,YAAY,CAAC;IACpC,kGAAkG;IAClG,IAAI,MAAM,GAAa,EAAE,CAAC;IAE1B,kGAAkG;IAClG,MAAM,aAAa,GAAG,CAAC,MAAc,EAAQ,EAAE;QAC7C,OAAO,WAAW,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,WAAW,CAAE,GAAG,MAAM,EAAE,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAE,EAAE,QAAQ,CAAC,WAAW,GAAG,CAAC,CAAE,EAAE,SAAS,CAAC,CAAC;YAC3E,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,EAAE;QACT,cAAc,CAAC,MAAM;YACnB,QAAQ,GAAG,MAAM,CAAC;YAClB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED;;;;WAIG;QACH,SAAS,CAAC,KAAK,EAAE,GAAG;YAClB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG;YACxB,MAAM,MAAM,GAAW,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACtC,IAAI,KAAK,KAAK,CAAC;gBAAE,QAAQ,GAAG,MAAM,CAAC;;gBAC9B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEzB,MAAM,IAAI,GAAc,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;YAC3D,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,2FAA2F;YAC3F,qCAAqC;YACrC,IAAI,GAAG,GAAG,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YAE/C,wFAAwF;YACxF,4FAA4F;YAC5F,yFAAyF;YACzF,0FAA0F;YAC1F,sFAAsF;YACtF,wFAAwF;YACxF,0FAA0F;YAC1F,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAE,CAAC;gBAC3B,IAAI,QAAQ,GAAG,GAAG;oBAAE,SAAS;gBAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC9D,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC;gBACxD,IAAI,IAAI,IAAI,EAAE;oBAAE,SAAS;gBACzB,aAAa,CAAC,IAAI,CAAC,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,aAAa,CAAC,QAAQ,CAAC,CAAC;YAExB,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACpB,WAAW,GAAG,CAAC,CAAC;YAChB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,SAAS,CAAC,MAAM;YACd,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAC/C,CAAC;QAED,UAAU,CAAC,MAAM;YACf,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,CAAC,KAAK,EAAE,GAAG;YAChB,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;QAED,YAAY,CAAC,GAAG,EAAE,YAAY;YAC5B,UAAU,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE;gBAChC,QAAQ;gBACR,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;gBACtD,MAAM;gBACN,YAAY;aACb,CAAC,CAAC;QACL,CAAC;KACF,CAAC,CAAC;IAEH,+FAA+F;IAC/F,kEAAkE;IAClE,aAAa,CAAC,QAAQ,CAAC,CAAC;IAExB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,YAAY,GAAW,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;AAElD;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A span of source text, half-open.
|
|
3
|
+
*
|
|
4
|
+
* `end` is one past the last character, so `text.slice(start, end)` is exactly the text the
|
|
5
|
+
* region selects, an empty region is `start === end`, and two adjacent regions share a number
|
|
6
|
+
* rather than straddling one. An empty region is what a cursor between two characters is.
|
|
7
|
+
*
|
|
8
|
+
* Offsets are indices into the JavaScript string, which are UTF-16 code units. Nothing converts
|
|
9
|
+
* them, because no consumer wants them converted: the Language Server Protocol's default
|
|
10
|
+
* `positionEncoding` is UTF-16, Monaco measures columns in UTF-16 code units, and CodeMirror
|
|
11
|
+
* addresses its document by the same offsets. The Python library counts code points instead,
|
|
12
|
+
* because Python string indices are code points; the two agree for every character below the
|
|
13
|
+
* astral planes and differ for anything above them, which in practice means an emoji in a
|
|
14
|
+
* comment. That divergence is registered rather than papered over.
|
|
15
|
+
*/
|
|
16
|
+
export interface Region {
|
|
17
|
+
/** Offset of the first character, into the source string. */
|
|
18
|
+
readonly start: number;
|
|
19
|
+
/** Offset one past the last character. */
|
|
20
|
+
readonly end: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* A position as a human reads it, derived from an offset rather than stored.
|
|
24
|
+
*
|
|
25
|
+
* Storing line and column on every region would double its size to hold two numbers that are a
|
|
26
|
+
* function of one. Both count from one, which is what the existing findings already report; a
|
|
27
|
+
* consumer whose own convention counts a column from zero subtracts one, and that subtraction is
|
|
28
|
+
* the consumer's, because a service that guessed which convention its caller wanted would be
|
|
29
|
+
* modelling a protocol.
|
|
30
|
+
*/
|
|
31
|
+
export interface LineColumn {
|
|
32
|
+
/** 1-based, counting from the start of the text. */
|
|
33
|
+
readonly line: number;
|
|
34
|
+
/** 1-based, in UTF-16 code units, from the start of the line. */
|
|
35
|
+
readonly column: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The minimum a position query needs: the text its offsets index into.
|
|
39
|
+
*
|
|
40
|
+
* Declared structurally rather than by importing `SyntaxLayer`, so that positions stay usable
|
|
41
|
+
* without one and this module depends on nothing. A `SyntaxLayer` holds its text and therefore
|
|
42
|
+
* satisfies this, which is what lets `lineColumnAt(layer, offset)` read as the contract writes it.
|
|
43
|
+
*/
|
|
44
|
+
export interface TextSource {
|
|
45
|
+
/** The text offsets are measured into. */
|
|
46
|
+
readonly text: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The line and column an offset falls on.
|
|
50
|
+
*
|
|
51
|
+
* Binary search over the line index, so a query costs the logarithm of the line count rather than
|
|
52
|
+
* a scan of the text. An offset outside `[0, text.length]` is clamped into range rather than
|
|
53
|
+
* throwing: a cursor arrives from an editor that may be a keystroke ahead of the text this layer
|
|
54
|
+
* was built from, and refusing to answer is worse than answering about the nearest character.
|
|
55
|
+
*
|
|
56
|
+
* The column of a *statement* is the column of its first non-blank character, not of the
|
|
57
|
+
* whitespace indenting it. This function does not enforce that rule, because a statement's region
|
|
58
|
+
* already starts at that character; it is stated here because every column this feature reports
|
|
59
|
+
* obeys it, and because the conformance corpus compares findings on their line and type name and
|
|
60
|
+
* would notice the number moving.
|
|
61
|
+
*/
|
|
62
|
+
export declare function lineColumnAt(source: TextSource, offset: number): LineColumn;
|
|
63
|
+
/**
|
|
64
|
+
* The offset a line and column names.
|
|
65
|
+
*
|
|
66
|
+
* The inverse of {@link lineColumnAt} for every position that exists, and clamped for every one
|
|
67
|
+
* that does not: a line past the end resolves on the last line, a column past the end of its line
|
|
68
|
+
* resolves at the line's last position, which is the offset of the break that ends it. Round
|
|
69
|
+
* tripping an out-of-range position therefore returns the clamped position rather than the one
|
|
70
|
+
* asked for, which is the only honest answer available.
|
|
71
|
+
*/
|
|
72
|
+
export declare function offsetAt(source: TextSource, position: LineColumn): number;
|
|
73
|
+
//# sourceMappingURL=region.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"region.d.ts","sourceRoot":"","sources":["../../src/syntax/region.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,MAAM;IACrB,6DAA6D;IAC7D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,0CAA0C;IAC1C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAgCD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAK3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,GAAG,MAAM,CAUzE"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Line index per source, built once and reused.
|
|
3
|
+
*
|
|
4
|
+
* Keyed by the source object rather than by its text, because a `SyntaxLayer` is immutable and
|
|
5
|
+
* lives as long as the document does, while keying on the string itself would hold megabytes of
|
|
6
|
+
* text alive for the sake of a few kilobytes of offsets. A caller that passes a fresh object
|
|
7
|
+
* literal each time gets a fresh index each time, which is the documented cost of doing that.
|
|
8
|
+
*/
|
|
9
|
+
const lineIndexes = new WeakMap();
|
|
10
|
+
/**
|
|
11
|
+
* Offsets at which each line begins, one entry per line, ascending.
|
|
12
|
+
*
|
|
13
|
+
* A line break is a line feed, and nothing else. A carriage return before it belongs to the line
|
|
14
|
+
* it ends, so a column at the end of a CRLF line is one greater than the same column in a file
|
|
15
|
+
* using line feeds alone, which is what every editor reports. A lone carriage return is *not* a
|
|
16
|
+
* break: the existing lexer does not treat it as one and Python's `_line_and_column` counts line
|
|
17
|
+
* feeds, and the conformance corpus compares a finding on its line, so a third opinion here would
|
|
18
|
+
* put the two libraries one line apart on a file no gate would explain.
|
|
19
|
+
*
|
|
20
|
+
* @internal
|
|
21
|
+
*/
|
|
22
|
+
export function lineStartsOf(source) {
|
|
23
|
+
const cached = lineIndexes.get(source);
|
|
24
|
+
if (cached !== undefined)
|
|
25
|
+
return cached;
|
|
26
|
+
const starts = buildLineStarts(source.text);
|
|
27
|
+
lineIndexes.set(source, starts);
|
|
28
|
+
return starts;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The line and column an offset falls on.
|
|
32
|
+
*
|
|
33
|
+
* Binary search over the line index, so a query costs the logarithm of the line count rather than
|
|
34
|
+
* a scan of the text. An offset outside `[0, text.length]` is clamped into range rather than
|
|
35
|
+
* throwing: a cursor arrives from an editor that may be a keystroke ahead of the text this layer
|
|
36
|
+
* was built from, and refusing to answer is worse than answering about the nearest character.
|
|
37
|
+
*
|
|
38
|
+
* The column of a *statement* is the column of its first non-blank character, not of the
|
|
39
|
+
* whitespace indenting it. This function does not enforce that rule, because a statement's region
|
|
40
|
+
* already starts at that character; it is stated here because every column this feature reports
|
|
41
|
+
* obeys it, and because the conformance corpus compares findings on their line and type name and
|
|
42
|
+
* would notice the number moving.
|
|
43
|
+
*/
|
|
44
|
+
export function lineColumnAt(source, offset) {
|
|
45
|
+
const clamped = clampOffset(offset, source.text.length);
|
|
46
|
+
const starts = lineStartsOf(source);
|
|
47
|
+
const line = lineIndexAt(starts, clamped);
|
|
48
|
+
return { line: line + 1, column: clamped - starts[line] + 1 };
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The offset a line and column names.
|
|
52
|
+
*
|
|
53
|
+
* The inverse of {@link lineColumnAt} for every position that exists, and clamped for every one
|
|
54
|
+
* that does not: a line past the end resolves on the last line, a column past the end of its line
|
|
55
|
+
* resolves at the line's last position, which is the offset of the break that ends it. Round
|
|
56
|
+
* tripping an out-of-range position therefore returns the clamped position rather than the one
|
|
57
|
+
* asked for, which is the only honest answer available.
|
|
58
|
+
*/
|
|
59
|
+
export function offsetAt(source, position) {
|
|
60
|
+
const text = source.text;
|
|
61
|
+
const starts = lineStartsOf(source);
|
|
62
|
+
const line = clampOffset(position.line - 1, starts.length - 1);
|
|
63
|
+
const lineStart = starts[line];
|
|
64
|
+
// One past the last position on this line is where the next line starts, so the last position on
|
|
65
|
+
// it is one before that: the offset of the line feed, at which a cursor is still on this line.
|
|
66
|
+
const lineEnd = line + 1 < starts.length ? starts[line + 1] - 1 : text.length;
|
|
67
|
+
const offset = lineStart + clampOffset(position.column - 1, text.length);
|
|
68
|
+
return offset > lineEnd ? lineEnd : offset;
|
|
69
|
+
}
|
|
70
|
+
/** Two passes so the array is allocated once at its exact size; `indexOf` does the scanning. */
|
|
71
|
+
function buildLineStarts(text) {
|
|
72
|
+
let count = 1;
|
|
73
|
+
for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1))
|
|
74
|
+
count += 1;
|
|
75
|
+
const starts = new Int32Array(count);
|
|
76
|
+
let line = 1;
|
|
77
|
+
for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1)) {
|
|
78
|
+
starts[line] = at + 1;
|
|
79
|
+
line += 1;
|
|
80
|
+
}
|
|
81
|
+
return starts;
|
|
82
|
+
}
|
|
83
|
+
/** Index of the greatest line start that is at or before `offset`. */
|
|
84
|
+
function lineIndexAt(starts, offset) {
|
|
85
|
+
let low = 0;
|
|
86
|
+
let high = starts.length - 1;
|
|
87
|
+
while (low < high) {
|
|
88
|
+
const middle = (low + high + 1) >> 1;
|
|
89
|
+
if (starts[middle] <= offset)
|
|
90
|
+
low = middle;
|
|
91
|
+
else
|
|
92
|
+
high = middle - 1;
|
|
93
|
+
}
|
|
94
|
+
return low;
|
|
95
|
+
}
|
|
96
|
+
/** Into `[0, max]`, whole. `NaN` lands at 0, since no position is nearer than another. */
|
|
97
|
+
function clampOffset(value, max) {
|
|
98
|
+
if (Number.isNaN(value))
|
|
99
|
+
return 0;
|
|
100
|
+
const whole = Math.trunc(value);
|
|
101
|
+
if (whole < 0)
|
|
102
|
+
return 0;
|
|
103
|
+
return whole > max ? max : whole;
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=region.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"region.js","sourceRoot":"","sources":["../../src/syntax/region.ts"],"names":[],"mappings":"AAkDA;;;;;;;GAOG;AACH,MAAM,WAAW,GAAG,IAAI,OAAO,EAA0B,CAAC;AAE1D;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5C,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB,EAAE,MAAc;IAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAE,GAAG,CAAC,EAAE,CAAC;AACjE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAkB,EAAE,QAAoB;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC;IAChC,iGAAiG;IACjG,+FAA+F;IAC/F,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAC/E,MAAM,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AAC7C,CAAC;AAED,gGAAgG;AAChG,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;QAC7E,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC;IACZ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,sEAAsE;AACtE,SAAS,WAAW,CAAC,MAAkB,EAAE,MAAc;IACrD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC7B,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,CAAE,IAAI,MAAM;YAAE,GAAG,GAAG,MAAM,CAAC;;YACvC,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,0FAA0F;AAC1F,SAAS,WAAW,CAAC,KAAa,EAAE,GAAW;IAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AACnC,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { Region } from './region.js';
|
|
2
|
+
/**
|
|
3
|
+
* What a span of text is, grammatically. Nothing here needs a schema: the layer records what the
|
|
4
|
+
* text says and never what it means, so a `value` is a value whether or not the field exists.
|
|
5
|
+
*/
|
|
6
|
+
export type TokenKind = 'typeName' | 'value' | 'separator' | 'terminator' | 'comment' | 'trivia';
|
|
7
|
+
/**
|
|
8
|
+
* One token, materialised.
|
|
9
|
+
*
|
|
10
|
+
* A `Token` *is* a `Region` rather than carrying one, which keeps a classified stream to one
|
|
11
|
+
* object per token instead of two. On a file of forty thousand lines that difference is the
|
|
12
|
+
* difference between an editor that repaints and one that stutters, and a token still passes
|
|
13
|
+
* anywhere a region is expected.
|
|
14
|
+
*
|
|
15
|
+
* A materialised token never crosses a line boundary; a stored `value` region may, because the
|
|
16
|
+
* format lets a field be written across two lines, and the classification view splits it.
|
|
17
|
+
*/
|
|
18
|
+
export interface Token extends Region {
|
|
19
|
+
readonly kind: TokenKind;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Every token of one text, packed into three parallel arrays.
|
|
23
|
+
*
|
|
24
|
+
* Nine bytes per token, against upwards of forty for an object: a start, an end, and a kind code,
|
|
25
|
+
* held in `Int32Array`, `Int32Array` and `Uint8Array`. On a reference model of ten thousand
|
|
26
|
+
* statements that is roughly 3.6 MB rather than upwards of 16 MB.
|
|
27
|
+
*
|
|
28
|
+
* This is a contract rather than an implementation note. The intended use is an editor holding
|
|
29
|
+
* several large files open at once, and a layer that allocated an object per token would pass
|
|
30
|
+
* every correctness test while being unusable there. So `Token` objects are materialised only when
|
|
31
|
+
* a caller asks for one, which for classification means only for the tokens actually drawn.
|
|
32
|
+
*
|
|
33
|
+
* The store knows nothing of the scanner that fills it or of the layer that holds it, and nothing
|
|
34
|
+
* of the text either: it holds positions, and the text they index stays with the layer.
|
|
35
|
+
*/
|
|
36
|
+
export declare class TokenStore implements Iterable<Token> {
|
|
37
|
+
#private;
|
|
38
|
+
constructor(capacity?: number);
|
|
39
|
+
/** How many tokens are stored. The arrays below are longer than this; they carry slack. */
|
|
40
|
+
get length(): number;
|
|
41
|
+
/**
|
|
42
|
+
* Start offsets, in source order.
|
|
43
|
+
*
|
|
44
|
+
* A view over the live buffer, trimmed to `length`, for a consumer that wants to walk the
|
|
45
|
+
* positions without materialising anything. It is invalidated by the next {@link push}, which in
|
|
46
|
+
* practice means it is safe for as long as the layer is: nothing appends to a built layer.
|
|
47
|
+
*/
|
|
48
|
+
get starts(): Int32Array;
|
|
49
|
+
/** End offsets, exclusive, in source order. A live view, like {@link starts}. */
|
|
50
|
+
get ends(): Int32Array;
|
|
51
|
+
/** Kind codes, indices into the `TokenKind` union. A live view, like {@link starts}. */
|
|
52
|
+
get kinds(): Uint8Array;
|
|
53
|
+
/** Append one token and return its index. Grows geometrically, so a scan stays linear. */
|
|
54
|
+
push(start: number, end: number, kind: TokenKind): number;
|
|
55
|
+
/** The start offset of one token. */
|
|
56
|
+
startAt(index: number): number;
|
|
57
|
+
/** The end offset of one token, exclusive. */
|
|
58
|
+
endAt(index: number): number;
|
|
59
|
+
/** The kind of one token. */
|
|
60
|
+
kindAt(index: number): TokenKind;
|
|
61
|
+
/** The stored kind code of one token, for a consumer encoding kinds numerically. */
|
|
62
|
+
kindCodeAt(index: number): number;
|
|
63
|
+
/**
|
|
64
|
+
* Build a `Token` object for one stored token.
|
|
65
|
+
*
|
|
66
|
+
* The only place a token becomes an object. Called for the tokens a consumer actually reads, and
|
|
67
|
+
* never on the way in.
|
|
68
|
+
*/
|
|
69
|
+
materialise(index: number): Token;
|
|
70
|
+
/**
|
|
71
|
+
* Every stored token, materialised one at a time.
|
|
72
|
+
*
|
|
73
|
+
* Convenient rather than cheap: it allocates per token, so a consumer colouring a viewport reads
|
|
74
|
+
* the arrays or goes through `classify`, which yields only what it is asked for. Note that this
|
|
75
|
+
* yields stored tokens alone, so the gaps between them are not covered; `trivia` is never stored
|
|
76
|
+
* and is computed as the complement.
|
|
77
|
+
*/
|
|
78
|
+
[Symbol.iterator](): IterableIterator<Token>;
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=tokens.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/syntax/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,SAAS,GACjB,UAAU,GACV,OAAO,GACP,WAAW,GACX,YAAY,GACZ,SAAS,GACT,QAAQ,CAAC;AAEb;;;;;;;;;;GAUG;AACH,MAAM,WAAW,KAAM,SAAQ,MAAM;IACnC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;CAC1B;AAwBD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,UAAW,YAAW,QAAQ,CAAC,KAAK,CAAC;;gBAMpC,QAAQ,GAAE,MAAyB;IAO/C,2FAA2F;IAC3F,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;;;;OAMG;IACH,IAAI,MAAM,IAAI,UAAU,CAEvB;IAED,iFAAiF;IACjF,IAAI,IAAI,IAAI,UAAU,CAErB;IAED,wFAAwF;IACxF,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,0FAA0F;IAC1F,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,MAAM;IAUzD,qCAAqC;IACrC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI9B,8CAA8C;IAC9C,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI5B,6BAA6B;IAC7B,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAIhC,oFAAoF;IACpF,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIjC;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK;IASjC;;;;;;;OAOG;IACF,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC;CAuB9C"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/** Kind codes, in the order the union declares them. The index is what `kinds` stores. */
|
|
2
|
+
const KIND_BY_CODE = [
|
|
3
|
+
'typeName',
|
|
4
|
+
'value',
|
|
5
|
+
'separator',
|
|
6
|
+
'terminator',
|
|
7
|
+
'comment',
|
|
8
|
+
'trivia',
|
|
9
|
+
];
|
|
10
|
+
const CODE_BY_KIND = {
|
|
11
|
+
typeName: 0,
|
|
12
|
+
value: 1,
|
|
13
|
+
separator: 2,
|
|
14
|
+
terminator: 3,
|
|
15
|
+
comment: 4,
|
|
16
|
+
trivia: 5,
|
|
17
|
+
};
|
|
18
|
+
/** Where growth starts. Small enough for a one-statement file, large enough to skip early copies. */
|
|
19
|
+
const INITIAL_CAPACITY = 64;
|
|
20
|
+
/**
|
|
21
|
+
* Every token of one text, packed into three parallel arrays.
|
|
22
|
+
*
|
|
23
|
+
* Nine bytes per token, against upwards of forty for an object: a start, an end, and a kind code,
|
|
24
|
+
* held in `Int32Array`, `Int32Array` and `Uint8Array`. On a reference model of ten thousand
|
|
25
|
+
* statements that is roughly 3.6 MB rather than upwards of 16 MB.
|
|
26
|
+
*
|
|
27
|
+
* This is a contract rather than an implementation note. The intended use is an editor holding
|
|
28
|
+
* several large files open at once, and a layer that allocated an object per token would pass
|
|
29
|
+
* every correctness test while being unusable there. So `Token` objects are materialised only when
|
|
30
|
+
* a caller asks for one, which for classification means only for the tokens actually drawn.
|
|
31
|
+
*
|
|
32
|
+
* The store knows nothing of the scanner that fills it or of the layer that holds it, and nothing
|
|
33
|
+
* of the text either: it holds positions, and the text they index stays with the layer.
|
|
34
|
+
*/
|
|
35
|
+
export class TokenStore {
|
|
36
|
+
#starts;
|
|
37
|
+
#ends;
|
|
38
|
+
#kinds;
|
|
39
|
+
#length = 0;
|
|
40
|
+
constructor(capacity = INITIAL_CAPACITY) {
|
|
41
|
+
const initial = Math.max(0, Math.trunc(capacity));
|
|
42
|
+
this.#starts = new Int32Array(initial);
|
|
43
|
+
this.#ends = new Int32Array(initial);
|
|
44
|
+
this.#kinds = new Uint8Array(initial);
|
|
45
|
+
}
|
|
46
|
+
/** How many tokens are stored. The arrays below are longer than this; they carry slack. */
|
|
47
|
+
get length() {
|
|
48
|
+
return this.#length;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Start offsets, in source order.
|
|
52
|
+
*
|
|
53
|
+
* A view over the live buffer, trimmed to `length`, for a consumer that wants to walk the
|
|
54
|
+
* positions without materialising anything. It is invalidated by the next {@link push}, which in
|
|
55
|
+
* practice means it is safe for as long as the layer is: nothing appends to a built layer.
|
|
56
|
+
*/
|
|
57
|
+
get starts() {
|
|
58
|
+
return this.#starts.subarray(0, this.#length);
|
|
59
|
+
}
|
|
60
|
+
/** End offsets, exclusive, in source order. A live view, like {@link starts}. */
|
|
61
|
+
get ends() {
|
|
62
|
+
return this.#ends.subarray(0, this.#length);
|
|
63
|
+
}
|
|
64
|
+
/** Kind codes, indices into the `TokenKind` union. A live view, like {@link starts}. */
|
|
65
|
+
get kinds() {
|
|
66
|
+
return this.#kinds.subarray(0, this.#length);
|
|
67
|
+
}
|
|
68
|
+
/** Append one token and return its index. Grows geometrically, so a scan stays linear. */
|
|
69
|
+
push(start, end, kind) {
|
|
70
|
+
const index = this.#length;
|
|
71
|
+
if (index === this.#starts.length)
|
|
72
|
+
this.#grow();
|
|
73
|
+
this.#starts[index] = start;
|
|
74
|
+
this.#ends[index] = end;
|
|
75
|
+
this.#kinds[index] = CODE_BY_KIND[kind];
|
|
76
|
+
this.#length = index + 1;
|
|
77
|
+
return index;
|
|
78
|
+
}
|
|
79
|
+
/** The start offset of one token. */
|
|
80
|
+
startAt(index) {
|
|
81
|
+
return this.#starts[this.#checked(index)];
|
|
82
|
+
}
|
|
83
|
+
/** The end offset of one token, exclusive. */
|
|
84
|
+
endAt(index) {
|
|
85
|
+
return this.#ends[this.#checked(index)];
|
|
86
|
+
}
|
|
87
|
+
/** The kind of one token. */
|
|
88
|
+
kindAt(index) {
|
|
89
|
+
return KIND_BY_CODE[this.#kinds[this.#checked(index)]];
|
|
90
|
+
}
|
|
91
|
+
/** The stored kind code of one token, for a consumer encoding kinds numerically. */
|
|
92
|
+
kindCodeAt(index) {
|
|
93
|
+
return this.#kinds[this.#checked(index)];
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Build a `Token` object for one stored token.
|
|
97
|
+
*
|
|
98
|
+
* The only place a token becomes an object. Called for the tokens a consumer actually reads, and
|
|
99
|
+
* never on the way in.
|
|
100
|
+
*/
|
|
101
|
+
materialise(index) {
|
|
102
|
+
const at = this.#checked(index);
|
|
103
|
+
return {
|
|
104
|
+
start: this.#starts[at],
|
|
105
|
+
end: this.#ends[at],
|
|
106
|
+
kind: KIND_BY_CODE[this.#kinds[at]],
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Every stored token, materialised one at a time.
|
|
111
|
+
*
|
|
112
|
+
* Convenient rather than cheap: it allocates per token, so a consumer colouring a viewport reads
|
|
113
|
+
* the arrays or goes through `classify`, which yields only what it is asked for. Note that this
|
|
114
|
+
* yields stored tokens alone, so the gaps between them are not covered; `trivia` is never stored
|
|
115
|
+
* and is computed as the complement.
|
|
116
|
+
*/
|
|
117
|
+
*[Symbol.iterator]() {
|
|
118
|
+
for (let index = 0; index < this.#length; index += 1)
|
|
119
|
+
yield this.materialise(index);
|
|
120
|
+
}
|
|
121
|
+
#checked(index) {
|
|
122
|
+
if (!Number.isInteger(index) || index < 0 || index >= this.#length) {
|
|
123
|
+
throw new RangeError(`No token at index ${index} (${this.#length} stored)`);
|
|
124
|
+
}
|
|
125
|
+
return index;
|
|
126
|
+
}
|
|
127
|
+
#grow() {
|
|
128
|
+
const capacity = this.#starts.length === 0 ? INITIAL_CAPACITY : this.#starts.length * 2;
|
|
129
|
+
const starts = new Int32Array(capacity);
|
|
130
|
+
const ends = new Int32Array(capacity);
|
|
131
|
+
const kinds = new Uint8Array(capacity);
|
|
132
|
+
starts.set(this.#starts);
|
|
133
|
+
ends.set(this.#ends);
|
|
134
|
+
kinds.set(this.#kinds);
|
|
135
|
+
this.#starts = starts;
|
|
136
|
+
this.#ends = ends;
|
|
137
|
+
this.#kinds = kinds;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=tokens.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/syntax/tokens.ts"],"names":[],"mappings":"AA6BA,0FAA0F;AAC1F,MAAM,YAAY,GAAyB;IACzC,UAAU;IACV,OAAO;IACP,WAAW;IACX,YAAY;IACZ,SAAS;IACT,QAAQ;CACT,CAAC;AAEF,MAAM,YAAY,GAAwC;IACxD,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;CACV,CAAC;AAEF,qGAAqG;AACrG,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,UAAU;IACrB,OAAO,CAAa;IACpB,KAAK,CAAa;IAClB,MAAM,CAAa;IACnB,OAAO,GAAG,CAAC,CAAC;IAEZ,YAAY,WAAmB,gBAAgB;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,2FAA2F;IAC3F,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,iFAAiF;IACjF,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,wFAAwF;IACxF,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,0FAA0F;IAC1F,IAAI,CAAC,KAAa,EAAE,GAAW,EAAE,IAAe;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3B,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAChD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC;QACzB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,qCAAqC;IACrC,OAAO,CAAC,KAAa;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC7C,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,KAAa;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC3C,CAAC;IAED,6BAA6B;IAC7B,MAAM,CAAC,KAAa;QAClB,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAE,CAAC;IAC3D,CAAC;IAED,oFAAoF;IACpF,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC5C,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,KAAa;QACvB,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE;YACxB,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAE;YACpB,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAE,CAAE;SACtC,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IACtF,CAAC;IAED,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,IAAI,UAAU,CAAC,qBAAqB,KAAK,KAAK,IAAI,CAAC,OAAO,UAAU,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;CACF"}
|