@idfkit/core 0.2.0-rc.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/conformance.d.ts +1 -1
  2. package/dist/conformance.js +1 -1
  3. package/dist/index.d.ts +7 -1
  4. package/dist/index.d.ts.map +1 -1
  5. package/dist/index.js +3 -0
  6. package/dist/index.js.map +1 -1
  7. package/dist/introspect/describe.d.ts +2 -2
  8. package/dist/introspect/describe.d.ts.map +1 -1
  9. package/dist/introspect/describe.js +62 -8
  10. package/dist/introspect/describe.js.map +1 -1
  11. package/dist/node.d.ts.map +1 -1
  12. package/dist/node.js +22 -2
  13. package/dist/node.js.map +1 -1
  14. package/dist/parse/idf.d.ts +21 -3
  15. package/dist/parse/idf.d.ts.map +1 -1
  16. package/dist/parse/idf.js +158 -5
  17. package/dist/parse/idf.js.map +1 -1
  18. package/dist/parse/lexer.d.ts +42 -13
  19. package/dist/parse/lexer.d.ts.map +1 -1
  20. package/dist/parse/lexer.js +65 -83
  21. package/dist/parse/lexer.js.map +1 -1
  22. package/dist/parse/scan.d.ts +34 -0
  23. package/dist/parse/scan.d.ts.map +1 -0
  24. package/dist/parse/scan.js +177 -0
  25. package/dist/parse/scan.js.map +1 -0
  26. package/dist/syntax/classify.d.ts +25 -0
  27. package/dist/syntax/classify.d.ts.map +1 -0
  28. package/dist/syntax/classify.js +59 -0
  29. package/dist/syntax/classify.js.map +1 -0
  30. package/dist/syntax/layer.d.ts +67 -0
  31. package/dist/syntax/layer.d.ts.map +1 -0
  32. package/dist/syntax/layer.js +134 -0
  33. package/dist/syntax/layer.js.map +1 -0
  34. package/dist/syntax/region.d.ts +73 -0
  35. package/dist/syntax/region.d.ts.map +1 -0
  36. package/dist/syntax/region.js +105 -0
  37. package/dist/syntax/region.js.map +1 -0
  38. package/dist/syntax/tokens.d.ts +80 -0
  39. package/dist/syntax/tokens.d.ts.map +1 -0
  40. package/dist/syntax/tokens.js +140 -0
  41. package/dist/syntax/tokens.js.map +1 -0
  42. package/dist/write/idf.d.ts +32 -0
  43. package/dist/write/idf.d.ts.map +1 -1
  44. package/dist/write/idf.js +24 -5
  45. package/dist/write/idf.js.map +1 -1
  46. package/package.json +4 -4
@@ -0,0 +1,67 @@
1
+ import type { Region } from './region.js';
2
+ import { TokenStore } from './tokens.js';
3
+ /**
4
+ * One statement as the text writes it, terminated or not.
5
+ *
6
+ * Not an object in the model and carrying no schema meaning (FR-006): a statement whose type the
7
+ * schema never heard of, or which carries twice the fields its type defines, is still a statement.
8
+ * Deciding what any of it means is the reader's job, and the layer is what a reader positions
9
+ * against.
10
+ */
11
+ export interface Statement {
12
+ /** The whole statement, from its first non-blank character through its terminator. */
13
+ readonly region: Region;
14
+ /** Just the type name. Empty, at the offset one would have begun, when none was written. */
15
+ readonly typeName: Region;
16
+ /** Type name text, exactly as written and not case-folded. */
17
+ readonly typeNameText: string;
18
+ /**
19
+ * Fields after the type name, in positional order, one region each.
20
+ *
21
+ * Index 0 is the first field after the type name, matching `RawObject.values`, which has already
22
+ * had the type name shifted off. A field written empty still gets a region, an empty one
23
+ * positioned where its value would have begun, so that positional indexing never shifts and a
24
+ * blank slot in the middle of an extensible group can still be pointed at.
25
+ *
26
+ * The count is what was written rather than what the type defines. A statement carrying more or
27
+ * fewer fields than its type is a finding, not a representation problem.
28
+ */
29
+ readonly fields: readonly Region[];
30
+ /** True when no terminator was found, meaning the statement runs to end of input. */
31
+ readonly unterminated: boolean;
32
+ }
33
+ /**
34
+ * Everything the text contains and where it was.
35
+ *
36
+ * The layer holds the text it was built from, which is what makes byte-identical reconstruction a
37
+ * consequence rather than a feature: a reconstruction defined as concatenating slices of that text
38
+ * returns the text by construction. What can actually break is the tiling, so that is what the
39
+ * corpus test asserts and what {@link classify} makes observable.
40
+ *
41
+ * Nothing builds a layer implicitly (FR-005). `lex` and `parseIdf` read the same characters
42
+ * through the same scan and construct none of this, so a caller who never names `scanIdf` pays
43
+ * neither its time nor its memory.
44
+ */
45
+ export interface SyntaxLayer {
46
+ /** The text this layer was built from. Held so regions can be resolved. */
47
+ readonly text: string;
48
+ /** Statements in source order. */
49
+ readonly statements: readonly Statement[];
50
+ /** Every meaningful token in source order, packed. Whitespace is not among them. */
51
+ readonly tokens: TokenStore;
52
+ }
53
+ /**
54
+ * Scan IDF text into a syntax layer.
55
+ *
56
+ * Text and nothing else: no schema, because the layer records what the text says and never what it
57
+ * means (FR-006). It never throws, for any input, and text that violates the grammar is
58
+ * represented rather than stopped at (FR-004): an unterminated final statement runs to end of
59
+ * input and says so, a statement written without a type name still tiles, and empty text produces
60
+ * a layer with no statements and no tokens, which satisfies the tiling invariant vacuously.
61
+ *
62
+ * One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a
63
+ * quarter over a plain read of the same text; the work above the scan is a push per token and a
64
+ * record per statement.
65
+ */
66
+ export declare function scanIdf(text: string): SyntaxLayer;
67
+ //# sourceMappingURL=layer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"layer.d.ts","sourceRoot":"","sources":["../../src/syntax/layer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,sFAAsF;IACtF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,4FAA4F;IAC5F,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,qFAAqF;IACrF,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C,oFAAoF;IACpF,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC7B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAiHjD"}
@@ -0,0 +1,134 @@
1
+ import { scan } from '../parse/scan.js';
2
+ import { TokenStore } from './tokens.js';
3
+ /**
4
+ * Scan IDF text into a syntax layer.
5
+ *
6
+ * Text and nothing else: no schema, because the layer records what the text says and never what it
7
+ * means (FR-006). It never throws, for any input, and text that violates the grammar is
8
+ * represented rather than stopped at (FR-004): an unterminated final statement runs to end of
9
+ * input and says so, a statement written without a type name still tiles, and empty text produces
10
+ * a layer with no statements and no tokens, which satisfies the tiling invariant vacuously.
11
+ *
12
+ * One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a
13
+ * quarter over a plain read of the same text; the work above the scan is a push per token and a
14
+ * record per statement.
15
+ */
16
+ export function scanIdf(text) {
17
+ const tokens = new TokenStore(initialCapacity(text.length));
18
+ const statements = [];
19
+ /**
20
+ * Comments seen since the last field closed, as flat `[start, end]` pairs in source order.
21
+ *
22
+ * They are buffered rather than pushed on sight because a comment can interrupt a field, in
23
+ * which case the scan reports it before the field it sits inside: the field's own bounds are
24
+ * only known once the field closes. Emitting them at that point, ordered against the value,
25
+ * is what keeps the store in source order.
26
+ */
27
+ const comments = [];
28
+ /** Index into {@link comments} of the first pair not yet pushed. */
29
+ let nextComment = 0;
30
+ /** Raw text runs of the field being read, as flat `[start, end]` pairs in source order. */
31
+ const runs = [];
32
+ /** Offset the current statement opened at. */
33
+ let openedAt = 0;
34
+ /** The current statement's type name, replaced when its field closes. */
35
+ let typeName = EMPTY_REGION;
36
+ /** The current statement's fields. Reassigned per statement, so a pushed record keeps its own. */
37
+ let fields = [];
38
+ /** Push every buffered comment that begins before `offset`, keeping the store in source order. */
39
+ const flushComments = (offset) => {
40
+ while (nextComment < comments.length && comments[nextComment] < offset) {
41
+ tokens.push(comments[nextComment], comments[nextComment + 1], 'comment');
42
+ nextComment += 2;
43
+ }
44
+ };
45
+ scan(text, {
46
+ statementStart(offset) {
47
+ openedAt = offset;
48
+ fields = [];
49
+ },
50
+ /**
51
+ * The runs are recorded only for the sake of a field written on both sides of a comment that
52
+ * interrupts it. Every other reader of this scan wants the field's value, which arrives whole
53
+ * below.
54
+ */
55
+ fieldText(start, end) {
56
+ runs.push(start, end);
57
+ },
58
+ fieldEnd(index, start, end) {
59
+ const region = { start, end };
60
+ if (index === 0)
61
+ typeName = region;
62
+ else
63
+ fields.push(region);
64
+ const kind = index === 0 ? 'typeName' : 'value';
65
+ flushComments(start);
66
+ // A field written empty has an empty region, which is a position rather than a span, so it
67
+ // is recorded above and is no token.
68
+ if (end > start)
69
+ tokens.push(start, end, kind);
70
+ // Only a comment can separate two runs of one field, so a run beginning at or after the
71
+ // value ends is text written after an interrupting comment: `A !- why\n B,` writes both `A`
72
+ // and `B` into one field. The scan bounds the value at `A` deliberately, so that a value
73
+ // region and a comment region can never overlap, which leaves `B` reached by no region at
74
+ // all, and a gap holding something other than whitespace is what the tiling invariant
75
+ // forbids. Every other run holds the value itself or the whitespace before it, and both
76
+ // begin before it ends, so a well-formed field never enters the body and never allocates.
77
+ for (let at = 0; at < runs.length; at += 2) {
78
+ const runStart = runs[at];
79
+ if (runStart < end)
80
+ continue;
81
+ const runEnd = runs[at + 1];
82
+ const raw = text.slice(runStart, runEnd);
83
+ const from = runStart + (raw.length - raw.trimStart().length);
84
+ const to = runEnd - (raw.length - raw.trimEnd().length);
85
+ if (from >= to)
86
+ continue;
87
+ flushComments(from);
88
+ tokens.push(from, to, kind);
89
+ }
90
+ flushComments(Infinity);
91
+ comments.length = 0;
92
+ nextComment = 0;
93
+ runs.length = 0;
94
+ },
95
+ separator(offset) {
96
+ tokens.push(offset, offset + 1, 'separator');
97
+ },
98
+ terminator(offset) {
99
+ tokens.push(offset, offset + 1, 'terminator');
100
+ },
101
+ comment(start, end) {
102
+ comments.push(start, end);
103
+ },
104
+ statementEnd(end, unterminated) {
105
+ statements.push({
106
+ region: { start: openedAt, end },
107
+ typeName,
108
+ typeNameText: text.slice(typeName.start, typeName.end),
109
+ fields,
110
+ unterminated,
111
+ });
112
+ },
113
+ });
114
+ // Comments after the last terminator close no field, so nothing has flushed them. Text that is
115
+ // only comments reaches here having reported no statement at all.
116
+ flushComments(Infinity);
117
+ return { text, statements, tokens };
118
+ }
119
+ /**
120
+ * Stands in until a statement's first field closes, which it always does before the statement
121
+ * ends, so no statement is ever recorded holding this.
122
+ */
123
+ const EMPTY_REGION = { start: 0, end: 0 };
124
+ /**
125
+ * A starting size for the token store, so a large file does not grow it a dozen times.
126
+ *
127
+ * A commented IDF line runs to about a dozen characters per token, so one per sixteen undershoots
128
+ * slightly and costs at most one doubling, which is the safer side to be wrong on: a file that is
129
+ * mostly whitespace or mostly comment would otherwise pay for capacity it never fills.
130
+ */
131
+ function initialCapacity(length) {
132
+ return length < 1024 ? 64 : length >> 4;
133
+ }
134
+ //# sourceMappingURL=layer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"layer.js","sourceRoot":"","sources":["../../src/syntax/layer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,OAAO,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAC;AAsDzD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAgB,EAAE,CAAC;IAEnC;;;;;;;OAOG;IACH,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,oEAAoE;IACpE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,2FAA2F;IAC3F,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,8CAA8C;IAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,yEAAyE;IACzE,IAAI,QAAQ,GAAW,YAAY,CAAC;IACpC,kGAAkG;IAClG,IAAI,MAAM,GAAa,EAAE,CAAC;IAE1B,kGAAkG;IAClG,MAAM,aAAa,GAAG,CAAC,MAAc,EAAQ,EAAE;QAC7C,OAAO,WAAW,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,WAAW,CAAE,GAAG,MAAM,EAAE,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAE,EAAE,QAAQ,CAAC,WAAW,GAAG,CAAC,CAAE,EAAE,SAAS,CAAC,CAAC;YAC3E,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,EAAE;QACT,cAAc,CAAC,MAAM;YACnB,QAAQ,GAAG,MAAM,CAAC;YAClB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED;;;;WAIG;QACH,SAAS,CAAC,KAAK,EAAE,GAAG;YAClB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG;YACxB,MAAM,MAAM,GAAW,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACtC,IAAI,KAAK,KAAK,CAAC;gBAAE,QAAQ,GAAG,MAAM,CAAC;;gBAC9B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEzB,MAAM,IAAI,GAAc,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;YAC3D,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,2FAA2F;YAC3F,qCAAqC;YACrC,IAAI,GAAG,GAAG,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YAE/C,wFAAwF;YACxF,4FAA4F;YAC5F,yFAAyF;YACzF,0FAA0F;YAC1F,sFAAsF;YACtF,wFAAwF;YACxF,0FAA0F;YAC1F,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAE,CAAC;gBAC3B,IAAI,QAAQ,GAAG,GAAG;oBAAE,SAAS;gBAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC9D,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC;gBACxD,IAAI,IAAI,IAAI,EAAE;oBAAE,SAAS;gBACzB,aAAa,CAAC,IAAI,CAAC,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,aAAa,CAAC,QAAQ,CAAC,CAAC;YAExB,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACpB,WAAW,GAAG,CAAC,CAAC;YAChB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,SAAS,CAAC,MAAM;YACd,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAC/C,CAAC;QAED,UAAU,CAAC,MAAM;YACf,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,CAAC,KAAK,EAAE,GAAG;YAChB,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;QAED,YAAY,CAAC,GAAG,EAAE,YAAY;YAC5B,UAAU,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE;gBAChC,QAAQ;gBACR,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;gBACtD,MAAM;gBACN,YAAY;aACb,CAAC,CAAC;QACL,CAAC;KACF,CAAC,CAAC;IAEH,+FAA+F;IAC/F,kEAAkE;IAClE,aAAa,CAAC,QAAQ,CAAC,CAAC;IAExB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,YAAY,GAAW,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;AAElD;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;AAC1C,CAAC"}
@@ -0,0 +1,73 @@
1
+ /**
2
+ * A span of source text, half-open.
3
+ *
4
+ * `end` is one past the last character, so `text.slice(start, end)` is exactly the text the
5
+ * region selects, an empty region is `start === end`, and two adjacent regions share a number
6
+ * rather than straddling one. An empty region is what a cursor between two characters is.
7
+ *
8
+ * Offsets are indices into the JavaScript string, which are UTF-16 code units. Nothing converts
9
+ * them, because no consumer wants them converted: the Language Server Protocol's default
10
+ * `positionEncoding` is UTF-16, Monaco measures columns in UTF-16 code units, and CodeMirror
11
+ * addresses its document by the same offsets. The Python library counts code points instead,
12
+ * because Python string indices are code points; the two agree for every character below the
13
+ * astral planes and differ for anything above them, which in practice means an emoji in a
14
+ * comment. That divergence is registered rather than papered over.
15
+ */
16
+ export interface Region {
17
+ /** Offset of the first character, into the source string. */
18
+ readonly start: number;
19
+ /** Offset one past the last character. */
20
+ readonly end: number;
21
+ }
22
+ /**
23
+ * A position as a human reads it, derived from an offset rather than stored.
24
+ *
25
+ * Storing line and column on every region would double its size to hold two numbers that are a
26
+ * function of one. Both count from one, which is what the existing findings already report; a
27
+ * consumer whose own convention counts a column from zero subtracts one, and that subtraction is
28
+ * the consumer's, because a service that guessed which convention its caller wanted would be
29
+ * modelling a protocol.
30
+ */
31
+ export interface LineColumn {
32
+ /** 1-based, counting from the start of the text. */
33
+ readonly line: number;
34
+ /** 1-based, in UTF-16 code units, from the start of the line. */
35
+ readonly column: number;
36
+ }
37
+ /**
38
+ * The minimum a position query needs: the text its offsets index into.
39
+ *
40
+ * Declared structurally rather than by importing `SyntaxLayer`, so that positions stay usable
41
+ * without one and this module depends on nothing. A `SyntaxLayer` holds its text and therefore
42
+ * satisfies this, which is what lets `lineColumnAt(layer, offset)` read as the contract writes it.
43
+ */
44
+ export interface TextSource {
45
+ /** The text offsets are measured into. */
46
+ readonly text: string;
47
+ }
48
+ /**
49
+ * The line and column an offset falls on.
50
+ *
51
+ * Binary search over the line index, so a query costs the logarithm of the line count rather than
52
+ * a scan of the text. An offset outside `[0, text.length]` is clamped into range rather than
53
+ * throwing: a cursor arrives from an editor that may be a keystroke ahead of the text this layer
54
+ * was built from, and refusing to answer is worse than answering about the nearest character.
55
+ *
56
+ * The column of a *statement* is the column of its first non-blank character, not of the
57
+ * whitespace indenting it. This function does not enforce that rule, because a statement's region
58
+ * already starts at that character; it is stated here because every column this feature reports
59
+ * obeys it, and because the conformance corpus compares findings on their line and type name and
60
+ * would notice the number moving.
61
+ */
62
+ export declare function lineColumnAt(source: TextSource, offset: number): LineColumn;
63
+ /**
64
+ * The offset a line and column names.
65
+ *
66
+ * The inverse of {@link lineColumnAt} for every position that exists, and clamped for every one
67
+ * that does not: a line past the end resolves on the last line, a column past the end of its line
68
+ * resolves at the line's last position, which is the offset of the break that ends it. Round
69
+ * tripping an out-of-range position therefore returns the clamped position rather than the one
70
+ * asked for, which is the only honest answer available.
71
+ */
72
+ export declare function offsetAt(source: TextSource, position: LineColumn): number;
73
+ //# sourceMappingURL=region.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"region.d.ts","sourceRoot":"","sources":["../../src/syntax/region.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,MAAM;IACrB,6DAA6D;IAC7D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,0CAA0C;IAC1C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAgCD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAK3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,GAAG,MAAM,CAUzE"}
@@ -0,0 +1,105 @@
1
+ /**
2
+ * Line index per source, built once and reused.
3
+ *
4
+ * Keyed by the source object rather than by its text, because a `SyntaxLayer` is immutable and
5
+ * lives as long as the document does, while keying on the string itself would hold megabytes of
6
+ * text alive for the sake of a few kilobytes of offsets. A caller that passes a fresh object
7
+ * literal each time gets a fresh index each time, which is the documented cost of doing that.
8
+ */
9
+ const lineIndexes = new WeakMap();
10
+ /**
11
+ * Offsets at which each line begins, one entry per line, ascending.
12
+ *
13
+ * A line break is a line feed, and nothing else. A carriage return before it belongs to the line
14
+ * it ends, so a column at the end of a CRLF line is one greater than the same column in a file
15
+ * using line feeds alone, which is what every editor reports. A lone carriage return is *not* a
16
+ * break: the existing lexer does not treat it as one and Python's `_line_and_column` counts line
17
+ * feeds, and the conformance corpus compares a finding on its line, so a third opinion here would
18
+ * put the two libraries one line apart on a file no gate would explain.
19
+ *
20
+ * @internal
21
+ */
22
+ export function lineStartsOf(source) {
23
+ const cached = lineIndexes.get(source);
24
+ if (cached !== undefined)
25
+ return cached;
26
+ const starts = buildLineStarts(source.text);
27
+ lineIndexes.set(source, starts);
28
+ return starts;
29
+ }
30
+ /**
31
+ * The line and column an offset falls on.
32
+ *
33
+ * Binary search over the line index, so a query costs the logarithm of the line count rather than
34
+ * a scan of the text. An offset outside `[0, text.length]` is clamped into range rather than
35
+ * throwing: a cursor arrives from an editor that may be a keystroke ahead of the text this layer
36
+ * was built from, and refusing to answer is worse than answering about the nearest character.
37
+ *
38
+ * The column of a *statement* is the column of its first non-blank character, not of the
39
+ * whitespace indenting it. This function does not enforce that rule, because a statement's region
40
+ * already starts at that character; it is stated here because every column this feature reports
41
+ * obeys it, and because the conformance corpus compares findings on their line and type name and
42
+ * would notice the number moving.
43
+ */
44
+ export function lineColumnAt(source, offset) {
45
+ const clamped = clampOffset(offset, source.text.length);
46
+ const starts = lineStartsOf(source);
47
+ const line = lineIndexAt(starts, clamped);
48
+ return { line: line + 1, column: clamped - starts[line] + 1 };
49
+ }
50
+ /**
51
+ * The offset a line and column names.
52
+ *
53
+ * The inverse of {@link lineColumnAt} for every position that exists, and clamped for every one
54
+ * that does not: a line past the end resolves on the last line, a column past the end of its line
55
+ * resolves at the line's last position, which is the offset of the break that ends it. Round
56
+ * tripping an out-of-range position therefore returns the clamped position rather than the one
57
+ * asked for, which is the only honest answer available.
58
+ */
59
+ export function offsetAt(source, position) {
60
+ const text = source.text;
61
+ const starts = lineStartsOf(source);
62
+ const line = clampOffset(position.line - 1, starts.length - 1);
63
+ const lineStart = starts[line];
64
+ // One past the last position on this line is where the next line starts, so the last position on
65
+ // it is one before that: the offset of the line feed, at which a cursor is still on this line.
66
+ const lineEnd = line + 1 < starts.length ? starts[line + 1] - 1 : text.length;
67
+ const offset = lineStart + clampOffset(position.column - 1, text.length);
68
+ return offset > lineEnd ? lineEnd : offset;
69
+ }
70
+ /** Two passes so the array is allocated once at its exact size; `indexOf` does the scanning. */
71
+ function buildLineStarts(text) {
72
+ let count = 1;
73
+ for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1))
74
+ count += 1;
75
+ const starts = new Int32Array(count);
76
+ let line = 1;
77
+ for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1)) {
78
+ starts[line] = at + 1;
79
+ line += 1;
80
+ }
81
+ return starts;
82
+ }
83
+ /** Index of the greatest line start that is at or before `offset`. */
84
+ function lineIndexAt(starts, offset) {
85
+ let low = 0;
86
+ let high = starts.length - 1;
87
+ while (low < high) {
88
+ const middle = (low + high + 1) >> 1;
89
+ if (starts[middle] <= offset)
90
+ low = middle;
91
+ else
92
+ high = middle - 1;
93
+ }
94
+ return low;
95
+ }
96
+ /** Into `[0, max]`, whole. `NaN` lands at 0, since no position is nearer than another. */
97
+ function clampOffset(value, max) {
98
+ if (Number.isNaN(value))
99
+ return 0;
100
+ const whole = Math.trunc(value);
101
+ if (whole < 0)
102
+ return 0;
103
+ return whole > max ? max : whole;
104
+ }
105
+ //# sourceMappingURL=region.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"region.js","sourceRoot":"","sources":["../../src/syntax/region.ts"],"names":[],"mappings":"AAkDA;;;;;;;GAOG;AACH,MAAM,WAAW,GAAG,IAAI,OAAO,EAA0B,CAAC;AAE1D;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5C,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB,EAAE,MAAc;IAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAE,GAAG,CAAC,EAAE,CAAC;AACjE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAkB,EAAE,QAAoB;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC;IAChC,iGAAiG;IACjG,+FAA+F;IAC/F,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAC/E,MAAM,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AAC7C,CAAC;AAED,gGAAgG;AAChG,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;QAC7E,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC;IACZ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,sEAAsE;AACtE,SAAS,WAAW,CAAC,MAAkB,EAAE,MAAc;IACrD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC7B,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,CAAE,IAAI,MAAM;YAAE,GAAG,GAAG,MAAM,CAAC;;YACvC,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,0FAA0F;AAC1F,SAAS,WAAW,CAAC,KAAa,EAAE,GAAW;IAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AACnC,CAAC"}
@@ -0,0 +1,80 @@
1
+ import type { Region } from './region.js';
2
+ /**
3
+ * What a span of text is, grammatically. Nothing here needs a schema: the layer records what the
4
+ * text says and never what it means, so a `value` is a value whether or not the field exists.
5
+ */
6
+ export type TokenKind = 'typeName' | 'value' | 'separator' | 'terminator' | 'comment' | 'trivia';
7
+ /**
8
+ * One token, materialised.
9
+ *
10
+ * A `Token` *is* a `Region` rather than carrying one, which keeps a classified stream to one
11
+ * object per token instead of two. On a file of forty thousand lines that difference is the
12
+ * difference between an editor that repaints and one that stutters, and a token still passes
13
+ * anywhere a region is expected.
14
+ *
15
+ * A materialised token never crosses a line boundary; a stored `value` region may, because the
16
+ * format lets a field be written across two lines, and the classification view splits it.
17
+ */
18
+ export interface Token extends Region {
19
+ readonly kind: TokenKind;
20
+ }
21
+ /**
22
+ * Every token of one text, packed into three parallel arrays.
23
+ *
24
+ * Nine bytes per token, against upwards of forty for an object: a start, an end, and a kind code,
25
+ * held in `Int32Array`, `Int32Array` and `Uint8Array`. On a reference model of ten thousand
26
+ * statements that is roughly 3.6 MB rather than upwards of 16 MB.
27
+ *
28
+ * This is a contract rather than an implementation note. The intended use is an editor holding
29
+ * several large files open at once, and a layer that allocated an object per token would pass
30
+ * every correctness test while being unusable there. So `Token` objects are materialised only when
31
+ * a caller asks for one, which for classification means only for the tokens actually drawn.
32
+ *
33
+ * The store knows nothing of the scanner that fills it or of the layer that holds it, and nothing
34
+ * of the text either: it holds positions, and the text they index stays with the layer.
35
+ */
36
+ export declare class TokenStore implements Iterable<Token> {
37
+ #private;
38
+ constructor(capacity?: number);
39
+ /** How many tokens are stored. The arrays below are longer than this; they carry slack. */
40
+ get length(): number;
41
+ /**
42
+ * Start offsets, in source order.
43
+ *
44
+ * A view over the live buffer, trimmed to `length`, for a consumer that wants to walk the
45
+ * positions without materialising anything. It is invalidated by the next {@link push}, which in
46
+ * practice means it is safe for as long as the layer is: nothing appends to a built layer.
47
+ */
48
+ get starts(): Int32Array;
49
+ /** End offsets, exclusive, in source order. A live view, like {@link starts}. */
50
+ get ends(): Int32Array;
51
+ /** Kind codes, indices into the `TokenKind` union. A live view, like {@link starts}. */
52
+ get kinds(): Uint8Array;
53
+ /** Append one token and return its index. Grows geometrically, so a scan stays linear. */
54
+ push(start: number, end: number, kind: TokenKind): number;
55
+ /** The start offset of one token. */
56
+ startAt(index: number): number;
57
+ /** The end offset of one token, exclusive. */
58
+ endAt(index: number): number;
59
+ /** The kind of one token. */
60
+ kindAt(index: number): TokenKind;
61
+ /** The stored kind code of one token, for a consumer encoding kinds numerically. */
62
+ kindCodeAt(index: number): number;
63
+ /**
64
+ * Build a `Token` object for one stored token.
65
+ *
66
+ * The only place a token becomes an object. Called for the tokens a consumer actually reads, and
67
+ * never on the way in.
68
+ */
69
+ materialise(index: number): Token;
70
+ /**
71
+ * Every stored token, materialised one at a time.
72
+ *
73
+ * Convenient rather than cheap: it allocates per token, so a consumer colouring a viewport reads
74
+ * the arrays or goes through `classify`, which yields only what it is asked for. Note that this
75
+ * yields stored tokens alone, so the gaps between them are not covered; `trivia` is never stored
76
+ * and is computed as the complement.
77
+ */
78
+ [Symbol.iterator](): IterableIterator<Token>;
79
+ }
80
+ //# sourceMappingURL=tokens.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/syntax/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,SAAS,GACjB,UAAU,GACV,OAAO,GACP,WAAW,GACX,YAAY,GACZ,SAAS,GACT,QAAQ,CAAC;AAEb;;;;;;;;;;GAUG;AACH,MAAM,WAAW,KAAM,SAAQ,MAAM;IACnC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;CAC1B;AAwBD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,UAAW,YAAW,QAAQ,CAAC,KAAK,CAAC;;gBAMpC,QAAQ,GAAE,MAAyB;IAO/C,2FAA2F;IAC3F,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;;;;OAMG;IACH,IAAI,MAAM,IAAI,UAAU,CAEvB;IAED,iFAAiF;IACjF,IAAI,IAAI,IAAI,UAAU,CAErB;IAED,wFAAwF;IACxF,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,0FAA0F;IAC1F,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,MAAM;IAUzD,qCAAqC;IACrC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI9B,8CAA8C;IAC9C,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI5B,6BAA6B;IAC7B,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAIhC,oFAAoF;IACpF,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIjC;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK;IASjC;;;;;;;OAOG;IACF,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC;CAuB9C"}
@@ -0,0 +1,140 @@
1
+ /** Kind codes, in the order the union declares them. The index is what `kinds` stores. */
2
+ const KIND_BY_CODE = [
3
+ 'typeName',
4
+ 'value',
5
+ 'separator',
6
+ 'terminator',
7
+ 'comment',
8
+ 'trivia',
9
+ ];
10
+ const CODE_BY_KIND = {
11
+ typeName: 0,
12
+ value: 1,
13
+ separator: 2,
14
+ terminator: 3,
15
+ comment: 4,
16
+ trivia: 5,
17
+ };
18
+ /** Where growth starts. Small enough for a one-statement file, large enough to skip early copies. */
19
+ const INITIAL_CAPACITY = 64;
20
+ /**
21
+ * Every token of one text, packed into three parallel arrays.
22
+ *
23
+ * Nine bytes per token, against upwards of forty for an object: a start, an end, and a kind code,
24
+ * held in `Int32Array`, `Int32Array` and `Uint8Array`. On a reference model of ten thousand
25
+ * statements that is roughly 3.6 MB rather than upwards of 16 MB.
26
+ *
27
+ * This is a contract rather than an implementation note. The intended use is an editor holding
28
+ * several large files open at once, and a layer that allocated an object per token would pass
29
+ * every correctness test while being unusable there. So `Token` objects are materialised only when
30
+ * a caller asks for one, which for classification means only for the tokens actually drawn.
31
+ *
32
+ * The store knows nothing of the scanner that fills it or of the layer that holds it, and nothing
33
+ * of the text either: it holds positions, and the text they index stays with the layer.
34
+ */
35
+ export class TokenStore {
36
+ #starts;
37
+ #ends;
38
+ #kinds;
39
+ #length = 0;
40
+ constructor(capacity = INITIAL_CAPACITY) {
41
+ const initial = Math.max(0, Math.trunc(capacity));
42
+ this.#starts = new Int32Array(initial);
43
+ this.#ends = new Int32Array(initial);
44
+ this.#kinds = new Uint8Array(initial);
45
+ }
46
+ /** How many tokens are stored. The arrays below are longer than this; they carry slack. */
47
+ get length() {
48
+ return this.#length;
49
+ }
50
+ /**
51
+ * Start offsets, in source order.
52
+ *
53
+ * A view over the live buffer, trimmed to `length`, for a consumer that wants to walk the
54
+ * positions without materialising anything. It is invalidated by the next {@link push}, which in
55
+ * practice means it is safe for as long as the layer is: nothing appends to a built layer.
56
+ */
57
+ get starts() {
58
+ return this.#starts.subarray(0, this.#length);
59
+ }
60
+ /** End offsets, exclusive, in source order. A live view, like {@link starts}. */
61
+ get ends() {
62
+ return this.#ends.subarray(0, this.#length);
63
+ }
64
+ /** Kind codes, indices into the `TokenKind` union. A live view, like {@link starts}. */
65
+ get kinds() {
66
+ return this.#kinds.subarray(0, this.#length);
67
+ }
68
+ /** Append one token and return its index. Grows geometrically, so a scan stays linear. */
69
+ push(start, end, kind) {
70
+ const index = this.#length;
71
+ if (index === this.#starts.length)
72
+ this.#grow();
73
+ this.#starts[index] = start;
74
+ this.#ends[index] = end;
75
+ this.#kinds[index] = CODE_BY_KIND[kind];
76
+ this.#length = index + 1;
77
+ return index;
78
+ }
79
+ /** The start offset of one token. */
80
+ startAt(index) {
81
+ return this.#starts[this.#checked(index)];
82
+ }
83
+ /** The end offset of one token, exclusive. */
84
+ endAt(index) {
85
+ return this.#ends[this.#checked(index)];
86
+ }
87
+ /** The kind of one token. */
88
+ kindAt(index) {
89
+ return KIND_BY_CODE[this.#kinds[this.#checked(index)]];
90
+ }
91
+ /** The stored kind code of one token, for a consumer encoding kinds numerically. */
92
+ kindCodeAt(index) {
93
+ return this.#kinds[this.#checked(index)];
94
+ }
95
+ /**
96
+ * Build a `Token` object for one stored token.
97
+ *
98
+ * The only place a token becomes an object. Called for the tokens a consumer actually reads, and
99
+ * never on the way in.
100
+ */
101
+ materialise(index) {
102
+ const at = this.#checked(index);
103
+ return {
104
+ start: this.#starts[at],
105
+ end: this.#ends[at],
106
+ kind: KIND_BY_CODE[this.#kinds[at]],
107
+ };
108
+ }
109
+ /**
110
+ * Every stored token, materialised one at a time.
111
+ *
112
+ * Convenient rather than cheap: it allocates per token, so a consumer colouring a viewport reads
113
+ * the arrays or goes through `classify`, which yields only what it is asked for. Note that this
114
+ * yields stored tokens alone, so the gaps between them are not covered; `trivia` is never stored
115
+ * and is computed as the complement.
116
+ */
117
+ *[Symbol.iterator]() {
118
+ for (let index = 0; index < this.#length; index += 1)
119
+ yield this.materialise(index);
120
+ }
121
+ #checked(index) {
122
+ if (!Number.isInteger(index) || index < 0 || index >= this.#length) {
123
+ throw new RangeError(`No token at index ${index} (${this.#length} stored)`);
124
+ }
125
+ return index;
126
+ }
127
+ #grow() {
128
+ const capacity = this.#starts.length === 0 ? INITIAL_CAPACITY : this.#starts.length * 2;
129
+ const starts = new Int32Array(capacity);
130
+ const ends = new Int32Array(capacity);
131
+ const kinds = new Uint8Array(capacity);
132
+ starts.set(this.#starts);
133
+ ends.set(this.#ends);
134
+ kinds.set(this.#kinds);
135
+ this.#starts = starts;
136
+ this.#ends = ends;
137
+ this.#kinds = kinds;
138
+ }
139
+ }
140
+ //# sourceMappingURL=tokens.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/syntax/tokens.ts"],"names":[],"mappings":"AA6BA,0FAA0F;AAC1F,MAAM,YAAY,GAAyB;IACzC,UAAU;IACV,OAAO;IACP,WAAW;IACX,YAAY;IACZ,SAAS;IACT,QAAQ;CACT,CAAC;AAEF,MAAM,YAAY,GAAwC;IACxD,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;CACV,CAAC;AAEF,qGAAqG;AACrG,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,UAAU;IACrB,OAAO,CAAa;IACpB,KAAK,CAAa;IAClB,MAAM,CAAa;IACnB,OAAO,GAAG,CAAC,CAAC;IAEZ,YAAY,WAAmB,gBAAgB;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,2FAA2F;IAC3F,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,iFAAiF;IACjF,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,wFAAwF;IACxF,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,0FAA0F;IAC1F,IAAI,CAAC,KAAa,EAAE,GAAW,EAAE,IAAe;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3B,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAChD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC;QACzB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,qCAAqC;IACrC,OAAO,CAAC,KAAa;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC7C,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,KAAa;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC3C,CAAC;IAED,6BAA6B;IAC7B,MAAM,CAAC,KAAa;QAClB,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAE,CAAC;IAC3D,CAAC;IAED,oFAAoF;IACpF,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAE,CAAC;IAC5C,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,KAAa;QACvB,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE;YACxB,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAE;YACpB,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAE,CAAE;SACtC,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IACtF,CAAC;IAED,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,IAAI,UAAU,CAAC,qBAAqB,KAAK,KAAK,IAAI,CAAC,OAAO,UAAU,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;CACF"}