@yozora/tokenizer-definition 1.2.2 → 2.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -6
- package/lib/cjs/index.js +149 -145
- package/lib/esm/index.js +148 -146
- package/lib/types/index.d.ts +4 -4
- package/lib/types/match.d.ts +18 -0
- package/lib/types/parse.d.ts +3 -0
- package/lib/types/tokenizer.d.ts +6 -38
- package/lib/types/types.d.ts +7 -6
- package/lib/types/util/link-destination.d.ts +5 -5
- package/lib/types/util/link-label.d.ts +5 -5
- package/lib/types/util/link-title.d.ts +5 -5
- package/package.json +5 -5
package/lib/esm/index.js
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import { DefinitionType } from '@yozora/ast';
|
|
2
1
|
import { AsciiCodePoint, VirtualCodePoint, isWhitespaceCharacter, isAsciiControlCharacter, calcStringFromNodePoints, calcEscapedStringFromNodePoints } from '@yozora/character';
|
|
3
|
-
import { eatOptionalWhitespaces,
|
|
4
|
-
|
|
5
|
-
const uniqueName = '@yozora/tokenizer-definition';
|
|
2
|
+
import { eatOptionalWhitespaces, calcEndYastNodePoint, resolveLabelToIdentifier, calcStartYastNodePoint, encodeLinkDestination, BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
|
|
3
|
+
import { DefinitionType } from '@yozora/ast';
|
|
6
4
|
|
|
7
|
-
function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex,
|
|
5
|
+
function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, state) {
|
|
8
6
|
let i = startIndex;
|
|
9
|
-
if (
|
|
10
|
-
|
|
7
|
+
if (state == null) {
|
|
8
|
+
state = {
|
|
11
9
|
saturated: false,
|
|
12
10
|
nodePoints: [],
|
|
13
11
|
hasOpenAngleBracket: false,
|
|
@@ -16,79 +14,78 @@ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
|
|
|
16
14
|
}
|
|
17
15
|
const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
|
|
18
16
|
if (firstNonWhitespaceIndex >= endIndex)
|
|
19
|
-
return { nextIndex: -1,
|
|
20
|
-
if (
|
|
17
|
+
return { nextIndex: -1, state: state };
|
|
18
|
+
if (state.nodePoints.length <= 0) {
|
|
21
19
|
i = firstNonWhitespaceIndex;
|
|
22
20
|
const p = nodePoints[i];
|
|
23
21
|
if (p.codePoint === AsciiCodePoint.OPEN_ANGLE) {
|
|
24
22
|
i += 1;
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
state.hasOpenAngleBracket = true;
|
|
24
|
+
state.nodePoints.push(p);
|
|
27
25
|
}
|
|
28
26
|
}
|
|
29
|
-
if (
|
|
27
|
+
if (state.hasOpenAngleBracket) {
|
|
30
28
|
for (; i < endIndex; ++i) {
|
|
31
29
|
const p = nodePoints[i];
|
|
32
30
|
switch (p.codePoint) {
|
|
33
31
|
case AsciiCodePoint.BACKSLASH:
|
|
34
32
|
if (i + 1 < endIndex) {
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
state.nodePoints.push(p);
|
|
34
|
+
state.nodePoints.push(nodePoints[i + 1]);
|
|
37
35
|
}
|
|
38
36
|
i += 1;
|
|
39
37
|
break;
|
|
40
38
|
case AsciiCodePoint.OPEN_ANGLE:
|
|
41
39
|
case VirtualCodePoint.LINE_END:
|
|
42
|
-
return { nextIndex: -1,
|
|
40
|
+
return { nextIndex: -1, state: state };
|
|
43
41
|
case AsciiCodePoint.CLOSE_ANGLE:
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return { nextIndex: i + 1,
|
|
42
|
+
state.saturated = true;
|
|
43
|
+
state.nodePoints.push(p);
|
|
44
|
+
return { nextIndex: i + 1, state: state };
|
|
47
45
|
default:
|
|
48
|
-
|
|
46
|
+
state.nodePoints.push(p);
|
|
49
47
|
}
|
|
50
48
|
}
|
|
51
|
-
return { nextIndex: i,
|
|
49
|
+
return { nextIndex: i, state: state };
|
|
52
50
|
}
|
|
53
51
|
for (; i < endIndex; ++i) {
|
|
54
52
|
const p = nodePoints[i];
|
|
55
53
|
switch (p.codePoint) {
|
|
56
54
|
case AsciiCodePoint.BACKSLASH:
|
|
57
55
|
if (i + 1 < endIndex) {
|
|
58
|
-
|
|
59
|
-
|
|
56
|
+
state.nodePoints.push(p);
|
|
57
|
+
state.nodePoints.push(nodePoints[i + 1]);
|
|
60
58
|
}
|
|
61
59
|
i += 1;
|
|
62
60
|
break;
|
|
63
61
|
case AsciiCodePoint.OPEN_PARENTHESIS:
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
state.openParensCount += 1;
|
|
63
|
+
state.nodePoints.push(p);
|
|
66
64
|
break;
|
|
67
65
|
case AsciiCodePoint.CLOSE_PARENTHESIS:
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
if (
|
|
71
|
-
return { nextIndex: i,
|
|
66
|
+
state.openParensCount -= 1;
|
|
67
|
+
state.nodePoints.push(p);
|
|
68
|
+
if (state.openParensCount < 0) {
|
|
69
|
+
return { nextIndex: i, state: state };
|
|
72
70
|
}
|
|
73
71
|
break;
|
|
74
72
|
default:
|
|
75
|
-
if (isWhitespaceCharacter(p.codePoint) ||
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return { nextIndex: i, token };
|
|
73
|
+
if (isWhitespaceCharacter(p.codePoint) || isAsciiControlCharacter(p.codePoint)) {
|
|
74
|
+
state.saturated = true;
|
|
75
|
+
return { nextIndex: i, state: state };
|
|
79
76
|
}
|
|
80
|
-
|
|
77
|
+
state.nodePoints.push(p);
|
|
81
78
|
break;
|
|
82
79
|
}
|
|
83
80
|
}
|
|
84
|
-
|
|
85
|
-
return { nextIndex: i,
|
|
81
|
+
state.saturated = true;
|
|
82
|
+
return { nextIndex: i, state: state };
|
|
86
83
|
}
|
|
87
84
|
|
|
88
|
-
function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex,
|
|
85
|
+
function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, state) {
|
|
89
86
|
let i = startIndex;
|
|
90
|
-
if (
|
|
91
|
-
|
|
87
|
+
if (state == null) {
|
|
88
|
+
state = {
|
|
92
89
|
saturated: false,
|
|
93
90
|
nodePoints: [],
|
|
94
91
|
hasNonWhitespaceCharacter: false,
|
|
@@ -96,50 +93,50 @@ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
|
|
|
96
93
|
}
|
|
97
94
|
const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
|
|
98
95
|
if (firstNonWhitespaceIndex >= endIndex)
|
|
99
|
-
return { nextIndex: -1,
|
|
100
|
-
if (
|
|
96
|
+
return { nextIndex: -1, state: state };
|
|
97
|
+
if (state.nodePoints.length <= 0) {
|
|
101
98
|
i = firstNonWhitespaceIndex;
|
|
102
99
|
const p = nodePoints[i];
|
|
103
100
|
if (p.codePoint !== AsciiCodePoint.OPEN_BRACKET) {
|
|
104
|
-
return { nextIndex: -1,
|
|
101
|
+
return { nextIndex: -1, state: state };
|
|
105
102
|
}
|
|
106
103
|
i += 1;
|
|
107
|
-
|
|
104
|
+
state.nodePoints.push(p);
|
|
108
105
|
}
|
|
109
106
|
for (; i < endIndex; ++i) {
|
|
110
107
|
const p = nodePoints[i];
|
|
111
108
|
switch (p.codePoint) {
|
|
112
109
|
case AsciiCodePoint.BACKSLASH:
|
|
113
|
-
|
|
110
|
+
state.hasNonWhitespaceCharacter = true;
|
|
114
111
|
if (i + 1 < endIndex) {
|
|
115
|
-
|
|
116
|
-
|
|
112
|
+
state.nodePoints.push(p);
|
|
113
|
+
state.nodePoints.push(nodePoints[i + 1]);
|
|
117
114
|
}
|
|
118
115
|
i += 1;
|
|
119
116
|
break;
|
|
120
117
|
case AsciiCodePoint.OPEN_BRACKET:
|
|
121
|
-
return { nextIndex: -1,
|
|
118
|
+
return { nextIndex: -1, state: state };
|
|
122
119
|
case AsciiCodePoint.CLOSE_BRACKET:
|
|
123
|
-
|
|
124
|
-
if (
|
|
125
|
-
|
|
126
|
-
return { nextIndex: i + 1,
|
|
120
|
+
state.nodePoints.push(p);
|
|
121
|
+
if (state.hasNonWhitespaceCharacter) {
|
|
122
|
+
state.saturated = true;
|
|
123
|
+
return { nextIndex: i + 1, state: state };
|
|
127
124
|
}
|
|
128
|
-
return { nextIndex: -1,
|
|
125
|
+
return { nextIndex: -1, state: state };
|
|
129
126
|
default:
|
|
130
127
|
if (!isWhitespaceCharacter(p.codePoint)) {
|
|
131
|
-
|
|
128
|
+
state.hasNonWhitespaceCharacter = true;
|
|
132
129
|
}
|
|
133
|
-
|
|
130
|
+
state.nodePoints.push(p);
|
|
134
131
|
}
|
|
135
132
|
}
|
|
136
|
-
return { nextIndex: 1,
|
|
133
|
+
return { nextIndex: 1, state: state };
|
|
137
134
|
}
|
|
138
135
|
|
|
139
|
-
function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex,
|
|
136
|
+
function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, state) {
|
|
140
137
|
let i = startIndex;
|
|
141
|
-
if (
|
|
142
|
-
|
|
138
|
+
if (state == null) {
|
|
139
|
+
state = {
|
|
143
140
|
saturated: false,
|
|
144
141
|
nodePoints: [],
|
|
145
142
|
wrapSymbol: null,
|
|
@@ -147,25 +144,25 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
|
|
|
147
144
|
}
|
|
148
145
|
const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
|
|
149
146
|
if (firstNonWhitespaceIndex >= endIndex)
|
|
150
|
-
return { nextIndex: -1,
|
|
151
|
-
if (
|
|
147
|
+
return { nextIndex: -1, state: state };
|
|
148
|
+
if (state.nodePoints.length <= 0) {
|
|
152
149
|
i = firstNonWhitespaceIndex;
|
|
153
150
|
const p = nodePoints[i];
|
|
154
151
|
switch (p.codePoint) {
|
|
155
152
|
case AsciiCodePoint.DOUBLE_QUOTE:
|
|
156
153
|
case AsciiCodePoint.SINGLE_QUOTE:
|
|
157
154
|
case AsciiCodePoint.OPEN_PARENTHESIS:
|
|
158
|
-
|
|
159
|
-
|
|
155
|
+
state.wrapSymbol = p.codePoint;
|
|
156
|
+
state.nodePoints.push(p);
|
|
160
157
|
i += 1;
|
|
161
158
|
break;
|
|
162
159
|
default:
|
|
163
|
-
return { nextIndex: -1,
|
|
160
|
+
return { nextIndex: -1, state: state };
|
|
164
161
|
}
|
|
165
162
|
}
|
|
166
|
-
if (
|
|
167
|
-
return { nextIndex: -1,
|
|
168
|
-
switch (
|
|
163
|
+
if (state.wrapSymbol == null)
|
|
164
|
+
return { nextIndex: -1, state: state };
|
|
165
|
+
switch (state.wrapSymbol) {
|
|
169
166
|
case AsciiCodePoint.DOUBLE_QUOTE:
|
|
170
167
|
case AsciiCodePoint.SINGLE_QUOTE: {
|
|
171
168
|
for (; i < endIndex; ++i) {
|
|
@@ -173,17 +170,17 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
|
|
|
173
170
|
switch (p.codePoint) {
|
|
174
171
|
case AsciiCodePoint.BACKSLASH:
|
|
175
172
|
if (i + 1 < endIndex) {
|
|
176
|
-
|
|
177
|
-
|
|
173
|
+
state.nodePoints.push(p);
|
|
174
|
+
state.nodePoints.push(nodePoints[i + 1]);
|
|
178
175
|
}
|
|
179
176
|
i += 1;
|
|
180
177
|
break;
|
|
181
|
-
case
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
return { nextIndex: i + 1,
|
|
178
|
+
case state.wrapSymbol:
|
|
179
|
+
state.saturated = true;
|
|
180
|
+
state.nodePoints.push(p);
|
|
181
|
+
return { nextIndex: i + 1, state: state };
|
|
185
182
|
default:
|
|
186
|
-
|
|
183
|
+
state.nodePoints.push(p);
|
|
187
184
|
}
|
|
188
185
|
}
|
|
189
186
|
break;
|
|
@@ -194,49 +191,46 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
|
|
|
194
191
|
switch (p.codePoint) {
|
|
195
192
|
case AsciiCodePoint.BACKSLASH:
|
|
196
193
|
if (i + 1 < endIndex) {
|
|
197
|
-
|
|
198
|
-
|
|
194
|
+
state.nodePoints.push(p);
|
|
195
|
+
state.nodePoints.push(nodePoints[i + 1]);
|
|
199
196
|
}
|
|
200
197
|
i += 1;
|
|
201
198
|
break;
|
|
202
199
|
case AsciiCodePoint.OPEN_PARENTHESIS:
|
|
203
|
-
return { nextIndex: -1,
|
|
200
|
+
return { nextIndex: -1, state: state };
|
|
204
201
|
case AsciiCodePoint.CLOSE_PARENTHESIS:
|
|
205
|
-
if (i + 1 >= endIndex ||
|
|
206
|
-
nodePoints
|
|
207
|
-
|
|
208
|
-
token.saturated = true;
|
|
202
|
+
if (i + 1 >= endIndex || nodePoints[i + 1].codePoint === VirtualCodePoint.LINE_END) {
|
|
203
|
+
state.nodePoints.push(p);
|
|
204
|
+
state.saturated = true;
|
|
209
205
|
break;
|
|
210
206
|
}
|
|
211
|
-
return { nextIndex: -1,
|
|
207
|
+
return { nextIndex: -1, state: state };
|
|
212
208
|
default:
|
|
213
|
-
|
|
209
|
+
state.nodePoints.push(p);
|
|
214
210
|
}
|
|
215
211
|
}
|
|
216
212
|
break;
|
|
217
213
|
}
|
|
218
214
|
}
|
|
219
|
-
return { nextIndex: endIndex,
|
|
215
|
+
return { nextIndex: endIndex, state: state };
|
|
220
216
|
}
|
|
221
217
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
}
|
|
231
|
-
eatOpener(line) {
|
|
218
|
+
const match = function (api) {
|
|
219
|
+
return {
|
|
220
|
+
isContainingBlock: false,
|
|
221
|
+
eatOpener,
|
|
222
|
+
eatContinuationText,
|
|
223
|
+
onClose,
|
|
224
|
+
};
|
|
225
|
+
function eatOpener(line) {
|
|
232
226
|
if (line.countOfPrecedeSpaces >= 4)
|
|
233
227
|
return null;
|
|
234
228
|
const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line;
|
|
235
229
|
if (firstNonWhitespaceIndex >= endIndex)
|
|
236
230
|
return null;
|
|
237
231
|
let i = firstNonWhitespaceIndex;
|
|
238
|
-
const
|
|
239
|
-
if (
|
|
232
|
+
const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
|
|
233
|
+
if (labelEndIndex < 0)
|
|
240
234
|
return null;
|
|
241
235
|
const lineNo = nodePoints[startIndex].line;
|
|
242
236
|
const createInitState = () => {
|
|
@@ -246,7 +240,7 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
246
240
|
start: calcStartYastNodePoint(nodePoints, startIndex),
|
|
247
241
|
end: calcEndYastNodePoint(nodePoints, endIndex - 1),
|
|
248
242
|
},
|
|
249
|
-
label:
|
|
243
|
+
label: labelState,
|
|
250
244
|
destination: null,
|
|
251
245
|
title: null,
|
|
252
246
|
lineNoOfLabel: lineNo,
|
|
@@ -256,11 +250,10 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
256
250
|
};
|
|
257
251
|
return token;
|
|
258
252
|
};
|
|
259
|
-
if (!
|
|
253
|
+
if (!labelState.saturated) {
|
|
260
254
|
const token = createInitState();
|
|
261
255
|
return { token, nextIndex: endIndex };
|
|
262
256
|
}
|
|
263
|
-
const labelEndIndex = linkLabelCollectResult.nextIndex;
|
|
264
257
|
if (labelEndIndex < 0 ||
|
|
265
258
|
labelEndIndex + 1 >= endIndex ||
|
|
266
259
|
nodePoints[labelEndIndex].codePoint !== AsciiCodePoint.COLON)
|
|
@@ -270,39 +263,36 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
270
263
|
const token = createInitState();
|
|
271
264
|
return { token, nextIndex: endIndex };
|
|
272
265
|
}
|
|
273
|
-
const
|
|
274
|
-
if (
|
|
266
|
+
const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
|
|
267
|
+
if (destinationEndIndex < 0)
|
|
275
268
|
return null;
|
|
276
|
-
if (!
|
|
277
|
-
linkDestinationCollectResult.nextIndex !== endIndex)
|
|
269
|
+
if (!destinationState.saturated && destinationEndIndex !== endIndex)
|
|
278
270
|
return null;
|
|
279
|
-
const destinationEndIndex = linkDestinationCollectResult.nextIndex;
|
|
280
271
|
i = eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
|
|
281
272
|
if (i >= endIndex) {
|
|
282
273
|
const token = createInitState();
|
|
283
|
-
token.destination =
|
|
274
|
+
token.destination = destinationState;
|
|
284
275
|
token.lineNoOfDestination = lineNo;
|
|
285
276
|
return { token, nextIndex: endIndex };
|
|
286
277
|
}
|
|
287
278
|
if (i === destinationEndIndex)
|
|
288
279
|
return null;
|
|
289
|
-
const
|
|
290
|
-
if (
|
|
291
|
-
i =
|
|
292
|
-
}
|
|
280
|
+
const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
|
|
281
|
+
if (titleEndIndex >= 0)
|
|
282
|
+
i = titleEndIndex;
|
|
293
283
|
if (i < endIndex) {
|
|
294
284
|
const k = eatOptionalWhitespaces(nodePoints, i, endIndex);
|
|
295
285
|
if (k < endIndex)
|
|
296
286
|
return null;
|
|
297
287
|
}
|
|
298
288
|
const token = createInitState();
|
|
299
|
-
token.destination =
|
|
300
|
-
token.title =
|
|
289
|
+
token.destination = destinationState;
|
|
290
|
+
token.title = titleState;
|
|
301
291
|
token.lineNoOfDestination = lineNo;
|
|
302
292
|
token.lineNoOfTitle = lineNo;
|
|
303
293
|
return { token, nextIndex: endIndex };
|
|
304
294
|
}
|
|
305
|
-
eatContinuationText(line, token) {
|
|
295
|
+
function eatContinuationText(line, token) {
|
|
306
296
|
var _a;
|
|
307
297
|
if (token.title != null && token.title.saturated)
|
|
308
298
|
return { status: 'notMatched' };
|
|
@@ -310,12 +300,11 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
310
300
|
const lineNo = nodePoints[startIndex].line;
|
|
311
301
|
let i = firstNonWhitespaceIndex;
|
|
312
302
|
if (!token.label.saturated) {
|
|
313
|
-
const
|
|
314
|
-
if (
|
|
303
|
+
const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
|
|
304
|
+
if (labelEndIndex < 0) {
|
|
315
305
|
return { status: 'failedAndRollback', lines: token.lines };
|
|
316
306
|
}
|
|
317
|
-
|
|
318
|
-
if (!linkLabelCollectResult.token.saturated) {
|
|
307
|
+
if (!labelState.saturated) {
|
|
319
308
|
token.lines.push(line);
|
|
320
309
|
return { status: 'opening', nextIndex: endIndex };
|
|
321
310
|
}
|
|
@@ -330,15 +319,13 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
330
319
|
if (i >= endIndex) {
|
|
331
320
|
return { status: 'failedAndRollback', lines: token.lines };
|
|
332
321
|
}
|
|
333
|
-
const
|
|
334
|
-
if (
|
|
335
|
-
!linkDestinationCollectResult.token.saturated) {
|
|
322
|
+
const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
|
|
323
|
+
if (destinationEndIndex < 0 || !destinationState.saturated) {
|
|
336
324
|
return { status: 'failedAndRollback', lines: token.lines };
|
|
337
325
|
}
|
|
338
|
-
const destinationEndIndex = linkDestinationCollectResult.nextIndex;
|
|
339
326
|
i = eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
|
|
340
327
|
if (i >= endIndex) {
|
|
341
|
-
token.destination =
|
|
328
|
+
token.destination = destinationState;
|
|
342
329
|
token.lines.push(line);
|
|
343
330
|
return { status: 'opening', nextIndex: endIndex };
|
|
344
331
|
}
|
|
@@ -348,12 +335,12 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
348
335
|
if (token.lineNoOfTitle < 0) {
|
|
349
336
|
token.lineNoOfTitle = lineNo;
|
|
350
337
|
}
|
|
351
|
-
const
|
|
352
|
-
token.title =
|
|
353
|
-
if (
|
|
354
|
-
|
|
355
|
-
(
|
|
356
|
-
eatOptionalWhitespaces(nodePoints,
|
|
338
|
+
const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
|
|
339
|
+
token.title = titleState;
|
|
340
|
+
if (titleEndIndex < 0 ||
|
|
341
|
+
titleState.nodePoints.length <= 0 ||
|
|
342
|
+
(titleState.saturated &&
|
|
343
|
+
eatOptionalWhitespaces(nodePoints, titleEndIndex, endIndex) < endIndex)) {
|
|
357
344
|
if (token.lineNoOfDestination === token.lineNoOfTitle) {
|
|
358
345
|
return { status: 'failedAndRollback', lines: token.lines };
|
|
359
346
|
}
|
|
@@ -369,7 +356,7 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
369
356
|
const saturated = (_a = token.title) === null || _a === void 0 ? void 0 : _a.saturated;
|
|
370
357
|
return { status: saturated ? 'closing' : 'opening', nextIndex: endIndex };
|
|
371
358
|
}
|
|
372
|
-
onClose(token
|
|
359
|
+
function onClose(token) {
|
|
373
360
|
let result;
|
|
374
361
|
if (token.title == null || !token.title.saturated) {
|
|
375
362
|
if (!token.label.saturated) {
|
|
@@ -397,26 +384,41 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
|
397
384
|
token._identifier = identifier;
|
|
398
385
|
return result;
|
|
399
386
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
387
|
+
};
|
|
388
|
+
|
|
389
|
+
const parse = function (api) {
|
|
390
|
+
return {
|
|
391
|
+
parse: tokens => tokens.map(token => {
|
|
392
|
+
const label = token._label;
|
|
393
|
+
const identifier = token._identifier;
|
|
394
|
+
const destinationPoints = token.destination.nodePoints;
|
|
395
|
+
const destination = destinationPoints[0].codePoint === AsciiCodePoint.OPEN_ANGLE
|
|
396
|
+
? calcEscapedStringFromNodePoints(destinationPoints, 1, destinationPoints.length - 1, true)
|
|
397
|
+
: calcEscapedStringFromNodePoints(destinationPoints, 0, destinationPoints.length, true);
|
|
398
|
+
const url = encodeLinkDestination(destination);
|
|
399
|
+
const title = token.title == null
|
|
400
|
+
? undefined
|
|
401
|
+
: calcEscapedStringFromNodePoints(token.title.nodePoints, 1, token.title.nodePoints.length - 1);
|
|
402
|
+
const node = api.shouldReservePosition
|
|
403
|
+
? { type: DefinitionType, position: token.position, identifier, label, url, title }
|
|
404
|
+
: { type: DefinitionType, identifier, label, url, title };
|
|
405
|
+
return node;
|
|
406
|
+
}),
|
|
407
|
+
};
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
const uniqueName = '@yozora/tokenizer-definition';
|
|
411
|
+
|
|
412
|
+
class DefinitionTokenizer extends BaseBlockTokenizer {
|
|
413
|
+
constructor(props = {}) {
|
|
414
|
+
var _a, _b;
|
|
415
|
+
super({
|
|
416
|
+
name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
|
|
417
|
+
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
|
|
418
|
+
});
|
|
419
|
+
this.match = match;
|
|
420
|
+
this.parse = parse;
|
|
419
421
|
}
|
|
420
422
|
}
|
|
421
423
|
|
|
422
|
-
export { DefinitionTokenizer, uniqueName as DefinitionTokenizerName, DefinitionTokenizer as default, eatAndCollectLinkDestination, eatAndCollectLinkLabel, eatAndCollectLinkTitle };
|
|
424
|
+
export { DefinitionTokenizer, uniqueName as DefinitionTokenizerName, DefinitionTokenizer as default, match as definitionMatch, parse as definitionParse, eatAndCollectLinkDestination, eatAndCollectLinkLabel, eatAndCollectLinkTitle };
|
package/lib/types/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { DefinitionTokenizer } from './tokenizer';
|
|
2
1
|
export * from './util/link-destination';
|
|
3
2
|
export * from './util/link-label';
|
|
4
3
|
export * from './util/link-title';
|
|
5
|
-
export {
|
|
4
|
+
export { match as definitionMatch } from './match';
|
|
5
|
+
export { parse as definitionParse } from './parse';
|
|
6
|
+
export { DefinitionTokenizer, DefinitionTokenizer as default } from './tokenizer';
|
|
6
7
|
export { uniqueName as DefinitionTokenizerName } from './types';
|
|
7
|
-
export type {
|
|
8
|
-
export default DefinitionTokenizer;
|
|
8
|
+
export type { IThis as IDefinitionHookContext, IToken as IDefinitionToken, ITokenizerProps as IDefinitionTokenizerProps, } from './types';
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { IMatchBlockHookCreator } from '@yozora/core-tokenizer';
|
|
2
|
+
import type { IThis, IToken, T } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* A link reference definition consists of a link label, indented up to three
|
|
5
|
+
* spaces, followed by a colon (:), optional whitespace (including up to one
|
|
6
|
+
* line ending), a link destination, optional whitespace (including up to one
|
|
7
|
+
* line ending), and an optional link title, which if it is present must be
|
|
8
|
+
* separated from the link destination by whitespace. No further non-whitespace
|
|
9
|
+
* characters may occur on the line.
|
|
10
|
+
*
|
|
11
|
+
* A link reference definition does not correspond to a structural element of
|
|
12
|
+
* a document. Instead, it defines a label which can be used in reference
|
|
13
|
+
* links and reference-style images elsewhere in the document. Link reference
|
|
14
|
+
* definitions can come either before or after the links that use them.
|
|
15
|
+
*
|
|
16
|
+
* @see https://github.github.com/gfm/#link-reference-definition
|
|
17
|
+
*/
|
|
18
|
+
export declare const match: IMatchBlockHookCreator<T, IToken, IThis>;
|
package/lib/types/tokenizer.d.ts
CHANGED
|
@@ -1,44 +1,12 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { IBlockTokenizer, IMatchBlockHookCreator, IParseBlockHookCreator } from '@yozora/core-tokenizer';
|
|
2
2
|
import { BaseBlockTokenizer } from '@yozora/core-tokenizer';
|
|
3
|
-
import type {
|
|
3
|
+
import type { INode, IThis, IToken, ITokenizerProps, T } from './types';
|
|
4
4
|
/**
|
|
5
5
|
* Lexical Analyzer for Definition.
|
|
6
|
-
*
|
|
7
|
-
* A link reference definition consists of a link label, indented up to three
|
|
8
|
-
* spaces, followed by a colon (:), optional whitespace (including up to one
|
|
9
|
-
* line ending), a link destination, optional whitespace (including up to one
|
|
10
|
-
* line ending), and an optional link title, which if it is present must be
|
|
11
|
-
* separated from the link destination by whitespace. No further non-whitespace
|
|
12
|
-
* characters may occur on the line.
|
|
13
|
-
*
|
|
14
|
-
* A link reference definition does not correspond to a structural element of
|
|
15
|
-
* a document. Instead, it defines a label which can be used in reference
|
|
16
|
-
* links and reference-style images elsewhere in the document. Link reference
|
|
17
|
-
* definitions can come either before or after the links that use them.
|
|
18
|
-
*
|
|
19
6
|
* @see https://github.github.com/gfm/#link-reference-definition
|
|
20
7
|
*/
|
|
21
|
-
export declare class DefinitionTokenizer extends BaseBlockTokenizer
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
* @override
|
|
26
|
-
* @see TokenizerMatchBlockHook
|
|
27
|
-
*/
|
|
28
|
-
eatOpener(line: Readonly<PhrasingContentLine>): ResultOfEatOpener<T, Token>;
|
|
29
|
-
/**
|
|
30
|
-
* @override
|
|
31
|
-
* @see TokenizerMatchBlockHook
|
|
32
|
-
*/
|
|
33
|
-
eatContinuationText(line: Readonly<PhrasingContentLine>, token: Token): ResultOfEatContinuationText;
|
|
34
|
-
/**
|
|
35
|
-
* @override
|
|
36
|
-
* @see TokenizerMatchBlockHook
|
|
37
|
-
*/
|
|
38
|
-
onClose(token: Token, api: Readonly<MatchBlockPhaseApi>): ResultOfOnClose;
|
|
39
|
-
/**
|
|
40
|
-
* @override
|
|
41
|
-
* @see TokenizerParseBlockHook
|
|
42
|
-
*/
|
|
43
|
-
parseBlock(token: Readonly<Token>): ResultOfParse<T, Node>;
|
|
8
|
+
export declare class DefinitionTokenizer extends BaseBlockTokenizer<T, IToken, INode, IThis> implements IBlockTokenizer<T, IToken, INode, IThis> {
|
|
9
|
+
constructor(props?: ITokenizerProps);
|
|
10
|
+
readonly match: IMatchBlockHookCreator<T, IToken, IThis>;
|
|
11
|
+
readonly parse: IParseBlockHookCreator<T, IToken, INode, IThis>;
|
|
44
12
|
}
|
package/lib/types/types.d.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
1
|
+
import type { DefinitionType, IDefinition } from '@yozora/ast';
|
|
2
|
+
import type { IBaseBlockTokenizerProps, IPartialYastBlockToken, IPhrasingContentLine, ITokenizer } from '@yozora/core-tokenizer';
|
|
3
3
|
import type { LinkDestinationCollectingState } from './util/link-destination';
|
|
4
4
|
import type { LinkLabelCollectingState } from './util/link-label';
|
|
5
5
|
import type { LinkTitleCollectingState } from './util/link-title';
|
|
6
6
|
export declare type T = DefinitionType;
|
|
7
|
-
export declare type
|
|
7
|
+
export declare type INode = IDefinition;
|
|
8
8
|
export declare const uniqueName = "@yozora/tokenizer-definition";
|
|
9
|
-
export interface
|
|
9
|
+
export interface IToken extends IPartialYastBlockToken<T> {
|
|
10
10
|
/**
|
|
11
11
|
*
|
|
12
12
|
*/
|
|
13
|
-
lines: Array<Readonly<
|
|
13
|
+
lines: Array<Readonly<IPhrasingContentLine>>;
|
|
14
14
|
/**
|
|
15
15
|
* Link label
|
|
16
16
|
* Trimmed, Case-Insensitive
|
|
@@ -45,4 +45,5 @@ export interface Token extends PartialYastBlockToken<T> {
|
|
|
45
45
|
*/
|
|
46
46
|
_identifier?: string;
|
|
47
47
|
}
|
|
48
|
-
export declare type
|
|
48
|
+
export declare type IThis = ITokenizer;
|
|
49
|
+
export declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
|