@yozora/tokenizer-definition 1.2.0 → 2.0.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/lib/cjs/index.js +153 -145
  2. package/lib/esm/index.js +152 -146
  3. package/lib/types/index.d.ts +4 -4
  4. package/lib/types/match.d.ts +18 -0
  5. package/lib/types/parse.d.ts +3 -0
  6. package/lib/types/tokenizer.d.ts +6 -38
  7. package/lib/types/types.d.ts +7 -6
  8. package/lib/types/util/link-destination.d.ts +5 -5
  9. package/lib/types/util/link-label.d.ts +5 -5
  10. package/lib/types/util/link-title.d.ts +5 -5
  11. package/package.json +5 -5
package/lib/cjs/index.js CHANGED
@@ -2,16 +2,14 @@
2
2
 
3
3
  Object.defineProperty(exports, '__esModule', { value: true });
4
4
 
5
- var ast = require('@yozora/ast');
6
5
  var character = require('@yozora/character');
7
6
  var coreTokenizer = require('@yozora/core-tokenizer');
7
+ var ast = require('@yozora/ast');
8
8
 
9
- const uniqueName = '@yozora/tokenizer-definition';
10
-
11
- function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
9
+ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, state) {
12
10
  let i = startIndex;
13
- if (token == null) {
14
- token = {
11
+ if (state == null) {
12
+ state = {
15
13
  saturated: false,
16
14
  nodePoints: [],
17
15
  hasOpenAngleBracket: false,
@@ -20,79 +18,78 @@ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
20
18
  }
21
19
  const firstNonWhitespaceIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
22
20
  if (firstNonWhitespaceIndex >= endIndex)
23
- return { nextIndex: -1, token };
24
- if (token.nodePoints.length <= 0) {
21
+ return { nextIndex: -1, state: state };
22
+ if (state.nodePoints.length <= 0) {
25
23
  i = firstNonWhitespaceIndex;
26
24
  const p = nodePoints[i];
27
25
  if (p.codePoint === character.AsciiCodePoint.OPEN_ANGLE) {
28
26
  i += 1;
29
- token.hasOpenAngleBracket = true;
30
- token.nodePoints.push(p);
27
+ state.hasOpenAngleBracket = true;
28
+ state.nodePoints.push(p);
31
29
  }
32
30
  }
33
- if (token.hasOpenAngleBracket) {
31
+ if (state.hasOpenAngleBracket) {
34
32
  for (; i < endIndex; ++i) {
35
33
  const p = nodePoints[i];
36
34
  switch (p.codePoint) {
37
35
  case character.AsciiCodePoint.BACKSLASH:
38
36
  if (i + 1 < endIndex) {
39
- token.nodePoints.push(p);
40
- token.nodePoints.push(nodePoints[i + 1]);
37
+ state.nodePoints.push(p);
38
+ state.nodePoints.push(nodePoints[i + 1]);
41
39
  }
42
40
  i += 1;
43
41
  break;
44
42
  case character.AsciiCodePoint.OPEN_ANGLE:
45
43
  case character.VirtualCodePoint.LINE_END:
46
- return { nextIndex: -1, token };
44
+ return { nextIndex: -1, state: state };
47
45
  case character.AsciiCodePoint.CLOSE_ANGLE:
48
- token.saturated = true;
49
- token.nodePoints.push(p);
50
- return { nextIndex: i + 1, token };
46
+ state.saturated = true;
47
+ state.nodePoints.push(p);
48
+ return { nextIndex: i + 1, state: state };
51
49
  default:
52
- token.nodePoints.push(p);
50
+ state.nodePoints.push(p);
53
51
  }
54
52
  }
55
- return { nextIndex: i, token };
53
+ return { nextIndex: i, state: state };
56
54
  }
57
55
  for (; i < endIndex; ++i) {
58
56
  const p = nodePoints[i];
59
57
  switch (p.codePoint) {
60
58
  case character.AsciiCodePoint.BACKSLASH:
61
59
  if (i + 1 < endIndex) {
62
- token.nodePoints.push(p);
63
- token.nodePoints.push(nodePoints[i + 1]);
60
+ state.nodePoints.push(p);
61
+ state.nodePoints.push(nodePoints[i + 1]);
64
62
  }
65
63
  i += 1;
66
64
  break;
67
65
  case character.AsciiCodePoint.OPEN_PARENTHESIS:
68
- token.openParensCount += 1;
69
- token.nodePoints.push(p);
66
+ state.openParensCount += 1;
67
+ state.nodePoints.push(p);
70
68
  break;
71
69
  case character.AsciiCodePoint.CLOSE_PARENTHESIS:
72
- token.openParensCount -= 1;
73
- token.nodePoints.push(p);
74
- if (token.openParensCount < 0) {
75
- return { nextIndex: i, token };
70
+ state.openParensCount -= 1;
71
+ state.nodePoints.push(p);
72
+ if (state.openParensCount < 0) {
73
+ return { nextIndex: i, state: state };
76
74
  }
77
75
  break;
78
76
  default:
79
- if (character.isWhitespaceCharacter(p.codePoint) ||
80
- character.isAsciiControlCharacter(p.codePoint)) {
81
- token.saturated = true;
82
- return { nextIndex: i, token };
77
+ if (character.isWhitespaceCharacter(p.codePoint) || character.isAsciiControlCharacter(p.codePoint)) {
78
+ state.saturated = true;
79
+ return { nextIndex: i, state: state };
83
80
  }
84
- token.nodePoints.push(p);
81
+ state.nodePoints.push(p);
85
82
  break;
86
83
  }
87
84
  }
88
- token.saturated = true;
89
- return { nextIndex: i, token };
85
+ state.saturated = true;
86
+ return { nextIndex: i, state: state };
90
87
  }
91
88
 
92
- function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
89
+ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, state) {
93
90
  let i = startIndex;
94
- if (token == null) {
95
- token = {
91
+ if (state == null) {
92
+ state = {
96
93
  saturated: false,
97
94
  nodePoints: [],
98
95
  hasNonWhitespaceCharacter: false,
@@ -100,50 +97,50 @@ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
100
97
  }
101
98
  const firstNonWhitespaceIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
102
99
  if (firstNonWhitespaceIndex >= endIndex)
103
- return { nextIndex: -1, token };
104
- if (token.nodePoints.length <= 0) {
100
+ return { nextIndex: -1, state: state };
101
+ if (state.nodePoints.length <= 0) {
105
102
  i = firstNonWhitespaceIndex;
106
103
  const p = nodePoints[i];
107
104
  if (p.codePoint !== character.AsciiCodePoint.OPEN_BRACKET) {
108
- return { nextIndex: -1, token };
105
+ return { nextIndex: -1, state: state };
109
106
  }
110
107
  i += 1;
111
- token.nodePoints.push(p);
108
+ state.nodePoints.push(p);
112
109
  }
113
110
  for (; i < endIndex; ++i) {
114
111
  const p = nodePoints[i];
115
112
  switch (p.codePoint) {
116
113
  case character.AsciiCodePoint.BACKSLASH:
117
- token.hasNonWhitespaceCharacter = true;
114
+ state.hasNonWhitespaceCharacter = true;
118
115
  if (i + 1 < endIndex) {
119
- token.nodePoints.push(p);
120
- token.nodePoints.push(nodePoints[i + 1]);
116
+ state.nodePoints.push(p);
117
+ state.nodePoints.push(nodePoints[i + 1]);
121
118
  }
122
119
  i += 1;
123
120
  break;
124
121
  case character.AsciiCodePoint.OPEN_BRACKET:
125
- return { nextIndex: -1, token };
122
+ return { nextIndex: -1, state: state };
126
123
  case character.AsciiCodePoint.CLOSE_BRACKET:
127
- token.nodePoints.push(p);
128
- if (token.hasNonWhitespaceCharacter) {
129
- token.saturated = true;
130
- return { nextIndex: i + 1, token };
124
+ state.nodePoints.push(p);
125
+ if (state.hasNonWhitespaceCharacter) {
126
+ state.saturated = true;
127
+ return { nextIndex: i + 1, state: state };
131
128
  }
132
- return { nextIndex: -1, token };
129
+ return { nextIndex: -1, state: state };
133
130
  default:
134
131
  if (!character.isWhitespaceCharacter(p.codePoint)) {
135
- token.hasNonWhitespaceCharacter = true;
132
+ state.hasNonWhitespaceCharacter = true;
136
133
  }
137
- token.nodePoints.push(p);
134
+ state.nodePoints.push(p);
138
135
  }
139
136
  }
140
- return { nextIndex: 1, token };
137
+ return { nextIndex: 1, state: state };
141
138
  }
142
139
 
143
- function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
140
+ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, state) {
144
141
  let i = startIndex;
145
- if (token == null) {
146
- token = {
142
+ if (state == null) {
143
+ state = {
147
144
  saturated: false,
148
145
  nodePoints: [],
149
146
  wrapSymbol: null,
@@ -151,25 +148,25 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
151
148
  }
152
149
  const firstNonWhitespaceIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
153
150
  if (firstNonWhitespaceIndex >= endIndex)
154
- return { nextIndex: -1, token };
155
- if (token.nodePoints.length <= 0) {
151
+ return { nextIndex: -1, state: state };
152
+ if (state.nodePoints.length <= 0) {
156
153
  i = firstNonWhitespaceIndex;
157
154
  const p = nodePoints[i];
158
155
  switch (p.codePoint) {
159
156
  case character.AsciiCodePoint.DOUBLE_QUOTE:
160
157
  case character.AsciiCodePoint.SINGLE_QUOTE:
161
158
  case character.AsciiCodePoint.OPEN_PARENTHESIS:
162
- token.wrapSymbol = p.codePoint;
163
- token.nodePoints.push(p);
159
+ state.wrapSymbol = p.codePoint;
160
+ state.nodePoints.push(p);
164
161
  i += 1;
165
162
  break;
166
163
  default:
167
- return { nextIndex: -1, token };
164
+ return { nextIndex: -1, state: state };
168
165
  }
169
166
  }
170
- if (token.wrapSymbol == null)
171
- return { nextIndex: -1, token };
172
- switch (token.wrapSymbol) {
167
+ if (state.wrapSymbol == null)
168
+ return { nextIndex: -1, state: state };
169
+ switch (state.wrapSymbol) {
173
170
  case character.AsciiCodePoint.DOUBLE_QUOTE:
174
171
  case character.AsciiCodePoint.SINGLE_QUOTE: {
175
172
  for (; i < endIndex; ++i) {
@@ -177,17 +174,17 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
177
174
  switch (p.codePoint) {
178
175
  case character.AsciiCodePoint.BACKSLASH:
179
176
  if (i + 1 < endIndex) {
180
- token.nodePoints.push(p);
181
- token.nodePoints.push(nodePoints[i + 1]);
177
+ state.nodePoints.push(p);
178
+ state.nodePoints.push(nodePoints[i + 1]);
182
179
  }
183
180
  i += 1;
184
181
  break;
185
- case token.wrapSymbol:
186
- token.saturated = true;
187
- token.nodePoints.push(p);
188
- return { nextIndex: i + 1, token };
182
+ case state.wrapSymbol:
183
+ state.saturated = true;
184
+ state.nodePoints.push(p);
185
+ return { nextIndex: i + 1, state: state };
189
186
  default:
190
- token.nodePoints.push(p);
187
+ state.nodePoints.push(p);
191
188
  }
192
189
  }
193
190
  break;
@@ -198,49 +195,46 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
198
195
  switch (p.codePoint) {
199
196
  case character.AsciiCodePoint.BACKSLASH:
200
197
  if (i + 1 < endIndex) {
201
- token.nodePoints.push(p);
202
- token.nodePoints.push(nodePoints[i + 1]);
198
+ state.nodePoints.push(p);
199
+ state.nodePoints.push(nodePoints[i + 1]);
203
200
  }
204
201
  i += 1;
205
202
  break;
206
203
  case character.AsciiCodePoint.OPEN_PARENTHESIS:
207
- return { nextIndex: -1, token };
204
+ return { nextIndex: -1, state: state };
208
205
  case character.AsciiCodePoint.CLOSE_PARENTHESIS:
209
- if (i + 1 >= endIndex ||
210
- nodePoints[i + 1].codePoint === character.VirtualCodePoint.LINE_END) {
211
- token.nodePoints.push(p);
212
- token.saturated = true;
206
+ if (i + 1 >= endIndex || nodePoints[i + 1].codePoint === character.VirtualCodePoint.LINE_END) {
207
+ state.nodePoints.push(p);
208
+ state.saturated = true;
213
209
  break;
214
210
  }
215
- return { nextIndex: -1, token };
211
+ return { nextIndex: -1, state: state };
216
212
  default:
217
- token.nodePoints.push(p);
213
+ state.nodePoints.push(p);
218
214
  }
219
215
  }
220
216
  break;
221
217
  }
222
218
  }
223
- return { nextIndex: endIndex, token };
219
+ return { nextIndex: endIndex, state: state };
224
220
  }
225
221
 
226
- class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
227
- constructor(props = {}) {
228
- var _a, _b;
229
- super({
230
- name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
231
- priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
232
- });
233
- this.isContainingBlock = false;
234
- }
235
- eatOpener(line) {
222
+ const match = function (api) {
223
+ return {
224
+ isContainingBlock: false,
225
+ eatOpener,
226
+ eatContinuationText,
227
+ onClose,
228
+ };
229
+ function eatOpener(line) {
236
230
  if (line.countOfPrecedeSpaces >= 4)
237
231
  return null;
238
232
  const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line;
239
233
  if (firstNonWhitespaceIndex >= endIndex)
240
234
  return null;
241
235
  let i = firstNonWhitespaceIndex;
242
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
243
- if (linkLabelCollectResult.nextIndex < 0)
236
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
237
+ if (labelEndIndex < 0)
244
238
  return null;
245
239
  const lineNo = nodePoints[startIndex].line;
246
240
  const createInitState = () => {
@@ -250,7 +244,7 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
250
244
  start: coreTokenizer.calcStartYastNodePoint(nodePoints, startIndex),
251
245
  end: coreTokenizer.calcEndYastNodePoint(nodePoints, endIndex - 1),
252
246
  },
253
- label: linkLabelCollectResult.token,
247
+ label: labelState,
254
248
  destination: null,
255
249
  title: null,
256
250
  lineNoOfLabel: lineNo,
@@ -260,11 +254,10 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
260
254
  };
261
255
  return token;
262
256
  };
263
- if (!linkLabelCollectResult.token.saturated) {
257
+ if (!labelState.saturated) {
264
258
  const token = createInitState();
265
259
  return { token, nextIndex: endIndex };
266
260
  }
267
- const labelEndIndex = linkLabelCollectResult.nextIndex;
268
261
  if (labelEndIndex < 0 ||
269
262
  labelEndIndex + 1 >= endIndex ||
270
263
  nodePoints[labelEndIndex].codePoint !== character.AsciiCodePoint.COLON)
@@ -274,39 +267,36 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
274
267
  const token = createInitState();
275
268
  return { token, nextIndex: endIndex };
276
269
  }
277
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
278
- if (linkDestinationCollectResult.nextIndex < 0)
270
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
271
+ if (destinationEndIndex < 0)
279
272
  return null;
280
- if (!linkDestinationCollectResult.token.saturated &&
281
- linkDestinationCollectResult.nextIndex !== endIndex)
273
+ if (!destinationState.saturated && destinationEndIndex !== endIndex)
282
274
  return null;
283
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
284
275
  i = coreTokenizer.eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
285
276
  if (i >= endIndex) {
286
277
  const token = createInitState();
287
- token.destination = linkDestinationCollectResult.token;
278
+ token.destination = destinationState;
288
279
  token.lineNoOfDestination = lineNo;
289
280
  return { token, nextIndex: endIndex };
290
281
  }
291
282
  if (i === destinationEndIndex)
292
283
  return null;
293
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
294
- if (linkTitleCollectResult.nextIndex >= 0) {
295
- i = linkTitleCollectResult.nextIndex;
296
- }
284
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
285
+ if (titleEndIndex >= 0)
286
+ i = titleEndIndex;
297
287
  if (i < endIndex) {
298
288
  const k = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
299
289
  if (k < endIndex)
300
290
  return null;
301
291
  }
302
292
  const token = createInitState();
303
- token.destination = linkDestinationCollectResult.token;
304
- token.title = linkTitleCollectResult.token;
293
+ token.destination = destinationState;
294
+ token.title = titleState;
305
295
  token.lineNoOfDestination = lineNo;
306
296
  token.lineNoOfTitle = lineNo;
307
297
  return { token, nextIndex: endIndex };
308
298
  }
309
- eatContinuationText(line, token) {
299
+ function eatContinuationText(line, token) {
310
300
  var _a;
311
301
  if (token.title != null && token.title.saturated)
312
302
  return { status: 'notMatched' };
@@ -314,12 +304,11 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
314
304
  const lineNo = nodePoints[startIndex].line;
315
305
  let i = firstNonWhitespaceIndex;
316
306
  if (!token.label.saturated) {
317
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
318
- if (linkLabelCollectResult.nextIndex < 0) {
307
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
308
+ if (labelEndIndex < 0) {
319
309
  return { status: 'failedAndRollback', lines: token.lines };
320
310
  }
321
- const labelEndIndex = linkLabelCollectResult.nextIndex;
322
- if (!linkLabelCollectResult.token.saturated) {
311
+ if (!labelState.saturated) {
323
312
  token.lines.push(line);
324
313
  return { status: 'opening', nextIndex: endIndex };
325
314
  }
@@ -334,15 +323,13 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
334
323
  if (i >= endIndex) {
335
324
  return { status: 'failedAndRollback', lines: token.lines };
336
325
  }
337
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
338
- if (linkDestinationCollectResult.nextIndex < 0 ||
339
- !linkDestinationCollectResult.token.saturated) {
326
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
327
+ if (destinationEndIndex < 0 || !destinationState.saturated) {
340
328
  return { status: 'failedAndRollback', lines: token.lines };
341
329
  }
342
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
343
330
  i = coreTokenizer.eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
344
331
  if (i >= endIndex) {
345
- token.destination = linkDestinationCollectResult.token;
332
+ token.destination = destinationState;
346
333
  token.lines.push(line);
347
334
  return { status: 'opening', nextIndex: endIndex };
348
335
  }
@@ -352,12 +339,12 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
352
339
  if (token.lineNoOfTitle < 0) {
353
340
  token.lineNoOfTitle = lineNo;
354
341
  }
355
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
356
- token.title = linkTitleCollectResult.token;
357
- if (linkTitleCollectResult.nextIndex < 0 ||
358
- linkTitleCollectResult.token.nodePoints.length <= 0 ||
359
- (linkTitleCollectResult.token.saturated &&
360
- coreTokenizer.eatOptionalWhitespaces(nodePoints, linkTitleCollectResult.nextIndex, endIndex) < endIndex)) {
342
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
343
+ token.title = titleState;
344
+ if (titleEndIndex < 0 ||
345
+ titleState.nodePoints.length <= 0 ||
346
+ (titleState.saturated &&
347
+ coreTokenizer.eatOptionalWhitespaces(nodePoints, titleEndIndex, endIndex) < endIndex)) {
361
348
  if (token.lineNoOfDestination === token.lineNoOfTitle) {
362
349
  return { status: 'failedAndRollback', lines: token.lines };
363
350
  }
@@ -373,7 +360,7 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
373
360
  const saturated = (_a = token.title) === null || _a === void 0 ? void 0 : _a.saturated;
374
361
  return { status: saturated ? 'closing' : 'opening', nextIndex: endIndex };
375
362
  }
376
- onClose(token, api) {
363
+ function onClose(token) {
377
364
  let result;
378
365
  if (token.title == null || !token.title.saturated) {
379
366
  if (!token.label.saturated) {
@@ -401,31 +388,52 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
401
388
  token._identifier = identifier;
402
389
  return result;
403
390
  }
404
- parseBlock(token) {
405
- const label = token._label;
406
- const identifier = token._identifier;
407
- const destinationPoints = token.destination.nodePoints;
408
- const destination = destinationPoints[0].codePoint === character.AsciiCodePoint.OPEN_ANGLE
409
- ? character.calcEscapedStringFromNodePoints(destinationPoints, 1, destinationPoints.length - 1, true)
410
- : character.calcEscapedStringFromNodePoints(destinationPoints, 0, destinationPoints.length, true);
411
- const url = coreTokenizer.encodeLinkDestination(destination);
412
- const title = token.title == null
413
- ? undefined
414
- : character.calcEscapedStringFromNodePoints(token.title.nodePoints, 1, token.title.nodePoints.length - 1);
415
- const node = {
416
- type: ast.DefinitionType,
417
- identifier,
418
- label,
419
- url,
420
- title,
421
- };
422
- return node;
391
+ };
392
+
393
+ const parse = function () {
394
+ return {
395
+ parse: token => {
396
+ const label = token._label;
397
+ const identifier = token._identifier;
398
+ const destinationPoints = token.destination.nodePoints;
399
+ const destination = destinationPoints[0].codePoint === character.AsciiCodePoint.OPEN_ANGLE
400
+ ? character.calcEscapedStringFromNodePoints(destinationPoints, 1, destinationPoints.length - 1, true)
401
+ : character.calcEscapedStringFromNodePoints(destinationPoints, 0, destinationPoints.length, true);
402
+ const url = coreTokenizer.encodeLinkDestination(destination);
403
+ const title = token.title == null
404
+ ? undefined
405
+ : character.calcEscapedStringFromNodePoints(token.title.nodePoints, 1, token.title.nodePoints.length - 1);
406
+ const node = {
407
+ type: ast.DefinitionType,
408
+ identifier,
409
+ label,
410
+ url,
411
+ title,
412
+ };
413
+ return node;
414
+ },
415
+ };
416
+ };
417
+
418
+ const uniqueName = '@yozora/tokenizer-definition';
419
+
420
+ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
421
+ constructor(props = {}) {
422
+ var _a, _b;
423
+ super({
424
+ name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
425
+ priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
426
+ });
427
+ this.match = match;
428
+ this.parse = parse;
423
429
  }
424
430
  }
425
431
 
426
432
  exports.DefinitionTokenizer = DefinitionTokenizer;
427
433
  exports.DefinitionTokenizerName = uniqueName;
428
- exports['default'] = DefinitionTokenizer;
434
+ exports["default"] = DefinitionTokenizer;
435
+ exports.definitionMatch = match;
436
+ exports.definitionParse = parse;
429
437
  exports.eatAndCollectLinkDestination = eatAndCollectLinkDestination;
430
438
  exports.eatAndCollectLinkLabel = eatAndCollectLinkLabel;
431
439
  exports.eatAndCollectLinkTitle = eatAndCollectLinkTitle;
package/lib/esm/index.js CHANGED
@@ -1,13 +1,11 @@
1
- import { DefinitionType } from '@yozora/ast';
2
1
  import { AsciiCodePoint, VirtualCodePoint, isWhitespaceCharacter, isAsciiControlCharacter, calcStringFromNodePoints, calcEscapedStringFromNodePoints } from '@yozora/character';
3
- import { eatOptionalWhitespaces, BaseBlockTokenizer, TokenizerPriority, calcEndYastNodePoint, resolveLabelToIdentifier, encodeLinkDestination, calcStartYastNodePoint } from '@yozora/core-tokenizer';
4
-
5
- const uniqueName = '@yozora/tokenizer-definition';
2
+ import { eatOptionalWhitespaces, calcEndYastNodePoint, resolveLabelToIdentifier, calcStartYastNodePoint, encodeLinkDestination, BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
3
+ import { DefinitionType } from '@yozora/ast';
6
4
 
7
- function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
5
+ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, state) {
8
6
  let i = startIndex;
9
- if (token == null) {
10
- token = {
7
+ if (state == null) {
8
+ state = {
11
9
  saturated: false,
12
10
  nodePoints: [],
13
11
  hasOpenAngleBracket: false,
@@ -16,79 +14,78 @@ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
16
14
  }
17
15
  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
18
16
  if (firstNonWhitespaceIndex >= endIndex)
19
- return { nextIndex: -1, token };
20
- if (token.nodePoints.length <= 0) {
17
+ return { nextIndex: -1, state: state };
18
+ if (state.nodePoints.length <= 0) {
21
19
  i = firstNonWhitespaceIndex;
22
20
  const p = nodePoints[i];
23
21
  if (p.codePoint === AsciiCodePoint.OPEN_ANGLE) {
24
22
  i += 1;
25
- token.hasOpenAngleBracket = true;
26
- token.nodePoints.push(p);
23
+ state.hasOpenAngleBracket = true;
24
+ state.nodePoints.push(p);
27
25
  }
28
26
  }
29
- if (token.hasOpenAngleBracket) {
27
+ if (state.hasOpenAngleBracket) {
30
28
  for (; i < endIndex; ++i) {
31
29
  const p = nodePoints[i];
32
30
  switch (p.codePoint) {
33
31
  case AsciiCodePoint.BACKSLASH:
34
32
  if (i + 1 < endIndex) {
35
- token.nodePoints.push(p);
36
- token.nodePoints.push(nodePoints[i + 1]);
33
+ state.nodePoints.push(p);
34
+ state.nodePoints.push(nodePoints[i + 1]);
37
35
  }
38
36
  i += 1;
39
37
  break;
40
38
  case AsciiCodePoint.OPEN_ANGLE:
41
39
  case VirtualCodePoint.LINE_END:
42
- return { nextIndex: -1, token };
40
+ return { nextIndex: -1, state: state };
43
41
  case AsciiCodePoint.CLOSE_ANGLE:
44
- token.saturated = true;
45
- token.nodePoints.push(p);
46
- return { nextIndex: i + 1, token };
42
+ state.saturated = true;
43
+ state.nodePoints.push(p);
44
+ return { nextIndex: i + 1, state: state };
47
45
  default:
48
- token.nodePoints.push(p);
46
+ state.nodePoints.push(p);
49
47
  }
50
48
  }
51
- return { nextIndex: i, token };
49
+ return { nextIndex: i, state: state };
52
50
  }
53
51
  for (; i < endIndex; ++i) {
54
52
  const p = nodePoints[i];
55
53
  switch (p.codePoint) {
56
54
  case AsciiCodePoint.BACKSLASH:
57
55
  if (i + 1 < endIndex) {
58
- token.nodePoints.push(p);
59
- token.nodePoints.push(nodePoints[i + 1]);
56
+ state.nodePoints.push(p);
57
+ state.nodePoints.push(nodePoints[i + 1]);
60
58
  }
61
59
  i += 1;
62
60
  break;
63
61
  case AsciiCodePoint.OPEN_PARENTHESIS:
64
- token.openParensCount += 1;
65
- token.nodePoints.push(p);
62
+ state.openParensCount += 1;
63
+ state.nodePoints.push(p);
66
64
  break;
67
65
  case AsciiCodePoint.CLOSE_PARENTHESIS:
68
- token.openParensCount -= 1;
69
- token.nodePoints.push(p);
70
- if (token.openParensCount < 0) {
71
- return { nextIndex: i, token };
66
+ state.openParensCount -= 1;
67
+ state.nodePoints.push(p);
68
+ if (state.openParensCount < 0) {
69
+ return { nextIndex: i, state: state };
72
70
  }
73
71
  break;
74
72
  default:
75
- if (isWhitespaceCharacter(p.codePoint) ||
76
- isAsciiControlCharacter(p.codePoint)) {
77
- token.saturated = true;
78
- return { nextIndex: i, token };
73
+ if (isWhitespaceCharacter(p.codePoint) || isAsciiControlCharacter(p.codePoint)) {
74
+ state.saturated = true;
75
+ return { nextIndex: i, state: state };
79
76
  }
80
- token.nodePoints.push(p);
77
+ state.nodePoints.push(p);
81
78
  break;
82
79
  }
83
80
  }
84
- token.saturated = true;
85
- return { nextIndex: i, token };
81
+ state.saturated = true;
82
+ return { nextIndex: i, state: state };
86
83
  }
87
84
 
88
- function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
85
+ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, state) {
89
86
  let i = startIndex;
90
- if (token == null) {
91
- token = {
87
+ if (state == null) {
88
+ state = {
92
89
  saturated: false,
93
90
  nodePoints: [],
94
91
  hasNonWhitespaceCharacter: false,
@@ -96,50 +93,50 @@ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
96
93
  }
97
94
  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
98
95
  if (firstNonWhitespaceIndex >= endIndex)
99
- return { nextIndex: -1, token };
100
- if (token.nodePoints.length <= 0) {
96
+ return { nextIndex: -1, state: state };
97
+ if (state.nodePoints.length <= 0) {
101
98
  i = firstNonWhitespaceIndex;
102
99
  const p = nodePoints[i];
103
100
  if (p.codePoint !== AsciiCodePoint.OPEN_BRACKET) {
104
- return { nextIndex: -1, token };
101
+ return { nextIndex: -1, state: state };
105
102
  }
106
103
  i += 1;
107
- token.nodePoints.push(p);
104
+ state.nodePoints.push(p);
108
105
  }
109
106
  for (; i < endIndex; ++i) {
110
107
  const p = nodePoints[i];
111
108
  switch (p.codePoint) {
112
109
  case AsciiCodePoint.BACKSLASH:
113
- token.hasNonWhitespaceCharacter = true;
110
+ state.hasNonWhitespaceCharacter = true;
114
111
  if (i + 1 < endIndex) {
115
- token.nodePoints.push(p);
116
- token.nodePoints.push(nodePoints[i + 1]);
112
+ state.nodePoints.push(p);
113
+ state.nodePoints.push(nodePoints[i + 1]);
117
114
  }
118
115
  i += 1;
119
116
  break;
120
117
  case AsciiCodePoint.OPEN_BRACKET:
121
- return { nextIndex: -1, token };
118
+ return { nextIndex: -1, state: state };
122
119
  case AsciiCodePoint.CLOSE_BRACKET:
123
- token.nodePoints.push(p);
124
- if (token.hasNonWhitespaceCharacter) {
125
- token.saturated = true;
126
- return { nextIndex: i + 1, token };
120
+ state.nodePoints.push(p);
121
+ if (state.hasNonWhitespaceCharacter) {
122
+ state.saturated = true;
123
+ return { nextIndex: i + 1, state: state };
127
124
  }
128
- return { nextIndex: -1, token };
125
+ return { nextIndex: -1, state: state };
129
126
  default:
130
127
  if (!isWhitespaceCharacter(p.codePoint)) {
131
- token.hasNonWhitespaceCharacter = true;
128
+ state.hasNonWhitespaceCharacter = true;
132
129
  }
133
- token.nodePoints.push(p);
130
+ state.nodePoints.push(p);
134
131
  }
135
132
  }
136
- return { nextIndex: 1, token };
133
+ return { nextIndex: 1, state: state };
137
134
  }
138
135
 
139
- function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
136
+ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, state) {
140
137
  let i = startIndex;
141
- if (token == null) {
142
- token = {
138
+ if (state == null) {
139
+ state = {
143
140
  saturated: false,
144
141
  nodePoints: [],
145
142
  wrapSymbol: null,
@@ -147,25 +144,25 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
147
144
  }
148
145
  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
149
146
  if (firstNonWhitespaceIndex >= endIndex)
150
- return { nextIndex: -1, token };
151
- if (token.nodePoints.length <= 0) {
147
+ return { nextIndex: -1, state: state };
148
+ if (state.nodePoints.length <= 0) {
152
149
  i = firstNonWhitespaceIndex;
153
150
  const p = nodePoints[i];
154
151
  switch (p.codePoint) {
155
152
  case AsciiCodePoint.DOUBLE_QUOTE:
156
153
  case AsciiCodePoint.SINGLE_QUOTE:
157
154
  case AsciiCodePoint.OPEN_PARENTHESIS:
158
- token.wrapSymbol = p.codePoint;
159
- token.nodePoints.push(p);
155
+ state.wrapSymbol = p.codePoint;
156
+ state.nodePoints.push(p);
160
157
  i += 1;
161
158
  break;
162
159
  default:
163
- return { nextIndex: -1, token };
160
+ return { nextIndex: -1, state: state };
164
161
  }
165
162
  }
166
- if (token.wrapSymbol == null)
167
- return { nextIndex: -1, token };
168
- switch (token.wrapSymbol) {
163
+ if (state.wrapSymbol == null)
164
+ return { nextIndex: -1, state: state };
165
+ switch (state.wrapSymbol) {
169
166
  case AsciiCodePoint.DOUBLE_QUOTE:
170
167
  case AsciiCodePoint.SINGLE_QUOTE: {
171
168
  for (; i < endIndex; ++i) {
@@ -173,17 +170,17 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
173
170
  switch (p.codePoint) {
174
171
  case AsciiCodePoint.BACKSLASH:
175
172
  if (i + 1 < endIndex) {
176
- token.nodePoints.push(p);
177
- token.nodePoints.push(nodePoints[i + 1]);
173
+ state.nodePoints.push(p);
174
+ state.nodePoints.push(nodePoints[i + 1]);
178
175
  }
179
176
  i += 1;
180
177
  break;
181
- case token.wrapSymbol:
182
- token.saturated = true;
183
- token.nodePoints.push(p);
184
- return { nextIndex: i + 1, token };
178
+ case state.wrapSymbol:
179
+ state.saturated = true;
180
+ state.nodePoints.push(p);
181
+ return { nextIndex: i + 1, state: state };
185
182
  default:
186
- token.nodePoints.push(p);
183
+ state.nodePoints.push(p);
187
184
  }
188
185
  }
189
186
  break;
@@ -194,49 +191,46 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
194
191
  switch (p.codePoint) {
195
192
  case AsciiCodePoint.BACKSLASH:
196
193
  if (i + 1 < endIndex) {
197
- token.nodePoints.push(p);
198
- token.nodePoints.push(nodePoints[i + 1]);
194
+ state.nodePoints.push(p);
195
+ state.nodePoints.push(nodePoints[i + 1]);
199
196
  }
200
197
  i += 1;
201
198
  break;
202
199
  case AsciiCodePoint.OPEN_PARENTHESIS:
203
- return { nextIndex: -1, token };
200
+ return { nextIndex: -1, state: state };
204
201
  case AsciiCodePoint.CLOSE_PARENTHESIS:
205
- if (i + 1 >= endIndex ||
206
- nodePoints[i + 1].codePoint === VirtualCodePoint.LINE_END) {
207
- token.nodePoints.push(p);
208
- token.saturated = true;
202
+ if (i + 1 >= endIndex || nodePoints[i + 1].codePoint === VirtualCodePoint.LINE_END) {
203
+ state.nodePoints.push(p);
204
+ state.saturated = true;
209
205
  break;
210
206
  }
211
- return { nextIndex: -1, token };
207
+ return { nextIndex: -1, state: state };
212
208
  default:
213
- token.nodePoints.push(p);
209
+ state.nodePoints.push(p);
214
210
  }
215
211
  }
216
212
  break;
217
213
  }
218
214
  }
219
- return { nextIndex: endIndex, token };
215
+ return { nextIndex: endIndex, state: state };
220
216
  }
221
217
 
222
- class DefinitionTokenizer extends BaseBlockTokenizer {
223
- constructor(props = {}) {
224
- var _a, _b;
225
- super({
226
- name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
227
- priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
228
- });
229
- this.isContainingBlock = false;
230
- }
231
- eatOpener(line) {
218
+ const match = function (api) {
219
+ return {
220
+ isContainingBlock: false,
221
+ eatOpener,
222
+ eatContinuationText,
223
+ onClose,
224
+ };
225
+ function eatOpener(line) {
232
226
  if (line.countOfPrecedeSpaces >= 4)
233
227
  return null;
234
228
  const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line;
235
229
  if (firstNonWhitespaceIndex >= endIndex)
236
230
  return null;
237
231
  let i = firstNonWhitespaceIndex;
238
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
239
- if (linkLabelCollectResult.nextIndex < 0)
232
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
233
+ if (labelEndIndex < 0)
240
234
  return null;
241
235
  const lineNo = nodePoints[startIndex].line;
242
236
  const createInitState = () => {
@@ -246,7 +240,7 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
246
240
  start: calcStartYastNodePoint(nodePoints, startIndex),
247
241
  end: calcEndYastNodePoint(nodePoints, endIndex - 1),
248
242
  },
249
- label: linkLabelCollectResult.token,
243
+ label: labelState,
250
244
  destination: null,
251
245
  title: null,
252
246
  lineNoOfLabel: lineNo,
@@ -256,11 +250,10 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
256
250
  };
257
251
  return token;
258
252
  };
259
- if (!linkLabelCollectResult.token.saturated) {
253
+ if (!labelState.saturated) {
260
254
  const token = createInitState();
261
255
  return { token, nextIndex: endIndex };
262
256
  }
263
- const labelEndIndex = linkLabelCollectResult.nextIndex;
264
257
  if (labelEndIndex < 0 ||
265
258
  labelEndIndex + 1 >= endIndex ||
266
259
  nodePoints[labelEndIndex].codePoint !== AsciiCodePoint.COLON)
@@ -270,39 +263,36 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
270
263
  const token = createInitState();
271
264
  return { token, nextIndex: endIndex };
272
265
  }
273
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
274
- if (linkDestinationCollectResult.nextIndex < 0)
266
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
267
+ if (destinationEndIndex < 0)
275
268
  return null;
276
- if (!linkDestinationCollectResult.token.saturated &&
277
- linkDestinationCollectResult.nextIndex !== endIndex)
269
+ if (!destinationState.saturated && destinationEndIndex !== endIndex)
278
270
  return null;
279
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
280
271
  i = eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
281
272
  if (i >= endIndex) {
282
273
  const token = createInitState();
283
- token.destination = linkDestinationCollectResult.token;
274
+ token.destination = destinationState;
284
275
  token.lineNoOfDestination = lineNo;
285
276
  return { token, nextIndex: endIndex };
286
277
  }
287
278
  if (i === destinationEndIndex)
288
279
  return null;
289
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
290
- if (linkTitleCollectResult.nextIndex >= 0) {
291
- i = linkTitleCollectResult.nextIndex;
292
- }
280
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
281
+ if (titleEndIndex >= 0)
282
+ i = titleEndIndex;
293
283
  if (i < endIndex) {
294
284
  const k = eatOptionalWhitespaces(nodePoints, i, endIndex);
295
285
  if (k < endIndex)
296
286
  return null;
297
287
  }
298
288
  const token = createInitState();
299
- token.destination = linkDestinationCollectResult.token;
300
- token.title = linkTitleCollectResult.token;
289
+ token.destination = destinationState;
290
+ token.title = titleState;
301
291
  token.lineNoOfDestination = lineNo;
302
292
  token.lineNoOfTitle = lineNo;
303
293
  return { token, nextIndex: endIndex };
304
294
  }
305
- eatContinuationText(line, token) {
295
+ function eatContinuationText(line, token) {
306
296
  var _a;
307
297
  if (token.title != null && token.title.saturated)
308
298
  return { status: 'notMatched' };
@@ -310,12 +300,11 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
310
300
  const lineNo = nodePoints[startIndex].line;
311
301
  let i = firstNonWhitespaceIndex;
312
302
  if (!token.label.saturated) {
313
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
314
- if (linkLabelCollectResult.nextIndex < 0) {
303
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
304
+ if (labelEndIndex < 0) {
315
305
  return { status: 'failedAndRollback', lines: token.lines };
316
306
  }
317
- const labelEndIndex = linkLabelCollectResult.nextIndex;
318
- if (!linkLabelCollectResult.token.saturated) {
307
+ if (!labelState.saturated) {
319
308
  token.lines.push(line);
320
309
  return { status: 'opening', nextIndex: endIndex };
321
310
  }
@@ -330,15 +319,13 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
330
319
  if (i >= endIndex) {
331
320
  return { status: 'failedAndRollback', lines: token.lines };
332
321
  }
333
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
334
- if (linkDestinationCollectResult.nextIndex < 0 ||
335
- !linkDestinationCollectResult.token.saturated) {
322
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
323
+ if (destinationEndIndex < 0 || !destinationState.saturated) {
336
324
  return { status: 'failedAndRollback', lines: token.lines };
337
325
  }
338
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
339
326
  i = eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
340
327
  if (i >= endIndex) {
341
- token.destination = linkDestinationCollectResult.token;
328
+ token.destination = destinationState;
342
329
  token.lines.push(line);
343
330
  return { status: 'opening', nextIndex: endIndex };
344
331
  }
@@ -348,12 +335,12 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
348
335
  if (token.lineNoOfTitle < 0) {
349
336
  token.lineNoOfTitle = lineNo;
350
337
  }
351
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
352
- token.title = linkTitleCollectResult.token;
353
- if (linkTitleCollectResult.nextIndex < 0 ||
354
- linkTitleCollectResult.token.nodePoints.length <= 0 ||
355
- (linkTitleCollectResult.token.saturated &&
356
- eatOptionalWhitespaces(nodePoints, linkTitleCollectResult.nextIndex, endIndex) < endIndex)) {
338
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
339
+ token.title = titleState;
340
+ if (titleEndIndex < 0 ||
341
+ titleState.nodePoints.length <= 0 ||
342
+ (titleState.saturated &&
343
+ eatOptionalWhitespaces(nodePoints, titleEndIndex, endIndex) < endIndex)) {
357
344
  if (token.lineNoOfDestination === token.lineNoOfTitle) {
358
345
  return { status: 'failedAndRollback', lines: token.lines };
359
346
  }
@@ -369,7 +356,7 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
369
356
  const saturated = (_a = token.title) === null || _a === void 0 ? void 0 : _a.saturated;
370
357
  return { status: saturated ? 'closing' : 'opening', nextIndex: endIndex };
371
358
  }
372
- onClose(token, api) {
359
+ function onClose(token) {
373
360
  let result;
374
361
  if (token.title == null || !token.title.saturated) {
375
362
  if (!token.label.saturated) {
@@ -397,26 +384,45 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
397
384
  token._identifier = identifier;
398
385
  return result;
399
386
  }
400
- parseBlock(token) {
401
- const label = token._label;
402
- const identifier = token._identifier;
403
- const destinationPoints = token.destination.nodePoints;
404
- const destination = destinationPoints[0].codePoint === AsciiCodePoint.OPEN_ANGLE
405
- ? calcEscapedStringFromNodePoints(destinationPoints, 1, destinationPoints.length - 1, true)
406
- : calcEscapedStringFromNodePoints(destinationPoints, 0, destinationPoints.length, true);
407
- const url = encodeLinkDestination(destination);
408
- const title = token.title == null
409
- ? undefined
410
- : calcEscapedStringFromNodePoints(token.title.nodePoints, 1, token.title.nodePoints.length - 1);
411
- const node = {
412
- type: DefinitionType,
413
- identifier,
414
- label,
415
- url,
416
- title,
417
- };
418
- return node;
387
+ };
388
+
389
+ const parse = function () {
390
+ return {
391
+ parse: token => {
392
+ const label = token._label;
393
+ const identifier = token._identifier;
394
+ const destinationPoints = token.destination.nodePoints;
395
+ const destination = destinationPoints[0].codePoint === AsciiCodePoint.OPEN_ANGLE
396
+ ? calcEscapedStringFromNodePoints(destinationPoints, 1, destinationPoints.length - 1, true)
397
+ : calcEscapedStringFromNodePoints(destinationPoints, 0, destinationPoints.length, true);
398
+ const url = encodeLinkDestination(destination);
399
+ const title = token.title == null
400
+ ? undefined
401
+ : calcEscapedStringFromNodePoints(token.title.nodePoints, 1, token.title.nodePoints.length - 1);
402
+ const node = {
403
+ type: DefinitionType,
404
+ identifier,
405
+ label,
406
+ url,
407
+ title,
408
+ };
409
+ return node;
410
+ },
411
+ };
412
+ };
413
+
414
+ const uniqueName = '@yozora/tokenizer-definition';
415
+
416
+ class DefinitionTokenizer extends BaseBlockTokenizer {
417
+ constructor(props = {}) {
418
+ var _a, _b;
419
+ super({
420
+ name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
421
+ priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
422
+ });
423
+ this.match = match;
424
+ this.parse = parse;
419
425
  }
420
426
  }
421
427
 
422
- export { DefinitionTokenizer, uniqueName as DefinitionTokenizerName, DefinitionTokenizer as default, eatAndCollectLinkDestination, eatAndCollectLinkLabel, eatAndCollectLinkTitle };
428
+ export { DefinitionTokenizer, uniqueName as DefinitionTokenizerName, DefinitionTokenizer as default, match as definitionMatch, parse as definitionParse, eatAndCollectLinkDestination, eatAndCollectLinkLabel, eatAndCollectLinkTitle };
package/lib/types/index.d.ts CHANGED
@@ -1,8 +1,8 @@
1
- import { DefinitionTokenizer } from './tokenizer';
2
1
  export * from './util/link-destination';
3
2
  export * from './util/link-label';
4
3
  export * from './util/link-title';
5
- export { DefinitionTokenizer } from './tokenizer';
4
+ export { match as definitionMatch } from './match';
5
+ export { parse as definitionParse } from './parse';
6
+ export { DefinitionTokenizer, DefinitionTokenizer as default } from './tokenizer';
6
7
  export { uniqueName as DefinitionTokenizerName } from './types';
7
- export type { Token as DefinitionToken, TokenizerProps as DefinitionTokenizerProps, } from './types';
8
- export default DefinitionTokenizer;
8
+ export type { IHookContext as IDefinitionHookContext, IToken as IDefinitionToken, ITokenizerProps as IDefinitionTokenizerProps, } from './types';
package/lib/types/match.d.ts ADDED
@@ -0,0 +1,18 @@
1
+ import type { IMatchBlockHookCreator } from '@yozora/core-tokenizer';
2
+ import type { IHookContext, IToken, T } from './types';
3
+ /**
4
+ * A link reference definition consists of a link label, indented up to three
5
+ * spaces, followed by a colon (:), optional whitespace (including up to one
6
+ * line ending), a link destination, optional whitespace (including up to one
7
+ * line ending), and an optional link title, which if it is present must be
8
+ * separated from the link destination by whitespace. No further non-whitespace
9
+ * characters may occur on the line.
10
+ *
11
+ * A link reference definition does not correspond to a structural element of
12
+ * a document. Instead, it defines a label which can be used in reference
13
+ * links and reference-style images elsewhere in the document. Link reference
14
+ * definitions can come either before or after the links that use them.
15
+ *
16
+ * @see https://github.github.com/gfm/#link-reference-definition
17
+ */
18
+ export declare const match: IMatchBlockHookCreator<T, IToken, IHookContext>;
package/lib/types/parse.d.ts ADDED
@@ -0,0 +1,3 @@
1
+ import type { IParseBlockHookCreator } from '@yozora/core-tokenizer';
2
+ import type { IHookContext, INode, IToken, T } from './types';
3
+ export declare const parse: IParseBlockHookCreator<T, IToken, INode, IHookContext>;
package/lib/types/tokenizer.d.ts CHANGED
@@ -1,44 +1,12 @@
1
- import type { MatchBlockPhaseApi, PhrasingContentLine, ResultOfEatContinuationText, ResultOfEatOpener, ResultOfOnClose, ResultOfParse, Tokenizer, TokenizerMatchBlockHook, TokenizerParseBlockHook } from '@yozora/core-tokenizer';
1
+ import type { IBlockTokenizer, IMatchBlockHookCreator, IParseBlockHookCreator } from '@yozora/core-tokenizer';
2
2
  import { BaseBlockTokenizer } from '@yozora/core-tokenizer';
3
- import type { Node, T, Token, TokenizerProps } from './types';
3
+ import type { IHookContext, INode, IToken, ITokenizerProps, T } from './types';
4
4
  /**
5
5
  * Lexical Analyzer for Definition.
6
- *
7
- * A link reference definition consists of a link label, indented up to three
8
- * spaces, followed by a colon (:), optional whitespace (including up to one
9
- * line ending), a link destination, optional whitespace (including up to one
10
- * line ending), and an optional link title, which if it is present must be
11
- * separated from the link destination by whitespace. No further non-whitespace
12
- * characters may occur on the line.
13
- *
14
- * A link reference definition does not correspond to a structural element of
15
- * a document. Instead, it defines a label which can be used in reference
16
- * links and reference-style images elsewhere in the document. Link reference
17
- * definitions can come either before or after the links that use them.
18
- *
19
6
  * @see https://github.github.com/gfm/#link-reference-definition
20
7
  */
21
- export declare class DefinitionTokenizer extends BaseBlockTokenizer implements Tokenizer, TokenizerMatchBlockHook<T, Token>, TokenizerParseBlockHook<T, Token, Node> {
22
- readonly isContainingBlock = false;
23
- constructor(props?: TokenizerProps);
24
- /**
25
- * @override
26
- * @see TokenizerMatchBlockHook
27
- */
28
- eatOpener(line: Readonly<PhrasingContentLine>): ResultOfEatOpener<T, Token>;
29
- /**
30
- * @override
31
- * @see TokenizerMatchBlockHook
32
- */
33
- eatContinuationText(line: Readonly<PhrasingContentLine>, token: Token): ResultOfEatContinuationText;
34
- /**
35
- * @override
36
- * @see TokenizerMatchBlockHook
37
- */
38
- onClose(token: Token, api: Readonly<MatchBlockPhaseApi>): ResultOfOnClose;
39
- /**
40
- * @override
41
- * @see TokenizerParseBlockHook
42
- */
43
- parseBlock(token: Readonly<Token>): ResultOfParse<T, Node>;
8
+ export declare class DefinitionTokenizer extends BaseBlockTokenizer<T, IToken, INode, IHookContext> implements IBlockTokenizer<T, IToken, INode, IHookContext> {
9
+ constructor(props?: ITokenizerProps);
10
+ readonly match: IMatchBlockHookCreator<T, IToken, IHookContext>;
11
+ readonly parse: IParseBlockHookCreator<T, IToken, INode, IHookContext>;
44
12
  }
package/lib/types/types.d.ts CHANGED
@@ -1,16 +1,16 @@
1
- import type { Definition, DefinitionType } from '@yozora/ast';
2
- import type { BaseBlockTokenizerProps, PartialYastBlockToken, PhrasingContentLine } from '@yozora/core-tokenizer';
1
+ import type { DefinitionType, IDefinition } from '@yozora/ast';
2
+ import type { IBaseBlockTokenizerProps, IPartialYastBlockToken, IPhrasingContentLine, ITokenizer } from '@yozora/core-tokenizer';
3
3
  import type { LinkDestinationCollectingState } from './util/link-destination';
4
4
  import type { LinkLabelCollectingState } from './util/link-label';
5
5
  import type { LinkTitleCollectingState } from './util/link-title';
6
6
  export declare type T = DefinitionType;
7
- export declare type Node = Definition;
7
+ export declare type INode = IDefinition;
8
8
  export declare const uniqueName = "@yozora/tokenizer-definition";
9
- export interface Token extends PartialYastBlockToken<T> {
9
+ export interface IToken extends IPartialYastBlockToken<T> {
10
10
  /**
11
11
  *
12
12
  */
13
- lines: Array<Readonly<PhrasingContentLine>>;
13
+ lines: Array<Readonly<IPhrasingContentLine>>;
14
14
  /**
15
15
  * Link label
16
16
  * Trimmed, Case-Insensitive
@@ -45,4 +45,5 @@ export interface Token extends PartialYastBlockToken<T> {
45
45
  */
46
46
  _identifier?: string;
47
47
  }
48
- export declare type TokenizerProps = Partial<BaseBlockTokenizerProps>;
48
+ export declare type IHookContext = ITokenizer;
49
+ export declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
package/lib/types/util/link-destination.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { NodePoint } from '@yozora/character';
1
+ import type { INodePoint } from '@yozora/character';
2
2
  /**
3
3
  * The processing token of eatAndCollectLinkDestination, used to save
4
4
  * intermediate data to support multiple codePosition fragment processing
@@ -13,7 +13,7 @@ export interface LinkDestinationCollectingState {
13
13
  /**
14
14
  * Collected token points
15
15
  */
16
- nodePoints: NodePoint[];
16
+ nodePoints: INodePoint[];
17
17
  /**
18
18
  * Whether an opening angle bracket has been matched
19
19
  */
@@ -28,10 +28,10 @@ export interface LinkDestinationCollectingState {
28
28
  * @param nodePoints
29
29
  * @param startIndex
30
30
  * @param endIndex
31
- * @param token
31
+ * @param state
32
32
  * @see https://github.github.com/gfm/#link-destination
33
33
  */
34
- export declare function eatAndCollectLinkDestination(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, token: LinkDestinationCollectingState | null): {
34
+ export declare function eatAndCollectLinkDestination(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: LinkDestinationCollectingState | null): {
35
35
  nextIndex: number;
36
- token: LinkDestinationCollectingState;
36
+ state: LinkDestinationCollectingState;
37
37
  };
package/lib/types/util/link-label.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { NodePoint } from '@yozora/character';
1
+ import type { INodePoint } from '@yozora/character';
2
2
  /**
3
3
  * The processing token of eatAndCollectLinkLabel, used to save
4
4
  * intermediate data to support multiple codePosition fragment processing
@@ -13,7 +13,7 @@ export interface LinkLabelCollectingState {
13
13
  /**
14
14
  * Collected token points
15
15
  */
16
- nodePoints: NodePoint[];
16
+ nodePoints: INodePoint[];
17
17
  /**
18
18
  * Does it contain non-blank characters
19
19
  */
@@ -35,10 +35,10 @@ export interface LinkLabelCollectingState {
35
35
  * @param nodePoints
36
36
  * @param startIndex
37
37
  * @param endIndex
38
- * @param token
38
+ * @param state
39
39
  * @see https://github.github.com/gfm/#link-label
40
40
  */
41
- export declare function eatAndCollectLinkLabel(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, token: LinkLabelCollectingState | null): {
41
+ export declare function eatAndCollectLinkLabel(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: LinkLabelCollectingState | null): {
42
42
  nextIndex: number;
43
- token: LinkLabelCollectingState;
43
+ state: LinkLabelCollectingState;
44
44
  };
package/lib/types/util/link-title.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { NodePoint } from '@yozora/character';
1
+ import type { INodePoint } from '@yozora/character';
2
2
  /**
3
3
  * The processing token of eatAndCollectLinkDestination, used to save
4
4
  * intermediate data to support multiple codePosition fragment processing.
@@ -13,7 +13,7 @@ export interface LinkTitleCollectingState {
13
13
  /**
14
14
  * Collected token points
15
15
  */
16
- nodePoints: NodePoint[];
16
+ nodePoints: INodePoint[];
17
17
  /**
18
18
  * Character that wrap link-title
19
19
  */
@@ -24,10 +24,10 @@ export interface LinkTitleCollectingState {
24
24
  * @param nodePoints
25
25
  * @param startIndex
26
26
  * @param endIndex
27
- * @param token
27
+ * @param state
28
28
  * @see https://github.github.com/gfm/#link-title
29
29
  */
30
- export declare function eatAndCollectLinkTitle(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, token: LinkTitleCollectingState | null): {
30
+ export declare function eatAndCollectLinkTitle(nodePoints: ReadonlyArray<INodePoint>, startIndex: number, endIndex: number, state: LinkTitleCollectingState | null): {
31
31
  nextIndex: number;
32
- token: LinkTitleCollectingState;
32
+ state: LinkTitleCollectingState;
33
33
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yozora/tokenizer-definition",
3
- "version": "1.2.0",
3
+ "version": "2.0.0-alpha.0",
4
4
  "author": {
5
5
  "name": "guanghechen",
6
6
  "url": "https://github.com/guanghechen/"
@@ -35,9 +35,9 @@
35
35
  "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
36
36
  },
37
37
  "dependencies": {
38
- "@yozora/ast": "^1.2.0",
39
- "@yozora/character": "^1.2.0",
40
- "@yozora/core-tokenizer": "^1.2.0"
38
+ "@yozora/ast": "^2.0.0-alpha.0",
39
+ "@yozora/character": "^2.0.0-alpha.0",
40
+ "@yozora/core-tokenizer": "^2.0.0-alpha.0"
41
41
  },
42
- "gitHead": "86da40e50d2fe9acace68695288e15e012e6cd0d"
42
+ "gitHead": "0171501339c49ffd02ed16a63447fa20a47a29a7"
43
43
  }