@yozora/tokenizer-definition 1.2.0-alpha.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/lib/cjs/index.js +99 -106
  2. package/lib/esm/index.js +99 -106
  3. package/lib/types/util/link-destination.d.ts +3 -3
  4. package/lib/types/util/link-label.d.ts +3 -3
  5. package/lib/types/util/link-title.d.ts +3 -3
  6. package/package.json +5 -5
package/lib/cjs/index.js CHANGED
@@ -8,10 +8,10 @@ var coreTokenizer = require('@yozora/core-tokenizer');
8
8
 
9
9
  const uniqueName = '@yozora/tokenizer-definition';
10
10
 
11
- function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
11
+ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, state) {
12
12
  let i = startIndex;
13
- if (token == null) {
14
- token = {
13
+ if (state == null) {
14
+ state = {
15
15
  saturated: false,
16
16
  nodePoints: [],
17
17
  hasOpenAngleBracket: false,
@@ -20,79 +20,79 @@ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
20
20
  }
21
21
  const firstNonWhitespaceIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
22
22
  if (firstNonWhitespaceIndex >= endIndex)
23
- return { nextIndex: -1, token };
24
- if (token.nodePoints.length <= 0) {
23
+ return { nextIndex: -1, state: state };
24
+ if (state.nodePoints.length <= 0) {
25
25
  i = firstNonWhitespaceIndex;
26
26
  const p = nodePoints[i];
27
27
  if (p.codePoint === character.AsciiCodePoint.OPEN_ANGLE) {
28
28
  i += 1;
29
- token.hasOpenAngleBracket = true;
30
- token.nodePoints.push(p);
29
+ state.hasOpenAngleBracket = true;
30
+ state.nodePoints.push(p);
31
31
  }
32
32
  }
33
- if (token.hasOpenAngleBracket) {
33
+ if (state.hasOpenAngleBracket) {
34
34
  for (; i < endIndex; ++i) {
35
35
  const p = nodePoints[i];
36
36
  switch (p.codePoint) {
37
37
  case character.AsciiCodePoint.BACKSLASH:
38
38
  if (i + 1 < endIndex) {
39
- token.nodePoints.push(p);
40
- token.nodePoints.push(nodePoints[i + 1]);
39
+ state.nodePoints.push(p);
40
+ state.nodePoints.push(nodePoints[i + 1]);
41
41
  }
42
42
  i += 1;
43
43
  break;
44
44
  case character.AsciiCodePoint.OPEN_ANGLE:
45
45
  case character.VirtualCodePoint.LINE_END:
46
- return { nextIndex: -1, token };
46
+ return { nextIndex: -1, state: state };
47
47
  case character.AsciiCodePoint.CLOSE_ANGLE:
48
- token.saturated = true;
49
- token.nodePoints.push(p);
50
- return { nextIndex: i + 1, token };
48
+ state.saturated = true;
49
+ state.nodePoints.push(p);
50
+ return { nextIndex: i + 1, state: state };
51
51
  default:
52
- token.nodePoints.push(p);
52
+ state.nodePoints.push(p);
53
53
  }
54
54
  }
55
- return { nextIndex: i, token };
55
+ return { nextIndex: i, state: state };
56
56
  }
57
57
  for (; i < endIndex; ++i) {
58
58
  const p = nodePoints[i];
59
59
  switch (p.codePoint) {
60
60
  case character.AsciiCodePoint.BACKSLASH:
61
61
  if (i + 1 < endIndex) {
62
- token.nodePoints.push(p);
63
- token.nodePoints.push(nodePoints[i + 1]);
62
+ state.nodePoints.push(p);
63
+ state.nodePoints.push(nodePoints[i + 1]);
64
64
  }
65
65
  i += 1;
66
66
  break;
67
67
  case character.AsciiCodePoint.OPEN_PARENTHESIS:
68
- token.openParensCount += 1;
69
- token.nodePoints.push(p);
68
+ state.openParensCount += 1;
69
+ state.nodePoints.push(p);
70
70
  break;
71
71
  case character.AsciiCodePoint.CLOSE_PARENTHESIS:
72
- token.openParensCount -= 1;
73
- token.nodePoints.push(p);
74
- if (token.openParensCount < 0) {
75
- return { nextIndex: i, token };
72
+ state.openParensCount -= 1;
73
+ state.nodePoints.push(p);
74
+ if (state.openParensCount < 0) {
75
+ return { nextIndex: i, state: state };
76
76
  }
77
77
  break;
78
78
  default:
79
79
  if (character.isWhitespaceCharacter(p.codePoint) ||
80
80
  character.isAsciiControlCharacter(p.codePoint)) {
81
- token.saturated = true;
82
- return { nextIndex: i, token };
81
+ state.saturated = true;
82
+ return { nextIndex: i, state: state };
83
83
  }
84
- token.nodePoints.push(p);
84
+ state.nodePoints.push(p);
85
85
  break;
86
86
  }
87
87
  }
88
- token.saturated = true;
89
- return { nextIndex: i, token };
88
+ state.saturated = true;
89
+ return { nextIndex: i, state: state };
90
90
  }
91
91
 
92
- function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
92
+ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, state) {
93
93
  let i = startIndex;
94
- if (token == null) {
95
- token = {
94
+ if (state == null) {
95
+ state = {
96
96
  saturated: false,
97
97
  nodePoints: [],
98
98
  hasNonWhitespaceCharacter: false,
@@ -100,50 +100,50 @@ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
100
100
  }
101
101
  const firstNonWhitespaceIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
102
102
  if (firstNonWhitespaceIndex >= endIndex)
103
- return { nextIndex: -1, token };
104
- if (token.nodePoints.length <= 0) {
103
+ return { nextIndex: -1, state: state };
104
+ if (state.nodePoints.length <= 0) {
105
105
  i = firstNonWhitespaceIndex;
106
106
  const p = nodePoints[i];
107
107
  if (p.codePoint !== character.AsciiCodePoint.OPEN_BRACKET) {
108
- return { nextIndex: -1, token };
108
+ return { nextIndex: -1, state: state };
109
109
  }
110
110
  i += 1;
111
- token.nodePoints.push(p);
111
+ state.nodePoints.push(p);
112
112
  }
113
113
  for (; i < endIndex; ++i) {
114
114
  const p = nodePoints[i];
115
115
  switch (p.codePoint) {
116
116
  case character.AsciiCodePoint.BACKSLASH:
117
- token.hasNonWhitespaceCharacter = true;
117
+ state.hasNonWhitespaceCharacter = true;
118
118
  if (i + 1 < endIndex) {
119
- token.nodePoints.push(p);
120
- token.nodePoints.push(nodePoints[i + 1]);
119
+ state.nodePoints.push(p);
120
+ state.nodePoints.push(nodePoints[i + 1]);
121
121
  }
122
122
  i += 1;
123
123
  break;
124
124
  case character.AsciiCodePoint.OPEN_BRACKET:
125
- return { nextIndex: -1, token };
125
+ return { nextIndex: -1, state: state };
126
126
  case character.AsciiCodePoint.CLOSE_BRACKET:
127
- token.nodePoints.push(p);
128
- if (token.hasNonWhitespaceCharacter) {
129
- token.saturated = true;
130
- return { nextIndex: i + 1, token };
127
+ state.nodePoints.push(p);
128
+ if (state.hasNonWhitespaceCharacter) {
129
+ state.saturated = true;
130
+ return { nextIndex: i + 1, state: state };
131
131
  }
132
- return { nextIndex: -1, token };
132
+ return { nextIndex: -1, state: state };
133
133
  default:
134
134
  if (!character.isWhitespaceCharacter(p.codePoint)) {
135
- token.hasNonWhitespaceCharacter = true;
135
+ state.hasNonWhitespaceCharacter = true;
136
136
  }
137
- token.nodePoints.push(p);
137
+ state.nodePoints.push(p);
138
138
  }
139
139
  }
140
- return { nextIndex: 1, token };
140
+ return { nextIndex: 1, state: state };
141
141
  }
142
142
 
143
- function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
143
+ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, state) {
144
144
  let i = startIndex;
145
- if (token == null) {
146
- token = {
145
+ if (state == null) {
146
+ state = {
147
147
  saturated: false,
148
148
  nodePoints: [],
149
149
  wrapSymbol: null,
@@ -151,25 +151,25 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
151
151
  }
152
152
  const firstNonWhitespaceIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
153
153
  if (firstNonWhitespaceIndex >= endIndex)
154
- return { nextIndex: -1, token };
155
- if (token.nodePoints.length <= 0) {
154
+ return { nextIndex: -1, state: state };
155
+ if (state.nodePoints.length <= 0) {
156
156
  i = firstNonWhitespaceIndex;
157
157
  const p = nodePoints[i];
158
158
  switch (p.codePoint) {
159
159
  case character.AsciiCodePoint.DOUBLE_QUOTE:
160
160
  case character.AsciiCodePoint.SINGLE_QUOTE:
161
161
  case character.AsciiCodePoint.OPEN_PARENTHESIS:
162
- token.wrapSymbol = p.codePoint;
163
- token.nodePoints.push(p);
162
+ state.wrapSymbol = p.codePoint;
163
+ state.nodePoints.push(p);
164
164
  i += 1;
165
165
  break;
166
166
  default:
167
- return { nextIndex: -1, token };
167
+ return { nextIndex: -1, state: state };
168
168
  }
169
169
  }
170
- if (token.wrapSymbol == null)
171
- return { nextIndex: -1, token };
172
- switch (token.wrapSymbol) {
170
+ if (state.wrapSymbol == null)
171
+ return { nextIndex: -1, state: state };
172
+ switch (state.wrapSymbol) {
173
173
  case character.AsciiCodePoint.DOUBLE_QUOTE:
174
174
  case character.AsciiCodePoint.SINGLE_QUOTE: {
175
175
  for (; i < endIndex; ++i) {
@@ -177,17 +177,17 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
177
177
  switch (p.codePoint) {
178
178
  case character.AsciiCodePoint.BACKSLASH:
179
179
  if (i + 1 < endIndex) {
180
- token.nodePoints.push(p);
181
- token.nodePoints.push(nodePoints[i + 1]);
180
+ state.nodePoints.push(p);
181
+ state.nodePoints.push(nodePoints[i + 1]);
182
182
  }
183
183
  i += 1;
184
184
  break;
185
- case token.wrapSymbol:
186
- token.saturated = true;
187
- token.nodePoints.push(p);
188
- return { nextIndex: i + 1, token };
185
+ case state.wrapSymbol:
186
+ state.saturated = true;
187
+ state.nodePoints.push(p);
188
+ return { nextIndex: i + 1, state: state };
189
189
  default:
190
- token.nodePoints.push(p);
190
+ state.nodePoints.push(p);
191
191
  }
192
192
  }
193
193
  break;
@@ -198,29 +198,29 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
198
198
  switch (p.codePoint) {
199
199
  case character.AsciiCodePoint.BACKSLASH:
200
200
  if (i + 1 < endIndex) {
201
- token.nodePoints.push(p);
202
- token.nodePoints.push(nodePoints[i + 1]);
201
+ state.nodePoints.push(p);
202
+ state.nodePoints.push(nodePoints[i + 1]);
203
203
  }
204
204
  i += 1;
205
205
  break;
206
206
  case character.AsciiCodePoint.OPEN_PARENTHESIS:
207
- return { nextIndex: -1, token };
207
+ return { nextIndex: -1, state: state };
208
208
  case character.AsciiCodePoint.CLOSE_PARENTHESIS:
209
209
  if (i + 1 >= endIndex ||
210
210
  nodePoints[i + 1].codePoint === character.VirtualCodePoint.LINE_END) {
211
- token.nodePoints.push(p);
212
- token.saturated = true;
211
+ state.nodePoints.push(p);
212
+ state.saturated = true;
213
213
  break;
214
214
  }
215
- return { nextIndex: -1, token };
215
+ return { nextIndex: -1, state: state };
216
216
  default:
217
- token.nodePoints.push(p);
217
+ state.nodePoints.push(p);
218
218
  }
219
219
  }
220
220
  break;
221
221
  }
222
222
  }
223
- return { nextIndex: endIndex, token };
223
+ return { nextIndex: endIndex, state: state };
224
224
  }
225
225
 
226
226
  class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
@@ -239,8 +239,8 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
239
239
  if (firstNonWhitespaceIndex >= endIndex)
240
240
  return null;
241
241
  let i = firstNonWhitespaceIndex;
242
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
243
- if (linkLabelCollectResult.nextIndex < 0)
242
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
243
+ if (labelEndIndex < 0)
244
244
  return null;
245
245
  const lineNo = nodePoints[startIndex].line;
246
246
  const createInitState = () => {
@@ -250,7 +250,7 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
250
250
  start: coreTokenizer.calcStartYastNodePoint(nodePoints, startIndex),
251
251
  end: coreTokenizer.calcEndYastNodePoint(nodePoints, endIndex - 1),
252
252
  },
253
- label: linkLabelCollectResult.token,
253
+ label: labelState,
254
254
  destination: null,
255
255
  title: null,
256
256
  lineNoOfLabel: lineNo,
@@ -260,11 +260,10 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
260
260
  };
261
261
  return token;
262
262
  };
263
- if (!linkLabelCollectResult.token.saturated) {
263
+ if (!labelState.saturated) {
264
264
  const token = createInitState();
265
265
  return { token, nextIndex: endIndex };
266
266
  }
267
- const labelEndIndex = linkLabelCollectResult.nextIndex;
268
267
  if (labelEndIndex < 0 ||
269
268
  labelEndIndex + 1 >= endIndex ||
270
269
  nodePoints[labelEndIndex].codePoint !== character.AsciiCodePoint.COLON)
@@ -274,34 +273,31 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
274
273
  const token = createInitState();
275
274
  return { token, nextIndex: endIndex };
276
275
  }
277
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
278
- if (linkDestinationCollectResult.nextIndex < 0)
276
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
277
+ if (destinationEndIndex < 0)
279
278
  return null;
280
- if (!linkDestinationCollectResult.token.saturated &&
281
- linkDestinationCollectResult.nextIndex !== endIndex)
279
+ if (!destinationState.saturated && destinationEndIndex !== endIndex)
282
280
  return null;
283
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
284
281
  i = coreTokenizer.eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
285
282
  if (i >= endIndex) {
286
283
  const token = createInitState();
287
- token.destination = linkDestinationCollectResult.token;
284
+ token.destination = destinationState;
288
285
  token.lineNoOfDestination = lineNo;
289
286
  return { token, nextIndex: endIndex };
290
287
  }
291
288
  if (i === destinationEndIndex)
292
289
  return null;
293
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
294
- if (linkTitleCollectResult.nextIndex >= 0) {
295
- i = linkTitleCollectResult.nextIndex;
296
- }
290
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
291
+ if (titleEndIndex >= 0)
292
+ i = titleEndIndex;
297
293
  if (i < endIndex) {
298
294
  const k = coreTokenizer.eatOptionalWhitespaces(nodePoints, i, endIndex);
299
295
  if (k < endIndex)
300
296
  return null;
301
297
  }
302
298
  const token = createInitState();
303
- token.destination = linkDestinationCollectResult.token;
304
- token.title = linkTitleCollectResult.token;
299
+ token.destination = destinationState;
300
+ token.title = titleState;
305
301
  token.lineNoOfDestination = lineNo;
306
302
  token.lineNoOfTitle = lineNo;
307
303
  return { token, nextIndex: endIndex };
@@ -314,12 +310,11 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
314
310
  const lineNo = nodePoints[startIndex].line;
315
311
  let i = firstNonWhitespaceIndex;
316
312
  if (!token.label.saturated) {
317
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
318
- if (linkLabelCollectResult.nextIndex < 0) {
313
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
314
+ if (labelEndIndex < 0) {
319
315
  return { status: 'failedAndRollback', lines: token.lines };
320
316
  }
321
- const labelEndIndex = linkLabelCollectResult.nextIndex;
322
- if (!linkLabelCollectResult.token.saturated) {
317
+ if (!labelState.saturated) {
323
318
  token.lines.push(line);
324
319
  return { status: 'opening', nextIndex: endIndex };
325
320
  }
@@ -334,15 +329,13 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
334
329
  if (i >= endIndex) {
335
330
  return { status: 'failedAndRollback', lines: token.lines };
336
331
  }
337
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
338
- if (linkDestinationCollectResult.nextIndex < 0 ||
339
- !linkDestinationCollectResult.token.saturated) {
332
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
333
+ if (destinationEndIndex < 0 || !destinationState.saturated) {
340
334
  return { status: 'failedAndRollback', lines: token.lines };
341
335
  }
342
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
343
336
  i = coreTokenizer.eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
344
337
  if (i >= endIndex) {
345
- token.destination = linkDestinationCollectResult.token;
338
+ token.destination = destinationState;
346
339
  token.lines.push(line);
347
340
  return { status: 'opening', nextIndex: endIndex };
348
341
  }
@@ -352,12 +345,12 @@ class DefinitionTokenizer extends coreTokenizer.BaseBlockTokenizer {
352
345
  if (token.lineNoOfTitle < 0) {
353
346
  token.lineNoOfTitle = lineNo;
354
347
  }
355
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
356
- token.title = linkTitleCollectResult.token;
357
- if (linkTitleCollectResult.nextIndex < 0 ||
358
- linkTitleCollectResult.token.nodePoints.length <= 0 ||
359
- (linkTitleCollectResult.token.saturated &&
360
- coreTokenizer.eatOptionalWhitespaces(nodePoints, linkTitleCollectResult.nextIndex, endIndex) < endIndex)) {
348
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
349
+ token.title = titleState;
350
+ if (titleEndIndex < 0 ||
351
+ titleState.nodePoints.length <= 0 ||
352
+ (titleState.saturated &&
353
+ coreTokenizer.eatOptionalWhitespaces(nodePoints, titleEndIndex, endIndex) < endIndex)) {
361
354
  if (token.lineNoOfDestination === token.lineNoOfTitle) {
362
355
  return { status: 'failedAndRollback', lines: token.lines };
363
356
  }
package/lib/esm/index.js CHANGED
@@ -4,10 +4,10 @@ import { eatOptionalWhitespaces, BaseBlockTokenizer, TokenizerPriority, calcEndY
4
4
 
5
5
  const uniqueName = '@yozora/tokenizer-definition';
6
6
 
7
- function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
7
+ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, state) {
8
8
  let i = startIndex;
9
- if (token == null) {
10
- token = {
9
+ if (state == null) {
10
+ state = {
11
11
  saturated: false,
12
12
  nodePoints: [],
13
13
  hasOpenAngleBracket: false,
@@ -16,79 +16,79 @@ function eatAndCollectLinkDestination(nodePoints, startIndex, endIndex, token) {
16
16
  }
17
17
  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
18
18
  if (firstNonWhitespaceIndex >= endIndex)
19
- return { nextIndex: -1, token };
20
- if (token.nodePoints.length <= 0) {
19
+ return { nextIndex: -1, state: state };
20
+ if (state.nodePoints.length <= 0) {
21
21
  i = firstNonWhitespaceIndex;
22
22
  const p = nodePoints[i];
23
23
  if (p.codePoint === AsciiCodePoint.OPEN_ANGLE) {
24
24
  i += 1;
25
- token.hasOpenAngleBracket = true;
26
- token.nodePoints.push(p);
25
+ state.hasOpenAngleBracket = true;
26
+ state.nodePoints.push(p);
27
27
  }
28
28
  }
29
- if (token.hasOpenAngleBracket) {
29
+ if (state.hasOpenAngleBracket) {
30
30
  for (; i < endIndex; ++i) {
31
31
  const p = nodePoints[i];
32
32
  switch (p.codePoint) {
33
33
  case AsciiCodePoint.BACKSLASH:
34
34
  if (i + 1 < endIndex) {
35
- token.nodePoints.push(p);
36
- token.nodePoints.push(nodePoints[i + 1]);
35
+ state.nodePoints.push(p);
36
+ state.nodePoints.push(nodePoints[i + 1]);
37
37
  }
38
38
  i += 1;
39
39
  break;
40
40
  case AsciiCodePoint.OPEN_ANGLE:
41
41
  case VirtualCodePoint.LINE_END:
42
- return { nextIndex: -1, token };
42
+ return { nextIndex: -1, state: state };
43
43
  case AsciiCodePoint.CLOSE_ANGLE:
44
- token.saturated = true;
45
- token.nodePoints.push(p);
46
- return { nextIndex: i + 1, token };
44
+ state.saturated = true;
45
+ state.nodePoints.push(p);
46
+ return { nextIndex: i + 1, state: state };
47
47
  default:
48
- token.nodePoints.push(p);
48
+ state.nodePoints.push(p);
49
49
  }
50
50
  }
51
- return { nextIndex: i, token };
51
+ return { nextIndex: i, state: state };
52
52
  }
53
53
  for (; i < endIndex; ++i) {
54
54
  const p = nodePoints[i];
55
55
  switch (p.codePoint) {
56
56
  case AsciiCodePoint.BACKSLASH:
57
57
  if (i + 1 < endIndex) {
58
- token.nodePoints.push(p);
59
- token.nodePoints.push(nodePoints[i + 1]);
58
+ state.nodePoints.push(p);
59
+ state.nodePoints.push(nodePoints[i + 1]);
60
60
  }
61
61
  i += 1;
62
62
  break;
63
63
  case AsciiCodePoint.OPEN_PARENTHESIS:
64
- token.openParensCount += 1;
65
- token.nodePoints.push(p);
64
+ state.openParensCount += 1;
65
+ state.nodePoints.push(p);
66
66
  break;
67
67
  case AsciiCodePoint.CLOSE_PARENTHESIS:
68
- token.openParensCount -= 1;
69
- token.nodePoints.push(p);
70
- if (token.openParensCount < 0) {
71
- return { nextIndex: i, token };
68
+ state.openParensCount -= 1;
69
+ state.nodePoints.push(p);
70
+ if (state.openParensCount < 0) {
71
+ return { nextIndex: i, state: state };
72
72
  }
73
73
  break;
74
74
  default:
75
75
  if (isWhitespaceCharacter(p.codePoint) ||
76
76
  isAsciiControlCharacter(p.codePoint)) {
77
- token.saturated = true;
78
- return { nextIndex: i, token };
77
+ state.saturated = true;
78
+ return { nextIndex: i, state: state };
79
79
  }
80
- token.nodePoints.push(p);
80
+ state.nodePoints.push(p);
81
81
  break;
82
82
  }
83
83
  }
84
- token.saturated = true;
85
- return { nextIndex: i, token };
84
+ state.saturated = true;
85
+ return { nextIndex: i, state: state };
86
86
  }
87
87
 
88
- function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
88
+ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, state) {
89
89
  let i = startIndex;
90
- if (token == null) {
91
- token = {
90
+ if (state == null) {
91
+ state = {
92
92
  saturated: false,
93
93
  nodePoints: [],
94
94
  hasNonWhitespaceCharacter: false,
@@ -96,50 +96,50 @@ function eatAndCollectLinkLabel(nodePoints, startIndex, endIndex, token) {
96
96
  }
97
97
  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
98
98
  if (firstNonWhitespaceIndex >= endIndex)
99
- return { nextIndex: -1, token };
100
- if (token.nodePoints.length <= 0) {
99
+ return { nextIndex: -1, state: state };
100
+ if (state.nodePoints.length <= 0) {
101
101
  i = firstNonWhitespaceIndex;
102
102
  const p = nodePoints[i];
103
103
  if (p.codePoint !== AsciiCodePoint.OPEN_BRACKET) {
104
- return { nextIndex: -1, token };
104
+ return { nextIndex: -1, state: state };
105
105
  }
106
106
  i += 1;
107
- token.nodePoints.push(p);
107
+ state.nodePoints.push(p);
108
108
  }
109
109
  for (; i < endIndex; ++i) {
110
110
  const p = nodePoints[i];
111
111
  switch (p.codePoint) {
112
112
  case AsciiCodePoint.BACKSLASH:
113
- token.hasNonWhitespaceCharacter = true;
113
+ state.hasNonWhitespaceCharacter = true;
114
114
  if (i + 1 < endIndex) {
115
- token.nodePoints.push(p);
116
- token.nodePoints.push(nodePoints[i + 1]);
115
+ state.nodePoints.push(p);
116
+ state.nodePoints.push(nodePoints[i + 1]);
117
117
  }
118
118
  i += 1;
119
119
  break;
120
120
  case AsciiCodePoint.OPEN_BRACKET:
121
- return { nextIndex: -1, token };
121
+ return { nextIndex: -1, state: state };
122
122
  case AsciiCodePoint.CLOSE_BRACKET:
123
- token.nodePoints.push(p);
124
- if (token.hasNonWhitespaceCharacter) {
125
- token.saturated = true;
126
- return { nextIndex: i + 1, token };
123
+ state.nodePoints.push(p);
124
+ if (state.hasNonWhitespaceCharacter) {
125
+ state.saturated = true;
126
+ return { nextIndex: i + 1, state: state };
127
127
  }
128
- return { nextIndex: -1, token };
128
+ return { nextIndex: -1, state: state };
129
129
  default:
130
130
  if (!isWhitespaceCharacter(p.codePoint)) {
131
- token.hasNonWhitespaceCharacter = true;
131
+ state.hasNonWhitespaceCharacter = true;
132
132
  }
133
- token.nodePoints.push(p);
133
+ state.nodePoints.push(p);
134
134
  }
135
135
  }
136
- return { nextIndex: 1, token };
136
+ return { nextIndex: 1, state: state };
137
137
  }
138
138
 
139
- function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
139
+ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, state) {
140
140
  let i = startIndex;
141
- if (token == null) {
142
- token = {
141
+ if (state == null) {
142
+ state = {
143
143
  saturated: false,
144
144
  nodePoints: [],
145
145
  wrapSymbol: null,
@@ -147,25 +147,25 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
147
147
  }
148
148
  const firstNonWhitespaceIndex = eatOptionalWhitespaces(nodePoints, i, endIndex);
149
149
  if (firstNonWhitespaceIndex >= endIndex)
150
- return { nextIndex: -1, token };
151
- if (token.nodePoints.length <= 0) {
150
+ return { nextIndex: -1, state: state };
151
+ if (state.nodePoints.length <= 0) {
152
152
  i = firstNonWhitespaceIndex;
153
153
  const p = nodePoints[i];
154
154
  switch (p.codePoint) {
155
155
  case AsciiCodePoint.DOUBLE_QUOTE:
156
156
  case AsciiCodePoint.SINGLE_QUOTE:
157
157
  case AsciiCodePoint.OPEN_PARENTHESIS:
158
- token.wrapSymbol = p.codePoint;
159
- token.nodePoints.push(p);
158
+ state.wrapSymbol = p.codePoint;
159
+ state.nodePoints.push(p);
160
160
  i += 1;
161
161
  break;
162
162
  default:
163
- return { nextIndex: -1, token };
163
+ return { nextIndex: -1, state: state };
164
164
  }
165
165
  }
166
- if (token.wrapSymbol == null)
167
- return { nextIndex: -1, token };
168
- switch (token.wrapSymbol) {
166
+ if (state.wrapSymbol == null)
167
+ return { nextIndex: -1, state: state };
168
+ switch (state.wrapSymbol) {
169
169
  case AsciiCodePoint.DOUBLE_QUOTE:
170
170
  case AsciiCodePoint.SINGLE_QUOTE: {
171
171
  for (; i < endIndex; ++i) {
@@ -173,17 +173,17 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
173
173
  switch (p.codePoint) {
174
174
  case AsciiCodePoint.BACKSLASH:
175
175
  if (i + 1 < endIndex) {
176
- token.nodePoints.push(p);
177
- token.nodePoints.push(nodePoints[i + 1]);
176
+ state.nodePoints.push(p);
177
+ state.nodePoints.push(nodePoints[i + 1]);
178
178
  }
179
179
  i += 1;
180
180
  break;
181
- case token.wrapSymbol:
182
- token.saturated = true;
183
- token.nodePoints.push(p);
184
- return { nextIndex: i + 1, token };
181
+ case state.wrapSymbol:
182
+ state.saturated = true;
183
+ state.nodePoints.push(p);
184
+ return { nextIndex: i + 1, state: state };
185
185
  default:
186
- token.nodePoints.push(p);
186
+ state.nodePoints.push(p);
187
187
  }
188
188
  }
189
189
  break;
@@ -194,29 +194,29 @@ function eatAndCollectLinkTitle(nodePoints, startIndex, endIndex, token) {
194
194
  switch (p.codePoint) {
195
195
  case AsciiCodePoint.BACKSLASH:
196
196
  if (i + 1 < endIndex) {
197
- token.nodePoints.push(p);
198
- token.nodePoints.push(nodePoints[i + 1]);
197
+ state.nodePoints.push(p);
198
+ state.nodePoints.push(nodePoints[i + 1]);
199
199
  }
200
200
  i += 1;
201
201
  break;
202
202
  case AsciiCodePoint.OPEN_PARENTHESIS:
203
- return { nextIndex: -1, token };
203
+ return { nextIndex: -1, state: state };
204
204
  case AsciiCodePoint.CLOSE_PARENTHESIS:
205
205
  if (i + 1 >= endIndex ||
206
206
  nodePoints[i + 1].codePoint === VirtualCodePoint.LINE_END) {
207
- token.nodePoints.push(p);
208
- token.saturated = true;
207
+ state.nodePoints.push(p);
208
+ state.saturated = true;
209
209
  break;
210
210
  }
211
- return { nextIndex: -1, token };
211
+ return { nextIndex: -1, state: state };
212
212
  default:
213
- token.nodePoints.push(p);
213
+ state.nodePoints.push(p);
214
214
  }
215
215
  }
216
216
  break;
217
217
  }
218
218
  }
219
- return { nextIndex: endIndex, token };
219
+ return { nextIndex: endIndex, state: state };
220
220
  }
221
221
 
222
222
  class DefinitionTokenizer extends BaseBlockTokenizer {
@@ -235,8 +235,8 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
235
235
  if (firstNonWhitespaceIndex >= endIndex)
236
236
  return null;
237
237
  let i = firstNonWhitespaceIndex;
238
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
239
- if (linkLabelCollectResult.nextIndex < 0)
238
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, null);
239
+ if (labelEndIndex < 0)
240
240
  return null;
241
241
  const lineNo = nodePoints[startIndex].line;
242
242
  const createInitState = () => {
@@ -246,7 +246,7 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
246
246
  start: calcStartYastNodePoint(nodePoints, startIndex),
247
247
  end: calcEndYastNodePoint(nodePoints, endIndex - 1),
248
248
  },
249
- label: linkLabelCollectResult.token,
249
+ label: labelState,
250
250
  destination: null,
251
251
  title: null,
252
252
  lineNoOfLabel: lineNo,
@@ -256,11 +256,10 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
256
256
  };
257
257
  return token;
258
258
  };
259
- if (!linkLabelCollectResult.token.saturated) {
259
+ if (!labelState.saturated) {
260
260
  const token = createInitState();
261
261
  return { token, nextIndex: endIndex };
262
262
  }
263
- const labelEndIndex = linkLabelCollectResult.nextIndex;
264
263
  if (labelEndIndex < 0 ||
265
264
  labelEndIndex + 1 >= endIndex ||
266
265
  nodePoints[labelEndIndex].codePoint !== AsciiCodePoint.COLON)
@@ -270,34 +269,31 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
270
269
  const token = createInitState();
271
270
  return { token, nextIndex: endIndex };
272
271
  }
273
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
274
- if (linkDestinationCollectResult.nextIndex < 0)
272
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
273
+ if (destinationEndIndex < 0)
275
274
  return null;
276
- if (!linkDestinationCollectResult.token.saturated &&
277
- linkDestinationCollectResult.nextIndex !== endIndex)
275
+ if (!destinationState.saturated && destinationEndIndex !== endIndex)
278
276
  return null;
279
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
280
277
  i = eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
281
278
  if (i >= endIndex) {
282
279
  const token = createInitState();
283
- token.destination = linkDestinationCollectResult.token;
280
+ token.destination = destinationState;
284
281
  token.lineNoOfDestination = lineNo;
285
282
  return { token, nextIndex: endIndex };
286
283
  }
287
284
  if (i === destinationEndIndex)
288
285
  return null;
289
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
290
- if (linkTitleCollectResult.nextIndex >= 0) {
291
- i = linkTitleCollectResult.nextIndex;
292
- }
286
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, null);
287
+ if (titleEndIndex >= 0)
288
+ i = titleEndIndex;
293
289
  if (i < endIndex) {
294
290
  const k = eatOptionalWhitespaces(nodePoints, i, endIndex);
295
291
  if (k < endIndex)
296
292
  return null;
297
293
  }
298
294
  const token = createInitState();
299
- token.destination = linkDestinationCollectResult.token;
300
- token.title = linkTitleCollectResult.token;
295
+ token.destination = destinationState;
296
+ token.title = titleState;
301
297
  token.lineNoOfDestination = lineNo;
302
298
  token.lineNoOfTitle = lineNo;
303
299
  return { token, nextIndex: endIndex };
@@ -310,12 +306,11 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
310
306
  const lineNo = nodePoints[startIndex].line;
311
307
  let i = firstNonWhitespaceIndex;
312
308
  if (!token.label.saturated) {
313
- const linkLabelCollectResult = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
314
- if (linkLabelCollectResult.nextIndex < 0) {
309
+ const { nextIndex: labelEndIndex, state: labelState } = eatAndCollectLinkLabel(nodePoints, i, endIndex, token.label);
310
+ if (labelEndIndex < 0) {
315
311
  return { status: 'failedAndRollback', lines: token.lines };
316
312
  }
317
- const labelEndIndex = linkLabelCollectResult.nextIndex;
318
- if (!linkLabelCollectResult.token.saturated) {
313
+ if (!labelState.saturated) {
319
314
  token.lines.push(line);
320
315
  return { status: 'opening', nextIndex: endIndex };
321
316
  }
@@ -330,15 +325,13 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
330
325
  if (i >= endIndex) {
331
326
  return { status: 'failedAndRollback', lines: token.lines };
332
327
  }
333
- const linkDestinationCollectResult = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
334
- if (linkDestinationCollectResult.nextIndex < 0 ||
335
- !linkDestinationCollectResult.token.saturated) {
328
+ const { nextIndex: destinationEndIndex, state: destinationState } = eatAndCollectLinkDestination(nodePoints, i, endIndex, null);
329
+ if (destinationEndIndex < 0 || !destinationState.saturated) {
336
330
  return { status: 'failedAndRollback', lines: token.lines };
337
331
  }
338
- const destinationEndIndex = linkDestinationCollectResult.nextIndex;
339
332
  i = eatOptionalWhitespaces(nodePoints, destinationEndIndex, endIndex);
340
333
  if (i >= endIndex) {
341
- token.destination = linkDestinationCollectResult.token;
334
+ token.destination = destinationState;
342
335
  token.lines.push(line);
343
336
  return { status: 'opening', nextIndex: endIndex };
344
337
  }
@@ -348,12 +341,12 @@ class DefinitionTokenizer extends BaseBlockTokenizer {
348
341
  if (token.lineNoOfTitle < 0) {
349
342
  token.lineNoOfTitle = lineNo;
350
343
  }
351
- const linkTitleCollectResult = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
352
- token.title = linkTitleCollectResult.token;
353
- if (linkTitleCollectResult.nextIndex < 0 ||
354
- linkTitleCollectResult.token.nodePoints.length <= 0 ||
355
- (linkTitleCollectResult.token.saturated &&
356
- eatOptionalWhitespaces(nodePoints, linkTitleCollectResult.nextIndex, endIndex) < endIndex)) {
344
+ const { nextIndex: titleEndIndex, state: titleState } = eatAndCollectLinkTitle(nodePoints, i, endIndex, token.title);
345
+ token.title = titleState;
346
+ if (titleEndIndex < 0 ||
347
+ titleState.nodePoints.length <= 0 ||
348
+ (titleState.saturated &&
349
+ eatOptionalWhitespaces(nodePoints, titleEndIndex, endIndex) < endIndex)) {
357
350
  if (token.lineNoOfDestination === token.lineNoOfTitle) {
358
351
  return { status: 'failedAndRollback', lines: token.lines };
359
352
  }
package/lib/types/util/link-destination.d.ts CHANGED
@@ -28,10 +28,10 @@ export interface LinkDestinationCollectingState {
28
28
  * @param nodePoints
29
29
  * @param startIndex
30
30
  * @param endIndex
31
- * @param token
31
+ * @param state
32
32
  * @see https://github.github.com/gfm/#link-destination
33
33
  */
34
- export declare function eatAndCollectLinkDestination(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, token: LinkDestinationCollectingState | null): {
34
+ export declare function eatAndCollectLinkDestination(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, state: LinkDestinationCollectingState | null): {
35
35
  nextIndex: number;
36
- token: LinkDestinationCollectingState;
36
+ state: LinkDestinationCollectingState;
37
37
  };
package/lib/types/util/link-label.d.ts CHANGED
@@ -35,10 +35,10 @@ export interface LinkLabelCollectingState {
35
35
  * @param nodePoints
36
36
  * @param startIndex
37
37
  * @param endIndex
38
- * @param token
38
+ * @param state
39
39
  * @see https://github.github.com/gfm/#link-label
40
40
  */
41
- export declare function eatAndCollectLinkLabel(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, token: LinkLabelCollectingState | null): {
41
+ export declare function eatAndCollectLinkLabel(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, state: LinkLabelCollectingState | null): {
42
42
  nextIndex: number;
43
- token: LinkLabelCollectingState;
43
+ state: LinkLabelCollectingState;
44
44
  };
package/lib/types/util/link-title.d.ts CHANGED
@@ -24,10 +24,10 @@ export interface LinkTitleCollectingState {
24
24
  * @param nodePoints
25
25
  * @param startIndex
26
26
  * @param endIndex
27
- * @param token
27
+ * @param state
28
28
  * @see https://github.github.com/gfm/#link-title
29
29
  */
30
- export declare function eatAndCollectLinkTitle(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, token: LinkTitleCollectingState | null): {
30
+ export declare function eatAndCollectLinkTitle(nodePoints: ReadonlyArray<NodePoint>, startIndex: number, endIndex: number, state: LinkTitleCollectingState | null): {
31
31
  nextIndex: number;
32
- token: LinkTitleCollectingState;
32
+ state: LinkTitleCollectingState;
33
33
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yozora/tokenizer-definition",
3
- "version": "1.2.0-alpha.1",
3
+ "version": "1.3.0",
4
4
  "author": {
5
5
  "name": "guanghechen",
6
6
  "url": "https://github.com/guanghechen/"
@@ -35,9 +35,9 @@
35
35
  "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
36
36
  },
37
37
  "dependencies": {
38
- "@yozora/ast": "^1.2.0-alpha.1",
39
- "@yozora/character": "^1.2.0-alpha.1",
40
- "@yozora/core-tokenizer": "^1.2.0-alpha.1"
38
+ "@yozora/ast": "^1.3.0",
39
+ "@yozora/character": "^1.3.0",
40
+ "@yozora/core-tokenizer": "^1.3.0"
41
41
  },
42
- "gitHead": "ce3c17330085f22404668a388f2bee6f57e6e958"
42
+ "gitHead": "18c9b167004ad97718b2f94f25139f80598cbf7a"
43
43
  }