@yozora/tokenizer-link 2.0.0-alpha.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -6
- package/lib/cjs/index.js +118 -115
- package/lib/esm/index.js +119 -116
- package/lib/types/index.d.ts +1 -1
- package/lib/types/match.d.ts +31 -0
- package/lib/types/parse.d.ts +3 -0
- package/lib/types/tokenizer.d.ts +4 -29
- package/lib/types/types.d.ts +4 -3
- package/package.json +5 -5
package/README.md
CHANGED
|
@@ -84,14 +84,14 @@ so you can use `YozoraParser` / `GfmExParser` / `GfmParser` directly.
|
|
|
84
84
|
registered in *YastParser* as a plugin-in before it can be used.
|
|
85
85
|
|
|
86
86
|
```typescript {4,9}
|
|
87
|
-
import {
|
|
87
|
+
import { DefaultParser } from '@yozora/core-parser'
|
|
88
88
|
import ParagraphTokenizer from '@yozora/tokenizer-paragraph'
|
|
89
89
|
import TextTokenizer from '@yozora/tokenizer-text'
|
|
90
90
|
import LinkTokenizer from '@yozora/tokenizer-link'
|
|
91
91
|
|
|
92
|
-
const parser = new
|
|
93
|
-
.
|
|
94
|
-
.
|
|
92
|
+
const parser = new DefaultParser()
|
|
93
|
+
.useFallbackTokenizer(new ParagraphTokenizer())
|
|
94
|
+
.useFallbackTokenizer(new TextTokenizer())
|
|
95
95
|
.useTokenizer(new LinkTokenizer())
|
|
96
96
|
|
|
97
97
|
// parse source markdown content
|
|
@@ -229,7 +229,6 @@ Name | Type | Required | Default
|
|
|
229
229
|
[@yozora/tokenizer-link]: https://github.com/yozorajs/yozora/tree/main/tokenizers/link#readme
|
|
230
230
|
[@yozora/tokenizer-link-reference]: https://github.com/yozorajs/yozora/tree/main/tokenizers/link-reference#readme
|
|
231
231
|
[@yozora/tokenizer-list]: https://github.com/yozorajs/yozora/tree/main/tokenizers/list#readme
|
|
232
|
-
[@yozora/tokenizer-list-item]: https://github.com/yozorajs/yozora/tree/main/tokenizers/list-item#readme
|
|
233
232
|
[@yozora/tokenizer-math]: https://github.com/yozorajs/yozora/tree/main/tokenizers/math#readme
|
|
234
233
|
[@yozora/tokenizer-paragraph]: https://github.com/yozorajs/yozora/tree/main/tokenizers/paragraph#readme
|
|
235
234
|
[@yozora/tokenizer-setext-heading]: https://github.com/yozorajs/yozora/tree/main/tokenizers/setext-heading#readme
|
|
@@ -289,7 +288,6 @@ Name | Type | Required | Default
|
|
|
289
288
|
[doc-@yozora/tokenizer-definition]: https://yozora.guanghechen.com/docs/package/tokenizer-definition
|
|
290
289
|
[doc-@yozora/tokenizer-link-reference]: https://yozora.guanghechen.com/docs/package/tokenizer-link-reference
|
|
291
290
|
[doc-@yozora/tokenizer-list]: https://yozora.guanghechen.com/docs/package/tokenizer-list
|
|
292
|
-
[doc-@yozora/tokenizer-list-item]: https://yozora.guanghechen.com/docs/package/tokenizer-list-item
|
|
293
291
|
[doc-@yozora/tokenizer-math]: https://yozora.guanghechen.com/docs/package/tokenizer-math
|
|
294
292
|
[doc-@yozora/tokenizer-paragraph]: https://yozora.guanghechen.com/docs/package/tokenizer-paragraph
|
|
295
293
|
[doc-@yozora/tokenizer-setext-heading]: https://yozora.guanghechen.com/docs/package/tokenizer-setext-heading
|
package/lib/cjs/index.js
CHANGED
|
@@ -151,6 +151,122 @@ function eatLinkTitle(nodePoints, startIndex, endIndex) {
|
|
|
151
151
|
return -1;
|
|
152
152
|
}
|
|
153
153
|
|
|
154
|
+
const match = function (api) {
|
|
155
|
+
return {
|
|
156
|
+
findDelimiter: () => coreTokenizer.genFindDelimiter(_findDelimiter),
|
|
157
|
+
isDelimiterPair,
|
|
158
|
+
processDelimiterPair,
|
|
159
|
+
};
|
|
160
|
+
function _findDelimiter(startIndex, endIndex) {
|
|
161
|
+
const nodePoints = api.getNodePoints();
|
|
162
|
+
const blockEndIndex = api.getBlockEndIndex();
|
|
163
|
+
for (let i = startIndex; i < endIndex; ++i) {
|
|
164
|
+
const p = nodePoints[i];
|
|
165
|
+
switch (p.codePoint) {
|
|
166
|
+
case character.AsciiCodePoint.BACKSLASH:
|
|
167
|
+
i += 1;
|
|
168
|
+
break;
|
|
169
|
+
case character.AsciiCodePoint.OPEN_BRACKET: {
|
|
170
|
+
const delimiter = {
|
|
171
|
+
type: 'opener',
|
|
172
|
+
startIndex: i,
|
|
173
|
+
endIndex: i + 1,
|
|
174
|
+
};
|
|
175
|
+
return delimiter;
|
|
176
|
+
}
|
|
177
|
+
case character.AsciiCodePoint.CLOSE_BRACKET: {
|
|
178
|
+
if (i + 1 >= endIndex || nodePoints[i + 1].codePoint !== character.AsciiCodePoint.OPEN_PARENTHESIS)
|
|
179
|
+
break;
|
|
180
|
+
const destinationStartIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i + 2, blockEndIndex);
|
|
181
|
+
const destinationEndIndex = eatLinkDestination(nodePoints, destinationStartIndex, blockEndIndex);
|
|
182
|
+
if (destinationEndIndex < 0)
|
|
183
|
+
break;
|
|
184
|
+
const titleStartIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, destinationEndIndex, blockEndIndex);
|
|
185
|
+
const titleEndIndex = eatLinkTitle(nodePoints, titleStartIndex, blockEndIndex);
|
|
186
|
+
if (titleEndIndex < 0)
|
|
187
|
+
break;
|
|
188
|
+
const _startIndex = i;
|
|
189
|
+
const _endIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, titleEndIndex, blockEndIndex) + 1;
|
|
190
|
+
if (_endIndex > blockEndIndex ||
|
|
191
|
+
nodePoints[_endIndex - 1].codePoint !== character.AsciiCodePoint.CLOSE_PARENTHESIS)
|
|
192
|
+
break;
|
|
193
|
+
return {
|
|
194
|
+
type: 'closer',
|
|
195
|
+
startIndex: _startIndex,
|
|
196
|
+
endIndex: _endIndex,
|
|
197
|
+
destinationContent: destinationStartIndex < destinationEndIndex
|
|
198
|
+
? {
|
|
199
|
+
startIndex: destinationStartIndex,
|
|
200
|
+
endIndex: destinationEndIndex,
|
|
201
|
+
}
|
|
202
|
+
: undefined,
|
|
203
|
+
titleContent: titleStartIndex < titleEndIndex
|
|
204
|
+
? { startIndex: titleStartIndex, endIndex: titleEndIndex }
|
|
205
|
+
: undefined,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return null;
|
|
211
|
+
}
|
|
212
|
+
function isDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
213
|
+
const nodePoints = api.getNodePoints();
|
|
214
|
+
const hasInternalLinkToken = internalTokens.find(coreTokenizer.isLinkToken) != null;
|
|
215
|
+
if (hasInternalLinkToken) {
|
|
216
|
+
return { paired: false, opener: false, closer: false };
|
|
217
|
+
}
|
|
218
|
+
const balancedBracketsStatus = checkBalancedBracketsStatus(openerDelimiter.endIndex, closerDelimiter.startIndex, internalTokens, nodePoints);
|
|
219
|
+
switch (balancedBracketsStatus) {
|
|
220
|
+
case -1:
|
|
221
|
+
return { paired: false, opener: false, closer: true };
|
|
222
|
+
case 0:
|
|
223
|
+
return { paired: true };
|
|
224
|
+
case 1:
|
|
225
|
+
return { paired: false, opener: true, closer: false };
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
function processDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
229
|
+
const children = api.resolveInternalTokens(internalTokens, openerDelimiter.endIndex, closerDelimiter.startIndex);
|
|
230
|
+
const token = {
|
|
231
|
+
nodeType: ast.LinkType,
|
|
232
|
+
startIndex: openerDelimiter.startIndex,
|
|
233
|
+
endIndex: closerDelimiter.endIndex,
|
|
234
|
+
destinationContent: closerDelimiter.destinationContent,
|
|
235
|
+
titleContent: closerDelimiter.titleContent,
|
|
236
|
+
children,
|
|
237
|
+
};
|
|
238
|
+
return { tokens: [token] };
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
const parse = function (api) {
|
|
243
|
+
return {
|
|
244
|
+
parse: tokens => tokens.map(token => {
|
|
245
|
+
const nodePoints = api.getNodePoints();
|
|
246
|
+
let url = '';
|
|
247
|
+
if (token.destinationContent != null) {
|
|
248
|
+
let { startIndex, endIndex } = token.destinationContent;
|
|
249
|
+
if (nodePoints[startIndex].codePoint === character.AsciiCodePoint.OPEN_ANGLE) {
|
|
250
|
+
startIndex += 1;
|
|
251
|
+
endIndex -= 1;
|
|
252
|
+
}
|
|
253
|
+
const destination = character.calcEscapedStringFromNodePoints(nodePoints, startIndex, endIndex, true);
|
|
254
|
+
url = coreTokenizer.encodeLinkDestination(destination);
|
|
255
|
+
}
|
|
256
|
+
let title;
|
|
257
|
+
if (token.titleContent != null) {
|
|
258
|
+
const { startIndex, endIndex } = token.titleContent;
|
|
259
|
+
title = character.calcEscapedStringFromNodePoints(nodePoints, startIndex + 1, endIndex - 1);
|
|
260
|
+
}
|
|
261
|
+
const children = api.parseInlineTokens(token.children);
|
|
262
|
+
const node = api.shouldReservePosition
|
|
263
|
+
? { type: ast.LinkType, position: api.calcPosition(token), url, title, children }
|
|
264
|
+
: { type: ast.LinkType, url, title, children };
|
|
265
|
+
return node;
|
|
266
|
+
}),
|
|
267
|
+
};
|
|
268
|
+
};
|
|
269
|
+
|
|
154
270
|
const uniqueName = '@yozora/tokenizer-link';
|
|
155
271
|
|
|
156
272
|
class LinkTokenizer extends coreTokenizer.BaseInlineTokenizer {
|
|
@@ -160,121 +276,8 @@ class LinkTokenizer extends coreTokenizer.BaseInlineTokenizer {
|
|
|
160
276
|
name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
|
|
161
277
|
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.LINKS,
|
|
162
278
|
});
|
|
163
|
-
this.match =
|
|
164
|
-
|
|
165
|
-
findDelimiter: () => coreTokenizer.genFindDelimiter(_findDelimiter),
|
|
166
|
-
isDelimiterPair,
|
|
167
|
-
processDelimiterPair,
|
|
168
|
-
};
|
|
169
|
-
function _findDelimiter(startIndex, endIndex) {
|
|
170
|
-
const nodePoints = api.getNodePoints();
|
|
171
|
-
const blockEndIndex = api.getBlockEndIndex();
|
|
172
|
-
for (let i = startIndex; i < endIndex; ++i) {
|
|
173
|
-
const p = nodePoints[i];
|
|
174
|
-
switch (p.codePoint) {
|
|
175
|
-
case character.AsciiCodePoint.BACKSLASH:
|
|
176
|
-
i += 1;
|
|
177
|
-
break;
|
|
178
|
-
case character.AsciiCodePoint.OPEN_BRACKET: {
|
|
179
|
-
const delimiter = {
|
|
180
|
-
type: 'opener',
|
|
181
|
-
startIndex: i,
|
|
182
|
-
endIndex: i + 1,
|
|
183
|
-
};
|
|
184
|
-
return delimiter;
|
|
185
|
-
}
|
|
186
|
-
case character.AsciiCodePoint.CLOSE_BRACKET: {
|
|
187
|
-
if (i + 1 >= endIndex ||
|
|
188
|
-
nodePoints[i + 1].codePoint !== character.AsciiCodePoint.OPEN_PARENTHESIS)
|
|
189
|
-
break;
|
|
190
|
-
const destinationStartIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, i + 2, blockEndIndex);
|
|
191
|
-
const destinationEndIndex = eatLinkDestination(nodePoints, destinationStartIndex, blockEndIndex);
|
|
192
|
-
if (destinationEndIndex < 0)
|
|
193
|
-
break;
|
|
194
|
-
const titleStartIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, destinationEndIndex, blockEndIndex);
|
|
195
|
-
const titleEndIndex = eatLinkTitle(nodePoints, titleStartIndex, blockEndIndex);
|
|
196
|
-
if (titleEndIndex < 0)
|
|
197
|
-
break;
|
|
198
|
-
const _startIndex = i;
|
|
199
|
-
const _endIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, titleEndIndex, blockEndIndex) + 1;
|
|
200
|
-
if (_endIndex > blockEndIndex ||
|
|
201
|
-
nodePoints[_endIndex - 1].codePoint !== character.AsciiCodePoint.CLOSE_PARENTHESIS)
|
|
202
|
-
break;
|
|
203
|
-
return {
|
|
204
|
-
type: 'closer',
|
|
205
|
-
startIndex: _startIndex,
|
|
206
|
-
endIndex: _endIndex,
|
|
207
|
-
destinationContent: destinationStartIndex < destinationEndIndex
|
|
208
|
-
? {
|
|
209
|
-
startIndex: destinationStartIndex,
|
|
210
|
-
endIndex: destinationEndIndex,
|
|
211
|
-
}
|
|
212
|
-
: undefined,
|
|
213
|
-
titleContent: titleStartIndex < titleEndIndex
|
|
214
|
-
? { startIndex: titleStartIndex, endIndex: titleEndIndex }
|
|
215
|
-
: undefined,
|
|
216
|
-
};
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
return null;
|
|
221
|
-
}
|
|
222
|
-
function isDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
223
|
-
const nodePoints = api.getNodePoints();
|
|
224
|
-
const hasInternalLinkToken = internalTokens.find(coreTokenizer.isLinkToken) != null;
|
|
225
|
-
if (hasInternalLinkToken) {
|
|
226
|
-
return { paired: false, opener: false, closer: false };
|
|
227
|
-
}
|
|
228
|
-
const balancedBracketsStatus = checkBalancedBracketsStatus(openerDelimiter.endIndex, closerDelimiter.startIndex, internalTokens, nodePoints);
|
|
229
|
-
switch (balancedBracketsStatus) {
|
|
230
|
-
case -1:
|
|
231
|
-
return { paired: false, opener: false, closer: true };
|
|
232
|
-
case 0:
|
|
233
|
-
return { paired: true };
|
|
234
|
-
case 1:
|
|
235
|
-
return { paired: false, opener: true, closer: false };
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
function processDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
239
|
-
const children = api.resolveInternalTokens(internalTokens, openerDelimiter.endIndex, closerDelimiter.startIndex);
|
|
240
|
-
const token = {
|
|
241
|
-
nodeType: ast.LinkType,
|
|
242
|
-
startIndex: openerDelimiter.startIndex,
|
|
243
|
-
endIndex: closerDelimiter.endIndex,
|
|
244
|
-
destinationContent: closerDelimiter.destinationContent,
|
|
245
|
-
titleContent: closerDelimiter.titleContent,
|
|
246
|
-
children,
|
|
247
|
-
};
|
|
248
|
-
return { tokens: [token] };
|
|
249
|
-
}
|
|
250
|
-
};
|
|
251
|
-
this.parse = api => ({
|
|
252
|
-
parse: (token, children) => {
|
|
253
|
-
const nodePoints = api.getNodePoints();
|
|
254
|
-
let url = '';
|
|
255
|
-
if (token.destinationContent != null) {
|
|
256
|
-
let { startIndex, endIndex } = token.destinationContent;
|
|
257
|
-
if (nodePoints[startIndex].codePoint === character.AsciiCodePoint.OPEN_ANGLE) {
|
|
258
|
-
startIndex += 1;
|
|
259
|
-
endIndex -= 1;
|
|
260
|
-
}
|
|
261
|
-
const destination = character.calcEscapedStringFromNodePoints(nodePoints, startIndex, endIndex, true);
|
|
262
|
-
url = coreTokenizer.encodeLinkDestination(destination);
|
|
263
|
-
}
|
|
264
|
-
let title;
|
|
265
|
-
if (token.titleContent != null) {
|
|
266
|
-
const { startIndex, endIndex } = token.titleContent;
|
|
267
|
-
title = character.calcEscapedStringFromNodePoints(nodePoints, startIndex + 1, endIndex - 1);
|
|
268
|
-
}
|
|
269
|
-
const result = {
|
|
270
|
-
type: ast.LinkType,
|
|
271
|
-
url,
|
|
272
|
-
title,
|
|
273
|
-
children,
|
|
274
|
-
};
|
|
275
|
-
return result;
|
|
276
|
-
},
|
|
277
|
-
});
|
|
279
|
+
this.match = match;
|
|
280
|
+
this.parse = parse;
|
|
278
281
|
}
|
|
279
282
|
}
|
|
280
283
|
|
package/lib/esm/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AsciiCodePoint, isWhitespaceCharacter, isAsciiControlCharacter, VirtualCodePoint, calcEscapedStringFromNodePoints } from '@yozora/character';
|
|
2
|
-
import { eatOptionalBlankLines,
|
|
2
|
+
import { eatOptionalBlankLines, genFindDelimiter, eatOptionalWhitespaces, isLinkToken, encodeLinkDestination, BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
|
|
3
3
|
import { LinkType } from '@yozora/ast';
|
|
4
4
|
|
|
5
5
|
const checkBalancedBracketsStatus = (startIndex, endIndex, internalTokens, nodePoints) => {
|
|
@@ -147,6 +147,122 @@ function eatLinkTitle(nodePoints, startIndex, endIndex) {
|
|
|
147
147
|
return -1;
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
+
const match = function (api) {
|
|
151
|
+
return {
|
|
152
|
+
findDelimiter: () => genFindDelimiter(_findDelimiter),
|
|
153
|
+
isDelimiterPair,
|
|
154
|
+
processDelimiterPair,
|
|
155
|
+
};
|
|
156
|
+
function _findDelimiter(startIndex, endIndex) {
|
|
157
|
+
const nodePoints = api.getNodePoints();
|
|
158
|
+
const blockEndIndex = api.getBlockEndIndex();
|
|
159
|
+
for (let i = startIndex; i < endIndex; ++i) {
|
|
160
|
+
const p = nodePoints[i];
|
|
161
|
+
switch (p.codePoint) {
|
|
162
|
+
case AsciiCodePoint.BACKSLASH:
|
|
163
|
+
i += 1;
|
|
164
|
+
break;
|
|
165
|
+
case AsciiCodePoint.OPEN_BRACKET: {
|
|
166
|
+
const delimiter = {
|
|
167
|
+
type: 'opener',
|
|
168
|
+
startIndex: i,
|
|
169
|
+
endIndex: i + 1,
|
|
170
|
+
};
|
|
171
|
+
return delimiter;
|
|
172
|
+
}
|
|
173
|
+
case AsciiCodePoint.CLOSE_BRACKET: {
|
|
174
|
+
if (i + 1 >= endIndex || nodePoints[i + 1].codePoint !== AsciiCodePoint.OPEN_PARENTHESIS)
|
|
175
|
+
break;
|
|
176
|
+
const destinationStartIndex = eatOptionalWhitespaces(nodePoints, i + 2, blockEndIndex);
|
|
177
|
+
const destinationEndIndex = eatLinkDestination(nodePoints, destinationStartIndex, blockEndIndex);
|
|
178
|
+
if (destinationEndIndex < 0)
|
|
179
|
+
break;
|
|
180
|
+
const titleStartIndex = eatOptionalWhitespaces(nodePoints, destinationEndIndex, blockEndIndex);
|
|
181
|
+
const titleEndIndex = eatLinkTitle(nodePoints, titleStartIndex, blockEndIndex);
|
|
182
|
+
if (titleEndIndex < 0)
|
|
183
|
+
break;
|
|
184
|
+
const _startIndex = i;
|
|
185
|
+
const _endIndex = eatOptionalWhitespaces(nodePoints, titleEndIndex, blockEndIndex) + 1;
|
|
186
|
+
if (_endIndex > blockEndIndex ||
|
|
187
|
+
nodePoints[_endIndex - 1].codePoint !== AsciiCodePoint.CLOSE_PARENTHESIS)
|
|
188
|
+
break;
|
|
189
|
+
return {
|
|
190
|
+
type: 'closer',
|
|
191
|
+
startIndex: _startIndex,
|
|
192
|
+
endIndex: _endIndex,
|
|
193
|
+
destinationContent: destinationStartIndex < destinationEndIndex
|
|
194
|
+
? {
|
|
195
|
+
startIndex: destinationStartIndex,
|
|
196
|
+
endIndex: destinationEndIndex,
|
|
197
|
+
}
|
|
198
|
+
: undefined,
|
|
199
|
+
titleContent: titleStartIndex < titleEndIndex
|
|
200
|
+
? { startIndex: titleStartIndex, endIndex: titleEndIndex }
|
|
201
|
+
: undefined,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
function isDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
209
|
+
const nodePoints = api.getNodePoints();
|
|
210
|
+
const hasInternalLinkToken = internalTokens.find(isLinkToken) != null;
|
|
211
|
+
if (hasInternalLinkToken) {
|
|
212
|
+
return { paired: false, opener: false, closer: false };
|
|
213
|
+
}
|
|
214
|
+
const balancedBracketsStatus = checkBalancedBracketsStatus(openerDelimiter.endIndex, closerDelimiter.startIndex, internalTokens, nodePoints);
|
|
215
|
+
switch (balancedBracketsStatus) {
|
|
216
|
+
case -1:
|
|
217
|
+
return { paired: false, opener: false, closer: true };
|
|
218
|
+
case 0:
|
|
219
|
+
return { paired: true };
|
|
220
|
+
case 1:
|
|
221
|
+
return { paired: false, opener: true, closer: false };
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function processDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
225
|
+
const children = api.resolveInternalTokens(internalTokens, openerDelimiter.endIndex, closerDelimiter.startIndex);
|
|
226
|
+
const token = {
|
|
227
|
+
nodeType: LinkType,
|
|
228
|
+
startIndex: openerDelimiter.startIndex,
|
|
229
|
+
endIndex: closerDelimiter.endIndex,
|
|
230
|
+
destinationContent: closerDelimiter.destinationContent,
|
|
231
|
+
titleContent: closerDelimiter.titleContent,
|
|
232
|
+
children,
|
|
233
|
+
};
|
|
234
|
+
return { tokens: [token] };
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
const parse = function (api) {
|
|
239
|
+
return {
|
|
240
|
+
parse: tokens => tokens.map(token => {
|
|
241
|
+
const nodePoints = api.getNodePoints();
|
|
242
|
+
let url = '';
|
|
243
|
+
if (token.destinationContent != null) {
|
|
244
|
+
let { startIndex, endIndex } = token.destinationContent;
|
|
245
|
+
if (nodePoints[startIndex].codePoint === AsciiCodePoint.OPEN_ANGLE) {
|
|
246
|
+
startIndex += 1;
|
|
247
|
+
endIndex -= 1;
|
|
248
|
+
}
|
|
249
|
+
const destination = calcEscapedStringFromNodePoints(nodePoints, startIndex, endIndex, true);
|
|
250
|
+
url = encodeLinkDestination(destination);
|
|
251
|
+
}
|
|
252
|
+
let title;
|
|
253
|
+
if (token.titleContent != null) {
|
|
254
|
+
const { startIndex, endIndex } = token.titleContent;
|
|
255
|
+
title = calcEscapedStringFromNodePoints(nodePoints, startIndex + 1, endIndex - 1);
|
|
256
|
+
}
|
|
257
|
+
const children = api.parseInlineTokens(token.children);
|
|
258
|
+
const node = api.shouldReservePosition
|
|
259
|
+
? { type: LinkType, position: api.calcPosition(token), url, title, children }
|
|
260
|
+
: { type: LinkType, url, title, children };
|
|
261
|
+
return node;
|
|
262
|
+
}),
|
|
263
|
+
};
|
|
264
|
+
};
|
|
265
|
+
|
|
150
266
|
const uniqueName = '@yozora/tokenizer-link';
|
|
151
267
|
|
|
152
268
|
class LinkTokenizer extends BaseInlineTokenizer {
|
|
@@ -156,121 +272,8 @@ class LinkTokenizer extends BaseInlineTokenizer {
|
|
|
156
272
|
name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
|
|
157
273
|
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.LINKS,
|
|
158
274
|
});
|
|
159
|
-
this.match =
|
|
160
|
-
|
|
161
|
-
findDelimiter: () => genFindDelimiter(_findDelimiter),
|
|
162
|
-
isDelimiterPair,
|
|
163
|
-
processDelimiterPair,
|
|
164
|
-
};
|
|
165
|
-
function _findDelimiter(startIndex, endIndex) {
|
|
166
|
-
const nodePoints = api.getNodePoints();
|
|
167
|
-
const blockEndIndex = api.getBlockEndIndex();
|
|
168
|
-
for (let i = startIndex; i < endIndex; ++i) {
|
|
169
|
-
const p = nodePoints[i];
|
|
170
|
-
switch (p.codePoint) {
|
|
171
|
-
case AsciiCodePoint.BACKSLASH:
|
|
172
|
-
i += 1;
|
|
173
|
-
break;
|
|
174
|
-
case AsciiCodePoint.OPEN_BRACKET: {
|
|
175
|
-
const delimiter = {
|
|
176
|
-
type: 'opener',
|
|
177
|
-
startIndex: i,
|
|
178
|
-
endIndex: i + 1,
|
|
179
|
-
};
|
|
180
|
-
return delimiter;
|
|
181
|
-
}
|
|
182
|
-
case AsciiCodePoint.CLOSE_BRACKET: {
|
|
183
|
-
if (i + 1 >= endIndex ||
|
|
184
|
-
nodePoints[i + 1].codePoint !== AsciiCodePoint.OPEN_PARENTHESIS)
|
|
185
|
-
break;
|
|
186
|
-
const destinationStartIndex = eatOptionalWhitespaces(nodePoints, i + 2, blockEndIndex);
|
|
187
|
-
const destinationEndIndex = eatLinkDestination(nodePoints, destinationStartIndex, blockEndIndex);
|
|
188
|
-
if (destinationEndIndex < 0)
|
|
189
|
-
break;
|
|
190
|
-
const titleStartIndex = eatOptionalWhitespaces(nodePoints, destinationEndIndex, blockEndIndex);
|
|
191
|
-
const titleEndIndex = eatLinkTitle(nodePoints, titleStartIndex, blockEndIndex);
|
|
192
|
-
if (titleEndIndex < 0)
|
|
193
|
-
break;
|
|
194
|
-
const _startIndex = i;
|
|
195
|
-
const _endIndex = eatOptionalWhitespaces(nodePoints, titleEndIndex, blockEndIndex) + 1;
|
|
196
|
-
if (_endIndex > blockEndIndex ||
|
|
197
|
-
nodePoints[_endIndex - 1].codePoint !== AsciiCodePoint.CLOSE_PARENTHESIS)
|
|
198
|
-
break;
|
|
199
|
-
return {
|
|
200
|
-
type: 'closer',
|
|
201
|
-
startIndex: _startIndex,
|
|
202
|
-
endIndex: _endIndex,
|
|
203
|
-
destinationContent: destinationStartIndex < destinationEndIndex
|
|
204
|
-
? {
|
|
205
|
-
startIndex: destinationStartIndex,
|
|
206
|
-
endIndex: destinationEndIndex,
|
|
207
|
-
}
|
|
208
|
-
: undefined,
|
|
209
|
-
titleContent: titleStartIndex < titleEndIndex
|
|
210
|
-
? { startIndex: titleStartIndex, endIndex: titleEndIndex }
|
|
211
|
-
: undefined,
|
|
212
|
-
};
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
return null;
|
|
217
|
-
}
|
|
218
|
-
function isDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
219
|
-
const nodePoints = api.getNodePoints();
|
|
220
|
-
const hasInternalLinkToken = internalTokens.find(isLinkToken) != null;
|
|
221
|
-
if (hasInternalLinkToken) {
|
|
222
|
-
return { paired: false, opener: false, closer: false };
|
|
223
|
-
}
|
|
224
|
-
const balancedBracketsStatus = checkBalancedBracketsStatus(openerDelimiter.endIndex, closerDelimiter.startIndex, internalTokens, nodePoints);
|
|
225
|
-
switch (balancedBracketsStatus) {
|
|
226
|
-
case -1:
|
|
227
|
-
return { paired: false, opener: false, closer: true };
|
|
228
|
-
case 0:
|
|
229
|
-
return { paired: true };
|
|
230
|
-
case 1:
|
|
231
|
-
return { paired: false, opener: true, closer: false };
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
function processDelimiterPair(openerDelimiter, closerDelimiter, internalTokens) {
|
|
235
|
-
const children = api.resolveInternalTokens(internalTokens, openerDelimiter.endIndex, closerDelimiter.startIndex);
|
|
236
|
-
const token = {
|
|
237
|
-
nodeType: LinkType,
|
|
238
|
-
startIndex: openerDelimiter.startIndex,
|
|
239
|
-
endIndex: closerDelimiter.endIndex,
|
|
240
|
-
destinationContent: closerDelimiter.destinationContent,
|
|
241
|
-
titleContent: closerDelimiter.titleContent,
|
|
242
|
-
children,
|
|
243
|
-
};
|
|
244
|
-
return { tokens: [token] };
|
|
245
|
-
}
|
|
246
|
-
};
|
|
247
|
-
this.parse = api => ({
|
|
248
|
-
parse: (token, children) => {
|
|
249
|
-
const nodePoints = api.getNodePoints();
|
|
250
|
-
let url = '';
|
|
251
|
-
if (token.destinationContent != null) {
|
|
252
|
-
let { startIndex, endIndex } = token.destinationContent;
|
|
253
|
-
if (nodePoints[startIndex].codePoint === AsciiCodePoint.OPEN_ANGLE) {
|
|
254
|
-
startIndex += 1;
|
|
255
|
-
endIndex -= 1;
|
|
256
|
-
}
|
|
257
|
-
const destination = calcEscapedStringFromNodePoints(nodePoints, startIndex, endIndex, true);
|
|
258
|
-
url = encodeLinkDestination(destination);
|
|
259
|
-
}
|
|
260
|
-
let title;
|
|
261
|
-
if (token.titleContent != null) {
|
|
262
|
-
const { startIndex, endIndex } = token.titleContent;
|
|
263
|
-
title = calcEscapedStringFromNodePoints(nodePoints, startIndex + 1, endIndex - 1);
|
|
264
|
-
}
|
|
265
|
-
const result = {
|
|
266
|
-
type: LinkType,
|
|
267
|
-
url,
|
|
268
|
-
title,
|
|
269
|
-
children,
|
|
270
|
-
};
|
|
271
|
-
return result;
|
|
272
|
-
},
|
|
273
|
-
});
|
|
275
|
+
this.match = match;
|
|
276
|
+
this.parse = parse;
|
|
274
277
|
}
|
|
275
278
|
}
|
|
276
279
|
|
package/lib/types/index.d.ts
CHANGED
|
@@ -3,4 +3,4 @@ export * from './util/link-destination';
|
|
|
3
3
|
export * from './util/link-title';
|
|
4
4
|
export { LinkTokenizer, LinkTokenizer as default } from './tokenizer';
|
|
5
5
|
export { uniqueName as LinkTokenizerName } from './types';
|
|
6
|
-
export type { IToken as ILinkToken, ITokenizerProps as ILinkTokenizerProps } from './types';
|
|
6
|
+
export type { IThis as ILinkHookContext, IToken as ILinkToken, ITokenizerProps as ILinkTokenizerProps, } from './types';
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { IMatchInlineHookCreator } from '@yozora/core-tokenizer';
|
|
2
|
+
import type { IDelimiter, IThis, IToken, T } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* An inline link consists of a link text followed immediately by a left
|
|
5
|
+
* parenthesis '(', optional whitespace, an optional link destination, an
|
|
6
|
+
* optional link title separated from the link destination by whitespace,
|
|
7
|
+
* optional whitespace, and a right parenthesis ')'. The link’s text consists
|
|
8
|
+
* of the inlines contained in the link text (excluding the enclosing square
|
|
9
|
+
* brackets).
|
|
10
|
+
* The link’s URI consists of the link destination, excluding enclosing '<...>'
|
|
11
|
+
* if present, with backslash-escapes in effect as described above. The link’s
|
|
12
|
+
* title consists of the link title, excluding its enclosing delimiters, with
|
|
13
|
+
* backslash-escapes in effect as described above.
|
|
14
|
+
*
|
|
15
|
+
* ------
|
|
16
|
+
*
|
|
17
|
+
* A 'opener' type delimiter is one of the following forms:
|
|
18
|
+
*
|
|
19
|
+
* - '['
|
|
20
|
+
*
|
|
21
|
+
* A 'closer' type delimiter is one of the following forms:
|
|
22
|
+
*
|
|
23
|
+
* - '](url)'
|
|
24
|
+
* - '](url "title")'
|
|
25
|
+
* - '](<url>)'
|
|
26
|
+
* - '](<url> "title")'
|
|
27
|
+
*
|
|
28
|
+
* @see https://github.com/syntax-tree/mdast#link
|
|
29
|
+
* @see https://github.github.com/gfm/#links
|
|
30
|
+
*/
|
|
31
|
+
export declare const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;
|
package/lib/types/tokenizer.d.ts
CHANGED
|
@@ -1,38 +1,13 @@
|
|
|
1
1
|
import type { IInlineTokenizer, IMatchInlineHookCreator, IParseInlineHookCreator } from '@yozora/core-tokenizer';
|
|
2
2
|
import { BaseInlineTokenizer } from '@yozora/core-tokenizer';
|
|
3
|
-
import type { IDelimiter, INode, IToken, ITokenizerProps, T } from './types';
|
|
3
|
+
import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types';
|
|
4
4
|
/**
|
|
5
5
|
* Lexical Analyzer for InlineLink.
|
|
6
|
-
*
|
|
7
|
-
* An inline link consists of a link text followed immediately by a left
|
|
8
|
-
* parenthesis '(', optional whitespace, an optional link destination, an
|
|
9
|
-
* optional link title separated from the link destination by whitespace,
|
|
10
|
-
* optional whitespace, and a right parenthesis ')'. The link’s text consists
|
|
11
|
-
* of the inlines contained in the link text (excluding the enclosing square
|
|
12
|
-
* brackets).
|
|
13
|
-
* The link’s URI consists of the link destination, excluding enclosing '<...>'
|
|
14
|
-
* if present, with backslash-escapes in effect as described above. The link’s
|
|
15
|
-
* title consists of the link title, excluding its enclosing delimiters, with
|
|
16
|
-
* backslash-escapes in effect as described above.
|
|
17
|
-
*
|
|
18
|
-
* ------
|
|
19
|
-
*
|
|
20
|
-
* A 'opener' type delimiter is one of the following forms:
|
|
21
|
-
*
|
|
22
|
-
* - '['
|
|
23
|
-
*
|
|
24
|
-
* A 'closer' type delimiter is one of the following forms:
|
|
25
|
-
*
|
|
26
|
-
* - '](url)'
|
|
27
|
-
* - '](url "title")'
|
|
28
|
-
* - '](<url>)'
|
|
29
|
-
* - '](<url> "title")'
|
|
30
|
-
*
|
|
31
6
|
* @see https://github.com/syntax-tree/mdast#link
|
|
32
7
|
* @see https://github.github.com/gfm/#links
|
|
33
8
|
*/
|
|
34
|
-
export declare class LinkTokenizer extends BaseInlineTokenizer<T, IDelimiter, IToken, INode> implements IInlineTokenizer<T, IDelimiter, IToken, INode> {
|
|
9
|
+
export declare class LinkTokenizer extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis> implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis> {
|
|
35
10
|
constructor(props?: ITokenizerProps);
|
|
36
|
-
readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken>;
|
|
37
|
-
readonly parse: IParseInlineHookCreator<T, IToken, INode>;
|
|
11
|
+
readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;
|
|
12
|
+
readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis>;
|
|
38
13
|
}
|
package/lib/types/types.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Link, LinkType } from '@yozora/ast';
|
|
2
2
|
import type { INodeInterval } from '@yozora/character';
|
|
3
|
-
import type { IBaseInlineTokenizerProps, IPartialYastInlineToken, IYastTokenDelimiter } from '@yozora/core-tokenizer';
|
|
3
|
+
import type { IBaseInlineTokenizerProps, IPartialYastInlineToken, ITokenizer, IYastTokenDelimiter } from '@yozora/core-tokenizer';
|
|
4
4
|
export declare type T = LinkType;
|
|
5
|
-
export declare type INode =
|
|
5
|
+
export declare type INode = Link;
|
|
6
6
|
export declare const uniqueName = "@yozora/tokenizer-link";
|
|
7
7
|
export interface IToken extends IPartialYastInlineToken<T> {
|
|
8
8
|
/**
|
|
@@ -28,4 +28,5 @@ export interface IDelimiter extends IYastTokenDelimiter {
|
|
|
28
28
|
*/
|
|
29
29
|
titleContent?: INodeInterval;
|
|
30
30
|
}
|
|
31
|
+
export declare type IThis = ITokenizer;
|
|
31
32
|
export declare type ITokenizerProps = Partial<IBaseInlineTokenizerProps>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yozora/tokenizer-link",
|
|
3
|
-
"version": "2.0.0
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"author": {
|
|
5
5
|
"name": "guanghechen",
|
|
6
6
|
"url": "https://github.com/guanghechen/"
|
|
@@ -35,9 +35,9 @@
|
|
|
35
35
|
"test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@yozora/ast": "^2.0.0
|
|
39
|
-
"@yozora/character": "^2.0.0
|
|
40
|
-
"@yozora/core-tokenizer": "^2.0.0
|
|
38
|
+
"@yozora/ast": "^2.0.0",
|
|
39
|
+
"@yozora/character": "^2.0.0",
|
|
40
|
+
"@yozora/core-tokenizer": "^2.0.0"
|
|
41
41
|
},
|
|
42
|
-
"gitHead": "
|
|
42
|
+
"gitHead": "65e99d1709fdd1c918465dce6b1e91de96bdab5e"
|
|
43
43
|
}
|