@yozora/tokenizer-inline-math 2.3.6 → 2.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +45 -44
- package/lib/cjs/index.cjs +104 -65
- package/lib/esm/index.mjs +105 -67
- package/lib/types/index.d.ts +3 -8
- package/package.json +6 -6
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,31 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file. See
|
|
4
4
|
[Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
## <small>2.3.8 (2024-12-03)</small>
|
|
7
|
+
|
|
8
|
+
- improve(inlineMath): support more than one dollar sign as delimiter
|
|
9
|
+
([bf17c33](https://github.com/yozorajs/yozora/commit/bf17c33))
|
|
10
|
+
- :bookmark: release: publish v2.3.7 ([ba79410](https://github.com/yozorajs/yozora/commit/ba79410))
|
|
11
|
+
- :bug: fix(inline-math): the inline syntax wihtout backticks could be interruptted by
|
|
12
|
+
emphasis/strong ([c347f97](https://github.com/yozorajs/yozora/commit/c347f97))
|
|
13
|
+
- :white_check_mark: test: fix tests ([dd9c762](https://github.com/yozorajs/yozora/commit/dd9c762))
|
|
14
|
+
|
|
15
|
+
# Change Log
|
|
16
|
+
|
|
17
|
+
All notable changes to this project will be documented in this file. See
|
|
18
|
+
[Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
19
|
+
|
|
20
|
+
## <small>2.3.7 (2024-11-06)</small>
|
|
21
|
+
|
|
22
|
+
- :bug: fix(inline-math): the inline syntax wihtout backticks could be interruptted by
|
|
23
|
+
emphasis/strong ([c347f97](https://github.com/yozorajs/yozora/commit/c347f97))
|
|
24
|
+
- :white_check_mark: test: fix tests ([dd9c762](https://github.com/yozorajs/yozora/commit/dd9c762))
|
|
25
|
+
|
|
26
|
+
# Change Log
|
|
27
|
+
|
|
28
|
+
All notable changes to this project will be documented in this file. See
|
|
29
|
+
[Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
30
|
+
|
|
6
31
|
## <small>2.3.6 (2024-10-23)</small>
|
|
7
32
|
|
|
8
33
|
- :bug: fix(tokenizer-inline-math): the dollar syntax should meet the opener and closer conditions
|
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
<header>
|
|
4
4
|
<h1 align="center">
|
|
5
|
-
<a href="https://github.com/yozorajs/yozora/tree/v2.3.
|
|
5
|
+
<a href="https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/inline-math#readme">@yozora/tokenizer-inline-math</a>
|
|
6
6
|
</h1>
|
|
7
7
|
<div align="center">
|
|
8
8
|
<a href="https://www.npmjs.com/package/@yozora/tokenizer-inline-math">
|
|
@@ -89,7 +89,8 @@ import InlineMathTokenizer from '@yozora/tokenizer-inline-math'
|
|
|
89
89
|
const parser = new DefaultParser()
|
|
90
90
|
.useFallbackTokenizer(new ParagraphTokenizer())
|
|
91
91
|
.useFallbackTokenizer(new TextTokenizer())
|
|
92
|
-
.useTokenizer(new InlineMathTokenizer())
|
|
92
|
+
.useTokenizer(new InlineMathTokenizer({ backtickRequired: true }))
|
|
93
|
+
.useTokenizer(new InlineMathTokenizer({ backtickRequired: false }))
|
|
93
94
|
|
|
94
95
|
// parse source markdown content
|
|
95
96
|
parser.parse("`$x^2 + y^2 = z^2, x < 0$`")
|
|
@@ -113,7 +114,7 @@ import GfmParser from '@yozora/parser-gfm'
|
|
|
113
114
|
import InlineMathTokenizer from '@yozora/tokenizer-inline-math'
|
|
114
115
|
|
|
115
116
|
const parser = new GfmParser()
|
|
116
|
-
parser.useTokenizer(new InlineMathTokenizer())
|
|
117
|
+
parser.useTokenizer(new InlineMathTokenizer({ backtickRequired: true }))
|
|
117
118
|
|
|
118
119
|
// parse source markdown content
|
|
119
120
|
parser.parse("`$x^2 + y^2 = z^2, x < 0$`")
|
|
@@ -180,80 +181,80 @@ parser.parse("`$x^2 + y^2 = z^2, x < 0$`")
|
|
|
180
181
|
|
|
181
182
|
[live-examples]: https://yozora.guanghechen.com/docs/package/tokenizer-inline-math#live-examples
|
|
182
183
|
[docpage]: https://yozora.guanghechen.com/docs/package/tokenizer-inline-math
|
|
183
|
-
[homepage]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
184
|
+
[homepage]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/inline-math#readme
|
|
184
185
|
[gfm-spec]: https://github.github.com/gfm
|
|
185
186
|
[mdast-homepage]: https://github.com/syntax-tree/mdast
|
|
186
|
-
[@yozora/ast]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
187
|
-
[@yozora/ast-util]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
188
|
-
[@yozora/character]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
187
|
+
[@yozora/ast]: https://github.com/yozorajs/yozora/tree/v2.3.8/packages/ast#readme
|
|
188
|
+
[@yozora/ast-util]: https://github.com/yozorajs/yozora/tree/v2.3.8/packages/ast-util#readme
|
|
189
|
+
[@yozora/character]: https://github.com/yozorajs/yozora/tree/v2.3.8/packages/character#readme
|
|
189
190
|
[@yozora/eslint-config]:
|
|
190
191
|
https://github.com/yozorajs/yozora/tree/release-2.x.x/packages/eslint-config#readme
|
|
191
|
-
[@yozora/core-parser]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
192
|
+
[@yozora/core-parser]: https://github.com/yozorajs/yozora/tree/v2.3.8/packages/core-parser#readme
|
|
192
193
|
[@yozora/core-tokenizer]:
|
|
193
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
194
|
-
[@yozora/invariant]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
194
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/packages/core-tokenizer#readme
|
|
195
|
+
[@yozora/invariant]: https://github.com/yozorajs/yozora/tree/v2.3.8/packages/invariant#readme
|
|
195
196
|
[@yozora/jest-for-tokenizer]:
|
|
196
197
|
https://github.com/yozorajs/yozora/tree/release-2.x.x/packages/jest-for-tokenizer#readme
|
|
197
|
-
[@yozora/parser]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
198
|
-
[@yozora/parser-gfm]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
198
|
+
[@yozora/parser]: https://github.com/yozorajs/yozora/tree/v2.3.8/packages/parser#readme
|
|
199
|
+
[@yozora/parser-gfm]: https://github.com/yozorajs/yozora/tree/v2.3.8/packages/parser-gfm#readme
|
|
199
200
|
[@yozora/parser-gfm-ex]:
|
|
200
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
201
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/packages/parser-gfm-ex#readme
|
|
201
202
|
[@yozora/template-tokenizer]:
|
|
202
203
|
https://github.com/yozorajs/yozora/tree/release-2.x.x/packages/template-tokenizer#readme
|
|
203
204
|
[@yozora/tokenizer-admonition]:
|
|
204
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
205
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/admonition#readme
|
|
205
206
|
[@yozora/tokenizer-autolink]:
|
|
206
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
207
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/autolink#readme
|
|
207
208
|
[@yozora/tokenizer-autolink-extension]:
|
|
208
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
209
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/autolink-extension#readme
|
|
209
210
|
[@yozora/tokenizer-blockquote]:
|
|
210
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
211
|
-
[@yozora/tokenizer-break]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
211
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/blockquote#readme
|
|
212
|
+
[@yozora/tokenizer-break]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/break#readme
|
|
212
213
|
[@yozora/tokenizer-definition]:
|
|
213
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
214
|
-
[@yozora/tokenizer-delete]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
214
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/definition#readme
|
|
215
|
+
[@yozora/tokenizer-delete]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/delete#readme
|
|
215
216
|
[@yozora/tokenizer-ecma-import]:
|
|
216
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
217
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/ecma-import#readme
|
|
217
218
|
[@yozora/tokenizer-emphasis]:
|
|
218
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
219
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/emphasis#readme
|
|
219
220
|
[@yozora/tokenizer-fenced-block]:
|
|
220
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
221
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/fenced-block#readme
|
|
221
222
|
[@yozora/tokenizer-fenced-code]:
|
|
222
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
223
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/fenced-code#readme
|
|
223
224
|
[@yozora/tokenizer-footnote]:
|
|
224
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
225
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/footnote#readme
|
|
225
226
|
[@yozora/tokenizer-footnote-definition]:
|
|
226
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
227
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/footnote-definition#readme
|
|
227
228
|
[@yozora/tokenizer-footnote-reference]:
|
|
228
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
229
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/footnote-reference#readme
|
|
229
230
|
[@yozora/tokenizer-heading]:
|
|
230
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
231
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/heading#readme
|
|
231
232
|
[@yozora/tokenizer-html-block]:
|
|
232
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
233
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/html-block#readme
|
|
233
234
|
[@yozora/tokenizer-html-inline]:
|
|
234
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
235
|
-
[@yozora/tokenizer-image]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
235
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/html-inline#readme
|
|
236
|
+
[@yozora/tokenizer-image]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/image#readme
|
|
236
237
|
[@yozora/tokenizer-image-reference]:
|
|
237
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
238
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/image-reference#readme
|
|
238
239
|
[@yozora/tokenizer-indented-code]:
|
|
239
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
240
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/indented-code#readme
|
|
240
241
|
[@yozora/tokenizer-inline-code]:
|
|
241
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
242
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/inline-code#readme
|
|
242
243
|
[@yozora/tokenizer-inline-math]:
|
|
243
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
244
|
-
[@yozora/tokenizer-link]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
244
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/inline-math#readme
|
|
245
|
+
[@yozora/tokenizer-link]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/link#readme
|
|
245
246
|
[@yozora/tokenizer-link-reference]:
|
|
246
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
247
|
-
[@yozora/tokenizer-list]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
248
|
-
[@yozora/tokenizer-math]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
247
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/link-reference#readme
|
|
248
|
+
[@yozora/tokenizer-list]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/list#readme
|
|
249
|
+
[@yozora/tokenizer-math]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/math#readme
|
|
249
250
|
[@yozora/tokenizer-paragraph]:
|
|
250
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
251
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/paragraph#readme
|
|
251
252
|
[@yozora/tokenizer-setext-heading]:
|
|
252
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
253
|
-
[@yozora/tokenizer-table]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
254
|
-
[@yozora/tokenizer-text]: https://github.com/yozorajs/yozora/tree/v2.3.
|
|
253
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/setext-heading#readme
|
|
254
|
+
[@yozora/tokenizer-table]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/table#readme
|
|
255
|
+
[@yozora/tokenizer-text]: https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/text#readme
|
|
255
256
|
[@yozora/tokenizer-thematic-break]:
|
|
256
|
-
https://github.com/yozorajs/yozora/tree/v2.3.
|
|
257
|
+
https://github.com/yozorajs/yozora/tree/v2.3.8/tokenizers/thematic-break#readme
|
|
257
258
|
[@yozora/react-admonition]:
|
|
258
259
|
https://github.com/yozorajs/yozora-react/tree/main/packages/admonition#readme
|
|
259
260
|
[@yozora/react-blockquote]:
|
package/lib/cjs/index.cjs
CHANGED
|
@@ -7,14 +7,101 @@ var character = require('@yozora/character');
|
|
|
7
7
|
var coreTokenizer = require('@yozora/core-tokenizer');
|
|
8
8
|
|
|
9
9
|
const match = function (api) {
|
|
10
|
-
|
|
10
|
+
return {
|
|
11
|
+
findDelimiter: () => coreTokenizer.genFindDelimiter(_findDelimiter),
|
|
12
|
+
isDelimiterPair,
|
|
13
|
+
processDelimiterPair,
|
|
14
|
+
};
|
|
15
|
+
function _findDelimiter(startIndex, endIndex) {
|
|
16
|
+
const nodePoints = api.getNodePoints();
|
|
17
|
+
const blockStartIndex = api.getBlockStartIndex();
|
|
18
|
+
const blockEndIndex = api.getBlockEndIndex();
|
|
19
|
+
for (let i = startIndex; i < endIndex; ++i) {
|
|
20
|
+
const c = nodePoints[i].codePoint;
|
|
21
|
+
switch (c) {
|
|
22
|
+
case character.AsciiCodePoint.BACKSLASH:
|
|
23
|
+
i += 1;
|
|
24
|
+
break;
|
|
25
|
+
case character.AsciiCodePoint.DOLLAR_SIGN: {
|
|
26
|
+
const leftCodePoint = i === blockStartIndex ? null : nodePoints[i - 1].codePoint;
|
|
27
|
+
const _startIndex = i;
|
|
28
|
+
i = coreTokenizer.eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, character.AsciiCodePoint.DOLLAR_SIGN);
|
|
29
|
+
const rightCodePoint = i === blockEndIndex ? null : nodePoints[i].codePoint;
|
|
30
|
+
const isPotentialOpener = (leftCodePoint === null || character.isWhitespaceCharacter(leftCodePoint)) &&
|
|
31
|
+
(rightCodePoint === null || !character.isWhitespaceCharacter(rightCodePoint));
|
|
32
|
+
const isPotentialCloser = (leftCodePoint === null || !character.isWhitespaceCharacter(leftCodePoint)) &&
|
|
33
|
+
(rightCodePoint === null || character.isWhitespaceCharacter(rightCodePoint));
|
|
34
|
+
if (!isPotentialOpener && !isPotentialCloser)
|
|
35
|
+
break;
|
|
36
|
+
const delimiterType = isPotentialOpener
|
|
37
|
+
? isPotentialCloser
|
|
38
|
+
? 'both'
|
|
39
|
+
: 'opener'
|
|
40
|
+
: 'closer';
|
|
41
|
+
const delimiter = {
|
|
42
|
+
type: delimiterType,
|
|
43
|
+
startIndex: _startIndex,
|
|
44
|
+
endIndex: i,
|
|
45
|
+
thickness: i - _startIndex,
|
|
46
|
+
};
|
|
47
|
+
return delimiter;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
function isDelimiterPair(openerDelimiter, closerDelimiter) {
|
|
54
|
+
if (openerDelimiter.thickness === closerDelimiter.thickness)
|
|
55
|
+
return { paired: true };
|
|
56
|
+
return { paired: false, opener: true, closer: true };
|
|
57
|
+
}
|
|
58
|
+
function processDelimiterPair(openerDelimiter, closerDelimiter) {
|
|
59
|
+
const token = {
|
|
60
|
+
nodeType: ast.InlineMathType,
|
|
61
|
+
startIndex: openerDelimiter.startIndex,
|
|
62
|
+
endIndex: closerDelimiter.endIndex,
|
|
63
|
+
thickness: openerDelimiter.thickness,
|
|
64
|
+
};
|
|
65
|
+
return { tokens: [token] };
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const parse = function (api) {
|
|
70
|
+
return {
|
|
71
|
+
parse: tokens => tokens.map(token => {
|
|
72
|
+
const nodePoints = api.getNodePoints();
|
|
73
|
+
let startIndex = token.startIndex + token.thickness;
|
|
74
|
+
let endIndex = token.endIndex - token.thickness;
|
|
75
|
+
let isAllSpace = true;
|
|
76
|
+
for (let i = startIndex; i < endIndex; ++i) {
|
|
77
|
+
if (character.isSpaceLike(nodePoints[i].codePoint))
|
|
78
|
+
continue;
|
|
79
|
+
isAllSpace = false;
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
if (!isAllSpace && startIndex + 2 < endIndex) {
|
|
83
|
+
const firstCharacter = nodePoints[startIndex].codePoint;
|
|
84
|
+
const lastCharacter = nodePoints[endIndex - 1].codePoint;
|
|
85
|
+
if (character.isSpaceLike(firstCharacter) && character.isSpaceLike(lastCharacter)) {
|
|
86
|
+
startIndex += 1;
|
|
87
|
+
endIndex -= 1;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const value = character.calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
|
|
91
|
+
const node = api.shouldReservePosition
|
|
92
|
+
? { type: ast.InlineMathType, position: api.calcPosition(token), value }
|
|
93
|
+
: { type: ast.InlineMathType, value };
|
|
94
|
+
return node;
|
|
95
|
+
}),
|
|
96
|
+
};
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
const matchWithBacktick = function (api) {
|
|
11
100
|
return { findDelimiter, processSingleDelimiter };
|
|
12
101
|
function* findDelimiter() {
|
|
13
102
|
const nodePoints = api.getNodePoints();
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const blockStartIndex = coreTokenizer.eatOptionalWhitespaces(nodePoints, originalBlockStartIndex, originalBlockEndIndex);
|
|
17
|
-
const blockEndIndex = coreTokenizer.eatOptionalWhitespacesReverse(nodePoints, originalBlockStartIndex, originalBlockEndIndex);
|
|
103
|
+
const blockStartIndex = api.getBlockStartIndex();
|
|
104
|
+
const blockEndIndex = api.getBlockEndIndex();
|
|
18
105
|
const potentialDelimiters = [];
|
|
19
106
|
for (let i = blockStartIndex; i < blockEndIndex; ++i) {
|
|
20
107
|
const c = nodePoints[i].codePoint;
|
|
@@ -43,29 +130,8 @@ const match = function (api) {
|
|
|
43
130
|
break;
|
|
44
131
|
}
|
|
45
132
|
const thickness = i - _startIndex;
|
|
46
|
-
if (thickness <= 1)
|
|
47
|
-
if (backtickRequired)
|
|
48
|
-
break;
|
|
49
|
-
const isPotentialOpener = _startIndex === blockStartIndex ||
|
|
50
|
-
character.isWhitespaceCharacter(nodePoints[_startIndex - 1].codePoint);
|
|
51
|
-
const isPotentialCloser = i === blockEndIndex ||
|
|
52
|
-
(_startIndex > blockStartIndex &&
|
|
53
|
-
!character.isWhitespaceCharacter(nodePoints[_startIndex - 1].codePoint));
|
|
54
|
-
if (!isPotentialOpener && !isPotentialCloser)
|
|
55
|
-
break;
|
|
56
|
-
const delimiterType = isPotentialOpener
|
|
57
|
-
? isPotentialCloser
|
|
58
|
-
? 'both'
|
|
59
|
-
: 'opener'
|
|
60
|
-
: 'closer';
|
|
61
|
-
const delimiter = {
|
|
62
|
-
type: delimiterType,
|
|
63
|
-
startIndex: _startIndex,
|
|
64
|
-
endIndex: i,
|
|
65
|
-
};
|
|
66
|
-
potentialDelimiters.push(delimiter);
|
|
133
|
+
if (thickness <= 1)
|
|
67
134
|
break;
|
|
68
|
-
}
|
|
69
135
|
const delimiter = {
|
|
70
136
|
type: 'closer',
|
|
71
137
|
startIndex: _startIndex,
|
|
@@ -131,53 +197,26 @@ const match = function (api) {
|
|
|
131
197
|
}
|
|
132
198
|
};
|
|
133
199
|
|
|
134
|
-
const parse = function (api) {
|
|
135
|
-
return {
|
|
136
|
-
parse: tokens => tokens.map(token => {
|
|
137
|
-
const nodePoints = api.getNodePoints();
|
|
138
|
-
let startIndex = token.startIndex + token.thickness;
|
|
139
|
-
let endIndex = token.endIndex - token.thickness;
|
|
140
|
-
let isAllSpace = true;
|
|
141
|
-
for (let i = startIndex; i < endIndex; ++i) {
|
|
142
|
-
if (character.isSpaceLike(nodePoints[i].codePoint))
|
|
143
|
-
continue;
|
|
144
|
-
isAllSpace = false;
|
|
145
|
-
break;
|
|
146
|
-
}
|
|
147
|
-
if (!isAllSpace && startIndex + 2 < endIndex) {
|
|
148
|
-
const firstCharacter = nodePoints[startIndex].codePoint;
|
|
149
|
-
const lastCharacter = nodePoints[endIndex - 1].codePoint;
|
|
150
|
-
if (character.isSpaceLike(firstCharacter) && character.isSpaceLike(lastCharacter)) {
|
|
151
|
-
startIndex += 1;
|
|
152
|
-
endIndex -= 1;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
const value = character.calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
|
|
156
|
-
const node = api.shouldReservePosition
|
|
157
|
-
? { type: ast.InlineMathType, position: api.calcPosition(token), value }
|
|
158
|
-
: { type: ast.InlineMathType, value };
|
|
159
|
-
return node;
|
|
160
|
-
}),
|
|
161
|
-
};
|
|
162
|
-
};
|
|
163
|
-
|
|
164
200
|
const uniqueName = '@yozora/tokenizer-inline-math';
|
|
201
|
+
const uniqueName_withBacktick = '@yozora/tokenizer-inline-math_with_backtick';
|
|
165
202
|
|
|
166
203
|
class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
|
|
167
|
-
|
|
204
|
+
match;
|
|
205
|
+
parse;
|
|
168
206
|
constructor(props = {}) {
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
207
|
+
const backtickRequired = props.backtickRequired ?? true;
|
|
208
|
+
const name = props.name ?? (backtickRequired ? uniqueName_withBacktick : uniqueName);
|
|
209
|
+
const priority = props.priority ??
|
|
210
|
+
(backtickRequired ? coreTokenizer.TokenizerPriority.ATOMIC : coreTokenizer.TokenizerPriority.INTERRUPTABLE_INLINE);
|
|
211
|
+
super({ name, priority });
|
|
212
|
+
this.match = backtickRequired ? matchWithBacktick : match;
|
|
213
|
+
this.parse = parse;
|
|
174
214
|
}
|
|
175
|
-
match = match;
|
|
176
|
-
parse = parse;
|
|
177
215
|
}
|
|
178
216
|
|
|
179
217
|
exports.InlineMathTokenizer = InlineMathTokenizer;
|
|
180
218
|
exports.InlineMathTokenizerName = uniqueName;
|
|
219
|
+
exports.InlineMathTokenizerName_withBacktick = uniqueName_withBacktick;
|
|
181
220
|
exports.default = InlineMathTokenizer;
|
|
182
221
|
exports.inlineMathMatch = match;
|
|
183
222
|
exports.inlineMathParse = parse;
|
package/lib/esm/index.mjs
CHANGED
|
@@ -1,16 +1,103 @@
|
|
|
1
1
|
import { InlineMathType } from '@yozora/ast';
|
|
2
2
|
import { AsciiCodePoint, isWhitespaceCharacter, isSpaceLike, calcStringFromNodePoints } from '@yozora/character';
|
|
3
|
-
import {
|
|
3
|
+
import { genFindDelimiter, eatOptionalCharacters, BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
|
|
4
4
|
|
|
5
5
|
const match = function (api) {
|
|
6
|
-
|
|
6
|
+
return {
|
|
7
|
+
findDelimiter: () => genFindDelimiter(_findDelimiter),
|
|
8
|
+
isDelimiterPair,
|
|
9
|
+
processDelimiterPair,
|
|
10
|
+
};
|
|
11
|
+
function _findDelimiter(startIndex, endIndex) {
|
|
12
|
+
const nodePoints = api.getNodePoints();
|
|
13
|
+
const blockStartIndex = api.getBlockStartIndex();
|
|
14
|
+
const blockEndIndex = api.getBlockEndIndex();
|
|
15
|
+
for (let i = startIndex; i < endIndex; ++i) {
|
|
16
|
+
const c = nodePoints[i].codePoint;
|
|
17
|
+
switch (c) {
|
|
18
|
+
case AsciiCodePoint.BACKSLASH:
|
|
19
|
+
i += 1;
|
|
20
|
+
break;
|
|
21
|
+
case AsciiCodePoint.DOLLAR_SIGN: {
|
|
22
|
+
const leftCodePoint = i === blockStartIndex ? null : nodePoints[i - 1].codePoint;
|
|
23
|
+
const _startIndex = i;
|
|
24
|
+
i = eatOptionalCharacters(nodePoints, i + 1, blockEndIndex, AsciiCodePoint.DOLLAR_SIGN);
|
|
25
|
+
const rightCodePoint = i === blockEndIndex ? null : nodePoints[i].codePoint;
|
|
26
|
+
const isPotentialOpener = (leftCodePoint === null || isWhitespaceCharacter(leftCodePoint)) &&
|
|
27
|
+
(rightCodePoint === null || !isWhitespaceCharacter(rightCodePoint));
|
|
28
|
+
const isPotentialCloser = (leftCodePoint === null || !isWhitespaceCharacter(leftCodePoint)) &&
|
|
29
|
+
(rightCodePoint === null || isWhitespaceCharacter(rightCodePoint));
|
|
30
|
+
if (!isPotentialOpener && !isPotentialCloser)
|
|
31
|
+
break;
|
|
32
|
+
const delimiterType = isPotentialOpener
|
|
33
|
+
? isPotentialCloser
|
|
34
|
+
? 'both'
|
|
35
|
+
: 'opener'
|
|
36
|
+
: 'closer';
|
|
37
|
+
const delimiter = {
|
|
38
|
+
type: delimiterType,
|
|
39
|
+
startIndex: _startIndex,
|
|
40
|
+
endIndex: i,
|
|
41
|
+
thickness: i - _startIndex,
|
|
42
|
+
};
|
|
43
|
+
return delimiter;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
function isDelimiterPair(openerDelimiter, closerDelimiter) {
|
|
50
|
+
if (openerDelimiter.thickness === closerDelimiter.thickness)
|
|
51
|
+
return { paired: true };
|
|
52
|
+
return { paired: false, opener: true, closer: true };
|
|
53
|
+
}
|
|
54
|
+
function processDelimiterPair(openerDelimiter, closerDelimiter) {
|
|
55
|
+
const token = {
|
|
56
|
+
nodeType: InlineMathType,
|
|
57
|
+
startIndex: openerDelimiter.startIndex,
|
|
58
|
+
endIndex: closerDelimiter.endIndex,
|
|
59
|
+
thickness: openerDelimiter.thickness,
|
|
60
|
+
};
|
|
61
|
+
return { tokens: [token] };
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const parse = function (api) {
|
|
66
|
+
return {
|
|
67
|
+
parse: tokens => tokens.map(token => {
|
|
68
|
+
const nodePoints = api.getNodePoints();
|
|
69
|
+
let startIndex = token.startIndex + token.thickness;
|
|
70
|
+
let endIndex = token.endIndex - token.thickness;
|
|
71
|
+
let isAllSpace = true;
|
|
72
|
+
for (let i = startIndex; i < endIndex; ++i) {
|
|
73
|
+
if (isSpaceLike(nodePoints[i].codePoint))
|
|
74
|
+
continue;
|
|
75
|
+
isAllSpace = false;
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
if (!isAllSpace && startIndex + 2 < endIndex) {
|
|
79
|
+
const firstCharacter = nodePoints[startIndex].codePoint;
|
|
80
|
+
const lastCharacter = nodePoints[endIndex - 1].codePoint;
|
|
81
|
+
if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
|
|
82
|
+
startIndex += 1;
|
|
83
|
+
endIndex -= 1;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
|
|
87
|
+
const node = api.shouldReservePosition
|
|
88
|
+
? { type: InlineMathType, position: api.calcPosition(token), value }
|
|
89
|
+
: { type: InlineMathType, value };
|
|
90
|
+
return node;
|
|
91
|
+
}),
|
|
92
|
+
};
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const matchWithBacktick = function (api) {
|
|
7
96
|
return { findDelimiter, processSingleDelimiter };
|
|
8
97
|
function* findDelimiter() {
|
|
9
98
|
const nodePoints = api.getNodePoints();
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const blockStartIndex = eatOptionalWhitespaces(nodePoints, originalBlockStartIndex, originalBlockEndIndex);
|
|
13
|
-
const blockEndIndex = eatOptionalWhitespacesReverse(nodePoints, originalBlockStartIndex, originalBlockEndIndex);
|
|
99
|
+
const blockStartIndex = api.getBlockStartIndex();
|
|
100
|
+
const blockEndIndex = api.getBlockEndIndex();
|
|
14
101
|
const potentialDelimiters = [];
|
|
15
102
|
for (let i = blockStartIndex; i < blockEndIndex; ++i) {
|
|
16
103
|
const c = nodePoints[i].codePoint;
|
|
@@ -39,29 +126,8 @@ const match = function (api) {
|
|
|
39
126
|
break;
|
|
40
127
|
}
|
|
41
128
|
const thickness = i - _startIndex;
|
|
42
|
-
if (thickness <= 1)
|
|
43
|
-
if (backtickRequired)
|
|
44
|
-
break;
|
|
45
|
-
const isPotentialOpener = _startIndex === blockStartIndex ||
|
|
46
|
-
isWhitespaceCharacter(nodePoints[_startIndex - 1].codePoint);
|
|
47
|
-
const isPotentialCloser = i === blockEndIndex ||
|
|
48
|
-
(_startIndex > blockStartIndex &&
|
|
49
|
-
!isWhitespaceCharacter(nodePoints[_startIndex - 1].codePoint));
|
|
50
|
-
if (!isPotentialOpener && !isPotentialCloser)
|
|
51
|
-
break;
|
|
52
|
-
const delimiterType = isPotentialOpener
|
|
53
|
-
? isPotentialCloser
|
|
54
|
-
? 'both'
|
|
55
|
-
: 'opener'
|
|
56
|
-
: 'closer';
|
|
57
|
-
const delimiter = {
|
|
58
|
-
type: delimiterType,
|
|
59
|
-
startIndex: _startIndex,
|
|
60
|
-
endIndex: i,
|
|
61
|
-
};
|
|
62
|
-
potentialDelimiters.push(delimiter);
|
|
129
|
+
if (thickness <= 1)
|
|
63
130
|
break;
|
|
64
|
-
}
|
|
65
131
|
const delimiter = {
|
|
66
132
|
type: 'closer',
|
|
67
133
|
startIndex: _startIndex,
|
|
@@ -127,49 +193,21 @@ const match = function (api) {
|
|
|
127
193
|
}
|
|
128
194
|
};
|
|
129
195
|
|
|
130
|
-
const parse = function (api) {
|
|
131
|
-
return {
|
|
132
|
-
parse: tokens => tokens.map(token => {
|
|
133
|
-
const nodePoints = api.getNodePoints();
|
|
134
|
-
let startIndex = token.startIndex + token.thickness;
|
|
135
|
-
let endIndex = token.endIndex - token.thickness;
|
|
136
|
-
let isAllSpace = true;
|
|
137
|
-
for (let i = startIndex; i < endIndex; ++i) {
|
|
138
|
-
if (isSpaceLike(nodePoints[i].codePoint))
|
|
139
|
-
continue;
|
|
140
|
-
isAllSpace = false;
|
|
141
|
-
break;
|
|
142
|
-
}
|
|
143
|
-
if (!isAllSpace && startIndex + 2 < endIndex) {
|
|
144
|
-
const firstCharacter = nodePoints[startIndex].codePoint;
|
|
145
|
-
const lastCharacter = nodePoints[endIndex - 1].codePoint;
|
|
146
|
-
if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
|
|
147
|
-
startIndex += 1;
|
|
148
|
-
endIndex -= 1;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
|
|
152
|
-
const node = api.shouldReservePosition
|
|
153
|
-
? { type: InlineMathType, position: api.calcPosition(token), value }
|
|
154
|
-
: { type: InlineMathType, value };
|
|
155
|
-
return node;
|
|
156
|
-
}),
|
|
157
|
-
};
|
|
158
|
-
};
|
|
159
|
-
|
|
160
196
|
const uniqueName = '@yozora/tokenizer-inline-math';
|
|
197
|
+
const uniqueName_withBacktick = '@yozora/tokenizer-inline-math_with_backtick';
|
|
161
198
|
|
|
162
199
|
class InlineMathTokenizer extends BaseInlineTokenizer {
|
|
163
|
-
|
|
200
|
+
match;
|
|
201
|
+
parse;
|
|
164
202
|
constructor(props = {}) {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
203
|
+
const backtickRequired = props.backtickRequired ?? true;
|
|
204
|
+
const name = props.name ?? (backtickRequired ? uniqueName_withBacktick : uniqueName);
|
|
205
|
+
const priority = props.priority ??
|
|
206
|
+
(backtickRequired ? TokenizerPriority.ATOMIC : TokenizerPriority.INTERRUPTABLE_INLINE);
|
|
207
|
+
super({ name, priority });
|
|
208
|
+
this.match = backtickRequired ? matchWithBacktick : match;
|
|
209
|
+
this.parse = parse;
|
|
170
210
|
}
|
|
171
|
-
match = match;
|
|
172
|
-
parse = parse;
|
|
173
211
|
}
|
|
174
212
|
|
|
175
|
-
export { InlineMathTokenizer, uniqueName as InlineMathTokenizerName, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
|
|
213
|
+
export { InlineMathTokenizer, uniqueName as InlineMathTokenizerName, uniqueName_withBacktick as InlineMathTokenizerName_withBacktick, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
|
package/lib/types/index.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { InlineMathType, InlineMath } from '@yozora/ast';
|
|
|
4
4
|
type T = InlineMathType;
|
|
5
5
|
type INode = InlineMath;
|
|
6
6
|
declare const uniqueName = "@yozora/tokenizer-inline-math";
|
|
7
|
+
declare const uniqueName_withBacktick = "@yozora/tokenizer-inline-math_with_backtick";
|
|
7
8
|
interface IToken extends IPartialInlineToken<T> {
|
|
8
9
|
/**
|
|
9
10
|
* Thickness of the InlineMathDelimiter
|
|
@@ -14,17 +15,12 @@ interface IToken extends IPartialInlineToken<T> {
|
|
|
14
15
|
* IDelimiter of InlineMathToken.
|
|
15
16
|
*/
|
|
16
17
|
interface IDelimiter extends ITokenDelimiter {
|
|
17
|
-
type: 'full';
|
|
18
18
|
/**
|
|
19
19
|
* Thickness of the InlineMathDelimiter
|
|
20
20
|
*/
|
|
21
21
|
thickness: number;
|
|
22
22
|
}
|
|
23
23
|
interface IThis extends ITokenizer {
|
|
24
|
-
/**
|
|
25
|
-
* Whether if the backtick mark wrapping necessary.
|
|
26
|
-
*/
|
|
27
|
-
readonly backtickRequired: boolean;
|
|
28
24
|
}
|
|
29
25
|
interface ITokenizerProps extends Partial<IBaseInlineTokenizerProps> {
|
|
30
26
|
/**
|
|
@@ -42,10 +38,9 @@ declare const parse: IParseInlineHookCreator<T, IToken, INode, IThis>;
|
|
|
42
38
|
* Lexical Analyzer for inlineMath.
|
|
43
39
|
*/
|
|
44
40
|
declare class InlineMathTokenizer extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis> implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis> {
|
|
45
|
-
readonly backtickRequired: boolean;
|
|
46
|
-
constructor(props?: ITokenizerProps);
|
|
47
41
|
readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;
|
|
48
42
|
readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis>;
|
|
43
|
+
constructor(props?: ITokenizerProps);
|
|
49
44
|
}
|
|
50
45
|
|
|
51
|
-
export { type IThis as IInlineMathHookContext, type IToken as IInlineMathToken, type ITokenizerProps as IInlineMathTokenizerProps, InlineMathTokenizer, uniqueName as InlineMathTokenizerName, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
|
|
46
|
+
export { type IThis as IInlineMathHookContext, type IToken as IInlineMathToken, type ITokenizerProps as IInlineMathTokenizerProps, InlineMathTokenizer, uniqueName as InlineMathTokenizerName, uniqueName_withBacktick as InlineMathTokenizerName_withBacktick, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yozora/tokenizer-inline-math",
|
|
3
|
-
"version": "2.3.
|
|
3
|
+
"version": "2.3.8",
|
|
4
4
|
"description": "Tokenizer for processing inline math (formulas)",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "guanghechen",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"url": "https://github.com/yozorajs/yozora.git",
|
|
12
12
|
"directory": "tokenizers/inline-math"
|
|
13
13
|
},
|
|
14
|
-
"homepage": "https://github.com/yozorajs/yozora/tree/v2.3.
|
|
14
|
+
"homepage": "https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/inline-math",
|
|
15
15
|
"keywords": [
|
|
16
16
|
"yozora",
|
|
17
17
|
"markdown",
|
|
@@ -44,9 +44,9 @@
|
|
|
44
44
|
"README.md"
|
|
45
45
|
],
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@yozora/ast": "^2.3.
|
|
48
|
-
"@yozora/character": "^2.3.
|
|
49
|
-
"@yozora/core-tokenizer": "^2.3.
|
|
47
|
+
"@yozora/ast": "^2.3.8",
|
|
48
|
+
"@yozora/character": "^2.3.8",
|
|
49
|
+
"@yozora/core-tokenizer": "^2.3.8"
|
|
50
50
|
},
|
|
51
|
-
"gitHead": "
|
|
51
|
+
"gitHead": "c4ddb8bfb317b74d481be80ce545d7dbe3add3f2"
|
|
52
52
|
}
|