@yozora/tokenizer-inline-math 2.3.5 → 2.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +45 -44
  3. package/lib/cjs/index.cjs +100 -49
  4. package/lib/esm/index.mjs +102 -52
  5. package/lib/types/index.d.ts +3 -8
  6. package/package.json +6 -6
package/CHANGELOG.md CHANGED
@@ -3,6 +3,27 @@
3
3
  All notable changes to this project will be documented in this file. See
4
4
  [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
5
5
 
6
+ ## <small>2.3.7 (2024-11-06)</small>
7
+
8
+ - :bug: fix(inline-math): the inline syntax wihtout backticks could be interruptted by
9
+ emphasis/strong ([c347f97](https://github.com/yozorajs/yozora/commit/c347f97))
10
+ - :white_check_mark: test: fix tests ([dd9c762](https://github.com/yozorajs/yozora/commit/dd9c762))
11
+
12
+ # Change Log
13
+
14
+ All notable changes to this project will be documented in this file. See
15
+ [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
16
+
17
+ ## <small>2.3.6 (2024-10-23)</small>
18
+
19
+ - :bug: fix(tokenizer-inline-math): the dollar syntax should meet the opener and closer conditions
20
+ ([aef429b](https://github.com/yozorajs/yozora/commit/aef429b))
21
+
22
+ # Change Log
23
+
24
+ All notable changes to this project will be documented in this file. See
25
+ [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
26
+
6
27
  ## <small>2.3.5 (2024-10-06)</small>
7
28
 
8
29
  - :bookmark: release: publish v2.3.3 ([be6d159](https://github.com/yozorajs/yozora/commit/be6d159))
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  <header>
4
4
  <h1 align="center">
5
- <a href="https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/inline-math#readme">@yozora/tokenizer-inline-math</a>
5
+ <a href="https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/inline-math#readme">@yozora/tokenizer-inline-math</a>
6
6
  </h1>
7
7
  <div align="center">
8
8
  <a href="https://www.npmjs.com/package/@yozora/tokenizer-inline-math">
@@ -89,7 +89,8 @@ import InlineMathTokenizer from '@yozora/tokenizer-inline-math'
89
89
  const parser = new DefaultParser()
90
90
  .useFallbackTokenizer(new ParagraphTokenizer())
91
91
  .useFallbackTokenizer(new TextTokenizer())
92
- .useTokenizer(new InlineMathTokenizer())
92
+ .useTokenizer(new InlineMathTokenizer({ backtickRequired: true }))
93
+ .useTokenizer(new InlineMathTokenizer({ backtickRequired: false }))
93
94
 
94
95
  // parse source markdown content
95
96
  parser.parse("`$x^2 + y^2 = z^2, x < 0$`")
@@ -113,7 +114,7 @@ import GfmParser from '@yozora/parser-gfm'
113
114
  import InlineMathTokenizer from '@yozora/tokenizer-inline-math'
114
115
 
115
116
  const parser = new GfmParser()
116
- parser.useTokenizer(new InlineMathTokenizer())
117
+ parser.useTokenizer(new InlineMathTokenizer({ backtickRequired: true }))
117
118
 
118
119
  // parse source markdown content
119
120
  parser.parse("`$x^2 + y^2 = z^2, x < 0$`")
@@ -180,80 +181,80 @@ parser.parse("`$x^2 + y^2 = z^2, x < 0$`")
180
181
 
181
182
  [live-examples]: https://yozora.guanghechen.com/docs/package/tokenizer-inline-math#live-examples
182
183
  [docpage]: https://yozora.guanghechen.com/docs/package/tokenizer-inline-math
183
- [homepage]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/inline-math#readme
184
+ [homepage]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/inline-math#readme
184
185
  [gfm-spec]: https://github.github.com/gfm
185
186
  [mdast-homepage]: https://github.com/syntax-tree/mdast
186
- [@yozora/ast]: https://github.com/yozorajs/yozora/tree/v2.3.5/packages/ast#readme
187
- [@yozora/ast-util]: https://github.com/yozorajs/yozora/tree/v2.3.5/packages/ast-util#readme
188
- [@yozora/character]: https://github.com/yozorajs/yozora/tree/v2.3.5/packages/character#readme
187
+ [@yozora/ast]: https://github.com/yozorajs/yozora/tree/v2.3.7/packages/ast#readme
188
+ [@yozora/ast-util]: https://github.com/yozorajs/yozora/tree/v2.3.7/packages/ast-util#readme
189
+ [@yozora/character]: https://github.com/yozorajs/yozora/tree/v2.3.7/packages/character#readme
189
190
  [@yozora/eslint-config]:
190
191
  https://github.com/yozorajs/yozora/tree/release-2.x.x/packages/eslint-config#readme
191
- [@yozora/core-parser]: https://github.com/yozorajs/yozora/tree/v2.3.5/packages/core-parser#readme
192
+ [@yozora/core-parser]: https://github.com/yozorajs/yozora/tree/v2.3.7/packages/core-parser#readme
192
193
  [@yozora/core-tokenizer]:
193
- https://github.com/yozorajs/yozora/tree/v2.3.5/packages/core-tokenizer#readme
194
- [@yozora/invariant]: https://github.com/yozorajs/yozora/tree/v2.3.5/packages/invariant#readme
194
+ https://github.com/yozorajs/yozora/tree/v2.3.7/packages/core-tokenizer#readme
195
+ [@yozora/invariant]: https://github.com/yozorajs/yozora/tree/v2.3.7/packages/invariant#readme
195
196
  [@yozora/jest-for-tokenizer]:
196
197
  https://github.com/yozorajs/yozora/tree/release-2.x.x/packages/jest-for-tokenizer#readme
197
- [@yozora/parser]: https://github.com/yozorajs/yozora/tree/v2.3.5/packages/parser#readme
198
- [@yozora/parser-gfm]: https://github.com/yozorajs/yozora/tree/v2.3.5/packages/parser-gfm#readme
198
+ [@yozora/parser]: https://github.com/yozorajs/yozora/tree/v2.3.7/packages/parser#readme
199
+ [@yozora/parser-gfm]: https://github.com/yozorajs/yozora/tree/v2.3.7/packages/parser-gfm#readme
199
200
  [@yozora/parser-gfm-ex]:
200
- https://github.com/yozorajs/yozora/tree/v2.3.5/packages/parser-gfm-ex#readme
201
+ https://github.com/yozorajs/yozora/tree/v2.3.7/packages/parser-gfm-ex#readme
201
202
  [@yozora/template-tokenizer]:
202
203
  https://github.com/yozorajs/yozora/tree/release-2.x.x/packages/template-tokenizer#readme
203
204
  [@yozora/tokenizer-admonition]:
204
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/admonition#readme
205
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/admonition#readme
205
206
  [@yozora/tokenizer-autolink]:
206
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/autolink#readme
207
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/autolink#readme
207
208
  [@yozora/tokenizer-autolink-extension]:
208
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/autolink-extension#readme
209
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/autolink-extension#readme
209
210
  [@yozora/tokenizer-blockquote]:
210
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/blockquote#readme
211
- [@yozora/tokenizer-break]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/break#readme
211
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/blockquote#readme
212
+ [@yozora/tokenizer-break]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/break#readme
212
213
  [@yozora/tokenizer-definition]:
213
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/definition#readme
214
- [@yozora/tokenizer-delete]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/delete#readme
214
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/definition#readme
215
+ [@yozora/tokenizer-delete]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/delete#readme
215
216
  [@yozora/tokenizer-ecma-import]:
216
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/ecma-import#readme
217
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/ecma-import#readme
217
218
  [@yozora/tokenizer-emphasis]:
218
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/emphasis#readme
219
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/emphasis#readme
219
220
  [@yozora/tokenizer-fenced-block]:
220
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/fenced-block#readme
221
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/fenced-block#readme
221
222
  [@yozora/tokenizer-fenced-code]:
222
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/fenced-code#readme
223
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/fenced-code#readme
223
224
  [@yozora/tokenizer-footnote]:
224
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/footnote#readme
225
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/footnote#readme
225
226
  [@yozora/tokenizer-footnote-definition]:
226
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/footnote-definition#readme
227
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/footnote-definition#readme
227
228
  [@yozora/tokenizer-footnote-reference]:
228
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/footnote-reference#readme
229
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/footnote-reference#readme
229
230
  [@yozora/tokenizer-heading]:
230
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/heading#readme
231
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/heading#readme
231
232
  [@yozora/tokenizer-html-block]:
232
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/html-block#readme
233
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/html-block#readme
233
234
  [@yozora/tokenizer-html-inline]:
234
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/html-inline#readme
235
- [@yozora/tokenizer-image]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/image#readme
235
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/html-inline#readme
236
+ [@yozora/tokenizer-image]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/image#readme
236
237
  [@yozora/tokenizer-image-reference]:
237
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/image-reference#readme
238
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/image-reference#readme
238
239
  [@yozora/tokenizer-indented-code]:
239
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/indented-code#readme
240
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/indented-code#readme
240
241
  [@yozora/tokenizer-inline-code]:
241
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/inline-code#readme
242
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/inline-code#readme
242
243
  [@yozora/tokenizer-inline-math]:
243
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/inline-math#readme
244
- [@yozora/tokenizer-link]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/link#readme
244
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/inline-math#readme
245
+ [@yozora/tokenizer-link]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/link#readme
245
246
  [@yozora/tokenizer-link-reference]:
246
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/link-reference#readme
247
- [@yozora/tokenizer-list]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/list#readme
248
- [@yozora/tokenizer-math]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/math#readme
247
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/link-reference#readme
248
+ [@yozora/tokenizer-list]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/list#readme
249
+ [@yozora/tokenizer-math]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/math#readme
249
250
  [@yozora/tokenizer-paragraph]:
250
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/paragraph#readme
251
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/paragraph#readme
251
252
  [@yozora/tokenizer-setext-heading]:
252
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/setext-heading#readme
253
- [@yozora/tokenizer-table]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/table#readme
254
- [@yozora/tokenizer-text]: https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/text#readme
253
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/setext-heading#readme
254
+ [@yozora/tokenizer-table]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/table#readme
255
+ [@yozora/tokenizer-text]: https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/text#readme
255
256
  [@yozora/tokenizer-thematic-break]:
256
- https://github.com/yozorajs/yozora/tree/v2.3.5/tokenizers/thematic-break#readme
257
+ https://github.com/yozorajs/yozora/tree/v2.3.7/tokenizers/thematic-break#readme
257
258
  [@yozora/react-admonition]:
258
259
  https://github.com/yozorajs/yozora-react/tree/main/packages/admonition#readme
259
260
  [@yozora/react-blockquote]:
package/lib/cjs/index.cjs CHANGED
@@ -7,7 +7,94 @@ var character = require('@yozora/character');
7
7
  var coreTokenizer = require('@yozora/core-tokenizer');
8
8
 
9
9
  const match = function (api) {
10
- const { backtickRequired } = this;
10
+ return {
11
+ findDelimiter: () => coreTokenizer.genFindDelimiter(_findDelimiter),
12
+ isDelimiterPair,
13
+ processDelimiterPair,
14
+ };
15
+ function _findDelimiter(startIndex, endIndex) {
16
+ const nodePoints = api.getNodePoints();
17
+ const blockStartIndex = api.getBlockStartIndex();
18
+ const blockEndIndex = api.getBlockEndIndex();
19
+ for (let i = startIndex; i < endIndex; ++i) {
20
+ const c = nodePoints[i].codePoint;
21
+ switch (c) {
22
+ case character.AsciiCodePoint.BACKSLASH:
23
+ i += 1;
24
+ break;
25
+ case character.AsciiCodePoint.DOLLAR_SIGN: {
26
+ const leftCodePoint = i === blockStartIndex ? null : nodePoints[i - 1].codePoint;
27
+ const rightCodePoint = i + 1 === blockEndIndex ? null : nodePoints[i + 1].codePoint;
28
+ const isPotentialOpener = (leftCodePoint === null || character.isWhitespaceCharacter(leftCodePoint)) &&
29
+ (rightCodePoint === null ||
30
+ (rightCodePoint !== character.AsciiCodePoint.DOLLAR_SIGN &&
31
+ !character.isWhitespaceCharacter(rightCodePoint)));
32
+ const isPotentialCloser = (leftCodePoint === null || !character.isWhitespaceCharacter(leftCodePoint)) &&
33
+ (rightCodePoint === null || character.isWhitespaceCharacter(rightCodePoint));
34
+ if (!isPotentialOpener && !isPotentialCloser)
35
+ break;
36
+ const delimiterType = isPotentialOpener
37
+ ? isPotentialCloser
38
+ ? 'both'
39
+ : 'opener'
40
+ : 'closer';
41
+ const delimiter = {
42
+ type: delimiterType,
43
+ startIndex: i,
44
+ endIndex: i + 1,
45
+ thickness: 1,
46
+ };
47
+ return delimiter;
48
+ }
49
+ }
50
+ }
51
+ return null;
52
+ }
53
+ function isDelimiterPair() {
54
+ return { paired: true };
55
+ }
56
+ function processDelimiterPair(openerDelimiter, closerDelimiter) {
57
+ const token = {
58
+ nodeType: ast.InlineMathType,
59
+ startIndex: openerDelimiter.startIndex,
60
+ endIndex: closerDelimiter.endIndex,
61
+ thickness: 1,
62
+ };
63
+ return { tokens: [token] };
64
+ }
65
+ };
66
+
67
+ const parse = function (api) {
68
+ return {
69
+ parse: tokens => tokens.map(token => {
70
+ const nodePoints = api.getNodePoints();
71
+ let startIndex = token.startIndex + token.thickness;
72
+ let endIndex = token.endIndex - token.thickness;
73
+ let isAllSpace = true;
74
+ for (let i = startIndex; i < endIndex; ++i) {
75
+ if (character.isSpaceLike(nodePoints[i].codePoint))
76
+ continue;
77
+ isAllSpace = false;
78
+ break;
79
+ }
80
+ if (!isAllSpace && startIndex + 2 < endIndex) {
81
+ const firstCharacter = nodePoints[startIndex].codePoint;
82
+ const lastCharacter = nodePoints[endIndex - 1].codePoint;
83
+ if (character.isSpaceLike(firstCharacter) && character.isSpaceLike(lastCharacter)) {
84
+ startIndex += 1;
85
+ endIndex -= 1;
86
+ }
87
+ }
88
+ const value = character.calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
89
+ const node = api.shouldReservePosition
90
+ ? { type: ast.InlineMathType, position: api.calcPosition(token), value }
91
+ : { type: ast.InlineMathType, value };
92
+ return node;
93
+ }),
94
+ };
95
+ };
96
+
97
+ const matchWithBacktick = function (api) {
11
98
  return { findDelimiter, processSingleDelimiter };
12
99
  function* findDelimiter() {
13
100
  const nodePoints = api.getNodePoints();
@@ -41,17 +128,8 @@ const match = function (api) {
41
128
  break;
42
129
  }
43
130
  const thickness = i - _startIndex;
44
- if (thickness <= 1) {
45
- if (backtickRequired)
46
- break;
47
- const delimiter = {
48
- type: 'both',
49
- startIndex: _startIndex,
50
- endIndex: i,
51
- };
52
- potentialDelimiters.push(delimiter);
131
+ if (thickness <= 1)
53
132
  break;
54
- }
55
133
  const delimiter = {
56
134
  type: 'closer',
57
135
  startIndex: _startIndex,
@@ -117,53 +195,26 @@ const match = function (api) {
117
195
  }
118
196
  };
119
197
 
120
- const parse = function (api) {
121
- return {
122
- parse: tokens => tokens.map(token => {
123
- const nodePoints = api.getNodePoints();
124
- let startIndex = token.startIndex + token.thickness;
125
- let endIndex = token.endIndex - token.thickness;
126
- let isAllSpace = true;
127
- for (let i = startIndex; i < endIndex; ++i) {
128
- if (character.isSpaceLike(nodePoints[i].codePoint))
129
- continue;
130
- isAllSpace = false;
131
- break;
132
- }
133
- if (!isAllSpace && startIndex + 2 < endIndex) {
134
- const firstCharacter = nodePoints[startIndex].codePoint;
135
- const lastCharacter = nodePoints[endIndex - 1].codePoint;
136
- if (character.isSpaceLike(firstCharacter) && character.isSpaceLike(lastCharacter)) {
137
- startIndex += 1;
138
- endIndex -= 1;
139
- }
140
- }
141
- const value = character.calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
142
- const node = api.shouldReservePosition
143
- ? { type: ast.InlineMathType, position: api.calcPosition(token), value }
144
- : { type: ast.InlineMathType, value };
145
- return node;
146
- }),
147
- };
148
- };
149
-
150
198
  const uniqueName = '@yozora/tokenizer-inline-math';
199
+ const uniqueName_withBacktick = '@yozora/tokenizer-inline-math_with_backtick';
151
200
 
152
201
  class InlineMathTokenizer extends coreTokenizer.BaseInlineTokenizer {
153
- backtickRequired;
202
+ match;
203
+ parse;
154
204
  constructor(props = {}) {
155
- super({
156
- name: props.name ?? uniqueName,
157
- priority: props.priority ?? coreTokenizer.TokenizerPriority.ATOMIC,
158
- });
159
- this.backtickRequired = props.backtickRequired ?? true;
205
+ const backtickRequired = props.backtickRequired ?? true;
206
+ const name = props.name ?? (backtickRequired ? uniqueName_withBacktick : uniqueName);
207
+ const priority = props.priority ??
208
+ (backtickRequired ? coreTokenizer.TokenizerPriority.ATOMIC : coreTokenizer.TokenizerPriority.INTERRUPTABLE_INLINE);
209
+ super({ name, priority });
210
+ this.match = backtickRequired ? matchWithBacktick : match;
211
+ this.parse = parse;
160
212
  }
161
- match = match;
162
- parse = parse;
163
213
  }
164
214
 
165
215
  exports.InlineMathTokenizer = InlineMathTokenizer;
166
216
  exports.InlineMathTokenizerName = uniqueName;
217
+ exports.InlineMathTokenizerName_withBacktick = uniqueName_withBacktick;
167
218
  exports.default = InlineMathTokenizer;
168
219
  exports.inlineMathMatch = match;
169
220
  exports.inlineMathParse = parse;
package/lib/esm/index.mjs CHANGED
@@ -1,9 +1,96 @@
1
1
  import { InlineMathType } from '@yozora/ast';
2
- import { AsciiCodePoint, isSpaceLike, calcStringFromNodePoints } from '@yozora/character';
3
- import { eatOptionalCharacters, BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
2
+ import { AsciiCodePoint, isWhitespaceCharacter, isSpaceLike, calcStringFromNodePoints } from '@yozora/character';
3
+ import { genFindDelimiter, eatOptionalCharacters, BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
4
4
 
5
5
  const match = function (api) {
6
- const { backtickRequired } = this;
6
+ return {
7
+ findDelimiter: () => genFindDelimiter(_findDelimiter),
8
+ isDelimiterPair,
9
+ processDelimiterPair,
10
+ };
11
+ function _findDelimiter(startIndex, endIndex) {
12
+ const nodePoints = api.getNodePoints();
13
+ const blockStartIndex = api.getBlockStartIndex();
14
+ const blockEndIndex = api.getBlockEndIndex();
15
+ for (let i = startIndex; i < endIndex; ++i) {
16
+ const c = nodePoints[i].codePoint;
17
+ switch (c) {
18
+ case AsciiCodePoint.BACKSLASH:
19
+ i += 1;
20
+ break;
21
+ case AsciiCodePoint.DOLLAR_SIGN: {
22
+ const leftCodePoint = i === blockStartIndex ? null : nodePoints[i - 1].codePoint;
23
+ const rightCodePoint = i + 1 === blockEndIndex ? null : nodePoints[i + 1].codePoint;
24
+ const isPotentialOpener = (leftCodePoint === null || isWhitespaceCharacter(leftCodePoint)) &&
25
+ (rightCodePoint === null ||
26
+ (rightCodePoint !== AsciiCodePoint.DOLLAR_SIGN &&
27
+ !isWhitespaceCharacter(rightCodePoint)));
28
+ const isPotentialCloser = (leftCodePoint === null || !isWhitespaceCharacter(leftCodePoint)) &&
29
+ (rightCodePoint === null || isWhitespaceCharacter(rightCodePoint));
30
+ if (!isPotentialOpener && !isPotentialCloser)
31
+ break;
32
+ const delimiterType = isPotentialOpener
33
+ ? isPotentialCloser
34
+ ? 'both'
35
+ : 'opener'
36
+ : 'closer';
37
+ const delimiter = {
38
+ type: delimiterType,
39
+ startIndex: i,
40
+ endIndex: i + 1,
41
+ thickness: 1,
42
+ };
43
+ return delimiter;
44
+ }
45
+ }
46
+ }
47
+ return null;
48
+ }
49
+ function isDelimiterPair() {
50
+ return { paired: true };
51
+ }
52
+ function processDelimiterPair(openerDelimiter, closerDelimiter) {
53
+ const token = {
54
+ nodeType: InlineMathType,
55
+ startIndex: openerDelimiter.startIndex,
56
+ endIndex: closerDelimiter.endIndex,
57
+ thickness: 1,
58
+ };
59
+ return { tokens: [token] };
60
+ }
61
+ };
62
+
63
+ const parse = function (api) {
64
+ return {
65
+ parse: tokens => tokens.map(token => {
66
+ const nodePoints = api.getNodePoints();
67
+ let startIndex = token.startIndex + token.thickness;
68
+ let endIndex = token.endIndex - token.thickness;
69
+ let isAllSpace = true;
70
+ for (let i = startIndex; i < endIndex; ++i) {
71
+ if (isSpaceLike(nodePoints[i].codePoint))
72
+ continue;
73
+ isAllSpace = false;
74
+ break;
75
+ }
76
+ if (!isAllSpace && startIndex + 2 < endIndex) {
77
+ const firstCharacter = nodePoints[startIndex].codePoint;
78
+ const lastCharacter = nodePoints[endIndex - 1].codePoint;
79
+ if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
80
+ startIndex += 1;
81
+ endIndex -= 1;
82
+ }
83
+ }
84
+ const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
85
+ const node = api.shouldReservePosition
86
+ ? { type: InlineMathType, position: api.calcPosition(token), value }
87
+ : { type: InlineMathType, value };
88
+ return node;
89
+ }),
90
+ };
91
+ };
92
+
93
+ const matchWithBacktick = function (api) {
7
94
  return { findDelimiter, processSingleDelimiter };
8
95
  function* findDelimiter() {
9
96
  const nodePoints = api.getNodePoints();
@@ -37,17 +124,8 @@ const match = function (api) {
37
124
  break;
38
125
  }
39
126
  const thickness = i - _startIndex;
40
- if (thickness <= 1) {
41
- if (backtickRequired)
42
- break;
43
- const delimiter = {
44
- type: 'both',
45
- startIndex: _startIndex,
46
- endIndex: i,
47
- };
48
- potentialDelimiters.push(delimiter);
127
+ if (thickness <= 1)
49
128
  break;
50
- }
51
129
  const delimiter = {
52
130
  type: 'closer',
53
131
  startIndex: _startIndex,
@@ -113,49 +191,21 @@ const match = function (api) {
113
191
  }
114
192
  };
115
193
 
116
- const parse = function (api) {
117
- return {
118
- parse: tokens => tokens.map(token => {
119
- const nodePoints = api.getNodePoints();
120
- let startIndex = token.startIndex + token.thickness;
121
- let endIndex = token.endIndex - token.thickness;
122
- let isAllSpace = true;
123
- for (let i = startIndex; i < endIndex; ++i) {
124
- if (isSpaceLike(nodePoints[i].codePoint))
125
- continue;
126
- isAllSpace = false;
127
- break;
128
- }
129
- if (!isAllSpace && startIndex + 2 < endIndex) {
130
- const firstCharacter = nodePoints[startIndex].codePoint;
131
- const lastCharacter = nodePoints[endIndex - 1].codePoint;
132
- if (isSpaceLike(firstCharacter) && isSpaceLike(lastCharacter)) {
133
- startIndex += 1;
134
- endIndex -= 1;
135
- }
136
- }
137
- const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex).replace(/\n/g, ' ');
138
- const node = api.shouldReservePosition
139
- ? { type: InlineMathType, position: api.calcPosition(token), value }
140
- : { type: InlineMathType, value };
141
- return node;
142
- }),
143
- };
144
- };
145
-
146
194
  const uniqueName = '@yozora/tokenizer-inline-math';
195
+ const uniqueName_withBacktick = '@yozora/tokenizer-inline-math_with_backtick';
147
196
 
148
197
  class InlineMathTokenizer extends BaseInlineTokenizer {
149
- backtickRequired;
198
+ match;
199
+ parse;
150
200
  constructor(props = {}) {
151
- super({
152
- name: props.name ?? uniqueName,
153
- priority: props.priority ?? TokenizerPriority.ATOMIC,
154
- });
155
- this.backtickRequired = props.backtickRequired ?? true;
201
+ const backtickRequired = props.backtickRequired ?? true;
202
+ const name = props.name ?? (backtickRequired ? uniqueName_withBacktick : uniqueName);
203
+ const priority = props.priority ??
204
+ (backtickRequired ? TokenizerPriority.ATOMIC : TokenizerPriority.INTERRUPTABLE_INLINE);
205
+ super({ name, priority });
206
+ this.match = backtickRequired ? matchWithBacktick : match;
207
+ this.parse = parse;
156
208
  }
157
- match = match;
158
- parse = parse;
159
209
  }
160
210
 
161
- export { InlineMathTokenizer, uniqueName as InlineMathTokenizerName, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
211
+ export { InlineMathTokenizer, uniqueName as InlineMathTokenizerName, uniqueName_withBacktick as InlineMathTokenizerName_withBacktick, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
package/lib/types/index.d.ts CHANGED
@@ -4,6 +4,7 @@ import { InlineMathType, InlineMath } from '@yozora/ast';
4
4
  type T = InlineMathType;
5
5
  type INode = InlineMath;
6
6
  declare const uniqueName = "@yozora/tokenizer-inline-math";
7
+ declare const uniqueName_withBacktick = "@yozora/tokenizer-inline-math_with_backtick";
7
8
  interface IToken extends IPartialInlineToken<T> {
8
9
  /**
9
10
  * Thickness of the InlineMathDelimiter
@@ -14,17 +15,12 @@ interface IToken extends IPartialInlineToken<T> {
14
15
  * IDelimiter of InlineMathToken.
15
16
  */
16
17
  interface IDelimiter extends ITokenDelimiter {
17
- type: 'full';
18
18
  /**
19
19
  * Thickness of the InlineMathDelimiter
20
20
  */
21
21
  thickness: number;
22
22
  }
23
23
  interface IThis extends ITokenizer {
24
- /**
25
- * Whether if the backtick mark wrapping necessary.
26
- */
27
- readonly backtickRequired: boolean;
28
24
  }
29
25
  interface ITokenizerProps extends Partial<IBaseInlineTokenizerProps> {
30
26
  /**
@@ -42,10 +38,9 @@ declare const parse: IParseInlineHookCreator<T, IToken, INode, IThis>;
42
38
  * Lexical Analyzer for inlineMath.
43
39
  */
44
40
  declare class InlineMathTokenizer extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis> implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis> {
45
- readonly backtickRequired: boolean;
46
- constructor(props?: ITokenizerProps);
47
41
  readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis>;
48
42
  readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis>;
43
+ constructor(props?: ITokenizerProps);
49
44
  }
50
45
 
51
- export { type IThis as IInlineMathHookContext, type IToken as IInlineMathToken, type ITokenizerProps as IInlineMathTokenizerProps, InlineMathTokenizer, uniqueName as InlineMathTokenizerName, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
46
+ export { type IThis as IInlineMathHookContext, type IToken as IInlineMathToken, type ITokenizerProps as IInlineMathTokenizerProps, InlineMathTokenizer, uniqueName as InlineMathTokenizerName, uniqueName_withBacktick as InlineMathTokenizerName_withBacktick, InlineMathTokenizer as default, match as inlineMathMatch, parse as inlineMathParse };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yozora/tokenizer-inline-math",
3
- "version": "2.3.5",
3
+ "version": "2.3.7",
4
4
  "description": "Tokenizer for processing inline math (formulas)",
5
5
  "author": {
6
6
  "name": "guanghechen",
@@ -11,7 +11,7 @@
11
11
  "url": "https://github.com/yozorajs/yozora.git",
12
12
  "directory": "tokenizers/inline-math"
13
13
  },
14
- "homepage": "https://github.com/yozorajs/yozora/tree/v2.3.4/tokenizers/inline-math",
14
+ "homepage": "https://github.com/yozorajs/yozora/tree/v2.3.6/tokenizers/inline-math",
15
15
  "keywords": [
16
16
  "yozora",
17
17
  "markdown",
@@ -44,9 +44,9 @@
44
44
  "README.md"
45
45
  ],
46
46
  "dependencies": {
47
- "@yozora/ast": "^2.3.5",
48
- "@yozora/character": "^2.3.5",
49
- "@yozora/core-tokenizer": "^2.3.5"
47
+ "@yozora/ast": "^2.3.7",
48
+ "@yozora/character": "^2.3.7",
49
+ "@yozora/core-tokenizer": "^2.3.7"
50
50
  },
51
- "gitHead": "251eac044927616fe211c86c98e44a39003bd664"
51
+ "gitHead": "ba794102161e339545274a42bb7b27e3021a5591"
52
52
  }