@yozora/tokenizer-table 1.3.0 → 2.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/index.js +87 -155
- package/lib/esm/index.js +86 -156
- package/lib/types/index.d.ts +4 -4
- package/lib/types/match.d.ts +26 -0
- package/lib/types/parse.d.ts +3 -0
- package/lib/types/tokenizer.d.ts +6 -51
- package/lib/types/types.d.ts +13 -12
- package/package.json +5 -5
package/lib/cjs/index.js
CHANGED
|
@@ -6,21 +6,18 @@ var ast = require('@yozora/ast');
|
|
|
6
6
|
var character = require('@yozora/character');
|
|
7
7
|
var coreTokenizer = require('@yozora/core-tokenizer');
|
|
8
8
|
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
this.isContainingBlock = false;
|
|
19
|
-
}
|
|
20
|
-
eatOpener() {
|
|
9
|
+
const match = function (api) {
|
|
10
|
+
const { name: _tokenizer } = this;
|
|
11
|
+
return {
|
|
12
|
+
isContainingBlock: false,
|
|
13
|
+
eatOpener,
|
|
14
|
+
eatAndInterruptPreviousSibling,
|
|
15
|
+
eatLazyContinuationText,
|
|
16
|
+
};
|
|
17
|
+
function eatOpener() {
|
|
21
18
|
return null;
|
|
22
19
|
}
|
|
23
|
-
eatAndInterruptPreviousSibling(line, prevSiblingToken
|
|
20
|
+
function eatAndInterruptPreviousSibling(line, prevSiblingToken) {
|
|
24
21
|
if (line.countOfPrecedeSpaces >= 4)
|
|
25
22
|
return null;
|
|
26
23
|
const { nodePoints, endIndex, firstNonWhitespaceIndex } = line;
|
|
@@ -28,9 +25,7 @@ class TableTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
28
25
|
return null;
|
|
29
26
|
const columns = [];
|
|
30
27
|
let c = nodePoints[firstNonWhitespaceIndex].codePoint;
|
|
31
|
-
let cIndex = c === character.AsciiCodePoint.VERTICAL_SLASH
|
|
32
|
-
? firstNonWhitespaceIndex + 1
|
|
33
|
-
: firstNonWhitespaceIndex;
|
|
28
|
+
let cIndex = c === character.AsciiCodePoint.VERTICAL_SLASH ? firstNonWhitespaceIndex + 1 : firstNonWhitespaceIndex;
|
|
34
29
|
for (; cIndex < endIndex;) {
|
|
35
30
|
for (; cIndex < endIndex; ++cIndex) {
|
|
36
31
|
c = nodePoints[cIndex].codePoint;
|
|
@@ -103,7 +98,7 @@ class TableTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
103
98
|
cellCount += 1;
|
|
104
99
|
if (cellCount !== columns.length)
|
|
105
100
|
return null;
|
|
106
|
-
const row =
|
|
101
|
+
const row = calcTableRow(previousLine, columns);
|
|
107
102
|
const nextIndex = endIndex;
|
|
108
103
|
const token = {
|
|
109
104
|
nodeType: ast.TableType,
|
|
@@ -120,149 +115,18 @@ class TableTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
120
115
|
remainingSibling: api.rollbackPhrasingLines(lines.slice(0, lines.length - 1), prevSiblingToken),
|
|
121
116
|
};
|
|
122
117
|
}
|
|
123
|
-
eatLazyContinuationText(line, token
|
|
118
|
+
function eatLazyContinuationText(line, token) {
|
|
124
119
|
if (line.firstNonWhitespaceIndex >= line.endIndex) {
|
|
125
120
|
return { status: 'notMatched' };
|
|
126
121
|
}
|
|
127
122
|
const tableToken = token;
|
|
128
|
-
const row =
|
|
123
|
+
const row = calcTableRow(line, tableToken.columns);
|
|
129
124
|
if (row == null)
|
|
130
125
|
return { status: 'notMatched' };
|
|
131
126
|
tableToken.children.push(row);
|
|
132
127
|
return { status: 'opening', nextIndex: line.endIndex };
|
|
133
128
|
}
|
|
134
|
-
|
|
135
|
-
let node;
|
|
136
|
-
switch (token.nodeType) {
|
|
137
|
-
case ast.TableType: {
|
|
138
|
-
node = {
|
|
139
|
-
type: ast.TableType,
|
|
140
|
-
columns: token.columns,
|
|
141
|
-
children: children,
|
|
142
|
-
};
|
|
143
|
-
break;
|
|
144
|
-
}
|
|
145
|
-
case ast.TableRowType: {
|
|
146
|
-
node = {
|
|
147
|
-
type: ast.TableRowType,
|
|
148
|
-
children: children,
|
|
149
|
-
};
|
|
150
|
-
break;
|
|
151
|
-
}
|
|
152
|
-
case ast.TableCellType: {
|
|
153
|
-
node = {
|
|
154
|
-
type: ast.TableCellType,
|
|
155
|
-
children: children,
|
|
156
|
-
};
|
|
157
|
-
for (const phrasingContent of node.children) {
|
|
158
|
-
if (phrasingContent.type !== coreTokenizer.PhrasingContentType)
|
|
159
|
-
continue;
|
|
160
|
-
const nextContents = [];
|
|
161
|
-
const endIndex = phrasingContent.contents.length;
|
|
162
|
-
for (let i = 0; i < endIndex; ++i) {
|
|
163
|
-
const p = phrasingContent.contents[i];
|
|
164
|
-
if (p.codePoint === character.AsciiCodePoint.BACKSLASH && i + 1 < endIndex) {
|
|
165
|
-
const q = phrasingContent.contents[i + 1];
|
|
166
|
-
if (q.codePoint !== character.AsciiCodePoint.VERTICAL_SLASH)
|
|
167
|
-
nextContents.push(p);
|
|
168
|
-
nextContents.push(q);
|
|
169
|
-
i += 1;
|
|
170
|
-
continue;
|
|
171
|
-
}
|
|
172
|
-
nextContents.push(p);
|
|
173
|
-
}
|
|
174
|
-
phrasingContent.contents = nextContents;
|
|
175
|
-
}
|
|
176
|
-
break;
|
|
177
|
-
}
|
|
178
|
-
default:
|
|
179
|
-
return null;
|
|
180
|
-
}
|
|
181
|
-
return node;
|
|
182
|
-
}
|
|
183
|
-
calcTableColumn(nodePoints, currentLine, previousLine) {
|
|
184
|
-
if (previousLine.firstNonWhitespaceIndex >= previousLine.endIndex) {
|
|
185
|
-
return null;
|
|
186
|
-
}
|
|
187
|
-
if (currentLine.firstNonWhitespaceIndex - currentLine.startIndex >= 4)
|
|
188
|
-
return null;
|
|
189
|
-
const columns = [];
|
|
190
|
-
let p = nodePoints[currentLine.firstNonWhitespaceIndex];
|
|
191
|
-
let cIndex = p.codePoint === character.AsciiCodePoint.VERTICAL_SLASH
|
|
192
|
-
? currentLine.firstNonWhitespaceIndex + 1
|
|
193
|
-
: currentLine.firstNonWhitespaceIndex;
|
|
194
|
-
for (; cIndex < currentLine.endIndex;) {
|
|
195
|
-
for (; cIndex < currentLine.endIndex; ++cIndex) {
|
|
196
|
-
p = nodePoints[cIndex];
|
|
197
|
-
if (!character.isWhitespaceCharacter(p.codePoint))
|
|
198
|
-
break;
|
|
199
|
-
}
|
|
200
|
-
if (cIndex >= currentLine.endIndex)
|
|
201
|
-
break;
|
|
202
|
-
let leftColon = false;
|
|
203
|
-
if (p.codePoint === character.AsciiCodePoint.COLON) {
|
|
204
|
-
leftColon = true;
|
|
205
|
-
cIndex += 1;
|
|
206
|
-
}
|
|
207
|
-
let hyphenCount = 0;
|
|
208
|
-
for (; cIndex < currentLine.endIndex; ++cIndex) {
|
|
209
|
-
p = nodePoints[cIndex];
|
|
210
|
-
if (p.codePoint !== character.AsciiCodePoint.MINUS_SIGN)
|
|
211
|
-
break;
|
|
212
|
-
hyphenCount += 1;
|
|
213
|
-
}
|
|
214
|
-
if (hyphenCount <= 0)
|
|
215
|
-
return null;
|
|
216
|
-
let rightColon = false;
|
|
217
|
-
if (cIndex < currentLine.endIndex &&
|
|
218
|
-
p.codePoint === character.AsciiCodePoint.COLON) {
|
|
219
|
-
rightColon = true;
|
|
220
|
-
cIndex += 1;
|
|
221
|
-
}
|
|
222
|
-
for (; cIndex < currentLine.endIndex; ++cIndex) {
|
|
223
|
-
p = nodePoints[cIndex];
|
|
224
|
-
if (character.isWhitespaceCharacter(p.codePoint))
|
|
225
|
-
continue;
|
|
226
|
-
if (p.codePoint === character.AsciiCodePoint.VERTICAL_SLASH) {
|
|
227
|
-
cIndex += 1;
|
|
228
|
-
break;
|
|
229
|
-
}
|
|
230
|
-
return null;
|
|
231
|
-
}
|
|
232
|
-
let align = null;
|
|
233
|
-
if (leftColon && rightColon)
|
|
234
|
-
align = 'center';
|
|
235
|
-
else if (leftColon)
|
|
236
|
-
align = 'left';
|
|
237
|
-
else if (rightColon)
|
|
238
|
-
align = 'right';
|
|
239
|
-
const column = { align };
|
|
240
|
-
columns.push(column);
|
|
241
|
-
}
|
|
242
|
-
if (columns.length <= 0)
|
|
243
|
-
return null;
|
|
244
|
-
let cellCount = 0, hasNonWhitespaceBeforePipe = false;
|
|
245
|
-
for (let pIndex = previousLine.startIndex; pIndex < previousLine.endIndex; ++pIndex) {
|
|
246
|
-
const p = nodePoints[pIndex];
|
|
247
|
-
if (character.isWhitespaceCharacter(p.codePoint))
|
|
248
|
-
continue;
|
|
249
|
-
if (p.codePoint === character.AsciiCodePoint.VERTICAL_SLASH) {
|
|
250
|
-
if (hasNonWhitespaceBeforePipe || cellCount > 0)
|
|
251
|
-
cellCount += 1;
|
|
252
|
-
hasNonWhitespaceBeforePipe = false;
|
|
253
|
-
continue;
|
|
254
|
-
}
|
|
255
|
-
hasNonWhitespaceBeforePipe = true;
|
|
256
|
-
if (p.codePoint === character.AsciiCodePoint.BACKSLASH)
|
|
257
|
-
pIndex += 1;
|
|
258
|
-
}
|
|
259
|
-
if (hasNonWhitespaceBeforePipe && columns.length > 1)
|
|
260
|
-
cellCount += 1;
|
|
261
|
-
if (cellCount !== columns.length)
|
|
262
|
-
return null;
|
|
263
|
-
return columns;
|
|
264
|
-
}
|
|
265
|
-
calcTableRow(api, line, columns) {
|
|
129
|
+
function calcTableRow(line, columns) {
|
|
266
130
|
const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line;
|
|
267
131
|
let p = nodePoints[firstNonWhitespaceIndex];
|
|
268
132
|
let i = p.codePoint === character.AsciiCodePoint.VERTICAL_SLASH
|
|
@@ -307,7 +171,7 @@ class TableTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
307
171
|
},
|
|
308
172
|
]);
|
|
309
173
|
const cell = {
|
|
310
|
-
_tokenizer
|
|
174
|
+
_tokenizer,
|
|
311
175
|
nodeType: ast.TableCellType,
|
|
312
176
|
position: { start: startPoint, end: endPoint },
|
|
313
177
|
children: phrasingContent == null ? [] : [phrasingContent],
|
|
@@ -320,7 +184,7 @@ class TableTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
320
184
|
const endPoint = coreTokenizer.calcEndYastNodePoint(nodePoints, endIndex - 1);
|
|
321
185
|
for (let c = cells.length; c < columns.length; ++c) {
|
|
322
186
|
const cell = {
|
|
323
|
-
_tokenizer
|
|
187
|
+
_tokenizer,
|
|
324
188
|
nodeType: ast.TableCellType,
|
|
325
189
|
position: { start: Object.assign({}, endPoint), end: Object.assign({}, endPoint) },
|
|
326
190
|
children: [],
|
|
@@ -328,15 +192,83 @@ class TableTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
328
192
|
cells.push(cell);
|
|
329
193
|
}
|
|
330
194
|
const row = {
|
|
331
|
-
_tokenizer
|
|
195
|
+
_tokenizer,
|
|
332
196
|
nodeType: ast.TableRowType,
|
|
333
197
|
position: { start: startPoint, end: endPoint },
|
|
334
198
|
children: cells,
|
|
335
199
|
};
|
|
336
200
|
return row;
|
|
337
201
|
}
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
const parse = () => ({
|
|
205
|
+
parse: (token, children) => {
|
|
206
|
+
let node;
|
|
207
|
+
switch (token.nodeType) {
|
|
208
|
+
case ast.TableType: {
|
|
209
|
+
node = {
|
|
210
|
+
type: ast.TableType,
|
|
211
|
+
columns: token.columns,
|
|
212
|
+
children: children,
|
|
213
|
+
};
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
case ast.TableRowType: {
|
|
217
|
+
node = {
|
|
218
|
+
type: ast.TableRowType,
|
|
219
|
+
children: children,
|
|
220
|
+
};
|
|
221
|
+
break;
|
|
222
|
+
}
|
|
223
|
+
case ast.TableCellType: {
|
|
224
|
+
node = {
|
|
225
|
+
type: ast.TableCellType,
|
|
226
|
+
children: children,
|
|
227
|
+
};
|
|
228
|
+
for (const phrasingContent of node.children) {
|
|
229
|
+
if (phrasingContent.type !== coreTokenizer.PhrasingContentType)
|
|
230
|
+
continue;
|
|
231
|
+
const nextContents = [];
|
|
232
|
+
const endIndex = phrasingContent.contents.length;
|
|
233
|
+
for (let i = 0; i < endIndex; ++i) {
|
|
234
|
+
const p = phrasingContent.contents[i];
|
|
235
|
+
if (p.codePoint === character.AsciiCodePoint.BACKSLASH && i + 1 < endIndex) {
|
|
236
|
+
const q = phrasingContent.contents[i + 1];
|
|
237
|
+
if (q.codePoint !== character.AsciiCodePoint.VERTICAL_SLASH)
|
|
238
|
+
nextContents.push(p);
|
|
239
|
+
nextContents.push(q);
|
|
240
|
+
i += 1;
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
nextContents.push(p);
|
|
244
|
+
}
|
|
245
|
+
phrasingContent.contents = nextContents;
|
|
246
|
+
}
|
|
247
|
+
break;
|
|
248
|
+
}
|
|
249
|
+
default:
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
return node;
|
|
253
|
+
},
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
const uniqueName = '@yozora/tokenizer-table';
|
|
257
|
+
|
|
258
|
+
class TableTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
259
|
+
constructor(props = {}) {
|
|
260
|
+
var _a, _b;
|
|
261
|
+
super({
|
|
262
|
+
name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
|
|
263
|
+
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.INTERRUPTABLE_BLOCK,
|
|
264
|
+
});
|
|
265
|
+
this.match = match;
|
|
266
|
+
this.parse = parse;
|
|
267
|
+
}
|
|
338
268
|
}
|
|
339
269
|
|
|
340
270
|
exports.TableTokenizer = TableTokenizer;
|
|
341
271
|
exports.TableTokenizerName = uniqueName;
|
|
342
|
-
exports[
|
|
272
|
+
exports["default"] = TableTokenizer;
|
|
273
|
+
exports.tableMatch = match;
|
|
274
|
+
exports.tableParse = parse;
|
package/lib/esm/index.js
CHANGED
|
@@ -1,22 +1,19 @@
|
|
|
1
1
|
import { TableType, TableCellType, TableRowType } from '@yozora/ast';
|
|
2
2
|
import { AsciiCodePoint, isWhitespaceCharacter } from '@yozora/character';
|
|
3
|
-
import {
|
|
3
|
+
import { calcStartYastNodePoint, calcEndYastNodePoint, PhrasingContentType, BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
|
|
4
4
|
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
this.isContainingBlock = false;
|
|
15
|
-
}
|
|
16
|
-
eatOpener() {
|
|
5
|
+
const match = function (api) {
|
|
6
|
+
const { name: _tokenizer } = this;
|
|
7
|
+
return {
|
|
8
|
+
isContainingBlock: false,
|
|
9
|
+
eatOpener,
|
|
10
|
+
eatAndInterruptPreviousSibling,
|
|
11
|
+
eatLazyContinuationText,
|
|
12
|
+
};
|
|
13
|
+
function eatOpener() {
|
|
17
14
|
return null;
|
|
18
15
|
}
|
|
19
|
-
eatAndInterruptPreviousSibling(line, prevSiblingToken
|
|
16
|
+
function eatAndInterruptPreviousSibling(line, prevSiblingToken) {
|
|
20
17
|
if (line.countOfPrecedeSpaces >= 4)
|
|
21
18
|
return null;
|
|
22
19
|
const { nodePoints, endIndex, firstNonWhitespaceIndex } = line;
|
|
@@ -24,9 +21,7 @@ class TableTokenizer extends BaseBlockTokenizer {
|
|
|
24
21
|
return null;
|
|
25
22
|
const columns = [];
|
|
26
23
|
let c = nodePoints[firstNonWhitespaceIndex].codePoint;
|
|
27
|
-
let cIndex = c === AsciiCodePoint.VERTICAL_SLASH
|
|
28
|
-
? firstNonWhitespaceIndex + 1
|
|
29
|
-
: firstNonWhitespaceIndex;
|
|
24
|
+
let cIndex = c === AsciiCodePoint.VERTICAL_SLASH ? firstNonWhitespaceIndex + 1 : firstNonWhitespaceIndex;
|
|
30
25
|
for (; cIndex < endIndex;) {
|
|
31
26
|
for (; cIndex < endIndex; ++cIndex) {
|
|
32
27
|
c = nodePoints[cIndex].codePoint;
|
|
@@ -99,7 +94,7 @@ class TableTokenizer extends BaseBlockTokenizer {
|
|
|
99
94
|
cellCount += 1;
|
|
100
95
|
if (cellCount !== columns.length)
|
|
101
96
|
return null;
|
|
102
|
-
const row =
|
|
97
|
+
const row = calcTableRow(previousLine, columns);
|
|
103
98
|
const nextIndex = endIndex;
|
|
104
99
|
const token = {
|
|
105
100
|
nodeType: TableType,
|
|
@@ -116,149 +111,18 @@ class TableTokenizer extends BaseBlockTokenizer {
|
|
|
116
111
|
remainingSibling: api.rollbackPhrasingLines(lines.slice(0, lines.length - 1), prevSiblingToken),
|
|
117
112
|
};
|
|
118
113
|
}
|
|
119
|
-
eatLazyContinuationText(line, token
|
|
114
|
+
function eatLazyContinuationText(line, token) {
|
|
120
115
|
if (line.firstNonWhitespaceIndex >= line.endIndex) {
|
|
121
116
|
return { status: 'notMatched' };
|
|
122
117
|
}
|
|
123
118
|
const tableToken = token;
|
|
124
|
-
const row =
|
|
119
|
+
const row = calcTableRow(line, tableToken.columns);
|
|
125
120
|
if (row == null)
|
|
126
121
|
return { status: 'notMatched' };
|
|
127
122
|
tableToken.children.push(row);
|
|
128
123
|
return { status: 'opening', nextIndex: line.endIndex };
|
|
129
124
|
}
|
|
130
|
-
|
|
131
|
-
let node;
|
|
132
|
-
switch (token.nodeType) {
|
|
133
|
-
case TableType: {
|
|
134
|
-
node = {
|
|
135
|
-
type: TableType,
|
|
136
|
-
columns: token.columns,
|
|
137
|
-
children: children,
|
|
138
|
-
};
|
|
139
|
-
break;
|
|
140
|
-
}
|
|
141
|
-
case TableRowType: {
|
|
142
|
-
node = {
|
|
143
|
-
type: TableRowType,
|
|
144
|
-
children: children,
|
|
145
|
-
};
|
|
146
|
-
break;
|
|
147
|
-
}
|
|
148
|
-
case TableCellType: {
|
|
149
|
-
node = {
|
|
150
|
-
type: TableCellType,
|
|
151
|
-
children: children,
|
|
152
|
-
};
|
|
153
|
-
for (const phrasingContent of node.children) {
|
|
154
|
-
if (phrasingContent.type !== PhrasingContentType)
|
|
155
|
-
continue;
|
|
156
|
-
const nextContents = [];
|
|
157
|
-
const endIndex = phrasingContent.contents.length;
|
|
158
|
-
for (let i = 0; i < endIndex; ++i) {
|
|
159
|
-
const p = phrasingContent.contents[i];
|
|
160
|
-
if (p.codePoint === AsciiCodePoint.BACKSLASH && i + 1 < endIndex) {
|
|
161
|
-
const q = phrasingContent.contents[i + 1];
|
|
162
|
-
if (q.codePoint !== AsciiCodePoint.VERTICAL_SLASH)
|
|
163
|
-
nextContents.push(p);
|
|
164
|
-
nextContents.push(q);
|
|
165
|
-
i += 1;
|
|
166
|
-
continue;
|
|
167
|
-
}
|
|
168
|
-
nextContents.push(p);
|
|
169
|
-
}
|
|
170
|
-
phrasingContent.contents = nextContents;
|
|
171
|
-
}
|
|
172
|
-
break;
|
|
173
|
-
}
|
|
174
|
-
default:
|
|
175
|
-
return null;
|
|
176
|
-
}
|
|
177
|
-
return node;
|
|
178
|
-
}
|
|
179
|
-
calcTableColumn(nodePoints, currentLine, previousLine) {
|
|
180
|
-
if (previousLine.firstNonWhitespaceIndex >= previousLine.endIndex) {
|
|
181
|
-
return null;
|
|
182
|
-
}
|
|
183
|
-
if (currentLine.firstNonWhitespaceIndex - currentLine.startIndex >= 4)
|
|
184
|
-
return null;
|
|
185
|
-
const columns = [];
|
|
186
|
-
let p = nodePoints[currentLine.firstNonWhitespaceIndex];
|
|
187
|
-
let cIndex = p.codePoint === AsciiCodePoint.VERTICAL_SLASH
|
|
188
|
-
? currentLine.firstNonWhitespaceIndex + 1
|
|
189
|
-
: currentLine.firstNonWhitespaceIndex;
|
|
190
|
-
for (; cIndex < currentLine.endIndex;) {
|
|
191
|
-
for (; cIndex < currentLine.endIndex; ++cIndex) {
|
|
192
|
-
p = nodePoints[cIndex];
|
|
193
|
-
if (!isWhitespaceCharacter(p.codePoint))
|
|
194
|
-
break;
|
|
195
|
-
}
|
|
196
|
-
if (cIndex >= currentLine.endIndex)
|
|
197
|
-
break;
|
|
198
|
-
let leftColon = false;
|
|
199
|
-
if (p.codePoint === AsciiCodePoint.COLON) {
|
|
200
|
-
leftColon = true;
|
|
201
|
-
cIndex += 1;
|
|
202
|
-
}
|
|
203
|
-
let hyphenCount = 0;
|
|
204
|
-
for (; cIndex < currentLine.endIndex; ++cIndex) {
|
|
205
|
-
p = nodePoints[cIndex];
|
|
206
|
-
if (p.codePoint !== AsciiCodePoint.MINUS_SIGN)
|
|
207
|
-
break;
|
|
208
|
-
hyphenCount += 1;
|
|
209
|
-
}
|
|
210
|
-
if (hyphenCount <= 0)
|
|
211
|
-
return null;
|
|
212
|
-
let rightColon = false;
|
|
213
|
-
if (cIndex < currentLine.endIndex &&
|
|
214
|
-
p.codePoint === AsciiCodePoint.COLON) {
|
|
215
|
-
rightColon = true;
|
|
216
|
-
cIndex += 1;
|
|
217
|
-
}
|
|
218
|
-
for (; cIndex < currentLine.endIndex; ++cIndex) {
|
|
219
|
-
p = nodePoints[cIndex];
|
|
220
|
-
if (isWhitespaceCharacter(p.codePoint))
|
|
221
|
-
continue;
|
|
222
|
-
if (p.codePoint === AsciiCodePoint.VERTICAL_SLASH) {
|
|
223
|
-
cIndex += 1;
|
|
224
|
-
break;
|
|
225
|
-
}
|
|
226
|
-
return null;
|
|
227
|
-
}
|
|
228
|
-
let align = null;
|
|
229
|
-
if (leftColon && rightColon)
|
|
230
|
-
align = 'center';
|
|
231
|
-
else if (leftColon)
|
|
232
|
-
align = 'left';
|
|
233
|
-
else if (rightColon)
|
|
234
|
-
align = 'right';
|
|
235
|
-
const column = { align };
|
|
236
|
-
columns.push(column);
|
|
237
|
-
}
|
|
238
|
-
if (columns.length <= 0)
|
|
239
|
-
return null;
|
|
240
|
-
let cellCount = 0, hasNonWhitespaceBeforePipe = false;
|
|
241
|
-
for (let pIndex = previousLine.startIndex; pIndex < previousLine.endIndex; ++pIndex) {
|
|
242
|
-
const p = nodePoints[pIndex];
|
|
243
|
-
if (isWhitespaceCharacter(p.codePoint))
|
|
244
|
-
continue;
|
|
245
|
-
if (p.codePoint === AsciiCodePoint.VERTICAL_SLASH) {
|
|
246
|
-
if (hasNonWhitespaceBeforePipe || cellCount > 0)
|
|
247
|
-
cellCount += 1;
|
|
248
|
-
hasNonWhitespaceBeforePipe = false;
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
251
|
-
hasNonWhitespaceBeforePipe = true;
|
|
252
|
-
if (p.codePoint === AsciiCodePoint.BACKSLASH)
|
|
253
|
-
pIndex += 1;
|
|
254
|
-
}
|
|
255
|
-
if (hasNonWhitespaceBeforePipe && columns.length > 1)
|
|
256
|
-
cellCount += 1;
|
|
257
|
-
if (cellCount !== columns.length)
|
|
258
|
-
return null;
|
|
259
|
-
return columns;
|
|
260
|
-
}
|
|
261
|
-
calcTableRow(api, line, columns) {
|
|
125
|
+
function calcTableRow(line, columns) {
|
|
262
126
|
const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex } = line;
|
|
263
127
|
let p = nodePoints[firstNonWhitespaceIndex];
|
|
264
128
|
let i = p.codePoint === AsciiCodePoint.VERTICAL_SLASH
|
|
@@ -303,7 +167,7 @@ class TableTokenizer extends BaseBlockTokenizer {
|
|
|
303
167
|
},
|
|
304
168
|
]);
|
|
305
169
|
const cell = {
|
|
306
|
-
_tokenizer
|
|
170
|
+
_tokenizer,
|
|
307
171
|
nodeType: TableCellType,
|
|
308
172
|
position: { start: startPoint, end: endPoint },
|
|
309
173
|
children: phrasingContent == null ? [] : [phrasingContent],
|
|
@@ -316,7 +180,7 @@ class TableTokenizer extends BaseBlockTokenizer {
|
|
|
316
180
|
const endPoint = calcEndYastNodePoint(nodePoints, endIndex - 1);
|
|
317
181
|
for (let c = cells.length; c < columns.length; ++c) {
|
|
318
182
|
const cell = {
|
|
319
|
-
_tokenizer
|
|
183
|
+
_tokenizer,
|
|
320
184
|
nodeType: TableCellType,
|
|
321
185
|
position: { start: Object.assign({}, endPoint), end: Object.assign({}, endPoint) },
|
|
322
186
|
children: [],
|
|
@@ -324,13 +188,79 @@ class TableTokenizer extends BaseBlockTokenizer {
|
|
|
324
188
|
cells.push(cell);
|
|
325
189
|
}
|
|
326
190
|
const row = {
|
|
327
|
-
_tokenizer
|
|
191
|
+
_tokenizer,
|
|
328
192
|
nodeType: TableRowType,
|
|
329
193
|
position: { start: startPoint, end: endPoint },
|
|
330
194
|
children: cells,
|
|
331
195
|
};
|
|
332
196
|
return row;
|
|
333
197
|
}
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
const parse = () => ({
|
|
201
|
+
parse: (token, children) => {
|
|
202
|
+
let node;
|
|
203
|
+
switch (token.nodeType) {
|
|
204
|
+
case TableType: {
|
|
205
|
+
node = {
|
|
206
|
+
type: TableType,
|
|
207
|
+
columns: token.columns,
|
|
208
|
+
children: children,
|
|
209
|
+
};
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
case TableRowType: {
|
|
213
|
+
node = {
|
|
214
|
+
type: TableRowType,
|
|
215
|
+
children: children,
|
|
216
|
+
};
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
case TableCellType: {
|
|
220
|
+
node = {
|
|
221
|
+
type: TableCellType,
|
|
222
|
+
children: children,
|
|
223
|
+
};
|
|
224
|
+
for (const phrasingContent of node.children) {
|
|
225
|
+
if (phrasingContent.type !== PhrasingContentType)
|
|
226
|
+
continue;
|
|
227
|
+
const nextContents = [];
|
|
228
|
+
const endIndex = phrasingContent.contents.length;
|
|
229
|
+
for (let i = 0; i < endIndex; ++i) {
|
|
230
|
+
const p = phrasingContent.contents[i];
|
|
231
|
+
if (p.codePoint === AsciiCodePoint.BACKSLASH && i + 1 < endIndex) {
|
|
232
|
+
const q = phrasingContent.contents[i + 1];
|
|
233
|
+
if (q.codePoint !== AsciiCodePoint.VERTICAL_SLASH)
|
|
234
|
+
nextContents.push(p);
|
|
235
|
+
nextContents.push(q);
|
|
236
|
+
i += 1;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
nextContents.push(p);
|
|
240
|
+
}
|
|
241
|
+
phrasingContent.contents = nextContents;
|
|
242
|
+
}
|
|
243
|
+
break;
|
|
244
|
+
}
|
|
245
|
+
default:
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
return node;
|
|
249
|
+
},
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
const uniqueName = '@yozora/tokenizer-table';
|
|
253
|
+
|
|
254
|
+
class TableTokenizer extends BaseBlockTokenizer {
|
|
255
|
+
constructor(props = {}) {
|
|
256
|
+
var _a, _b;
|
|
257
|
+
super({
|
|
258
|
+
name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
|
|
259
|
+
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.INTERRUPTABLE_BLOCK,
|
|
260
|
+
});
|
|
261
|
+
this.match = match;
|
|
262
|
+
this.parse = parse;
|
|
263
|
+
}
|
|
334
264
|
}
|
|
335
265
|
|
|
336
|
-
export { TableTokenizer, uniqueName as TableTokenizerName, TableTokenizer as default };
|
|
266
|
+
export { TableTokenizer, uniqueName as TableTokenizerName, TableTokenizer as default, match as tableMatch, parse as tableParse };
|
package/lib/types/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
export {
|
|
1
|
+
export { match as tableMatch } from './match';
|
|
2
|
+
export { parse as tableParse } from './parse';
|
|
3
|
+
export { TableTokenizer, TableTokenizer as default } from './tokenizer';
|
|
3
4
|
export { uniqueName as TableTokenizerName } from './types';
|
|
4
|
-
export type {
|
|
5
|
-
export default TableTokenizer;
|
|
5
|
+
export type { IHookContext as ITableHookContext, ITableCellToken, ITableRowToken, ITableToken, ITokenizerProps as ITableTokenizerProps, } from './types';
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { IMatchBlockHookCreator } from '@yozora/core-tokenizer';
|
|
2
|
+
import type { IHookContext, IToken, T } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* A table is an arrangement of data with rows and columns, consisting of
|
|
5
|
+
* a single header row, a delimiter row separating the header from the data,
|
|
6
|
+
* and zero or more data rows.
|
|
7
|
+
*
|
|
8
|
+
* Each row consists of cells containing arbitrary text, in which inlines
|
|
9
|
+
* are parsed, separated by pipes (|). A leading and trailing pipe is also
|
|
10
|
+
* recommended for clarity of reading, and if there’s otherwise parsing
|
|
11
|
+
* ambiguity. Spaces between pipes and cell content are trimmed. Block-level
|
|
12
|
+
* elements cannot be inserted in a table.
|
|
13
|
+
*
|
|
14
|
+
* @see https://github.github.com/gfm/#table
|
|
15
|
+
* @see https://github.com/syntax-tree/mdast#tablerow
|
|
16
|
+
* @see https://github.com/syntax-tree/mdast#tablecell
|
|
17
|
+
*/
|
|
18
|
+
export declare const match: IMatchBlockHookCreator<T, IToken, IHookContext>;
|
|
19
|
+
/**
|
|
20
|
+
* Find delimiter row
|
|
21
|
+
*
|
|
22
|
+
* The delimiter row consists of cells whose only content are
|
|
23
|
+
* hyphens (-), and optionally, a leading or trailing colon (:),
|
|
24
|
+
* or both, to indicate left, right, or center alignment respectively.
|
|
25
|
+
* @see https://github.github.com/gfm/#delimiter-row
|
|
26
|
+
*/
|
package/lib/types/tokenizer.d.ts
CHANGED
|
@@ -1,59 +1,14 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { NodePoint } from '@yozora/character';
|
|
3
|
-
import type { MatchBlockPhaseApi, PhrasingContentLine, ResultOfEatAndInterruptPreviousSibling, ResultOfEatLazyContinuationText, ResultOfEatOpener, ResultOfParse, Tokenizer, TokenizerMatchBlockHook, TokenizerParseBlockHook, YastBlockToken } from '@yozora/core-tokenizer';
|
|
1
|
+
import type { IBlockTokenizer, IMatchBlockHookCreator, IParseBlockHookCreator } from '@yozora/core-tokenizer';
|
|
4
2
|
import { BaseBlockTokenizer } from '@yozora/core-tokenizer';
|
|
5
|
-
import type {
|
|
3
|
+
import type { IHookContext, INode, IToken, ITokenizerProps, T } from './types';
|
|
6
4
|
/**
|
|
7
5
|
* Lexical Analyzer for Table, table-row and table-cell.
|
|
8
|
-
*
|
|
9
|
-
* A table is an arrangement of data with rows and columns, consisting of
|
|
10
|
-
* a single header row, a delimiter row separating the header from the data,
|
|
11
|
-
* and zero or more data rows.
|
|
12
|
-
*
|
|
13
|
-
* Each row consists of cells containing arbitrary text, in which inlines
|
|
14
|
-
* are parsed, separated by pipes (|). A leading and trailing pipe is also
|
|
15
|
-
* recommended for clarity of reading, and if there’s otherwise parsing
|
|
16
|
-
* ambiguity. Spaces between pipes and cell content are trimmed. Block-level
|
|
17
|
-
* elements cannot be inserted in a table.
|
|
18
|
-
*
|
|
19
6
|
* @see https://github.github.com/gfm/#table
|
|
20
7
|
* @see https://github.com/syntax-tree/mdast#tablerow
|
|
21
8
|
* @see https://github.com/syntax-tree/mdast#tablecell
|
|
22
9
|
*/
|
|
23
|
-
export declare class TableTokenizer extends BaseBlockTokenizer
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
* @override
|
|
28
|
-
* @see TokenizerMatchBlockHook
|
|
29
|
-
*/
|
|
30
|
-
eatOpener(): ResultOfEatOpener<T, Token>;
|
|
31
|
-
/**
|
|
32
|
-
* @override
|
|
33
|
-
* @see TokenizerMatchBlockHook
|
|
34
|
-
*/
|
|
35
|
-
eatAndInterruptPreviousSibling(line: Readonly<PhrasingContentLine>, prevSiblingToken: Readonly<YastBlockToken>, parentToken: Readonly<YastBlockToken>, api: Readonly<MatchBlockPhaseApi>): ResultOfEatAndInterruptPreviousSibling<T, Token>;
|
|
36
|
-
/**
|
|
37
|
-
* @override
|
|
38
|
-
* @see TokenizerMatchBlockHook
|
|
39
|
-
*/
|
|
40
|
-
eatLazyContinuationText(line: Readonly<PhrasingContentLine>, token: Token, parentToken: Readonly<YastBlockToken>, api: Readonly<MatchBlockPhaseApi>): ResultOfEatLazyContinuationText;
|
|
41
|
-
/**
|
|
42
|
-
* @override
|
|
43
|
-
* @see TokenizerParseBlockHook
|
|
44
|
-
*/
|
|
45
|
-
parseBlock(token: Readonly<Token>, children: YastNode[]): ResultOfParse<T, Node>;
|
|
46
|
-
/**
|
|
47
|
-
* Find delimiter row
|
|
48
|
-
*
|
|
49
|
-
* The delimiter row consists of cells whose only content are
|
|
50
|
-
* hyphens (-), and optionally, a leading or trailing colon (:),
|
|
51
|
-
* or both, to indicate left, right, or center alignment respectively.
|
|
52
|
-
* @see https://github.github.com/gfm/#delimiter-row
|
|
53
|
-
*/
|
|
54
|
-
protected calcTableColumn(nodePoints: ReadonlyArray<NodePoint>, currentLine: PhrasingContentLine, previousLine: PhrasingContentLine): TableColumn[] | null;
|
|
55
|
-
/**
|
|
56
|
-
* process table row
|
|
57
|
-
*/
|
|
58
|
-
protected calcTableRow(api: Readonly<MatchBlockPhaseApi>, line: PhrasingContentLine, columns: TableColumn[]): TableRowToken;
|
|
10
|
+
export declare class TableTokenizer extends BaseBlockTokenizer<T, IToken, INode, IHookContext> implements IBlockTokenizer<T, IToken, INode, IHookContext> {
|
|
11
|
+
constructor(props?: ITokenizerProps);
|
|
12
|
+
readonly match: IMatchBlockHookCreator<T, IToken, IHookContext>;
|
|
13
|
+
readonly parse: IParseBlockHookCreator<T, IToken, INode, IHookContext>;
|
|
59
14
|
}
|
package/lib/types/types.d.ts
CHANGED
|
@@ -1,29 +1,30 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
1
|
+
import type { ITable, ITableCell, ITableColumn, ITableRow, TableCellType, TableRowType, TableType } from '@yozora/ast';
|
|
2
|
+
import type { IBaseBlockTokenizerProps, IPartialYastBlockToken, IPhrasingContentToken, ITokenizer, IYastBlockToken } from '@yozora/core-tokenizer';
|
|
3
3
|
export declare type T = TableType | TableRowType | TableCellType;
|
|
4
|
-
export declare type
|
|
5
|
-
export declare type
|
|
4
|
+
export declare type INode = ITable | ITableRow | ITableCell;
|
|
5
|
+
export declare type IToken = ITableToken | ITableRowToken | ITableCellToken;
|
|
6
6
|
export declare const uniqueName = "@yozora/tokenizer-table";
|
|
7
|
-
export interface
|
|
7
|
+
export interface ITableToken extends IPartialYastBlockToken<TableType> {
|
|
8
8
|
/**
|
|
9
9
|
* Table column configuration items
|
|
10
10
|
*/
|
|
11
|
-
columns:
|
|
11
|
+
columns: ITableColumn[];
|
|
12
12
|
/**
|
|
13
13
|
* Table rows
|
|
14
14
|
*/
|
|
15
|
-
children:
|
|
15
|
+
children: ITableRowToken[];
|
|
16
16
|
}
|
|
17
|
-
export interface
|
|
17
|
+
export interface ITableRowToken extends IYastBlockToken<TableRowType> {
|
|
18
18
|
/**
|
|
19
19
|
* Table cells
|
|
20
20
|
*/
|
|
21
|
-
children:
|
|
21
|
+
children: ITableCellToken[];
|
|
22
22
|
}
|
|
23
|
-
export interface
|
|
23
|
+
export interface ITableCellToken extends IYastBlockToken<TableCellType> {
|
|
24
24
|
/**
|
|
25
25
|
* Contents of table cell.
|
|
26
26
|
*/
|
|
27
|
-
children:
|
|
27
|
+
children: IPhrasingContentToken[];
|
|
28
28
|
}
|
|
29
|
-
export declare type
|
|
29
|
+
export declare type IHookContext = ITokenizer;
|
|
30
|
+
export declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yozora/tokenizer-table",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0-alpha.0",
|
|
4
4
|
"author": {
|
|
5
5
|
"name": "guanghechen",
|
|
6
6
|
"url": "https://github.com/guanghechen/"
|
|
@@ -35,9 +35,9 @@
|
|
|
35
35
|
"test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@yozora/ast": "^
|
|
39
|
-
"@yozora/character": "^
|
|
40
|
-
"@yozora/core-tokenizer": "^
|
|
38
|
+
"@yozora/ast": "^2.0.0-alpha.0",
|
|
39
|
+
"@yozora/character": "^2.0.0-alpha.0",
|
|
40
|
+
"@yozora/core-tokenizer": "^2.0.0-alpha.0"
|
|
41
41
|
},
|
|
42
|
-
"gitHead": "
|
|
42
|
+
"gitHead": "0171501339c49ffd02ed16a63447fa20a47a29a7"
|
|
43
43
|
}
|