@yozora/tokenizer-fenced-code 1.3.0 → 2.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/index.js +37 -24
- package/lib/esm/index.js +36 -25
- package/lib/types/index.d.ts +4 -4
- package/lib/types/match.d.ts +11 -0
- package/lib/types/parse.d.ts +3 -0
- package/lib/types/tokenizer.d.ts +6 -15
- package/lib/types/types.d.ts +7 -6
- package/package.json +6 -6
package/lib/cjs/index.js
CHANGED
|
@@ -2,18 +2,48 @@
|
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
|
+
var FencedBlockTokenizer = require('@yozora/tokenizer-fenced-block');
|
|
5
6
|
var ast = require('@yozora/ast');
|
|
6
7
|
var character = require('@yozora/character');
|
|
7
8
|
var coreTokenizer = require('@yozora/core-tokenizer');
|
|
8
|
-
var FencedBlockTokenizer = require('@yozora/tokenizer-fenced-block');
|
|
9
9
|
|
|
10
10
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
11
11
|
|
|
12
12
|
var FencedBlockTokenizer__default = /*#__PURE__*/_interopDefaultLegacy(FencedBlockTokenizer);
|
|
13
13
|
|
|
14
|
+
const match = function (api) {
|
|
15
|
+
const hook = FencedBlockTokenizer.fencedMatch.call(this, api);
|
|
16
|
+
return Object.assign(Object.assign({}, hook), { isContainingBlock: false });
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const parse = function () {
|
|
20
|
+
return {
|
|
21
|
+
parse: token => {
|
|
22
|
+
const infoString = token.infoString;
|
|
23
|
+
let i = 0;
|
|
24
|
+
const lang = [];
|
|
25
|
+
for (; i < infoString.length; ++i) {
|
|
26
|
+
const p = infoString[i];
|
|
27
|
+
if (character.isUnicodeWhitespaceCharacter(p.codePoint))
|
|
28
|
+
break;
|
|
29
|
+
lang.push(p);
|
|
30
|
+
}
|
|
31
|
+
i = coreTokenizer.eatOptionalWhitespaces(infoString, i, infoString.length);
|
|
32
|
+
const contents = coreTokenizer.mergeContentLinesFaithfully(token.lines);
|
|
33
|
+
const node = {
|
|
34
|
+
type: ast.CodeType,
|
|
35
|
+
lang: character.calcEscapedStringFromNodePoints(lang, 0, lang.length, true),
|
|
36
|
+
meta: character.calcEscapedStringFromNodePoints(infoString, i, infoString.length, true),
|
|
37
|
+
value: character.calcStringFromNodePoints(contents),
|
|
38
|
+
};
|
|
39
|
+
return node;
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
|
|
14
44
|
const uniqueName = '@yozora/tokenizer-fenced-code';
|
|
15
45
|
|
|
16
|
-
class FencedCodeTokenizer extends FencedBlockTokenizer__default[
|
|
46
|
+
class FencedCodeTokenizer extends FencedBlockTokenizer__default["default"] {
|
|
17
47
|
constructor(props = {}) {
|
|
18
48
|
var _a, _b;
|
|
19
49
|
super({
|
|
@@ -32,30 +62,13 @@ class FencedCodeTokenizer extends FencedBlockTokenizer__default['default'] {
|
|
|
32
62
|
return true;
|
|
33
63
|
},
|
|
34
64
|
});
|
|
35
|
-
this.
|
|
36
|
-
|
|
37
|
-
parseBlock(token) {
|
|
38
|
-
const infoString = token.infoString;
|
|
39
|
-
let i = 0;
|
|
40
|
-
const lang = [];
|
|
41
|
-
for (; i < infoString.length; ++i) {
|
|
42
|
-
const p = infoString[i];
|
|
43
|
-
if (character.isUnicodeWhitespaceCharacter(p.codePoint))
|
|
44
|
-
break;
|
|
45
|
-
lang.push(p);
|
|
46
|
-
}
|
|
47
|
-
i = coreTokenizer.eatOptionalWhitespaces(infoString, i, infoString.length);
|
|
48
|
-
const contents = coreTokenizer.mergeContentLinesFaithfully(token.lines);
|
|
49
|
-
const node = {
|
|
50
|
-
type: ast.CodeType,
|
|
51
|
-
lang: character.calcEscapedStringFromNodePoints(lang, 0, lang.length, true),
|
|
52
|
-
meta: character.calcEscapedStringFromNodePoints(infoString, i, infoString.length, true),
|
|
53
|
-
value: character.calcStringFromNodePoints(contents),
|
|
54
|
-
};
|
|
55
|
-
return node;
|
|
65
|
+
this.match = match;
|
|
66
|
+
this.parse = parse;
|
|
56
67
|
}
|
|
57
68
|
}
|
|
58
69
|
|
|
59
70
|
exports.FencedCodeTokenizer = FencedCodeTokenizer;
|
|
60
71
|
exports.FencedCodeTokenizerName = uniqueName;
|
|
61
|
-
exports[
|
|
72
|
+
exports["default"] = FencedCodeTokenizer;
|
|
73
|
+
exports.fencedCodeMatch = match;
|
|
74
|
+
exports.fencedCodeParse = parse;
|
package/lib/esm/index.js
CHANGED
|
@@ -1,7 +1,37 @@
|
|
|
1
|
+
import FencedBlockTokenizer, { fencedMatch } from '@yozora/tokenizer-fenced-block';
|
|
1
2
|
import { CodeType } from '@yozora/ast';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import { isUnicodeWhitespaceCharacter, calcEscapedStringFromNodePoints, calcStringFromNodePoints, AsciiCodePoint } from '@yozora/character';
|
|
4
|
+
import { eatOptionalWhitespaces, mergeContentLinesFaithfully, TokenizerPriority } from '@yozora/core-tokenizer';
|
|
5
|
+
|
|
6
|
+
const match = function (api) {
|
|
7
|
+
const hook = fencedMatch.call(this, api);
|
|
8
|
+
return Object.assign(Object.assign({}, hook), { isContainingBlock: false });
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const parse = function () {
|
|
12
|
+
return {
|
|
13
|
+
parse: token => {
|
|
14
|
+
const infoString = token.infoString;
|
|
15
|
+
let i = 0;
|
|
16
|
+
const lang = [];
|
|
17
|
+
for (; i < infoString.length; ++i) {
|
|
18
|
+
const p = infoString[i];
|
|
19
|
+
if (isUnicodeWhitespaceCharacter(p.codePoint))
|
|
20
|
+
break;
|
|
21
|
+
lang.push(p);
|
|
22
|
+
}
|
|
23
|
+
i = eatOptionalWhitespaces(infoString, i, infoString.length);
|
|
24
|
+
const contents = mergeContentLinesFaithfully(token.lines);
|
|
25
|
+
const node = {
|
|
26
|
+
type: CodeType,
|
|
27
|
+
lang: calcEscapedStringFromNodePoints(lang, 0, lang.length, true),
|
|
28
|
+
meta: calcEscapedStringFromNodePoints(infoString, i, infoString.length, true),
|
|
29
|
+
value: calcStringFromNodePoints(contents),
|
|
30
|
+
};
|
|
31
|
+
return node;
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
};
|
|
5
35
|
|
|
6
36
|
const uniqueName = '@yozora/tokenizer-fenced-code';
|
|
7
37
|
|
|
@@ -24,28 +54,9 @@ class FencedCodeTokenizer extends FencedBlockTokenizer {
|
|
|
24
54
|
return true;
|
|
25
55
|
},
|
|
26
56
|
});
|
|
27
|
-
this.
|
|
28
|
-
|
|
29
|
-
parseBlock(token) {
|
|
30
|
-
const infoString = token.infoString;
|
|
31
|
-
let i = 0;
|
|
32
|
-
const lang = [];
|
|
33
|
-
for (; i < infoString.length; ++i) {
|
|
34
|
-
const p = infoString[i];
|
|
35
|
-
if (isUnicodeWhitespaceCharacter(p.codePoint))
|
|
36
|
-
break;
|
|
37
|
-
lang.push(p);
|
|
38
|
-
}
|
|
39
|
-
i = eatOptionalWhitespaces(infoString, i, infoString.length);
|
|
40
|
-
const contents = mergeContentLinesFaithfully(token.lines);
|
|
41
|
-
const node = {
|
|
42
|
-
type: CodeType,
|
|
43
|
-
lang: calcEscapedStringFromNodePoints(lang, 0, lang.length, true),
|
|
44
|
-
meta: calcEscapedStringFromNodePoints(infoString, i, infoString.length, true),
|
|
45
|
-
value: calcStringFromNodePoints(contents),
|
|
46
|
-
};
|
|
47
|
-
return node;
|
|
57
|
+
this.match = match;
|
|
58
|
+
this.parse = parse;
|
|
48
59
|
}
|
|
49
60
|
}
|
|
50
61
|
|
|
51
|
-
export { FencedCodeTokenizer, uniqueName as FencedCodeTokenizerName, FencedCodeTokenizer as default };
|
|
62
|
+
export { FencedCodeTokenizer, uniqueName as FencedCodeTokenizerName, FencedCodeTokenizer as default, match as fencedCodeMatch, parse as fencedCodeParse };
|
package/lib/types/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
export {
|
|
1
|
+
export { match as fencedCodeMatch } from './match';
|
|
2
|
+
export { parse as fencedCodeParse } from './parse';
|
|
3
|
+
export { FencedCodeTokenizer, FencedCodeTokenizer as default } from './tokenizer';
|
|
3
4
|
export { uniqueName as FencedCodeTokenizerName } from './types';
|
|
4
|
-
export type {
|
|
5
|
-
export default FencedCodeTokenizer;
|
|
5
|
+
export type { IHookContext as IFencedCodeHookContext, IToken as IFencedCodeToken, ITokenizerProps as IFencedCodeTokenizerProps, } from './types';
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { IMatchBlockHookCreator } from '@yozora/core-tokenizer';
|
|
2
|
+
import type { IHookContext, IToken, T } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* A code fence is a sequence of at least three consecutive backtick characters
|
|
5
|
+
* (`) or tildes (~). (Tildes and backticks cannot be mixed.) A fenced code
|
|
6
|
+
* block begins with a code fence, indented no more than three spaces.
|
|
7
|
+
*
|
|
8
|
+
* @see https://github.com/syntax-tree/mdast#code
|
|
9
|
+
* @see https://github.github.com/gfm/#code-fence
|
|
10
|
+
*/
|
|
11
|
+
export declare const match: IMatchBlockHookCreator<T, IToken, IHookContext>;
|
package/lib/types/tokenizer.d.ts
CHANGED
|
@@ -1,22 +1,13 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { IBlockTokenizer, IMatchBlockHookCreator, IParseBlockHookCreator } from '@yozora/core-tokenizer';
|
|
2
2
|
import FencedBlockTokenizer from '@yozora/tokenizer-fenced-block';
|
|
3
|
-
import type {
|
|
3
|
+
import type { IHookContext, INode, IToken, ITokenizerProps, T } from './types';
|
|
4
4
|
/**
|
|
5
5
|
* Lexical Analyzer for FencedCode.
|
|
6
|
-
*
|
|
7
|
-
* A code fence is a sequence of at least three consecutive backtick characters
|
|
8
|
-
* (`) or tildes (~). (Tildes and backticks cannot be mixed.) A fenced code
|
|
9
|
-
* block begins with a code fence, indented no more than three spaces.
|
|
10
|
-
*
|
|
11
6
|
* @see https://github.com/syntax-tree/mdast#code
|
|
12
7
|
* @see https://github.github.com/gfm/#code-fence
|
|
13
8
|
*/
|
|
14
|
-
export declare class FencedCodeTokenizer extends FencedBlockTokenizer<T> implements
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
* @override
|
|
19
|
-
* @see TokenizerParseBlockHook
|
|
20
|
-
*/
|
|
21
|
-
parseBlock(token: Token): ResultOfParse<T, Node>;
|
|
9
|
+
export declare class FencedCodeTokenizer extends FencedBlockTokenizer<T, INode, IHookContext> implements IBlockTokenizer<T, IToken, INode, IHookContext> {
|
|
10
|
+
constructor(props?: ITokenizerProps);
|
|
11
|
+
readonly match: IMatchBlockHookCreator<T, IToken, IHookContext>;
|
|
12
|
+
readonly parse: IParseBlockHookCreator<T, IToken, INode, IHookContext>;
|
|
22
13
|
}
|
package/lib/types/types.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
1
|
+
import type { CodeType, ICode } from '@yozora/ast';
|
|
2
|
+
import type { IBaseBlockTokenizerProps } from '@yozora/core-tokenizer';
|
|
3
|
+
import type { IFencedBlockHookContext, IFencedBlockToken } from '@yozora/tokenizer-fenced-block';
|
|
4
4
|
export declare type T = CodeType;
|
|
5
|
-
export declare type
|
|
5
|
+
export declare type INode = ICode;
|
|
6
6
|
export declare const uniqueName = "@yozora/tokenizer-fenced-code";
|
|
7
|
-
export declare type
|
|
8
|
-
export declare type
|
|
7
|
+
export declare type IToken = IFencedBlockToken<T>;
|
|
8
|
+
export declare type IHookContext = IFencedBlockHookContext<T>;
|
|
9
|
+
export declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yozora/tokenizer-fenced-code",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0-alpha.0",
|
|
4
4
|
"author": {
|
|
5
5
|
"name": "guanghechen",
|
|
6
6
|
"url": "https://github.com/guanghechen/"
|
|
@@ -35,10 +35,10 @@
|
|
|
35
35
|
"test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@yozora/ast": "^
|
|
39
|
-
"@yozora/character": "^
|
|
40
|
-
"@yozora/core-tokenizer": "^
|
|
41
|
-
"@yozora/tokenizer-fenced-block": "^
|
|
38
|
+
"@yozora/ast": "^2.0.0-alpha.0",
|
|
39
|
+
"@yozora/character": "^2.0.0-alpha.0",
|
|
40
|
+
"@yozora/core-tokenizer": "^2.0.0-alpha.0",
|
|
41
|
+
"@yozora/tokenizer-fenced-block": "^2.0.0-alpha.0"
|
|
42
42
|
},
|
|
43
|
-
"gitHead": "
|
|
43
|
+
"gitHead": "0171501339c49ffd02ed16a63447fa20a47a29a7"
|
|
44
44
|
}
|