@yozora/tokenizer-indented-code 1.2.0 → 2.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/index.js +30 -16
- package/lib/esm/index.js +29 -17
- package/lib/types/index.d.ts +4 -4
- package/lib/types/match.d.ts +12 -0
- package/lib/types/parse.d.ts +3 -0
- package/lib/types/tokenizer.d.ts +6 -27
- package/lib/types/types.d.ts +7 -6
- package/package.json +5 -5
package/lib/cjs/index.js
CHANGED
|
@@ -6,18 +6,13 @@ var ast = require('@yozora/ast');
|
|
|
6
6
|
var character = require('@yozora/character');
|
|
7
7
|
var coreTokenizer = require('@yozora/core-tokenizer');
|
|
8
8
|
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
|
|
17
|
-
});
|
|
18
|
-
this.isContainingBlock = false;
|
|
19
|
-
}
|
|
20
|
-
eatOpener(line) {
|
|
9
|
+
const match = function () {
|
|
10
|
+
return {
|
|
11
|
+
isContainingBlock: false,
|
|
12
|
+
eatOpener,
|
|
13
|
+
eatContinuationText,
|
|
14
|
+
};
|
|
15
|
+
function eatOpener(line) {
|
|
21
16
|
if (line.countOfPrecedeSpaces < 4)
|
|
22
17
|
return null;
|
|
23
18
|
const { nodePoints, startIndex, firstNonWhitespaceIndex, endIndex } = line;
|
|
@@ -50,8 +45,8 @@ class IndentedCodeTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
50
45
|
};
|
|
51
46
|
return { token, nextIndex };
|
|
52
47
|
}
|
|
53
|
-
eatContinuationText(line, token) {
|
|
54
|
-
const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex, countOfPrecedeSpaces
|
|
48
|
+
function eatContinuationText(line, token) {
|
|
49
|
+
const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex, countOfPrecedeSpaces } = line;
|
|
55
50
|
if (countOfPrecedeSpaces < 4 && firstNonWhitespaceIndex < endIndex)
|
|
56
51
|
return { status: 'notMatched' };
|
|
57
52
|
const firstIndex = Math.min(endIndex - 1, startIndex + 4);
|
|
@@ -64,7 +59,10 @@ class IndentedCodeTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
64
59
|
});
|
|
65
60
|
return { status: 'opening', nextIndex: endIndex };
|
|
66
61
|
}
|
|
67
|
-
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const parse = () => ({
|
|
65
|
+
parse: token => {
|
|
68
66
|
const { lines } = token;
|
|
69
67
|
let startLineIndex = 0, endLineIndex = lines.length;
|
|
70
68
|
for (; startLineIndex < endLineIndex; ++startLineIndex) {
|
|
@@ -83,9 +81,25 @@ class IndentedCodeTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
|
83
81
|
value: character.calcStringFromNodePoints(contents),
|
|
84
82
|
};
|
|
85
83
|
return node;
|
|
84
|
+
},
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
const uniqueName = '@yozora/tokenizer-indented-code';
|
|
88
|
+
|
|
89
|
+
class IndentedCodeTokenizer extends coreTokenizer.BaseBlockTokenizer {
|
|
90
|
+
constructor(props = {}) {
|
|
91
|
+
var _a, _b;
|
|
92
|
+
super({
|
|
93
|
+
name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
|
|
94
|
+
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.ATOMIC,
|
|
95
|
+
});
|
|
96
|
+
this.match = match;
|
|
97
|
+
this.parse = parse;
|
|
86
98
|
}
|
|
87
99
|
}
|
|
88
100
|
|
|
89
101
|
exports.IndentedCodeTokenizer = IndentedCodeTokenizer;
|
|
90
102
|
exports.IndentedCodeTokenizerName = uniqueName;
|
|
91
|
-
exports[
|
|
103
|
+
exports["default"] = IndentedCodeTokenizer;
|
|
104
|
+
exports.indentedCodeMatch = match;
|
|
105
|
+
exports.indentedCodeParse = parse;
|
package/lib/esm/index.js
CHANGED
|
@@ -1,19 +1,14 @@
|
|
|
1
1
|
import { CodeType } from '@yozora/ast';
|
|
2
2
|
import { AsciiCodePoint, VirtualCodePoint, calcStringFromNodePoints } from '@yozora/character';
|
|
3
|
-
import {
|
|
3
|
+
import { calcStartYastNodePoint, calcEndYastNodePoint, mergeContentLinesFaithfully, BaseBlockTokenizer, TokenizerPriority } from '@yozora/core-tokenizer';
|
|
4
4
|
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
|
|
13
|
-
});
|
|
14
|
-
this.isContainingBlock = false;
|
|
15
|
-
}
|
|
16
|
-
eatOpener(line) {
|
|
5
|
+
const match = function () {
|
|
6
|
+
return {
|
|
7
|
+
isContainingBlock: false,
|
|
8
|
+
eatOpener,
|
|
9
|
+
eatContinuationText,
|
|
10
|
+
};
|
|
11
|
+
function eatOpener(line) {
|
|
17
12
|
if (line.countOfPrecedeSpaces < 4)
|
|
18
13
|
return null;
|
|
19
14
|
const { nodePoints, startIndex, firstNonWhitespaceIndex, endIndex } = line;
|
|
@@ -46,8 +41,8 @@ class IndentedCodeTokenizer extends BaseBlockTokenizer {
|
|
|
46
41
|
};
|
|
47
42
|
return { token, nextIndex };
|
|
48
43
|
}
|
|
49
|
-
eatContinuationText(line, token) {
|
|
50
|
-
const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex, countOfPrecedeSpaces
|
|
44
|
+
function eatContinuationText(line, token) {
|
|
45
|
+
const { nodePoints, startIndex, endIndex, firstNonWhitespaceIndex, countOfPrecedeSpaces } = line;
|
|
51
46
|
if (countOfPrecedeSpaces < 4 && firstNonWhitespaceIndex < endIndex)
|
|
52
47
|
return { status: 'notMatched' };
|
|
53
48
|
const firstIndex = Math.min(endIndex - 1, startIndex + 4);
|
|
@@ -60,7 +55,10 @@ class IndentedCodeTokenizer extends BaseBlockTokenizer {
|
|
|
60
55
|
});
|
|
61
56
|
return { status: 'opening', nextIndex: endIndex };
|
|
62
57
|
}
|
|
63
|
-
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const parse = () => ({
|
|
61
|
+
parse: token => {
|
|
64
62
|
const { lines } = token;
|
|
65
63
|
let startLineIndex = 0, endLineIndex = lines.length;
|
|
66
64
|
for (; startLineIndex < endLineIndex; ++startLineIndex) {
|
|
@@ -79,7 +77,21 @@ class IndentedCodeTokenizer extends BaseBlockTokenizer {
|
|
|
79
77
|
value: calcStringFromNodePoints(contents),
|
|
80
78
|
};
|
|
81
79
|
return node;
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const uniqueName = '@yozora/tokenizer-indented-code';
|
|
84
|
+
|
|
85
|
+
class IndentedCodeTokenizer extends BaseBlockTokenizer {
|
|
86
|
+
constructor(props = {}) {
|
|
87
|
+
var _a, _b;
|
|
88
|
+
super({
|
|
89
|
+
name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
|
|
90
|
+
priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.ATOMIC,
|
|
91
|
+
});
|
|
92
|
+
this.match = match;
|
|
93
|
+
this.parse = parse;
|
|
82
94
|
}
|
|
83
95
|
}
|
|
84
96
|
|
|
85
|
-
export { IndentedCodeTokenizer, uniqueName as IndentedCodeTokenizerName, IndentedCodeTokenizer as default };
|
|
97
|
+
export { IndentedCodeTokenizer, uniqueName as IndentedCodeTokenizerName, IndentedCodeTokenizer as default, match as indentedCodeMatch, parse as indentedCodeParse };
|
package/lib/types/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
export {
|
|
1
|
+
export { match as indentedCodeMatch } from './match';
|
|
2
|
+
export { parse as indentedCodeParse } from './parse';
|
|
3
|
+
export { IndentedCodeTokenizer, IndentedCodeTokenizer as default } from './tokenizer';
|
|
3
4
|
export { uniqueName as IndentedCodeTokenizerName } from './types';
|
|
4
|
-
export type {
|
|
5
|
-
export default IndentedCodeTokenizer;
|
|
5
|
+
export type { IHookContext as IIndentedCodeHookContext, IToken as IIndentedCodeToken, ITokenizerProps as IIndentedCodeTokenizerProps, } from './types';
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { IMatchBlockHookCreator } from '@yozora/core-tokenizer';
|
|
2
|
+
import type { IHookContext, IToken, T } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* An indented code block is composed of one or more indented chunks
|
|
5
|
+
* separated by blank lines. An indented chunk is a sequence of non-blank
|
|
6
|
+
* lines, each indented four or more spaces. The contents of the code block
|
|
7
|
+
* are the literal contents of the lines, including trailing line endings,
|
|
8
|
+
* minus four spaces of indentation.
|
|
9
|
+
*
|
|
10
|
+
* @see https://github.github.com/gfm/#indented-code-block
|
|
11
|
+
*/
|
|
12
|
+
export declare const match: IMatchBlockHookCreator<T, IToken, IHookContext>;
|
package/lib/types/tokenizer.d.ts
CHANGED
|
@@ -1,33 +1,12 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { IBlockTokenizer, IMatchBlockHookCreator, IParseBlockHookCreator } from '@yozora/core-tokenizer';
|
|
2
2
|
import { BaseBlockTokenizer } from '@yozora/core-tokenizer';
|
|
3
|
-
import type {
|
|
3
|
+
import type { IHookContext, INode, IToken, ITokenizerProps, T } from './types';
|
|
4
4
|
/**
|
|
5
5
|
* Lexical Analyzer for IndentedCode.
|
|
6
|
-
*
|
|
7
|
-
* An indented code block is composed of one or more indented chunks
|
|
8
|
-
* separated by blank lines. An indented chunk is a sequence of non-blank
|
|
9
|
-
* lines, each indented four or more spaces. The contents of the code block
|
|
10
|
-
* are the literal contents of the lines, including trailing line endings,
|
|
11
|
-
* minus four spaces of indentation.
|
|
12
|
-
*
|
|
13
6
|
* @see https://github.github.com/gfm/#indented-code-block
|
|
14
7
|
*/
|
|
15
|
-
export declare class IndentedCodeTokenizer extends BaseBlockTokenizer
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
* @override
|
|
20
|
-
* @see TokenizerMatchBlockHook
|
|
21
|
-
*/
|
|
22
|
-
eatOpener(line: Readonly<PhrasingContentLine>): ResultOfEatOpener<T, Token>;
|
|
23
|
-
/**
|
|
24
|
-
* @override
|
|
25
|
-
* @see TokenizerMatchBlockHook
|
|
26
|
-
*/
|
|
27
|
-
eatContinuationText(line: Readonly<PhrasingContentLine>, token: Token): ResultOfEatContinuationText;
|
|
28
|
-
/**
|
|
29
|
-
* @override
|
|
30
|
-
* @see TokenizerParseBlockHook
|
|
31
|
-
*/
|
|
32
|
-
parseBlock(token: Readonly<Token>): ResultOfParse<T, Node>;
|
|
8
|
+
export declare class IndentedCodeTokenizer extends BaseBlockTokenizer<T, IToken, INode, IHookContext> implements IBlockTokenizer<T, IToken, INode, IHookContext> {
|
|
9
|
+
constructor(props?: ITokenizerProps);
|
|
10
|
+
readonly match: IMatchBlockHookCreator<T, IToken, IHookContext>;
|
|
11
|
+
readonly parse: IParseBlockHookCreator<T, IToken, INode, IHookContext>;
|
|
33
12
|
}
|
package/lib/types/types.d.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
1
|
+
import type { CodeType, ICode } from '@yozora/ast';
|
|
2
|
+
import type { IBaseBlockTokenizerProps, IPartialYastBlockToken, IPhrasingContentLine, ITokenizer } from '@yozora/core-tokenizer';
|
|
3
3
|
export declare type T = CodeType;
|
|
4
|
-
export declare type
|
|
4
|
+
export declare type INode = ICode;
|
|
5
5
|
export declare const uniqueName = "@yozora/tokenizer-indented-code";
|
|
6
|
-
export interface
|
|
6
|
+
export interface IToken extends IPartialYastBlockToken<T> {
|
|
7
7
|
/**
|
|
8
8
|
* Lines to construct the contents of a paragraph.
|
|
9
9
|
*/
|
|
10
|
-
lines:
|
|
10
|
+
lines: IPhrasingContentLine[];
|
|
11
11
|
}
|
|
12
|
-
export declare type
|
|
12
|
+
export declare type IHookContext = ITokenizer;
|
|
13
|
+
export declare type ITokenizerProps = Partial<IBaseBlockTokenizerProps>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yozora/tokenizer-indented-code",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0-alpha.0",
|
|
4
4
|
"author": {
|
|
5
5
|
"name": "guanghechen",
|
|
6
6
|
"url": "https://github.com/guanghechen/"
|
|
@@ -35,9 +35,9 @@
|
|
|
35
35
|
"test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@yozora/ast": "^
|
|
39
|
-
"@yozora/character": "^
|
|
40
|
-
"@yozora/core-tokenizer": "^
|
|
38
|
+
"@yozora/ast": "^2.0.0-alpha.0",
|
|
39
|
+
"@yozora/character": "^2.0.0-alpha.0",
|
|
40
|
+
"@yozora/core-tokenizer": "^2.0.0-alpha.0"
|
|
41
41
|
},
|
|
42
|
-
"gitHead": "
|
|
42
|
+
"gitHead": "0171501339c49ffd02ed16a63447fa20a47a29a7"
|
|
43
43
|
}
|