@lvce-editor/shared-process 0.0.14 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/builtin.language-basics-cpp/README.md +14 -0
- package/extensions/builtin.language-basics-cpp/extension.json +13 -0
- package/extensions/builtin.language-basics-cpp/src/tokenizeCpp.js +222 -0
- package/extensions/builtin.language-basics-css/README.md +14 -0
- package/extensions/builtin.language-basics-css/extension.json +12 -0
- package/extensions/builtin.language-basics-css/src/tokenizeCss.js +378 -0
- package/extensions/builtin.language-basics-dart/README.md +14 -0
- package/extensions/builtin.language-basics-dart/extension.json +12 -0
- package/extensions/builtin.language-basics-dart/src/tokenizeDart.js +132 -0
- package/extensions/builtin.language-basics-dotenv/README.md +14 -0
- package/extensions/builtin.language-basics-dotenv/extension.json +12 -0
- package/extensions/builtin.language-basics-dotenv/src/tokenizeDotenv.js +297 -0
- package/extensions/builtin.language-basics-elixir/README.md +15 -0
- package/extensions/builtin.language-basics-elixir/extension.json +12 -0
- package/extensions/builtin.language-basics-elixir/src/tokenizeElixir.js +150 -0
- package/extensions/builtin.language-basics-go/README.md +14 -0
- package/extensions/builtin.language-basics-go/extension.json +12 -0
- package/extensions/builtin.language-basics-go/src/tokenizeGo.js +161 -0
- package/extensions/builtin.language-basics-html/README.md +14 -0
- package/extensions/builtin.language-basics-html/extension.json +13 -0
- package/extensions/builtin.language-basics-html/src/tokenizeHtml.js +278 -0
- package/extensions/builtin.language-basics-java/README.md +14 -0
- package/extensions/builtin.language-basics-java/extension.json +12 -0
- package/extensions/builtin.language-basics-java/src/tokenizeJava.js +135 -0
- package/extensions/builtin.language-basics-javascript/README.md +14 -0
- package/extensions/builtin.language-basics-javascript/extension.json +13 -0
- package/extensions/builtin.language-basics-javascript/src/tokenizeJavaScript.js +295 -0
- package/extensions/builtin.language-basics-json/README.md +14 -0
- package/extensions/builtin.language-basics-json/extension.json +12 -0
- package/extensions/builtin.language-basics-json/src/tokenizeJson.js +323 -0
- package/extensions/builtin.language-basics-julia/README.md +18 -0
- package/extensions/builtin.language-basics-julia/extension.json +12 -0
- package/extensions/builtin.language-basics-julia/src/tokenizeJulia.js +135 -0
- package/extensions/builtin.language-basics-kotlin/README.md +15 -0
- package/extensions/builtin.language-basics-kotlin/extension.json +12 -0
- package/extensions/builtin.language-basics-kotlin/src/tokenizeKotlin.js +127 -0
- package/extensions/builtin.language-basics-python/README.md +14 -0
- package/extensions/builtin.language-basics-python/extension.json +13 -0
- package/extensions/builtin.language-basics-python/src/tokenizePython.js +201 -0
- package/extensions/builtin.language-basics-shellscript/README.md +14 -0
- package/extensions/builtin.language-basics-shellscript/extension.json +12 -0
- package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js +127 -0
- package/package.json +3 -3
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @enum number
|
|
3
|
+
*/
|
|
4
|
+
const State = {
|
|
5
|
+
None: 0,
|
|
6
|
+
TopLevelContent: 1,
|
|
7
|
+
InsideSingleQuoteString: 2,
|
|
8
|
+
InsideDoubleQuoteString: 3,
|
|
9
|
+
InsideTripleQuoteString: 4,
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @enum number
|
|
14
|
+
*/
|
|
15
|
+
export const TokenType = {
|
|
16
|
+
None: 99999999,
|
|
17
|
+
Keyword: 951,
|
|
18
|
+
Whitespace: 0,
|
|
19
|
+
NewLine: 771,
|
|
20
|
+
VariableName: 2,
|
|
21
|
+
Punctuation: 3,
|
|
22
|
+
String: 4,
|
|
23
|
+
Numeric: 5,
|
|
24
|
+
Attribute: 6,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export const TokenMap = {
|
|
28
|
+
[TokenType.None]: 'None',
|
|
29
|
+
[TokenType.Keyword]: 'Keyword',
|
|
30
|
+
[TokenType.Whitespace]: 'Whitespace',
|
|
31
|
+
[TokenType.NewLine]: 'NewLine',
|
|
32
|
+
[TokenType.VariableName]: 'VariableName',
|
|
33
|
+
[TokenType.Punctuation]: 'Punctuation',
|
|
34
|
+
[TokenType.String]: 'String',
|
|
35
|
+
[TokenType.Numeric]: 'Numeric',
|
|
36
|
+
[TokenType.Attribute]: 'Attribute',
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export const initialLineState = {
|
|
40
|
+
state: State.TopLevelContent,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const RE_KEYWORD =
|
|
44
|
+
/^(?:baremodule|begin|break|catch|ccall|const|continue|do|else|elseif|end|export|false|finally|for|function|global|if|import|in|isa|let|local|macro|module|quote|return|true|try|using|where|while)\b/
|
|
45
|
+
const RE_WHITESPACE = /^\s+/
|
|
46
|
+
const RE_VARIABLE_NAME = /^[A-Za-z_\u00A1-\uFFFF][A-Za-z_0-9\u00A1-\uFFFF]*/
|
|
47
|
+
const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>\^\+\-\<\?]/
|
|
48
|
+
const RE_QUOTE_SINGLE = /^'/
|
|
49
|
+
const RE_QUOTE_DOUBLE = /^"/
|
|
50
|
+
const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
|
|
51
|
+
const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
|
|
52
|
+
const RE_NUMERIC = /^\d+/
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* @param {string} line
|
|
56
|
+
* @param {any} lineState
|
|
57
|
+
*/
|
|
58
|
+
export const tokenizeLine = (line, lineState) => {
|
|
59
|
+
let next = null
|
|
60
|
+
let index = 0
|
|
61
|
+
let tokens = []
|
|
62
|
+
let token = TokenType.None
|
|
63
|
+
let state = lineState.state
|
|
64
|
+
while (index < line.length) {
|
|
65
|
+
const part = line.slice(index)
|
|
66
|
+
switch (state) {
|
|
67
|
+
case State.TopLevelContent:
|
|
68
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
69
|
+
token = TokenType.Whitespace
|
|
70
|
+
state = State.TopLevelContent
|
|
71
|
+
} else if ((next = part.match(RE_KEYWORD))) {
|
|
72
|
+
token = TokenType.Keyword
|
|
73
|
+
state = State.TopLevelContent
|
|
74
|
+
} else if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
75
|
+
token = TokenType.VariableName
|
|
76
|
+
state = State.TopLevelContent
|
|
77
|
+
} else if ((next = part.match(RE_PUNCTUATION))) {
|
|
78
|
+
token = TokenType.Punctuation
|
|
79
|
+
state = State.TopLevelContent
|
|
80
|
+
} else if ((next = part.match(RE_NUMERIC))) {
|
|
81
|
+
token = TokenType.Numeric
|
|
82
|
+
state = State.TopLevelContent
|
|
83
|
+
} else if ((next = part.match(RE_QUOTE_SINGLE))) {
|
|
84
|
+
token = TokenType.Punctuation
|
|
85
|
+
state = State.InsideSingleQuoteString
|
|
86
|
+
} else if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
87
|
+
token = TokenType.Punctuation
|
|
88
|
+
state = State.InsideDoubleQuoteString
|
|
89
|
+
} else {
|
|
90
|
+
part //?
|
|
91
|
+
throw new Error('no')
|
|
92
|
+
}
|
|
93
|
+
break
|
|
94
|
+
case State.InsideSingleQuoteString:
|
|
95
|
+
if ((next = part.match(RE_QUOTE_SINGLE))) {
|
|
96
|
+
token = TokenType.Punctuation
|
|
97
|
+
state = State.TopLevelContent
|
|
98
|
+
} else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
|
|
99
|
+
token = TokenType.String
|
|
100
|
+
state = State.InsideSingleQuoteString
|
|
101
|
+
} else {
|
|
102
|
+
throw new Error('no')
|
|
103
|
+
}
|
|
104
|
+
break
|
|
105
|
+
case State.InsideDoubleQuoteString:
|
|
106
|
+
if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
107
|
+
token = TokenType.Punctuation
|
|
108
|
+
state = State.TopLevelContent
|
|
109
|
+
} else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
|
|
110
|
+
token = TokenType.String
|
|
111
|
+
state = State.InsideDoubleQuoteString
|
|
112
|
+
} else {
|
|
113
|
+
throw new Error('no')
|
|
114
|
+
}
|
|
115
|
+
break
|
|
116
|
+
default:
|
|
117
|
+
state
|
|
118
|
+
throw new Error('no')
|
|
119
|
+
}
|
|
120
|
+
index += next[0].length
|
|
121
|
+
tokens.push({
|
|
122
|
+
type: token,
|
|
123
|
+
length: next[0].length,
|
|
124
|
+
})
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
state,
|
|
128
|
+
tokens,
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
tokenizeLine(
|
|
133
|
+
`abs(mandelbrot(complex(x, y))) < 2 ? print("*") : print(" ")`,
|
|
134
|
+
initialLineState
|
|
135
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Language Basics Kotlin
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
## Contributing
|
|
5
|
+
|
|
6
|
+
```sh
|
|
7
|
+
git clone git@github.com:lvce-editor/language-basics-kotlin.git &&
|
|
8
|
+
cd language-basics-kotlin &&
|
|
9
|
+
npm ci &&
|
|
10
|
+
npm test
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Gitpod
|
|
14
|
+
|
|
15
|
+
[](https://gitpod.io/#https://github.com/lvce-editor/language-basics-kotlin)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "builtin.language-basics-kotlin",
|
|
3
|
+
"name": "Language Basics Kotlin",
|
|
4
|
+
"description": "Provides syntax highlighting and bracket matching in Kotlin files.",
|
|
5
|
+
"languages": [
|
|
6
|
+
{
|
|
7
|
+
"id": "kotlin",
|
|
8
|
+
"extensions": [".kt"],
|
|
9
|
+
"tokenize": "src/tokenizeKotlin.js"
|
|
10
|
+
}
|
|
11
|
+
]
|
|
12
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @enum number
|
|
3
|
+
*/
|
|
4
|
+
export const State = {
|
|
5
|
+
TopLevelContent: 1,
|
|
6
|
+
InsideString: 2,
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export const StateMap = {
|
|
10
|
+
[State.TopLevelContent]: 'TopLevelContent',
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @enum number
|
|
15
|
+
*/
|
|
16
|
+
export const TokenType = {
|
|
17
|
+
None: 1,
|
|
18
|
+
Whitespace: 2,
|
|
19
|
+
PunctuationString: 3,
|
|
20
|
+
String: 4,
|
|
21
|
+
Keyword: 5,
|
|
22
|
+
Numeric: 6,
|
|
23
|
+
Punctuation: 7,
|
|
24
|
+
VariableName: 8,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export const TokenMap = {
|
|
28
|
+
[TokenType.None]: 'None',
|
|
29
|
+
[TokenType.Whitespace]: 'Whitespace',
|
|
30
|
+
[TokenType.PunctuationString]: 'PunctuationString',
|
|
31
|
+
[TokenType.String]: 'String',
|
|
32
|
+
[TokenType.Keyword]: 'Keyword',
|
|
33
|
+
[TokenType.Numeric]: 'Numeric',
|
|
34
|
+
[TokenType.Punctuation]: 'Punctuation',
|
|
35
|
+
[TokenType.VariableName]: 'VariableName',
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
|
|
39
|
+
const RE_WHITESPACE = /^ +/
|
|
40
|
+
const RE_CURLY_OPEN = /^\{/
|
|
41
|
+
const RE_CURLY_CLOSE = /^\}/
|
|
42
|
+
const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
|
|
43
|
+
const RE_COLON = /^:/
|
|
44
|
+
const RE_PROPERTY_VALUE = /^[^;\}]+/
|
|
45
|
+
const RE_SEMICOLON = /^;/
|
|
46
|
+
const RE_COMMA = /^,/
|
|
47
|
+
const RE_ANYTHING = /^.*/
|
|
48
|
+
const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
|
|
49
|
+
const RE_QUOTE_DOUBLE = /^"/
|
|
50
|
+
const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
|
|
51
|
+
const RE_KEYWORD =
|
|
52
|
+
/^(?:while|when|var|val|typeof|typealias|try|true|throw|this|super|return|package|object|null|is|interface|in|if|fun|for|false|else|do|continue|class|break|as)\b/
|
|
53
|
+
|
|
54
|
+
const RE_VARIABLE_NAME = /^[a-zA-Z]+/
|
|
55
|
+
const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
|
|
56
|
+
const RE_NUMERIC = /^\d+/
|
|
57
|
+
|
|
58
|
+
export const initialLineState = {
|
|
59
|
+
state: State.TopLevelContent,
|
|
60
|
+
tokens: [],
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export const isEqualLineState = (lineStateA, lineStateB) => {
|
|
64
|
+
return lineStateA.state === lineStateB.state
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* @param {string} line
|
|
69
|
+
*/
|
|
70
|
+
export const tokenizeLine = (line, lineState) => {
|
|
71
|
+
let next = null
|
|
72
|
+
let index = 0
|
|
73
|
+
let tokens = []
|
|
74
|
+
let token = TokenType.None
|
|
75
|
+
let state = lineState.state
|
|
76
|
+
while (index < line.length) {
|
|
77
|
+
const part = line.slice(index)
|
|
78
|
+
switch (state) {
|
|
79
|
+
case State.TopLevelContent:
|
|
80
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
81
|
+
token = TokenType.Whitespace
|
|
82
|
+
state = State.TopLevelContent
|
|
83
|
+
} else if ((next = part.match(RE_KEYWORD))) {
|
|
84
|
+
token = TokenType.Keyword
|
|
85
|
+
state = State.TopLevelContent
|
|
86
|
+
} else if ((next = part.match(RE_PUNCTUATION))) {
|
|
87
|
+
token = TokenType.Punctuation
|
|
88
|
+
state = State.TopLevelContent
|
|
89
|
+
} else if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
90
|
+
token = TokenType.VariableName
|
|
91
|
+
state = State.TopLevelContent
|
|
92
|
+
} else if ((next = part.match(RE_NUMERIC))) {
|
|
93
|
+
token = TokenType.Numeric
|
|
94
|
+
state = State.TopLevelContent
|
|
95
|
+
} else if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
96
|
+
token = TokenType.PunctuationString
|
|
97
|
+
state = State.InsideString
|
|
98
|
+
} else {
|
|
99
|
+
part //?
|
|
100
|
+
throw new Error('no')
|
|
101
|
+
}
|
|
102
|
+
break
|
|
103
|
+
case State.InsideString:
|
|
104
|
+
if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
105
|
+
token = TokenType.PunctuationString
|
|
106
|
+
state = State.TopLevelContent
|
|
107
|
+
} else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
|
|
108
|
+
token = TokenType.String
|
|
109
|
+
state = State.InsideString
|
|
110
|
+
} else {
|
|
111
|
+
throw new Error('no')
|
|
112
|
+
}
|
|
113
|
+
break
|
|
114
|
+
default:
|
|
115
|
+
throw new Error('no')
|
|
116
|
+
}
|
|
117
|
+
index += next[0].length
|
|
118
|
+
tokens.push({
|
|
119
|
+
type: token,
|
|
120
|
+
length: next[0].length,
|
|
121
|
+
})
|
|
122
|
+
}
|
|
123
|
+
return {
|
|
124
|
+
state,
|
|
125
|
+
tokens,
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Language Basics Python
|
|
2
|
+
|
|
3
|
+
## Contributing
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
git clone git@github.com:lvce-editor/language-basics-python.git &&
|
|
7
|
+
cd language-basics-python &&
|
|
8
|
+
npm ci &&
|
|
9
|
+
npm test
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Gitpod
|
|
13
|
+
|
|
14
|
+
[](https://gitpod.io/#https://github.com/lvce-editor/language-basics-python)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "builtin.language-basics-python",
|
|
3
|
+
"name": "Language Basics Python",
|
|
4
|
+
"description": "Provides syntax highlighting and bracket matching in Python files.",
|
|
5
|
+
"languages": [
|
|
6
|
+
{
|
|
7
|
+
"id": "python",
|
|
8
|
+
"extensions": [".py"],
|
|
9
|
+
"configuration": "./languageConfiguration.json",
|
|
10
|
+
"tokenize": "src/tokenizePython.js"
|
|
11
|
+
}
|
|
12
|
+
]
|
|
13
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @enum number
|
|
3
|
+
*/
|
|
4
|
+
export const State = {
|
|
5
|
+
None: 0,
|
|
6
|
+
TopLevelContent: 1,
|
|
7
|
+
AfterOpeningAngleBracket: 2,
|
|
8
|
+
InsideOpeningTag: 3,
|
|
9
|
+
InsideOpeningTagAndHasSeenWhitespace: 4,
|
|
10
|
+
AfterClosingTagAngleBrackets: 5,
|
|
11
|
+
AfterClosingTagName: 6,
|
|
12
|
+
AfterAttributeName: 7,
|
|
13
|
+
AfterAttributeEqualSign: 8,
|
|
14
|
+
InsideAttributeDoubleQuote: 9,
|
|
15
|
+
InsideBlockComment: 10,
|
|
16
|
+
InsideDoubleQuoteString: 11,
|
|
17
|
+
InsideLineComment: 12,
|
|
18
|
+
InsideSingleQuoteString: 13,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @enum number
|
|
23
|
+
*/
|
|
24
|
+
export const TokenType = {
|
|
25
|
+
None: 99999999,
|
|
26
|
+
Numeric: 30,
|
|
27
|
+
String: 50,
|
|
28
|
+
Whitespace: 0,
|
|
29
|
+
Comment: 60,
|
|
30
|
+
Text: 117,
|
|
31
|
+
PunctuationTag: 228,
|
|
32
|
+
TagName: 118,
|
|
33
|
+
AttributeName: 119,
|
|
34
|
+
Punctuation: 10,
|
|
35
|
+
Error: 141,
|
|
36
|
+
PunctuationString: 11,
|
|
37
|
+
NewLine: 771,
|
|
38
|
+
Keyword: 951,
|
|
39
|
+
VariableName: 952,
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export const TokenMap = {
|
|
43
|
+
[TokenType.None]: 'None',
|
|
44
|
+
[TokenType.Numeric]: 'Numeric',
|
|
45
|
+
[TokenType.String]: 'String',
|
|
46
|
+
[TokenType.Whitespace]: 'Whitespace',
|
|
47
|
+
[TokenType.Comment]: 'Comment',
|
|
48
|
+
[TokenType.Text]: 'Text',
|
|
49
|
+
[TokenType.PunctuationTag]: 'PunctuationTag',
|
|
50
|
+
[TokenType.TagName]: 'TagName',
|
|
51
|
+
[TokenType.AttributeName]: 'AttributeName',
|
|
52
|
+
[TokenType.Punctuation]: 'Punctuation',
|
|
53
|
+
[TokenType.Error]: 'Error',
|
|
54
|
+
[TokenType.PunctuationString]: 'PunctuationString',
|
|
55
|
+
[TokenType.NewLine]: 'NewLine',
|
|
56
|
+
[TokenType.Keyword]: 'Keyword',
|
|
57
|
+
[TokenType.VariableName]: 'VariableName',
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const RE_WHITESPACE = /^\s+/
|
|
61
|
+
const RE_DOUBLE_QUOTE = /^"/
|
|
62
|
+
const RE_SINGLE_QUOTE = /^'/
|
|
63
|
+
const RE_ANGLE_BRACKET_OPEN = /^<(?!\s)/
|
|
64
|
+
const RE_ANGLE_BRACKET_CLOSE = /^>/
|
|
65
|
+
const RE_TAGNAME = /^[!\w]+/
|
|
66
|
+
const RE_TEXT = /^.+/
|
|
67
|
+
const RE_ANY_TEXT = /^[^\n]+/
|
|
68
|
+
const RE_SLASH = /^\//
|
|
69
|
+
const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-]+/
|
|
70
|
+
const RE_EQUAL_SIGN = /^=/
|
|
71
|
+
const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
|
|
72
|
+
const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
|
|
73
|
+
const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
|
|
74
|
+
const RE_INVALID_INSIDE_OPENING_TAG = /^[^a-zA-Z>]/
|
|
75
|
+
const RE_INVALID_INSIDE_ClOSING_TAG = /^[^a-zA-Z>]/
|
|
76
|
+
const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
|
|
77
|
+
const RE_WORD = /^[^\s]+/
|
|
78
|
+
const RE_ANGLE_BRACKET_ONLY = /^</
|
|
79
|
+
const RE_NEWLINE = /^\n/
|
|
80
|
+
const RE_EXCLAMATION_MARK = /^!/
|
|
81
|
+
const RE_SELF_CLOSING = /^\/>/
|
|
82
|
+
const RE_DASH_DASH = /^\-\-/
|
|
83
|
+
const RE_TAG_TEXT = /^[^\s>]+/
|
|
84
|
+
const RE_LINE_COMMENT = /^#/
|
|
85
|
+
const RE_NEWLINE_WHITESPACE = /^\n\s*/
|
|
86
|
+
const RE_BLOCK_COMMENT_START = /^\/\*/
|
|
87
|
+
const RE_BLOCK_COMMENT_CONTENT = /^.+(?=\*\/)/s
|
|
88
|
+
const RE_BLOCK_COMMENT_END = /^\*\//
|
|
89
|
+
const RE_UNKNOWN_VALUE = /^[^\}\{\s,"]+/
|
|
90
|
+
const RE_KEYWORD =
|
|
91
|
+
/^(?:__debug__|Ellipsis|yield|with|while|try|true|True|return|raise|pass|or|NotImplemented|not|nonlocal|None|lambda|is|in|import|if|global|from|for|finally|False|false|except|else|elif|del|def|continue|class|break|await|async|assert|as|and)\b/
|
|
92
|
+
const RE_FUNCTION_NAME = /^[a-zA-Z\d]+/
|
|
93
|
+
const RE_ROUND_OPEN = /^\(/
|
|
94
|
+
const RE_VARIABLE_NAME = /^[_a-zA-Z][a-zA-Z\d_]*/
|
|
95
|
+
const RE_ROUND_CLOSE = /^\)/
|
|
96
|
+
const RE_OPERATOR = /^(\+|\-|\*|\/)/
|
|
97
|
+
const RE_NUMERIC =
|
|
98
|
+
/^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
|
|
99
|
+
const RE_PUNCTUATION = /^[\(\)\{\}:,\+\-\*&=\/\\\[\]!\.<>%]+/
|
|
100
|
+
|
|
101
|
+
export const initialLineState = {
|
|
102
|
+
state: State.TopLevelContent,
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// TODO maybe reverse order of parameters -> line, context (context may be optional)
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
*
|
|
109
|
+
* @param {string} line
|
|
110
|
+
* @param {{state: State}} lineState
|
|
111
|
+
* @returns
|
|
112
|
+
*/
|
|
113
|
+
export const tokenizeLine = (line, lineState) => {
|
|
114
|
+
let next = null
|
|
115
|
+
let index = 0
|
|
116
|
+
let tokens = []
|
|
117
|
+
let token = TokenType.None
|
|
118
|
+
let state = lineState.state
|
|
119
|
+
while (index < line.length) {
|
|
120
|
+
const part = line.slice(index)
|
|
121
|
+
switch (state) {
|
|
122
|
+
case State.TopLevelContent:
|
|
123
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
124
|
+
token = TokenType.Whitespace
|
|
125
|
+
state = State.TopLevelContent
|
|
126
|
+
} else if ((next = part.match(RE_KEYWORD))) {
|
|
127
|
+
token = TokenType.Keyword
|
|
128
|
+
state = State.TopLevelContent
|
|
129
|
+
} else if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
130
|
+
token = TokenType.VariableName
|
|
131
|
+
state = State.TopLevelContent
|
|
132
|
+
} else if ((next = part.match(RE_PUNCTUATION))) {
|
|
133
|
+
token = TokenType.Punctuation
|
|
134
|
+
state = State.TopLevelContent
|
|
135
|
+
} else if ((next = part.match(RE_NUMERIC))) {
|
|
136
|
+
token = TokenType.Numeric
|
|
137
|
+
state = State.TopLevelContent
|
|
138
|
+
} else if ((next = part.match(RE_DOUBLE_QUOTE))) {
|
|
139
|
+
token = TokenType.PunctuationString
|
|
140
|
+
state = State.InsideDoubleQuoteString
|
|
141
|
+
} else if ((next = part.match(RE_SINGLE_QUOTE))) {
|
|
142
|
+
token = TokenType.PunctuationString
|
|
143
|
+
state = State.InsideSingleQuoteString
|
|
144
|
+
} else if ((next = part.match(RE_LINE_COMMENT))) {
|
|
145
|
+
token = TokenType.Comment
|
|
146
|
+
state = State.InsideLineComment
|
|
147
|
+
} else if ((next = part.match(RE_TEXT))) {
|
|
148
|
+
token = TokenType.Text
|
|
149
|
+
state = State.TopLevelContent
|
|
150
|
+
} else {
|
|
151
|
+
part //?
|
|
152
|
+
throw new Error('no')
|
|
153
|
+
}
|
|
154
|
+
break
|
|
155
|
+
case State.InsideDoubleQuoteString:
|
|
156
|
+
if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
|
|
157
|
+
token = TokenType.String
|
|
158
|
+
state = State.InsideDoubleQuoteString
|
|
159
|
+
} else if ((next = part.match(RE_DOUBLE_QUOTE))) {
|
|
160
|
+
token = TokenType.PunctuationString
|
|
161
|
+
state = State.TopLevelContent
|
|
162
|
+
} else {
|
|
163
|
+
throw new Error('no')
|
|
164
|
+
}
|
|
165
|
+
break
|
|
166
|
+
case State.InsideSingleQuoteString:
|
|
167
|
+
if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
|
|
168
|
+
token = TokenType.String
|
|
169
|
+
state = State.InsideSingleQuoteString
|
|
170
|
+
} else if ((next = part.match(RE_SINGLE_QUOTE))) {
|
|
171
|
+
token = TokenType.PunctuationString
|
|
172
|
+
state = State.TopLevelContent
|
|
173
|
+
} else {
|
|
174
|
+
throw new Error('no')
|
|
175
|
+
}
|
|
176
|
+
break
|
|
177
|
+
case State.InsideLineComment:
|
|
178
|
+
if ((next = part.match(RE_ANY_TEXT))) {
|
|
179
|
+
token = TokenType.Comment
|
|
180
|
+
state = State.TopLevelContent
|
|
181
|
+
} else {
|
|
182
|
+
throw new Error('no')
|
|
183
|
+
}
|
|
184
|
+
break
|
|
185
|
+
default:
|
|
186
|
+
state
|
|
187
|
+
throw new Error('no')
|
|
188
|
+
}
|
|
189
|
+
index += next[0].length
|
|
190
|
+
tokens.push({
|
|
191
|
+
type: token,
|
|
192
|
+
length: next[0].length,
|
|
193
|
+
})
|
|
194
|
+
}
|
|
195
|
+
return {
|
|
196
|
+
state,
|
|
197
|
+
tokens,
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// tokenizeLine(initialContext, `def add(a,b):`) //?
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Language Basics Shell Script
|
|
2
|
+
|
|
3
|
+
## Contributing
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
git clone git@github.com:lvce-editor/language-basics-shellscript.git &&
|
|
7
|
+
cd language-basics-shellscript &&
|
|
8
|
+
npm ci &&
|
|
9
|
+
npm test
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Gitpod
|
|
13
|
+
|
|
14
|
+
[](https://gitpod.io/#https://github.com/lvce-editor/language-basics-shellscript)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "builtin.language-basics-shellscript",
|
|
3
|
+
"name": "Language Basics Shell Script",
|
|
4
|
+
"description": "Provides syntax highlighting and bracket matching in Shell Script files.",
|
|
5
|
+
"languages": [
|
|
6
|
+
{
|
|
7
|
+
"id": "shellscript",
|
|
8
|
+
"extensions": [".sh"],
|
|
9
|
+
"tokenize": "src/tokenizeShellScript.js"
|
|
10
|
+
}
|
|
11
|
+
]
|
|
12
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @enum number
|
|
3
|
+
*/
|
|
4
|
+
export const State = {
|
|
5
|
+
TopLevelContent: 1,
|
|
6
|
+
InsideString: 2,
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export const StateMap = {
|
|
10
|
+
[State.TopLevelContent]: 'TopLevelContent',
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @enum number
|
|
15
|
+
*/
|
|
16
|
+
export const TokenType = {
|
|
17
|
+
None: 1,
|
|
18
|
+
Whitespace: 2,
|
|
19
|
+
PunctuationString: 3,
|
|
20
|
+
String: 4,
|
|
21
|
+
Keyword: 5,
|
|
22
|
+
Numeric: 6,
|
|
23
|
+
Punctuation: 7,
|
|
24
|
+
VariableName: 8,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export const TokenMap = {
|
|
28
|
+
[TokenType.None]: 'None',
|
|
29
|
+
[TokenType.Whitespace]: 'Whitespace',
|
|
30
|
+
[TokenType.PunctuationString]: 'PunctuationString',
|
|
31
|
+
[TokenType.String]: 'String',
|
|
32
|
+
[TokenType.Keyword]: 'Keyword',
|
|
33
|
+
[TokenType.Numeric]: 'Numeric',
|
|
34
|
+
[TokenType.Punctuation]: 'Punctuation',
|
|
35
|
+
[TokenType.VariableName]: 'VariableName',
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
|
|
39
|
+
const RE_WHITESPACE = /^ +/
|
|
40
|
+
const RE_CURLY_OPEN = /^\{/
|
|
41
|
+
const RE_CURLY_CLOSE = /^\}/
|
|
42
|
+
const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
|
|
43
|
+
const RE_COLON = /^:/
|
|
44
|
+
const RE_PROPERTY_VALUE = /^[^;\}]+/
|
|
45
|
+
const RE_SEMICOLON = /^;/
|
|
46
|
+
const RE_COMMA = /^,/
|
|
47
|
+
const RE_ANYTHING = /^.*/
|
|
48
|
+
const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
|
|
49
|
+
const RE_QUOTE_DOUBLE = /^"/
|
|
50
|
+
const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
|
|
51
|
+
const RE_KEYWORD =
|
|
52
|
+
/^(?:alias|bg|bind|break|builtin|caller|cd|command|compgen|complete|dirs|disown|echo|enable|eval|exec|exit|false|fc|fg|getopts|hash|help|history|jobs|kill|let|logout|popd|printf|pushd|pwd|read|readonly|set|shift|shopt|source|suspend|test|times|trap|true|type|ulimit|umask|unalias|unset|wait)\b/
|
|
53
|
+
|
|
54
|
+
const RE_VARIABLE_NAME = /^[a-zA-Z]+/
|
|
55
|
+
const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
|
|
56
|
+
const RE_NUMERIC = /^\d+/
|
|
57
|
+
|
|
58
|
+
export const initialLineState = {
|
|
59
|
+
state: State.TopLevelContent,
|
|
60
|
+
tokens: [],
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export const isEqualLineState = (lineStateA, lineStateB) => {
|
|
64
|
+
return lineStateA.state === lineStateB.state
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* @param {string} line
|
|
69
|
+
*/
|
|
70
|
+
export const tokenizeLine = (line, lineState) => {
|
|
71
|
+
let next = null
|
|
72
|
+
let index = 0
|
|
73
|
+
let tokens = []
|
|
74
|
+
let token = TokenType.None
|
|
75
|
+
let state = lineState.state
|
|
76
|
+
while (index < line.length) {
|
|
77
|
+
const part = line.slice(index)
|
|
78
|
+
switch (state) {
|
|
79
|
+
case State.TopLevelContent:
|
|
80
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
81
|
+
token = TokenType.Whitespace
|
|
82
|
+
state = State.TopLevelContent
|
|
83
|
+
} else if ((next = part.match(RE_KEYWORD))) {
|
|
84
|
+
token = TokenType.Keyword
|
|
85
|
+
state = State.TopLevelContent
|
|
86
|
+
} else if ((next = part.match(RE_PUNCTUATION))) {
|
|
87
|
+
token = TokenType.Punctuation
|
|
88
|
+
state = State.TopLevelContent
|
|
89
|
+
} else if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
90
|
+
token = TokenType.VariableName
|
|
91
|
+
state = State.TopLevelContent
|
|
92
|
+
} else if ((next = part.match(RE_NUMERIC))) {
|
|
93
|
+
token = TokenType.Numeric
|
|
94
|
+
state = State.TopLevelContent
|
|
95
|
+
} else if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
96
|
+
token = TokenType.PunctuationString
|
|
97
|
+
state = State.InsideString
|
|
98
|
+
} else {
|
|
99
|
+
part //?
|
|
100
|
+
throw new Error('no')
|
|
101
|
+
}
|
|
102
|
+
break
|
|
103
|
+
case State.InsideString:
|
|
104
|
+
if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
105
|
+
token = TokenType.PunctuationString
|
|
106
|
+
state = State.TopLevelContent
|
|
107
|
+
} else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
|
|
108
|
+
token = TokenType.String
|
|
109
|
+
state = State.InsideString
|
|
110
|
+
} else {
|
|
111
|
+
throw new Error('no')
|
|
112
|
+
}
|
|
113
|
+
break
|
|
114
|
+
default:
|
|
115
|
+
throw new Error('no')
|
|
116
|
+
}
|
|
117
|
+
index += next[0].length
|
|
118
|
+
tokens.push({
|
|
119
|
+
type: token,
|
|
120
|
+
length: next[0].length,
|
|
121
|
+
})
|
|
122
|
+
}
|
|
123
|
+
return {
|
|
124
|
+
state,
|
|
125
|
+
tokens,
|
|
126
|
+
}
|
|
127
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lvce-editor/shared-process",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.17",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -32,8 +32,8 @@
|
|
|
32
32
|
"verror": "^1.10.1",
|
|
33
33
|
"ws": "^8.8.0",
|
|
34
34
|
"xdg-basedir": "^5.1.0",
|
|
35
|
-
"@lvce-editor/pty-host": "0.0.
|
|
36
|
-
"@lvce-editor/extension-host": "0.0.
|
|
35
|
+
"@lvce-editor/pty-host": "0.0.17",
|
|
36
|
+
"@lvce-editor/extension-host": "0.0.17"
|
|
37
37
|
},
|
|
38
38
|
"optionalDependencies": {
|
|
39
39
|
"electron-clipboard-ex": "^1.3.3",
|