@lvce-editor/shared-process 0.0.13 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/builtin.language-basics-cpp/README.md +14 -0
- package/extensions/builtin.language-basics-cpp/extension.json +13 -0
- package/extensions/builtin.language-basics-cpp/src/tokenizeCpp.js +222 -0
- package/extensions/builtin.language-basics-css/README.md +14 -0
- package/extensions/builtin.language-basics-css/extension.json +12 -0
- package/extensions/builtin.language-basics-css/src/tokenizeCss.js +378 -0
- package/extensions/builtin.language-basics-dart/README.md +14 -0
- package/extensions/builtin.language-basics-dart/extension.json +12 -0
- package/extensions/builtin.language-basics-dart/src/tokenizeDart.js +132 -0
- package/extensions/builtin.language-basics-dotenv/README.md +14 -0
- package/extensions/builtin.language-basics-dotenv/extension.json +12 -0
- package/extensions/builtin.language-basics-dotenv/src/tokenizeDotenv.js +297 -0
- package/extensions/builtin.language-basics-elixir/README.md +15 -0
- package/extensions/builtin.language-basics-elixir/extension.json +12 -0
- package/extensions/builtin.language-basics-elixir/src/tokenizeElixir.js +150 -0
- package/extensions/builtin.language-basics-go/README.md +14 -0
- package/extensions/builtin.language-basics-go/extension.json +12 -0
- package/extensions/builtin.language-basics-go/src/tokenizeGo.js +161 -0
- package/extensions/builtin.language-basics-html/README.md +14 -0
- package/extensions/builtin.language-basics-html/extension.json +13 -0
- package/extensions/builtin.language-basics-html/src/tokenizeHtml.js +278 -0
- package/extensions/builtin.language-basics-java/README.md +14 -0
- package/extensions/builtin.language-basics-java/extension.json +12 -0
- package/extensions/builtin.language-basics-java/src/tokenizeJava.js +135 -0
- package/extensions/builtin.language-basics-javascript/README.md +14 -0
- package/extensions/builtin.language-basics-javascript/extension.json +13 -0
- package/extensions/builtin.language-basics-javascript/src/tokenizeJavaScript.js +295 -0
- package/extensions/builtin.language-basics-julia/README.md +18 -0
- package/extensions/builtin.language-basics-julia/extension.json +12 -0
- package/extensions/builtin.language-basics-julia/src/tokenizeJulia.js +135 -0
- package/extensions/builtin.language-basics-kotlin/README.md +15 -0
- package/extensions/builtin.language-basics-kotlin/extension.json +12 -0
- package/extensions/builtin.language-basics-kotlin/src/tokenizeKotlin.js +127 -0
- package/extensions/builtin.language-basics-python/README.md +14 -0
- package/extensions/builtin.language-basics-python/extension.json +13 -0
- package/extensions/builtin.language-basics-python/src/tokenizePython.js +201 -0
- package/extensions/builtin.language-basics-shellscript/README.md +15 -0
- package/extensions/builtin.language-basics-shellscript/extension.json +12 -0
- package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js +127 -0
- package/package.json +3 -3
- package/src/parts/Native/Native.ipc.js +1 -1
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @enum number
|
|
3
|
+
*/
|
|
4
|
+
const State = {
|
|
5
|
+
None: 0,
|
|
6
|
+
TopLevelContent: 1,
|
|
7
|
+
InsideSingleQuoteString: 2,
|
|
8
|
+
InsideDoubleQuoteString: 3,
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @enum number
|
|
13
|
+
*/
|
|
14
|
+
export const TokenType = {
|
|
15
|
+
None: 99999999,
|
|
16
|
+
Keyword: 951,
|
|
17
|
+
Whitespace: 0,
|
|
18
|
+
NewLine: 771,
|
|
19
|
+
VariableName: 2,
|
|
20
|
+
Punctuation: 3,
|
|
21
|
+
String: 4,
|
|
22
|
+
Numeric: 5,
|
|
23
|
+
Attribute: 6,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export const TokenMap = {
|
|
27
|
+
[TokenType.None]: 'None',
|
|
28
|
+
[TokenType.Keyword]: 'Keyword',
|
|
29
|
+
[TokenType.Whitespace]: 'Whitespace',
|
|
30
|
+
[TokenType.NewLine]: 'NewLine',
|
|
31
|
+
[TokenType.VariableName]: 'VariableName',
|
|
32
|
+
[TokenType.Punctuation]: 'Punctuation',
|
|
33
|
+
[TokenType.String]: 'String',
|
|
34
|
+
[TokenType.Numeric]: 'Numeric',
|
|
35
|
+
[TokenType.Attribute]: 'Attribute',
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export const initialLineState = {
|
|
39
|
+
state: State.TopLevelContent,
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const RE_KEYWORD =
|
|
43
|
+
/^(?:abstract|as|assert|async|await|break|case|catch|class|const|continue|covariant|default|deferred|do|dynamic|else|enum|export|extends|extension|external|factory|false|final|finally|for|Function|get|hide|if|implements|import|in|interface|is|late|library|mixin|new|null|on|operator|part|required|rethrow|return|set|show|static|super|switch|sync|this|throw|true|try|typedef|var|void|while|with|yield)\b/
|
|
44
|
+
const RE_WHITESPACE = /^\s+/
|
|
45
|
+
const RE_VARIABLE_NAME = /^[a-zA-Z]+/
|
|
46
|
+
const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>]/
|
|
47
|
+
const RE_QUOTE_SINGLE = /^'/
|
|
48
|
+
const RE_QUOTE_DOUBLE = /^"/
|
|
49
|
+
const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
|
|
50
|
+
const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
|
|
51
|
+
const RE_NUMERIC = /^\d+/
|
|
52
|
+
const RE_TRIPLE_DOUBLE_QUOTE = /^"""/
|
|
53
|
+
const RE_STRING_TRIPLE_CONTENT = /^.+?(?="""|$)/s
|
|
54
|
+
const RE_ATTRIBUTE = /^@\w+/
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {string} line
|
|
58
|
+
* @param {any} lineState
|
|
59
|
+
*/
|
|
60
|
+
export const tokenizeLine = (line, lineState) => {
|
|
61
|
+
let next = null
|
|
62
|
+
let index = 0
|
|
63
|
+
let tokens = []
|
|
64
|
+
let token = TokenType.None
|
|
65
|
+
let state = lineState.state
|
|
66
|
+
while (index < line.length) {
|
|
67
|
+
const part = line.slice(index)
|
|
68
|
+
switch (state) {
|
|
69
|
+
case State.TopLevelContent:
|
|
70
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
71
|
+
token = TokenType.Whitespace
|
|
72
|
+
state = State.TopLevelContent
|
|
73
|
+
} else if ((next = part.match(RE_KEYWORD))) {
|
|
74
|
+
token = TokenType.Keyword
|
|
75
|
+
state = State.TopLevelContent
|
|
76
|
+
} else if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
77
|
+
token = TokenType.VariableName
|
|
78
|
+
state = State.TopLevelContent
|
|
79
|
+
} else if ((next = part.match(RE_PUNCTUATION))) {
|
|
80
|
+
token = TokenType.Punctuation
|
|
81
|
+
state = State.TopLevelContent
|
|
82
|
+
} else if ((next = part.match(RE_NUMERIC))) {
|
|
83
|
+
token = TokenType.Numeric
|
|
84
|
+
state = State.TopLevelContent
|
|
85
|
+
} else if ((next = part.match(RE_QUOTE_SINGLE))) {
|
|
86
|
+
token = TokenType.Punctuation
|
|
87
|
+
state = State.InsideSingleQuoteString
|
|
88
|
+
} else if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
89
|
+
token = TokenType.Punctuation
|
|
90
|
+
state = State.InsideDoubleQuoteString
|
|
91
|
+
} else {
|
|
92
|
+
part //?
|
|
93
|
+
throw new Error('no')
|
|
94
|
+
}
|
|
95
|
+
break
|
|
96
|
+
case State.InsideSingleQuoteString:
|
|
97
|
+
if ((next = part.match(RE_QUOTE_SINGLE))) {
|
|
98
|
+
token = TokenType.Punctuation
|
|
99
|
+
state = State.TopLevelContent
|
|
100
|
+
} else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
|
|
101
|
+
token = TokenType.String
|
|
102
|
+
state = State.InsideSingleQuoteString
|
|
103
|
+
} else {
|
|
104
|
+
throw new Error('no')
|
|
105
|
+
}
|
|
106
|
+
break
|
|
107
|
+
case State.InsideDoubleQuoteString:
|
|
108
|
+
if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
109
|
+
token = TokenType.Punctuation
|
|
110
|
+
state = State.TopLevelContent
|
|
111
|
+
} else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
|
|
112
|
+
token = TokenType.String
|
|
113
|
+
state = State.InsideDoubleQuoteString
|
|
114
|
+
} else {
|
|
115
|
+
throw new Error('no')
|
|
116
|
+
}
|
|
117
|
+
break
|
|
118
|
+
default:
|
|
119
|
+
state
|
|
120
|
+
throw new Error('no')
|
|
121
|
+
}
|
|
122
|
+
index += next[0].length
|
|
123
|
+
tokens.push({
|
|
124
|
+
type: token,
|
|
125
|
+
length: next[0].length,
|
|
126
|
+
})
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
state,
|
|
130
|
+
tokens,
|
|
131
|
+
}
|
|
132
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Language Basics Dotenv
|
|
2
|
+
|
|
3
|
+
## Contributing
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
git clone git@github.com:lvce-editor/language-basics-dotenv.git &&
|
|
7
|
+
cd language-basics-dotenv &&
|
|
8
|
+
npm ci &&
|
|
9
|
+
npm test
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Gitpod
|
|
13
|
+
|
|
14
|
+
[](https://gitpod.io/#https://github.com/lvce-editor/language-basics-dotenv)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "builtin.language-basics-dotenv",
|
|
3
|
+
"name": "Language Basics Dotenv",
|
|
4
|
+
"description": "Provides syntax highlighting and bracket matching in dotenv files.",
|
|
5
|
+
"languages": [
|
|
6
|
+
{
|
|
7
|
+
"id": "dotenv",
|
|
8
|
+
"extensions": [".env", ".env.local"],
|
|
9
|
+
"configuration": "./languageConfiguration.json"
|
|
10
|
+
}
|
|
11
|
+
]
|
|
12
|
+
}
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @enum number
|
|
3
|
+
*/
|
|
4
|
+
export const State = {
|
|
5
|
+
TopLevelContent: 1,
|
|
6
|
+
InsideLineComment: 2,
|
|
7
|
+
AfterVariableName: 3,
|
|
8
|
+
AfterAssignmentEqualSign: 4,
|
|
9
|
+
AfterVariableValue: 5,
|
|
10
|
+
AfterKeyword: 6,
|
|
11
|
+
AfterKeywordAndHasSeenWhitespace: 7,
|
|
12
|
+
InsideDoubleQuoteString: 8,
|
|
13
|
+
InsideSingleQuoteString: 9,
|
|
14
|
+
InsideDoubleQuoteStringAfterBackslash: 10,
|
|
15
|
+
InsideSingleQuoteStringAfterBackslash: 11,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @enum {number}
|
|
20
|
+
*/
|
|
21
|
+
export const TokenType = {
|
|
22
|
+
Whitespace /* Whitespace */: 0,
|
|
23
|
+
TypeName /* TypeName */: 35,
|
|
24
|
+
PlainText /* Text */: 117,
|
|
25
|
+
Comment /* Comment */: 60,
|
|
26
|
+
String /* String */: 50,
|
|
27
|
+
Numeric /* Numeric */: 30,
|
|
28
|
+
FunctionName /* FunctionName */: 112,
|
|
29
|
+
TypeNamePrimitive /* TypeNamePrimitive */: 36,
|
|
30
|
+
Text /* Text */: 117,
|
|
31
|
+
PunctuationTag /* PunctuationTag */: 228,
|
|
32
|
+
TagName /* TagName */: 118,
|
|
33
|
+
AttributeName /* AttributeName */: 119,
|
|
34
|
+
Punctuation /* Punctuation */: 10,
|
|
35
|
+
Error /* Error */: 141,
|
|
36
|
+
PunctuationString /* PunctuationString */: 11,
|
|
37
|
+
Keyword: /* Keyword */ 951,
|
|
38
|
+
VariableName: /* VariableName */ 952,
|
|
39
|
+
None: 57,
|
|
40
|
+
KeywordExport: 90,
|
|
41
|
+
VariableValueString: 91,
|
|
42
|
+
LanguageConstant: 92,
|
|
43
|
+
VariableValue: 93,
|
|
44
|
+
NewLine: 94,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export const TokenMap = {
|
|
48
|
+
[TokenType.Whitespace]: 'Whitespace',
|
|
49
|
+
[TokenType.TypeName]: 'TypeName',
|
|
50
|
+
[TokenType.PlainText]: 'PlainText',
|
|
51
|
+
[TokenType.Comment]: 'Comment',
|
|
52
|
+
[TokenType.String]: 'String',
|
|
53
|
+
[TokenType.Numeric]: 'Numeric',
|
|
54
|
+
[TokenType.FunctionName]: 'FunctionName',
|
|
55
|
+
[TokenType.TypeNamePrimitive]: 'TypeNamePrimitive',
|
|
56
|
+
[TokenType.Text]: 'Text',
|
|
57
|
+
[TokenType.PunctuationTag]: 'PunctuationTag',
|
|
58
|
+
[TokenType.TagName]: 'TagName',
|
|
59
|
+
[TokenType.AttributeName]: 'AttributeName',
|
|
60
|
+
[TokenType.Punctuation]: 'Punctuation',
|
|
61
|
+
[TokenType.Error]: 'Error',
|
|
62
|
+
[TokenType.PunctuationString]: 'PunctuationString',
|
|
63
|
+
[TokenType.Keyword]: 'Keyword',
|
|
64
|
+
[TokenType.None]: 'None',
|
|
65
|
+
[TokenType.KeywordExport]: 'KeywordExport',
|
|
66
|
+
[TokenType.VariableValueString]: 'String',
|
|
67
|
+
[TokenType.LanguageConstant]: 'LanguageConstant',
|
|
68
|
+
[TokenType.VariableValue]: 'VariableValue',
|
|
69
|
+
[TokenType.VariableName]: 'VariableName',
|
|
70
|
+
[TokenType.NewLine]: 'NewLine',
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const RE_WHITESPACE_NEWLINE = /^\n/
|
|
74
|
+
const RE_CURLY_OPEN = /^\{/
|
|
75
|
+
const RE_CURLY_CLOSE = /^\}/
|
|
76
|
+
const RE_SQUARE_OPEN = /^\[/
|
|
77
|
+
const RE_SQUARE_CLOSE = /^\]/
|
|
78
|
+
const RE_COMMA = /^,/
|
|
79
|
+
const RE_COLON = /^:/
|
|
80
|
+
const RE_NUMERIC =
|
|
81
|
+
/^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)/
|
|
82
|
+
const RE_NEWLINE_WHITESPACE = /^\n\s*/
|
|
83
|
+
const RE_BLOCK_COMMENT_START = /^\/\*/
|
|
84
|
+
const RE_BLOCK_COMMENT_CONTENT = /^.+(?=\*\/|$)/s
|
|
85
|
+
const RE_BLOCK_COMMENT_END = /^\*\//
|
|
86
|
+
const RE_UNKNOWN_VALUE = /^[^\}\{\s,"]+/
|
|
87
|
+
const RE_IMPORT = /^[a-zA-Z\.]+/
|
|
88
|
+
const RE_SEMICOLON = /^;/
|
|
89
|
+
const RE_LINE_COMMENT = /^\/\//
|
|
90
|
+
const RE_ROUND_OPEN = /^\(/
|
|
91
|
+
const RE_ROUND_CLOSE = /^\)/
|
|
92
|
+
const RE_DOT = /^\./
|
|
93
|
+
const RE_PUNCTUATION = /^[\(\)=\+\-><\.,\/\*\^\[\]\{\}\|:]/
|
|
94
|
+
const RE_ANYTHING_UNTIL_END = /^.+/s
|
|
95
|
+
const RE_START_OF_FUNCTION = /^( )*\(/
|
|
96
|
+
const RE_COLON_COLON = /^::/
|
|
97
|
+
const RE_BASH_SLASH = /^\\/
|
|
98
|
+
const RE_ANY_CHAR = /^./
|
|
99
|
+
const RE_SQUARE_OPEN_SQUARE_OPEN = /^\[\[/
|
|
100
|
+
const RE_SQUARE_CLOSE_SQUARE_CLOSE = /^\]\]/
|
|
101
|
+
const RE_STRING_MULTILINE_CONTENT = /^.+?(?=\]\]|$)/s
|
|
102
|
+
const RE_KEYWORD =
|
|
103
|
+
/^(?:var|type|switch|struct|select|return|range|package|map|interface|import|if|goto|go|func|default|continue|const|chan|case|break)\b/
|
|
104
|
+
const RE_TEXT = /^.+/s
|
|
105
|
+
const RE_WHITESPACE = /^\s+/
|
|
106
|
+
const RE_WHITESPACE_SINGLE_LINE = /^( |\t)+/
|
|
107
|
+
const RE_DOUBLE_QUOTE = /^"/
|
|
108
|
+
const RE_LINE_COMMENT_START = /^#/
|
|
109
|
+
const RE_LINE_COMMENT_CONTENT = /^[^\n]+/
|
|
110
|
+
const RE_VARIABLE_NAME = /^[a-zA-Z\_]+/
|
|
111
|
+
const RE_EQUAL_SIGN = /^=/
|
|
112
|
+
const RE_NUMBER =
|
|
113
|
+
/^\b((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
|
|
114
|
+
const RE_KEYWORD_EXPORT = /^export(?=(\t| ))/
|
|
115
|
+
const RE_CONSTANT = /^(true|false|null)(?=\s|$)/i
|
|
116
|
+
const RE_VARIABLE_VALUE = /^[^\n"'#]+/
|
|
117
|
+
const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^\n\\"]+/
|
|
118
|
+
const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^\n\\']+/
|
|
119
|
+
const RE_SINGLE_QUOTE = /^'/
|
|
120
|
+
const RE_NEWLINE_AND_WHITESPACE = /^\n\s*/
|
|
121
|
+
const RE_BACKSLASH = /^\\/
|
|
122
|
+
const RE_EVERYTHING = /^./
|
|
123
|
+
|
|
124
|
+
export const initialLineState = {
|
|
125
|
+
state: State.TopLevelContent,
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* @param {string} line
|
|
130
|
+
*/
|
|
131
|
+
export const tokenizeLine = (line, lineState) => {
|
|
132
|
+
let next = null
|
|
133
|
+
let index = 0
|
|
134
|
+
let tokens = []
|
|
135
|
+
let token = TokenType.None
|
|
136
|
+
let state = lineState.state
|
|
137
|
+
while (index < line.length) {
|
|
138
|
+
const part = line.slice(index)
|
|
139
|
+
switch (state) {
|
|
140
|
+
case State.TopLevelContent:
|
|
141
|
+
if ((next = part.match(RE_KEYWORD_EXPORT))) {
|
|
142
|
+
token = TokenType.KeywordExport
|
|
143
|
+
state = State.AfterKeyword
|
|
144
|
+
} else if ((next = part.match(RE_LINE_COMMENT_START))) {
|
|
145
|
+
token = TokenType.Comment
|
|
146
|
+
state = State.InsideLineComment
|
|
147
|
+
} else if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
148
|
+
token = TokenType.VariableName
|
|
149
|
+
state = State.AfterVariableName
|
|
150
|
+
} else if ((next = part.match(RE_WHITESPACE))) {
|
|
151
|
+
token = TokenType.Whitespace
|
|
152
|
+
state = State.TopLevelContent
|
|
153
|
+
} else {
|
|
154
|
+
throw new Error('no')
|
|
155
|
+
}
|
|
156
|
+
break
|
|
157
|
+
case State.AfterKeyword:
|
|
158
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
159
|
+
token = TokenType.Whitespace
|
|
160
|
+
state = State.AfterKeywordAndHasSeenWhitespace
|
|
161
|
+
} else {
|
|
162
|
+
line
|
|
163
|
+
throw new Error('no')
|
|
164
|
+
}
|
|
165
|
+
break
|
|
166
|
+
case State.AfterKeywordAndHasSeenWhitespace:
|
|
167
|
+
if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
168
|
+
token = TokenType.VariableName
|
|
169
|
+
state = State.AfterVariableName
|
|
170
|
+
} else {
|
|
171
|
+
throw new Error('no')
|
|
172
|
+
}
|
|
173
|
+
break
|
|
174
|
+
case State.AfterVariableName:
|
|
175
|
+
if ((next = part.match(RE_EQUAL_SIGN))) {
|
|
176
|
+
token = TokenType.Punctuation
|
|
177
|
+
state = State.AfterAssignmentEqualSign
|
|
178
|
+
} else if ((next = part.match(RE_WHITESPACE))) {
|
|
179
|
+
token = TokenType.Whitespace
|
|
180
|
+
// TODO what about newline
|
|
181
|
+
state = State.AfterVariableName
|
|
182
|
+
} else {
|
|
183
|
+
throw new Error('no')
|
|
184
|
+
}
|
|
185
|
+
break
|
|
186
|
+
case State.InsideDoubleQuoteString:
|
|
187
|
+
if ((next = part.match(RE_DOUBLE_QUOTE))) {
|
|
188
|
+
token = TokenType.PunctuationString
|
|
189
|
+
state = State.AfterVariableValue
|
|
190
|
+
} else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
|
|
191
|
+
token = TokenType.VariableValueString
|
|
192
|
+
state = State.InsideDoubleQuoteString
|
|
193
|
+
} else if ((next = part.match(RE_BACKSLASH))) {
|
|
194
|
+
token = TokenType.VariableValueString
|
|
195
|
+
state = State.InsideDoubleQuoteStringAfterBackslash
|
|
196
|
+
} else {
|
|
197
|
+
part.startsWith('\n') //?
|
|
198
|
+
throw new Error('no')
|
|
199
|
+
}
|
|
200
|
+
break
|
|
201
|
+
case State.InsideDoubleQuoteStringAfterBackslash:
|
|
202
|
+
if ((next = part.match(RE_EVERYTHING))) {
|
|
203
|
+
token = TokenType.VariableValueString
|
|
204
|
+
state = State.InsideDoubleQuoteString
|
|
205
|
+
} else {
|
|
206
|
+
throw new Error('no')
|
|
207
|
+
}
|
|
208
|
+
break
|
|
209
|
+
case State.InsideSingleQuoteStringAfterBackslash:
|
|
210
|
+
if ((next = part.match(RE_EVERYTHING))) {
|
|
211
|
+
token = TokenType.VariableValueString
|
|
212
|
+
state = State.InsideSingleQuoteString
|
|
213
|
+
} else {
|
|
214
|
+
throw new Error('no')
|
|
215
|
+
}
|
|
216
|
+
break
|
|
217
|
+
case State.InsideSingleQuoteString:
|
|
218
|
+
if ((next = part.match(RE_SINGLE_QUOTE))) {
|
|
219
|
+
token = TokenType.Punctuation
|
|
220
|
+
state = State.AfterVariableValue
|
|
221
|
+
} else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
|
|
222
|
+
token = TokenType.VariableValueString
|
|
223
|
+
state = State.InsideSingleQuoteString
|
|
224
|
+
} else if ((next = part.match(RE_BACKSLASH))) {
|
|
225
|
+
token = TokenType.VariableValueString
|
|
226
|
+
state = State.InsideSingleQuoteStringAfterBackslash
|
|
227
|
+
} else {
|
|
228
|
+
throw new Error('no')
|
|
229
|
+
}
|
|
230
|
+
break
|
|
231
|
+
case State.AfterAssignmentEqualSign:
|
|
232
|
+
if ((next = part.match(RE_DOUBLE_QUOTE))) {
|
|
233
|
+
token = TokenType.PunctuationString
|
|
234
|
+
state = State.InsideDoubleQuoteString
|
|
235
|
+
} else if ((next = part.match(RE_SINGLE_QUOTE))) {
|
|
236
|
+
token = TokenType.Punctuation
|
|
237
|
+
state = State.InsideSingleQuoteString
|
|
238
|
+
} else if ((next = part.match(RE_NUMBER))) {
|
|
239
|
+
token = TokenType.Numeric
|
|
240
|
+
state = State.AfterVariableValue
|
|
241
|
+
} else if ((next = part.match(RE_CONSTANT))) {
|
|
242
|
+
token = TokenType.LanguageConstant
|
|
243
|
+
state = State.AfterVariableValue
|
|
244
|
+
} else if ((next = part.match(RE_WHITESPACE_SINGLE_LINE))) {
|
|
245
|
+
token = TokenType.Whitespace
|
|
246
|
+
state = State.AfterAssignmentEqualSign
|
|
247
|
+
} else if ((next = part.match(RE_VARIABLE_VALUE))) {
|
|
248
|
+
token = TokenType.VariableValue
|
|
249
|
+
state = State.AfterVariableValue
|
|
250
|
+
} else if ((next = part.match(RE_NEWLINE_AND_WHITESPACE))) {
|
|
251
|
+
token = TokenType.Whitespace
|
|
252
|
+
state = State.TopLevelContent
|
|
253
|
+
} else {
|
|
254
|
+
throw new Error('no')
|
|
255
|
+
}
|
|
256
|
+
break
|
|
257
|
+
case State.InsideLineComment:
|
|
258
|
+
if ((next = part.match(RE_LINE_COMMENT_CONTENT))) {
|
|
259
|
+
token = TokenType.Comment
|
|
260
|
+
state = State.TopLevelContent
|
|
261
|
+
} else {
|
|
262
|
+
throw new Error('no')
|
|
263
|
+
}
|
|
264
|
+
break
|
|
265
|
+
case State.AfterVariableValue:
|
|
266
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
267
|
+
token = TokenType.Whitespace
|
|
268
|
+
state = State.TopLevelContent
|
|
269
|
+
} else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
|
|
270
|
+
token = TokenType.Error
|
|
271
|
+
state = State.TopLevelContent
|
|
272
|
+
} else {
|
|
273
|
+
throw new Error('no')
|
|
274
|
+
}
|
|
275
|
+
break
|
|
276
|
+
default:
|
|
277
|
+
state
|
|
278
|
+
throw new Error('no')
|
|
279
|
+
}
|
|
280
|
+
index += next[0].length
|
|
281
|
+
tokens.push({
|
|
282
|
+
type: token,
|
|
283
|
+
length: next[0].length,
|
|
284
|
+
})
|
|
285
|
+
}
|
|
286
|
+
if (state === State.AfterVariableValue) {
|
|
287
|
+
state = State.TopLevelContent
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
state,
|
|
291
|
+
tokens,
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// tokenizeLine(initialContext, '# comment') //?
|
|
296
|
+
|
|
297
|
+
// tokenizeLine('KEY=42"', initialLineState) //?
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Language Basics Elixir
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
## Contributing
|
|
5
|
+
|
|
6
|
+
```sh
|
|
7
|
+
git clone git@github.com:lvce-editor/language-basics-elixir.git &&
|
|
8
|
+
cd language-basics-elixir &&
|
|
9
|
+
npm ci &&
|
|
10
|
+
npm test
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Gitpod
|
|
14
|
+
|
|
15
|
+
[](https://gitpod.io/#https://github.com/lvce-editor/language-basics-elixir)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "builtin.language-basics-elixir",
|
|
3
|
+
"name": "Language Basics Elixir",
|
|
4
|
+
"description": "Provides syntax highlighting and bracket matching in Elixir files.",
|
|
5
|
+
"languages": [
|
|
6
|
+
{
|
|
7
|
+
"id": "elixir",
|
|
8
|
+
"extensions": [".ex"],
|
|
9
|
+
"tokenize": "src/tokenizeElixir.js"
|
|
10
|
+
}
|
|
11
|
+
]
|
|
12
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @enum number
|
|
3
|
+
*/
|
|
4
|
+
const State = {
|
|
5
|
+
None: 0,
|
|
6
|
+
TopLevelContent: 1,
|
|
7
|
+
InsideSingleQuoteString: 2,
|
|
8
|
+
InsideDoubleQuoteString: 3,
|
|
9
|
+
InsideTripleQuoteString: 4,
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @enum number
|
|
14
|
+
*/
|
|
15
|
+
export const TokenType = {
|
|
16
|
+
None: 99999999,
|
|
17
|
+
Keyword: 951,
|
|
18
|
+
Whitespace: 0,
|
|
19
|
+
NewLine: 771,
|
|
20
|
+
VariableName: 2,
|
|
21
|
+
Punctuation: 3,
|
|
22
|
+
String: 4,
|
|
23
|
+
Numeric: 5,
|
|
24
|
+
Attribute: 6,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export const TokenMap = {
|
|
28
|
+
[TokenType.None]: 'None',
|
|
29
|
+
[TokenType.Keyword]: 'Keyword',
|
|
30
|
+
[TokenType.Whitespace]: 'Whitespace',
|
|
31
|
+
[TokenType.NewLine]: 'NewLine',
|
|
32
|
+
[TokenType.VariableName]: 'VariableName',
|
|
33
|
+
[TokenType.Punctuation]: 'Punctuation',
|
|
34
|
+
[TokenType.String]: 'String',
|
|
35
|
+
[TokenType.Numeric]: 'Numeric',
|
|
36
|
+
[TokenType.Attribute]: 'Attribute',
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export const initialLineState = {
|
|
40
|
+
state: State.TopLevelContent,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const RE_KEYWORD =
|
|
44
|
+
/^(?:after|alias|and|case|catch|cond|defmodule|defimpl|defprotocol|defrecord|defstruct|do|else|end|fn|for|if|import|in|not|or|quote|raise|receive|require|reraise|rescue|try|unless|unquote|unquote_splicing|use|when|with\|0)\b/
|
|
45
|
+
const RE_WHITESPACE = /^\s+/
|
|
46
|
+
const RE_VARIABLE_NAME = /^[a-zA-Z]+/
|
|
47
|
+
const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>]/
|
|
48
|
+
const RE_QUOTE_SINGLE = /^'/
|
|
49
|
+
const RE_QUOTE_DOUBLE = /^"/
|
|
50
|
+
const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
|
|
51
|
+
const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
|
|
52
|
+
const RE_NUMERIC = /^\d+/
|
|
53
|
+
const RE_TRIPLE_DOUBLE_QUOTE = /^"""/
|
|
54
|
+
const RE_STRING_TRIPLE_CONTENT = /^.+?(?="""|$)/s
|
|
55
|
+
const RE_ATTRIBUTE = /^@\w+/
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* @param {string} line
|
|
59
|
+
* @param {any} lineState
|
|
60
|
+
*/
|
|
61
|
+
export const tokenizeLine = (line, lineState) => {
|
|
62
|
+
let next = null
|
|
63
|
+
let index = 0
|
|
64
|
+
let tokens = []
|
|
65
|
+
let token = TokenType.None
|
|
66
|
+
let state = lineState.state
|
|
67
|
+
while (index < line.length) {
|
|
68
|
+
const part = line.slice(index)
|
|
69
|
+
switch (state) {
|
|
70
|
+
case State.TopLevelContent:
|
|
71
|
+
if ((next = part.match(RE_WHITESPACE))) {
|
|
72
|
+
token = TokenType.Whitespace
|
|
73
|
+
state = State.TopLevelContent
|
|
74
|
+
} else if ((next = part.match(RE_KEYWORD))) {
|
|
75
|
+
token = TokenType.Keyword
|
|
76
|
+
state = State.TopLevelContent
|
|
77
|
+
} else if ((next = part.match(RE_VARIABLE_NAME))) {
|
|
78
|
+
token = TokenType.VariableName
|
|
79
|
+
state = State.TopLevelContent
|
|
80
|
+
} else if ((next = part.match(RE_PUNCTUATION))) {
|
|
81
|
+
token = TokenType.Punctuation
|
|
82
|
+
state = State.TopLevelContent
|
|
83
|
+
} else if ((next = part.match(RE_NUMERIC))) {
|
|
84
|
+
token = TokenType.Numeric
|
|
85
|
+
state = State.TopLevelContent
|
|
86
|
+
} else if ((next = part.match(RE_QUOTE_SINGLE))) {
|
|
87
|
+
token = TokenType.Punctuation
|
|
88
|
+
state = State.InsideSingleQuoteString
|
|
89
|
+
} else if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
|
|
90
|
+
token = TokenType.Punctuation
|
|
91
|
+
state = State.InsideTripleQuoteString
|
|
92
|
+
} else if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
93
|
+
token = TokenType.Punctuation
|
|
94
|
+
state = State.InsideDoubleQuoteString
|
|
95
|
+
} else if ((next = part.match(RE_ATTRIBUTE))) {
|
|
96
|
+
token = TokenType.Attribute
|
|
97
|
+
state = State.TopLevelContent
|
|
98
|
+
} else {
|
|
99
|
+
part //?
|
|
100
|
+
throw new Error('no')
|
|
101
|
+
}
|
|
102
|
+
break
|
|
103
|
+
case State.InsideTripleQuoteString:
|
|
104
|
+
if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
|
|
105
|
+
token = TokenType.Punctuation
|
|
106
|
+
state = State.TopLevelContent
|
|
107
|
+
} else if ((next = part.match(RE_STRING_TRIPLE_CONTENT))) {
|
|
108
|
+
token = TokenType.String
|
|
109
|
+
state = State.InsideTripleQuoteString
|
|
110
|
+
} else {
|
|
111
|
+
throw new Error('no')
|
|
112
|
+
}
|
|
113
|
+
break
|
|
114
|
+
case State.InsideSingleQuoteString:
|
|
115
|
+
if ((next = part.match(RE_QUOTE_SINGLE))) {
|
|
116
|
+
token = TokenType.Punctuation
|
|
117
|
+
state = State.TopLevelContent
|
|
118
|
+
} else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
|
|
119
|
+
token = TokenType.String
|
|
120
|
+
state = State.InsideSingleQuoteString
|
|
121
|
+
} else {
|
|
122
|
+
throw new Error('no')
|
|
123
|
+
}
|
|
124
|
+
break
|
|
125
|
+
case State.InsideDoubleQuoteString:
|
|
126
|
+
if ((next = part.match(RE_QUOTE_DOUBLE))) {
|
|
127
|
+
token = TokenType.Punctuation
|
|
128
|
+
state = State.TopLevelContent
|
|
129
|
+
} else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
|
|
130
|
+
token = TokenType.String
|
|
131
|
+
state = State.InsideDoubleQuoteString
|
|
132
|
+
} else {
|
|
133
|
+
throw new Error('no')
|
|
134
|
+
}
|
|
135
|
+
break
|
|
136
|
+
default:
|
|
137
|
+
state
|
|
138
|
+
throw new Error('no')
|
|
139
|
+
}
|
|
140
|
+
index += next[0].length
|
|
141
|
+
tokens.push({
|
|
142
|
+
type: token,
|
|
143
|
+
length: next[0].length,
|
|
144
|
+
})
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
state,
|
|
148
|
+
tokens,
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Language Basics Go
|
|
2
|
+
|
|
3
|
+
## Contributing
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
git clone git@github.com:lvce-editor/language-basics-go.git &&
|
|
7
|
+
cd language-basics-go &&
|
|
8
|
+
npm ci &&
|
|
9
|
+
npm test
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Gitpod
|
|
13
|
+
|
|
14
|
+
[](https://gitpod.io/#https://github.com/lvce-editor/language-basics-go)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "builtin.language-basics-go",
|
|
3
|
+
"name": "Language Basics Go",
|
|
4
|
+
"description": "Provides syntax highlighting and bracket matching in Gp files.",
|
|
5
|
+
"languages": [
|
|
6
|
+
{
|
|
7
|
+
"id": "go",
|
|
8
|
+
"extensions": [".go"],
|
|
9
|
+
"configuration": "./languageConfiguration.json"
|
|
10
|
+
}
|
|
11
|
+
]
|
|
12
|
+
}
|