@lvce-editor/shared-process 0.0.13 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/extensions/builtin.language-basics-cpp/README.md +14 -0
  2. package/extensions/builtin.language-basics-cpp/extension.json +13 -0
  3. package/extensions/builtin.language-basics-cpp/src/tokenizeCpp.js +222 -0
  4. package/extensions/builtin.language-basics-css/README.md +14 -0
  5. package/extensions/builtin.language-basics-css/extension.json +12 -0
  6. package/extensions/builtin.language-basics-css/src/tokenizeCss.js +378 -0
  7. package/extensions/builtin.language-basics-dart/README.md +14 -0
  8. package/extensions/builtin.language-basics-dart/extension.json +12 -0
  9. package/extensions/builtin.language-basics-dart/src/tokenizeDart.js +132 -0
  10. package/extensions/builtin.language-basics-dotenv/README.md +14 -0
  11. package/extensions/builtin.language-basics-dotenv/extension.json +12 -0
  12. package/extensions/builtin.language-basics-dotenv/src/tokenizeDotenv.js +297 -0
  13. package/extensions/builtin.language-basics-elixir/README.md +15 -0
  14. package/extensions/builtin.language-basics-elixir/extension.json +12 -0
  15. package/extensions/builtin.language-basics-elixir/src/tokenizeElixir.js +150 -0
  16. package/extensions/builtin.language-basics-go/README.md +14 -0
  17. package/extensions/builtin.language-basics-go/extension.json +12 -0
  18. package/extensions/builtin.language-basics-go/src/tokenizeGo.js +161 -0
  19. package/extensions/builtin.language-basics-html/README.md +14 -0
  20. package/extensions/builtin.language-basics-html/extension.json +13 -0
  21. package/extensions/builtin.language-basics-html/src/tokenizeHtml.js +278 -0
  22. package/extensions/builtin.language-basics-java/README.md +14 -0
  23. package/extensions/builtin.language-basics-java/extension.json +12 -0
  24. package/extensions/builtin.language-basics-java/src/tokenizeJava.js +135 -0
  25. package/extensions/builtin.language-basics-javascript/README.md +14 -0
  26. package/extensions/builtin.language-basics-javascript/extension.json +13 -0
  27. package/extensions/builtin.language-basics-javascript/src/tokenizeJavaScript.js +295 -0
  28. package/extensions/builtin.language-basics-julia/README.md +18 -0
  29. package/extensions/builtin.language-basics-julia/extension.json +12 -0
  30. package/extensions/builtin.language-basics-julia/src/tokenizeJulia.js +135 -0
  31. package/extensions/builtin.language-basics-kotlin/README.md +15 -0
  32. package/extensions/builtin.language-basics-kotlin/extension.json +12 -0
  33. package/extensions/builtin.language-basics-kotlin/src/tokenizeKotlin.js +127 -0
  34. package/extensions/builtin.language-basics-python/README.md +14 -0
  35. package/extensions/builtin.language-basics-python/extension.json +13 -0
  36. package/extensions/builtin.language-basics-python/src/tokenizePython.js +201 -0
  37. package/extensions/builtin.language-basics-shellscript/README.md +15 -0
  38. package/extensions/builtin.language-basics-shellscript/extension.json +12 -0
  39. package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js +127 -0
  40. package/package.json +3 -3
  41. package/src/parts/Native/Native.ipc.js +1 -1
@@ -0,0 +1,132 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ const State = {
5
+ None: 0,
6
+ TopLevelContent: 1,
7
+ InsideSingleQuoteString: 2,
8
+ InsideDoubleQuoteString: 3,
9
+ }
10
+
11
+ /**
12
+ * @enum number
13
+ */
14
+ export const TokenType = {
15
+ None: 99999999,
16
+ Keyword: 951,
17
+ Whitespace: 0,
18
+ NewLine: 771,
19
+ VariableName: 2,
20
+ Punctuation: 3,
21
+ String: 4,
22
+ Numeric: 5,
23
+ Attribute: 6,
24
+ }
25
+
26
+ export const TokenMap = {
27
+ [TokenType.None]: 'None',
28
+ [TokenType.Keyword]: 'Keyword',
29
+ [TokenType.Whitespace]: 'Whitespace',
30
+ [TokenType.NewLine]: 'NewLine',
31
+ [TokenType.VariableName]: 'VariableName',
32
+ [TokenType.Punctuation]: 'Punctuation',
33
+ [TokenType.String]: 'String',
34
+ [TokenType.Numeric]: 'Numeric',
35
+ [TokenType.Attribute]: 'Attribute',
36
+ }
37
+
38
+ export const initialLineState = {
39
+ state: State.TopLevelContent,
40
+ }
41
+
42
+ const RE_KEYWORD =
43
+ /^(?:abstract|as|assert|async|await|break|case|catch|class|const|continue|covariant|default|deferred|do|dynamic|else|enum|export|extends|extension|external|factory|false|final|finally|for|Function|get|hide|if|implements|import|in|interface|is|late|library|mixin|new|null|on|operator|part|required|rethrow|return|set|show|static|super|switch|sync|this|throw|true|try|typedef|var|void|while|with|yield)\b/
44
+ const RE_WHITESPACE = /^\s+/
45
+ const RE_VARIABLE_NAME = /^[a-zA-Z]+/
46
+ const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>]/
47
+ const RE_QUOTE_SINGLE = /^'/
48
+ const RE_QUOTE_DOUBLE = /^"/
49
+ const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
50
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
51
+ const RE_NUMERIC = /^\d+/
52
+ const RE_TRIPLE_DOUBLE_QUOTE = /^"""/
53
+ const RE_STRING_TRIPLE_CONTENT = /^.+?(?="""|$)/s
54
+ const RE_ATTRIBUTE = /^@\w+/
55
+
56
+ /**
57
+ * @param {string} line
58
+ * @param {any} lineState
59
+ */
60
+ export const tokenizeLine = (line, lineState) => {
61
+ let next = null
62
+ let index = 0
63
+ let tokens = []
64
+ let token = TokenType.None
65
+ let state = lineState.state
66
+ while (index < line.length) {
67
+ const part = line.slice(index)
68
+ switch (state) {
69
+ case State.TopLevelContent:
70
+ if ((next = part.match(RE_WHITESPACE))) {
71
+ token = TokenType.Whitespace
72
+ state = State.TopLevelContent
73
+ } else if ((next = part.match(RE_KEYWORD))) {
74
+ token = TokenType.Keyword
75
+ state = State.TopLevelContent
76
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
77
+ token = TokenType.VariableName
78
+ state = State.TopLevelContent
79
+ } else if ((next = part.match(RE_PUNCTUATION))) {
80
+ token = TokenType.Punctuation
81
+ state = State.TopLevelContent
82
+ } else if ((next = part.match(RE_NUMERIC))) {
83
+ token = TokenType.Numeric
84
+ state = State.TopLevelContent
85
+ } else if ((next = part.match(RE_QUOTE_SINGLE))) {
86
+ token = TokenType.Punctuation
87
+ state = State.InsideSingleQuoteString
88
+ } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
89
+ token = TokenType.Punctuation
90
+ state = State.InsideDoubleQuoteString
91
+ } else {
92
+ part //?
93
+ throw new Error('no')
94
+ }
95
+ break
96
+ case State.InsideSingleQuoteString:
97
+ if ((next = part.match(RE_QUOTE_SINGLE))) {
98
+ token = TokenType.Punctuation
99
+ state = State.TopLevelContent
100
+ } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
101
+ token = TokenType.String
102
+ state = State.InsideSingleQuoteString
103
+ } else {
104
+ throw new Error('no')
105
+ }
106
+ break
107
+ case State.InsideDoubleQuoteString:
108
+ if ((next = part.match(RE_QUOTE_DOUBLE))) {
109
+ token = TokenType.Punctuation
110
+ state = State.TopLevelContent
111
+ } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
112
+ token = TokenType.String
113
+ state = State.InsideDoubleQuoteString
114
+ } else {
115
+ throw new Error('no')
116
+ }
117
+ break
118
+ default:
119
+ state
120
+ throw new Error('no')
121
+ }
122
+ index += next[0].length
123
+ tokens.push({
124
+ type: token,
125
+ length: next[0].length,
126
+ })
127
+ }
128
+ return {
129
+ state,
130
+ tokens,
131
+ }
132
+ }
@@ -0,0 +1,14 @@
1
+ # Language Basics Dotenv
2
+
3
+ ## Contributing
4
+
5
+ ```sh
6
+ git clone git@github.com:lvce-editor/language-basics-dotenv.git &&
7
+ cd language-basics-dotenv &&
8
+ npm ci &&
9
+ npm test
10
+ ```
11
+
12
+ ## Gitpod
13
+
14
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-dotenv)
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "builtin.language-basics-dotenv",
3
+ "name": "Language Basics Dotenv",
4
+ "description": "Provides syntax highlighting and bracket matching in dotenv files.",
5
+ "languages": [
6
+ {
7
+ "id": "dotenv",
8
+ "extensions": [".env", ".env.local"],
9
+ "configuration": "./languageConfiguration.json"
10
+ }
11
+ ]
12
+ }
@@ -0,0 +1,297 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ export const State = {
5
+ TopLevelContent: 1,
6
+ InsideLineComment: 2,
7
+ AfterVariableName: 3,
8
+ AfterAssignmentEqualSign: 4,
9
+ AfterVariableValue: 5,
10
+ AfterKeyword: 6,
11
+ AfterKeywordAndHasSeenWhitespace: 7,
12
+ InsideDoubleQuoteString: 8,
13
+ InsideSingleQuoteString: 9,
14
+ InsideDoubleQuoteStringAfterBackslash: 10,
15
+ InsideSingleQuoteStringAfterBackslash: 11,
16
+ }
17
+
18
+ /**
19
+ * @enum {number}
20
+ */
21
+ export const TokenType = {
22
+ Whitespace /* Whitespace */: 0,
23
+ TypeName /* TypeName */: 35,
24
+ PlainText /* Text */: 117,
25
+ Comment /* Comment */: 60,
26
+ String /* String */: 50,
27
+ Numeric /* Numeric */: 30,
28
+ FunctionName /* FunctionName */: 112,
29
+ TypeNamePrimitive /* TypeNamePrimitive */: 36,
30
+ Text /* Text */: 117,
31
+ PunctuationTag /* PunctuationTag */: 228,
32
+ TagName /* TagName */: 118,
33
+ AttributeName /* AttributeName */: 119,
34
+ Punctuation /* Punctuation */: 10,
35
+ Error /* Error */: 141,
36
+ PunctuationString /* PunctuationString */: 11,
37
+ Keyword: /* Keyword */ 951,
38
+ VariableName: /* VariableName */ 952,
39
+ None: 57,
40
+ KeywordExport: 90,
41
+ VariableValueString: 91,
42
+ LanguageConstant: 92,
43
+ VariableValue: 93,
44
+ NewLine: 94,
45
+ }
46
+
47
+ export const TokenMap = {
48
+ [TokenType.Whitespace]: 'Whitespace',
49
+ [TokenType.TypeName]: 'TypeName',
50
+ [TokenType.PlainText]: 'PlainText',
51
+ [TokenType.Comment]: 'Comment',
52
+ [TokenType.String]: 'String',
53
+ [TokenType.Numeric]: 'Numeric',
54
+ [TokenType.FunctionName]: 'FunctionName',
55
+ [TokenType.TypeNamePrimitive]: 'TypeNamePrimitive',
56
+ [TokenType.Text]: 'Text',
57
+ [TokenType.PunctuationTag]: 'PunctuationTag',
58
+ [TokenType.TagName]: 'TagName',
59
+ [TokenType.AttributeName]: 'AttributeName',
60
+ [TokenType.Punctuation]: 'Punctuation',
61
+ [TokenType.Error]: 'Error',
62
+ [TokenType.PunctuationString]: 'PunctuationString',
63
+ [TokenType.Keyword]: 'Keyword',
64
+ [TokenType.None]: 'None',
65
+ [TokenType.KeywordExport]: 'KeywordExport',
66
+ [TokenType.VariableValueString]: 'String',
67
+ [TokenType.LanguageConstant]: 'LanguageConstant',
68
+ [TokenType.VariableValue]: 'VariableValue',
69
+ [TokenType.VariableName]: 'VariableName',
70
+ [TokenType.NewLine]: 'NewLine',
71
+ }
72
+
73
+ const RE_WHITESPACE_NEWLINE = /^\n/
74
+ const RE_CURLY_OPEN = /^\{/
75
+ const RE_CURLY_CLOSE = /^\}/
76
+ const RE_SQUARE_OPEN = /^\[/
77
+ const RE_SQUARE_CLOSE = /^\]/
78
+ const RE_COMMA = /^,/
79
+ const RE_COLON = /^:/
80
+ const RE_NUMERIC =
81
+ /^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)/
82
+ const RE_NEWLINE_WHITESPACE = /^\n\s*/
83
+ const RE_BLOCK_COMMENT_START = /^\/\*/
84
+ const RE_BLOCK_COMMENT_CONTENT = /^.+(?=\*\/|$)/s
85
+ const RE_BLOCK_COMMENT_END = /^\*\//
86
+ const RE_UNKNOWN_VALUE = /^[^\}\{\s,"]+/
87
+ const RE_IMPORT = /^[a-zA-Z\.]+/
88
+ const RE_SEMICOLON = /^;/
89
+ const RE_LINE_COMMENT = /^\/\//
90
+ const RE_ROUND_OPEN = /^\(/
91
+ const RE_ROUND_CLOSE = /^\)/
92
+ const RE_DOT = /^\./
93
+ const RE_PUNCTUATION = /^[\(\)=\+\-><\.,\/\*\^\[\]\{\}\|:]/
94
+ const RE_ANYTHING_UNTIL_END = /^.+/s
95
+ const RE_START_OF_FUNCTION = /^( )*\(/
96
+ const RE_COLON_COLON = /^::/
97
+ const RE_BASH_SLASH = /^\\/
98
+ const RE_ANY_CHAR = /^./
99
+ const RE_SQUARE_OPEN_SQUARE_OPEN = /^\[\[/
100
+ const RE_SQUARE_CLOSE_SQUARE_CLOSE = /^\]\]/
101
+ const RE_STRING_MULTILINE_CONTENT = /^.+?(?=\]\]|$)/s
102
+ const RE_KEYWORD =
103
+ /^(?:var|type|switch|struct|select|return|range|package|map|interface|import|if|goto|go|func|default|continue|const|chan|case|break)\b/
104
+ const RE_TEXT = /^.+/s
105
+ const RE_WHITESPACE = /^\s+/
106
+ const RE_WHITESPACE_SINGLE_LINE = /^( |\t)+/
107
+ const RE_DOUBLE_QUOTE = /^"/
108
+ const RE_LINE_COMMENT_START = /^#/
109
+ const RE_LINE_COMMENT_CONTENT = /^[^\n]+/
110
+ const RE_VARIABLE_NAME = /^[a-zA-Z\_]+/
111
+ const RE_EQUAL_SIGN = /^=/
112
+ const RE_NUMBER =
113
+ /^\b((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
114
+ const RE_KEYWORD_EXPORT = /^export(?=(\t| ))/
115
+ const RE_CONSTANT = /^(true|false|null)(?=\s|$)/i
116
+ const RE_VARIABLE_VALUE = /^[^\n"'#]+/
117
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^\n\\"]+/
118
+ const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^\n\\']+/
119
+ const RE_SINGLE_QUOTE = /^'/
120
+ const RE_NEWLINE_AND_WHITESPACE = /^\n\s*/
121
+ const RE_BACKSLASH = /^\\/
122
+ const RE_EVERYTHING = /^./
123
+
124
+ export const initialLineState = {
125
+ state: State.TopLevelContent,
126
+ }
127
+
128
+ /**
129
+ * @param {string} line
130
+ */
131
+ export const tokenizeLine = (line, lineState) => {
132
+ let next = null
133
+ let index = 0
134
+ let tokens = []
135
+ let token = TokenType.None
136
+ let state = lineState.state
137
+ while (index < line.length) {
138
+ const part = line.slice(index)
139
+ switch (state) {
140
+ case State.TopLevelContent:
141
+ if ((next = part.match(RE_KEYWORD_EXPORT))) {
142
+ token = TokenType.KeywordExport
143
+ state = State.AfterKeyword
144
+ } else if ((next = part.match(RE_LINE_COMMENT_START))) {
145
+ token = TokenType.Comment
146
+ state = State.InsideLineComment
147
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
148
+ token = TokenType.VariableName
149
+ state = State.AfterVariableName
150
+ } else if ((next = part.match(RE_WHITESPACE))) {
151
+ token = TokenType.Whitespace
152
+ state = State.TopLevelContent
153
+ } else {
154
+ throw new Error('no')
155
+ }
156
+ break
157
+ case State.AfterKeyword:
158
+ if ((next = part.match(RE_WHITESPACE))) {
159
+ token = TokenType.Whitespace
160
+ state = State.AfterKeywordAndHasSeenWhitespace
161
+ } else {
162
+ line
163
+ throw new Error('no')
164
+ }
165
+ break
166
+ case State.AfterKeywordAndHasSeenWhitespace:
167
+ if ((next = part.match(RE_VARIABLE_NAME))) {
168
+ token = TokenType.VariableName
169
+ state = State.AfterVariableName
170
+ } else {
171
+ throw new Error('no')
172
+ }
173
+ break
174
+ case State.AfterVariableName:
175
+ if ((next = part.match(RE_EQUAL_SIGN))) {
176
+ token = TokenType.Punctuation
177
+ state = State.AfterAssignmentEqualSign
178
+ } else if ((next = part.match(RE_WHITESPACE))) {
179
+ token = TokenType.Whitespace
180
+ // TODO what about newline
181
+ state = State.AfterVariableName
182
+ } else {
183
+ throw new Error('no')
184
+ }
185
+ break
186
+ case State.InsideDoubleQuoteString:
187
+ if ((next = part.match(RE_DOUBLE_QUOTE))) {
188
+ token = TokenType.PunctuationString
189
+ state = State.AfterVariableValue
190
+ } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
191
+ token = TokenType.VariableValueString
192
+ state = State.InsideDoubleQuoteString
193
+ } else if ((next = part.match(RE_BACKSLASH))) {
194
+ token = TokenType.VariableValueString
195
+ state = State.InsideDoubleQuoteStringAfterBackslash
196
+ } else {
197
+ part.startsWith('\n') //?
198
+ throw new Error('no')
199
+ }
200
+ break
201
+ case State.InsideDoubleQuoteStringAfterBackslash:
202
+ if ((next = part.match(RE_EVERYTHING))) {
203
+ token = TokenType.VariableValueString
204
+ state = State.InsideDoubleQuoteString
205
+ } else {
206
+ throw new Error('no')
207
+ }
208
+ break
209
+ case State.InsideSingleQuoteStringAfterBackslash:
210
+ if ((next = part.match(RE_EVERYTHING))) {
211
+ token = TokenType.VariableValueString
212
+ state = State.InsideSingleQuoteString
213
+ } else {
214
+ throw new Error('no')
215
+ }
216
+ break
217
+ case State.InsideSingleQuoteString:
218
+ if ((next = part.match(RE_SINGLE_QUOTE))) {
219
+ token = TokenType.Punctuation
220
+ state = State.AfterVariableValue
221
+ } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
222
+ token = TokenType.VariableValueString
223
+ state = State.InsideSingleQuoteString
224
+ } else if ((next = part.match(RE_BACKSLASH))) {
225
+ token = TokenType.VariableValueString
226
+ state = State.InsideSingleQuoteStringAfterBackslash
227
+ } else {
228
+ throw new Error('no')
229
+ }
230
+ break
231
+ case State.AfterAssignmentEqualSign:
232
+ if ((next = part.match(RE_DOUBLE_QUOTE))) {
233
+ token = TokenType.PunctuationString
234
+ state = State.InsideDoubleQuoteString
235
+ } else if ((next = part.match(RE_SINGLE_QUOTE))) {
236
+ token = TokenType.Punctuation
237
+ state = State.InsideSingleQuoteString
238
+ } else if ((next = part.match(RE_NUMBER))) {
239
+ token = TokenType.Numeric
240
+ state = State.AfterVariableValue
241
+ } else if ((next = part.match(RE_CONSTANT))) {
242
+ token = TokenType.LanguageConstant
243
+ state = State.AfterVariableValue
244
+ } else if ((next = part.match(RE_WHITESPACE_SINGLE_LINE))) {
245
+ token = TokenType.Whitespace
246
+ state = State.AfterAssignmentEqualSign
247
+ } else if ((next = part.match(RE_VARIABLE_VALUE))) {
248
+ token = TokenType.VariableValue
249
+ state = State.AfterVariableValue
250
+ } else if ((next = part.match(RE_NEWLINE_AND_WHITESPACE))) {
251
+ token = TokenType.Whitespace
252
+ state = State.TopLevelContent
253
+ } else {
254
+ throw new Error('no')
255
+ }
256
+ break
257
+ case State.InsideLineComment:
258
+ if ((next = part.match(RE_LINE_COMMENT_CONTENT))) {
259
+ token = TokenType.Comment
260
+ state = State.TopLevelContent
261
+ } else {
262
+ throw new Error('no')
263
+ }
264
+ break
265
+ case State.AfterVariableValue:
266
+ if ((next = part.match(RE_WHITESPACE))) {
267
+ token = TokenType.Whitespace
268
+ state = State.TopLevelContent
269
+ } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
270
+ token = TokenType.Error
271
+ state = State.TopLevelContent
272
+ } else {
273
+ throw new Error('no')
274
+ }
275
+ break
276
+ default:
277
+ state
278
+ throw new Error('no')
279
+ }
280
+ index += next[0].length
281
+ tokens.push({
282
+ type: token,
283
+ length: next[0].length,
284
+ })
285
+ }
286
+ if (state === State.AfterVariableValue) {
287
+ state = State.TopLevelContent
288
+ }
289
+ return {
290
+ state,
291
+ tokens,
292
+ }
293
+ }
294
+
295
+ // tokenizeLine(initialContext, '# comment') //?
296
+
297
+ // tokenizeLine('KEY=42"', initialLineState) //?
@@ -0,0 +1,15 @@
1
+ # Language Basics Elixir
2
+
3
+
4
+ ## Contributing
5
+
6
+ ```sh
7
+ git clone git@github.com:lvce-editor/language-basics-elixir.git &&
8
+ cd language-basics-elixir &&
9
+ npm ci &&
10
+ npm test
11
+ ```
12
+
13
+ ## Gitpod
14
+
15
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-elixir)
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "builtin.language-basics-elixir",
3
+ "name": "Language Basics Elixir",
4
+ "description": "Provides syntax highlighting and bracket matching in Elixir files.",
5
+ "languages": [
6
+ {
7
+ "id": "elixir",
8
+ "extensions": [".ex"],
9
+ "tokenize": "src/tokenizeElixir.js"
10
+ }
11
+ ]
12
+ }
@@ -0,0 +1,150 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ const State = {
5
+ None: 0,
6
+ TopLevelContent: 1,
7
+ InsideSingleQuoteString: 2,
8
+ InsideDoubleQuoteString: 3,
9
+ InsideTripleQuoteString: 4,
10
+ }
11
+
12
+ /**
13
+ * @enum number
14
+ */
15
+ export const TokenType = {
16
+ None: 99999999,
17
+ Keyword: 951,
18
+ Whitespace: 0,
19
+ NewLine: 771,
20
+ VariableName: 2,
21
+ Punctuation: 3,
22
+ String: 4,
23
+ Numeric: 5,
24
+ Attribute: 6,
25
+ }
26
+
27
+ export const TokenMap = {
28
+ [TokenType.None]: 'None',
29
+ [TokenType.Keyword]: 'Keyword',
30
+ [TokenType.Whitespace]: 'Whitespace',
31
+ [TokenType.NewLine]: 'NewLine',
32
+ [TokenType.VariableName]: 'VariableName',
33
+ [TokenType.Punctuation]: 'Punctuation',
34
+ [TokenType.String]: 'String',
35
+ [TokenType.Numeric]: 'Numeric',
36
+ [TokenType.Attribute]: 'Attribute',
37
+ }
38
+
39
+ export const initialLineState = {
40
+ state: State.TopLevelContent,
41
+ }
42
+
43
+ const RE_KEYWORD =
44
+ /^(?:after|alias|and|case|catch|cond|defmodule|defimpl|defprotocol|defrecord|defstruct|do|else|end|fn|for|if|import|in|not|or|quote|raise|receive|require|reraise|rescue|try|unless|unquote|unquote_splicing|use|when|with\|0)\b/
45
+ const RE_WHITESPACE = /^\s+/
46
+ const RE_VARIABLE_NAME = /^[a-zA-Z]+/
47
+ const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>]/
48
+ const RE_QUOTE_SINGLE = /^'/
49
+ const RE_QUOTE_DOUBLE = /^"/
50
+ const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
51
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
52
+ const RE_NUMERIC = /^\d+/
53
+ const RE_TRIPLE_DOUBLE_QUOTE = /^"""/
54
+ const RE_STRING_TRIPLE_CONTENT = /^.+?(?="""|$)/s
55
+ const RE_ATTRIBUTE = /^@\w+/
56
+
57
+ /**
58
+ * @param {string} line
59
+ * @param {any} lineState
60
+ */
61
+ export const tokenizeLine = (line, lineState) => {
62
+ let next = null
63
+ let index = 0
64
+ let tokens = []
65
+ let token = TokenType.None
66
+ let state = lineState.state
67
+ while (index < line.length) {
68
+ const part = line.slice(index)
69
+ switch (state) {
70
+ case State.TopLevelContent:
71
+ if ((next = part.match(RE_WHITESPACE))) {
72
+ token = TokenType.Whitespace
73
+ state = State.TopLevelContent
74
+ } else if ((next = part.match(RE_KEYWORD))) {
75
+ token = TokenType.Keyword
76
+ state = State.TopLevelContent
77
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
78
+ token = TokenType.VariableName
79
+ state = State.TopLevelContent
80
+ } else if ((next = part.match(RE_PUNCTUATION))) {
81
+ token = TokenType.Punctuation
82
+ state = State.TopLevelContent
83
+ } else if ((next = part.match(RE_NUMERIC))) {
84
+ token = TokenType.Numeric
85
+ state = State.TopLevelContent
86
+ } else if ((next = part.match(RE_QUOTE_SINGLE))) {
87
+ token = TokenType.Punctuation
88
+ state = State.InsideSingleQuoteString
89
+ } else if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
90
+ token = TokenType.Punctuation
91
+ state = State.InsideTripleQuoteString
92
+ } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
93
+ token = TokenType.Punctuation
94
+ state = State.InsideDoubleQuoteString
95
+ } else if ((next = part.match(RE_ATTRIBUTE))) {
96
+ token = TokenType.Attribute
97
+ state = State.TopLevelContent
98
+ } else {
99
+ part //?
100
+ throw new Error('no')
101
+ }
102
+ break
103
+ case State.InsideTripleQuoteString:
104
+ if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
105
+ token = TokenType.Punctuation
106
+ state = State.TopLevelContent
107
+ } else if ((next = part.match(RE_STRING_TRIPLE_CONTENT))) {
108
+ token = TokenType.String
109
+ state = State.InsideTripleQuoteString
110
+ } else {
111
+ throw new Error('no')
112
+ }
113
+ break
114
+ case State.InsideSingleQuoteString:
115
+ if ((next = part.match(RE_QUOTE_SINGLE))) {
116
+ token = TokenType.Punctuation
117
+ state = State.TopLevelContent
118
+ } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
119
+ token = TokenType.String
120
+ state = State.InsideSingleQuoteString
121
+ } else {
122
+ throw new Error('no')
123
+ }
124
+ break
125
+ case State.InsideDoubleQuoteString:
126
+ if ((next = part.match(RE_QUOTE_DOUBLE))) {
127
+ token = TokenType.Punctuation
128
+ state = State.TopLevelContent
129
+ } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
130
+ token = TokenType.String
131
+ state = State.InsideDoubleQuoteString
132
+ } else {
133
+ throw new Error('no')
134
+ }
135
+ break
136
+ default:
137
+ state
138
+ throw new Error('no')
139
+ }
140
+ index += next[0].length
141
+ tokens.push({
142
+ type: token,
143
+ length: next[0].length,
144
+ })
145
+ }
146
+ return {
147
+ state,
148
+ tokens,
149
+ }
150
+ }
@@ -0,0 +1,14 @@
1
+ # Language Basics Go
2
+
3
+ ## Contributing
4
+
5
+ ```sh
6
+ git clone git@github.com:lvce-editor/language-basics-go.git &&
7
+ cd language-basics-go &&
8
+ npm ci &&
9
+ npm test
10
+ ```
11
+
12
+ ## Gitpod
13
+
14
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-go)
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "builtin.language-basics-go",
3
+ "name": "Language Basics Go",
4
+ "description": "Provides syntax highlighting and bracket matching in Gp files.",
5
+ "languages": [
6
+ {
7
+ "id": "go",
8
+ "extensions": [".go"],
9
+ "configuration": "./languageConfiguration.json"
10
+ }
11
+ ]
12
+ }