@lvce-editor/shared-process 0.0.15 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/extensions/builtin.language-basics-cpp/README.md +14 -0
  2. package/extensions/builtin.language-basics-cpp/extension.json +13 -0
  3. package/extensions/builtin.language-basics-cpp/src/tokenizeCpp.js +222 -0
  4. package/extensions/builtin.language-basics-css/README.md +14 -0
  5. package/extensions/builtin.language-basics-css/extension.json +12 -0
  6. package/extensions/builtin.language-basics-css/src/tokenizeCss.js +378 -0
  7. package/extensions/builtin.language-basics-dart/README.md +14 -0
  8. package/extensions/builtin.language-basics-dart/extension.json +12 -0
  9. package/extensions/builtin.language-basics-dart/src/tokenizeDart.js +132 -0
  10. package/extensions/builtin.language-basics-dotenv/README.md +14 -0
  11. package/extensions/builtin.language-basics-dotenv/extension.json +12 -0
  12. package/extensions/builtin.language-basics-dotenv/src/tokenizeDotenv.js +297 -0
  13. package/extensions/builtin.language-basics-elixir/README.md +15 -0
  14. package/extensions/builtin.language-basics-elixir/extension.json +12 -0
  15. package/extensions/builtin.language-basics-elixir/src/tokenizeElixir.js +150 -0
  16. package/extensions/builtin.language-basics-go/README.md +14 -0
  17. package/extensions/builtin.language-basics-go/extension.json +12 -0
  18. package/extensions/builtin.language-basics-go/src/tokenizeGo.js +161 -0
  19. package/extensions/builtin.language-basics-html/README.md +14 -0
  20. package/extensions/builtin.language-basics-html/extension.json +13 -0
  21. package/extensions/builtin.language-basics-html/src/tokenizeHtml.js +278 -0
  22. package/extensions/builtin.language-basics-java/README.md +14 -0
  23. package/extensions/builtin.language-basics-java/extension.json +12 -0
  24. package/extensions/builtin.language-basics-java/src/tokenizeJava.js +135 -0
  25. package/extensions/builtin.language-basics-javascript/README.md +14 -0
  26. package/extensions/builtin.language-basics-javascript/extension.json +13 -0
  27. package/extensions/builtin.language-basics-javascript/src/tokenizeJavaScript.js +295 -0
  28. package/extensions/builtin.language-basics-json/README.md +14 -0
  29. package/extensions/builtin.language-basics-json/extension.json +12 -0
  30. package/extensions/builtin.language-basics-json/src/tokenizeJson.js +323 -0
  31. package/extensions/builtin.language-basics-julia/README.md +18 -0
  32. package/extensions/builtin.language-basics-julia/extension.json +12 -0
  33. package/extensions/builtin.language-basics-julia/src/tokenizeJulia.js +135 -0
  34. package/extensions/builtin.language-basics-kotlin/README.md +15 -0
  35. package/extensions/builtin.language-basics-kotlin/extension.json +12 -0
  36. package/extensions/builtin.language-basics-kotlin/src/tokenizeKotlin.js +127 -0
  37. package/extensions/builtin.language-basics-python/README.md +14 -0
  38. package/extensions/builtin.language-basics-python/extension.json +13 -0
  39. package/extensions/builtin.language-basics-python/src/tokenizePython.js +201 -0
  40. package/extensions/builtin.language-basics-shellscript/README.md +14 -0
  41. package/extensions/builtin.language-basics-shellscript/extension.json +12 -0
  42. package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js +127 -0
  43. package/extensions/builtin.language-basics-typescript/README.md +14 -0
  44. package/extensions/builtin.language-basics-typescript/extension.json +12 -0
  45. package/extensions/builtin.language-basics-typescript/src/tokenizeTypeScript.js +551 -0
  46. package/package.json +3 -3
@@ -0,0 +1,295 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ const State = {
5
+ None: 0,
6
+ TopLevelContent: 1,
7
+ AfterOpeningAngleBracket: 2,
8
+ InsideOpeningTag: 3,
9
+ InsideOpeningTagAndHasSeenWhitespace: 4,
10
+ AfterClosingTagAngleBrackets: 5,
11
+ AfterClosingTagName: 6,
12
+ AfterAttributeName: 7,
13
+ AfterAttributeEqualSign: 8,
14
+ InsideAttributeDoubleQuote: 9,
15
+ InsideBlockComment: 10,
16
+ InsideString: 11,
17
+ AfterKeyword: 12,
18
+ Keyword: 13,
19
+ InsideLineComment: 14,
20
+ InsideSingleQuoteString: 15,
21
+ InsideDoubleQuoteString: 16,
22
+ InsideBacktickString: 17,
23
+ }
24
+
25
+ /**
26
+ * @enum number
27
+ */
28
+ export const TokenType = {
29
+ None: 99999999,
30
+ Numeric: 30,
31
+ String: 50,
32
+ Whitespace: 777,
33
+ Comment: 60,
34
+ Text: 117,
35
+ PunctuationTag: 228,
36
+ TagName: 118,
37
+ AttributeName: 119,
38
+ Punctuation: 10,
39
+ Error: 141,
40
+ PunctuationString: 11,
41
+ NewLine: 771,
42
+ Keyword: 951,
43
+ VariableName: 952,
44
+ LanguageConstant: 13,
45
+ Regex: 14,
46
+ KeywordImport: 215,
47
+ KeywordControl: 881,
48
+ KeywordModifier: 882,
49
+ KeywordReturn: 883,
50
+ KeywordNew: 884,
51
+ }
52
+
53
+ export const TokenMap = {
54
+ [TokenType.None]: 'None',
55
+ [TokenType.Numeric]: 'Numeric',
56
+ [TokenType.String]: 'String',
57
+ [TokenType.Whitespace]: 'Whitespace',
58
+ [TokenType.Comment]: 'Comment',
59
+ [TokenType.Text]: 'Text',
60
+ [TokenType.PunctuationTag]: 'PunctuationTag',
61
+ [TokenType.TagName]: 'TagName',
62
+ [TokenType.AttributeName]: 'AttributeName',
63
+ [TokenType.Punctuation]: 'Punctuation',
64
+ [TokenType.Error]: 'Error',
65
+ [TokenType.PunctuationString]: 'PunctuationString',
66
+ [TokenType.NewLine]: 'NewLine',
67
+ [TokenType.Keyword]: 'Keyword',
68
+ [TokenType.VariableName]: 'VariableName',
69
+ [TokenType.LanguageConstant]: 'LanguageConstant',
70
+ [TokenType.Regex]: 'Regex',
71
+ [TokenType.KeywordImport]: 'KeywordImport',
72
+ [TokenType.KeywordControl]: 'KeywordControl',
73
+ [TokenType.KeywordModifier]: 'KeywordModifier',
74
+ [TokenType.KeywordReturn]: 'KeywordReturn',
75
+ [TokenType.KeywordNew]: 'KeywordNew',
76
+ }
77
+
78
+ const RE_KEYWORD =
79
+ /^(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|false|finally|for|from|function|if|implements|import|in|instanceof|interface|let|new|null|package|private|protected|public|return|super|switch|static|this|throw|try|true|typeof|var|void|while|with|yield)\b/
80
+ const RE_CURLY_OPEN = /^\{/
81
+ const RE_CURLY_CLOSE = /^\}/
82
+ const RE_SQUARE_OPEN = /^\[/
83
+ const RE_SQUARE_CLOSE = /^\]/
84
+ const RE_COMMA = /^,/
85
+ const RE_COLON = /^:/
86
+ const RE_NUMERIC =
87
+ /^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
88
+ const RE_LINE_COMMENT_START = /^\/\//
89
+ const RE_LINE_COMMENT_CONTENT = /^[^\n]+/
90
+ const RE_NEWLINE_WHITESPACE = /^\n\s*/
91
+ const RE_BLOCK_COMMENT_START = /^\/\*/
92
+ const RE_BLOCK_COMMENT_CONTENT = /^.+?(?=\*\/)/
93
+ const RE_BLOCK_COMMENT_END = /^\*\//
94
+ const RE_UNKNOWN_VALUE = /^[^\}\{\s,"]+/
95
+ const RE_IMPORT = /^[a-zA-Z\.]+/
96
+ const RE_SEMICOLON = /^;/
97
+ const RE_VARIABLE_NAME = /^[a-zA-Z_][a-zA-Z\d\_]*/
98
+ const RE_LINE_COMMENT = /^\/\/[^\n]*/
99
+ const RE_ROUND_OPEN = /^\(/
100
+ const RE_ROUND_CLOSE = /^\)/
101
+ const RE_DOT = /^\./
102
+ const RE_EQUAL_SIGN = /^=/
103
+ const RE_SINGLE_QUOTE = /^'/
104
+ const RE_PUNCTUATION = /^[\(\)=\+\-><\.:;\{\}\[\]!,&\|\^\?\*]/
105
+ const RE_SLASH = /^\//
106
+ const RE_ANYTHING_UNTIL_END = /^.+/s
107
+ const RE_WHITESPACE = /^\s+/
108
+ const RE_QUOTE_SINGLE = /^'/
109
+ const RE_QUOTE_DOUBLE = /^"/
110
+ const RE_QUOTE_BACKTICK = /^`/
111
+ const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
112
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
113
+ const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`]+/
114
+
115
+ // copied from https://github.com/PrismJS/prism/blob/master/components/prism-javascript.js#L57
116
+ const RE_REGEX =
117
+ /((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/
118
+
119
+ export const initialLineState = {
120
+ state: State.TopLevelContent,
121
+ }
122
+
123
+ export const tokenizeLine = (line, lineState) => {
124
+ let next = null
125
+ let index = 0
126
+ let tokens = []
127
+ let token = TokenType.None
128
+ let state = lineState.state
129
+ while (index < line.length) {
130
+ const part = line.slice(index)
131
+ switch (state) {
132
+ case State.TopLevelContent:
133
+ if ((next = part.match(RE_WHITESPACE))) {
134
+ token = TokenType.Whitespace
135
+ state = State.TopLevelContent
136
+ } else if ((next = part.match(RE_KEYWORD))) {
137
+ switch (next[0]) {
138
+ case 'true':
139
+ case 'false':
140
+ case 'null':
141
+ token = TokenType.LanguageConstant
142
+ break
143
+ case 'import':
144
+ case 'export':
145
+ case 'from':
146
+ token = TokenType.KeywordImport
147
+ break
148
+ case 'as':
149
+ case 'switch':
150
+ case 'default':
151
+ case 'case':
152
+ case 'else':
153
+ case 'if':
154
+ case 'break':
155
+ case 'throw':
156
+ token = TokenType.KeywordControl
157
+ break
158
+ case 'async':
159
+ case 'await':
160
+ token = TokenType.KeywordModifier
161
+ break
162
+ case 'return':
163
+ token = TokenType.KeywordReturn
164
+ break
165
+ case 'new':
166
+ token = TokenType.KeywordNew
167
+ break
168
+ default:
169
+ token = TokenType.Keyword
170
+ break
171
+ }
172
+ state = State.TopLevelContent
173
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
174
+ token = TokenType.VariableName
175
+ state = State.TopLevelContent
176
+ } else if ((next = part.match(RE_NUMERIC))) {
177
+ token = TokenType.Numeric
178
+ state = State.TopLevelContent
179
+ } else if ((next = part.match(RE_PUNCTUATION))) {
180
+ token = TokenType.Punctuation
181
+ state = State.TopLevelContent
182
+ } else if ((next = part.match(RE_SLASH))) {
183
+ if ((next = part.match(RE_BLOCK_COMMENT_START))) {
184
+ token = TokenType.Comment
185
+ state = State.InsideBlockComment
186
+ } else if ((next = part.match(RE_REGEX))) {
187
+ token = TokenType.Regex
188
+ state = State.TopLevelContent
189
+ } else if ((next = part.match(RE_LINE_COMMENT_START))) {
190
+ token = TokenType.Comment
191
+ state = State.InsideLineComment
192
+ } else {
193
+ next = part.match(RE_PUNCTUATION)
194
+ token = TokenType.Punctuation
195
+ state = State.TopLevelContent
196
+ }
197
+ } else if ((next = part.match(RE_QUOTE_SINGLE))) {
198
+ token = TokenType.Punctuation
199
+ state = State.InsideSingleQuoteString
200
+ } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
201
+ token = TokenType.Punctuation
202
+ state = State.InsideDoubleQuoteString
203
+ } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
204
+ token = TokenType.Punctuation
205
+ state = State.InsideBacktickString
206
+ } else {
207
+ line.slice(0, index) //?
208
+ part //?
209
+ throw new Error('no')
210
+ }
211
+ break
212
+ case State.Keyword:
213
+ const keyword = next[0]
214
+ switch (keyword) {
215
+ case 'true':
216
+ case 'false':
217
+ break
218
+ default:
219
+ token = TokenType.Keyword
220
+ state = State.TopLevelContent
221
+ break
222
+ }
223
+ break
224
+ case State.InsideLineComment:
225
+ if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
226
+ token = TokenType.Comment
227
+ state = State.TopLevelContent
228
+ } else {
229
+ throw new Error('no')
230
+ }
231
+ break
232
+ case State.InsideBlockComment:
233
+ if ((next = part.match(RE_BLOCK_COMMENT_CONTENT))) {
234
+ token = TokenType.Comment
235
+ state = State.InsideBlockComment
236
+ } else if ((next = part.match(RE_BLOCK_COMMENT_END))) {
237
+ token = TokenType.Comment
238
+ state = State.TopLevelContent
239
+ } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
240
+ token = TokenType.Comment
241
+ state = State.InsideBlockComment
242
+ } else {
243
+ throw new Error('no')
244
+ }
245
+ break
246
+ case State.InsideSingleQuoteString:
247
+ if ((next = part.match(RE_QUOTE_SINGLE))) {
248
+ token = TokenType.Punctuation
249
+ state = State.TopLevelContent
250
+ } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
251
+ token = TokenType.String
252
+ state = State.InsideSingleQuoteString
253
+ } else {
254
+ throw new Error('no')
255
+ }
256
+ break
257
+ case State.InsideDoubleQuoteString:
258
+ if ((next = part.match(RE_QUOTE_DOUBLE))) {
259
+ token = TokenType.Punctuation
260
+ state = State.TopLevelContent
261
+ } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
262
+ token = TokenType.String
263
+ state = State.InsideDoubleQuoteString
264
+ } else {
265
+ throw new Error('no')
266
+ }
267
+ break
268
+ case State.InsideBacktickString:
269
+ if ((next = part.match(RE_QUOTE_BACKTICK))) {
270
+ token = TokenType.Punctuation
271
+ state = State.TopLevelContent
272
+ } else if ((next = part.match(RE_STRING_BACKTICK_QUOTE_CONTENT))) {
273
+ token = TokenType.String
274
+ state = State.InsideBacktickString
275
+ } else {
276
+ throw new Error('no')
277
+ }
278
+ break
279
+ default:
280
+ state
281
+ throw new Error('no')
282
+ }
283
+ index += next[0].length
284
+ tokens.push({
285
+ type: token,
286
+ length: next[0].length,
287
+ })
288
+ }
289
+ return {
290
+ state,
291
+ tokens,
292
+ }
293
+ }
294
+
295
+ tokenizeLine(`\`\``, initialLineState) //?
@@ -0,0 +1,14 @@
1
+ # Language Basics JSON
2
+
3
+ ## Contributing
4
+
5
+ ```sh
6
+ git clone git@github.com:lvce-editor/language-basics-json.git &&
7
+ cd language-basics-json &&
8
+ npm ci &&
9
+ npm test
10
+ ```
11
+
12
+ ## Gitpod
13
+
14
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-json)
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "builtin.language-basics-json",
3
+ "name": "Language Basics JSON",
4
+ "description": "Provides syntax highlighting and bracket matching in JSON files.",
5
+ "languages": [
6
+ {
7
+ "id": "json",
8
+ "extensions": [".json"],
9
+ "tokenize": "src/tokenizeJson.js"
10
+ }
11
+ ]
12
+ }
@@ -0,0 +1,323 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ export const State = {
5
+ TopLevelContent: 1,
6
+ AfterCurlyOpen: 2,
7
+ InsideString: 3,
8
+ AfterPropertyName: 4,
9
+ AfterPropertyNameAfterColon: 5,
10
+ AfterPropertyValue: 6,
11
+ JsonPropertyKey: 7,
12
+ InsideLineComment: 8,
13
+ InsidePropertyNameString: 9,
14
+ }
15
+
16
+ export const StateMap = {
17
+ [State.TopLevelContent]: 'TopLevelContent',
18
+ [State.AfterCurlyOpen]: 'AfterCurlyOpen',
19
+ [State.InsideString]: 'InsideString',
20
+ [State.AfterPropertyName]: 'AfterPropertyName',
21
+ [State.AfterPropertyNameAfterColon]: 'AfterPropertyNameAfterColon',
22
+ [State.AfterPropertyValue]: 'AfterPropertyValue',
23
+ [State.JsonPropertyKey]: 'JsonPropertyKey',
24
+ }
25
+
26
+ /**
27
+ * @enum number
28
+ */
29
+ export const TokenType = {
30
+ CssSelector: 1,
31
+ Whitespace: 2,
32
+ Punctuation: 3,
33
+ PropertyName: 4,
34
+ PropertyValue: 5,
35
+ CurlyOpen: 6,
36
+ CurlyClose: 7,
37
+ PropertyColon: 8,
38
+ PropertySemicolon: 9,
39
+ String: 10,
40
+ LanguageConstant: 11,
41
+ NewLine: 12,
42
+ JsonPropertyName: 13,
43
+ PropertyValueString: 14,
44
+ Numeric: 15,
45
+ LanguageConstantBoolean: 16,
46
+ Comment: 17,
47
+ Error: 18,
48
+ Unknown: 19,
49
+ Text: 20,
50
+ }
51
+
52
+ export const TokenMap = {
53
+ [TokenType.CssSelector]: 'CssSelector',
54
+ [TokenType.Whitespace]: 'Whitespace',
55
+ [TokenType.Punctuation]: 'Punctuation',
56
+ [TokenType.PropertyName]: 'PropertyName',
57
+ [TokenType.PropertyValue]: 'PropertyValue',
58
+ [TokenType.CurlyOpen]: 'CurlyOpen',
59
+ [TokenType.CurlyClose]: 'CurlyClose',
60
+ [TokenType.PropertyColon]: 'PropertyColon',
61
+ [TokenType.PropertySemicolon]: 'PropertySemicolon',
62
+ [TokenType.String]: 'String',
63
+ [TokenType.LanguageConstant]: 'LanguageConstant',
64
+ [TokenType.LanguageConstantBoolean]: 'LanguageConstantBoolean',
65
+ [TokenType.JsonPropertyName]: 'JsonPropertyName',
66
+ [TokenType.PropertyValueString]: 'PropertyValueString',
67
+ [TokenType.Comment]: 'Comment',
68
+ [TokenType.Error]: 'Error',
69
+ [TokenType.Unknown]: 'Unknown',
70
+ [TokenType.NewLine]: 'NewLine',
71
+ [TokenType.Text]: 'Text',
72
+ [TokenType.Numeric]: 'Numeric',
73
+ }
74
+
75
+ const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
76
+ const RE_WHITESPACE = /^\s+/
77
+ const RE_CURLY_OPEN = /^\{/
78
+ const RE_CURLY_CLOSE = /^\}/
79
+ const RE_PROPERTY_NAME = /^[a-zA-Z\-]+/
80
+ const RE_COLON = /^:/
81
+ const RE_PROPERTY_VALUE = /^[^\n;]+/
82
+ const RE_SEMICOLON = /^;/
83
+ const RE_COMMA = /^,/
84
+ const RE_DOUBLE_QUOTE = /^"/
85
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
86
+ const RE_LANGUAGE_CONSTANT = /^(?:true|false|null)/
87
+ const RE_SQUARE_OPEN = /^\[/
88
+ const RE_SQUARE_CLOSE = /^\]/
89
+ const RE_LINE_COMMENT = /^\/\/.*/
90
+ const RE_ANYTHING = /^.+/s
91
+ const RE_NUMERIC =
92
+ /^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
93
+ const RE_TEXT = /^[^\s\{\}\[\]]+/
94
+ const RE_ESCAPED_QUOTE = /^\\"/
95
+ const RE_BACK_SLASH = /^\\/
96
+
97
+ export const initialLineState = {
98
+ state: State.TopLevelContent,
99
+ stack: [],
100
+ }
101
+
102
+ /**
103
+ * @param {string} line
104
+ */
105
+ export const tokenizeLine = (line, lineState) => {
106
+ let next
107
+ let index = 0
108
+ let token
109
+ let state = lineState.state
110
+ let __r = 0
111
+ const tokens = []
112
+ const stack = lineState.stack
113
+ while (index < line.length) {
114
+ if (__r++ > 100_000_000_000) {
115
+ throw new Error('endless loop')
116
+ }
117
+ const part = line.slice(index)
118
+ switch (state) {
119
+ case State.TopLevelContent:
120
+ if ((next = part.match(RE_CURLY_OPEN))) {
121
+ token = TokenType.Punctuation
122
+ state = State.AfterCurlyOpen
123
+ stack.push(State.TopLevelContent)
124
+ } else if ((next = part.match(RE_SQUARE_OPEN))) {
125
+ token = TokenType.Punctuation
126
+ state = State.TopLevelContent
127
+ stack.push(State.TopLevelContent)
128
+ } else if ((next = part.match(RE_WHITESPACE))) {
129
+ token = TokenType.Whitespace
130
+ state = State.TopLevelContent
131
+ } else if ((next = part.match(RE_SQUARE_CLOSE))) {
132
+ token = TokenType.Punctuation
133
+ state = stack.pop()
134
+ } else if ((next = part.match(RE_COMMA))) {
135
+ token = TokenType.Punctuation
136
+ state = State.TopLevelContent
137
+ } else if ((next = part.match(RE_NUMERIC))) {
138
+ token = TokenType.Numeric
139
+ state = State.TopLevelContent
140
+ } else if ((next = part.match(RE_LANGUAGE_CONSTANT))) {
141
+ token = TokenType.LanguageConstant
142
+ state = State.TopLevelContent
143
+ } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
144
+ token = TokenType.Punctuation
145
+ state = State.InsideString
146
+ stack.push(State.TopLevelContent)
147
+ } else if ((next = part.match(RE_LINE_COMMENT))) {
148
+ token = TokenType.Comment
149
+ state = State.InsideLineComment
150
+ } else if ((next = part.match(RE_CURLY_CLOSE))) {
151
+ token = TokenType.CurlyClose
152
+ state = stack.pop()
153
+ } else if ((next = part.match(RE_ANYTHING))) {
154
+ token = TokenType.Text
155
+ state = State.TopLevelContent
156
+ } else {
157
+ part //?
158
+ throw new Error('no')
159
+ }
160
+ break
161
+ case State.AfterCurlyOpen:
162
+ if ((next = part.match(RE_WHITESPACE))) {
163
+ token = TokenType.Whitespace
164
+ state = State.AfterCurlyOpen
165
+ } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
166
+ token = TokenType.Punctuation
167
+ state = State.InsidePropertyNameString
168
+ stack.push(State.AfterPropertyName)
169
+ } else if ((next = part.match(RE_CURLY_CLOSE))) {
170
+ token = TokenType.Punctuation
171
+ state = stack.pop()
172
+ if (!state) {
173
+ throw new Error('imbalanced json')
174
+ }
175
+ } else if ((next = part.match(RE_LINE_COMMENT))) {
176
+ token = TokenType.Comment
177
+ state = State.AfterCurlyOpen
178
+ } else {
179
+ part //?
180
+ throw new Error('no')
181
+ }
182
+ break
183
+ case State.InsidePropertyNameString:
184
+ if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
185
+ token = TokenType.JsonPropertyName
186
+ state = State.InsidePropertyNameString
187
+ } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
188
+ token = TokenType.Punctuation
189
+ state = stack.pop()
190
+ } else if ((next = part.match(RE_ESCAPED_QUOTE))) {
191
+ token = TokenType.JsonPropertyName
192
+ state = State.InsidePropertyNameString
193
+ } else if ((next = part.match(RE_BACK_SLASH))) {
194
+ token = TokenType.JsonPropertyName
195
+ state = State.InsidePropertyNameString
196
+ } else {
197
+ throw new Error('no')
198
+ }
199
+ break
200
+ case State.InsideString:
201
+ if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
202
+ token = TokenType.String
203
+ state = State.InsideString
204
+ } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
205
+ token = TokenType.Punctuation
206
+ state = stack.pop()
207
+ } else if ((next = part.match(RE_ESCAPED_QUOTE))) {
208
+ token = TokenType.String
209
+ state = State.InsideString
210
+ } else if ((next = part.match(RE_BACK_SLASH))) {
211
+ token = TokenType.String
212
+ state = State.InsideString
213
+ } else {
214
+ line.slice(0, index) //?
215
+ part //?
216
+ throw new Error('no')
217
+ }
218
+ break
219
+ case State.AfterPropertyName:
220
+ if ((next = part.match(RE_COLON))) {
221
+ token = TokenType.Punctuation
222
+ state = State.AfterPropertyNameAfterColon
223
+ } else {
224
+ throw new Error('no')
225
+ }
226
+ break
227
+ case State.AfterPropertyNameAfterColon:
228
+ if ((next = part.match(RE_WHITESPACE))) {
229
+ token = TokenType.Whitespace
230
+ state = State.AfterPropertyNameAfterColon
231
+ } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
232
+ line.slice(0, index) //?
233
+ token = TokenType.Punctuation
234
+ state = State.InsideString
235
+ stack.push(State.AfterPropertyValue)
236
+ } else if ((next = part.match(RE_LANGUAGE_CONSTANT))) {
237
+ token = TokenType.LanguageConstant
238
+ state = State.AfterPropertyValue
239
+ } else if ((next = part.match(RE_NUMERIC))) {
240
+ token = TokenType.Numeric
241
+ state = State.AfterPropertyValue
242
+ } else if ((next = part.match(RE_TEXT))) {
243
+ token = TokenType.Text
244
+ state = State.TopLevelContent
245
+ } else if ((next = part.match(RE_CURLY_OPEN))) {
246
+ token = TokenType.Punctuation
247
+ state = State.AfterCurlyOpen
248
+ stack.push(State.AfterPropertyValue)
249
+ } else if ((next = part.match(RE_SQUARE_OPEN))) {
250
+ token = TokenType.Punctuation
251
+ state = State.TopLevelContent
252
+ stack.push(State.AfterPropertyValue)
253
+ } else {
254
+ line.slice(0, index) //?
255
+ throw new Error('no')
256
+ }
257
+ break
258
+ case State.AfterPropertyValue:
259
+ if ((next = part.match(RE_CURLY_CLOSE))) {
260
+ token = TokenType.Punctuation
261
+ state = stack.pop()
262
+ if (!state) {
263
+ throw new Error('inbalanced json')
264
+ }
265
+ } else if ((next = part.match(RE_WHITESPACE))) {
266
+ token = TokenType.Whitespace
267
+ state = State.AfterPropertyValue
268
+ } else if ((next = part.match(RE_COMMA))) {
269
+ token = TokenType.Punctuation
270
+ state = State.AfterCurlyOpen
271
+ } else {
272
+ part //?
273
+ throw new Error('no')
274
+ }
275
+ break
276
+ case State.InsideLineComment:
277
+ if ((next = part.match(RE_ANYTHING))) {
278
+ token = TokenType.Comment
279
+ state = State.TopLevelContent
280
+ } else {
281
+ throw new Error('no')
282
+ }
283
+ break
284
+ default:
285
+ state
286
+ throw new Error('no')
287
+ }
288
+
289
+ const currentTokenText = next[0]
290
+ const currentTokenLength = currentTokenText.length
291
+ index += currentTokenLength
292
+ tokens.push({
293
+ type: token,
294
+ length: currentTokenLength,
295
+ })
296
+ }
297
+ if (state === State.InsideLineComment) {
298
+ state = State.TopLevelContent
299
+ }
300
+ return {
301
+ state,
302
+ stack,
303
+ tokens,
304
+ }
305
+ }
306
+
307
+ // const result = tokenizeLine(` "a": "b",`, {
308
+ // state: State.AfterCurlyOpen,
309
+ // }) //?
310
+
311
+ const result = tokenizeLine(
312
+ `{ "testRegex": "/scripts/jest/dont-run-jest-directly\\.js$"`,
313
+ {
314
+ state: State.TopLevelContent,
315
+ stack: [],
316
+ }
317
+ ) //?
318
+
319
+ result.tokens.map((t) => TokenMap[t.type]) //?
320
+
321
+ // tokenizeLine(`{
322
+ // "a": "b"
323
+ // }`) //?
@@ -0,0 +1,18 @@
1
+ # Language Basics Julia
2
+
3
+ ## Contributing
4
+
5
+ ```sh
6
+ git clone git@github.com:lvce-editor/language-basics-julia.git &&
7
+ cd language-basics-julia &&
8
+ npm ci &&
9
+ npm test
10
+ ```
11
+
12
+ ## Gitpod
13
+
14
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-julia)
15
+
16
+ ## Credits
17
+
18
+ Based on https://github.com/highlightjs/highlight.js/blob/main/src/languages/julia.js by @isagalaev and contributers.
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "builtin.language-basics-julia",
3
+ "name": "Language Basics Julia",
4
+ "description": "Provides syntax highlighting and bracket matching in Julia files.",
5
+ "languages": [
6
+ {
7
+ "id": "julia",
8
+ "extensions": [".jl"],
9
+ "tokenize": "src/tokenizeJulia.js"
10
+ }
11
+ ]
12
+ }