@lvce-editor/shared-process 0.0.14 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/extensions/builtin.language-basics-cpp/README.md +14 -0
  2. package/extensions/builtin.language-basics-cpp/extension.json +13 -0
  3. package/extensions/builtin.language-basics-cpp/src/tokenizeCpp.js +222 -0
  4. package/extensions/builtin.language-basics-css/README.md +14 -0
  5. package/extensions/builtin.language-basics-css/extension.json +12 -0
  6. package/extensions/builtin.language-basics-css/src/tokenizeCss.js +378 -0
  7. package/extensions/builtin.language-basics-dart/README.md +14 -0
  8. package/extensions/builtin.language-basics-dart/extension.json +12 -0
  9. package/extensions/builtin.language-basics-dart/src/tokenizeDart.js +132 -0
  10. package/extensions/builtin.language-basics-dotenv/README.md +14 -0
  11. package/extensions/builtin.language-basics-dotenv/extension.json +12 -0
  12. package/extensions/builtin.language-basics-dotenv/src/tokenizeDotenv.js +297 -0
  13. package/extensions/builtin.language-basics-elixir/README.md +15 -0
  14. package/extensions/builtin.language-basics-elixir/extension.json +12 -0
  15. package/extensions/builtin.language-basics-elixir/src/tokenizeElixir.js +150 -0
  16. package/extensions/builtin.language-basics-go/README.md +14 -0
  17. package/extensions/builtin.language-basics-go/extension.json +12 -0
  18. package/extensions/builtin.language-basics-go/src/tokenizeGo.js +161 -0
  19. package/extensions/builtin.language-basics-html/README.md +14 -0
  20. package/extensions/builtin.language-basics-html/extension.json +13 -0
  21. package/extensions/builtin.language-basics-html/src/tokenizeHtml.js +278 -0
  22. package/extensions/builtin.language-basics-java/README.md +14 -0
  23. package/extensions/builtin.language-basics-java/extension.json +12 -0
  24. package/extensions/builtin.language-basics-java/src/tokenizeJava.js +135 -0
  25. package/extensions/builtin.language-basics-javascript/README.md +14 -0
  26. package/extensions/builtin.language-basics-javascript/extension.json +13 -0
  27. package/extensions/builtin.language-basics-javascript/src/tokenizeJavaScript.js +295 -0
  28. package/extensions/builtin.language-basics-json/README.md +14 -0
  29. package/extensions/builtin.language-basics-json/extension.json +12 -0
  30. package/extensions/builtin.language-basics-json/src/tokenizeJson.js +323 -0
  31. package/extensions/builtin.language-basics-julia/README.md +18 -0
  32. package/extensions/builtin.language-basics-julia/extension.json +12 -0
  33. package/extensions/builtin.language-basics-julia/src/tokenizeJulia.js +135 -0
  34. package/extensions/builtin.language-basics-kotlin/README.md +15 -0
  35. package/extensions/builtin.language-basics-kotlin/extension.json +12 -0
  36. package/extensions/builtin.language-basics-kotlin/src/tokenizeKotlin.js +127 -0
  37. package/extensions/builtin.language-basics-python/README.md +14 -0
  38. package/extensions/builtin.language-basics-python/extension.json +13 -0
  39. package/extensions/builtin.language-basics-python/src/tokenizePython.js +201 -0
  40. package/extensions/builtin.language-basics-shellscript/README.md +14 -0
  41. package/extensions/builtin.language-basics-shellscript/extension.json +12 -0
  42. package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js +127 -0
  43. package/package.json +3 -3
@@ -0,0 +1,135 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ const State = {
5
+ None: 0,
6
+ TopLevelContent: 1,
7
+ InsideSingleQuoteString: 2,
8
+ InsideDoubleQuoteString: 3,
9
+ InsideTripleQuoteString: 4,
10
+ }
11
+
12
+ /**
13
+ * @enum number
14
+ */
15
+ export const TokenType = {
16
+ None: 99999999,
17
+ Keyword: 951,
18
+ Whitespace: 0,
19
+ NewLine: 771,
20
+ VariableName: 2,
21
+ Punctuation: 3,
22
+ String: 4,
23
+ Numeric: 5,
24
+ Attribute: 6,
25
+ }
26
+
27
+ export const TokenMap = {
28
+ [TokenType.None]: 'None',
29
+ [TokenType.Keyword]: 'Keyword',
30
+ [TokenType.Whitespace]: 'Whitespace',
31
+ [TokenType.NewLine]: 'NewLine',
32
+ [TokenType.VariableName]: 'VariableName',
33
+ [TokenType.Punctuation]: 'Punctuation',
34
+ [TokenType.String]: 'String',
35
+ [TokenType.Numeric]: 'Numeric',
36
+ [TokenType.Attribute]: 'Attribute',
37
+ }
38
+
39
+ export const initialLineState = {
40
+ state: State.TopLevelContent,
41
+ }
42
+
43
+ const RE_KEYWORD =
44
+ /^(?:baremodule|begin|break|catch|ccall|const|continue|do|else|elseif|end|export|false|finally|for|function|global|if|import|in|isa|let|local|macro|module|quote|return|true|try|using|where|while)\b/
45
+ const RE_WHITESPACE = /^\s+/
46
+ const RE_VARIABLE_NAME = /^[A-Za-z_\u00A1-\uFFFF][A-Za-z_0-9\u00A1-\uFFFF]*/
47
+ const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>\^\+\-\<\?]/
48
+ const RE_QUOTE_SINGLE = /^'/
49
+ const RE_QUOTE_DOUBLE = /^"/
50
+ const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
51
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
52
+ const RE_NUMERIC = /^\d+/
53
+
54
+ /**
55
+ * @param {string} line
56
+ * @param {any} lineState
57
+ */
58
+ export const tokenizeLine = (line, lineState) => {
59
+ let next = null
60
+ let index = 0
61
+ let tokens = []
62
+ let token = TokenType.None
63
+ let state = lineState.state
64
+ while (index < line.length) {
65
+ const part = line.slice(index)
66
+ switch (state) {
67
+ case State.TopLevelContent:
68
+ if ((next = part.match(RE_WHITESPACE))) {
69
+ token = TokenType.Whitespace
70
+ state = State.TopLevelContent
71
+ } else if ((next = part.match(RE_KEYWORD))) {
72
+ token = TokenType.Keyword
73
+ state = State.TopLevelContent
74
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
75
+ token = TokenType.VariableName
76
+ state = State.TopLevelContent
77
+ } else if ((next = part.match(RE_PUNCTUATION))) {
78
+ token = TokenType.Punctuation
79
+ state = State.TopLevelContent
80
+ } else if ((next = part.match(RE_NUMERIC))) {
81
+ token = TokenType.Numeric
82
+ state = State.TopLevelContent
83
+ } else if ((next = part.match(RE_QUOTE_SINGLE))) {
84
+ token = TokenType.Punctuation
85
+ state = State.InsideSingleQuoteString
86
+ } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
87
+ token = TokenType.Punctuation
88
+ state = State.InsideDoubleQuoteString
89
+ } else {
90
+ part //?
91
+ throw new Error('no')
92
+ }
93
+ break
94
+ case State.InsideSingleQuoteString:
95
+ if ((next = part.match(RE_QUOTE_SINGLE))) {
96
+ token = TokenType.Punctuation
97
+ state = State.TopLevelContent
98
+ } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
99
+ token = TokenType.String
100
+ state = State.InsideSingleQuoteString
101
+ } else {
102
+ throw new Error('no')
103
+ }
104
+ break
105
+ case State.InsideDoubleQuoteString:
106
+ if ((next = part.match(RE_QUOTE_DOUBLE))) {
107
+ token = TokenType.Punctuation
108
+ state = State.TopLevelContent
109
+ } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
110
+ token = TokenType.String
111
+ state = State.InsideDoubleQuoteString
112
+ } else {
113
+ throw new Error('no')
114
+ }
115
+ break
116
+ default:
117
+ state
118
+ throw new Error('no')
119
+ }
120
+ index += next[0].length
121
+ tokens.push({
122
+ type: token,
123
+ length: next[0].length,
124
+ })
125
+ }
126
+ return {
127
+ state,
128
+ tokens,
129
+ }
130
+ }
131
+
132
+ tokenizeLine(
133
+ `abs(mandelbrot(complex(x, y))) < 2 ? print("*") : print(" ")`,
134
+ initialLineState
135
+ )
@@ -0,0 +1,15 @@
1
+ # Language Basics Kotlin
2
+
3
+
4
+ ## Contributing
5
+
6
+ ```sh
7
+ git clone git@github.com:lvce-editor/language-basics-kotlin.git &&
8
+ cd language-basics-kotlin &&
9
+ npm ci &&
10
+ npm test
11
+ ```
12
+
13
+ ## Gitpod
14
+
15
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-kotlin)
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "builtin.language-basics-kotlin",
3
+ "name": "Language Basics Kotlin",
4
+ "description": "Provides syntax highlighting and bracket matching in Kotlin files.",
5
+ "languages": [
6
+ {
7
+ "id": "kotlin",
8
+ "extensions": [".kt"],
9
+ "tokenize": "src/tokenizeKotlin.js"
10
+ }
11
+ ]
12
+ }
@@ -0,0 +1,127 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ export const State = {
5
+ TopLevelContent: 1,
6
+ InsideString: 2,
7
+ }
8
+
9
+ export const StateMap = {
10
+ [State.TopLevelContent]: 'TopLevelContent',
11
+ }
12
+
13
+ /**
14
+ * @enum number
15
+ */
16
+ export const TokenType = {
17
+ None: 1,
18
+ Whitespace: 2,
19
+ PunctuationString: 3,
20
+ String: 4,
21
+ Keyword: 5,
22
+ Numeric: 6,
23
+ Punctuation: 7,
24
+ VariableName: 8,
25
+ }
26
+
27
+ export const TokenMap = {
28
+ [TokenType.None]: 'None',
29
+ [TokenType.Whitespace]: 'Whitespace',
30
+ [TokenType.PunctuationString]: 'PunctuationString',
31
+ [TokenType.String]: 'String',
32
+ [TokenType.Keyword]: 'Keyword',
33
+ [TokenType.Numeric]: 'Numeric',
34
+ [TokenType.Punctuation]: 'Punctuation',
35
+ [TokenType.VariableName]: 'VariableName',
36
+ }
37
+
38
+ const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
39
+ const RE_WHITESPACE = /^ +/
40
+ const RE_CURLY_OPEN = /^\{/
41
+ const RE_CURLY_CLOSE = /^\}/
42
+ const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
43
+ const RE_COLON = /^:/
44
+ const RE_PROPERTY_VALUE = /^[^;\}]+/
45
+ const RE_SEMICOLON = /^;/
46
+ const RE_COMMA = /^,/
47
+ const RE_ANYTHING = /^.*/
48
+ const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
49
+ const RE_QUOTE_DOUBLE = /^"/
50
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
51
+ const RE_KEYWORD =
52
+ /^(?:while|when|var|val|typeof|typealias|try|true|throw|this|super|return|package|object|null|is|interface|in|if|fun|for|false|else|do|continue|class|break|as)\b/
53
+
54
+ const RE_VARIABLE_NAME = /^[a-zA-Z]+/
55
+ const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
56
+ const RE_NUMERIC = /^\d+/
57
+
58
+ export const initialLineState = {
59
+ state: State.TopLevelContent,
60
+ tokens: [],
61
+ }
62
+
63
+ export const isEqualLineState = (lineStateA, lineStateB) => {
64
+ return lineStateA.state === lineStateB.state
65
+ }
66
+
67
+ /**
68
+ * @param {string} line
69
+ */
70
+ export const tokenizeLine = (line, lineState) => {
71
+ let next = null
72
+ let index = 0
73
+ let tokens = []
74
+ let token = TokenType.None
75
+ let state = lineState.state
76
+ while (index < line.length) {
77
+ const part = line.slice(index)
78
+ switch (state) {
79
+ case State.TopLevelContent:
80
+ if ((next = part.match(RE_WHITESPACE))) {
81
+ token = TokenType.Whitespace
82
+ state = State.TopLevelContent
83
+ } else if ((next = part.match(RE_KEYWORD))) {
84
+ token = TokenType.Keyword
85
+ state = State.TopLevelContent
86
+ } else if ((next = part.match(RE_PUNCTUATION))) {
87
+ token = TokenType.Punctuation
88
+ state = State.TopLevelContent
89
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
90
+ token = TokenType.VariableName
91
+ state = State.TopLevelContent
92
+ } else if ((next = part.match(RE_NUMERIC))) {
93
+ token = TokenType.Numeric
94
+ state = State.TopLevelContent
95
+ } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
96
+ token = TokenType.PunctuationString
97
+ state = State.InsideString
98
+ } else {
99
+ part //?
100
+ throw new Error('no')
101
+ }
102
+ break
103
+ case State.InsideString:
104
+ if ((next = part.match(RE_QUOTE_DOUBLE))) {
105
+ token = TokenType.PunctuationString
106
+ state = State.TopLevelContent
107
+ } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
108
+ token = TokenType.String
109
+ state = State.InsideString
110
+ } else {
111
+ throw new Error('no')
112
+ }
113
+ break
114
+ default:
115
+ throw new Error('no')
116
+ }
117
+ index += next[0].length
118
+ tokens.push({
119
+ type: token,
120
+ length: next[0].length,
121
+ })
122
+ }
123
+ return {
124
+ state,
125
+ tokens,
126
+ }
127
+ }
@@ -0,0 +1,14 @@
1
+ # Language Basics Python
2
+
3
+ ## Contributing
4
+
5
+ ```sh
6
+ git clone git@github.com:lvce-editor/language-basics-python.git &&
7
+ cd language-basics-python &&
8
+ npm ci &&
9
+ npm test
10
+ ```
11
+
12
+ ## Gitpod
13
+
14
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-python)
@@ -0,0 +1,13 @@
1
+ {
2
+ "id": "builtin.language-basics-python",
3
+ "name": "Language Basics Python",
4
+ "description": "Provides syntax highlighting and bracket matching in Python files.",
5
+ "languages": [
6
+ {
7
+ "id": "python",
8
+ "extensions": [".py"],
9
+ "configuration": "./languageConfiguration.json",
10
+ "tokenize": "src/tokenizePython.js"
11
+ }
12
+ ]
13
+ }
@@ -0,0 +1,201 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ export const State = {
5
+ None: 0,
6
+ TopLevelContent: 1,
7
+ AfterOpeningAngleBracket: 2,
8
+ InsideOpeningTag: 3,
9
+ InsideOpeningTagAndHasSeenWhitespace: 4,
10
+ AfterClosingTagAngleBrackets: 5,
11
+ AfterClosingTagName: 6,
12
+ AfterAttributeName: 7,
13
+ AfterAttributeEqualSign: 8,
14
+ InsideAttributeDoubleQuote: 9,
15
+ InsideBlockComment: 10,
16
+ InsideDoubleQuoteString: 11,
17
+ InsideLineComment: 12,
18
+ InsideSingleQuoteString: 13,
19
+ }
20
+
21
+ /**
22
+ * @enum number
23
+ */
24
+ export const TokenType = {
25
+ None: 99999999,
26
+ Numeric: 30,
27
+ String: 50,
28
+ Whitespace: 0,
29
+ Comment: 60,
30
+ Text: 117,
31
+ PunctuationTag: 228,
32
+ TagName: 118,
33
+ AttributeName: 119,
34
+ Punctuation: 10,
35
+ Error: 141,
36
+ PunctuationString: 11,
37
+ NewLine: 771,
38
+ Keyword: 951,
39
+ VariableName: 952,
40
+ }
41
+
42
+ export const TokenMap = {
43
+ [TokenType.None]: 'None',
44
+ [TokenType.Numeric]: 'Numeric',
45
+ [TokenType.String]: 'String',
46
+ [TokenType.Whitespace]: 'Whitespace',
47
+ [TokenType.Comment]: 'Comment',
48
+ [TokenType.Text]: 'Text',
49
+ [TokenType.PunctuationTag]: 'PunctuationTag',
50
+ [TokenType.TagName]: 'TagName',
51
+ [TokenType.AttributeName]: 'AttributeName',
52
+ [TokenType.Punctuation]: 'Punctuation',
53
+ [TokenType.Error]: 'Error',
54
+ [TokenType.PunctuationString]: 'PunctuationString',
55
+ [TokenType.NewLine]: 'NewLine',
56
+ [TokenType.Keyword]: 'Keyword',
57
+ [TokenType.VariableName]: 'VariableName',
58
+ }
59
+
60
+ const RE_WHITESPACE = /^\s+/
61
+ const RE_DOUBLE_QUOTE = /^"/
62
+ const RE_SINGLE_QUOTE = /^'/
63
+ const RE_ANGLE_BRACKET_OPEN = /^<(?!\s)/
64
+ const RE_ANGLE_BRACKET_CLOSE = /^>/
65
+ const RE_TAGNAME = /^[!\w]+/
66
+ const RE_TEXT = /^.+/
67
+ const RE_ANY_TEXT = /^[^\n]+/
68
+ const RE_SLASH = /^\//
69
+ const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-]+/
70
+ const RE_EQUAL_SIGN = /^=/
71
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
72
+ const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
73
+ const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
74
+ const RE_INVALID_INSIDE_OPENING_TAG = /^[^a-zA-Z>]/
75
+ const RE_INVALID_INSIDE_ClOSING_TAG = /^[^a-zA-Z>]/
76
+ const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
77
+ const RE_WORD = /^[^\s]+/
78
+ const RE_ANGLE_BRACKET_ONLY = /^</
79
+ const RE_NEWLINE = /^\n/
80
+ const RE_EXCLAMATION_MARK = /^!/
81
+ const RE_SELF_CLOSING = /^\/>/
82
+ const RE_DASH_DASH = /^\-\-/
83
+ const RE_TAG_TEXT = /^[^\s>]+/
84
+ const RE_LINE_COMMENT = /^#/
85
+ const RE_NEWLINE_WHITESPACE = /^\n\s*/
86
+ const RE_BLOCK_COMMENT_START = /^\/\*/
87
+ const RE_BLOCK_COMMENT_CONTENT = /^.+(?=\*\/)/s
88
+ const RE_BLOCK_COMMENT_END = /^\*\//
89
+ const RE_UNKNOWN_VALUE = /^[^\}\{\s,"]+/
90
+ const RE_KEYWORD =
91
+ /^(?:__debug__|Ellipsis|yield|with|while|try|true|True|return|raise|pass|or|NotImplemented|not|nonlocal|None|lambda|is|in|import|if|global|from|for|finally|False|false|except|else|elif|del|def|continue|class|break|await|async|assert|as|and)\b/
92
+ const RE_FUNCTION_NAME = /^[a-zA-Z\d]+/
93
+ const RE_ROUND_OPEN = /^\(/
94
+ const RE_VARIABLE_NAME = /^[_a-zA-Z][a-zA-Z\d_]*/
95
+ const RE_ROUND_CLOSE = /^\)/
96
+ const RE_OPERATOR = /^(\+|\-|\*|\/)/
97
+ const RE_NUMERIC =
98
+ /^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
99
+ const RE_PUNCTUATION = /^[\(\)\{\}:,\+\-\*&=\/\\\[\]!\.<>%]+/
100
+
101
+ export const initialLineState = {
102
+ state: State.TopLevelContent,
103
+ }
104
+
105
+ // TODO maybe reverse order of parameters -> line, context (context may be optional)
106
+
107
+ /**
108
+ *
109
+ * @param {string} line
110
+ * @param {{state: State}} lineState
111
+ * @returns
112
+ */
113
+ export const tokenizeLine = (line, lineState) => {
114
+ let next = null
115
+ let index = 0
116
+ let tokens = []
117
+ let token = TokenType.None
118
+ let state = lineState.state
119
+ while (index < line.length) {
120
+ const part = line.slice(index)
121
+ switch (state) {
122
+ case State.TopLevelContent:
123
+ if ((next = part.match(RE_WHITESPACE))) {
124
+ token = TokenType.Whitespace
125
+ state = State.TopLevelContent
126
+ } else if ((next = part.match(RE_KEYWORD))) {
127
+ token = TokenType.Keyword
128
+ state = State.TopLevelContent
129
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
130
+ token = TokenType.VariableName
131
+ state = State.TopLevelContent
132
+ } else if ((next = part.match(RE_PUNCTUATION))) {
133
+ token = TokenType.Punctuation
134
+ state = State.TopLevelContent
135
+ } else if ((next = part.match(RE_NUMERIC))) {
136
+ token = TokenType.Numeric
137
+ state = State.TopLevelContent
138
+ } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
139
+ token = TokenType.PunctuationString
140
+ state = State.InsideDoubleQuoteString
141
+ } else if ((next = part.match(RE_SINGLE_QUOTE))) {
142
+ token = TokenType.PunctuationString
143
+ state = State.InsideSingleQuoteString
144
+ } else if ((next = part.match(RE_LINE_COMMENT))) {
145
+ token = TokenType.Comment
146
+ state = State.InsideLineComment
147
+ } else if ((next = part.match(RE_TEXT))) {
148
+ token = TokenType.Text
149
+ state = State.TopLevelContent
150
+ } else {
151
+ part //?
152
+ throw new Error('no')
153
+ }
154
+ break
155
+ case State.InsideDoubleQuoteString:
156
+ if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
157
+ token = TokenType.String
158
+ state = State.InsideDoubleQuoteString
159
+ } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
160
+ token = TokenType.PunctuationString
161
+ state = State.TopLevelContent
162
+ } else {
163
+ throw new Error('no')
164
+ }
165
+ break
166
+ case State.InsideSingleQuoteString:
167
+ if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
168
+ token = TokenType.String
169
+ state = State.InsideSingleQuoteString
170
+ } else if ((next = part.match(RE_SINGLE_QUOTE))) {
171
+ token = TokenType.PunctuationString
172
+ state = State.TopLevelContent
173
+ } else {
174
+ throw new Error('no')
175
+ }
176
+ break
177
+ case State.InsideLineComment:
178
+ if ((next = part.match(RE_ANY_TEXT))) {
179
+ token = TokenType.Comment
180
+ state = State.TopLevelContent
181
+ } else {
182
+ throw new Error('no')
183
+ }
184
+ break
185
+ default:
186
+ state
187
+ throw new Error('no')
188
+ }
189
+ index += next[0].length
190
+ tokens.push({
191
+ type: token,
192
+ length: next[0].length,
193
+ })
194
+ }
195
+ return {
196
+ state,
197
+ tokens,
198
+ }
199
+ }
200
+
201
+ // tokenizeLine(initialContext, `def add(a,b):`) //?
@@ -0,0 +1,14 @@
1
+ # Language Basics Shell Script
2
+
3
+ ## Contributing
4
+
5
+ ```sh
6
+ git clone git@github.com:lvce-editor/language-basics-shellscript.git &&
7
+ cd language-basics-shellscript &&
8
+ npm ci &&
9
+ npm test
10
+ ```
11
+
12
+ ## Gitpod
13
+
14
+ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-shellscript)
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "builtin.language-basics-shellscript",
3
+ "name": "Language Basics Shell Script",
4
+ "description": "Provides syntax highlighting and bracket matching in Shell Script files.",
5
+ "languages": [
6
+ {
7
+ "id": "shellscript",
8
+ "extensions": [".sh"],
9
+ "tokenize": "src/tokenizeShellScript.js"
10
+ }
11
+ ]
12
+ }
@@ -0,0 +1,127 @@
1
+ /**
2
+ * @enum number
3
+ */
4
+ export const State = {
5
+ TopLevelContent: 1,
6
+ InsideString: 2,
7
+ }
8
+
9
+ export const StateMap = {
10
+ [State.TopLevelContent]: 'TopLevelContent',
11
+ }
12
+
13
+ /**
14
+ * @enum number
15
+ */
16
+ export const TokenType = {
17
+ None: 1,
18
+ Whitespace: 2,
19
+ PunctuationString: 3,
20
+ String: 4,
21
+ Keyword: 5,
22
+ Numeric: 6,
23
+ Punctuation: 7,
24
+ VariableName: 8,
25
+ }
26
+
27
+ export const TokenMap = {
28
+ [TokenType.None]: 'None',
29
+ [TokenType.Whitespace]: 'Whitespace',
30
+ [TokenType.PunctuationString]: 'PunctuationString',
31
+ [TokenType.String]: 'String',
32
+ [TokenType.Keyword]: 'Keyword',
33
+ [TokenType.Numeric]: 'Numeric',
34
+ [TokenType.Punctuation]: 'Punctuation',
35
+ [TokenType.VariableName]: 'VariableName',
36
+ }
37
+
38
+ const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
39
+ const RE_WHITESPACE = /^ +/
40
+ const RE_CURLY_OPEN = /^\{/
41
+ const RE_CURLY_CLOSE = /^\}/
42
+ const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
43
+ const RE_COLON = /^:/
44
+ const RE_PROPERTY_VALUE = /^[^;\}]+/
45
+ const RE_SEMICOLON = /^;/
46
+ const RE_COMMA = /^,/
47
+ const RE_ANYTHING = /^.*/
48
+ const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
49
+ const RE_QUOTE_DOUBLE = /^"/
50
+ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
51
+ const RE_KEYWORD =
52
+ /^(?:alias|bg|bind|break|builtin|caller|cd|command|compgen|complete|dirs|disown|echo|enable|eval|exec|exit|false|fc|fg|getopts|hash|help|history|jobs|kill|let|logout|popd|printf|pushd|pwd|read|readonly|set|shift|shopt|source|suspend|test|times|trap|true|type|ulimit|umask|unalias|unset|wait)\b/
53
+
54
+ const RE_VARIABLE_NAME = /^[a-zA-Z]+/
55
+ const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
56
+ const RE_NUMERIC = /^\d+/
57
+
58
+ export const initialLineState = {
59
+ state: State.TopLevelContent,
60
+ tokens: [],
61
+ }
62
+
63
+ export const isEqualLineState = (lineStateA, lineStateB) => {
64
+ return lineStateA.state === lineStateB.state
65
+ }
66
+
67
+ /**
68
+ * @param {string} line
69
+ */
70
+ export const tokenizeLine = (line, lineState) => {
71
+ let next = null
72
+ let index = 0
73
+ let tokens = []
74
+ let token = TokenType.None
75
+ let state = lineState.state
76
+ while (index < line.length) {
77
+ const part = line.slice(index)
78
+ switch (state) {
79
+ case State.TopLevelContent:
80
+ if ((next = part.match(RE_WHITESPACE))) {
81
+ token = TokenType.Whitespace
82
+ state = State.TopLevelContent
83
+ } else if ((next = part.match(RE_KEYWORD))) {
84
+ token = TokenType.Keyword
85
+ state = State.TopLevelContent
86
+ } else if ((next = part.match(RE_PUNCTUATION))) {
87
+ token = TokenType.Punctuation
88
+ state = State.TopLevelContent
89
+ } else if ((next = part.match(RE_VARIABLE_NAME))) {
90
+ token = TokenType.VariableName
91
+ state = State.TopLevelContent
92
+ } else if ((next = part.match(RE_NUMERIC))) {
93
+ token = TokenType.Numeric
94
+ state = State.TopLevelContent
95
+ } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
96
+ token = TokenType.PunctuationString
97
+ state = State.InsideString
98
+ } else {
99
+ part //?
100
+ throw new Error('no')
101
+ }
102
+ break
103
+ case State.InsideString:
104
+ if ((next = part.match(RE_QUOTE_DOUBLE))) {
105
+ token = TokenType.PunctuationString
106
+ state = State.TopLevelContent
107
+ } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
108
+ token = TokenType.String
109
+ state = State.InsideString
110
+ } else {
111
+ throw new Error('no')
112
+ }
113
+ break
114
+ default:
115
+ throw new Error('no')
116
+ }
117
+ index += next[0].length
118
+ tokens.push({
119
+ type: token,
120
+ length: next[0].length,
121
+ })
122
+ }
123
+ return {
124
+ state,
125
+ tokens,
126
+ }
127
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lvce-editor/shared-process",
3
- "version": "0.0.14",
3
+ "version": "0.0.17",
4
4
  "description": "",
5
5
  "main": "index.js",
6
6
  "type": "module",
@@ -32,8 +32,8 @@
32
32
  "verror": "^1.10.1",
33
33
  "ws": "^8.8.0",
34
34
  "xdg-basedir": "^5.1.0",
35
- "@lvce-editor/pty-host": "0.0.14",
36
- "@lvce-editor/extension-host": "0.0.14"
35
+ "@lvce-editor/pty-host": "0.0.17",
36
+ "@lvce-editor/extension-host": "0.0.17"
37
37
  },
38
38
  "optionalDependencies": {
39
39
  "electron-clipboard-ex": "^1.3.3",