@codua/codeoutline-win32-x64 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISTRIBUTION.md +98 -0
- package/LICENSE +24 -0
- package/README.md +132 -0
- package/README.zh-CN.md +132 -0
- package/THIRD_PARTY.md +32 -0
- package/build-info.json +1 -0
- package/codeoutline +4 -0
- package/codeoutline.cmd +3 -0
- package/licenses/libdeflate.txt +22 -0
- package/licenses/lua-cmsgpack.txt +22 -0
- package/licenses/luajit.txt +56 -0
- package/licenses/mbedtls.txt +553 -0
- package/licenses/rpmalloc.txt +12 -0
- package/licenses/xnet2lua.txt +24 -0
- package/licenses/yyjson.txt +21 -0
- package/package.json +1 -0
- package/scripts/codeoutline/cli.lua +32 -0
- package/scripts/codeoutline/command.lua +155 -0
- package/scripts/codeoutline/common.lua +124 -0
- package/scripts/codeoutline/control.lua +4 -0
- package/scripts/codeoutline/explore.lua +500 -0
- package/scripts/codeoutline/graph.lua +380 -0
- package/scripts/codeoutline/http.lua +156 -0
- package/scripts/codeoutline/index.lua +404 -0
- package/scripts/codeoutline/index_worker.lua +43 -0
- package/scripts/codeoutline/lang/c.lua +362 -0
- package/scripts/codeoutline/lang/cpp.lua +539 -0
- package/scripts/codeoutline/lang/csharp.lua +334 -0
- package/scripts/codeoutline/lang/go.lua +233 -0
- package/scripts/codeoutline/lang/java.lua +274 -0
- package/scripts/codeoutline/lang/js.lua +466 -0
- package/scripts/codeoutline/lang/lua.lua +218 -0
- package/scripts/codeoutline/lang/python.lua +184 -0
- package/scripts/codeoutline/lang/rust.lua +276 -0
- package/scripts/codeoutline/main.lua +104 -0
- package/scripts/codeoutline/mcp.lua +188 -0
- package/scripts/codeoutline/parse.lua +49 -0
- package/scripts/codeoutline/path.lua +36 -0
- package/scripts/codeoutline/service.lua +135 -0
- package/scripts/codeoutline/text.lua +43 -0
- package/scripts/codeoutline/version.lua +1 -0
- package/scripts/codeoutline/xscan.lua +625 -0
- package/xnet2lua/bin/xnet.exe +0 -0
- package/xnet2lua/scripts/core/share/xhttp_codec.lua +835 -0
|
@@ -0,0 +1,625 @@
|
|
|
1
|
+
-- xscan.lua — configurable tokenizer + bracket matcher, pure-Lua reference
|
|
2
|
+
-- implementation. The API is the contract for a later C port: language
|
|
3
|
+
-- parsers (lang/*.lua) only touch the Tokens methods below, so swapping in
|
|
4
|
+
-- the C module needs no parser changes.
|
|
5
|
+
--
|
|
6
|
+
-- local L = xscan.lang{ ...config... }
|
|
7
|
+
-- local t = L:tokenize(src)
|
|
8
|
+
-- t.n, t:kind(i), t:text(i), t:line(i), t:eline(i), t:match(i), t:calls(i, j)
|
|
9
|
+
--
|
|
10
|
+
-- Token kinds: 'id' identifier, 'kw' keyword, 'op' operator/punctuation,
|
|
11
|
+
-- 'str' string/char literal, 'num' number, 'dir' preprocessor line,
|
|
12
|
+
-- 'nl' logical newline, 'indent'/'dedent' (indent mode only).
|
|
13
|
+
--
|
|
14
|
+
-- Config keys (all optional):
|
|
15
|
+
-- ident_start, ident_char character ranges, e.g. 'A-Za-z_' (bytes >= 0x80 always count)
|
|
16
|
+
-- keywords list of reserved words
|
|
17
|
+
-- ops multi-char operators; longest match wins
|
|
18
|
+
-- line_comment list of prefixes ('//', '#')
|
|
19
|
+
-- block_comment list of {open, close}
|
|
20
|
+
-- strings list of {open, close, escape = '\\', multiline = bool}
|
|
21
|
+
-- string_prefixes letters allowed before a quote (Python r/b/f/u)
|
|
22
|
+
-- directive char that starts a whole-line directive at line start ('#')
|
|
23
|
+
-- pp_first_branch C preprocessor: keep only the first live #if branch
|
|
24
|
+
-- indent emit nl/indent/dedent (Python)
|
|
25
|
+
-- long_brackets Lua [==[ strings ]==] and --[==[ comments ]==]
|
|
26
|
+
-- template_literals JS `a ${b} c` (nested templates in ${})
|
|
27
|
+
-- regex_literals JS /re/flags where an operand may start
|
|
28
|
+
-- lifetimes Rust 'a lifetimes (vs 'a' chars)
|
|
29
|
+
-- raw_strings Rust r"..", r#"..."#, br#"..."#
|
|
30
|
+
-- nested_comments Rust: block comments nest
|
|
31
|
+
|
|
32
|
+
-- The runtime's native module (xlua/lua_xscan.c) implements this same
|
|
33
|
+
-- contract; prefer it unless XSCAN_PURE_LUA is set (parity testing).
|
|
34
|
+
if type(_G.xscan) == 'table' and _G.xscan.lang and os.getenv('XSCAN_PURE_LUA') ~= '1' then
|
|
35
|
+
return _G.xscan
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
local M = {}
|
|
39
|
+
|
|
40
|
+
local Lang = {}
|
|
41
|
+
Lang.__index = Lang
|
|
42
|
+
|
|
43
|
+
local function char_set(ranges)
|
|
44
|
+
local set = {}
|
|
45
|
+
for i = 128, 255 do set[i] = true end
|
|
46
|
+
local k = 1
|
|
47
|
+
while k <= #ranges do
|
|
48
|
+
local a, b = ranges:byte(k), ranges:byte(k + 2)
|
|
49
|
+
if ranges:sub(k + 1, k + 1) == '-' and b then
|
|
50
|
+
for c = a, b do set[c] = true end
|
|
51
|
+
k = k + 3
|
|
52
|
+
else
|
|
53
|
+
set[a] = true
|
|
54
|
+
k = k + 1
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
return set
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
function M.lang(cfg)
|
|
61
|
+
local L = setmetatable({ cfg = cfg }, Lang)
|
|
62
|
+
L.id_start = char_set(cfg.ident_start or 'A-Za-z_')
|
|
63
|
+
L.id_char = char_set(cfg.ident_char or 'A-Za-z0-9_')
|
|
64
|
+
L.keywords = {}
|
|
65
|
+
for _, k in ipairs(cfg.keywords or {}) do L.keywords[k] = true end
|
|
66
|
+
-- Operators bucketed by first byte, longest first.
|
|
67
|
+
L.ops = {}
|
|
68
|
+
for _, op in ipairs(cfg.ops or {}) do
|
|
69
|
+
local b = op:byte(1)
|
|
70
|
+
L.ops[b] = L.ops[b] or {}
|
|
71
|
+
table.insert(L.ops[b], op)
|
|
72
|
+
end
|
|
73
|
+
for _, list in pairs(L.ops) do table.sort(list, function(a, b) return #a > #b end) end
|
|
74
|
+
L.line_comment = cfg.line_comment or {}
|
|
75
|
+
L.block_comment = cfg.block_comment or {}
|
|
76
|
+
L.strings = cfg.strings or {}
|
|
77
|
+
L.prefixes = {}
|
|
78
|
+
for c in (cfg.string_prefixes or ''):gmatch('.') do
|
|
79
|
+
L.prefixes[c:byte()] = true
|
|
80
|
+
L.prefixes[c:upper():byte()] = true
|
|
81
|
+
end
|
|
82
|
+
return L
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
local Tokens = {}
|
|
86
|
+
Tokens.__index = Tokens
|
|
87
|
+
|
|
88
|
+
function Tokens:kind(i) return self.k[i] end
|
|
89
|
+
function Tokens:text(i)
|
|
90
|
+
local a = self.s[i]
|
|
91
|
+
if not a then return '' end
|
|
92
|
+
return self.src:sub(a, self.e[i])
|
|
93
|
+
end
|
|
94
|
+
function Tokens:line(i) return self.l[i] end
|
|
95
|
+
function Tokens:eline(i) return self.el[i] end
|
|
96
|
+
function Tokens:match(i) return self.m[i] end
|
|
97
|
+
function Tokens:is(i, text)
|
|
98
|
+
local a = self.s[i]
|
|
99
|
+
return a ~= nil and self.k[i] ~= 'str' and self.src:sub(a, self.e[i]) == text
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
-- Identifiers in (i, j) that are directly followed by '(' — call sites.
|
|
103
|
+
-- Returned as token indices; the parser inspects neighbors for receivers.
|
|
104
|
+
function Tokens:calls(i, j)
|
|
105
|
+
local out = {}
|
|
106
|
+
local k, s, src = self.k, self.s, self.src
|
|
107
|
+
for x = i, j - 1 do
|
|
108
|
+
if k[x] == 'id' and k[x + 1] == 'op' and src:byte(s[x + 1]) == 40 and self.e[x + 1] == s[x + 1] then
|
|
109
|
+
out[#out + 1] = x
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
return out
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
-- Identifier tokens in (i, j) not followed by '(' — candidate references
|
|
116
|
+
-- (function pointers, callbacks registered in tables).
|
|
117
|
+
function Tokens:idents(i, j)
|
|
118
|
+
local out = {}
|
|
119
|
+
local k, s, src = self.k, self.s, self.src
|
|
120
|
+
for x = i, j do
|
|
121
|
+
if k[x] == 'id' and not (k[x + 1] == 'op' and src:byte(s[x + 1]) == 40) then
|
|
122
|
+
out[#out + 1] = x
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
return out
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
local function starts(src, pos, lit)
|
|
129
|
+
return src:sub(pos, pos + #lit - 1) == lit
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
local function count_nl(src, a, b)
|
|
133
|
+
local n, p = 0, a
|
|
134
|
+
while true do
|
|
135
|
+
p = src:find('\n', p, true)
|
|
136
|
+
if not p or p > b then return n end
|
|
137
|
+
n = n + 1
|
|
138
|
+
p = p + 1
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
-- Lua long bracket opening at p: '[' '='* '['. Returns its level (the
|
|
143
|
+
-- number of '='), or nil when p does not open one.
|
|
144
|
+
local function long_open(src, p)
|
|
145
|
+
if src:byte(p) ~= 91 then return nil end
|
|
146
|
+
local q, level = p + 1, 0
|
|
147
|
+
while src:byte(q) == 61 do q = q + 1; level = level + 1 end
|
|
148
|
+
if src:byte(q) == 91 then return level end
|
|
149
|
+
return nil
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
-- Last byte of the long bracket opened at p: the first ']' '='*level ']'
|
|
153
|
+
-- after it, or the end of the source when it is never closed.
|
|
154
|
+
local function long_close(src, p, level)
|
|
155
|
+
local _, stop = src:find(']' .. string.rep('=', level) .. ']', p + level + 2, true)
|
|
156
|
+
return stop or #src
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
-- End of a quoted run opened at p by quote byte q (backslash escapes; stops
|
|
160
|
+
-- at a newline so a stray quote cannot swallow the file).
|
|
161
|
+
local function skip_quoted(src, p, q)
|
|
162
|
+
local x, len = p + 1, #src
|
|
163
|
+
while x <= len do
|
|
164
|
+
local b = src:byte(x)
|
|
165
|
+
if b == 92 then x = x + 2
|
|
166
|
+
elseif b == q or b == 10 then return x
|
|
167
|
+
else x = x + 1 end
|
|
168
|
+
end
|
|
169
|
+
return len
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
local skip_template
|
|
173
|
+
|
|
174
|
+
-- Matching '}' of the `${` expression whose '{' is at p.
|
|
175
|
+
local function skip_braces(src, p)
|
|
176
|
+
local depth, x, len = 0, p, #src
|
|
177
|
+
while x <= len do
|
|
178
|
+
local b = src:byte(x)
|
|
179
|
+
if b == 123 then depth = depth + 1
|
|
180
|
+
elseif b == 125 then
|
|
181
|
+
depth = depth - 1
|
|
182
|
+
if depth == 0 then return x end
|
|
183
|
+
elseif b == 96 then x = skip_template(src, x)
|
|
184
|
+
elseif b == 34 or b == 39 then x = skip_quoted(src, x, b)
|
|
185
|
+
elseif b == 47 and src:byte(x + 1) == 47 then
|
|
186
|
+
while x <= len and src:byte(x) ~= 10 do x = x + 1 end -- // comment
|
|
187
|
+
elseif b == 47 and src:byte(x + 1) == 42 then
|
|
188
|
+
x = x + 2 -- block comment
|
|
189
|
+
while x < len and not (src:byte(x) == 42 and src:byte(x + 1) == 47) do x = x + 1 end
|
|
190
|
+
x = x + 1
|
|
191
|
+
end
|
|
192
|
+
x = x + 1
|
|
193
|
+
end
|
|
194
|
+
return len
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
-- Closing backtick of a JS template literal opened at p; `${ ... }` parts
|
|
198
|
+
-- may nest strings and further templates.
|
|
199
|
+
skip_template = function(src, p)
|
|
200
|
+
local x, len = p + 1, #src
|
|
201
|
+
while x <= len do
|
|
202
|
+
local b = src:byte(x)
|
|
203
|
+
if b == 92 then x = x + 2
|
|
204
|
+
elseif b == 96 then return x
|
|
205
|
+
elseif b == 36 and src:byte(x + 1) == 123 then x = skip_braces(src, x + 1) + 1
|
|
206
|
+
else x = x + 1 end
|
|
207
|
+
end
|
|
208
|
+
return len
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
-- A '/' starts a regex only where an operand may start: not after an
|
|
212
|
+
-- identifier, number, string, `)`, `]`, `}`, or a value keyword.
|
|
213
|
+
local REGEX_AFTER_VALUE_KW = { this = true, super = true, ['true'] = true, ['false'] = true, null = true }
|
|
214
|
+
local function regex_allowed(src, K, S, E, n, ctrl_close)
|
|
215
|
+
if n == 0 then return true end
|
|
216
|
+
local k = K[n]
|
|
217
|
+
if k == 'id' or k == 'num' or k == 'str' then return false end
|
|
218
|
+
local t = src:sub(S[n], E[n])
|
|
219
|
+
-- `if (x) /re/.test(s)`: a ')' closing a control condition ends no operand
|
|
220
|
+
if k == 'op' and t == ')' then return ctrl_close == n end
|
|
221
|
+
if k == 'op' then return not (t == ']' or t == '}') end
|
|
222
|
+
if k == 'kw' then return not REGEX_AFTER_VALUE_KW[t] end
|
|
223
|
+
return true
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
-- Last byte (flags included) of a regex literal opened at p, or nil when
|
|
227
|
+
-- the line ends first (then the '/' was a division after all).
|
|
228
|
+
local function scan_regex(src, p, id_char)
|
|
229
|
+
local x, len, in_class = p + 1, #src, false
|
|
230
|
+
while x <= len do
|
|
231
|
+
local b = src:byte(x)
|
|
232
|
+
if b == 92 then
|
|
233
|
+
x = x + 2
|
|
234
|
+
elseif b == 10 then
|
|
235
|
+
return nil
|
|
236
|
+
else
|
|
237
|
+
if in_class then
|
|
238
|
+
if b == 93 then in_class = false end
|
|
239
|
+
elseif b == 91 then
|
|
240
|
+
in_class = true
|
|
241
|
+
elseif b == 47 then
|
|
242
|
+
x = x + 1
|
|
243
|
+
while x <= len and id_char[src:byte(x)] do x = x + 1 end
|
|
244
|
+
return x - 1
|
|
245
|
+
end
|
|
246
|
+
x = x + 1
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
return nil
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
-- Rust raw string at p: [bc]?r#*" ... "#*. Returns its last byte or nil.
|
|
253
|
+
local function scan_raw_string(src, p, id_char)
|
|
254
|
+
if p > 1 and id_char[src:byte(p - 1)] then return nil end
|
|
255
|
+
local x = p
|
|
256
|
+
local c = src:byte(x)
|
|
257
|
+
if c == 98 or c == 99 then x = x + 1 end
|
|
258
|
+
if src:byte(x) ~= 114 then return nil end
|
|
259
|
+
x = x + 1
|
|
260
|
+
local hashes = 0
|
|
261
|
+
while src:byte(x) == 35 do hashes = hashes + 1; x = x + 1 end
|
|
262
|
+
if src:byte(x) ~= 34 then return nil end
|
|
263
|
+
local _, stop = src:find('"' .. string.rep('#', hashes), x + 1, true)
|
|
264
|
+
return stop or #src
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
-- Rust `'a` lifetime (not a `'a'` char) at p: returns its last byte or nil.
|
|
268
|
+
local function scan_lifetime(src, p, id_start, id_char)
|
|
269
|
+
local c = src:byte(p + 1)
|
|
270
|
+
if not c or c == 92 or not id_start[c] then return nil end
|
|
271
|
+
local w = c < 0x80 and 1 or c >= 0xF0 and 4 or c >= 0xE0 and 3 or 2
|
|
272
|
+
if src:byte(p + 1 + w) == 39 then return nil end
|
|
273
|
+
local x = p + 1
|
|
274
|
+
while x <= #src and id_char[src:byte(x)] do x = x + 1 end
|
|
275
|
+
return x - 1
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
local OPEN = { [40] = 41, [91] = 93, [123] = 125 } -- ( [ {
|
|
279
|
+
local CLOSE = { [41] = 40, [93] = 91, [125] = 123 }
|
|
280
|
+
|
|
281
|
+
function Lang:tokenize(src)
|
|
282
|
+
local cfg = self.cfg
|
|
283
|
+
local T = setmetatable({ src = src, k = {}, s = {}, e = {}, l = {}, el = {}, m = {}, n = 0 }, Tokens)
|
|
284
|
+
local K, S, E, Ln, EL = T.k, T.s, T.e, T.l, T.el
|
|
285
|
+
local n, len, pos, line = 0, #src, 1, 1
|
|
286
|
+
local bol = true -- only whitespace seen on this line
|
|
287
|
+
local depth = 0 -- bracket depth (indent mode ignores newlines inside)
|
|
288
|
+
local skip = false -- inside a dead preprocessor branch
|
|
289
|
+
local pp = {} -- #if frames
|
|
290
|
+
local indents = { 0 }
|
|
291
|
+
local indent_mode = cfg.indent
|
|
292
|
+
local long_brackets = cfg.long_brackets
|
|
293
|
+
local template_literals, regex_literals = cfg.template_literals, cfg.regex_literals
|
|
294
|
+
local lifetimes, raw_strings = cfg.lifetimes, cfg.raw_strings
|
|
295
|
+
local nested_comments = cfg.nested_comments
|
|
296
|
+
-- regex_literals: per open '(' whether it follows if/while/for/with, and
|
|
297
|
+
-- the index of the last ')' closing such a condition (see regex_allowed)
|
|
298
|
+
local parens, ctrl_close = {}, 0
|
|
299
|
+
local dir_byte = cfg.directive and cfg.directive:byte()
|
|
300
|
+
local id_start, id_char, keywords, ops = self.id_start, self.id_char, self.keywords, self.ops
|
|
301
|
+
|
|
302
|
+
local function push(kind, a, b, l1, l2)
|
|
303
|
+
if skip then return end
|
|
304
|
+
n = n + 1
|
|
305
|
+
K[n], S[n], E[n], Ln[n], EL[n] = kind, a, b, l1, l2 or l1
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
-- At file scope (depth 0) every #if arm is kept: platform variants of
|
|
309
|
+
-- whole functions (#ifdef _WIN32 ... #else ...) are separate, balanced
|
|
310
|
+
-- definitions worth indexing. Inside brackets only the first live arm is
|
|
311
|
+
-- kept, because that is where arms split a construct
|
|
312
|
+
-- (`#ifdef X if (a) { #else if (b) { #endif`) and would unbalance braces.
|
|
313
|
+
local function directive(text)
|
|
314
|
+
if not cfg.pp_first_branch then return end
|
|
315
|
+
local word = text:match('^#%s*(%a+)')
|
|
316
|
+
if word == 'if' or word == 'ifdef' or word == 'ifndef' then
|
|
317
|
+
local zero = word == 'if' and text:match('^#%s*if%s+0%s*$') ~= nil
|
|
318
|
+
pp[#pp + 1] = { parent = skip, taken = not zero, keep_all = depth == 0 and not zero }
|
|
319
|
+
if zero then skip = true end
|
|
320
|
+
elseif (word == 'elif' or word == 'else') and #pp > 0 then
|
|
321
|
+
local f = pp[#pp]
|
|
322
|
+
if f.keep_all then skip = f.parent
|
|
323
|
+
elseif f.taken then skip = true
|
|
324
|
+
else skip = f.parent; f.taken = true end
|
|
325
|
+
elseif word == 'endif' and #pp > 0 then
|
|
326
|
+
skip = table.remove(pp).parent
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
while pos <= len do
|
|
331
|
+
local c = src:byte(pos)
|
|
332
|
+
if c == 10 then
|
|
333
|
+
if indent_mode and depth == 0 and n > 0 and K[n] ~= 'nl' and not skip then
|
|
334
|
+
push('nl', pos, pos, line)
|
|
335
|
+
end
|
|
336
|
+
line = line + 1
|
|
337
|
+
pos = pos + 1
|
|
338
|
+
bol = true
|
|
339
|
+
elseif c == 32 or c == 9 or c == 13 or c == 12 or c == 11 then
|
|
340
|
+
pos = pos + 1
|
|
341
|
+
elseif indent_mode and c == 92 and (src:byte(pos + 1) == 10
|
|
342
|
+
or (src:byte(pos + 1) == 13 and src:byte(pos + 2) == 10)) then
|
|
343
|
+
-- Python `\` continuation: the next line joins this logical line
|
|
344
|
+
pos = pos + (src:byte(pos + 1) == 10 and 2 or 3)
|
|
345
|
+
line = line + 1
|
|
346
|
+
else
|
|
347
|
+
-- Indentation (Python): measured at the first real token of a line.
|
|
348
|
+
if indent_mode and bol and depth == 0 then
|
|
349
|
+
local col_start = pos
|
|
350
|
+
while col_start > 1 and src:byte(col_start - 1) ~= 10 do col_start = col_start - 1 end
|
|
351
|
+
local width = 0
|
|
352
|
+
for x = col_start, pos - 1 do
|
|
353
|
+
width = width + ((src:byte(x) == 9) and (8 - width % 8) or 1)
|
|
354
|
+
end
|
|
355
|
+
local is_comment = false
|
|
356
|
+
for _, lc in ipairs(self.line_comment) do
|
|
357
|
+
if starts(src, pos, lc) then is_comment = true end
|
|
358
|
+
end
|
|
359
|
+
if not is_comment then
|
|
360
|
+
if width > indents[#indents] then
|
|
361
|
+
indents[#indents + 1] = width
|
|
362
|
+
push('indent', pos, pos - 1, line)
|
|
363
|
+
else
|
|
364
|
+
while width < indents[#indents] do
|
|
365
|
+
indents[#indents] = nil
|
|
366
|
+
push('dedent', pos, pos - 1, line)
|
|
367
|
+
end
|
|
368
|
+
end
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
local handled = false
|
|
373
|
+
-- Directive line (C preprocessor), with '\' continuations.
|
|
374
|
+
if dir_byte and c == dir_byte and bol then
|
|
375
|
+
local p = pos
|
|
376
|
+
local l0 = line
|
|
377
|
+
while true do
|
|
378
|
+
local nl = src:find('\n', p, true) or (len + 1)
|
|
379
|
+
local q = nl - 1
|
|
380
|
+
if src:byte(q) == 13 then q = q - 1 end
|
|
381
|
+
if src:byte(q) == 92 and nl <= len then
|
|
382
|
+
line = line + 1
|
|
383
|
+
p = nl + 1
|
|
384
|
+
else
|
|
385
|
+
p = nl
|
|
386
|
+
break
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
local text = src:sub(pos, p - 1)
|
|
390
|
+
-- A directive is kept when either side of it is live code, so
|
|
391
|
+
-- #include/#define in taken branches still reach the parser.
|
|
392
|
+
local was_skip = skip
|
|
393
|
+
directive(text)
|
|
394
|
+
if not (was_skip and skip) then
|
|
395
|
+
local now = skip
|
|
396
|
+
skip = false
|
|
397
|
+
push('dir', pos, p - 1, l0, line)
|
|
398
|
+
skip = now
|
|
399
|
+
end
|
|
400
|
+
pos = p
|
|
401
|
+
handled = true
|
|
402
|
+
end
|
|
403
|
+
bol = false
|
|
404
|
+
|
|
405
|
+
if not handled then
|
|
406
|
+
for _, lc in ipairs(self.line_comment) do
|
|
407
|
+
if starts(src, pos, lc) then
|
|
408
|
+
local lb = long_brackets and long_open(src, pos + #lc)
|
|
409
|
+
if lb then
|
|
410
|
+
-- --[==[ block comment ]==]
|
|
411
|
+
local stop = long_close(src, pos + #lc, lb)
|
|
412
|
+
line = line + count_nl(src, pos, stop)
|
|
413
|
+
pos = stop + 1
|
|
414
|
+
else
|
|
415
|
+
pos = src:find('\n', pos, true) or (len + 1)
|
|
416
|
+
end
|
|
417
|
+
handled = true
|
|
418
|
+
break
|
|
419
|
+
end
|
|
420
|
+
end
|
|
421
|
+
end
|
|
422
|
+
if not handled and long_brackets then
|
|
423
|
+
local lb = long_open(src, pos)
|
|
424
|
+
if lb then
|
|
425
|
+
local stop = long_close(src, pos, lb)
|
|
426
|
+
local l0 = line
|
|
427
|
+
line = line + count_nl(src, pos, stop)
|
|
428
|
+
push('str', pos, stop, l0, line)
|
|
429
|
+
pos = stop + 1
|
|
430
|
+
handled = true
|
|
431
|
+
end
|
|
432
|
+
end
|
|
433
|
+
if not handled then
|
|
434
|
+
for _, bc in ipairs(self.block_comment) do
|
|
435
|
+
if starts(src, pos, bc[1]) then
|
|
436
|
+
local stop
|
|
437
|
+
if nested_comments then
|
|
438
|
+
local nest, p = 1, pos + #bc[1]
|
|
439
|
+
while p + #bc[2] - 1 <= len do
|
|
440
|
+
if starts(src, p, bc[1]) then
|
|
441
|
+
nest = nest + 1
|
|
442
|
+
p = p + #bc[1]
|
|
443
|
+
elseif starts(src, p, bc[2]) then
|
|
444
|
+
nest = nest - 1
|
|
445
|
+
if nest == 0 then stop = p + #bc[2] - 1; break end
|
|
446
|
+
p = p + #bc[2]
|
|
447
|
+
else
|
|
448
|
+
p = p + 1
|
|
449
|
+
end
|
|
450
|
+
end
|
|
451
|
+
else
|
|
452
|
+
local _
|
|
453
|
+
_, stop = src:find(bc[2], pos + #bc[1], true)
|
|
454
|
+
end
|
|
455
|
+
stop = stop or len
|
|
456
|
+
line = line + count_nl(src, pos, stop)
|
|
457
|
+
pos = stop + 1
|
|
458
|
+
handled = true
|
|
459
|
+
break
|
|
460
|
+
end
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
if not handled and template_literals and c == 96 then
|
|
464
|
+
local stop = skip_template(src, pos)
|
|
465
|
+
local l0 = line
|
|
466
|
+
line = line + count_nl(src, pos, stop)
|
|
467
|
+
push('str', pos, stop, l0, line)
|
|
468
|
+
pos = stop + 1
|
|
469
|
+
handled = true
|
|
470
|
+
end
|
|
471
|
+
if not handled and regex_literals and c == 47 and regex_allowed(src, K, S, E, n, ctrl_close) then
|
|
472
|
+
local stop = scan_regex(src, pos, id_char)
|
|
473
|
+
if stop then
|
|
474
|
+
push('str', pos, stop, line)
|
|
475
|
+
pos = stop + 1
|
|
476
|
+
handled = true
|
|
477
|
+
end
|
|
478
|
+
end
|
|
479
|
+
if not handled and lifetimes and c == 39 then
|
|
480
|
+
local stop = scan_lifetime(src, pos, id_start, id_char)
|
|
481
|
+
if stop then
|
|
482
|
+
push('id', pos, stop, line)
|
|
483
|
+
pos = stop + 1
|
|
484
|
+
handled = true
|
|
485
|
+
end
|
|
486
|
+
end
|
|
487
|
+
if not handled and raw_strings and (c == 114 or c == 98 or c == 99) then
|
|
488
|
+
local stop = scan_raw_string(src, pos, id_char)
|
|
489
|
+
if stop then
|
|
490
|
+
local l0 = line
|
|
491
|
+
line = line + count_nl(src, pos, stop)
|
|
492
|
+
push('str', pos, stop, l0, line)
|
|
493
|
+
pos = stop + 1
|
|
494
|
+
handled = true
|
|
495
|
+
end
|
|
496
|
+
end
|
|
497
|
+
if not handled then
|
|
498
|
+
-- String, optionally after a prefix (r"", b'', f"""...""").
|
|
499
|
+
local q = pos
|
|
500
|
+
while self.prefixes[src:byte(q)] and q - pos < 2 do q = q + 1 end
|
|
501
|
+
for _, st in ipairs(self.strings) do
|
|
502
|
+
local open = st[1]
|
|
503
|
+
if starts(src, q, open) and (q == pos or not id_char[src:byte(pos - 1) or 0]) then
|
|
504
|
+
local close, esc = st[2], st.escape
|
|
505
|
+
local p = q + #open
|
|
506
|
+
local stop
|
|
507
|
+
while p <= len do
|
|
508
|
+
local b = src:byte(p)
|
|
509
|
+
if esc and b == esc:byte() then
|
|
510
|
+
p = p + 2
|
|
511
|
+
elseif starts(src, p, close) then
|
|
512
|
+
stop = p + #close - 1
|
|
513
|
+
break
|
|
514
|
+
elseif b == 10 and not st.multiline then
|
|
515
|
+
stop = p - 1
|
|
516
|
+
break
|
|
517
|
+
else
|
|
518
|
+
p = p + 1
|
|
519
|
+
end
|
|
520
|
+
end
|
|
521
|
+
stop = stop or len
|
|
522
|
+
local l0 = line
|
|
523
|
+
line = line + count_nl(src, pos, stop)
|
|
524
|
+
push('str', pos, stop, l0, line)
|
|
525
|
+
pos = stop + 1
|
|
526
|
+
handled = true
|
|
527
|
+
break
|
|
528
|
+
end
|
|
529
|
+
end
|
|
530
|
+
end
|
|
531
|
+
if not handled then
|
|
532
|
+
if id_start[c] then
|
|
533
|
+
local p = pos + 1
|
|
534
|
+
while p <= len and id_char[src:byte(p)] do p = p + 1 end
|
|
535
|
+
local word = src:sub(pos, p - 1)
|
|
536
|
+
push(keywords[word] and 'kw' or 'id', pos, p - 1, line)
|
|
537
|
+
pos = p
|
|
538
|
+
elseif (c >= 48 and c <= 57) or (c == 46 and (src:byte(pos + 1) or 0) >= 48 and (src:byte(pos + 1) or 0) <= 57) then
|
|
539
|
+
local p = pos + 1
|
|
540
|
+
while p <= len do
|
|
541
|
+
local b = src:byte(p)
|
|
542
|
+
if id_char[b] or b == 46 or b == 39 then
|
|
543
|
+
p = p + 1
|
|
544
|
+
elseif (b == 43 or b == 45) and (src:byte(p - 1) == 101 or src:byte(p - 1) == 69
|
|
545
|
+
or src:byte(p - 1) == 112 or src:byte(p - 1) == 80) then
|
|
546
|
+
p = p + 1 -- exponent sign: 1e-5, 0x1p+3
|
|
547
|
+
else
|
|
548
|
+
break
|
|
549
|
+
end
|
|
550
|
+
end
|
|
551
|
+
push('num', pos, p - 1, line)
|
|
552
|
+
pos = p
|
|
553
|
+
else
|
|
554
|
+
local w = 1
|
|
555
|
+
local list = ops[c]
|
|
556
|
+
if list then
|
|
557
|
+
for _, op in ipairs(list) do
|
|
558
|
+
if starts(src, pos, op) then w = #op; break end
|
|
559
|
+
end
|
|
560
|
+
end
|
|
561
|
+
if w == 1 and not skip then
|
|
562
|
+
if OPEN[c] then depth = depth + 1
|
|
563
|
+
elseif CLOSE[c] and depth > 0 then depth = depth - 1 end
|
|
564
|
+
end
|
|
565
|
+
push('op', pos, pos + w - 1, line)
|
|
566
|
+
if regex_literals and w == 1 and not skip then
|
|
567
|
+
if c == 40 then
|
|
568
|
+
local prev = n - 1
|
|
569
|
+
local ctrl = false
|
|
570
|
+
if prev >= 1 and K[prev] == 'kw' then
|
|
571
|
+
local pt = src:sub(S[prev], E[prev])
|
|
572
|
+
ctrl = pt == 'if' or pt == 'while' or pt == 'for' or pt == 'with'
|
|
573
|
+
end
|
|
574
|
+
parens[#parens + 1] = ctrl
|
|
575
|
+
elseif c == 41 then
|
|
576
|
+
local ctrl = table.remove(parens)
|
|
577
|
+
ctrl_close = ctrl and n or 0
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
pos = pos + w
|
|
581
|
+
end
|
|
582
|
+
end
|
|
583
|
+
end
|
|
584
|
+
end
|
|
585
|
+
if indent_mode then
|
|
586
|
+
if n > 0 and K[n] ~= 'nl' then push('nl', len + 1, len, line) end
|
|
587
|
+
for _ = 2, #indents do push('dedent', len + 1, len, line) end
|
|
588
|
+
end
|
|
589
|
+
T.n = n
|
|
590
|
+
T:compute_matches()
|
|
591
|
+
return T
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
-- Pair brackets (and indent/dedent). An unbalanced closer is left unmatched
|
|
595
|
+
-- instead of unwinding the stack, so one stray brace (usually from a
|
|
596
|
+
-- preprocessor branch) doesn't break every later match.
|
|
597
|
+
function Tokens:compute_matches()
|
|
598
|
+
local stack, m = {}, self.m
|
|
599
|
+
local K, S, E, src = self.k, self.s, self.e, self.src
|
|
600
|
+
for i = 1, self.n do
|
|
601
|
+
local kind = K[i]
|
|
602
|
+
if kind == 'op' and S[i] == E[i] then
|
|
603
|
+
local c = src:byte(S[i])
|
|
604
|
+
if OPEN[c] then
|
|
605
|
+
stack[#stack + 1] = i
|
|
606
|
+
elseif CLOSE[c] then
|
|
607
|
+
local top = stack[#stack]
|
|
608
|
+
if top and K[top] == 'op' and src:byte(S[top]) == CLOSE[c] then
|
|
609
|
+
stack[#stack] = nil
|
|
610
|
+
m[top], m[i] = i, top
|
|
611
|
+
end
|
|
612
|
+
end
|
|
613
|
+
elseif kind == 'indent' then
|
|
614
|
+
stack[#stack + 1] = i
|
|
615
|
+
elseif kind == 'dedent' then
|
|
616
|
+
local top = stack[#stack]
|
|
617
|
+
if top and K[top] == 'indent' then
|
|
618
|
+
stack[#stack] = nil
|
|
619
|
+
m[top], m[i] = i, top
|
|
620
|
+
end
|
|
621
|
+
end
|
|
622
|
+
end
|
|
623
|
+
end
|
|
624
|
+
|
|
625
|
+
return M
|
|
Binary file
|