@codua/codeoutline-win32-x64 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/DISTRIBUTION.md +98 -0
  2. package/LICENSE +24 -0
  3. package/README.md +132 -0
  4. package/README.zh-CN.md +132 -0
  5. package/THIRD_PARTY.md +32 -0
  6. package/build-info.json +1 -0
  7. package/codeoutline +4 -0
  8. package/codeoutline.cmd +3 -0
  9. package/licenses/libdeflate.txt +22 -0
  10. package/licenses/lua-cmsgpack.txt +22 -0
  11. package/licenses/luajit.txt +56 -0
  12. package/licenses/mbedtls.txt +553 -0
  13. package/licenses/rpmalloc.txt +12 -0
  14. package/licenses/xnet2lua.txt +24 -0
  15. package/licenses/yyjson.txt +21 -0
  16. package/package.json +1 -0
  17. package/scripts/codeoutline/cli.lua +32 -0
  18. package/scripts/codeoutline/command.lua +155 -0
  19. package/scripts/codeoutline/common.lua +124 -0
  20. package/scripts/codeoutline/control.lua +4 -0
  21. package/scripts/codeoutline/explore.lua +500 -0
  22. package/scripts/codeoutline/graph.lua +380 -0
  23. package/scripts/codeoutline/http.lua +156 -0
  24. package/scripts/codeoutline/index.lua +404 -0
  25. package/scripts/codeoutline/index_worker.lua +43 -0
  26. package/scripts/codeoutline/lang/c.lua +362 -0
  27. package/scripts/codeoutline/lang/cpp.lua +539 -0
  28. package/scripts/codeoutline/lang/csharp.lua +334 -0
  29. package/scripts/codeoutline/lang/go.lua +233 -0
  30. package/scripts/codeoutline/lang/java.lua +274 -0
  31. package/scripts/codeoutline/lang/js.lua +466 -0
  32. package/scripts/codeoutline/lang/lua.lua +218 -0
  33. package/scripts/codeoutline/lang/python.lua +184 -0
  34. package/scripts/codeoutline/lang/rust.lua +276 -0
  35. package/scripts/codeoutline/main.lua +104 -0
  36. package/scripts/codeoutline/mcp.lua +188 -0
  37. package/scripts/codeoutline/parse.lua +49 -0
  38. package/scripts/codeoutline/path.lua +36 -0
  39. package/scripts/codeoutline/service.lua +135 -0
  40. package/scripts/codeoutline/text.lua +43 -0
  41. package/scripts/codeoutline/version.lua +1 -0
  42. package/scripts/codeoutline/xscan.lua +625 -0
  43. package/xnet2lua/bin/xnet.exe +0 -0
  44. package/xnet2lua/scripts/core/share/xhttp_codec.lua +835 -0
@@ -0,0 +1,625 @@
1
+ -- xscan.lua — configurable tokenizer + bracket matcher, pure-Lua reference
2
+ -- implementation. The API is the contract for a later C port: language
3
+ -- parsers (lang/*.lua) only touch the Tokens methods below, so swapping in
4
+ -- the C module needs no parser changes.
5
+ --
6
+ -- local L = xscan.lang{ ...config... }
7
+ -- local t = L:tokenize(src)
8
+ -- t.n, t:kind(i), t:text(i), t:line(i), t:eline(i), t:match(i), t:calls(i, j)
9
+ --
10
+ -- Token kinds: 'id' identifier, 'kw' keyword, 'op' operator/punctuation,
11
+ -- 'str' string/char literal, 'num' number, 'dir' preprocessor line,
12
+ -- 'nl' logical newline, 'indent'/'dedent' (indent mode only).
13
+ --
14
+ -- Config keys (all optional):
15
+ -- ident_start, ident_char character ranges, e.g. 'A-Za-z_' (bytes >= 0x80 always count)
16
+ -- keywords list of reserved words
17
+ -- ops multi-char operators; longest match wins
18
+ -- line_comment list of prefixes ('//', '#')
19
+ -- block_comment list of {open, close}
20
+ -- strings list of {open, close, escape = '\\', multiline = bool}
21
+ -- string_prefixes letters allowed before a quote (Python r/b/f/u)
22
+ -- directive char that starts a whole-line directive at line start ('#')
23
+ -- pp_first_branch C preprocessor: keep only the first live #if branch
24
+ -- indent emit nl/indent/dedent (Python)
25
+ -- long_brackets Lua [==[ strings ]==] and --[==[ comments ]==]
26
+ -- template_literals JS `a ${b} c` (nested templates in ${})
27
+ -- regex_literals JS /re/flags where an operand may start
28
+ -- lifetimes Rust 'a lifetimes (vs 'a' chars)
29
+ -- raw_strings Rust r"..", r#"..."#, br#"..."#
30
+ -- nested_comments Rust: block comments nest
31
+
32
+ -- The runtime's native module (xlua/lua_xscan.c) implements this same
33
+ -- contract; prefer it unless XSCAN_PURE_LUA is set (parity testing).
34
+ if type(_G.xscan) == 'table' and _G.xscan.lang and os.getenv('XSCAN_PURE_LUA') ~= '1' then
35
+ return _G.xscan
36
+ end
37
+
38
+ local M = {}
39
+
40
+ local Lang = {}
41
+ Lang.__index = Lang
42
+
43
+ local function char_set(ranges)
44
+ local set = {}
45
+ for i = 128, 255 do set[i] = true end
46
+ local k = 1
47
+ while k <= #ranges do
48
+ local a, b = ranges:byte(k), ranges:byte(k + 2)
49
+ if ranges:sub(k + 1, k + 1) == '-' and b then
50
+ for c = a, b do set[c] = true end
51
+ k = k + 3
52
+ else
53
+ set[a] = true
54
+ k = k + 1
55
+ end
56
+ end
57
+ return set
58
+ end
59
+
60
+ function M.lang(cfg)
61
+ local L = setmetatable({ cfg = cfg }, Lang)
62
+ L.id_start = char_set(cfg.ident_start or 'A-Za-z_')
63
+ L.id_char = char_set(cfg.ident_char or 'A-Za-z0-9_')
64
+ L.keywords = {}
65
+ for _, k in ipairs(cfg.keywords or {}) do L.keywords[k] = true end
66
+ -- Operators bucketed by first byte, longest first.
67
+ L.ops = {}
68
+ for _, op in ipairs(cfg.ops or {}) do
69
+ local b = op:byte(1)
70
+ L.ops[b] = L.ops[b] or {}
71
+ table.insert(L.ops[b], op)
72
+ end
73
+ for _, list in pairs(L.ops) do table.sort(list, function(a, b) return #a > #b end) end
74
+ L.line_comment = cfg.line_comment or {}
75
+ L.block_comment = cfg.block_comment or {}
76
+ L.strings = cfg.strings or {}
77
+ L.prefixes = {}
78
+ for c in (cfg.string_prefixes or ''):gmatch('.') do
79
+ L.prefixes[c:byte()] = true
80
+ L.prefixes[c:upper():byte()] = true
81
+ end
82
+ return L
83
+ end
84
+
85
+ local Tokens = {}
86
+ Tokens.__index = Tokens
87
+
88
+ function Tokens:kind(i) return self.k[i] end
89
+ function Tokens:text(i)
90
+ local a = self.s[i]
91
+ if not a then return '' end
92
+ return self.src:sub(a, self.e[i])
93
+ end
94
+ function Tokens:line(i) return self.l[i] end
95
+ function Tokens:eline(i) return self.el[i] end
96
+ function Tokens:match(i) return self.m[i] end
97
+ function Tokens:is(i, text)
98
+ local a = self.s[i]
99
+ return a ~= nil and self.k[i] ~= 'str' and self.src:sub(a, self.e[i]) == text
100
+ end
101
+
102
+ -- Identifiers in (i, j) that are directly followed by '(' — call sites.
103
+ -- Returned as token indices; the parser inspects neighbors for receivers.
104
+ function Tokens:calls(i, j)
105
+ local out = {}
106
+ local k, s, src = self.k, self.s, self.src
107
+ for x = i, j - 1 do
108
+ if k[x] == 'id' and k[x + 1] == 'op' and src:byte(s[x + 1]) == 40 and self.e[x + 1] == s[x + 1] then
109
+ out[#out + 1] = x
110
+ end
111
+ end
112
+ return out
113
+ end
114
+
115
+ -- Identifier tokens in (i, j) not followed by '(' — candidate references
116
+ -- (function pointers, callbacks registered in tables).
117
+ function Tokens:idents(i, j)
118
+ local out = {}
119
+ local k, s, src = self.k, self.s, self.src
120
+ for x = i, j do
121
+ if k[x] == 'id' and not (k[x + 1] == 'op' and src:byte(s[x + 1]) == 40) then
122
+ out[#out + 1] = x
123
+ end
124
+ end
125
+ return out
126
+ end
127
+
128
+ local function starts(src, pos, lit)
129
+ return src:sub(pos, pos + #lit - 1) == lit
130
+ end
131
+
132
+ local function count_nl(src, a, b)
133
+ local n, p = 0, a
134
+ while true do
135
+ p = src:find('\n', p, true)
136
+ if not p or p > b then return n end
137
+ n = n + 1
138
+ p = p + 1
139
+ end
140
+ end
141
+
142
+ -- Lua long bracket opening at p: '[' '='* '['. Returns its level (the
143
+ -- number of '='), or nil when p does not open one.
144
+ local function long_open(src, p)
145
+ if src:byte(p) ~= 91 then return nil end
146
+ local q, level = p + 1, 0
147
+ while src:byte(q) == 61 do q = q + 1; level = level + 1 end
148
+ if src:byte(q) == 91 then return level end
149
+ return nil
150
+ end
151
+
152
+ -- Last byte of the long bracket opened at p: the first ']' '='*level ']'
153
+ -- after it, or the end of the source when it is never closed.
154
+ local function long_close(src, p, level)
155
+ local _, stop = src:find(']' .. string.rep('=', level) .. ']', p + level + 2, true)
156
+ return stop or #src
157
+ end
158
+
159
+ -- End of a quoted run opened at p by quote byte q (backslash escapes; stops
160
+ -- at a newline so a stray quote cannot swallow the file).
161
+ local function skip_quoted(src, p, q)
162
+ local x, len = p + 1, #src
163
+ while x <= len do
164
+ local b = src:byte(x)
165
+ if b == 92 then x = x + 2
166
+ elseif b == q or b == 10 then return x
167
+ else x = x + 1 end
168
+ end
169
+ return len
170
+ end
171
+
172
+ local skip_template
173
+
174
+ -- Matching '}' of the `${` expression whose '{' is at p.
175
+ local function skip_braces(src, p)
176
+ local depth, x, len = 0, p, #src
177
+ while x <= len do
178
+ local b = src:byte(x)
179
+ if b == 123 then depth = depth + 1
180
+ elseif b == 125 then
181
+ depth = depth - 1
182
+ if depth == 0 then return x end
183
+ elseif b == 96 then x = skip_template(src, x)
184
+ elseif b == 34 or b == 39 then x = skip_quoted(src, x, b)
185
+ elseif b == 47 and src:byte(x + 1) == 47 then
186
+ while x <= len and src:byte(x) ~= 10 do x = x + 1 end -- // comment
187
+ elseif b == 47 and src:byte(x + 1) == 42 then
188
+ x = x + 2 -- block comment
189
+ while x < len and not (src:byte(x) == 42 and src:byte(x + 1) == 47) do x = x + 1 end
190
+ x = x + 1
191
+ end
192
+ x = x + 1
193
+ end
194
+ return len
195
+ end
196
+
197
+ -- Closing backtick of a JS template literal opened at p; `${ ... }` parts
198
+ -- may nest strings and further templates.
199
+ skip_template = function(src, p)
200
+ local x, len = p + 1, #src
201
+ while x <= len do
202
+ local b = src:byte(x)
203
+ if b == 92 then x = x + 2
204
+ elseif b == 96 then return x
205
+ elseif b == 36 and src:byte(x + 1) == 123 then x = skip_braces(src, x + 1) + 1
206
+ else x = x + 1 end
207
+ end
208
+ return len
209
+ end
210
+
211
+ -- A '/' starts a regex only where an operand may start: not after an
212
+ -- identifier, number, string, `)`, `]`, `}`, or a value keyword.
213
+ local REGEX_AFTER_VALUE_KW = { this = true, super = true, ['true'] = true, ['false'] = true, null = true }
214
+ local function regex_allowed(src, K, S, E, n, ctrl_close)
215
+ if n == 0 then return true end
216
+ local k = K[n]
217
+ if k == 'id' or k == 'num' or k == 'str' then return false end
218
+ local t = src:sub(S[n], E[n])
219
+ -- `if (x) /re/.test(s)`: a ')' closing a control condition ends no operand
220
+ if k == 'op' and t == ')' then return ctrl_close == n end
221
+ if k == 'op' then return not (t == ']' or t == '}') end
222
+ if k == 'kw' then return not REGEX_AFTER_VALUE_KW[t] end
223
+ return true
224
+ end
225
+
226
+ -- Last byte (flags included) of a regex literal opened at p, or nil when
227
+ -- the line ends first (then the '/' was a division after all).
228
+ local function scan_regex(src, p, id_char)
229
+ local x, len, in_class = p + 1, #src, false
230
+ while x <= len do
231
+ local b = src:byte(x)
232
+ if b == 92 then
233
+ x = x + 2
234
+ elseif b == 10 then
235
+ return nil
236
+ else
237
+ if in_class then
238
+ if b == 93 then in_class = false end
239
+ elseif b == 91 then
240
+ in_class = true
241
+ elseif b == 47 then
242
+ x = x + 1
243
+ while x <= len and id_char[src:byte(x)] do x = x + 1 end
244
+ return x - 1
245
+ end
246
+ x = x + 1
247
+ end
248
+ end
249
+ return nil
250
+ end
251
+
252
+ -- Rust raw string at p: [bc]?r#*" ... "#*. Returns its last byte or nil.
253
+ local function scan_raw_string(src, p, id_char)
254
+ if p > 1 and id_char[src:byte(p - 1)] then return nil end
255
+ local x = p
256
+ local c = src:byte(x)
257
+ if c == 98 or c == 99 then x = x + 1 end
258
+ if src:byte(x) ~= 114 then return nil end
259
+ x = x + 1
260
+ local hashes = 0
261
+ while src:byte(x) == 35 do hashes = hashes + 1; x = x + 1 end
262
+ if src:byte(x) ~= 34 then return nil end
263
+ local _, stop = src:find('"' .. string.rep('#', hashes), x + 1, true)
264
+ return stop or #src
265
+ end
266
+
267
+ -- Rust `'a` lifetime (not a `'a'` char) at p: returns its last byte or nil.
268
+ local function scan_lifetime(src, p, id_start, id_char)
269
+ local c = src:byte(p + 1)
270
+ if not c or c == 92 or not id_start[c] then return nil end
271
+ local w = c < 0x80 and 1 or c >= 0xF0 and 4 or c >= 0xE0 and 3 or 2
272
+ if src:byte(p + 1 + w) == 39 then return nil end
273
+ local x = p + 1
274
+ while x <= #src and id_char[src:byte(x)] do x = x + 1 end
275
+ return x - 1
276
+ end
277
+
278
+ local OPEN = { [40] = 41, [91] = 93, [123] = 125 } -- ( [ {
279
+ local CLOSE = { [41] = 40, [93] = 91, [125] = 123 }
280
+
281
+ function Lang:tokenize(src)
282
+ local cfg = self.cfg
283
+ local T = setmetatable({ src = src, k = {}, s = {}, e = {}, l = {}, el = {}, m = {}, n = 0 }, Tokens)
284
+ local K, S, E, Ln, EL = T.k, T.s, T.e, T.l, T.el
285
+ local n, len, pos, line = 0, #src, 1, 1
286
+ local bol = true -- only whitespace seen on this line
287
+ local depth = 0 -- bracket depth (indent mode ignores newlines inside)
288
+ local skip = false -- inside a dead preprocessor branch
289
+ local pp = {} -- #if frames
290
+ local indents = { 0 }
291
+ local indent_mode = cfg.indent
292
+ local long_brackets = cfg.long_brackets
293
+ local template_literals, regex_literals = cfg.template_literals, cfg.regex_literals
294
+ local lifetimes, raw_strings = cfg.lifetimes, cfg.raw_strings
295
+ local nested_comments = cfg.nested_comments
296
+ -- regex_literals: per open '(' whether it follows if/while/for/with, and
297
+ -- the index of the last ')' closing such a condition (see regex_allowed)
298
+ local parens, ctrl_close = {}, 0
299
+ local dir_byte = cfg.directive and cfg.directive:byte()
300
+ local id_start, id_char, keywords, ops = self.id_start, self.id_char, self.keywords, self.ops
301
+
302
+ local function push(kind, a, b, l1, l2)
303
+ if skip then return end
304
+ n = n + 1
305
+ K[n], S[n], E[n], Ln[n], EL[n] = kind, a, b, l1, l2 or l1
306
+ end
307
+
308
+ -- At file scope (depth 0) every #if arm is kept: platform variants of
309
+ -- whole functions (#ifdef _WIN32 ... #else ...) are separate, balanced
310
+ -- definitions worth indexing. Inside brackets only the first live arm is
311
+ -- kept, because that is where arms split a construct
312
+ -- (`#ifdef X if (a) { #else if (b) { #endif`) and would unbalance braces.
313
+ local function directive(text)
314
+ if not cfg.pp_first_branch then return end
315
+ local word = text:match('^#%s*(%a+)')
316
+ if word == 'if' or word == 'ifdef' or word == 'ifndef' then
317
+ local zero = word == 'if' and text:match('^#%s*if%s+0%s*$') ~= nil
318
+ pp[#pp + 1] = { parent = skip, taken = not zero, keep_all = depth == 0 and not zero }
319
+ if zero then skip = true end
320
+ elseif (word == 'elif' or word == 'else') and #pp > 0 then
321
+ local f = pp[#pp]
322
+ if f.keep_all then skip = f.parent
323
+ elseif f.taken then skip = true
324
+ else skip = f.parent; f.taken = true end
325
+ elseif word == 'endif' and #pp > 0 then
326
+ skip = table.remove(pp).parent
327
+ end
328
+ end
329
+
330
+ while pos <= len do
331
+ local c = src:byte(pos)
332
+ if c == 10 then
333
+ if indent_mode and depth == 0 and n > 0 and K[n] ~= 'nl' and not skip then
334
+ push('nl', pos, pos, line)
335
+ end
336
+ line = line + 1
337
+ pos = pos + 1
338
+ bol = true
339
+ elseif c == 32 or c == 9 or c == 13 or c == 12 or c == 11 then
340
+ pos = pos + 1
341
+ elseif indent_mode and c == 92 and (src:byte(pos + 1) == 10
342
+ or (src:byte(pos + 1) == 13 and src:byte(pos + 2) == 10)) then
343
+ -- Python `\` continuation: the next line joins this logical line
344
+ pos = pos + (src:byte(pos + 1) == 10 and 2 or 3)
345
+ line = line + 1
346
+ else
347
+ -- Indentation (Python): measured at the first real token of a line.
348
+ if indent_mode and bol and depth == 0 then
349
+ local col_start = pos
350
+ while col_start > 1 and src:byte(col_start - 1) ~= 10 do col_start = col_start - 1 end
351
+ local width = 0
352
+ for x = col_start, pos - 1 do
353
+ width = width + ((src:byte(x) == 9) and (8 - width % 8) or 1)
354
+ end
355
+ local is_comment = false
356
+ for _, lc in ipairs(self.line_comment) do
357
+ if starts(src, pos, lc) then is_comment = true end
358
+ end
359
+ if not is_comment then
360
+ if width > indents[#indents] then
361
+ indents[#indents + 1] = width
362
+ push('indent', pos, pos - 1, line)
363
+ else
364
+ while width < indents[#indents] do
365
+ indents[#indents] = nil
366
+ push('dedent', pos, pos - 1, line)
367
+ end
368
+ end
369
+ end
370
+ end
371
+
372
+ local handled = false
373
+ -- Directive line (C preprocessor), with '\' continuations.
374
+ if dir_byte and c == dir_byte and bol then
375
+ local p = pos
376
+ local l0 = line
377
+ while true do
378
+ local nl = src:find('\n', p, true) or (len + 1)
379
+ local q = nl - 1
380
+ if src:byte(q) == 13 then q = q - 1 end
381
+ if src:byte(q) == 92 and nl <= len then
382
+ line = line + 1
383
+ p = nl + 1
384
+ else
385
+ p = nl
386
+ break
387
+ end
388
+ end
389
+ local text = src:sub(pos, p - 1)
390
+ -- A directive is kept when either side of it is live code, so
391
+ -- #include/#define in taken branches still reach the parser.
392
+ local was_skip = skip
393
+ directive(text)
394
+ if not (was_skip and skip) then
395
+ local now = skip
396
+ skip = false
397
+ push('dir', pos, p - 1, l0, line)
398
+ skip = now
399
+ end
400
+ pos = p
401
+ handled = true
402
+ end
403
+ bol = false
404
+
405
+ if not handled then
406
+ for _, lc in ipairs(self.line_comment) do
407
+ if starts(src, pos, lc) then
408
+ local lb = long_brackets and long_open(src, pos + #lc)
409
+ if lb then
410
+ -- --[==[ block comment ]==]
411
+ local stop = long_close(src, pos + #lc, lb)
412
+ line = line + count_nl(src, pos, stop)
413
+ pos = stop + 1
414
+ else
415
+ pos = src:find('\n', pos, true) or (len + 1)
416
+ end
417
+ handled = true
418
+ break
419
+ end
420
+ end
421
+ end
422
+ if not handled and long_brackets then
423
+ local lb = long_open(src, pos)
424
+ if lb then
425
+ local stop = long_close(src, pos, lb)
426
+ local l0 = line
427
+ line = line + count_nl(src, pos, stop)
428
+ push('str', pos, stop, l0, line)
429
+ pos = stop + 1
430
+ handled = true
431
+ end
432
+ end
433
+ if not handled then
434
+ for _, bc in ipairs(self.block_comment) do
435
+ if starts(src, pos, bc[1]) then
436
+ local stop
437
+ if nested_comments then
438
+ local nest, p = 1, pos + #bc[1]
439
+ while p + #bc[2] - 1 <= len do
440
+ if starts(src, p, bc[1]) then
441
+ nest = nest + 1
442
+ p = p + #bc[1]
443
+ elseif starts(src, p, bc[2]) then
444
+ nest = nest - 1
445
+ if nest == 0 then stop = p + #bc[2] - 1; break end
446
+ p = p + #bc[2]
447
+ else
448
+ p = p + 1
449
+ end
450
+ end
451
+ else
452
+ local _
453
+ _, stop = src:find(bc[2], pos + #bc[1], true)
454
+ end
455
+ stop = stop or len
456
+ line = line + count_nl(src, pos, stop)
457
+ pos = stop + 1
458
+ handled = true
459
+ break
460
+ end
461
+ end
462
+ end
463
+ if not handled and template_literals and c == 96 then
464
+ local stop = skip_template(src, pos)
465
+ local l0 = line
466
+ line = line + count_nl(src, pos, stop)
467
+ push('str', pos, stop, l0, line)
468
+ pos = stop + 1
469
+ handled = true
470
+ end
471
+ if not handled and regex_literals and c == 47 and regex_allowed(src, K, S, E, n, ctrl_close) then
472
+ local stop = scan_regex(src, pos, id_char)
473
+ if stop then
474
+ push('str', pos, stop, line)
475
+ pos = stop + 1
476
+ handled = true
477
+ end
478
+ end
479
+ if not handled and lifetimes and c == 39 then
480
+ local stop = scan_lifetime(src, pos, id_start, id_char)
481
+ if stop then
482
+ push('id', pos, stop, line)
483
+ pos = stop + 1
484
+ handled = true
485
+ end
486
+ end
487
+ if not handled and raw_strings and (c == 114 or c == 98 or c == 99) then
488
+ local stop = scan_raw_string(src, pos, id_char)
489
+ if stop then
490
+ local l0 = line
491
+ line = line + count_nl(src, pos, stop)
492
+ push('str', pos, stop, l0, line)
493
+ pos = stop + 1
494
+ handled = true
495
+ end
496
+ end
497
+ if not handled then
498
+ -- String, optionally after a prefix (r"", b'', f"""...""").
499
+ local q = pos
500
+ while self.prefixes[src:byte(q)] and q - pos < 2 do q = q + 1 end
501
+ for _, st in ipairs(self.strings) do
502
+ local open = st[1]
503
+ if starts(src, q, open) and (q == pos or not id_char[src:byte(pos - 1) or 0]) then
504
+ local close, esc = st[2], st.escape
505
+ local p = q + #open
506
+ local stop
507
+ while p <= len do
508
+ local b = src:byte(p)
509
+ if esc and b == esc:byte() then
510
+ p = p + 2
511
+ elseif starts(src, p, close) then
512
+ stop = p + #close - 1
513
+ break
514
+ elseif b == 10 and not st.multiline then
515
+ stop = p - 1
516
+ break
517
+ else
518
+ p = p + 1
519
+ end
520
+ end
521
+ stop = stop or len
522
+ local l0 = line
523
+ line = line + count_nl(src, pos, stop)
524
+ push('str', pos, stop, l0, line)
525
+ pos = stop + 1
526
+ handled = true
527
+ break
528
+ end
529
+ end
530
+ end
531
+ if not handled then
532
+ if id_start[c] then
533
+ local p = pos + 1
534
+ while p <= len and id_char[src:byte(p)] do p = p + 1 end
535
+ local word = src:sub(pos, p - 1)
536
+ push(keywords[word] and 'kw' or 'id', pos, p - 1, line)
537
+ pos = p
538
+ elseif (c >= 48 and c <= 57) or (c == 46 and (src:byte(pos + 1) or 0) >= 48 and (src:byte(pos + 1) or 0) <= 57) then
539
+ local p = pos + 1
540
+ while p <= len do
541
+ local b = src:byte(p)
542
+ if id_char[b] or b == 46 or b == 39 then
543
+ p = p + 1
544
+ elseif (b == 43 or b == 45) and (src:byte(p - 1) == 101 or src:byte(p - 1) == 69
545
+ or src:byte(p - 1) == 112 or src:byte(p - 1) == 80) then
546
+ p = p + 1 -- exponent sign: 1e-5, 0x1p+3
547
+ else
548
+ break
549
+ end
550
+ end
551
+ push('num', pos, p - 1, line)
552
+ pos = p
553
+ else
554
+ local w = 1
555
+ local list = ops[c]
556
+ if list then
557
+ for _, op in ipairs(list) do
558
+ if starts(src, pos, op) then w = #op; break end
559
+ end
560
+ end
561
+ if w == 1 and not skip then
562
+ if OPEN[c] then depth = depth + 1
563
+ elseif CLOSE[c] and depth > 0 then depth = depth - 1 end
564
+ end
565
+ push('op', pos, pos + w - 1, line)
566
+ if regex_literals and w == 1 and not skip then
567
+ if c == 40 then
568
+ local prev = n - 1
569
+ local ctrl = false
570
+ if prev >= 1 and K[prev] == 'kw' then
571
+ local pt = src:sub(S[prev], E[prev])
572
+ ctrl = pt == 'if' or pt == 'while' or pt == 'for' or pt == 'with'
573
+ end
574
+ parens[#parens + 1] = ctrl
575
+ elseif c == 41 then
576
+ local ctrl = table.remove(parens)
577
+ ctrl_close = ctrl and n or 0
578
+ end
579
+ end
580
+ pos = pos + w
581
+ end
582
+ end
583
+ end
584
+ end
585
+ if indent_mode then
586
+ if n > 0 and K[n] ~= 'nl' then push('nl', len + 1, len, line) end
587
+ for _ = 2, #indents do push('dedent', len + 1, len, line) end
588
+ end
589
+ T.n = n
590
+ T:compute_matches()
591
+ return T
592
+ end
593
+
594
+ -- Pair brackets (and indent/dedent). An unbalanced closer is left unmatched
595
+ -- instead of unwinding the stack, so one stray brace (usually from a
596
+ -- preprocessor branch) doesn't break every later match.
597
+ function Tokens:compute_matches()
598
+ local stack, m = {}, self.m
599
+ local K, S, E, src = self.k, self.s, self.e, self.src
600
+ for i = 1, self.n do
601
+ local kind = K[i]
602
+ if kind == 'op' and S[i] == E[i] then
603
+ local c = src:byte(S[i])
604
+ if OPEN[c] then
605
+ stack[#stack + 1] = i
606
+ elseif CLOSE[c] then
607
+ local top = stack[#stack]
608
+ if top and K[top] == 'op' and src:byte(S[top]) == CLOSE[c] then
609
+ stack[#stack] = nil
610
+ m[top], m[i] = i, top
611
+ end
612
+ end
613
+ elseif kind == 'indent' then
614
+ stack[#stack + 1] = i
615
+ elseif kind == 'dedent' then
616
+ local top = stack[#stack]
617
+ if top and K[top] == 'indent' then
618
+ stack[#stack] = nil
619
+ m[top], m[i] = i, top
620
+ end
621
+ end
622
+ end
623
+ end
624
+
625
+ return M
Binary file