@codua/codeoutline-win32-x64 0.1.9 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +38 -6
  2. package/README.zh-CN.md +36 -6
  3. package/THIRD_PARTY.md +3 -3
  4. package/bin/xnet.exe +0 -0
  5. package/build-info.json +1 -1
  6. package/package.json +1 -1
  7. package/scripts/codeoutline/cli.lua +2 -2
  8. package/scripts/codeoutline/command.lua +9 -1
  9. package/scripts/codeoutline/common.lua +45 -5
  10. package/scripts/codeoutline/completion.lua +936 -0
  11. package/scripts/codeoutline/control.lua +5 -1
  12. package/scripts/codeoutline/documents.lua +154 -0
  13. package/scripts/codeoutline/explore.lua +35 -8
  14. package/scripts/codeoutline/graph.lua +321 -102
  15. package/scripts/codeoutline/index.lua +170 -50
  16. package/scripts/codeoutline/index_worker.lua +108 -17
  17. package/scripts/codeoutline/indirect.lua +201 -0
  18. package/scripts/codeoutline/js_evidence.lua +175 -0
  19. package/scripts/codeoutline/js_resolution.lua +201 -0
  20. package/scripts/codeoutline/lang/c.lua +22 -12
  21. package/scripts/codeoutline/lang/cpp.lua +13 -12
  22. package/scripts/codeoutline/lang/csharp.lua +21 -21
  23. package/scripts/codeoutline/lang/go.lua +10 -10
  24. package/scripts/codeoutline/lang/java.lua +9 -9
  25. package/scripts/codeoutline/lang/js.lua +15 -14
  26. package/scripts/codeoutline/lang/kotlin.lua +384 -0
  27. package/scripts/codeoutline/lang/lua.lua +11 -6
  28. package/scripts/codeoutline/lang/python.lua +6 -6
  29. package/scripts/codeoutline/lang/rust.lua +23 -15
  30. package/scripts/codeoutline/lsp.lua +574 -0
  31. package/scripts/codeoutline/lua_bindings.lua +222 -0
  32. package/scripts/codeoutline/main.lua +28 -5
  33. package/scripts/codeoutline/navigation.lua +1046 -0
  34. package/scripts/codeoutline/parse.lua +9 -2
  35. package/scripts/codeoutline/parse_pool.lua +189 -0
  36. package/scripts/codeoutline/parse_worker.lua +24 -0
  37. package/scripts/codeoutline/service.lua +82 -31
  38. package/scripts/codeoutline/update_sequence.lua +1 -1
  39. package/scripts/codeoutline/version.lua +1 -1
  40. package/licenses/rpmalloc.txt +0 -12
@@ -19,6 +19,7 @@ M.EXTENSIONS = {
19
19
  c = 'c', h = 'c',
20
20
  py = 'python', pyi = 'python',
21
21
  java = 'java',
22
+ kt = 'kotlin', kts = 'kotlin',
22
23
  lua = 'lua',
23
24
  cpp = 'cpp', cc = 'cpp', cxx = 'cpp', hpp = 'cpp', hh = 'cpp', hxx = 'cpp', ipp = 'cpp',
24
25
  go = 'go',
@@ -36,14 +37,20 @@ function M.language(path)
36
37
  return ext and M.EXTENSIONS[ext:lower()]
37
38
  end
38
39
 
39
- function M.parse(path, src)
40
+ -- Returns the record and its tokens; positions in both exclude a UTF-8 BOM.
41
+ function M.parse_tokens(path, src, opts)
40
42
  local lang = M.language(path)
41
43
  if not lang then return nil end
42
44
  -- Strip a UTF-8 BOM so it is not read as part of the first token.
43
45
  if src:sub(1, 3) == '\239\187\191' then src = src:sub(4) end
44
46
  -- .h is shared by C and C++: sniff for C++-only syntax.
45
47
  if lang == 'c' and path:match('%.[hH]$') and parser('cpp').looks_cpp(src) then lang = 'cpp' end
46
- return (parser(MODULE[lang] or lang).parse(path, src, lang))
48
+ return parser(MODULE[lang] or lang).parse(path, src, lang, opts)
49
+ end
50
+
51
+ -- opts.ranges=false emits compact index nodes; default parsing retains positions.
52
+ function M.parse(path, src, opts)
53
+ return (M.parse_tokens(path, src, opts))
47
54
  end
48
55
 
49
56
  return M
@@ -0,0 +1,189 @@
1
+ -- parse_pool.lua — spreads a full refresh's file reads and parses over
2
+ -- parse threads (parse_worker.lua) started by a long-lived host.
3
+ --
4
+ -- pool.configure(script, count) -- create threads on the first parallel pass
5
+ -- pool.bind(co) -- the job coroutine allowed to wait
6
+ -- -- on every 'parsed' message: pool.receive(token), even after cancellation
7
+ -- -- then, while pool.waiting():
8
+ -- coroutine.resume(co, token, results, failure)
9
+ -- pool.sweep() -- host update: reclaim idle threads
10
+ --
11
+ -- The index stays owned by the host thread: workers only read and parse,
12
+ -- and every commit happens here as results arrive. Files are split up front
13
+ -- by size (largest first onto the least-loaded thread) and posted in chunks,
14
+ -- so results stream back and a cancelled pass stops within a chunk.
15
+
16
+ local control = require('codeoutline.control')
17
+
18
+ local M = {}
19
+ M.THREADS = package.config:sub(1, 1) == '\\' and 8 or 6
20
+ M.FIRST_ID = 20 -- xnet2lua XTHR_WORKER_GRP1
21
+ M.CHUNK_FILES = 32
22
+ M.CHUNK_BYTES = 1048576
23
+ M.IDLE_MS = 60000
24
+
25
+ local threads, owner, shared = {}, nil, nil
26
+ local job, waiting, token = nil, false, 0
27
+ local script_path, thread_count
28
+ local outstanding, running, idle_since = {}, false, nil
29
+
30
+ local function log(format, ...)
31
+ xthread.log_info('[codeoutline parse] ' .. format, ...)
32
+ end
33
+
34
+ -- Configure without allocating threads; zero disables parallel parsing.
35
+ function M.configure(script, count)
36
+ script_path, thread_count = script, count or M.THREADS
37
+ end
38
+
39
+ local function start_threads()
40
+ shared = assert(xshared.dict('codeoutline_control'))
41
+ owner = xthread.current_id()
42
+ for i = 1, thread_count do
43
+ local id = M.FIRST_ID + i - 1
44
+ local ok, err = xthread.create_thread(id, 'PARSE' .. i, script_path)
45
+ if ok then threads[#threads + 1] = id
46
+ else io.stderr:write('[codeoutline] parse thread ', i, ': ', tostring(err), '\n') end
47
+ end
48
+ idle_since = xtimer.now_ms()
49
+ log('pool started: threads=%d requested=%d', #threads, thread_count)
50
+ end
51
+
52
+ local function stop_threads(reason)
53
+ local started, count, stopped = xtimer.now_ms(), #threads, 0
54
+ if shared then shared:set('parse_abort', token) end
55
+ for _, id in ipairs(threads) do
56
+ local ok, err = xthread.shutdown_thread(id)
57
+ if ok then stopped = stopped + 1
58
+ else log('thread stop failed: id=%d reason=%s', id, tostring(err)) end
59
+ end
60
+ if count > 0 then
61
+ log('pool stopped: threads=%d/%d reason=%s elapsed_ms=%d',
62
+ stopped, count, reason or 'shutdown', xtimer.now_ms() - started)
63
+ end
64
+ threads = {}
65
+ outstanding, idle_since = {}, nil
66
+ -- Keep token monotonic across restarts: parse_abort and late messages
67
+ -- may still refer to the previous pool's passes.
68
+ end
69
+
70
+ -- xthread releases its ownership reference only from the Lua state that
71
+ -- created the thread (coroutines have distinct lua_State pointers). Keep
72
+ -- both lifecycle calls on one coroutine across jobs and host updates.
73
+ local lifecycle = coroutine.wrap(function(action, reason)
74
+ while true do
75
+ if action == 'start' then start_threads() else stop_threads(reason) end
76
+ action, reason = coroutine.yield()
77
+ end
78
+ end)
79
+
80
+ function M.stop(reason) lifecycle('stop', reason) end
81
+
82
+ function M.bind(co) job = co end
83
+
84
+ -- Whether the running code may hand files to the pool and wait for them.
85
+ function M.ready()
86
+ if job == nil or coroutine.running() ~= job or not script_path or thread_count < 1 then return false end
87
+ if #threads == 0 then lifecycle('start') end
88
+ return #threads > 0
89
+ end
90
+
91
+ function M.waiting() return waiting end
92
+
93
+ -- Account for every reply, including abandoned passes with no job to resume.
94
+ function M.receive(got)
95
+ local pending = outstanding[got]
96
+ if not pending then return false end
97
+ outstanding[got] = pending > 1 and pending - 1 or nil
98
+ if not running and not next(outstanding) then idle_since = xtimer.now_ms() end
99
+ return true
100
+ end
101
+
102
+ function M.sweep()
103
+ if not running and not next(outstanding) and idle_since
104
+ and xtimer.now_ms() - idle_since >= M.IDLE_MS then
105
+ M.stop(string.format('idle %dms', xtimer.now_ms() - idle_since))
106
+ end
107
+ end
108
+
109
+ -- Size-balanced split: largest first, each onto the least-loaded thread.
110
+ local function partition(jobs, n)
111
+ local order = {}
112
+ for i, j in ipairs(jobs) do order[i] = j end
113
+ table.sort(order, function(a, b)
114
+ if a.weight ~= b.weight then return a.weight > b.weight end
115
+ return a.rel < b.rel
116
+ end)
117
+ local buckets, loads = {}, {}
118
+ for i = 1, n do buckets[i], loads[i] = {}, 0 end
119
+ for _, j in ipairs(order) do
120
+ local best = 1
121
+ for b = 2, n do if loads[b] < loads[best] then best = b end end
122
+ local bucket = buckets[best]
123
+ bucket[#bucket + 1] = j
124
+ loads[best] = loads[best] + j.weight
125
+ end
126
+ return buckets, loads
127
+ end
128
+ M.partition = partition
129
+
130
+ -- jobs: { rel, abs, weight, size?, crc? }; on_result(rel, load_file result)
131
+ -- runs in this thread, in arrival order. Must be called from the bound job.
132
+ function M.run(jobs, max_bytes, on_result)
133
+ assert(M.ready(), 'parse pool is not available here')
134
+ token = token + 1
135
+ local current, pending = token, 0
136
+ local started, completed = xtimer.now_ms(), 0
137
+ local counts = { parsed = 0, unchanged = 0, skipped = 0, failed = 0 }
138
+ log('pass started: token=%d files=%d threads=%d', current, #jobs, #threads)
139
+ running, idle_since = true, nil
140
+ local ok, err = pcall(function()
141
+ for i, bucket in ipairs(partition(jobs, #threads)) do
142
+ local chunk, bytes = {}, 0
143
+ local function flush()
144
+ if #chunk == 0 then return end
145
+ local posted, why = xthread.post(threads[i], 'parse', owner, current, max_bytes, chunk)
146
+ assert(posted, 'cannot post to parse thread: ' .. tostring(why))
147
+ pending = pending + 1
148
+ outstanding[current] = (outstanding[current] or 0) + 1
149
+ chunk, bytes = {}, 0
150
+ end
151
+ for _, j in ipairs(bucket) do
152
+ chunk[#chunk + 1] = { j.rel, j.abs, j.size or false, j.crc or false }
153
+ bytes = bytes + j.weight
154
+ if #chunk >= M.CHUNK_FILES or bytes >= M.CHUNK_BYTES then flush() end
155
+ end
156
+ flush()
157
+ end
158
+ while pending > 0 do
159
+ waiting = true
160
+ local got, results, failure = coroutine.yield()
161
+ waiting = false
162
+ if got == current then
163
+ pending = pending - 1
164
+ if failure then error('parse thread failed: ' .. tostring(failure), 0) end
165
+ control.check()
166
+ for _, res in ipairs(results) do
167
+ on_result(res.rel, res)
168
+ completed = completed + 1
169
+ counts[res.status] = counts[res.status] + 1
170
+ end
171
+ end
172
+ end
173
+ end)
174
+ waiting = false
175
+ running = false
176
+ if not next(outstanding) then idle_since = xtimer.now_ms() end
177
+ -- Wall time includes dispatch, waiting for workers and applying results.
178
+ local status = ok and 'completed' or (control.is_interrupted(err) and err.code or 'failed')
179
+ log('pass %s: token=%d files=%d/%d parsed=%d unchanged=%d skipped=%d failed=%d elapsed_ms=%d',
180
+ status, current, completed, #jobs, counts.parsed, counts.unchanged, counts.skipped, counts.failed,
181
+ xtimer.now_ms() - started)
182
+ if not ok then
183
+ -- Workers skip the rest of this pass; late results carry a stale token.
184
+ shared:set('parse_abort', current)
185
+ error(err, 0)
186
+ end
187
+ end
188
+
189
+ return M
@@ -0,0 +1,24 @@
1
+ -- Parse thread for parse_pool.lua: reads and parses the files it is posted
2
+ -- and posts the results back to the thread that owns the index.
3
+ local source = assert(xutils.realpath(debug.getinfo(1, 'S').source:sub(2))):gsub('\\', '/')
4
+ local scripts = assert(source:match('^(.*)/codeoutline/[^/]+$'))
5
+ package.path = scripts .. '/?.lua;' .. package.path
6
+ local index = require('codeoutline.index')
7
+ local shared = assert(xshared.dict('codeoutline_control'))
8
+
9
+ return {
10
+ __thread_handle = function(_, op, owner, token, max_bytes, files)
11
+ if op ~= 'parse' then return end
12
+ local results = {}
13
+ local ok, err = pcall(function()
14
+ for i, f in ipairs(files) do
15
+ -- An abandoned pass (cancelled, failed, shutting down) skips its rest.
16
+ if shared:get('shutdown') or (shared:get('parse_abort') or 0) >= token then return end
17
+ local res = index.load_file(f[2], f[1], max_bytes, f[3] or nil, f[4] or nil)
18
+ res.rel = f[1]
19
+ results[i] = res
20
+ end
21
+ end)
22
+ xthread.post(owner, 'parsed', token, results, not ok and tostring(err) or nil)
23
+ end,
24
+ }
@@ -1,8 +1,8 @@
1
1
  -- Requests within one Lua state are serialized by the owning transport.
2
2
  local index = require('codeoutline.index')
3
- local graph = require('codeoutline.graph')
4
3
  local explore = require('codeoutline.explore')
5
4
  local paths = require('codeoutline.path')
5
+ local control = require('codeoutline.control')
6
6
  local M = {}
7
7
  local projects = {}
8
8
  local sequence = 0
@@ -10,11 +10,18 @@ local limits = { max_projects = 8, idle_seconds = 900 }
10
10
  -- Long-lived hosts watch resident projects so a query refreshes only what
11
11
  -- changed; one-shot commands would pay for a watcher they never read.
12
12
  local watch = false
13
+ -- LuaJIT starts a collection cycle only once the heap has doubled since the
14
+ -- last one. A resident index makes the heap large, so the garbage of polls
15
+ -- and requests (and a dropped project's whole index) would build up toward
16
+ -- the index's own size; hosts collect it on idle ticks with M.collect.
17
+ local gc = { live = collectgarbage('count'), running = false }
18
+ -- A 512 KB step takes about 1 ms (6 ms at most) on a 650 MB heap.
19
+ M.GC_MIN_KB, M.GC_STEP_KB = 16384, 512
13
20
 
14
21
  local function drop(key)
15
22
  local p = projects[key]
16
23
  projects[key] = nil
17
- if p then p.idx:close() end
24
+ if p then p.idx:close(); gc.running = true end
18
25
  end
19
26
 
20
27
  function M.configure(opts)
@@ -32,14 +39,17 @@ function M.configure(opts)
32
39
  M.sweep()
33
40
  end
34
41
 
42
+ -- Hosts sweep on every tick: allocate only when something is dropped.
35
43
  function M.sweep(now)
36
44
  now = now or os.time()
37
- local count = 0
38
- local idle = {}
45
+ local count, idle = 0, nil
39
46
  for key, p in pairs(projects) do
40
- if now - p.used >= limits.idle_seconds then idle[#idle + 1] = key else count = count + 1 end
47
+ if now - p.used >= limits.idle_seconds then
48
+ idle = idle or {}
49
+ idle[#idle + 1] = key
50
+ else count = count + 1 end
41
51
  end
42
- for _, key in ipairs(idle) do drop(key) end
52
+ if idle then for _, key in ipairs(idle) do drop(key) end end
43
53
  while count > limits.max_projects do
44
54
  local oldest
45
55
  for key, p in pairs(projects) do
@@ -51,13 +61,28 @@ function M.sweep(now)
51
61
  return count
52
62
  end
53
63
 
64
+ -- Run one bounded incremental GC step on an idle tick; returns whether a cycle
65
+ -- is still in progress. A cycle starts once the garbage since the last one
66
+ -- reaches a sixteenth of the live heap (at least GC_MIN_KB), or a project is
67
+ -- dropped, and then advances one step per call until it completes.
68
+ function M.collect()
69
+ if not gc.running then
70
+ if collectgarbage('count') - gc.live < math.max(M.GC_MIN_KB, gc.live / 16) then return false end
71
+ gc.running = true
72
+ end
73
+ if collectgarbage('step', M.GC_STEP_KB) then gc.running, gc.live = false, collectgarbage('count') end
74
+ return gc.running
75
+ end
76
+
54
77
  local function key_for(root)
55
78
  local canonical, err = index.check_root(root)
56
79
  assert(canonical, err)
57
80
  return paths.key(canonical), canonical
58
81
  end
59
82
 
60
- function M.get(root, opts)
83
+ -- Files commit one at a time, so an interrupted refresh needs no rollback:
84
+ -- the index stays consistent and the next refresh resumes with a full pass.
85
+ local function refresh(root, opts, with_symbols)
61
86
  opts = opts or {}
62
87
  local key, canonical = key_for(root)
63
88
  M.sweep()
@@ -75,53 +100,77 @@ function M.get(root, opts)
75
100
  if watch then p.idx:watch() end
76
101
  end
77
102
  local t0 = os.clock()
78
- local stats = p.idx:refresh()
79
- if not p.G or p.G.generation ~= p.idx.generation then p.G = graph.build(p.idx) end
103
+ local ok, stats = pcall(function()
104
+ local refresh_opts = opts
105
+ if not projects[key] and opts.paths then refresh_opts = { full_interval = opts.full_interval } end
106
+ local result = p.idx:refresh(refresh_opts)
107
+ if with_symbols then p.idx:symbols() end
108
+ return result
109
+ end)
110
+ if not ok then
111
+ if not projects[key] then p.idx:close() end
112
+ error(stats, 0)
113
+ end
114
+ sequence = sequence + 1
115
+ p.used, p.stats, p.order = os.time(), stats, sequence
116
+ projects[key] = p
117
+ M.sweep()
80
118
  -- A failed save leaves the index behind its cache, so the next call retries.
81
119
  if p.idx:needs_save() then
82
120
  local saved, err = p.idx:save()
83
121
  stats.cache_saved, stats.cache_error = saved, err
84
122
  end
85
123
  stats.seconds = os.clock() - t0
86
- sequence = sequence + 1
87
- p.used, p.stats, p.order = os.time(), stats, sequence
88
- projects[key] = p
89
- M.sweep()
90
- return p.idx, p.G, stats
124
+ return p.idx, with_symbols and p.idx:symbols() or nil, stats
125
+ end
126
+
127
+ -- LSP refreshes records; symbol tables are kept current only once built.
128
+ function M.refresh(root, opts)
129
+ local idx, _, stats = refresh(root, opts, false)
130
+ return idx, stats
131
+ end
132
+
133
+ -- Refresh, and return the symbol tables relationships resolve against.
134
+ function M.get(root, opts)
135
+ return refresh(root, opts, true)
136
+ end
137
+
138
+ -- Worker-owned resident state; callers must serialize access with get/explore.
139
+ -- This does not refresh the filesystem or build symbol tables.
140
+ function M.resident(root)
141
+ local key = key_for(root)
142
+ local p = assert(projects[key], 'Project is indexing; retry when indexing completes')
143
+ p.used = os.time()
144
+ return p.idx, p.idx.symbol_tables
91
145
  end
92
146
 
93
147
  function M.explore(root, query, opts)
94
148
  explore.validate(query, opts)
95
- local idx, G, stats = M.get(root, opts)
149
+ local idx, _, stats = refresh(root, opts, true)
96
150
  idx.query_sources = {}
97
- local ok, text, info = pcall(function()
151
+ local ok, result = pcall(function()
98
152
  while true do
99
- local output, details = explore.run(G, idx, query, opts)
100
- if not details.encoding_retry then return output, details end
101
- G = graph.build(idx)
153
+ -- A GBK repair commits the decoded file, which renumbers its nodes.
154
+ local output, details = explore.run(idx:symbols(), idx, query, opts)
155
+ control.check()
156
+ if not details.encoding_retry then return { text = output, info = details } end
102
157
  end
103
158
  end)
104
159
  idx.query_sources = nil
105
- -- A partial encoding repair can precede an error in another file. Keep
106
- -- the resident graph consistent even when the query itself fails.
107
- local p = projects[paths.key(idx.root)]
108
- if p.G.generation ~= idx.generation then
109
- p.G = G.generation == idx.generation and G or graph.build(idx)
110
- end
160
+ if not ok then error(result, 0) end
111
161
  if idx:needs_save() then
112
162
  local saved, err = idx:save()
113
163
  stats.cache_saved, stats.cache_error = saved, err
114
164
  end
115
- if not ok then error(text, 2) end
116
- info.refresh = stats
117
- return text, info
165
+ result.info.refresh = stats
166
+ return result.text, result.info
118
167
  end
119
168
 
120
169
  function M.status(root, opts)
121
170
  local idx, G, stats = M.get(root, opts)
122
171
  local languages = {}
123
- for _, rec in ipairs(G.files) do languages[rec.language] = (languages[rec.language] or 0) + 1 end
124
- return { projectPath = idx.root, files = #G.files, symbols = #G.nodes, edges = G.edge_count,
172
+ for _, rec in pairs(idx.files) do languages[rec.language] = (languages[rec.language] or 0) + 1 end
173
+ return { projectPath = idx.root, files = G.file_count, symbols = G.node_count,
125
174
  generation = idx.generation, languages = languages, refresh = stats,
126
175
  cachePath = idx.cache_path, cacheLoaded = idx.cache_loaded, cacheWarning = idx.cache_error,
127
176
  enumerator = idx.enumerator, scanner = xscan and os.getenv('XSCAN_PURE_LUA') ~= '1' and 'native' or 'lua',
@@ -130,7 +179,9 @@ end
130
179
 
131
180
  -- Forget drops memory only. Rebuild bypasses cache and publishes after success.
132
181
  function M.forget(root)
133
- local key = key_for(root)
182
+ -- A resident canonical root can disappear after a checkout or deletion.
183
+ local key = type(root) == 'string' and paths.key(root)
184
+ if not key or not projects[key] then key = key_for(root) end
134
185
  local existed = projects[key] ~= nil
135
186
  drop(key)
136
187
  return existed
@@ -1 +1 @@
1
- return 9
1
+ return 11
@@ -1 +1 @@
1
- return '0.1.9'
1
+ return '0.2.1'
@@ -1,12 +0,0 @@
1
- /* rpmalloc.c - Memory allocator - Public Domain - 2016-2020 Mattias
2
- * Jansson
3
- *
4
- * This library provides a cross-platform lock free thread caching malloc
5
- * implementation in C11. The latest source code is always available at
6
- *
7
- * https://github.com/mjansson/rpmalloc
8
- *
9
- * This library is put in the public domain; you can redistribute it and/or
10
- * modify it without any restrictions.
11
- *
12
- */