@codua/codeoutline-win32-x64 0.1.5 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISTRIBUTION.md +60 -15
- package/README.md +7 -7
- package/README.zh-CN.md +7 -7
- package/bin/xnet.exe +0 -0
- package/build-info.json +1 -1
- package/codeoutline +3 -1
- package/codeoutline.cmd +5 -1
- package/package.json +1 -1
- package/scripts/codeoutline/command.lua +103 -7
- package/scripts/codeoutline/common.lua +10 -2
- package/scripts/codeoutline/daemon.lua +221 -0
- package/scripts/codeoutline/explore.lua +46 -42
- package/scripts/codeoutline/graph.lua +89 -54
- package/scripts/codeoutline/http.lua +5 -0
- package/scripts/codeoutline/index.lua +246 -66
- package/scripts/codeoutline/index_worker.lua +1 -0
- package/scripts/codeoutline/main.lua +71 -11
- package/scripts/codeoutline/service.lua +25 -4
- package/scripts/codeoutline/update_sequence.lua +1 -1
- package/scripts/codeoutline/update_worker.lua +17 -7
- package/scripts/codeoutline/updater.lua +7 -4
- package/scripts/codeoutline/version.lua +1 -1
- package/scripts/codeoutline/xscan.lua +41 -2
- package/scripts/xupgate/client.lua +77 -1
|
@@ -5,11 +5,14 @@
|
|
|
5
5
|
-- local idx = index.open('C:/src/proj') -- loads the cache if present
|
|
6
6
|
-- local stats = idx:refresh() -- reparses only changed files
|
|
7
7
|
-- idx:save()
|
|
8
|
+
-- idx:watch() -- long-lived hosts: later
|
|
9
|
+
-- -- refreshes visit only changes
|
|
8
10
|
--
|
|
9
11
|
-- Change detection trusts an unchanged size + mtime once the file was last
|
|
10
12
|
-- read strictly after that mtime second, skipping the read entirely; otherwise
|
|
11
13
|
-- it reads the file and compares size + CRC-32, which still skips the parse.
|
|
12
|
-
-- Tools that restore an older mtime at the same size are not detected
|
|
14
|
+
-- Tools that restore an older mtime at the same size are not detected by a
|
|
15
|
+
-- full refresh; a watcher reports them.
|
|
13
16
|
|
|
14
17
|
local ci = require('codeoutline.parse')
|
|
15
18
|
local paths = require('codeoutline.path')
|
|
@@ -32,6 +35,10 @@ end)()
|
|
|
32
35
|
-- Generated amalgamations (sqlite3.c, minified bundles) cost more to index
|
|
33
36
|
-- than they give back; skip files above this size unless configured.
|
|
34
37
|
M.MAX_FILE_BYTES = 1500000
|
|
38
|
+
-- Caches are deflated: msgpack records shrink to ~1/6. Level 1 is within a
|
|
39
|
+
-- few percent of level 6 at under half the time; inflating is cheap next to
|
|
40
|
+
-- unpacking, and the checksum then covers the smaller compressed bytes.
|
|
41
|
+
M.CACHE_COMPRESSION_LEVEL = 1
|
|
35
42
|
|
|
36
43
|
local Index = {}
|
|
37
44
|
Index.__index = Index
|
|
@@ -102,12 +109,18 @@ end
|
|
|
102
109
|
|
|
103
110
|
function Index:load()
|
|
104
111
|
local data = slurp(self.cache_path)
|
|
105
|
-
if not data or not cmsgpack then return false end
|
|
112
|
+
if not data or not cmsgpack or not (xcompress and xcompress.inflate) then return false end
|
|
106
113
|
local ok, files = pcall(function()
|
|
107
114
|
local envelope = cmsgpack.unpack(data)
|
|
108
115
|
assert(type(envelope) == 'table' and type(envelope.payload) == 'string')
|
|
109
116
|
assert(envelope.sha256 == xutils.sha256_hex(envelope.payload))
|
|
110
|
-
|
|
117
|
+
-- Older uncompressed caches have no codec and are rebuilt.
|
|
118
|
+
assert(envelope.codec == 'deflate')
|
|
119
|
+
local size = envelope.size
|
|
120
|
+
assert(type(size) == 'number' and size >= 1 and size % 1 == 0)
|
|
121
|
+
local payload = assert(xcompress.inflate(envelope.payload, size))
|
|
122
|
+
assert(#payload == size)
|
|
123
|
+
local t = cmsgpack.unpack(payload)
|
|
111
124
|
assert(t.version == M.VERSION and t.root == paths.key(self.root) and type(t.files) == 'table')
|
|
112
125
|
local records = {}
|
|
113
126
|
for _, rec in ipairs(t.files) do
|
|
@@ -121,7 +134,17 @@ function Index:load()
|
|
|
121
134
|
assert(type(n.name) == 'string' and type(n.qualified) == 'string' and type(n.kind) == 'string')
|
|
122
135
|
assert(type(n.line) == 'number' and type(n.end_line) == 'number' and n.line >= 1 and n.end_line >= n.line)
|
|
123
136
|
end
|
|
124
|
-
|
|
137
|
+
local refs = rec.refs
|
|
138
|
+
assert(type(refs.name) == 'table' and type(refs.from) == 'table' and type(refs.kind) == 'table'
|
|
139
|
+
and type(refs.line) == 'table' and type(refs.recv) == 'table')
|
|
140
|
+
local nrefs = #refs.name
|
|
141
|
+
assert(#refs.from == nrefs and #refs.kind == nrefs and #refs.line == nrefs and #refs.recv == nrefs)
|
|
142
|
+
for i = 1, nrefs do
|
|
143
|
+
assert(type(refs.name[i]) == 'string' and type(refs.from[i]) == 'number'
|
|
144
|
+
and type(refs.kind[i]) == 'string' and type(refs.line[i]) == 'number')
|
|
145
|
+
local recv = refs.recv[i]
|
|
146
|
+
assert(recv == false or type(recv) == 'string')
|
|
147
|
+
end
|
|
125
148
|
for _, imp in ipairs(rec.imports) do assert(type(imp.path) == 'string') end
|
|
126
149
|
records[rec.path] = rec
|
|
127
150
|
end
|
|
@@ -141,14 +164,15 @@ function Index:needs_save()
|
|
|
141
164
|
end
|
|
142
165
|
|
|
143
166
|
function Index:save()
|
|
144
|
-
if not (cmsgpack and xutils.temp_file and xutils.replace_file) then
|
|
145
|
-
return false, 'runtime requires cmsgpack, temp_file and replace_file'
|
|
167
|
+
if not (cmsgpack and xcompress and xcompress.deflate and xutils.temp_file and xutils.replace_file) then
|
|
168
|
+
return false, 'runtime requires cmsgpack, xcompress, temp_file and replace_file'
|
|
146
169
|
end
|
|
147
170
|
local list = {}
|
|
148
171
|
for _, rec in pairs(self.files) do list[#list + 1] = rec end
|
|
149
172
|
table.sort(list, function(a, b) return a.path < b.path end)
|
|
150
173
|
local payload = cmsgpack.pack({ version = M.VERSION, root = paths.key(self.root), files = list })
|
|
151
|
-
local
|
|
174
|
+
local packed = xcompress.deflate(payload, M.CACHE_COMPRESSION_LEVEL)
|
|
175
|
+
local blob = cmsgpack.pack({ codec = 'deflate', size = #payload, sha256 = xutils.sha256_hex(packed), payload = packed })
|
|
152
176
|
local d = self.cache_path:match('^(.*)/[^/]*$') or '.'
|
|
153
177
|
local made, err = xutils.mkdir_p(d)
|
|
154
178
|
if not made then return false, err end
|
|
@@ -264,8 +288,9 @@ local function ignored(rules, rel, name, is_dir)
|
|
|
264
288
|
end
|
|
265
289
|
|
|
266
290
|
-- Supported files under the root without a subprocess (xutils.list_dir).
|
|
291
|
+
-- Also returns each file's stat fields when the runtime lists them.
|
|
267
292
|
function Index:walk_files()
|
|
268
|
-
local out = {}
|
|
293
|
+
local out, meta = {}, {}
|
|
269
294
|
assert(xutils and xutils.list_dir, 'file enumeration unavailable: install rg or use a runtime with list_dir')
|
|
270
295
|
local rules = load_ignore(self.root)
|
|
271
296
|
local dirs = { '' }
|
|
@@ -273,7 +298,8 @@ function Index:walk_files()
|
|
|
273
298
|
while #dirs > 0 do
|
|
274
299
|
control.check()
|
|
275
300
|
local rel_dir = table.remove(dirs)
|
|
276
|
-
local entries, truncated = xutils.list_dir(rel_dir == '' and self.root or (self.root .. '/' .. rel_dir),
|
|
301
|
+
local entries, truncated = xutils.list_dir(rel_dir == '' and self.root or (self.root .. '/' .. rel_dir),
|
|
302
|
+
M.MAX_WALK_FILES + 1, true)
|
|
277
303
|
assert(entries, 'cannot enumerate directory: ' .. rel_dir .. ': ' .. tostring(truncated))
|
|
278
304
|
assert(not truncated, 'directory enumeration limit exceeded')
|
|
279
305
|
for _, e in ipairs(entries) do
|
|
@@ -281,18 +307,19 @@ function Index:walk_files()
|
|
|
281
307
|
assert(visited <= M.MAX_WALK_FILES, 'directory enumeration limit exceeded')
|
|
282
308
|
local name = e.name
|
|
283
309
|
local rel = rel_dir == '' and name or (rel_dir .. '/' .. name)
|
|
284
|
-
local st = xutils.stat(self.root .. '/' .. rel)
|
|
310
|
+
local st = e.type and e or xutils.stat(self.root .. '/' .. rel)
|
|
285
311
|
if st and st.type ~= 'link' and name:sub(1, 1) ~= '.' and not ignored(rules, rel, name, e.dir) then
|
|
286
312
|
if e.dir then
|
|
287
313
|
if not M.WALK_SKIP[name] then dirs[#dirs + 1] = rel end
|
|
288
314
|
elseif ci.language(name) then
|
|
289
315
|
out[#out + 1] = rel
|
|
316
|
+
meta[rel] = st
|
|
290
317
|
end
|
|
291
318
|
end
|
|
292
319
|
end
|
|
293
320
|
end
|
|
294
321
|
table.sort(out)
|
|
295
|
-
return out
|
|
322
|
+
return out, meta
|
|
296
323
|
end
|
|
297
324
|
|
|
298
325
|
function Index:list_files()
|
|
@@ -326,66 +353,219 @@ function Index:list_files()
|
|
|
326
353
|
return out
|
|
327
354
|
end
|
|
328
355
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
356
|
+
local MISSING = { exists = false }
|
|
357
|
+
|
|
358
|
+
-- Stat fields for every listed file, read one directory at a time:
|
|
359
|
+
-- list_dir reports them per entry, so a tree costs a call per directory
|
|
360
|
+
-- rather than a stat per file. Older runtimes fall back to stat.
|
|
361
|
+
function Index:stat_listing(list)
|
|
362
|
+
local by_dir = {}
|
|
363
|
+
for _, rel in ipairs(list) do
|
|
364
|
+
local dir, name = rel:match('^(.*)/([^/]*)$')
|
|
365
|
+
if not dir then dir, name = '', rel end
|
|
366
|
+
local names = by_dir[dir]
|
|
367
|
+
if not names then names = {}; by_dir[dir] = names end
|
|
368
|
+
names[#names + 1] = name
|
|
369
|
+
end
|
|
370
|
+
local meta = {}
|
|
371
|
+
for dir, names in pairs(by_dir) do
|
|
372
|
+
control.check()
|
|
373
|
+
local entries = xutils.list_dir(dir == '' and self.root or (self.root .. '/' .. dir), nil, true)
|
|
374
|
+
local by_name
|
|
375
|
+
if entries and (entries[1] == nil or entries[1].size ~= nil) then
|
|
376
|
+
by_name = {}
|
|
377
|
+
for _, e in ipairs(entries) do by_name[e.name] = e end
|
|
378
|
+
end
|
|
379
|
+
for _, name in ipairs(names) do
|
|
380
|
+
local rel = dir == '' and name or (dir .. '/' .. name)
|
|
381
|
+
meta[rel] = by_name and (by_name[name] or MISSING) or xutils.stat(self:abs(rel))
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
return meta
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
local function new_stats(mode)
|
|
388
|
+
return { parsed = 0, unchanged = 0, removed = 0, skipped = 0, failed = 0, mode = mode }
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
-- Bring one listed file in line with the tree, given its stat fields (nil
|
|
392
|
+
-- when the runtime has no stat). Returns false when it no longer belongs in
|
|
393
|
+
-- the index: gone, too large, unreadable or unparsable.
|
|
394
|
+
function Index:sync_file(rel, st, stats)
|
|
395
|
+
local rec = self.files[rel]
|
|
396
|
+
if st and (st.exists == false or st.type ~= 'file') then
|
|
397
|
+
return false
|
|
398
|
+
elseif st and st.size and st.size > self.max_bytes then
|
|
399
|
+
stats.skipped = stats.skipped + 1
|
|
400
|
+
return false
|
|
401
|
+
elseif st and rec and st.mtime and rec.size == st.size and rec.mtime == st.mtime
|
|
402
|
+
and (rec.checked or 0) > st.mtime + 1 then
|
|
403
|
+
-- (size, mtime) proves "unchanged" only once the content was read
|
|
404
|
+
-- strictly after that mtime second: an edit within the same second
|
|
405
|
+
-- keeps both equal forever (the "racy git" problem).
|
|
406
|
+
stats.unchanged = stats.unchanged + 1
|
|
407
|
+
return true
|
|
408
|
+
end
|
|
409
|
+
local src = slurp(self:abs(rel))
|
|
410
|
+
if not src or #src > self.max_bytes then
|
|
411
|
+
stats.skipped = stats.skipped + 1
|
|
412
|
+
return false
|
|
413
|
+
end
|
|
414
|
+
local crc = checksum(src)
|
|
415
|
+
if rec and rec.size == #src and rec.crc == crc then
|
|
416
|
+
rec.mtime = st and st.mtime or rec.mtime
|
|
417
|
+
rec.checked = os.time()
|
|
418
|
+
stats.unchanged = stats.unchanged + 1
|
|
419
|
+
return true
|
|
420
|
+
end
|
|
421
|
+
local ok, r = pcall(ci.parse, rel, src)
|
|
422
|
+
control.check()
|
|
423
|
+
-- A parser exception must not permanently hide a GBK file.
|
|
424
|
+
-- Successful speculative parsing does not scan encoding here.
|
|
425
|
+
if not ok or not r then
|
|
426
|
+
if xutils.to_utf8 then
|
|
427
|
+
local decoded, encoding = xutils.to_utf8(src)
|
|
428
|
+
if decoded then
|
|
429
|
+
ok, r = pcall(ci.parse, rel, decoded)
|
|
430
|
+
if ok and r then r.encoding = encoding end
|
|
431
|
+
end
|
|
432
|
+
end
|
|
433
|
+
end
|
|
434
|
+
if not (ok and r) then
|
|
435
|
+
stats.failed = stats.failed + 1
|
|
436
|
+
return false
|
|
437
|
+
end
|
|
438
|
+
r.size, r.crc, r.mtime = #src, crc, st and st.mtime or nil
|
|
439
|
+
r.checked = os.time()
|
|
440
|
+
self.files[rel] = r
|
|
441
|
+
stats.parsed = stats.parsed + 1
|
|
442
|
+
return true
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
-- Start change notification so later refreshes visit only what changed.
|
|
446
|
+
-- Call it before the next refresh: that one stays a full pass, and anything
|
|
447
|
+
-- changing while it runs is reported to the pass after. Returns false when
|
|
448
|
+
-- the runtime or the filesystem cannot watch the root.
|
|
449
|
+
function Index:watch()
|
|
450
|
+
if not (xwatch and xwatch.open) then return false end
|
|
451
|
+
self.watching = true
|
|
452
|
+
return self.watcher ~= nil or self:open_watcher()
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
-- A root that cannot be watched (inotify limit, network share) stays on full
|
|
456
|
+
-- refreshes; retry now and then rather than paying a failed open per query.
|
|
457
|
+
M.WATCH_RETRY_SECONDS = 300
|
|
458
|
+
|
|
459
|
+
function Index:open_watcher()
|
|
460
|
+
local w, err = xwatch.open(self.root, { skip_hidden = true })
|
|
461
|
+
self.watcher, self.watch_error, self.watch_ready = w, err, false
|
|
462
|
+
if not w then self.watch_retry = os.time() + M.WATCH_RETRY_SECONDS end
|
|
463
|
+
return w ~= nil
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
function Index:close()
|
|
467
|
+
if self.watcher then self.watcher:close() end
|
|
468
|
+
self.watcher, self.watch_ready, self.watching = nil, false, false
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
-- rg reads these anywhere in the tree; a change can admit or hide files.
|
|
472
|
+
local IGNORE_FILES = { ['.gitignore'] = true, ['.ignore'] = true, ['.rgignore'] = true }
|
|
473
|
+
|
|
474
|
+
-- Changed paths that can matter, or nil when only a full refresh is safe.
|
|
475
|
+
-- Hidden paths are never listed (rg and the walk both skip them).
|
|
476
|
+
local function relevant(paths)
|
|
477
|
+
local changed = {}
|
|
478
|
+
for _, rel in ipairs(paths) do
|
|
479
|
+
if IGNORE_FILES[rel:match('[^/]*$')] then return nil end
|
|
480
|
+
if rel:sub(1, 1) ~= '.' and not rel:find('/.', 1, true) then changed[rel] = true end
|
|
481
|
+
end
|
|
482
|
+
return changed
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
-- Apply watcher-reported changes. Creations, removals and renames re-run the
|
|
486
|
+
-- listing (only rg decides what .gitignore admits) but still visit only the
|
|
487
|
+
-- files that differ from the index.
|
|
488
|
+
function Index:refresh_changed(changed, structural)
|
|
489
|
+
local stats = new_stats('changes')
|
|
490
|
+
if not structural then
|
|
491
|
+
-- A supported file the index lacks may have shrunk under the size
|
|
492
|
+
-- limit or been fixed after a failed parse: let the listing decide.
|
|
493
|
+
for rel in pairs(changed) do
|
|
494
|
+
if not self.files[rel] and ci.language(rel) then structural = true; break end
|
|
495
|
+
end
|
|
496
|
+
end
|
|
497
|
+
local listed
|
|
498
|
+
if structural then
|
|
499
|
+
stats.mode = 'relisted'
|
|
500
|
+
listed = {}
|
|
501
|
+
for _, rel in ipairs(self:list_files()) do
|
|
502
|
+
listed[rel] = true
|
|
503
|
+
if not self.files[rel] then changed[rel] = true end
|
|
504
|
+
end
|
|
505
|
+
for rel in pairs(self.files) do
|
|
506
|
+
if not listed[rel] then changed[rel] = true end
|
|
507
|
+
end
|
|
508
|
+
end
|
|
333
509
|
local has_stat = xutils and xutils.stat
|
|
334
|
-
for
|
|
510
|
+
for rel in pairs(changed) do
|
|
335
511
|
control.check()
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
seen[rel] = nil
|
|
342
|
-
elseif st and st.size and st.size > self.max_bytes then
|
|
343
|
-
stats.skipped = stats.skipped + 1
|
|
344
|
-
seen[rel] = nil
|
|
345
|
-
elseif st and rec and st.mtime and rec.size == st.size and rec.mtime == st.mtime
|
|
346
|
-
and (rec.checked or 0) > st.mtime + 1 then
|
|
347
|
-
-- (size, mtime) proves "unchanged" only once the content was read
|
|
348
|
-
-- strictly after that mtime second: an edit within the same second
|
|
349
|
-
-- keeps both equal forever (the "racy git" problem).
|
|
350
|
-
stats.unchanged = stats.unchanged + 1
|
|
351
|
-
else
|
|
352
|
-
local src = slurp(abs)
|
|
353
|
-
if not src or #src > self.max_bytes then
|
|
354
|
-
stats.skipped = stats.skipped + 1
|
|
355
|
-
seen[rel] = nil
|
|
356
|
-
else
|
|
357
|
-
local crc = checksum(src)
|
|
358
|
-
if rec and rec.size == #src and rec.crc == crc then
|
|
359
|
-
rec.mtime = st and st.mtime or rec.mtime
|
|
360
|
-
rec.checked = os.time()
|
|
361
|
-
stats.unchanged = stats.unchanged + 1
|
|
362
|
-
else
|
|
363
|
-
local ok, r = pcall(ci.parse, rel, src)
|
|
364
|
-
control.check()
|
|
365
|
-
-- A parser exception must not permanently hide a GBK file.
|
|
366
|
-
-- Successful speculative parsing does not scan encoding here.
|
|
367
|
-
if not ok or not r then
|
|
368
|
-
if xutils.to_utf8 then
|
|
369
|
-
local decoded, encoding = xutils.to_utf8(src)
|
|
370
|
-
if decoded then
|
|
371
|
-
ok, r = pcall(ci.parse, rel, decoded)
|
|
372
|
-
if ok and r then r.encoding = encoding end
|
|
373
|
-
end
|
|
374
|
-
end
|
|
375
|
-
end
|
|
376
|
-
if ok and r then
|
|
377
|
-
r.size, r.crc, r.mtime = #src, crc, st and st.mtime or nil
|
|
378
|
-
r.checked = os.time()
|
|
379
|
-
self.files[rel] = r
|
|
380
|
-
stats.parsed = stats.parsed + 1
|
|
381
|
-
else
|
|
382
|
-
stats.failed = stats.failed + 1
|
|
383
|
-
seen[rel] = nil
|
|
384
|
-
end
|
|
385
|
-
end
|
|
512
|
+
local known = self.files[rel] ~= nil
|
|
513
|
+
if listed and listed[rel] or (not listed and known) then
|
|
514
|
+
if not self:sync_file(rel, has_stat and xutils.stat(self:abs(rel)) or nil, stats) and known then
|
|
515
|
+
self.files[rel] = nil
|
|
516
|
+
stats.removed = stats.removed + 1
|
|
386
517
|
end
|
|
518
|
+
elseif known then
|
|
519
|
+
self.files[rel] = nil -- no longer listed
|
|
520
|
+
stats.removed = stats.removed + 1
|
|
387
521
|
end
|
|
388
522
|
end
|
|
523
|
+
local total = 0
|
|
524
|
+
for _ in pairs(self.files) do total = total + 1 end
|
|
525
|
+
stats.unchanged = total - stats.parsed
|
|
526
|
+
return stats
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
-- Bring the index in line with the tree. Returns counts of what changed.
|
|
530
|
+
-- With a watcher (see Index:watch) only reported paths are visited; an
|
|
531
|
+
-- overflow or an ignore-file change falls back to a full pass, and a failed
|
|
532
|
+
-- watcher is reopened before one.
|
|
533
|
+
function Index:refresh()
|
|
534
|
+
if self.watcher and self.watch_ready then
|
|
535
|
+
local paths, structural, overflow = self.watcher:read()
|
|
536
|
+
local changed = paths and not overflow and relevant(paths)
|
|
537
|
+
if changed then
|
|
538
|
+
-- Not ready again until this pass completes: the events are
|
|
539
|
+
-- drained, so a pass that fails midway leaves a full one next.
|
|
540
|
+
self.watch_ready = false
|
|
541
|
+
local stats = self:refresh_changed(changed, structural)
|
|
542
|
+
if stats.parsed > 0 or stats.removed > 0 then self.generation = self.generation + 1 end
|
|
543
|
+
self.watch_ready = true
|
|
544
|
+
return stats
|
|
545
|
+
end
|
|
546
|
+
if not paths then
|
|
547
|
+
-- A watcher that worked and then failed (root moved, queue
|
|
548
|
+
-- error) is reopened right away.
|
|
549
|
+
self.watcher:close()
|
|
550
|
+
self.watcher, self.watch_error, self.watch_retry = nil, structural, 0
|
|
551
|
+
end
|
|
552
|
+
end
|
|
553
|
+
if self.watching and not self.watcher and os.time() >= (self.watch_retry or 0) then self:open_watcher() end
|
|
554
|
+
if self.watcher then self.watch_ready = false end
|
|
555
|
+
local stats = self:refresh_all()
|
|
556
|
+
if self.watcher then self.watch_ready = true end
|
|
557
|
+
return stats
|
|
558
|
+
end
|
|
559
|
+
|
|
560
|
+
function Index:refresh_all()
|
|
561
|
+
local stats = new_stats('full')
|
|
562
|
+
local seen = {}
|
|
563
|
+
local list, meta = self:list_files()
|
|
564
|
+
if not meta and xutils and xutils.stat then meta = self:stat_listing(list) end
|
|
565
|
+
for _, rel in ipairs(list) do
|
|
566
|
+
control.check()
|
|
567
|
+
if self:sync_file(rel, meta and meta[rel], stats) then seen[rel] = true end
|
|
568
|
+
end
|
|
389
569
|
for rel in pairs(self.files) do
|
|
390
570
|
if not seen[rel] then
|
|
391
571
|
self.files[rel] = nil
|
|
@@ -4,6 +4,7 @@ package.path = scripts .. '/?.lua;' .. package.path
|
|
|
4
4
|
local service = require('codeoutline.service')
|
|
5
5
|
local paths = require('codeoutline.path')
|
|
6
6
|
local control = require('codeoutline.control')
|
|
7
|
+
service.configure({ watch = true })
|
|
7
8
|
local shared = assert(xshared.dict('codeoutline_control'))
|
|
8
9
|
|
|
9
10
|
local function execute(req)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
-- Native entry point: xnet main.lua STDIO=1 PROJECT=... or HTTP=1 PORT=19876.
|
|
2
|
+
-- DAEMON=1 (with HTTP=1) runs the shared background service; see daemon.lua.
|
|
2
3
|
local source = assert(xutils.realpath(debug.getinfo(1, 'S').source:sub(2))):gsub('\\', '/')
|
|
3
4
|
local scripts = assert(source:match('^(.*)/codeoutline/[^/]+$'))
|
|
4
5
|
local install = assert(scripts:match('^(.*)/scripts$'))
|
|
@@ -6,6 +7,7 @@ package.path = scripts .. '/?.lua;' .. package.path
|
|
|
6
7
|
local paths = require('codeoutline.path')
|
|
7
8
|
local mcp = require('codeoutline.mcp')
|
|
8
9
|
local http = require('codeoutline.http')
|
|
10
|
+
local daemon = require('codeoutline.daemon')
|
|
9
11
|
local options = { ALLOW_ROOT = {}, ALLOW_HOST = {}, ALLOW_ORIGIN = {} }
|
|
10
12
|
for _, argument in ipairs(arg or {}) do
|
|
11
13
|
local key, value = argument:match('^([A-Z_]+)=(.*)$')
|
|
@@ -14,6 +16,8 @@ for _, argument in ipairs(arg or {}) do
|
|
|
14
16
|
end
|
|
15
17
|
end
|
|
16
18
|
assert((options.STDIO == '1') ~= (options.HTTP == '1'), 'Specify exactly one of STDIO=1 or HTTP=1')
|
|
19
|
+
local managed = options.DAEMON == '1'
|
|
20
|
+
assert(not managed or options.HTTP == '1', 'DAEMON=1 requires HTTP=1')
|
|
17
21
|
local config = { roots = {}, project = options.PROJECT and assert(paths.canonical(options.PROJECT)),
|
|
18
22
|
workdir = options.STDIO == '1' and paths.canonical('.') or nil, host = options.HOST or '127.0.0.1', port = tonumber(options.PORT or '19876'),
|
|
19
23
|
token = os.getenv('CODEOUTLINE_TOKEN'), hosts = options.ALLOW_HOST, origins = options.ALLOW_ORIGIN }
|
|
@@ -45,6 +49,16 @@ local jobs, next_id, job_count, worker_ready = {}, 0, 0, false
|
|
|
45
49
|
local endpoint, stdio, input = nil, nil, ''
|
|
46
50
|
local updating = false
|
|
47
51
|
local startup = xtimer.now_ms()
|
|
52
|
+
-- After installing an update, exit once idle so the next start runs it: MCP
|
|
53
|
+
-- clients restart stdio servers on demand, and the background service starts
|
|
54
|
+
-- its replacement itself. A manually started HTTP service keeps running unless asked.
|
|
55
|
+
local exit_on_update = os.getenv('CODEOUTLINE_EXIT_ON_UPDATE')
|
|
56
|
+
if managed then exit_on_update = true
|
|
57
|
+
elseif exit_on_update == nil or exit_on_update == '' then exit_on_update = options.STDIO == '1'
|
|
58
|
+
else exit_on_update = exit_on_update == '1' end
|
|
59
|
+
local installed_update
|
|
60
|
+
-- Background service: listening starts once the port proved free.
|
|
61
|
+
local codec, next_stop_check = nil, 0
|
|
48
62
|
|
|
49
63
|
local function submit(req, done)
|
|
50
64
|
if job_count >= 64 then return nil, 'Request queue is full; retry later' end
|
|
@@ -62,14 +76,36 @@ local function cancel(id) if id and jobs[id] then shared:set('cancel:' .. id, tr
|
|
|
62
76
|
local function send(response)
|
|
63
77
|
if response then io.stdout:write(xutils.json_pack(response), '\n'); io.stdout:flush() end
|
|
64
78
|
end
|
|
79
|
+
local function listen()
|
|
80
|
+
endpoint = http.start(config, submit, cancel, codec)
|
|
81
|
+
-- A status line, not an error: xnet routes io.stderr through the
|
|
82
|
+
-- error log. HTTP mode leaves stdout free (stdio mode owns it).
|
|
83
|
+
io.stdout:write(string.format('[codeoutline] listening http://%s:%d/mcp\n', config.host, config.port))
|
|
84
|
+
io.stdout:flush()
|
|
85
|
+
end
|
|
86
|
+
local stopping, indexing = false, false
|
|
87
|
+
local function stop(code) stopping = true; xthread.stop(code) end
|
|
65
88
|
|
|
66
89
|
return {
|
|
67
90
|
__tick_ms = 5,
|
|
68
91
|
__init = function()
|
|
69
92
|
assert(xnet.init())
|
|
93
|
+
if managed then
|
|
94
|
+
xtimer.init(16)
|
|
95
|
+
-- One background service runs the newest installed version.
|
|
96
|
+
if not daemon.is_current(install) then
|
|
97
|
+
notice('starting the newest installed version')
|
|
98
|
+
assert(daemon.spawn(daemon.current_root(install), config.port), 'cannot start the newest version')
|
|
99
|
+
stop(0); return
|
|
100
|
+
end
|
|
101
|
+
os.remove(daemon.stop_file(config.port))
|
|
102
|
+
end
|
|
70
103
|
assert(xthread.create_thread(2, 'INDEX', scripts .. '/codeoutline/index_worker.lua'))
|
|
71
|
-
|
|
72
|
-
|
|
104
|
+
indexing = true
|
|
105
|
+
-- Launchers enable this for serve; the background service updates unless
|
|
106
|
+
-- disabled. Updating never delays or blocks requests.
|
|
107
|
+
local auto_update = os.getenv('CODEOUTLINE_AUTO_UPDATE')
|
|
108
|
+
if auto_update == '1' or (managed and auto_update ~= '0') then
|
|
73
109
|
updating = xthread.create_thread(3, 'UPDATE', scripts .. '/codeoutline/update_worker.lua') and true
|
|
74
110
|
end
|
|
75
111
|
if options.STDIO == '1' then
|
|
@@ -83,12 +119,17 @@ return {
|
|
|
83
119
|
local probe = io.open(codec_path, 'rb')
|
|
84
120
|
if probe then probe:close()
|
|
85
121
|
else codec_path = install .. '/xnet2lua/scripts/core/share/xhttp_codec.lua' end
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
122
|
+
codec = dofile(codec_path)
|
|
123
|
+
if not managed then listen()
|
|
124
|
+
else
|
|
125
|
+
-- Windows lets a second listener bind the same port, so the
|
|
126
|
+
-- background service checks for a running one first.
|
|
127
|
+
daemon.probe(config.port, function(listening)
|
|
128
|
+
if stopping then return end
|
|
129
|
+
if listening then notice('a service already listens on port ' .. config.port); stop(0); return end
|
|
130
|
+
listen()
|
|
131
|
+
end)
|
|
132
|
+
end
|
|
92
133
|
end
|
|
93
134
|
end,
|
|
94
135
|
__thread_handle = function(_, op, id, ok, result)
|
|
@@ -104,16 +145,35 @@ return {
|
|
|
104
145
|
elseif op == 'update_result' then
|
|
105
146
|
-- Arguments: error, installed version (nil when already current).
|
|
106
147
|
if id then notice('update unavailable: ' .. tostring(id))
|
|
107
|
-
elseif ok then
|
|
148
|
+
elseif ok then
|
|
149
|
+
installed_update = ok
|
|
150
|
+
notice('update ' .. tostring(ok) .. ' installed; ' .. (exit_on_update and 'exiting when idle' or 'restart to use it'))
|
|
151
|
+
end
|
|
108
152
|
end
|
|
109
153
|
end,
|
|
110
154
|
__update = function()
|
|
155
|
+
if stopping then return end
|
|
111
156
|
if not worker_ready and xtimer.now_ms() - startup > 10000 then error('Index worker failed to start') end
|
|
157
|
+
if installed_update and exit_on_update and job_count == 0
|
|
158
|
+
and not (stdio and next(stdio.pending)) and not (endpoint and endpoint.busy()) then
|
|
159
|
+
notice('exiting to start update ' .. tostring(installed_update))
|
|
160
|
+
if managed then
|
|
161
|
+
-- Free the port, then start the update as this service's replacement.
|
|
162
|
+
if endpoint then endpoint.close(); endpoint = nil end
|
|
163
|
+
if not daemon.spawn(daemon.current_root(install), config.port) then notice('cannot start update ' .. tostring(installed_update)) end
|
|
164
|
+
end
|
|
165
|
+
stop(0); return
|
|
166
|
+
end
|
|
167
|
+
if managed and xtimer.now_ms() >= next_stop_check then
|
|
168
|
+
next_stop_check = xtimer.now_ms() + 1000
|
|
169
|
+
if xutils.stat(daemon.stop_file(config.port)).exists then notice('stop requested'); stop(0); return end
|
|
170
|
+
end
|
|
112
171
|
if stdio then
|
|
113
172
|
local data, err = xutils.read_stdin(65536)
|
|
114
173
|
if not data then
|
|
115
174
|
local incomplete = err == 'eof' and input:find('%S') ~= nil
|
|
116
175
|
if incomplete then send(mcp.error(nil, -32700, 'EOF before message newline')) end
|
|
176
|
+
if err == 'stdio requires redirected stdin' then err = 'run by an MCP client; see codeoutline --help' end
|
|
117
177
|
if err ~= 'eof' then io.stderr:write(tostring(err) .. '\n') end
|
|
118
178
|
stdio:close(); shared:set('shutdown', true); xthread.stop(err == 'eof' and not incomplete and 0 or 1); return
|
|
119
179
|
end
|
|
@@ -129,13 +189,13 @@ return {
|
|
|
129
189
|
end
|
|
130
190
|
if #input > 1048576 then send(mcp.error(nil, -32600, 'Message too large')); xthread.stop(1); return end
|
|
131
191
|
stdio:tick()
|
|
132
|
-
|
|
192
|
+
elseif endpoint then endpoint.tick() end
|
|
133
193
|
end,
|
|
134
194
|
__uninit = function()
|
|
135
195
|
shared:set('shutdown', true)
|
|
136
196
|
if stdio then stdio:close() end
|
|
137
197
|
if endpoint then endpoint.close() end
|
|
138
|
-
xthread.shutdown_thread(2)
|
|
198
|
+
if indexing then xthread.shutdown_thread(2) end
|
|
139
199
|
if updating then xthread.shutdown_thread(3) end
|
|
140
200
|
xnet.uninit()
|
|
141
201
|
end,
|
|
@@ -7,8 +7,21 @@ local M = {}
|
|
|
7
7
|
local projects = {}
|
|
8
8
|
local sequence = 0
|
|
9
9
|
local limits = { max_projects = 8, idle_seconds = 900 }
|
|
10
|
+
-- Long-lived hosts watch resident projects so a query refreshes only what
|
|
11
|
+
-- changed; one-shot commands would pay for a watcher they never read.
|
|
12
|
+
local watch = false
|
|
13
|
+
|
|
14
|
+
local function drop(key)
|
|
15
|
+
local p = projects[key]
|
|
16
|
+
projects[key] = nil
|
|
17
|
+
if p then p.idx:close() end
|
|
18
|
+
end
|
|
10
19
|
|
|
11
20
|
function M.configure(opts)
|
|
21
|
+
if opts.watch ~= nil then
|
|
22
|
+
assert(type(opts.watch) == 'boolean', 'watch must be a boolean')
|
|
23
|
+
watch = opts.watch
|
|
24
|
+
end
|
|
12
25
|
for _, name in ipairs({ 'max_projects', 'idle_seconds' }) do
|
|
13
26
|
local v = opts[name]
|
|
14
27
|
if v ~= nil then
|
|
@@ -22,15 +35,17 @@ end
|
|
|
22
35
|
function M.sweep(now)
|
|
23
36
|
now = now or os.time()
|
|
24
37
|
local count = 0
|
|
38
|
+
local idle = {}
|
|
25
39
|
for key, p in pairs(projects) do
|
|
26
|
-
if now - p.used >= limits.idle_seconds then
|
|
40
|
+
if now - p.used >= limits.idle_seconds then idle[#idle + 1] = key else count = count + 1 end
|
|
27
41
|
end
|
|
42
|
+
for _, key in ipairs(idle) do drop(key) end
|
|
28
43
|
while count > limits.max_projects do
|
|
29
44
|
local oldest
|
|
30
45
|
for key, p in pairs(projects) do
|
|
31
46
|
if not oldest or p.order < projects[oldest].order then oldest = key end
|
|
32
47
|
end
|
|
33
|
-
|
|
48
|
+
drop(oldest)
|
|
34
49
|
count = count - 1
|
|
35
50
|
end
|
|
36
51
|
return count
|
|
@@ -57,6 +72,7 @@ function M.get(root, opts)
|
|
|
57
72
|
p = { idx = index.open(canonical, opts), opts = {
|
|
58
73
|
cache_path = opts.cache_path, max_file_bytes = opts.max_file_bytes, lister = opts.lister,
|
|
59
74
|
} }
|
|
75
|
+
if watch then p.idx:watch() end
|
|
60
76
|
end
|
|
61
77
|
local t0 = os.clock()
|
|
62
78
|
local stats = p.idx:refresh()
|
|
@@ -116,7 +132,7 @@ end
|
|
|
116
132
|
function M.forget(root)
|
|
117
133
|
local key = key_for(root)
|
|
118
134
|
local existed = projects[key] ~= nil
|
|
119
|
-
|
|
135
|
+
drop(key)
|
|
120
136
|
return existed
|
|
121
137
|
end
|
|
122
138
|
|
|
@@ -128,7 +144,12 @@ function M.rebuild(root, opts)
|
|
|
128
144
|
config.rebuild = true
|
|
129
145
|
projects[key] = nil
|
|
130
146
|
local ok, result = pcall(M.status, root, config)
|
|
131
|
-
if not ok then
|
|
147
|
+
if not ok then
|
|
148
|
+
drop(key)
|
|
149
|
+
projects[key] = previous
|
|
150
|
+
error(result, 2)
|
|
151
|
+
end
|
|
152
|
+
if previous then previous.idx:close() end
|
|
132
153
|
return result
|
|
133
154
|
end
|
|
134
155
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
return
|
|
1
|
+
return 8
|