@codua/codeoutline-win32-x64 0.3.8 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +49 -0
  2. package/README.zh-CN.md +37 -0
  3. package/bin/xnet.exe +0 -0
  4. package/build-info.json +1 -1
  5. package/editors/codeoutline.vsix +0 -0
  6. package/editors/zed/extension.wasm +0 -0
  7. package/keys/update-ca.crt +18 -18
  8. package/package.json +1 -1
  9. package/scripts/codeoutline/analysis/graph.lua +7 -16
  10. package/scripts/codeoutline/analysis/indirect.lua +18 -12
  11. package/scripts/codeoutline/analysis/lua/lua_bindings.lua +28 -0
  12. package/scripts/codeoutline/bm25/automatic.lua +100 -0
  13. package/scripts/codeoutline/bm25/collect.lua +148 -0
  14. package/scripts/codeoutline/bm25/index.lua +548 -0
  15. package/scripts/codeoutline/bm25/snapshot.lua +45 -0
  16. package/scripts/codeoutline/command.lua +3 -3
  17. package/scripts/codeoutline/indexing/parse_pool.lua +12 -6
  18. package/scripts/codeoutline/main.lua +5 -1
  19. package/scripts/codeoutline/protocol/lsp.lua +3 -1
  20. package/scripts/codeoutline/protocol/mcp.lua +5 -2
  21. package/scripts/codeoutline/query/coverage.lua +2 -0
  22. package/scripts/codeoutline/query/definition_candidates.lua +36 -12
  23. package/scripts/codeoutline/query/explore.lua +110 -18
  24. package/scripts/codeoutline/query/navigation.lua +213 -26
  25. package/scripts/codeoutline/query/query_filters.lua +5 -1
  26. package/scripts/codeoutline/runtime/bm25_worker.lua +86 -0
  27. package/scripts/codeoutline/runtime/index_worker.lua +18 -5
  28. package/scripts/codeoutline/runtime/parse_worker.lua +10 -3
  29. package/scripts/codeoutline/service.lua +6 -0
  30. package/scripts/codeoutline/shared/rules.lua +9 -1
  31. package/scripts/codeoutline/update_sequence.lua +1 -1
  32. package/scripts/codeoutline/version.lua +1 -1
@@ -224,7 +224,9 @@ local function resolve_site(V, f, from, name, kind, recv, evidence, offset, inde
224
224
  end
225
225
  -- The resolver never binds a ref to its enclosing node, which hides recursion.
226
226
  local receiver_bound = recv and evidence and evidence ~= '' or reason and reason:match('^receiver:')
227
- if #targets == 0 and not receiver_bound and reason ~= 'unknown or unsupported macro visibility' and src and V.nodes[src].name == name and (kind == 'call' or recursive(V.nodes[src], recv)) then
227
+ if #targets == 0 and not receiver_bound and reason ~= 'unknown or unsupported macro visibility'
228
+ and reason ~= 'receiver binding unresolved' and src and V.nodes[src].name == name
229
+ and (kind == 'call' or recursive(V.nodes[src], recv)) then
228
230
  targets = { src }
229
231
  end
230
232
  -- Definitions too far apart to choose between: a visible prototype still
@@ -805,27 +807,30 @@ end
805
807
  -- Decode literals without running source code. Lua long strings and quoted escapes.
806
808
  local string_value = require('codeoutline.analysis.lua.lua_literal').decode
807
809
 
808
- local function call_name(T, i)
810
+ local function call_name(T, i, qualified_separators)
809
811
  if T.k[i] ~= 'id' or T:text(i + 1) ~= '(' then return nil end
812
+ local separators = qualified_separators or LUA_MEMBER
810
813
  local name, first = T:text(i), i
811
- while T.k[first - 2] == 'id' and (T:text(first - 1) == '.' or T:text(first - 1) == ':') do
814
+ while T.k[first - 2] == 'id' and separators[T:text(first - 1)] do
812
815
  name = T:text(first - 2) .. T:text(first - 1) .. name
813
816
  first = first - 2
814
817
  end
815
- if T:text(first - 1) == 'function' then return nil end
818
+ local previous = T:text(first - 1)
819
+ if previous == 'function' or qualified_separators and qualified_separators[previous] then return nil end
816
820
  return name
817
821
  end
818
822
 
819
823
  -- Find a single literal argument, skipping nested brackets and anonymous functions.
820
- local function literal_argument(T, call, argument)
824
+ local function literal_argument(T, call, argument, selected)
821
825
  local open, number, first = call + 1, 1, call + 2
822
826
  local close = T.m[open]
823
827
  if not close then return nil end
824
828
  local i = first
825
829
  while i <= close do
830
+ if selected and i > selected + 1 then return nil end
826
831
  local token = T:text(i)
827
832
  if i == close or token == ',' then
828
- if number == argument then
833
+ if number == argument or first == selected then
829
834
  if i == first + 1 and T.k[first] == 'str' then
830
835
  return string_value(T:text(first)), T.s[first] - 1, T.e[first]
831
836
  end
@@ -907,6 +912,7 @@ end
907
912
  local function shared_definition(V, doc, offset, rules)
908
913
  local record = doc:parse()
909
914
  local language = record.language
915
+ local indirect = require('codeoutline.analysis.indirect')
910
916
  local names, applicable, literals = {}, {}, {}
911
917
  local function literal_position(name, argument)
912
918
  literals[name] = literals[name] or {}
@@ -914,7 +920,7 @@ local function shared_definition(V, doc, offset, rules)
914
920
  end
915
921
  for _, rule in ipairs(rules) do
916
922
  if rule.shared and (not rule.language or rule.language == language)
917
- and require('codeoutline.analysis.indirect').candidate(record, rule.call) then
923
+ and indirect.candidate(record, rule.call) then
918
924
  applicable[#applicable + 1] = rule
919
925
  names[rule.call] = true
920
926
  if rule.register then names[rule.register] = true end
@@ -974,7 +980,7 @@ local function shared_definition(V, doc, offset, rules)
974
980
  end
975
981
  end
976
982
  if #matched == 0 then return nil end
977
- local result, seen, events = {}, {}, {}
983
+ local result, seen, events, event_keys = {}, {}, {}, {}
978
984
  local deadline = os.clock() + M.SCAN_SECONDS
979
985
  local function add(loc)
980
986
  local key = loc.uri .. ':' .. loc.range.start.line .. ':' .. loc.range.start.character
@@ -985,7 +991,9 @@ local function shared_definition(V, doc, offset, rules)
985
991
  local resolver
986
992
  for _, match in ipairs(matched) do
987
993
  local rule = match.rule
988
- if rule.kind == 'event' then events[#events + 1] = match
994
+ if rule.kind == 'event' then
995
+ events[#events + 1] = match
996
+ event_keys[match.arg.literal] = true
989
997
  elseif rule.kind == 'target' then
990
998
  local name = match.arg.literal
991
999
  local services = language == 'lua' and service_files(V, match.site, rule)
@@ -1004,7 +1012,7 @@ local function shared_definition(V, doc, offset, rules)
1004
1012
  end
1005
1013
  end
1006
1014
  else
1007
- resolver = resolver or require('codeoutline.analysis.indirect').query(V, { rule }).resolve_targets
1015
+ resolver = resolver or indirect.query(V, { rule }).resolve_targets
1008
1016
  for _, id in ipairs(resolver(f, match.site, match.arg, rule.kind == 'target')) do
1009
1017
  control.check()
1010
1018
  if os.clock() > deadline then return result, true end
@@ -1023,12 +1031,12 @@ local function shared_definition(V, doc, offset, rules)
1023
1031
  local candidate = false
1024
1032
  if V.files[file].language == language then
1025
1033
  for _, match in ipairs(events) do
1026
- if require('codeoutline.analysis.indirect').candidate(V.files[file], match.rule.register) then candidate = true; break end
1034
+ if indirect.candidate(V.files[file], match.rule.register) then candidate = true; break end
1027
1035
  end
1028
1036
  end
1029
1037
  if candidate then
1030
1038
  local source = source_doc(V, file)
1031
- if source then
1039
+ if source and (language ~= 'lua' or indirect.has_lua_literal(source.source, event_keys)) then
1032
1040
  for _, site in ipairs(sites(source)) do
1033
1041
  for _, match in ipairs(events) do
1034
1042
  local rule = match.rule
@@ -1099,16 +1107,31 @@ local function legacy_definition(V, doc, offset, rules)
1099
1107
  end
1100
1108
  end
1101
1109
  if #calls == 0 and #keys == 0 then return result, false end
1102
- for _, f in ipairs(V.file_order) do files[#files + 1] = f end
1103
- for _, f in ipairs(V.new_files) do files[#files + 1] = f end
1110
+ local scoped = #calls == 0
1111
+ local seen_files = {}
1112
+ for _, match in ipairs(keys) do
1113
+ local path = match[1].targetPath
1114
+ if not path then scoped = false; break end
1115
+ local f = V.file_index[path]
1116
+ if f and not seen_files[f] then files[#files + 1] = f; seen_files[f] = true end
1117
+ end
1118
+ if not scoped then
1119
+ files = {}
1120
+ for _, f in ipairs(V.file_order) do files[#files + 1] = f end
1121
+ for _, f in ipairs(V.new_files) do files[#files + 1] = f end
1122
+ end
1104
1123
  table.sort(files, function(a, b) return V.files[a].path < V.files[b].path end)
1105
1124
  for _, f in ipairs(files) do
1106
1125
  control.check()
1107
1126
  if os.clock() > deadline then return result, true end
1108
1127
  if V.files[f].language == 'lua' then
1128
+ local file_keys = {}
1129
+ for _, match in ipairs(keys) do
1130
+ if not match[1].targetPath or match[1].targetPath == V.files[f].path then file_keys[#file_keys + 1] = match end
1131
+ end
1109
1132
  local target = source_doc(V, f)
1110
1133
  if target then
1111
- local candidate = #keys > 0
1134
+ local candidate = #file_keys > 0
1112
1135
  for _, match in ipairs(calls) do
1113
1136
  local tail = match[1].target:match('([%w_]+)$')
1114
1137
  if target.source:find(tail, 1, true) then candidate = true; break end
@@ -1120,7 +1143,7 @@ local function legacy_definition(V, doc, offset, rules)
1120
1143
  control.check()
1121
1144
  if os.clock() > deadline then return result, true end
1122
1145
  end
1123
- local name = call_name(T, i)
1146
+ local name = #calls > 0 and call_name(T, i)
1124
1147
  for _, match in ipairs(calls) do
1125
1148
  if call_matches(name, match[1].target) then
1126
1149
  local value, first, last = literal_argument(T, i, match[1].targetArgument)
@@ -1131,10 +1154,10 @@ local function legacy_definition(V, doc, offset, rules)
1131
1154
  end
1132
1155
  end
1133
1156
  end
1134
- if #keys > 0 then
1157
+ if #file_keys > 0 then
1135
1158
  local value, owner = key_literal(T, i)
1136
1159
  if value then
1137
- for _, match in ipairs(keys) do
1160
+ for _, match in ipairs(file_keys) do
1138
1161
  if value == match[2] and qualifier_matches(match[1], owner)
1139
1162
  and add({ uri = target.uri, range = target:range(T.s[i] - 1, T.e[i]) }) then
1140
1163
  return result, true
@@ -1196,9 +1219,9 @@ local function lua_self_member(V, f, c)
1196
1219
  and (c.kind == 'member_call' or c.kind == 'member_symbol')
1197
1220
  end
1198
1221
 
1199
- local function definition_tags(V, f, c)
1222
+ local function definition_tags(V, f, c, candidates)
1200
1223
  local self_fallback = lua_self_member(V, f, c) and c.evidence and c.evidence:match('^lua%-self:%d+$')
1201
- local ids, info = require('codeoutline.query.definition_candidates').find(V, f, c, V.js_context, self_fallback, true)
1224
+ local ids, info = require('codeoutline.query.definition_candidates').find(V, f, c, V.js_context, self_fallback, true, candidates)
1202
1225
  local result, seen, bytes, started, count = {}, {}, 0, os.clock(), 0
1203
1226
  for _, id in ipairs(ids) do
1204
1227
  local g = V.node_file[id]
@@ -1273,9 +1296,7 @@ function handlers.lsp_implementation(req)
1273
1296
  return result, relationships.incomplete
1274
1297
  end
1275
1298
 
1276
- function handlers.lsp_definition(req)
1277
- local V, doc, c, targets, f, offset = at(req)
1278
- if not V then return false end
1299
+ local function resolved_definition(V, doc, c, targets, f, offset)
1279
1300
  if not c and (V.files[f].language == 'javascript' or V.files[f].language == 'typescript') then
1280
1301
  local events = require('codeoutline.analysis.shared_events').context(V, V.js_context)
1281
1302
  local found, recognized = events.definition(f, offset, doc:position(offset).line + 1)
@@ -1331,7 +1352,7 @@ function handlers.lsp_definition(req)
1331
1352
  local result = {}
1332
1353
  for _, id in ipairs(ids) do result[#result + 1] = (location(V, id)) end
1333
1354
  return result, truncated or V.js_context.include_incomplete or V.js_context.workspace_incomplete
1334
- or V.js_context.go_incomplete or V.js_context.go_methods_incomplete or V.js_context.cpp_receivers_incomplete or V.js_context.rust_incomplete or V.js_context.receiver_incomplete
1355
+ or V.js_context.go_incomplete or V.js_context.go_methods_incomplete or V.js_context.cpp_receivers_incomplete or V.js_context.rust_incomplete or V.js_context.receiver_incomplete, true
1335
1356
  end
1336
1357
  if c.decl and is_decl(V.nodes[targets[1]]) then
1337
1358
  -- From a declaration, go to its definitions.
@@ -1346,6 +1367,172 @@ function handlers.lsp_definition(req)
1346
1367
  return result, V.js_context.include_incomplete or V.js_context.workspace_incomplete or V.js_context.go_incomplete or V.js_context.go_methods_incomplete or V.js_context.cpp_receivers_incomplete or V.js_context.rust_incomplete or V.js_context.receiver_incomplete
1347
1368
  end
1348
1369
 
1370
+ local function token_after(T, offset)
1371
+ local low, high = 1, T.n + 1
1372
+ while low < high do
1373
+ local middle = math.floor((low + high) / 2)
1374
+ if T.e[middle] <= offset then low = middle + 1 else high = middle end
1375
+ end
1376
+ return low
1377
+ end
1378
+
1379
+ local function lexical_literal(T, token, language)
1380
+ if T.k[token] ~= 'str' or T.e[token] - T.s[token] > 1024 then return nil end
1381
+ local value = T:text(token)
1382
+ if language == 'lua' then return string_value(value) end
1383
+ local quote = value:sub(1, 1)
1384
+ if #value >= 2 and (quote == '"' or quote == "'" or quote == '`') and value:sub(-1) == quote then
1385
+ return value:sub(2, -2)
1386
+ end
1387
+ return value
1388
+ end
1389
+
1390
+ -- A direct string argument belongs to its innermost call. Skip balanced
1391
+ -- expressions in earlier arguments when looking for the opening parenthesis.
1392
+ local function literal_call(T, token, language)
1393
+ local i, after = token - 1, token + 1
1394
+ while T.k[i] == 'nl' do i = i - 1 end
1395
+ while T.k[after] == 'nl' do after = after + 1 end
1396
+ local first, last = T:text(i), T:text(after)
1397
+ if (first ~= '(' and first ~= ',') or (last ~= ')' and last ~= ',') then return nil end
1398
+ local scanned = 0
1399
+ while i > 0 do
1400
+ scanned = scanned + 1
1401
+ if scanned % 128 == 0 then control.check() end
1402
+ local word, pair = T:text(i), T.m[i]
1403
+ if pair and pair < i then
1404
+ i = pair
1405
+ elseif word == '(' then
1406
+ if not pair or pair <= token then return nil end
1407
+ local lua = language == 'lua'
1408
+ local name = call_name(T, i - 1, lua and LUA_MEMBER or MEMBER)
1409
+ if name and (not lua or literal_argument(T, i - 1, nil, token)) then return name end
1410
+ return nil
1411
+ elseif word == '[' or word == '{' then
1412
+ return nil
1413
+ end
1414
+ i = i - 1
1415
+ end
1416
+ end
1417
+
1418
+ local function lexical_ids(V, hits)
1419
+ local ids, seen = {}, {}
1420
+ for _, hit in ipairs(hits) do
1421
+ control.check()
1422
+ local file = V.file_index[hit.path]
1423
+ for i, id in ipairs(V.id_of[file] or {}) do
1424
+ if i % 128 == 0 then control.check() end
1425
+ local node = V.nodes[id]
1426
+ if not seen[id] and node.kind == hit.kind and node.qualified == hit.symbol
1427
+ and (V.drafts[file] or node.line == hit.start_line) then
1428
+ ids[#ids + 1], seen[id] = id, true
1429
+ end
1430
+ end
1431
+ end
1432
+ return ids
1433
+ end
1434
+
1435
+ local function lexical_locations(V, ids, query, origin, offset, origin_call)
1436
+ local comparisons, other, seen = {}, {}, {}
1437
+ local literals = { [query] = true }
1438
+ local has_lua_literal = require('codeoutline.analysis.indirect').has_lua_literal
1439
+ local truncated = false
1440
+ for _, id in ipairs(ids) do
1441
+ control.check()
1442
+ local file = V.node_file[id]
1443
+ local record, draft = V.files[file], V.drafts[file]
1444
+ if (draft and #draft.source or record.size or 0) > M.MAX_FILE_BYTES then
1445
+ truncated = true
1446
+ else
1447
+ local offsets = seen[file]
1448
+ if offsets == nil then
1449
+ local doc = source_doc(V, file)
1450
+ local relevant = doc and (record.language == 'lua' and has_lua_literal(doc.source, literals)
1451
+ or doc.source:find(query, 1, true))
1452
+ offsets = relevant and {} or false
1453
+ seen[file] = offsets
1454
+ end
1455
+ if offsets then
1456
+ -- Populate the shared position cache before locating the node.
1457
+ local _, T = M.full(V, file)
1458
+ local doc, node = locate(V, id)
1459
+ if T and node and node.start_byte and node.end_byte then
1460
+ local start, stop = token_after(T, node.start_byte), token_after(T, node.end_byte) - 1
1461
+ for i = start, stop do
1462
+ if i % 128 == 0 then control.check() end
1463
+ local first, last = T.s[i] - 1, T.e[i]
1464
+ if not offsets[first] and (file ~= origin or offset < first or offset >= last)
1465
+ and lexical_literal(T, i, record.language) == query
1466
+ and (not origin_call or literal_call(T, i, record.language) ~= origin_call) then
1467
+ offsets[first] = true
1468
+ local matches = (T:text(i - 1) == '==' or T:text(i + 1) == '==') and comparisons or other
1469
+ if #matches < 20 then
1470
+ matches[#matches + 1] = {uri = doc.uri, range = doc:range(first, last)}
1471
+ else truncated = true end
1472
+ end
1473
+ end
1474
+ end
1475
+ end
1476
+ end
1477
+ end
1478
+ for _, result in ipairs(other) do
1479
+ if #comparisons >= 20 then truncated = true; break end
1480
+ comparisons[#comparisons + 1] = result
1481
+ end
1482
+ return comparisons, truncated
1483
+ end
1484
+
1485
+ local function lexical_definition(req, V, c, f, offset, deadline, allow_tags)
1486
+ local query = c and c.name
1487
+ local origin_call
1488
+ if not query then
1489
+ local _, T = M.full(V, f)
1490
+ if T then
1491
+ local token = token_after(T, offset)
1492
+ if token <= T.n and T.s[token] - 1 <= offset then
1493
+ local language = V.files[f].language
1494
+ query = lexical_literal(T, token, language)
1495
+ if query then origin_call = literal_call(T, token, language) end
1496
+ end
1497
+ end
1498
+ end
1499
+ if not query or #query > 256 or not query:find('[%w_]') then return nil end
1500
+ local hits, incomplete = req.bm25_definition(V.idx, query, deadline, c and c.recv)
1501
+ if not hits or #hits == 0 then return {}, incomplete end
1502
+ if c and not allow_tags then return {}, incomplete end
1503
+ local ids = lexical_ids(V, hits)
1504
+ if not c then
1505
+ local locations, truncated = lexical_locations(V, ids, query, f, offset, origin_call)
1506
+ return locations, incomplete or truncated
1507
+ end
1508
+ local found, truncated = definition_tags(V, f, c, ids)
1509
+ local locations = {}
1510
+ for _, id in ipairs(found) do locations[#locations + 1] = (location(V, id)) end
1511
+ return locations, incomplete or truncated
1512
+ end
1513
+
1514
+ function handlers.lsp_definition(req)
1515
+ local started = xtimer.now_ms()
1516
+ local V, doc, c, targets, f, offset = at(req)
1517
+ if not V then return false end
1518
+ local primary, incomplete, allow_tags = resolved_definition(V, doc, c, targets, f, offset)
1519
+ if #primary > 0 or not req.bm25_definition then return primary, incomplete end
1520
+ local deadline = math.min(started + 900, xtimer.now_ms() + 250)
1521
+ if xtimer.now_ms() >= deadline then return primary, incomplete end
1522
+ local previous, expired = control.callback, {}
1523
+ control.callback = function()
1524
+ if previous then previous() end
1525
+ if xtimer.now_ms() >= deadline then error(expired, 0) end
1526
+ end
1527
+ local ok, result, limited = pcall(lexical_definition, req, V, c, f, offset, deadline, allow_tags)
1528
+ control.callback = previous
1529
+ if not ok then
1530
+ if result == expired or tostring(result):find('source changed during query', 1, true) then return primary, true end
1531
+ error(result, 0)
1532
+ end
1533
+ return result or primary, incomplete or not not limited, limited == 'bm25_building'
1534
+ end
1535
+
1349
1536
  -- Display tags may be truncated before a heavily annotated SDK declaration's
1350
1537
  -- argument list. Recover only the selected function's bounded source span.
1351
1538
  local function external_signature(entry)
@@ -1735,9 +1922,9 @@ function M.handles(method) return handlers[method] ~= nil end
1735
1922
 
1736
1923
  -- Returns { result = value (false for null), incomplete = bool } or { error = message }.
1737
1924
  function M.execute(req)
1738
- local result, incomplete = handlers[req.method](req)
1925
+ local result, incomplete, bm25_building = handlers[req.method](req)
1739
1926
  if type(result) == 'table' and result.error then return result end
1740
- return { result = result, incomplete = incomplete or false }
1927
+ return { result = result, incomplete = incomplete or false, bm25_building = bm25_building or nil }
1741
1928
  end
1742
1929
 
1743
1930
  return M
@@ -54,8 +54,12 @@ function M.file(filters, rec)
54
54
  return matches(filters.languages, rec.language, true) and matches(filters.paths, (rec.path:gsub('\\', '/')))
55
55
  end
56
56
 
57
+ function M.node_fields(filters, rec, kind, qualified)
58
+ return M.file(filters, rec) and matches(filters.kinds, kind, true) and matches(filters.names, qualified)
59
+ end
60
+
57
61
  function M.node(filters, rec, node)
58
- return M.file(filters, rec) and matches(filters.kinds, node.kind, true) and matches(filters.names, node.qualified)
62
+ return M.node_fields(filters, rec, node.kind, node.qualified)
59
63
  end
60
64
 
61
65
  function M.id(filters, G, id)
@@ -0,0 +1,86 @@
1
+ local source = assert(xutils.realpath(debug.getinfo(1, 'S').source:sub(2))):gsub('\\', '/')
2
+ local scripts = assert(source:match('^(.*)/codeoutline/runtime/[^/]+$'))
3
+ package.path = scripts .. '/?.lua;' .. package.path
4
+ local indexing = require('codeoutline.indexing.index')
5
+ local bm25 = require('codeoutline.bm25.index')
6
+ local snapshot = require('codeoutline.bm25.snapshot')
7
+ local control = require('codeoutline.shared.control')
8
+ local pool = require('codeoutline.indexing.parse_pool')
9
+ local shared = assert(xshared.dict('codeoutline_control'))
10
+ local job
11
+
12
+ local function build(item)
13
+ control.check()
14
+ local index = indexing.open(item.root, {cache_path = item.cache})
15
+ local ok, err = pcall(function()
16
+ assert(index.cache_loaded and snapshot.stamp(index) == item.stamp, 'base snapshot changed before BM25 build')
17
+ if item.repair or not snapshot.current(item.file, item.stamp) then bm25.build(index, item.file) end
18
+ end)
19
+ index:close()
20
+ if not ok then error(err, 0) end
21
+ end
22
+
23
+ local function advance(...)
24
+ local ok, err = coroutine.resume(job, ...)
25
+ if not ok then
26
+ io.stderr:write('[codeoutline bm25] ', tostring(err), '\n')
27
+ job = nil
28
+ pool.bind(nil)
29
+ elseif coroutine.status(job) == 'dead' then
30
+ job = nil
31
+ pool.bind(nil)
32
+ end
33
+ end
34
+
35
+ local function lookup(request)
36
+ local previous = control.callback
37
+ control.callback = function()
38
+ previous()
39
+ if xtimer.now_ms() >= request.deadline then error(control.deadline(), 0) end
40
+ end
41
+ local ok, result = pcall(function()
42
+ control.check()
43
+ local hits = bm25.search(snapshot.reader(request.root, request.stamp), request.file, request.query, {
44
+ match = 'all', limit = 20, prefix = request.prefix,
45
+ })
46
+ control.check()
47
+ return hits
48
+ end)
49
+ control.callback = previous
50
+ return ok, ok and result or tostring(result)
51
+ end
52
+
53
+ return {
54
+ __init = function()
55
+ assert(xnet.init())
56
+ -- Keep worker IDs separate from the foreground index's parse pool.
57
+ pool.FIRST_ID = 256
58
+ pool.configure(scripts .. '/codeoutline/runtime/parse_worker.lua', pool.configured_threads())
59
+ control.callback = function()
60
+ if shared:get('shutdown') then error(control.cancelled(), 0) end
61
+ end
62
+ end,
63
+ __thread_handle = function(_, op, owner_or_token, item_or_results, failure)
64
+ if op == 'bm25_lookup' then
65
+ local ok, result = false, 'BM25 builder is busy'
66
+ if not job then ok, result = lookup(failure) end
67
+ xthread.post(owner_or_token, 'bm25_lookup_done', item_or_results, ok, result)
68
+ elseif op == 'bm25_build' then
69
+ local owner, item = owner_or_token, item_or_results
70
+ job = coroutine.create(function()
71
+ local ok, err = pcall(build, item)
72
+ if not ok and not control.is_interrupted(err) then
73
+ io.stderr:write('[codeoutline bm25] ', tostring(err), '\n')
74
+ end
75
+ pool.stop('BM25 snapshot complete')
76
+ collectgarbage('collect')
77
+ xthread.post(owner, 'bm25_done', item.file, item.stamp, ok)
78
+ end)
79
+ pool.bind(job)
80
+ advance()
81
+ elseif op == 'parsed' and pool.receive(owner_or_token) and job and pool.waiting() then
82
+ advance(owner_or_token, item_or_results, failure)
83
+ end
84
+ end,
85
+ __uninit = function() pool.stop(); xnet.uninit() end,
86
+ }
@@ -11,9 +11,13 @@ local pool = require('codeoutline.indexing.parse_pool')
11
11
  require('codeoutline.query.completion') -- registers lsp_completion
12
12
  service.configure({ watch = true })
13
13
  local shared = assert(xshared.dict('codeoutline_control'))
14
+ local bm25 = shared:get('bm25_worker') and require('codeoutline.bm25.automatic')
14
15
 
15
16
  local function execute(req)
16
- if navigation.handles(req.method) then return navigation.execute(req) end
17
+ if navigation.handles(req.method) then
18
+ if bm25 and req.method == 'lsp_definition' then req.bm25_definition = bm25.definition end
19
+ return navigation.execute(req)
20
+ end
17
21
  if req.method == 'lsp_close' then navigation.close(req.session); return true end
18
22
  if req.method == 'lsp_symbols' then
19
23
  local result = {}
@@ -62,9 +66,10 @@ local function execute(req)
62
66
  return true
63
67
  elseif req.method == 'explore' or req.method == 'callers' or req.method == 'type_hierarchy' or req.method == 'type_references' or req.method == 'affected' then
64
68
  local text, info = service[req.method](root, req.method == 'affected' and req.files or req.query, { budget = req.budget, depth = req.depth,
65
- max_nodes = req.max_nodes, direction = req.direction })
69
+ max_nodes = req.max_nodes, direction = req.direction, bm25_match = req.bm25_match,
70
+ prepare_bm25 = req.method == 'explore' and bm25 and bm25.prepare or nil })
66
71
  return { text = text, truncated = info.truncated or false, coverage = info.coverage,
67
- resolution = info.resolution, generation = info.generation, refresh = info.refresh }
72
+ resolution = info.resolution, generation = info.generation, refresh = info.refresh, bm25 = info.bm25 }
68
73
  elseif req.method == 'status' then return service.status(root)
69
74
  elseif req.method == 'rebuild' then return service.rebuild(root)
70
75
  else error('unknown operation') end
@@ -140,15 +145,16 @@ return {
140
145
  __init = function()
141
146
  assert(xnet.init())
142
147
  xtimer.init()
143
- local threads = tonumber(os.getenv('CODEOUTLINE_INDEX_THREADS') or '') or pool.THREADS
144
- pool.configure(scripts .. '/codeoutline/runtime/parse_worker.lua', math.max(0, math.floor(threads)))
148
+ pool.configure(scripts .. '/codeoutline/runtime/parse_worker.lua', pool.configured_threads())
145
149
  assert(xthread.post(1, 'worker_ready'))
146
150
  end,
147
151
  -- A suspended job still holds its project: sweeping waits for it. Idle
148
152
  -- ticks also collect garbage, a step at a time (see service.collect).
149
153
  __update = function()
154
+ if bm25 and bm25.expire_lookup() and active then resume(false); pump() end
150
155
  pool.sweep()
151
156
  if not active and #queue == 0 then
157
+ if bm25 then bm25.pump() end
152
158
  service.sweep()
153
159
  -- Timer callbacks own pending cycles. Retain idle collection for
154
160
  -- allocations/dropped projects that happen after a timer finishes.
@@ -170,7 +176,14 @@ return {
170
176
  elseif op == 'parsed' then
171
177
  -- (token, results, failure); stale tokens are dropped by the pool.
172
178
  if pool.receive(id) and active and pool.waiting() then resume(id, req, failure); pump() end
179
+ elseif op == 'bm25_done' and bm25 then
180
+ bm25.finish(id, req, failure)
181
+ elseif op == 'bm25_lookup_done' and bm25 then
182
+ if bm25.receive_lookup(id) and active then resume(req, failure); pump() end
173
183
  end
184
+ -- Dispatch after foreground processing, even when a project is queried
185
+ -- by name and will never consult the lexical index in this request.
186
+ if bm25 and not active and #queue == 0 then bm25.pump() end
174
187
  end,
175
188
  __uninit = function()
176
189
  timer_gc.close()
@@ -8,13 +8,20 @@ local shared = assert(xshared.dict('codeoutline_control'))
8
8
 
9
9
  return {
10
10
  __thread_handle = function(_, op, owner, token, max_bytes, files)
11
- if op ~= 'parse' then return end
11
+ if op ~= 'parse' and op ~= 'bm25_collect' then return end
12
12
  local results = {}
13
+ local abort_key = 'parse_abort:' .. owner
13
14
  local ok, err = pcall(function()
15
+ local load = op == 'parse' and index.load_file or require('codeoutline.bm25.collect').file
14
16
  for i, f in ipairs(files) do
15
17
  -- An abandoned pass (cancelled, failed, shutting down) skips its rest.
16
- if shared:get('shutdown') or (shared:get('parse_abort') or 0) >= token then return end
17
- local res = index.load_file(f[2], f[1], max_bytes, f[3] or nil, f[4] or nil, f[5] or nil)
18
+ if shared:get('shutdown') or (shared:get(abort_key) or 0) >= token then return end
19
+ local res
20
+ if op == 'parse' then
21
+ res = load(f[2], f[1], max_bytes, f[3] or nil, f[4] or nil, f[5] or nil)
22
+ else
23
+ res = load(f[2], f[1], f[3], f[4], f[5], f[6])
24
+ end
18
25
  res.rel = f[1]
19
26
  results[i] = res
20
27
  end
@@ -182,6 +182,12 @@ end
182
182
  local function execute_query(engine, root, query, opts)
183
183
  engine.validate(query, opts)
184
184
  local idx, _, stats = refresh(root, opts, true)
185
+ if engine == explore and opts and opts.prepare_bm25 then
186
+ local query_opts = {}
187
+ for key, value in pairs(opts) do query_opts[key] = value end
188
+ query_opts.bm25_fallback = opts.prepare_bm25(idx)
189
+ opts = query_opts
190
+ end
185
191
  idx.query_sources = {}
186
192
  local ok, result = pcall(function()
187
193
  while true do
@@ -61,6 +61,14 @@ local function definition(r, label)
61
61
  result.targetKind = r.targetKind or 'call'
62
62
  assert(result.targetKind == 'call' or result.targetKind == 'declaration' or result.targetKind == 'key',
63
63
  "invalid " .. label .. '.targetKind')
64
+ if r.targetPath ~= nil then
65
+ assert(result.targetKind == 'key' and type(r.targetPath) == 'string' and r.targetPath ~= ''
66
+ and not r.targetPath:find('[\\:%z\r\n]') and not r.targetPath:match('^/'),
67
+ label .. '.targetPath must be a project-relative key target path')
68
+ for part in r.targetPath:gmatch('[^/]+') do
69
+ assert(part ~= '.' and part ~= '..', label .. '.targetPath must stay inside the project')
70
+ end
71
+ end
64
72
  if result.targetKind == 'call' then
65
73
  call(r, 'target', label); position(r, 'targetArgument', label)
66
74
  else
@@ -83,7 +91,7 @@ function M.normalize(config)
83
91
  local seen = { definitions = {}, dispatch = {} }
84
92
  local function add(consumer, r)
85
93
  local fields = consumer == 'dispatch' and { 'kind', 'language', 'call', 'argument', 'register', 'keyArgument', 'handlerArgument', 'serviceArgument' }
86
- or { 'language', 'call', 'argument', 'targetKind', 'target', 'targetArgument', 'serviceArgument' }
94
+ or { 'language', 'call', 'argument', 'targetKind', 'target', 'targetArgument', 'serviceArgument', 'targetPath' }
87
95
  local key = {}
88
96
  for _, f in ipairs(fields) do key[#key + 1] = tostring(r[f]) end
89
97
  if r.qualifiers then
@@ -1 +1 @@
1
- return 27
1
+ return 29
@@ -1 +1 @@
1
- return '0.3.8'
1
+ return '0.5.0'