@codua/codeoutline-win32-x64 0.3.9 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/README.zh-CN.md +33 -0
- package/bin/xnet.exe +0 -0
- package/build-info.json +1 -1
- package/editors/codeoutline.vsix +0 -0
- package/editors/zed/extension.wasm +0 -0
- package/package.json +1 -1
- package/scripts/codeoutline/analysis/indirect.lua +18 -12
- package/scripts/codeoutline/analysis/lua/lua_bindings.lua +28 -0
- package/scripts/codeoutline/bm25/automatic.lua +100 -0
- package/scripts/codeoutline/bm25/collect.lua +148 -0
- package/scripts/codeoutline/bm25/index.lua +548 -0
- package/scripts/codeoutline/bm25/snapshot.lua +45 -0
- package/scripts/codeoutline/indexing/parse_pool.lua +12 -6
- package/scripts/codeoutline/main.lua +5 -1
- package/scripts/codeoutline/protocol/lsp.lua +3 -1
- package/scripts/codeoutline/protocol/mcp.lua +5 -2
- package/scripts/codeoutline/query/coverage.lua +2 -0
- package/scripts/codeoutline/query/definition_candidates.lua +24 -4
- package/scripts/codeoutline/query/explore.lua +110 -18
- package/scripts/codeoutline/query/navigation.lua +189 -19
- package/scripts/codeoutline/query/query_filters.lua +5 -1
- package/scripts/codeoutline/runtime/bm25_worker.lua +86 -0
- package/scripts/codeoutline/runtime/index_worker.lua +18 -5
- package/scripts/codeoutline/runtime/parse_worker.lua +10 -3
- package/scripts/codeoutline/service.lua +6 -0
- package/scripts/codeoutline/update_sequence.lua +1 -1
- package/scripts/codeoutline/version.lua +1 -1
|
@@ -0,0 +1,548 @@
|
|
|
1
|
+
-- Public API: build(index, file) and search(index, file, query, options).
|
|
2
|
+
-- Pass a refreshed CodeOutline index. The file is a lexical snapshot of that
|
|
3
|
+
-- project's source records and must be rebuilt when those records change.
|
|
4
|
+
local indexing = require('codeoutline.indexing.index')
|
|
5
|
+
local pool = require('codeoutline.indexing.parse_pool')
|
|
6
|
+
local collect = require('codeoutline.bm25.collect')
|
|
7
|
+
local control = require('codeoutline.shared.control')
|
|
8
|
+
local paths = require('codeoutline.shared.path')
|
|
9
|
+
local text_util = require('codeoutline.shared.text')
|
|
10
|
+
local snapshot = require('codeoutline.bm25.snapshot')
|
|
11
|
+
|
|
12
|
+
local M = {}
|
|
13
|
+
local TRAILER_SIZE = 80
|
|
14
|
+
local PASSAGE_BLOCK = 32
|
|
15
|
+
local GC_BATCH_BYTES = 512 * 1024
|
|
16
|
+
local GC_STEP_KB = 512
|
|
17
|
+
local cached_digest, cached_header
|
|
18
|
+
|
|
19
|
+
-- Drive GC by completed work so temporary batches can be reclaimed while the
|
|
20
|
+
-- live postings grow. Keep the collector settings of the calling Lua state.
|
|
21
|
+
local function build_gc()
|
|
22
|
+
local processed = 0
|
|
23
|
+
return function(bytes)
|
|
24
|
+
processed = processed + bytes
|
|
25
|
+
if processed >= GC_BATCH_BYTES then
|
|
26
|
+
processed = 0
|
|
27
|
+
collectgarbage('step', GC_STEP_KB)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
local function plan(index, source_paths)
|
|
33
|
+
local jobs, files, locations = {}, {}, {}
|
|
34
|
+
for _, path in ipairs(source_paths) do
|
|
35
|
+
control.check()
|
|
36
|
+
local file = index.files[path]
|
|
37
|
+
local documents = {}
|
|
38
|
+
for _, node in ipairs(file.nodes) do
|
|
39
|
+
if node.kind == 'function' or node.kind == 'method' then
|
|
40
|
+
documents[#documents + 1] = {false, text_util.valid(node.qualified or node.name),
|
|
41
|
+
node.line, math.max(node.line, node.end_line or node.line), node.kind}
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
if #documents > 0 then
|
|
45
|
+
table.sort(documents, function(a, b)
|
|
46
|
+
if a[3] ~= b[3] then return a[3] < b[3] end
|
|
47
|
+
if a[2] ~= b[2] then return a[2] < b[2] end
|
|
48
|
+
if a[4] ~= b[4] then return a[4] < b[4] end
|
|
49
|
+
return a[5] < b[5]
|
|
50
|
+
end)
|
|
51
|
+
local file_id = #files + 1
|
|
52
|
+
files[file_id] = path
|
|
53
|
+
for _, doc in ipairs(documents) do
|
|
54
|
+
doc[1] = #locations + 1
|
|
55
|
+
locations[doc[1]] = {file_id, doc[2], doc[3], doc[4], doc[5]}
|
|
56
|
+
end
|
|
57
|
+
jobs[#jobs + 1] = {rel = path, abs = index:abs(path), size = file.size, crc = file.crc,
|
|
58
|
+
language = file.language, documents = documents, weight = file.size + indexing.FILE_WEIGHT}
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
return jobs, files, locations
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
local function collect_terms(index, jobs, spool, gc_step)
|
|
65
|
+
local totals, lengths, comments = {0, 0, 0, 0}, {}, 0
|
|
66
|
+
local function accept(_, result)
|
|
67
|
+
local encoded = result.terms
|
|
68
|
+
assert(spool:write(encoded))
|
|
69
|
+
lengths[#lengths + 1] = #encoded
|
|
70
|
+
comments = comments + result.comments
|
|
71
|
+
for i, length in ipairs(result.totals) do totals[i] = totals[i] + length end
|
|
72
|
+
gc_step(#encoded)
|
|
73
|
+
end
|
|
74
|
+
if #jobs >= indexing.PARALLEL_MIN_FILES and pool.ready() then
|
|
75
|
+
pool.run(jobs, index.max_bytes, accept, 'bm25_collect')
|
|
76
|
+
else
|
|
77
|
+
for _, entry in ipairs(jobs) do
|
|
78
|
+
accept(entry.rel, collect.file(entry.abs, entry.rel, entry.size, entry.crc, entry.language, entry.documents))
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
assert(#lengths == #jobs, 'incomplete BM25 source pass')
|
|
82
|
+
return totals, lengths, comments
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
local function read_at(file, offset, length)
|
|
86
|
+
assert(file:seek('set', offset) == offset, 'cannot seek BM25 index')
|
|
87
|
+
local data = assert(file:read(length), 'truncated BM25 index')
|
|
88
|
+
assert(#data == length, 'truncated BM25 index')
|
|
89
|
+
return data
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
local function merge_terms(spool, totals, lengths, count, write_passages, gc_step)
|
|
93
|
+
local postings, averages = {}, {}
|
|
94
|
+
for i, total in ipairs(totals) do averages[i] = count > 0 and total / count or 0 end
|
|
95
|
+
assert(spool:seek('set', 0) == 0)
|
|
96
|
+
for _, length in ipairs(lengths) do
|
|
97
|
+
control.check()
|
|
98
|
+
gc_step(length)
|
|
99
|
+
local data = assert(spool:read(length), 'truncated BM25 staging file')
|
|
100
|
+
assert(#data == length, 'truncated BM25 staging file')
|
|
101
|
+
local batch = cmsgpack.unpack(data)
|
|
102
|
+
if #batch > 0 then write_passages(batch) end
|
|
103
|
+
for _, doc in ipairs(batch) do
|
|
104
|
+
local id, fields = table.unpack(doc)
|
|
105
|
+
for i, field in ipairs(fields) do
|
|
106
|
+
if averages[i] > 0 then
|
|
107
|
+
local terms, length = table.unpack(field)
|
|
108
|
+
local normalization = 1.2 * (0.25 + 0.75 * length / averages[i])
|
|
109
|
+
local weight = collect.weights[i]
|
|
110
|
+
for word, frequency in pairs(terms) do
|
|
111
|
+
local matches = postings[word]
|
|
112
|
+
if not matches then matches = {}; postings[word] = matches end
|
|
113
|
+
local tf = weight * 2.2 * frequency / (frequency + normalization)
|
|
114
|
+
matches[id] = (matches[id] or 0) + tf
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
return postings
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
local function exact_keys(files, locations)
|
|
124
|
+
local exact = {}
|
|
125
|
+
for id, location in ipairs(locations) do
|
|
126
|
+
if id % 128 == 0 then control.check() end
|
|
127
|
+
local path = files[location[1]]:lower()
|
|
128
|
+
local name = location[2]:lower()
|
|
129
|
+
for _, key in ipairs({path, path:match('[^/]+$'), name, name:match('[^.:]+$') or name}) do
|
|
130
|
+
local matches = exact[key]
|
|
131
|
+
if not matches then matches = {}; exact[key] = matches end
|
|
132
|
+
matches[id] = true
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
return exact
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
local function write_index(output, spool, totals, lengths, header, gc_step)
|
|
139
|
+
local prefix = snapshot.prefix(header.version) .. header.stamp
|
|
140
|
+
assert(output:write(prefix))
|
|
141
|
+
local offset, count = #prefix, #header.locations
|
|
142
|
+
local function write_data(data)
|
|
143
|
+
local location = {offset, #data, xcompress.crc32(data)}
|
|
144
|
+
assert(output:write(data))
|
|
145
|
+
offset = offset + #data
|
|
146
|
+
return location
|
|
147
|
+
end
|
|
148
|
+
local function write_block(matches)
|
|
149
|
+
return write_data(cmsgpack.pack(matches))
|
|
150
|
+
end
|
|
151
|
+
local compressor = xcompress.new_compressor(1)
|
|
152
|
+
local function write_passages(batch)
|
|
153
|
+
local first_id = batch[1][1]
|
|
154
|
+
local blocks = {first_id}
|
|
155
|
+
header.passages[header.locations[first_id][1]] = blocks
|
|
156
|
+
for start = 1, #batch, PASSAGE_BLOCK do
|
|
157
|
+
control.check()
|
|
158
|
+
local passages = {}
|
|
159
|
+
for i = start, math.min(start + PASSAGE_BLOCK - 1, #batch) do
|
|
160
|
+
passages[#passages + 1] = {batch[i][3], batch[i][4]}
|
|
161
|
+
end
|
|
162
|
+
local data = cmsgpack.pack(passages)
|
|
163
|
+
local entry = write_data(assert(compressor:deflate(data)))
|
|
164
|
+
entry[4] = #data
|
|
165
|
+
blocks[#blocks + 1] = entry
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
local postings = merge_terms(spool, totals, lengths, count, write_passages, gc_step)
|
|
169
|
+
compressor:close()
|
|
170
|
+
local records = 0
|
|
171
|
+
for word, matches in pairs(postings) do
|
|
172
|
+
control.check()
|
|
173
|
+
local frequency = 0
|
|
174
|
+
for _ in pairs(matches) do frequency = frequency + 1 end
|
|
175
|
+
local entry = write_block(matches)
|
|
176
|
+
entry[4] = math.log(1 + (count - frequency + 0.5) / (frequency + 0.5))
|
|
177
|
+
header.words[word] = entry
|
|
178
|
+
records = records + frequency
|
|
179
|
+
postings[word] = nil
|
|
180
|
+
matches = nil
|
|
181
|
+
gc_step(entry[2])
|
|
182
|
+
end
|
|
183
|
+
postings = nil
|
|
184
|
+
local exact = exact_keys(header.files, header.locations)
|
|
185
|
+
for key, matches in pairs(exact) do
|
|
186
|
+
control.check()
|
|
187
|
+
local entry = write_block(matches)
|
|
188
|
+
header.exact[key] = entry
|
|
189
|
+
exact[key] = nil
|
|
190
|
+
matches = nil
|
|
191
|
+
gc_step(entry[2])
|
|
192
|
+
end
|
|
193
|
+
exact = nil
|
|
194
|
+
local encoded = cmsgpack.pack(header)
|
|
195
|
+
assert(output:write(encoded, xutils.sha256_hex(encoded), string.format('%016d', #encoded)))
|
|
196
|
+
return records, offset + #encoded + TRAILER_SIZE
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
local function output_path(file)
|
|
200
|
+
assert(type(file) == 'string' and file ~= '' and not file:find('%z'), 'BM25 index file is required')
|
|
201
|
+
return paths.normalize(file)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
local function check_destination(file)
|
|
205
|
+
local existing = io.open(file, 'rb')
|
|
206
|
+
if not existing then return end
|
|
207
|
+
local magic = existing:read(#snapshot.magic)
|
|
208
|
+
existing:close()
|
|
209
|
+
assert(magic == snapshot.magic, 'refusing to replace a non-BM25 file: ' .. file)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
function M.build(index, file)
|
|
213
|
+
file = output_path(file)
|
|
214
|
+
assert(not index.cache_path or paths.key(file) ~= paths.key(index.cache_path),
|
|
215
|
+
'BM25 must use a separate file from the base index')
|
|
216
|
+
check_destination(file)
|
|
217
|
+
local parent = file:match('^(.*)/[^/]+$') or '.'
|
|
218
|
+
if parent == '' or parent:match('^%a:$') then parent = parent .. '/' end
|
|
219
|
+
assert(xutils.mkdir_p(parent))
|
|
220
|
+
local generation = index.generation
|
|
221
|
+
local stamp, source_paths = snapshot.stamp(index)
|
|
222
|
+
local jobs, files, locations = plan(index, source_paths)
|
|
223
|
+
local spool_name, staged, spool, output
|
|
224
|
+
local ok, result = pcall(function()
|
|
225
|
+
spool_name = assert(xutils.temp_file(parent))
|
|
226
|
+
spool = assert(io.open(spool_name, 'w+b'))
|
|
227
|
+
local gc_step = build_gc()
|
|
228
|
+
local totals, lengths, comments = collect_terms(index, jobs, spool, gc_step)
|
|
229
|
+
jobs = nil
|
|
230
|
+
staged = assert(xutils.temp_file(parent))
|
|
231
|
+
output = assert(io.open(staged, 'wb'))
|
|
232
|
+
local records, bytes = write_index(output, spool, totals, lengths, {version = snapshot.version,
|
|
233
|
+
stamp = stamp, files = files, locations = locations, words = {}, exact = {}, passages = {}}, gc_step)
|
|
234
|
+
assert(spool:close()); spool = nil
|
|
235
|
+
assert(output:close()); output = nil
|
|
236
|
+
control.check()
|
|
237
|
+
assert(index.generation == generation, 'base index changed during BM25 build')
|
|
238
|
+
check_destination(file)
|
|
239
|
+
assert(xutils.replace_file(staged, file))
|
|
240
|
+
staged = nil
|
|
241
|
+
return {files = #files, functions = #locations, comments = comments, postings = records, bytes = bytes}
|
|
242
|
+
end)
|
|
243
|
+
if spool then spool:close() end
|
|
244
|
+
if output then output:close() end
|
|
245
|
+
if spool_name then os.remove(spool_name) end
|
|
246
|
+
if staged then os.remove(staged) end
|
|
247
|
+
if not ok then error(result, 0) end
|
|
248
|
+
return result
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
local function open_header(file)
|
|
252
|
+
local size = assert(file:seek('end'))
|
|
253
|
+
local magic = snapshot.magic
|
|
254
|
+
assert(size > #magic + TRAILER_SIZE and read_at(file, 0, #magic) == magic, 'invalid BM25 index')
|
|
255
|
+
local trailer = read_at(file, size - TRAILER_SIZE, TRAILER_SIZE)
|
|
256
|
+
local digest, length = trailer:sub(1, 64), tonumber(trailer:sub(65))
|
|
257
|
+
assert(length and length % 1 == 0 and length > 0 and length <= size - #magic - TRAILER_SIZE,
|
|
258
|
+
'invalid BM25 header length')
|
|
259
|
+
local offset = size - TRAILER_SIZE - length
|
|
260
|
+
if digest ~= cached_digest then
|
|
261
|
+
local encoded = read_at(file, offset, length)
|
|
262
|
+
assert(xutils.sha256_hex(encoded) == digest, 'corrupt BM25 header')
|
|
263
|
+
local header = cmsgpack.unpack(encoded)
|
|
264
|
+
assert(header.version == 1 or header.version == 2 or header.version == snapshot.version,
|
|
265
|
+
'unsupported BM25 index version; rebuild it')
|
|
266
|
+
cached_digest, cached_header = digest, header
|
|
267
|
+
end
|
|
268
|
+
local first = #magic
|
|
269
|
+
if cached_header.version >= 2 then
|
|
270
|
+
local prefix = snapshot.prefix(cached_header.version) .. cached_header.stamp
|
|
271
|
+
assert(offset >= #prefix and read_at(file, 0, #prefix) == prefix, 'corrupt BM25 snapshot prefix')
|
|
272
|
+
first = #prefix
|
|
273
|
+
end
|
|
274
|
+
return cached_header, first, offset
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
local function read_data(file, entry, first, boundary)
|
|
278
|
+
local offset, length = entry[1], entry[2]
|
|
279
|
+
assert(offset >= first and length > 0 and offset + length <= boundary, 'invalid BM25 block range')
|
|
280
|
+
local encoded = read_at(file, offset, length)
|
|
281
|
+
assert(xcompress.crc32(encoded) == entry[3], 'corrupt BM25 postings')
|
|
282
|
+
return encoded
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
local function read_block(file, entry, first, boundary)
|
|
286
|
+
return cmsgpack.unpack(read_data(file, entry, first, boundary))
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
local function scope_key(index, path)
|
|
290
|
+
if path == nil or path == '' or path == '.' then return nil end
|
|
291
|
+
assert(type(path) == 'string', 'BM25 path filter must be a string')
|
|
292
|
+
path = paths.key(path):gsub('^%./', '')
|
|
293
|
+
if path:match('^/') or path:match('^%a:/') then
|
|
294
|
+
local root = paths.key(index.root)
|
|
295
|
+
assert(paths.contains(root, path), 'BM25 path filter is outside the project')
|
|
296
|
+
if root == path then return nil end
|
|
297
|
+
return path:sub(#root + (root:sub(-1) == '/' and 1 or 2))
|
|
298
|
+
end
|
|
299
|
+
return path
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
local function rank(scores, exact, literal, limit)
|
|
303
|
+
local function better(a, b)
|
|
304
|
+
if exact[a] ~= exact[b] then return exact[a] == true end
|
|
305
|
+
if literal[a] ~= literal[b] then return literal[a] == true end
|
|
306
|
+
if scores[a] ~= scores[b] then return scores[a] > scores[b] end
|
|
307
|
+
return a < b
|
|
308
|
+
end
|
|
309
|
+
-- The root of the bounded heap is the worst retained candidate.
|
|
310
|
+
local heap, visited = {}, 0
|
|
311
|
+
for id in pairs(scores) do
|
|
312
|
+
visited = visited + 1
|
|
313
|
+
if visited % 1024 == 0 then control.check() end
|
|
314
|
+
if #heap < limit then
|
|
315
|
+
local i = #heap + 1
|
|
316
|
+
while i > 1 do
|
|
317
|
+
local parent = math.floor(i / 2)
|
|
318
|
+
if not better(heap[parent], id) then break end
|
|
319
|
+
heap[i], i = heap[parent], parent
|
|
320
|
+
end
|
|
321
|
+
heap[i] = id
|
|
322
|
+
elseif better(id, heap[1]) then
|
|
323
|
+
local i, child = 1, 2
|
|
324
|
+
while child <= #heap do
|
|
325
|
+
if child < #heap and better(heap[child], heap[child + 1]) then child = child + 1 end
|
|
326
|
+
if not better(id, heap[child]) then break end
|
|
327
|
+
heap[i], i = heap[child], child
|
|
328
|
+
child = i * 2
|
|
329
|
+
end
|
|
330
|
+
heap[i] = id
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
table.sort(heap, better)
|
|
334
|
+
return heap
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
local function score_terms(file, entries, words, first, boundary, accepts, match)
|
|
338
|
+
local scores = {}
|
|
339
|
+
local counts = match ~= 'all' and #words > 1 and {} or nil
|
|
340
|
+
local rarest, seed
|
|
341
|
+
if match == 'all' and #words > 1 then
|
|
342
|
+
for _, word in ipairs(words) do
|
|
343
|
+
local entry = entries[word]
|
|
344
|
+
if not entry then return scores end
|
|
345
|
+
-- IDF decreases as the number of documents for a word increases.
|
|
346
|
+
if not rarest or entry[4] > entries[rarest][4] then rarest = word end
|
|
347
|
+
end
|
|
348
|
+
seed = read_block(file, entries[rarest], first, boundary)
|
|
349
|
+
local scanned = 0
|
|
350
|
+
for id in pairs(seed) do
|
|
351
|
+
scanned = scanned + 1
|
|
352
|
+
if scanned % 1024 == 0 then control.check() end
|
|
353
|
+
if accepts(id) then scores[id] = 0 end
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
-- Accumulate terms in the same order for union and intersection searches,
|
|
357
|
+
-- so a surviving document keeps the same BM25 score and tie-breaking.
|
|
358
|
+
for _, word in ipairs(words) do
|
|
359
|
+
control.check()
|
|
360
|
+
if rarest and not next(scores) then break end
|
|
361
|
+
local entry = entries[word]
|
|
362
|
+
if entry then
|
|
363
|
+
local matches = word == rarest and seed or read_block(file, entry, first, boundary)
|
|
364
|
+
local scanned = 0
|
|
365
|
+
if rarest then
|
|
366
|
+
for id, score in pairs(scores) do
|
|
367
|
+
scanned = scanned + 1
|
|
368
|
+
if scanned % 1024 == 0 then control.check() end
|
|
369
|
+
local tf = matches[id]
|
|
370
|
+
scores[id] = tf and (score + entry[4] * tf) or nil
|
|
371
|
+
end
|
|
372
|
+
if word == rarest then seed = nil end
|
|
373
|
+
else
|
|
374
|
+
for id, tf in pairs(matches) do
|
|
375
|
+
scanned = scanned + 1
|
|
376
|
+
if scanned % 1024 == 0 then control.check() end
|
|
377
|
+
if accepts(id) then
|
|
378
|
+
scores[id] = (scores[id] or 0) + entry[4] * tf
|
|
379
|
+
if counts then counts[id] = (counts[id] or 0) + 1 end
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
end
|
|
385
|
+
return scores, counts
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
local function contains_literal(text, literal, left_boundary, right_boundary)
|
|
389
|
+
if text == '' then return false end
|
|
390
|
+
text = text:lower()
|
|
391
|
+
local offset = 1
|
|
392
|
+
while true do
|
|
393
|
+
local first, last = text:find(literal, offset, true)
|
|
394
|
+
if not first then return false end
|
|
395
|
+
if (not left_boundary or first == 1 or not text:sub(first - 1, first - 1):match('[%w_$]'))
|
|
396
|
+
and (not right_boundary or not text:sub(last + 1, last + 1):match('[%w_$]')) then
|
|
397
|
+
return true
|
|
398
|
+
end
|
|
399
|
+
offset = first + 1
|
|
400
|
+
end
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
local function boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
|
|
404
|
+
local exact = {}
|
|
405
|
+
if not header.passages or #words == 0 or literal == '' or not next(scores) then return exact end
|
|
406
|
+
local strength = 0
|
|
407
|
+
for _, word in ipairs(words) do
|
|
408
|
+
local entry = header.words[word]
|
|
409
|
+
if not entry then return exact end
|
|
410
|
+
strength = strength + entry[4]
|
|
411
|
+
end
|
|
412
|
+
-- Group by block so each compressed passage is read at most once per query.
|
|
413
|
+
local groups, scanned = {}, 0
|
|
414
|
+
for id in pairs(scores) do
|
|
415
|
+
scanned = scanned + 1
|
|
416
|
+
if scanned % 128 == 0 then control.check() end
|
|
417
|
+
if not counts or counts[id] == #words then
|
|
418
|
+
local blocks = header.passages[header.locations[id][1]]
|
|
419
|
+
local offset = id - blocks[1]
|
|
420
|
+
local entry = blocks[2 + math.floor(offset / PASSAGE_BLOCK)]
|
|
421
|
+
local group = groups[entry]
|
|
422
|
+
if not group then group = {}; groups[entry] = group end
|
|
423
|
+
group[#group + 1] = id
|
|
424
|
+
end
|
|
425
|
+
end
|
|
426
|
+
if not next(groups) then return exact end
|
|
427
|
+
local decoder = xcompress.new_decompressor()
|
|
428
|
+
local left_boundary, right_boundary = literal:sub(1, 1):match('[%w_$]'), literal:sub(-1):match('[%w_$]')
|
|
429
|
+
for entry, ids in pairs(groups) do
|
|
430
|
+
control.check()
|
|
431
|
+
local data = read_data(file, entry, first, boundary)
|
|
432
|
+
local passages = cmsgpack.unpack(assert(decoder:deflate(data, entry[4]), 'corrupt BM25 passages'))
|
|
433
|
+
for _, id in ipairs(ids) do
|
|
434
|
+
local location = header.locations[id]
|
|
435
|
+
local offset = id - header.passages[location[1]][1]
|
|
436
|
+
local passage = passages[1 + offset % PASSAGE_BLOCK]
|
|
437
|
+
local best
|
|
438
|
+
for _, text in ipairs({location[2], header.files[location[1]], passage[1], passage[2]}) do
|
|
439
|
+
if not exact[id] and contains_literal(text, literal, left_boundary, right_boundary) then exact[id] = true end
|
|
440
|
+
if #words > 1 and best ~= 0 and text ~= '' then
|
|
441
|
+
local gap = collect.gap(text, terms, #words)
|
|
442
|
+
if gap and (not best or gap < best) then best = gap end
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
if best then scores[id] = scores[id] + strength / (1 + best) end
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
decoder:close()
|
|
449
|
+
return exact
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
-- Use the existing symbol and path tables. Prefix hints change ranking only.
|
|
453
|
+
local function boost_prefix(header, scores, words, prefix)
|
|
454
|
+
if not prefix or prefix == '' then return end
|
|
455
|
+
prefix = prefix:lower()
|
|
456
|
+
local strength = 0
|
|
457
|
+
for _, word in ipairs(words) do
|
|
458
|
+
local entry = header.words[word]
|
|
459
|
+
if entry then strength = strength + entry[4] end
|
|
460
|
+
end
|
|
461
|
+
strength = math.max(strength, 1)
|
|
462
|
+
local path_scores, scanned = {}, 0
|
|
463
|
+
local directory = '/' .. prefix .. '/'
|
|
464
|
+
for id, score in pairs(scores) do
|
|
465
|
+
scanned = scanned + 1
|
|
466
|
+
if scanned % 128 == 0 then control.check() end
|
|
467
|
+
local location = header.locations[id]
|
|
468
|
+
local file_id = location[1]
|
|
469
|
+
local weight = path_scores[file_id]
|
|
470
|
+
if not weight then
|
|
471
|
+
local path = header.files[file_id]:lower()
|
|
472
|
+
local stem = path:match('[^/]+$'):gsub('%.[^%.]+$', '')
|
|
473
|
+
if stem == prefix then weight = 4
|
|
474
|
+
elseif ('/' .. path):find(directory, 1, true) then weight = 3
|
|
475
|
+
elseif stem:sub(1, #prefix) == prefix and stem:sub(#prefix + 1, #prefix + 1):match('[_%.%-]') then
|
|
476
|
+
weight = 1
|
|
477
|
+
else weight = 0 end
|
|
478
|
+
path_scores[file_id] = weight
|
|
479
|
+
end
|
|
480
|
+
local owner = location[2]:match('([^.:]+)[.:]+[^.:]+$')
|
|
481
|
+
if owner and owner:lower() == prefix then weight = 5 end
|
|
482
|
+
if weight > 0 then scores[id] = score + weight * strength end
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
local function search_file(index, file, query, options, limit)
|
|
487
|
+
local header, first, boundary = open_header(file)
|
|
488
|
+
assert(header.stamp == snapshot.stamp(index), 'BM25 index is stale; rebuild it')
|
|
489
|
+
local scope, allowed = scope_key(index, options.path), {}
|
|
490
|
+
local function in_scope(id)
|
|
491
|
+
if not scope then return true end
|
|
492
|
+
local file_id = header.locations[id][1]
|
|
493
|
+
if allowed[file_id] == nil then allowed[file_id] = paths.contains(scope, header.files[file_id]) end
|
|
494
|
+
return allowed[file_id]
|
|
495
|
+
end
|
|
496
|
+
local accepts = in_scope
|
|
497
|
+
if options.accept then
|
|
498
|
+
local accepted = {}
|
|
499
|
+
accepts = function(id)
|
|
500
|
+
if accepted[id] == nil then
|
|
501
|
+
local location = header.locations[id]
|
|
502
|
+
accepted[id] = not not (in_scope(id)
|
|
503
|
+
and options.accept(header.files[location[1]], location[2], location[5]))
|
|
504
|
+
end
|
|
505
|
+
return accepted[id]
|
|
506
|
+
end
|
|
507
|
+
end
|
|
508
|
+
local words, terms = {}, collect.tokenize(query)
|
|
509
|
+
for token in pairs(terms) do words[#words + 1] = token end
|
|
510
|
+
table.sort(words)
|
|
511
|
+
local scores, counts = score_terms(file, header.words, words, first, boundary, accepts, options.match)
|
|
512
|
+
local mark = string.char(96)
|
|
513
|
+
local literal = query:match('^%s*(.-)%s*$'):gsub('^' .. mark .. '+', ''):gsub(mark .. '+$', ''):lower()
|
|
514
|
+
local key = literal:gsub('\\', '/')
|
|
515
|
+
local entry = header.exact[key]
|
|
516
|
+
local exact = entry and read_block(file, entry, first, boundary) or {}
|
|
517
|
+
local literal_matches = boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
|
|
518
|
+
if options.match ~= 'all' or #words == 0 then
|
|
519
|
+
for id in pairs(exact) do if accepts(id) then scores[id] = scores[id] or 0 end end
|
|
520
|
+
end
|
|
521
|
+
boost_prefix(header, scores, words, options.prefix)
|
|
522
|
+
local results = {}
|
|
523
|
+
for rank_number, id in ipairs(rank(scores, exact, literal_matches, limit)) do
|
|
524
|
+
local location = header.locations[id]
|
|
525
|
+
results[rank_number] = {path = header.files[location[1]], symbol = location[2],
|
|
526
|
+
start_line = location[3], end_line = location[4], score = scores[id], kind = location[5]}
|
|
527
|
+
end
|
|
528
|
+
return results
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
function M.search(index, file, query, options)
|
|
532
|
+
assert(type(query) == 'string', 'BM25 query must be a string')
|
|
533
|
+
options = options or {}
|
|
534
|
+
assert(options.match == nil or options.match == 'any' or options.match == 'all',
|
|
535
|
+
'BM25 match must be any or all')
|
|
536
|
+
assert(options.prefix == nil or type(options.prefix) == 'string', 'BM25 prefix hint must be a string')
|
|
537
|
+
local limit = options.limit == nil and 20 or options.limit
|
|
538
|
+
assert(type(limit) == 'number' and limit >= 0 and limit < math.huge and limit % 1 == 0,
|
|
539
|
+
'BM25 limit must be a nonnegative integer')
|
|
540
|
+
if limit == 0 then return {} end
|
|
541
|
+
local input = assert(io.open(output_path(file), 'rb'))
|
|
542
|
+
local ok, result = pcall(search_file, index, input, query, options, limit)
|
|
543
|
+
input:close()
|
|
544
|
+
if not ok then error(result, 0) end
|
|
545
|
+
return result
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
return M
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
-- Internal source identity shared by readers and the background builder.
|
|
2
|
+
local indexing = require('codeoutline.indexing.index')
|
|
3
|
+
local paths = require('codeoutline.shared.path')
|
|
4
|
+
local control = require('codeoutline.shared.control')
|
|
5
|
+
local M = {magic = 'CODEOUTLINE-BM25\n', version = 3}
|
|
6
|
+
|
|
7
|
+
function M.prefix(version)
|
|
8
|
+
return M.magic .. version .. '\n'
|
|
9
|
+
end
|
|
10
|
+
local cache = setmetatable({}, {__mode = 'k'})
|
|
11
|
+
|
|
12
|
+
function M.stamp(index)
|
|
13
|
+
local cached = cache[index]
|
|
14
|
+
if cached and cached.generation == index.generation then return cached.stamp, cached.files end
|
|
15
|
+
local files, records = {}, {}
|
|
16
|
+
for path in pairs(index.files) do files[#files + 1] = path end
|
|
17
|
+
table.sort(files)
|
|
18
|
+
for i, path in ipairs(files) do
|
|
19
|
+
if i % 128 == 0 then control.check() end
|
|
20
|
+
local file = index.files[path]
|
|
21
|
+
records[i] = {path, file.size, file.crc, file.language, file.cpp_fragment or false}
|
|
22
|
+
end
|
|
23
|
+
local stamp = xutils.sha256_hex(cmsgpack.pack({indexing.VERSION, paths.key(index.root), records}))
|
|
24
|
+
cache[index] = {generation = index.generation, stamp = stamp, files = files}
|
|
25
|
+
return stamp, files
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
function M.current(path, stamp)
|
|
29
|
+
local file = io.open(path, 'rb')
|
|
30
|
+
if not file then return false end
|
|
31
|
+
local expected = M.prefix(M.version) .. stamp
|
|
32
|
+
local prefix = file:read(#expected)
|
|
33
|
+
file:close()
|
|
34
|
+
return prefix == expected
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
-- The background reader validates the disk snapshot against the foreground
|
|
38
|
+
-- index's identity without loading a second copy of its base records.
|
|
39
|
+
function M.reader(root, stamp)
|
|
40
|
+
local index = {root = root}
|
|
41
|
+
cache[index] = {stamp = stamp}
|
|
42
|
+
return index
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
return M
|
|
@@ -22,7 +22,7 @@ M.CHUNK_FILES = 32
|
|
|
22
22
|
M.CHUNK_BYTES = 1048576
|
|
23
23
|
M.IDLE_MS = 60000
|
|
24
24
|
|
|
25
|
-
local threads, owner, shared = {}, nil, nil
|
|
25
|
+
local threads, owner, shared, abort_key = {}, nil, nil, nil
|
|
26
26
|
local job, waiting, token = nil, false, 0
|
|
27
27
|
local script_path, thread_count
|
|
28
28
|
local outstanding, running, idle_since = {}, false, nil
|
|
@@ -36,9 +36,15 @@ function M.configure(script, count)
|
|
|
36
36
|
script_path, thread_count = script, count or M.THREADS
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
+
function M.configured_threads()
|
|
40
|
+
local count = tonumber(os.getenv('CODEOUTLINE_INDEX_THREADS') or '') or M.THREADS
|
|
41
|
+
return math.max(0, math.floor(count))
|
|
42
|
+
end
|
|
43
|
+
|
|
39
44
|
local function start_threads()
|
|
40
45
|
shared = assert(xshared.dict('codeoutline_control'))
|
|
41
46
|
owner = xthread.current_id()
|
|
47
|
+
abort_key = 'parse_abort:' .. owner
|
|
42
48
|
for i = 1, thread_count do
|
|
43
49
|
local id = M.FIRST_ID + i - 1
|
|
44
50
|
local ok, err = xthread.create_thread(id, 'PARSE' .. i, script_path)
|
|
@@ -51,7 +57,7 @@ end
|
|
|
51
57
|
|
|
52
58
|
local function stop_threads(reason)
|
|
53
59
|
local started, count, stopped = xtimer.now_ms(), #threads, 0
|
|
54
|
-
if shared then shared:set(
|
|
60
|
+
if shared then shared:set(abort_key, token) end
|
|
55
61
|
for _, id in ipairs(threads) do
|
|
56
62
|
local ok, err = xthread.shutdown_thread(id)
|
|
57
63
|
if ok then stopped = stopped + 1
|
|
@@ -129,7 +135,7 @@ M.partition = partition
|
|
|
129
135
|
|
|
130
136
|
-- jobs: { rel, abs, weight, size?, crc? }; on_result(rel, load_file result)
|
|
131
137
|
-- runs in this thread, in arrival order. Must be called from the bound job.
|
|
132
|
-
function M.run(jobs, max_bytes, on_result)
|
|
138
|
+
function M.run(jobs, max_bytes, on_result, operation)
|
|
133
139
|
assert(M.ready(), 'parse pool is not available here')
|
|
134
140
|
token = token + 1
|
|
135
141
|
local current, pending = token, 0
|
|
@@ -142,14 +148,14 @@ function M.run(jobs, max_bytes, on_result)
|
|
|
142
148
|
local chunk, bytes = {}, 0
|
|
143
149
|
local function flush()
|
|
144
150
|
if #chunk == 0 then return end
|
|
145
|
-
local posted, why = xthread.post(threads[i], 'parse', owner, current, max_bytes, chunk)
|
|
151
|
+
local posted, why = xthread.post(threads[i], operation or 'parse', owner, current, max_bytes, chunk)
|
|
146
152
|
assert(posted, 'cannot post to parse thread: ' .. tostring(why))
|
|
147
153
|
pending = pending + 1
|
|
148
154
|
outstanding[current] = (outstanding[current] or 0) + 1
|
|
149
155
|
chunk, bytes = {}, 0
|
|
150
156
|
end
|
|
151
157
|
for _, j in ipairs(bucket) do
|
|
152
|
-
chunk[#chunk + 1] = { j.rel, j.abs, j.size or false, j.crc or false, j.language or false }
|
|
158
|
+
chunk[#chunk + 1] = { j.rel, j.abs, j.size or false, j.crc or false, j.language or false, j.documents }
|
|
153
159
|
bytes = bytes + j.weight
|
|
154
160
|
if #chunk >= M.CHUNK_FILES or bytes >= M.CHUNK_BYTES then flush() end
|
|
155
161
|
end
|
|
@@ -181,7 +187,7 @@ function M.run(jobs, max_bytes, on_result)
|
|
|
181
187
|
xtimer.now_ms() - started)
|
|
182
188
|
if not ok then
|
|
183
189
|
-- Workers skip the rest of this pass; late results carry a stale token.
|
|
184
|
-
shared:set(
|
|
190
|
+
shared:set(abort_key, current)
|
|
185
191
|
error(err, 0)
|
|
186
192
|
end
|
|
187
193
|
end
|
|
@@ -50,7 +50,7 @@ if #config.roots == 0 then
|
|
|
50
50
|
end
|
|
51
51
|
local shared = assert(xshared.create('codeoutline_control', 65536, 256))
|
|
52
52
|
local jobs, next_id, job_count, worker_ready = {}, 0, 0, false
|
|
53
|
-
local stopping, indexing = false, false
|
|
53
|
+
local stopping, indexing, building_bm25 = false, false, false
|
|
54
54
|
local endpoint, stdio, input = nil, nil, ''
|
|
55
55
|
local updating = false
|
|
56
56
|
local startup = xtimer.now_ms()
|
|
@@ -129,6 +129,9 @@ return {
|
|
|
129
129
|
end
|
|
130
130
|
os.remove(daemon.stop_file(config.port))
|
|
131
131
|
end
|
|
132
|
+
local bm25 = require('codeoutline.bm25.automatic')
|
|
133
|
+
building_bm25 = xthread.create_thread(bm25.worker_id, 'BM25', scripts .. '/codeoutline/runtime/bm25_worker.lua')
|
|
134
|
+
shared:set('bm25_worker', building_bm25 and true or false)
|
|
132
135
|
assert(xthread.create_thread(2, 'INDEX', scripts .. '/codeoutline/runtime/index_worker.lua'))
|
|
133
136
|
indexing = true
|
|
134
137
|
-- Launchers enable this for serve; the background service updates unless
|
|
@@ -264,6 +267,7 @@ return {
|
|
|
264
267
|
shared:set('shutdown', true)
|
|
265
268
|
if stdio then stdio:close() end
|
|
266
269
|
if endpoint then endpoint.close() end
|
|
270
|
+
if building_bm25 then xthread.shutdown_thread(require('codeoutline.bm25.automatic').worker_id) end
|
|
267
271
|
if indexing then xthread.shutdown_thread(2) end
|
|
268
272
|
if updating then xthread.shutdown_thread(3) end
|
|
269
273
|
xnet.uninit()
|