@codua/codeoutline-win32-x64 0.3.8 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +49 -0
  2. package/README.zh-CN.md +37 -0
  3. package/bin/xnet.exe +0 -0
  4. package/build-info.json +1 -1
  5. package/editors/codeoutline.vsix +0 -0
  6. package/editors/zed/extension.wasm +0 -0
  7. package/keys/update-ca.crt +18 -18
  8. package/package.json +1 -1
  9. package/scripts/codeoutline/analysis/graph.lua +7 -16
  10. package/scripts/codeoutline/analysis/indirect.lua +18 -12
  11. package/scripts/codeoutline/analysis/lua/lua_bindings.lua +28 -0
  12. package/scripts/codeoutline/bm25/automatic.lua +100 -0
  13. package/scripts/codeoutline/bm25/collect.lua +148 -0
  14. package/scripts/codeoutline/bm25/index.lua +548 -0
  15. package/scripts/codeoutline/bm25/snapshot.lua +45 -0
  16. package/scripts/codeoutline/command.lua +3 -3
  17. package/scripts/codeoutline/indexing/parse_pool.lua +12 -6
  18. package/scripts/codeoutline/main.lua +5 -1
  19. package/scripts/codeoutline/protocol/lsp.lua +3 -1
  20. package/scripts/codeoutline/protocol/mcp.lua +5 -2
  21. package/scripts/codeoutline/query/coverage.lua +2 -0
  22. package/scripts/codeoutline/query/definition_candidates.lua +36 -12
  23. package/scripts/codeoutline/query/explore.lua +110 -18
  24. package/scripts/codeoutline/query/navigation.lua +213 -26
  25. package/scripts/codeoutline/query/query_filters.lua +5 -1
  26. package/scripts/codeoutline/runtime/bm25_worker.lua +86 -0
  27. package/scripts/codeoutline/runtime/index_worker.lua +18 -5
  28. package/scripts/codeoutline/runtime/parse_worker.lua +10 -3
  29. package/scripts/codeoutline/service.lua +6 -0
  30. package/scripts/codeoutline/shared/rules.lua +9 -1
  31. package/scripts/codeoutline/update_sequence.lua +1 -1
  32. package/scripts/codeoutline/version.lua +1 -1
@@ -0,0 +1,148 @@
1
+ -- Internal lexical document extraction. Base index records stay unchanged.
2
+ local control = require('codeoutline.shared.control')
3
+ local M = { weights = {3, 2, 1, 1} }
4
+
5
+ local stop_words = {}
6
+ for word in ([[
7
+ a an and are as at be by can could do does during for from had has have how
8
+ i if in is it of on or that the their them this to was were when where which
9
+ who will with would you your
10
+ ]]):gmatch('%S+') do
11
+ stop_words[word] = true
12
+ end
13
+
14
+ local function scan_words(text, visit)
15
+ text = text:gsub('([A-Z])([A-Z][a-z])', '%1 %2')
16
+ text = text:gsub('([a-z0-9])([A-Z])', '%1 %2'):lower()
17
+ local ordinal = 0
18
+ for group in text:gmatch('[a-z0-9]+') do
19
+ local pos = 1
20
+ while pos <= #group do
21
+ local token = group:match('^[a-z]+', pos) or group:match('^[0-9]+', pos)
22
+ pos = pos + #token
23
+ ordinal = ordinal + 1
24
+ if not stop_words[token] then
25
+ local ending = token:sub(-2)
26
+ if #token > 4 and token:sub(-1) == 's'
27
+ and ending ~= 'ss' and ending ~= 'us' and ending ~= 'is' then
28
+ token = token:sub(1, -2)
29
+ end
30
+ if visit(token, ordinal) == false then return end
31
+ end
32
+ end
33
+ end
34
+ end
35
+
36
+ function M.tokenize(text)
37
+ local counts, length = {}, 0
38
+ scan_words(text, function(word)
39
+ counts[word] = (counts[word] or 0) + 1
40
+ length = length + 1
41
+ end)
42
+ return counts, length
43
+ end
44
+
45
+ -- Smallest window containing every query term, regardless of order.
46
+ -- Positions count stop words so that skipping them does not shrink a gap.
47
+ function M.gap(text, query, needed)
48
+ local words, positions, counts = {}, {}, {}
49
+ local left, distinct, best = 1, 0, nil
50
+ scan_words(text, function(word, position)
51
+ if not query[word] then return end
52
+ local right = #words + 1
53
+ words[right], positions[right] = word, position
54
+ if not counts[word] or counts[word] == 0 then distinct = distinct + 1 end
55
+ counts[word] = (counts[word] or 0) + 1
56
+ while distinct == needed do
57
+ local gap = position - positions[left] + 1 - needed
58
+ if not best or gap < best then best = gap end
59
+ if best == 0 then return false end
60
+ local first = words[left]
61
+ counts[first] = counts[first] - 1
62
+ if counts[first] == 0 then distinct = distinct - 1 end
63
+ left = left + 1
64
+ end
65
+ end)
66
+ return best
67
+ end
68
+
69
+ local function line_offsets(source)
70
+ local offsets, pos = {1}, 1
71
+ while true do
72
+ local eol = source:find('\n', pos, true)
73
+ if not eol then return offsets end
74
+ pos = eol + 1
75
+ offsets[#offsets + 1] = pos
76
+ end
77
+ end
78
+
79
+ -- Scan only the lexical gap above the declaration. The tokenizer excludes
80
+ -- comments but keeps string spans, including Lua long strings.
81
+ local function leading_lua_doc(tokens, start, previous_end)
82
+ local gap = tokens.src:sub(previous_end + 1, start - 1)
83
+ if not gap:find('--', 1, true) then return '' end
84
+ local comments, pos, standalone = {}, 1, previous_end == 0
85
+ while pos <= #gap do
86
+ local space = gap:match('^%s*', pos)
87
+ pos = pos + #space
88
+ local _, breaks = space:gsub('\n', '')
89
+ if breaks > 1 then comments = {} end
90
+ if breaks > 0 then standalone = true end
91
+ if pos > #gap then break end
92
+ if gap:sub(pos, pos + 1) ~= '--' then return '' end
93
+ local _, open_end, equals = gap:find('^%-%-%[(=*)%[', pos)
94
+ local first, last
95
+ if open_end then
96
+ local close_start, close_end = gap:find(']' .. equals .. ']', open_end + 1, true)
97
+ if not close_start then return '' end
98
+ first, last, pos = open_end + 1, close_start - 1, close_end + 1
99
+ else
100
+ local eol = gap:find('\n', pos, true) or #gap + 1
101
+ first, last, pos = pos + 2, eol - 1, eol
102
+ end
103
+ if standalone then comments[#comments + 1] = gap:sub(first, last) end
104
+ standalone = false
105
+ end
106
+ return table.concat(comments, '\n'):sub(1, 512)
107
+ end
108
+
109
+ function M.file(abs, rel, size, crc, language, documents)
110
+ control.check()
111
+ local file = assert(io.open(abs, 'rb'))
112
+ local raw = file:read(size + 1) or ''
113
+ assert(file:close())
114
+ local actual = xcompress and xcompress.crc32 and xcompress.crc32(raw) or xutils.sha256_hex(raw)
115
+ assert(#raw == size and actual == crc, 'source changed during BM25 build: ' .. rel)
116
+ -- The tokenizer only indexes ASCII. Retaining raw bytes if decoding is
117
+ -- unavailable keeps ASCII terms and the base index's line coordinates.
118
+ local source = xutils.to_utf8(raw) or raw
119
+ if source:sub(1, 3) == '\239\187\191' then source = source:sub(4) end
120
+ local offsets = line_offsets(source)
121
+ local tokens = language == 'lua' and require('codeoutline.lang.lua').lang:tokenize(source)
122
+ local totals, encoded, comment_count = {0, 0, 0, 0}, {}, 0
123
+ local token = 1
124
+ for i, document in ipairs(documents) do
125
+ if i % 128 == 0 then control.check() end
126
+ local id, name, line, end_line = table.unpack(document)
127
+ local first = assert(offsets[line], 'invalid BM25 source line: ' .. rel)
128
+ local last = math.min((offsets[end_line + 1] or #source + 1) - 1,
129
+ (offsets[line + 40] or #source + 1) - 1, first + 2047)
130
+ local doc = ''
131
+ if tokens then
132
+ while token <= tokens.n and tokens.s[token] < first do token = token + 1 end
133
+ doc = leading_lua_doc(tokens, first, tokens.e[token - 1] or 0)
134
+ if doc:find('%S') then comment_count = comment_count + 1 end
135
+ end
136
+ local fields = {}
137
+ local body = source:sub(first, last)
138
+ for field, value in ipairs({name, rel, body, doc}) do
139
+ local terms, length = M.tokenize(value)
140
+ fields[field] = {terms, length}
141
+ totals[field] = totals[field] + length
142
+ end
143
+ encoded[i] = {id, fields, body, doc}
144
+ end
145
+ return {status = 'parsed', terms = cmsgpack.pack(encoded), totals = totals, comments = comment_count}
146
+ end
147
+
148
+ return M
@@ -0,0 +1,548 @@
1
+ -- Public API: build(index, file) and search(index, file, query, options).
2
+ -- Pass a refreshed CodeOutline index. The file is a lexical snapshot of that
3
+ -- project's source records and must be rebuilt when those records change.
4
+ local indexing = require('codeoutline.indexing.index')
5
+ local pool = require('codeoutline.indexing.parse_pool')
6
+ local collect = require('codeoutline.bm25.collect')
7
+ local control = require('codeoutline.shared.control')
8
+ local paths = require('codeoutline.shared.path')
9
+ local text_util = require('codeoutline.shared.text')
10
+ local snapshot = require('codeoutline.bm25.snapshot')
11
+
12
+ local M = {}
13
+ local TRAILER_SIZE = 80
14
+ local PASSAGE_BLOCK = 32
15
+ local GC_BATCH_BYTES = 512 * 1024
16
+ local GC_STEP_KB = 512
17
+ local cached_digest, cached_header
18
+
19
+ -- Drive GC by completed work so temporary batches can be reclaimed while the
20
+ -- live postings grow. Keep the collector settings of the calling Lua state.
21
+ local function build_gc()
22
+ local processed = 0
23
+ return function(bytes)
24
+ processed = processed + bytes
25
+ if processed >= GC_BATCH_BYTES then
26
+ processed = 0
27
+ collectgarbage('step', GC_STEP_KB)
28
+ end
29
+ end
30
+ end
31
+
32
+ local function plan(index, source_paths)
33
+ local jobs, files, locations = {}, {}, {}
34
+ for _, path in ipairs(source_paths) do
35
+ control.check()
36
+ local file = index.files[path]
37
+ local documents = {}
38
+ for _, node in ipairs(file.nodes) do
39
+ if node.kind == 'function' or node.kind == 'method' then
40
+ documents[#documents + 1] = {false, text_util.valid(node.qualified or node.name),
41
+ node.line, math.max(node.line, node.end_line or node.line), node.kind}
42
+ end
43
+ end
44
+ if #documents > 0 then
45
+ table.sort(documents, function(a, b)
46
+ if a[3] ~= b[3] then return a[3] < b[3] end
47
+ if a[2] ~= b[2] then return a[2] < b[2] end
48
+ if a[4] ~= b[4] then return a[4] < b[4] end
49
+ return a[5] < b[5]
50
+ end)
51
+ local file_id = #files + 1
52
+ files[file_id] = path
53
+ for _, doc in ipairs(documents) do
54
+ doc[1] = #locations + 1
55
+ locations[doc[1]] = {file_id, doc[2], doc[3], doc[4], doc[5]}
56
+ end
57
+ jobs[#jobs + 1] = {rel = path, abs = index:abs(path), size = file.size, crc = file.crc,
58
+ language = file.language, documents = documents, weight = file.size + indexing.FILE_WEIGHT}
59
+ end
60
+ end
61
+ return jobs, files, locations
62
+ end
63
+
64
+ local function collect_terms(index, jobs, spool, gc_step)
65
+ local totals, lengths, comments = {0, 0, 0, 0}, {}, 0
66
+ local function accept(_, result)
67
+ local encoded = result.terms
68
+ assert(spool:write(encoded))
69
+ lengths[#lengths + 1] = #encoded
70
+ comments = comments + result.comments
71
+ for i, length in ipairs(result.totals) do totals[i] = totals[i] + length end
72
+ gc_step(#encoded)
73
+ end
74
+ if #jobs >= indexing.PARALLEL_MIN_FILES and pool.ready() then
75
+ pool.run(jobs, index.max_bytes, accept, 'bm25_collect')
76
+ else
77
+ for _, entry in ipairs(jobs) do
78
+ accept(entry.rel, collect.file(entry.abs, entry.rel, entry.size, entry.crc, entry.language, entry.documents))
79
+ end
80
+ end
81
+ assert(#lengths == #jobs, 'incomplete BM25 source pass')
82
+ return totals, lengths, comments
83
+ end
84
+
85
+ local function read_at(file, offset, length)
86
+ assert(file:seek('set', offset) == offset, 'cannot seek BM25 index')
87
+ local data = assert(file:read(length), 'truncated BM25 index')
88
+ assert(#data == length, 'truncated BM25 index')
89
+ return data
90
+ end
91
+
92
+ local function merge_terms(spool, totals, lengths, count, write_passages, gc_step)
93
+ local postings, averages = {}, {}
94
+ for i, total in ipairs(totals) do averages[i] = count > 0 and total / count or 0 end
95
+ assert(spool:seek('set', 0) == 0)
96
+ for _, length in ipairs(lengths) do
97
+ control.check()
98
+ gc_step(length)
99
+ local data = assert(spool:read(length), 'truncated BM25 staging file')
100
+ assert(#data == length, 'truncated BM25 staging file')
101
+ local batch = cmsgpack.unpack(data)
102
+ if #batch > 0 then write_passages(batch) end
103
+ for _, doc in ipairs(batch) do
104
+ local id, fields = table.unpack(doc)
105
+ for i, field in ipairs(fields) do
106
+ if averages[i] > 0 then
107
+ local terms, length = table.unpack(field)
108
+ local normalization = 1.2 * (0.25 + 0.75 * length / averages[i])
109
+ local weight = collect.weights[i]
110
+ for word, frequency in pairs(terms) do
111
+ local matches = postings[word]
112
+ if not matches then matches = {}; postings[word] = matches end
113
+ local tf = weight * 2.2 * frequency / (frequency + normalization)
114
+ matches[id] = (matches[id] or 0) + tf
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
120
+ return postings
121
+ end
122
+
123
+ local function exact_keys(files, locations)
124
+ local exact = {}
125
+ for id, location in ipairs(locations) do
126
+ if id % 128 == 0 then control.check() end
127
+ local path = files[location[1]]:lower()
128
+ local name = location[2]:lower()
129
+ for _, key in ipairs({path, path:match('[^/]+$'), name, name:match('[^.:]+$') or name}) do
130
+ local matches = exact[key]
131
+ if not matches then matches = {}; exact[key] = matches end
132
+ matches[id] = true
133
+ end
134
+ end
135
+ return exact
136
+ end
137
+
138
+ local function write_index(output, spool, totals, lengths, header, gc_step)
139
+ local prefix = snapshot.prefix(header.version) .. header.stamp
140
+ assert(output:write(prefix))
141
+ local offset, count = #prefix, #header.locations
142
+ local function write_data(data)
143
+ local location = {offset, #data, xcompress.crc32(data)}
144
+ assert(output:write(data))
145
+ offset = offset + #data
146
+ return location
147
+ end
148
+ local function write_block(matches)
149
+ return write_data(cmsgpack.pack(matches))
150
+ end
151
+ local compressor = xcompress.new_compressor(1)
152
+ local function write_passages(batch)
153
+ local first_id = batch[1][1]
154
+ local blocks = {first_id}
155
+ header.passages[header.locations[first_id][1]] = blocks
156
+ for start = 1, #batch, PASSAGE_BLOCK do
157
+ control.check()
158
+ local passages = {}
159
+ for i = start, math.min(start + PASSAGE_BLOCK - 1, #batch) do
160
+ passages[#passages + 1] = {batch[i][3], batch[i][4]}
161
+ end
162
+ local data = cmsgpack.pack(passages)
163
+ local entry = write_data(assert(compressor:deflate(data)))
164
+ entry[4] = #data
165
+ blocks[#blocks + 1] = entry
166
+ end
167
+ end
168
+ local postings = merge_terms(spool, totals, lengths, count, write_passages, gc_step)
169
+ compressor:close()
170
+ local records = 0
171
+ for word, matches in pairs(postings) do
172
+ control.check()
173
+ local frequency = 0
174
+ for _ in pairs(matches) do frequency = frequency + 1 end
175
+ local entry = write_block(matches)
176
+ entry[4] = math.log(1 + (count - frequency + 0.5) / (frequency + 0.5))
177
+ header.words[word] = entry
178
+ records = records + frequency
179
+ postings[word] = nil
180
+ matches = nil
181
+ gc_step(entry[2])
182
+ end
183
+ postings = nil
184
+ local exact = exact_keys(header.files, header.locations)
185
+ for key, matches in pairs(exact) do
186
+ control.check()
187
+ local entry = write_block(matches)
188
+ header.exact[key] = entry
189
+ exact[key] = nil
190
+ matches = nil
191
+ gc_step(entry[2])
192
+ end
193
+ exact = nil
194
+ local encoded = cmsgpack.pack(header)
195
+ assert(output:write(encoded, xutils.sha256_hex(encoded), string.format('%016d', #encoded)))
196
+ return records, offset + #encoded + TRAILER_SIZE
197
+ end
198
+
199
+ local function output_path(file)
200
+ assert(type(file) == 'string' and file ~= '' and not file:find('%z'), 'BM25 index file is required')
201
+ return paths.normalize(file)
202
+ end
203
+
204
+ local function check_destination(file)
205
+ local existing = io.open(file, 'rb')
206
+ if not existing then return end
207
+ local magic = existing:read(#snapshot.magic)
208
+ existing:close()
209
+ assert(magic == snapshot.magic, 'refusing to replace a non-BM25 file: ' .. file)
210
+ end
211
+
212
+ function M.build(index, file)
213
+ file = output_path(file)
214
+ assert(not index.cache_path or paths.key(file) ~= paths.key(index.cache_path),
215
+ 'BM25 must use a separate file from the base index')
216
+ check_destination(file)
217
+ local parent = file:match('^(.*)/[^/]+$') or '.'
218
+ if parent == '' or parent:match('^%a:$') then parent = parent .. '/' end
219
+ assert(xutils.mkdir_p(parent))
220
+ local generation = index.generation
221
+ local stamp, source_paths = snapshot.stamp(index)
222
+ local jobs, files, locations = plan(index, source_paths)
223
+ local spool_name, staged, spool, output
224
+ local ok, result = pcall(function()
225
+ spool_name = assert(xutils.temp_file(parent))
226
+ spool = assert(io.open(spool_name, 'w+b'))
227
+ local gc_step = build_gc()
228
+ local totals, lengths, comments = collect_terms(index, jobs, spool, gc_step)
229
+ jobs = nil
230
+ staged = assert(xutils.temp_file(parent))
231
+ output = assert(io.open(staged, 'wb'))
232
+ local records, bytes = write_index(output, spool, totals, lengths, {version = snapshot.version,
233
+ stamp = stamp, files = files, locations = locations, words = {}, exact = {}, passages = {}}, gc_step)
234
+ assert(spool:close()); spool = nil
235
+ assert(output:close()); output = nil
236
+ control.check()
237
+ assert(index.generation == generation, 'base index changed during BM25 build')
238
+ check_destination(file)
239
+ assert(xutils.replace_file(staged, file))
240
+ staged = nil
241
+ return {files = #files, functions = #locations, comments = comments, postings = records, bytes = bytes}
242
+ end)
243
+ if spool then spool:close() end
244
+ if output then output:close() end
245
+ if spool_name then os.remove(spool_name) end
246
+ if staged then os.remove(staged) end
247
+ if not ok then error(result, 0) end
248
+ return result
249
+ end
250
+
251
+ local function open_header(file)
252
+ local size = assert(file:seek('end'))
253
+ local magic = snapshot.magic
254
+ assert(size > #magic + TRAILER_SIZE and read_at(file, 0, #magic) == magic, 'invalid BM25 index')
255
+ local trailer = read_at(file, size - TRAILER_SIZE, TRAILER_SIZE)
256
+ local digest, length = trailer:sub(1, 64), tonumber(trailer:sub(65))
257
+ assert(length and length % 1 == 0 and length > 0 and length <= size - #magic - TRAILER_SIZE,
258
+ 'invalid BM25 header length')
259
+ local offset = size - TRAILER_SIZE - length
260
+ if digest ~= cached_digest then
261
+ local encoded = read_at(file, offset, length)
262
+ assert(xutils.sha256_hex(encoded) == digest, 'corrupt BM25 header')
263
+ local header = cmsgpack.unpack(encoded)
264
+ assert(header.version == 1 or header.version == 2 or header.version == snapshot.version,
265
+ 'unsupported BM25 index version; rebuild it')
266
+ cached_digest, cached_header = digest, header
267
+ end
268
+ local first = #magic
269
+ if cached_header.version >= 2 then
270
+ local prefix = snapshot.prefix(cached_header.version) .. cached_header.stamp
271
+ assert(offset >= #prefix and read_at(file, 0, #prefix) == prefix, 'corrupt BM25 snapshot prefix')
272
+ first = #prefix
273
+ end
274
+ return cached_header, first, offset
275
+ end
276
+
277
+ local function read_data(file, entry, first, boundary)
278
+ local offset, length = entry[1], entry[2]
279
+ assert(offset >= first and length > 0 and offset + length <= boundary, 'invalid BM25 block range')
280
+ local encoded = read_at(file, offset, length)
281
+ assert(xcompress.crc32(encoded) == entry[3], 'corrupt BM25 postings')
282
+ return encoded
283
+ end
284
+
285
+ local function read_block(file, entry, first, boundary)
286
+ return cmsgpack.unpack(read_data(file, entry, first, boundary))
287
+ end
288
+
289
+ local function scope_key(index, path)
290
+ if path == nil or path == '' or path == '.' then return nil end
291
+ assert(type(path) == 'string', 'BM25 path filter must be a string')
292
+ path = paths.key(path):gsub('^%./', '')
293
+ if path:match('^/') or path:match('^%a:/') then
294
+ local root = paths.key(index.root)
295
+ assert(paths.contains(root, path), 'BM25 path filter is outside the project')
296
+ if root == path then return nil end
297
+ return path:sub(#root + (root:sub(-1) == '/' and 1 or 2))
298
+ end
299
+ return path
300
+ end
301
+
302
+ local function rank(scores, exact, literal, limit)
303
+ local function better(a, b)
304
+ if exact[a] ~= exact[b] then return exact[a] == true end
305
+ if literal[a] ~= literal[b] then return literal[a] == true end
306
+ if scores[a] ~= scores[b] then return scores[a] > scores[b] end
307
+ return a < b
308
+ end
309
+ -- The root of the bounded heap is the worst retained candidate.
310
+ local heap, visited = {}, 0
311
+ for id in pairs(scores) do
312
+ visited = visited + 1
313
+ if visited % 1024 == 0 then control.check() end
314
+ if #heap < limit then
315
+ local i = #heap + 1
316
+ while i > 1 do
317
+ local parent = math.floor(i / 2)
318
+ if not better(heap[parent], id) then break end
319
+ heap[i], i = heap[parent], parent
320
+ end
321
+ heap[i] = id
322
+ elseif better(id, heap[1]) then
323
+ local i, child = 1, 2
324
+ while child <= #heap do
325
+ if child < #heap and better(heap[child], heap[child + 1]) then child = child + 1 end
326
+ if not better(id, heap[child]) then break end
327
+ heap[i], i = heap[child], child
328
+ child = i * 2
329
+ end
330
+ heap[i] = id
331
+ end
332
+ end
333
+ table.sort(heap, better)
334
+ return heap
335
+ end
336
+
337
+ local function score_terms(file, entries, words, first, boundary, accepts, match)
338
+ local scores = {}
339
+ local counts = match ~= 'all' and #words > 1 and {} or nil
340
+ local rarest, seed
341
+ if match == 'all' and #words > 1 then
342
+ for _, word in ipairs(words) do
343
+ local entry = entries[word]
344
+ if not entry then return scores end
345
+ -- IDF decreases as the number of documents for a word increases.
346
+ if not rarest or entry[4] > entries[rarest][4] then rarest = word end
347
+ end
348
+ seed = read_block(file, entries[rarest], first, boundary)
349
+ local scanned = 0
350
+ for id in pairs(seed) do
351
+ scanned = scanned + 1
352
+ if scanned % 1024 == 0 then control.check() end
353
+ if accepts(id) then scores[id] = 0 end
354
+ end
355
+ end
356
+ -- Accumulate terms in the same order for union and intersection searches,
357
+ -- so a surviving document keeps the same BM25 score and tie-breaking.
358
+ for _, word in ipairs(words) do
359
+ control.check()
360
+ if rarest and not next(scores) then break end
361
+ local entry = entries[word]
362
+ if entry then
363
+ local matches = word == rarest and seed or read_block(file, entry, first, boundary)
364
+ local scanned = 0
365
+ if rarest then
366
+ for id, score in pairs(scores) do
367
+ scanned = scanned + 1
368
+ if scanned % 1024 == 0 then control.check() end
369
+ local tf = matches[id]
370
+ scores[id] = tf and (score + entry[4] * tf) or nil
371
+ end
372
+ if word == rarest then seed = nil end
373
+ else
374
+ for id, tf in pairs(matches) do
375
+ scanned = scanned + 1
376
+ if scanned % 1024 == 0 then control.check() end
377
+ if accepts(id) then
378
+ scores[id] = (scores[id] or 0) + entry[4] * tf
379
+ if counts then counts[id] = (counts[id] or 0) + 1 end
380
+ end
381
+ end
382
+ end
383
+ end
384
+ end
385
+ return scores, counts
386
+ end
387
+
388
+ local function contains_literal(text, literal, left_boundary, right_boundary)
389
+ if text == '' then return false end
390
+ text = text:lower()
391
+ local offset = 1
392
+ while true do
393
+ local first, last = text:find(literal, offset, true)
394
+ if not first then return false end
395
+ if (not left_boundary or first == 1 or not text:sub(first - 1, first - 1):match('[%w_$]'))
396
+ and (not right_boundary or not text:sub(last + 1, last + 1):match('[%w_$]')) then
397
+ return true
398
+ end
399
+ offset = first + 1
400
+ end
401
+ end
402
+
403
+ local function boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
404
+ local exact = {}
405
+ if not header.passages or #words == 0 or literal == '' or not next(scores) then return exact end
406
+ local strength = 0
407
+ for _, word in ipairs(words) do
408
+ local entry = header.words[word]
409
+ if not entry then return exact end
410
+ strength = strength + entry[4]
411
+ end
412
+ -- Group by block so each compressed passage is read at most once per query.
413
+ local groups, scanned = {}, 0
414
+ for id in pairs(scores) do
415
+ scanned = scanned + 1
416
+ if scanned % 128 == 0 then control.check() end
417
+ if not counts or counts[id] == #words then
418
+ local blocks = header.passages[header.locations[id][1]]
419
+ local offset = id - blocks[1]
420
+ local entry = blocks[2 + math.floor(offset / PASSAGE_BLOCK)]
421
+ local group = groups[entry]
422
+ if not group then group = {}; groups[entry] = group end
423
+ group[#group + 1] = id
424
+ end
425
+ end
426
+ if not next(groups) then return exact end
427
+ local decoder = xcompress.new_decompressor()
428
+ local left_boundary, right_boundary = literal:sub(1, 1):match('[%w_$]'), literal:sub(-1):match('[%w_$]')
429
+ for entry, ids in pairs(groups) do
430
+ control.check()
431
+ local data = read_data(file, entry, first, boundary)
432
+ local passages = cmsgpack.unpack(assert(decoder:deflate(data, entry[4]), 'corrupt BM25 passages'))
433
+ for _, id in ipairs(ids) do
434
+ local location = header.locations[id]
435
+ local offset = id - header.passages[location[1]][1]
436
+ local passage = passages[1 + offset % PASSAGE_BLOCK]
437
+ local best
438
+ for _, text in ipairs({location[2], header.files[location[1]], passage[1], passage[2]}) do
439
+ if not exact[id] and contains_literal(text, literal, left_boundary, right_boundary) then exact[id] = true end
440
+ if #words > 1 and best ~= 0 and text ~= '' then
441
+ local gap = collect.gap(text, terms, #words)
442
+ if gap and (not best or gap < best) then best = gap end
443
+ end
444
+ end
445
+ if best then scores[id] = scores[id] + strength / (1 + best) end
446
+ end
447
+ end
448
+ decoder:close()
449
+ return exact
450
+ end
451
+
452
+ -- Use the existing symbol and path tables. Prefix hints change ranking only.
453
+ local function boost_prefix(header, scores, words, prefix)
454
+ if not prefix or prefix == '' then return end
455
+ prefix = prefix:lower()
456
+ local strength = 0
457
+ for _, word in ipairs(words) do
458
+ local entry = header.words[word]
459
+ if entry then strength = strength + entry[4] end
460
+ end
461
+ strength = math.max(strength, 1)
462
+ local path_scores, scanned = {}, 0
463
+ local directory = '/' .. prefix .. '/'
464
+ for id, score in pairs(scores) do
465
+ scanned = scanned + 1
466
+ if scanned % 128 == 0 then control.check() end
467
+ local location = header.locations[id]
468
+ local file_id = location[1]
469
+ local weight = path_scores[file_id]
470
+ if not weight then
471
+ local path = header.files[file_id]:lower()
472
+ local stem = path:match('[^/]+$'):gsub('%.[^%.]+$', '')
473
+ if stem == prefix then weight = 4
474
+ elseif ('/' .. path):find(directory, 1, true) then weight = 3
475
+ elseif stem:sub(1, #prefix) == prefix and stem:sub(#prefix + 1, #prefix + 1):match('[_%.%-]') then
476
+ weight = 1
477
+ else weight = 0 end
478
+ path_scores[file_id] = weight
479
+ end
480
+ local owner = location[2]:match('([^.:]+)[.:]+[^.:]+$')
481
+ if owner and owner:lower() == prefix then weight = 5 end
482
+ if weight > 0 then scores[id] = score + weight * strength end
483
+ end
484
+ end
485
+
486
+ local function search_file(index, file, query, options, limit)
487
+ local header, first, boundary = open_header(file)
488
+ assert(header.stamp == snapshot.stamp(index), 'BM25 index is stale; rebuild it')
489
+ local scope, allowed = scope_key(index, options.path), {}
490
+ local function in_scope(id)
491
+ if not scope then return true end
492
+ local file_id = header.locations[id][1]
493
+ if allowed[file_id] == nil then allowed[file_id] = paths.contains(scope, header.files[file_id]) end
494
+ return allowed[file_id]
495
+ end
496
+ local accepts = in_scope
497
+ if options.accept then
498
+ local accepted = {}
499
+ accepts = function(id)
500
+ if accepted[id] == nil then
501
+ local location = header.locations[id]
502
+ accepted[id] = not not (in_scope(id)
503
+ and options.accept(header.files[location[1]], location[2], location[5]))
504
+ end
505
+ return accepted[id]
506
+ end
507
+ end
508
+ local words, terms = {}, collect.tokenize(query)
509
+ for token in pairs(terms) do words[#words + 1] = token end
510
+ table.sort(words)
511
+ local scores, counts = score_terms(file, header.words, words, first, boundary, accepts, options.match)
512
+ local mark = string.char(96)
513
+ local literal = query:match('^%s*(.-)%s*$'):gsub('^' .. mark .. '+', ''):gsub(mark .. '+$', ''):lower()
514
+ local key = literal:gsub('\\', '/')
515
+ local entry = header.exact[key]
516
+ local exact = entry and read_block(file, entry, first, boundary) or {}
517
+ local literal_matches = boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
518
+ if options.match ~= 'all' or #words == 0 then
519
+ for id in pairs(exact) do if accepts(id) then scores[id] = scores[id] or 0 end end
520
+ end
521
+ boost_prefix(header, scores, words, options.prefix)
522
+ local results = {}
523
+ for rank_number, id in ipairs(rank(scores, exact, literal_matches, limit)) do
524
+ local location = header.locations[id]
525
+ results[rank_number] = {path = header.files[location[1]], symbol = location[2],
526
+ start_line = location[3], end_line = location[4], score = scores[id], kind = location[5]}
527
+ end
528
+ return results
529
+ end
530
+
531
+ function M.search(index, file, query, options)
532
+ assert(type(query) == 'string', 'BM25 query must be a string')
533
+ options = options or {}
534
+ assert(options.match == nil or options.match == 'any' or options.match == 'all',
535
+ 'BM25 match must be any or all')
536
+ assert(options.prefix == nil or type(options.prefix) == 'string', 'BM25 prefix hint must be a string')
537
+ local limit = options.limit == nil and 20 or options.limit
538
+ assert(type(limit) == 'number' and limit >= 0 and limit < math.huge and limit % 1 == 0,
539
+ 'BM25 limit must be a nonnegative integer')
540
+ if limit == 0 then return {} end
541
+ local input = assert(io.open(output_path(file), 'rb'))
542
+ local ok, result = pcall(search_file, index, input, query, options, limit)
543
+ input:close()
544
+ if not ok then error(result, 0) end
545
+ return result
546
+ end
547
+
548
+ return M