@codua/codeoutline-win32-x64 0.3.8 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -0
- package/README.zh-CN.md +37 -0
- package/bin/xnet.exe +0 -0
- package/build-info.json +1 -1
- package/editors/codeoutline.vsix +0 -0
- package/editors/zed/extension.wasm +0 -0
- package/keys/update-ca.crt +18 -18
- package/package.json +1 -1
- package/scripts/codeoutline/analysis/graph.lua +7 -16
- package/scripts/codeoutline/analysis/indirect.lua +18 -12
- package/scripts/codeoutline/analysis/lua/lua_bindings.lua +28 -0
- package/scripts/codeoutline/bm25/automatic.lua +100 -0
- package/scripts/codeoutline/bm25/collect.lua +148 -0
- package/scripts/codeoutline/bm25/index.lua +548 -0
- package/scripts/codeoutline/bm25/snapshot.lua +45 -0
- package/scripts/codeoutline/command.lua +3 -3
- package/scripts/codeoutline/indexing/parse_pool.lua +12 -6
- package/scripts/codeoutline/main.lua +5 -1
- package/scripts/codeoutline/protocol/lsp.lua +3 -1
- package/scripts/codeoutline/protocol/mcp.lua +5 -2
- package/scripts/codeoutline/query/coverage.lua +2 -0
- package/scripts/codeoutline/query/definition_candidates.lua +36 -12
- package/scripts/codeoutline/query/explore.lua +110 -18
- package/scripts/codeoutline/query/navigation.lua +213 -26
- package/scripts/codeoutline/query/query_filters.lua +5 -1
- package/scripts/codeoutline/runtime/bm25_worker.lua +86 -0
- package/scripts/codeoutline/runtime/index_worker.lua +18 -5
- package/scripts/codeoutline/runtime/parse_worker.lua +10 -3
- package/scripts/codeoutline/service.lua +6 -0
- package/scripts/codeoutline/shared/rules.lua +9 -1
- package/scripts/codeoutline/update_sequence.lua +1 -1
- package/scripts/codeoutline/version.lua +1 -1
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
-- Internal lexical document extraction. Base index records stay unchanged.
|
|
2
|
+
local control = require('codeoutline.shared.control')
|
|
3
|
+
local M = { weights = {3, 2, 1, 1} }
|
|
4
|
+
|
|
5
|
+
local stop_words = {}
|
|
6
|
+
for word in ([[
|
|
7
|
+
a an and are as at be by can could do does during for from had has have how
|
|
8
|
+
i if in is it of on or that the their them this to was were when where which
|
|
9
|
+
who will with would you your
|
|
10
|
+
]]):gmatch('%S+') do
|
|
11
|
+
stop_words[word] = true
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
local function scan_words(text, visit)
|
|
15
|
+
text = text:gsub('([A-Z])([A-Z][a-z])', '%1 %2')
|
|
16
|
+
text = text:gsub('([a-z0-9])([A-Z])', '%1 %2'):lower()
|
|
17
|
+
local ordinal = 0
|
|
18
|
+
for group in text:gmatch('[a-z0-9]+') do
|
|
19
|
+
local pos = 1
|
|
20
|
+
while pos <= #group do
|
|
21
|
+
local token = group:match('^[a-z]+', pos) or group:match('^[0-9]+', pos)
|
|
22
|
+
pos = pos + #token
|
|
23
|
+
ordinal = ordinal + 1
|
|
24
|
+
if not stop_words[token] then
|
|
25
|
+
local ending = token:sub(-2)
|
|
26
|
+
if #token > 4 and token:sub(-1) == 's'
|
|
27
|
+
and ending ~= 'ss' and ending ~= 'us' and ending ~= 'is' then
|
|
28
|
+
token = token:sub(1, -2)
|
|
29
|
+
end
|
|
30
|
+
if visit(token, ordinal) == false then return end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
function M.tokenize(text)
|
|
37
|
+
local counts, length = {}, 0
|
|
38
|
+
scan_words(text, function(word)
|
|
39
|
+
counts[word] = (counts[word] or 0) + 1
|
|
40
|
+
length = length + 1
|
|
41
|
+
end)
|
|
42
|
+
return counts, length
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
-- Smallest window containing every query term, regardless of order.
|
|
46
|
+
-- Positions count stop words so that skipping them does not shrink a gap.
|
|
47
|
+
function M.gap(text, query, needed)
|
|
48
|
+
local words, positions, counts = {}, {}, {}
|
|
49
|
+
local left, distinct, best = 1, 0, nil
|
|
50
|
+
scan_words(text, function(word, position)
|
|
51
|
+
if not query[word] then return end
|
|
52
|
+
local right = #words + 1
|
|
53
|
+
words[right], positions[right] = word, position
|
|
54
|
+
if not counts[word] or counts[word] == 0 then distinct = distinct + 1 end
|
|
55
|
+
counts[word] = (counts[word] or 0) + 1
|
|
56
|
+
while distinct == needed do
|
|
57
|
+
local gap = position - positions[left] + 1 - needed
|
|
58
|
+
if not best or gap < best then best = gap end
|
|
59
|
+
if best == 0 then return false end
|
|
60
|
+
local first = words[left]
|
|
61
|
+
counts[first] = counts[first] - 1
|
|
62
|
+
if counts[first] == 0 then distinct = distinct - 1 end
|
|
63
|
+
left = left + 1
|
|
64
|
+
end
|
|
65
|
+
end)
|
|
66
|
+
return best
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
local function line_offsets(source)
|
|
70
|
+
local offsets, pos = {1}, 1
|
|
71
|
+
while true do
|
|
72
|
+
local eol = source:find('\n', pos, true)
|
|
73
|
+
if not eol then return offsets end
|
|
74
|
+
pos = eol + 1
|
|
75
|
+
offsets[#offsets + 1] = pos
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
-- Scan only the lexical gap above the declaration. The tokenizer excludes
|
|
80
|
+
-- comments but keeps string spans, including Lua long strings.
|
|
81
|
+
local function leading_lua_doc(tokens, start, previous_end)
|
|
82
|
+
local gap = tokens.src:sub(previous_end + 1, start - 1)
|
|
83
|
+
if not gap:find('--', 1, true) then return '' end
|
|
84
|
+
local comments, pos, standalone = {}, 1, previous_end == 0
|
|
85
|
+
while pos <= #gap do
|
|
86
|
+
local space = gap:match('^%s*', pos)
|
|
87
|
+
pos = pos + #space
|
|
88
|
+
local _, breaks = space:gsub('\n', '')
|
|
89
|
+
if breaks > 1 then comments = {} end
|
|
90
|
+
if breaks > 0 then standalone = true end
|
|
91
|
+
if pos > #gap then break end
|
|
92
|
+
if gap:sub(pos, pos + 1) ~= '--' then return '' end
|
|
93
|
+
local _, open_end, equals = gap:find('^%-%-%[(=*)%[', pos)
|
|
94
|
+
local first, last
|
|
95
|
+
if open_end then
|
|
96
|
+
local close_start, close_end = gap:find(']' .. equals .. ']', open_end + 1, true)
|
|
97
|
+
if not close_start then return '' end
|
|
98
|
+
first, last, pos = open_end + 1, close_start - 1, close_end + 1
|
|
99
|
+
else
|
|
100
|
+
local eol = gap:find('\n', pos, true) or #gap + 1
|
|
101
|
+
first, last, pos = pos + 2, eol - 1, eol
|
|
102
|
+
end
|
|
103
|
+
if standalone then comments[#comments + 1] = gap:sub(first, last) end
|
|
104
|
+
standalone = false
|
|
105
|
+
end
|
|
106
|
+
return table.concat(comments, '\n'):sub(1, 512)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
function M.file(abs, rel, size, crc, language, documents)
|
|
110
|
+
control.check()
|
|
111
|
+
local file = assert(io.open(abs, 'rb'))
|
|
112
|
+
local raw = file:read(size + 1) or ''
|
|
113
|
+
assert(file:close())
|
|
114
|
+
local actual = xcompress and xcompress.crc32 and xcompress.crc32(raw) or xutils.sha256_hex(raw)
|
|
115
|
+
assert(#raw == size and actual == crc, 'source changed during BM25 build: ' .. rel)
|
|
116
|
+
-- The tokenizer only indexes ASCII. Retaining raw bytes if decoding is
|
|
117
|
+
-- unavailable keeps ASCII terms and the base index's line coordinates.
|
|
118
|
+
local source = xutils.to_utf8(raw) or raw
|
|
119
|
+
if source:sub(1, 3) == '\239\187\191' then source = source:sub(4) end
|
|
120
|
+
local offsets = line_offsets(source)
|
|
121
|
+
local tokens = language == 'lua' and require('codeoutline.lang.lua').lang:tokenize(source)
|
|
122
|
+
local totals, encoded, comment_count = {0, 0, 0, 0}, {}, 0
|
|
123
|
+
local token = 1
|
|
124
|
+
for i, document in ipairs(documents) do
|
|
125
|
+
if i % 128 == 0 then control.check() end
|
|
126
|
+
local id, name, line, end_line = table.unpack(document)
|
|
127
|
+
local first = assert(offsets[line], 'invalid BM25 source line: ' .. rel)
|
|
128
|
+
local last = math.min((offsets[end_line + 1] or #source + 1) - 1,
|
|
129
|
+
(offsets[line + 40] or #source + 1) - 1, first + 2047)
|
|
130
|
+
local doc = ''
|
|
131
|
+
if tokens then
|
|
132
|
+
while token <= tokens.n and tokens.s[token] < first do token = token + 1 end
|
|
133
|
+
doc = leading_lua_doc(tokens, first, tokens.e[token - 1] or 0)
|
|
134
|
+
if doc:find('%S') then comment_count = comment_count + 1 end
|
|
135
|
+
end
|
|
136
|
+
local fields = {}
|
|
137
|
+
local body = source:sub(first, last)
|
|
138
|
+
for field, value in ipairs({name, rel, body, doc}) do
|
|
139
|
+
local terms, length = M.tokenize(value)
|
|
140
|
+
fields[field] = {terms, length}
|
|
141
|
+
totals[field] = totals[field] + length
|
|
142
|
+
end
|
|
143
|
+
encoded[i] = {id, fields, body, doc}
|
|
144
|
+
end
|
|
145
|
+
return {status = 'parsed', terms = cmsgpack.pack(encoded), totals = totals, comments = comment_count}
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
return M
|
|
@@ -0,0 +1,548 @@
|
|
|
1
|
+
-- Public API: build(index, file) and search(index, file, query, options).
|
|
2
|
+
-- Pass a refreshed CodeOutline index. The file is a lexical snapshot of that
|
|
3
|
+
-- project's source records and must be rebuilt when those records change.
|
|
4
|
+
local indexing = require('codeoutline.indexing.index')
|
|
5
|
+
local pool = require('codeoutline.indexing.parse_pool')
|
|
6
|
+
local collect = require('codeoutline.bm25.collect')
|
|
7
|
+
local control = require('codeoutline.shared.control')
|
|
8
|
+
local paths = require('codeoutline.shared.path')
|
|
9
|
+
local text_util = require('codeoutline.shared.text')
|
|
10
|
+
local snapshot = require('codeoutline.bm25.snapshot')
|
|
11
|
+
|
|
12
|
+
local M = {}
|
|
13
|
+
local TRAILER_SIZE = 80
|
|
14
|
+
local PASSAGE_BLOCK = 32
|
|
15
|
+
local GC_BATCH_BYTES = 512 * 1024
|
|
16
|
+
local GC_STEP_KB = 512
|
|
17
|
+
local cached_digest, cached_header
|
|
18
|
+
|
|
19
|
+
-- Drive GC by completed work so temporary batches can be reclaimed while the
|
|
20
|
+
-- live postings grow. Keep the collector settings of the calling Lua state.
|
|
21
|
+
local function build_gc()
|
|
22
|
+
local processed = 0
|
|
23
|
+
return function(bytes)
|
|
24
|
+
processed = processed + bytes
|
|
25
|
+
if processed >= GC_BATCH_BYTES then
|
|
26
|
+
processed = 0
|
|
27
|
+
collectgarbage('step', GC_STEP_KB)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
local function plan(index, source_paths)
|
|
33
|
+
local jobs, files, locations = {}, {}, {}
|
|
34
|
+
for _, path in ipairs(source_paths) do
|
|
35
|
+
control.check()
|
|
36
|
+
local file = index.files[path]
|
|
37
|
+
local documents = {}
|
|
38
|
+
for _, node in ipairs(file.nodes) do
|
|
39
|
+
if node.kind == 'function' or node.kind == 'method' then
|
|
40
|
+
documents[#documents + 1] = {false, text_util.valid(node.qualified or node.name),
|
|
41
|
+
node.line, math.max(node.line, node.end_line or node.line), node.kind}
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
if #documents > 0 then
|
|
45
|
+
table.sort(documents, function(a, b)
|
|
46
|
+
if a[3] ~= b[3] then return a[3] < b[3] end
|
|
47
|
+
if a[2] ~= b[2] then return a[2] < b[2] end
|
|
48
|
+
if a[4] ~= b[4] then return a[4] < b[4] end
|
|
49
|
+
return a[5] < b[5]
|
|
50
|
+
end)
|
|
51
|
+
local file_id = #files + 1
|
|
52
|
+
files[file_id] = path
|
|
53
|
+
for _, doc in ipairs(documents) do
|
|
54
|
+
doc[1] = #locations + 1
|
|
55
|
+
locations[doc[1]] = {file_id, doc[2], doc[3], doc[4], doc[5]}
|
|
56
|
+
end
|
|
57
|
+
jobs[#jobs + 1] = {rel = path, abs = index:abs(path), size = file.size, crc = file.crc,
|
|
58
|
+
language = file.language, documents = documents, weight = file.size + indexing.FILE_WEIGHT}
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
return jobs, files, locations
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
local function collect_terms(index, jobs, spool, gc_step)
|
|
65
|
+
local totals, lengths, comments = {0, 0, 0, 0}, {}, 0
|
|
66
|
+
local function accept(_, result)
|
|
67
|
+
local encoded = result.terms
|
|
68
|
+
assert(spool:write(encoded))
|
|
69
|
+
lengths[#lengths + 1] = #encoded
|
|
70
|
+
comments = comments + result.comments
|
|
71
|
+
for i, length in ipairs(result.totals) do totals[i] = totals[i] + length end
|
|
72
|
+
gc_step(#encoded)
|
|
73
|
+
end
|
|
74
|
+
if #jobs >= indexing.PARALLEL_MIN_FILES and pool.ready() then
|
|
75
|
+
pool.run(jobs, index.max_bytes, accept, 'bm25_collect')
|
|
76
|
+
else
|
|
77
|
+
for _, entry in ipairs(jobs) do
|
|
78
|
+
accept(entry.rel, collect.file(entry.abs, entry.rel, entry.size, entry.crc, entry.language, entry.documents))
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
assert(#lengths == #jobs, 'incomplete BM25 source pass')
|
|
82
|
+
return totals, lengths, comments
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
local function read_at(file, offset, length)
|
|
86
|
+
assert(file:seek('set', offset) == offset, 'cannot seek BM25 index')
|
|
87
|
+
local data = assert(file:read(length), 'truncated BM25 index')
|
|
88
|
+
assert(#data == length, 'truncated BM25 index')
|
|
89
|
+
return data
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
local function merge_terms(spool, totals, lengths, count, write_passages, gc_step)
|
|
93
|
+
local postings, averages = {}, {}
|
|
94
|
+
for i, total in ipairs(totals) do averages[i] = count > 0 and total / count or 0 end
|
|
95
|
+
assert(spool:seek('set', 0) == 0)
|
|
96
|
+
for _, length in ipairs(lengths) do
|
|
97
|
+
control.check()
|
|
98
|
+
gc_step(length)
|
|
99
|
+
local data = assert(spool:read(length), 'truncated BM25 staging file')
|
|
100
|
+
assert(#data == length, 'truncated BM25 staging file')
|
|
101
|
+
local batch = cmsgpack.unpack(data)
|
|
102
|
+
if #batch > 0 then write_passages(batch) end
|
|
103
|
+
for _, doc in ipairs(batch) do
|
|
104
|
+
local id, fields = table.unpack(doc)
|
|
105
|
+
for i, field in ipairs(fields) do
|
|
106
|
+
if averages[i] > 0 then
|
|
107
|
+
local terms, length = table.unpack(field)
|
|
108
|
+
local normalization = 1.2 * (0.25 + 0.75 * length / averages[i])
|
|
109
|
+
local weight = collect.weights[i]
|
|
110
|
+
for word, frequency in pairs(terms) do
|
|
111
|
+
local matches = postings[word]
|
|
112
|
+
if not matches then matches = {}; postings[word] = matches end
|
|
113
|
+
local tf = weight * 2.2 * frequency / (frequency + normalization)
|
|
114
|
+
matches[id] = (matches[id] or 0) + tf
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
return postings
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
local function exact_keys(files, locations)
|
|
124
|
+
local exact = {}
|
|
125
|
+
for id, location in ipairs(locations) do
|
|
126
|
+
if id % 128 == 0 then control.check() end
|
|
127
|
+
local path = files[location[1]]:lower()
|
|
128
|
+
local name = location[2]:lower()
|
|
129
|
+
for _, key in ipairs({path, path:match('[^/]+$'), name, name:match('[^.:]+$') or name}) do
|
|
130
|
+
local matches = exact[key]
|
|
131
|
+
if not matches then matches = {}; exact[key] = matches end
|
|
132
|
+
matches[id] = true
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
return exact
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
local function write_index(output, spool, totals, lengths, header, gc_step)
|
|
139
|
+
local prefix = snapshot.prefix(header.version) .. header.stamp
|
|
140
|
+
assert(output:write(prefix))
|
|
141
|
+
local offset, count = #prefix, #header.locations
|
|
142
|
+
local function write_data(data)
|
|
143
|
+
local location = {offset, #data, xcompress.crc32(data)}
|
|
144
|
+
assert(output:write(data))
|
|
145
|
+
offset = offset + #data
|
|
146
|
+
return location
|
|
147
|
+
end
|
|
148
|
+
local function write_block(matches)
|
|
149
|
+
return write_data(cmsgpack.pack(matches))
|
|
150
|
+
end
|
|
151
|
+
local compressor = xcompress.new_compressor(1)
|
|
152
|
+
local function write_passages(batch)
|
|
153
|
+
local first_id = batch[1][1]
|
|
154
|
+
local blocks = {first_id}
|
|
155
|
+
header.passages[header.locations[first_id][1]] = blocks
|
|
156
|
+
for start = 1, #batch, PASSAGE_BLOCK do
|
|
157
|
+
control.check()
|
|
158
|
+
local passages = {}
|
|
159
|
+
for i = start, math.min(start + PASSAGE_BLOCK - 1, #batch) do
|
|
160
|
+
passages[#passages + 1] = {batch[i][3], batch[i][4]}
|
|
161
|
+
end
|
|
162
|
+
local data = cmsgpack.pack(passages)
|
|
163
|
+
local entry = write_data(assert(compressor:deflate(data)))
|
|
164
|
+
entry[4] = #data
|
|
165
|
+
blocks[#blocks + 1] = entry
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
local postings = merge_terms(spool, totals, lengths, count, write_passages, gc_step)
|
|
169
|
+
compressor:close()
|
|
170
|
+
local records = 0
|
|
171
|
+
for word, matches in pairs(postings) do
|
|
172
|
+
control.check()
|
|
173
|
+
local frequency = 0
|
|
174
|
+
for _ in pairs(matches) do frequency = frequency + 1 end
|
|
175
|
+
local entry = write_block(matches)
|
|
176
|
+
entry[4] = math.log(1 + (count - frequency + 0.5) / (frequency + 0.5))
|
|
177
|
+
header.words[word] = entry
|
|
178
|
+
records = records + frequency
|
|
179
|
+
postings[word] = nil
|
|
180
|
+
matches = nil
|
|
181
|
+
gc_step(entry[2])
|
|
182
|
+
end
|
|
183
|
+
postings = nil
|
|
184
|
+
local exact = exact_keys(header.files, header.locations)
|
|
185
|
+
for key, matches in pairs(exact) do
|
|
186
|
+
control.check()
|
|
187
|
+
local entry = write_block(matches)
|
|
188
|
+
header.exact[key] = entry
|
|
189
|
+
exact[key] = nil
|
|
190
|
+
matches = nil
|
|
191
|
+
gc_step(entry[2])
|
|
192
|
+
end
|
|
193
|
+
exact = nil
|
|
194
|
+
local encoded = cmsgpack.pack(header)
|
|
195
|
+
assert(output:write(encoded, xutils.sha256_hex(encoded), string.format('%016d', #encoded)))
|
|
196
|
+
return records, offset + #encoded + TRAILER_SIZE
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
local function output_path(file)
|
|
200
|
+
assert(type(file) == 'string' and file ~= '' and not file:find('%z'), 'BM25 index file is required')
|
|
201
|
+
return paths.normalize(file)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
local function check_destination(file)
|
|
205
|
+
local existing = io.open(file, 'rb')
|
|
206
|
+
if not existing then return end
|
|
207
|
+
local magic = existing:read(#snapshot.magic)
|
|
208
|
+
existing:close()
|
|
209
|
+
assert(magic == snapshot.magic, 'refusing to replace a non-BM25 file: ' .. file)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
function M.build(index, file)
|
|
213
|
+
file = output_path(file)
|
|
214
|
+
assert(not index.cache_path or paths.key(file) ~= paths.key(index.cache_path),
|
|
215
|
+
'BM25 must use a separate file from the base index')
|
|
216
|
+
check_destination(file)
|
|
217
|
+
local parent = file:match('^(.*)/[^/]+$') or '.'
|
|
218
|
+
if parent == '' or parent:match('^%a:$') then parent = parent .. '/' end
|
|
219
|
+
assert(xutils.mkdir_p(parent))
|
|
220
|
+
local generation = index.generation
|
|
221
|
+
local stamp, source_paths = snapshot.stamp(index)
|
|
222
|
+
local jobs, files, locations = plan(index, source_paths)
|
|
223
|
+
local spool_name, staged, spool, output
|
|
224
|
+
local ok, result = pcall(function()
|
|
225
|
+
spool_name = assert(xutils.temp_file(parent))
|
|
226
|
+
spool = assert(io.open(spool_name, 'w+b'))
|
|
227
|
+
local gc_step = build_gc()
|
|
228
|
+
local totals, lengths, comments = collect_terms(index, jobs, spool, gc_step)
|
|
229
|
+
jobs = nil
|
|
230
|
+
staged = assert(xutils.temp_file(parent))
|
|
231
|
+
output = assert(io.open(staged, 'wb'))
|
|
232
|
+
local records, bytes = write_index(output, spool, totals, lengths, {version = snapshot.version,
|
|
233
|
+
stamp = stamp, files = files, locations = locations, words = {}, exact = {}, passages = {}}, gc_step)
|
|
234
|
+
assert(spool:close()); spool = nil
|
|
235
|
+
assert(output:close()); output = nil
|
|
236
|
+
control.check()
|
|
237
|
+
assert(index.generation == generation, 'base index changed during BM25 build')
|
|
238
|
+
check_destination(file)
|
|
239
|
+
assert(xutils.replace_file(staged, file))
|
|
240
|
+
staged = nil
|
|
241
|
+
return {files = #files, functions = #locations, comments = comments, postings = records, bytes = bytes}
|
|
242
|
+
end)
|
|
243
|
+
if spool then spool:close() end
|
|
244
|
+
if output then output:close() end
|
|
245
|
+
if spool_name then os.remove(spool_name) end
|
|
246
|
+
if staged then os.remove(staged) end
|
|
247
|
+
if not ok then error(result, 0) end
|
|
248
|
+
return result
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
local function open_header(file)
|
|
252
|
+
local size = assert(file:seek('end'))
|
|
253
|
+
local magic = snapshot.magic
|
|
254
|
+
assert(size > #magic + TRAILER_SIZE and read_at(file, 0, #magic) == magic, 'invalid BM25 index')
|
|
255
|
+
local trailer = read_at(file, size - TRAILER_SIZE, TRAILER_SIZE)
|
|
256
|
+
local digest, length = trailer:sub(1, 64), tonumber(trailer:sub(65))
|
|
257
|
+
assert(length and length % 1 == 0 and length > 0 and length <= size - #magic - TRAILER_SIZE,
|
|
258
|
+
'invalid BM25 header length')
|
|
259
|
+
local offset = size - TRAILER_SIZE - length
|
|
260
|
+
if digest ~= cached_digest then
|
|
261
|
+
local encoded = read_at(file, offset, length)
|
|
262
|
+
assert(xutils.sha256_hex(encoded) == digest, 'corrupt BM25 header')
|
|
263
|
+
local header = cmsgpack.unpack(encoded)
|
|
264
|
+
assert(header.version == 1 or header.version == 2 or header.version == snapshot.version,
|
|
265
|
+
'unsupported BM25 index version; rebuild it')
|
|
266
|
+
cached_digest, cached_header = digest, header
|
|
267
|
+
end
|
|
268
|
+
local first = #magic
|
|
269
|
+
if cached_header.version >= 2 then
|
|
270
|
+
local prefix = snapshot.prefix(cached_header.version) .. cached_header.stamp
|
|
271
|
+
assert(offset >= #prefix and read_at(file, 0, #prefix) == prefix, 'corrupt BM25 snapshot prefix')
|
|
272
|
+
first = #prefix
|
|
273
|
+
end
|
|
274
|
+
return cached_header, first, offset
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
local function read_data(file, entry, first, boundary)
|
|
278
|
+
local offset, length = entry[1], entry[2]
|
|
279
|
+
assert(offset >= first and length > 0 and offset + length <= boundary, 'invalid BM25 block range')
|
|
280
|
+
local encoded = read_at(file, offset, length)
|
|
281
|
+
assert(xcompress.crc32(encoded) == entry[3], 'corrupt BM25 postings')
|
|
282
|
+
return encoded
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
local function read_block(file, entry, first, boundary)
|
|
286
|
+
return cmsgpack.unpack(read_data(file, entry, first, boundary))
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
local function scope_key(index, path)
|
|
290
|
+
if path == nil or path == '' or path == '.' then return nil end
|
|
291
|
+
assert(type(path) == 'string', 'BM25 path filter must be a string')
|
|
292
|
+
path = paths.key(path):gsub('^%./', '')
|
|
293
|
+
if path:match('^/') or path:match('^%a:/') then
|
|
294
|
+
local root = paths.key(index.root)
|
|
295
|
+
assert(paths.contains(root, path), 'BM25 path filter is outside the project')
|
|
296
|
+
if root == path then return nil end
|
|
297
|
+
return path:sub(#root + (root:sub(-1) == '/' and 1 or 2))
|
|
298
|
+
end
|
|
299
|
+
return path
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
local function rank(scores, exact, literal, limit)
|
|
303
|
+
local function better(a, b)
|
|
304
|
+
if exact[a] ~= exact[b] then return exact[a] == true end
|
|
305
|
+
if literal[a] ~= literal[b] then return literal[a] == true end
|
|
306
|
+
if scores[a] ~= scores[b] then return scores[a] > scores[b] end
|
|
307
|
+
return a < b
|
|
308
|
+
end
|
|
309
|
+
-- The root of the bounded heap is the worst retained candidate.
|
|
310
|
+
local heap, visited = {}, 0
|
|
311
|
+
for id in pairs(scores) do
|
|
312
|
+
visited = visited + 1
|
|
313
|
+
if visited % 1024 == 0 then control.check() end
|
|
314
|
+
if #heap < limit then
|
|
315
|
+
local i = #heap + 1
|
|
316
|
+
while i > 1 do
|
|
317
|
+
local parent = math.floor(i / 2)
|
|
318
|
+
if not better(heap[parent], id) then break end
|
|
319
|
+
heap[i], i = heap[parent], parent
|
|
320
|
+
end
|
|
321
|
+
heap[i] = id
|
|
322
|
+
elseif better(id, heap[1]) then
|
|
323
|
+
local i, child = 1, 2
|
|
324
|
+
while child <= #heap do
|
|
325
|
+
if child < #heap and better(heap[child], heap[child + 1]) then child = child + 1 end
|
|
326
|
+
if not better(id, heap[child]) then break end
|
|
327
|
+
heap[i], i = heap[child], child
|
|
328
|
+
child = i * 2
|
|
329
|
+
end
|
|
330
|
+
heap[i] = id
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
table.sort(heap, better)
|
|
334
|
+
return heap
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
local function score_terms(file, entries, words, first, boundary, accepts, match)
|
|
338
|
+
local scores = {}
|
|
339
|
+
local counts = match ~= 'all' and #words > 1 and {} or nil
|
|
340
|
+
local rarest, seed
|
|
341
|
+
if match == 'all' and #words > 1 then
|
|
342
|
+
for _, word in ipairs(words) do
|
|
343
|
+
local entry = entries[word]
|
|
344
|
+
if not entry then return scores end
|
|
345
|
+
-- IDF decreases as the number of documents for a word increases.
|
|
346
|
+
if not rarest or entry[4] > entries[rarest][4] then rarest = word end
|
|
347
|
+
end
|
|
348
|
+
seed = read_block(file, entries[rarest], first, boundary)
|
|
349
|
+
local scanned = 0
|
|
350
|
+
for id in pairs(seed) do
|
|
351
|
+
scanned = scanned + 1
|
|
352
|
+
if scanned % 1024 == 0 then control.check() end
|
|
353
|
+
if accepts(id) then scores[id] = 0 end
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
-- Accumulate terms in the same order for union and intersection searches,
|
|
357
|
+
-- so a surviving document keeps the same BM25 score and tie-breaking.
|
|
358
|
+
for _, word in ipairs(words) do
|
|
359
|
+
control.check()
|
|
360
|
+
if rarest and not next(scores) then break end
|
|
361
|
+
local entry = entries[word]
|
|
362
|
+
if entry then
|
|
363
|
+
local matches = word == rarest and seed or read_block(file, entry, first, boundary)
|
|
364
|
+
local scanned = 0
|
|
365
|
+
if rarest then
|
|
366
|
+
for id, score in pairs(scores) do
|
|
367
|
+
scanned = scanned + 1
|
|
368
|
+
if scanned % 1024 == 0 then control.check() end
|
|
369
|
+
local tf = matches[id]
|
|
370
|
+
scores[id] = tf and (score + entry[4] * tf) or nil
|
|
371
|
+
end
|
|
372
|
+
if word == rarest then seed = nil end
|
|
373
|
+
else
|
|
374
|
+
for id, tf in pairs(matches) do
|
|
375
|
+
scanned = scanned + 1
|
|
376
|
+
if scanned % 1024 == 0 then control.check() end
|
|
377
|
+
if accepts(id) then
|
|
378
|
+
scores[id] = (scores[id] or 0) + entry[4] * tf
|
|
379
|
+
if counts then counts[id] = (counts[id] or 0) + 1 end
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
end
|
|
385
|
+
return scores, counts
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
local function contains_literal(text, literal, left_boundary, right_boundary)
|
|
389
|
+
if text == '' then return false end
|
|
390
|
+
text = text:lower()
|
|
391
|
+
local offset = 1
|
|
392
|
+
while true do
|
|
393
|
+
local first, last = text:find(literal, offset, true)
|
|
394
|
+
if not first then return false end
|
|
395
|
+
if (not left_boundary or first == 1 or not text:sub(first - 1, first - 1):match('[%w_$]'))
|
|
396
|
+
and (not right_boundary or not text:sub(last + 1, last + 1):match('[%w_$]')) then
|
|
397
|
+
return true
|
|
398
|
+
end
|
|
399
|
+
offset = first + 1
|
|
400
|
+
end
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
local function boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
|
|
404
|
+
local exact = {}
|
|
405
|
+
if not header.passages or #words == 0 or literal == '' or not next(scores) then return exact end
|
|
406
|
+
local strength = 0
|
|
407
|
+
for _, word in ipairs(words) do
|
|
408
|
+
local entry = header.words[word]
|
|
409
|
+
if not entry then return exact end
|
|
410
|
+
strength = strength + entry[4]
|
|
411
|
+
end
|
|
412
|
+
-- Group by block so each compressed passage is read at most once per query.
|
|
413
|
+
local groups, scanned = {}, 0
|
|
414
|
+
for id in pairs(scores) do
|
|
415
|
+
scanned = scanned + 1
|
|
416
|
+
if scanned % 128 == 0 then control.check() end
|
|
417
|
+
if not counts or counts[id] == #words then
|
|
418
|
+
local blocks = header.passages[header.locations[id][1]]
|
|
419
|
+
local offset = id - blocks[1]
|
|
420
|
+
local entry = blocks[2 + math.floor(offset / PASSAGE_BLOCK)]
|
|
421
|
+
local group = groups[entry]
|
|
422
|
+
if not group then group = {}; groups[entry] = group end
|
|
423
|
+
group[#group + 1] = id
|
|
424
|
+
end
|
|
425
|
+
end
|
|
426
|
+
if not next(groups) then return exact end
|
|
427
|
+
local decoder = xcompress.new_decompressor()
|
|
428
|
+
local left_boundary, right_boundary = literal:sub(1, 1):match('[%w_$]'), literal:sub(-1):match('[%w_$]')
|
|
429
|
+
for entry, ids in pairs(groups) do
|
|
430
|
+
control.check()
|
|
431
|
+
local data = read_data(file, entry, first, boundary)
|
|
432
|
+
local passages = cmsgpack.unpack(assert(decoder:deflate(data, entry[4]), 'corrupt BM25 passages'))
|
|
433
|
+
for _, id in ipairs(ids) do
|
|
434
|
+
local location = header.locations[id]
|
|
435
|
+
local offset = id - header.passages[location[1]][1]
|
|
436
|
+
local passage = passages[1 + offset % PASSAGE_BLOCK]
|
|
437
|
+
local best
|
|
438
|
+
for _, text in ipairs({location[2], header.files[location[1]], passage[1], passage[2]}) do
|
|
439
|
+
if not exact[id] and contains_literal(text, literal, left_boundary, right_boundary) then exact[id] = true end
|
|
440
|
+
if #words > 1 and best ~= 0 and text ~= '' then
|
|
441
|
+
local gap = collect.gap(text, terms, #words)
|
|
442
|
+
if gap and (not best or gap < best) then best = gap end
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
if best then scores[id] = scores[id] + strength / (1 + best) end
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
decoder:close()
|
|
449
|
+
return exact
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
-- Use the existing symbol and path tables. Prefix hints change ranking only.
|
|
453
|
+
local function boost_prefix(header, scores, words, prefix)
|
|
454
|
+
if not prefix or prefix == '' then return end
|
|
455
|
+
prefix = prefix:lower()
|
|
456
|
+
local strength = 0
|
|
457
|
+
for _, word in ipairs(words) do
|
|
458
|
+
local entry = header.words[word]
|
|
459
|
+
if entry then strength = strength + entry[4] end
|
|
460
|
+
end
|
|
461
|
+
strength = math.max(strength, 1)
|
|
462
|
+
local path_scores, scanned = {}, 0
|
|
463
|
+
local directory = '/' .. prefix .. '/'
|
|
464
|
+
for id, score in pairs(scores) do
|
|
465
|
+
scanned = scanned + 1
|
|
466
|
+
if scanned % 128 == 0 then control.check() end
|
|
467
|
+
local location = header.locations[id]
|
|
468
|
+
local file_id = location[1]
|
|
469
|
+
local weight = path_scores[file_id]
|
|
470
|
+
if not weight then
|
|
471
|
+
local path = header.files[file_id]:lower()
|
|
472
|
+
local stem = path:match('[^/]+$'):gsub('%.[^%.]+$', '')
|
|
473
|
+
if stem == prefix then weight = 4
|
|
474
|
+
elseif ('/' .. path):find(directory, 1, true) then weight = 3
|
|
475
|
+
elseif stem:sub(1, #prefix) == prefix and stem:sub(#prefix + 1, #prefix + 1):match('[_%.%-]') then
|
|
476
|
+
weight = 1
|
|
477
|
+
else weight = 0 end
|
|
478
|
+
path_scores[file_id] = weight
|
|
479
|
+
end
|
|
480
|
+
local owner = location[2]:match('([^.:]+)[.:]+[^.:]+$')
|
|
481
|
+
if owner and owner:lower() == prefix then weight = 5 end
|
|
482
|
+
if weight > 0 then scores[id] = score + weight * strength end
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
local function search_file(index, file, query, options, limit)
|
|
487
|
+
local header, first, boundary = open_header(file)
|
|
488
|
+
assert(header.stamp == snapshot.stamp(index), 'BM25 index is stale; rebuild it')
|
|
489
|
+
local scope, allowed = scope_key(index, options.path), {}
|
|
490
|
+
local function in_scope(id)
|
|
491
|
+
if not scope then return true end
|
|
492
|
+
local file_id = header.locations[id][1]
|
|
493
|
+
if allowed[file_id] == nil then allowed[file_id] = paths.contains(scope, header.files[file_id]) end
|
|
494
|
+
return allowed[file_id]
|
|
495
|
+
end
|
|
496
|
+
local accepts = in_scope
|
|
497
|
+
if options.accept then
|
|
498
|
+
local accepted = {}
|
|
499
|
+
accepts = function(id)
|
|
500
|
+
if accepted[id] == nil then
|
|
501
|
+
local location = header.locations[id]
|
|
502
|
+
accepted[id] = not not (in_scope(id)
|
|
503
|
+
and options.accept(header.files[location[1]], location[2], location[5]))
|
|
504
|
+
end
|
|
505
|
+
return accepted[id]
|
|
506
|
+
end
|
|
507
|
+
end
|
|
508
|
+
local words, terms = {}, collect.tokenize(query)
|
|
509
|
+
for token in pairs(terms) do words[#words + 1] = token end
|
|
510
|
+
table.sort(words)
|
|
511
|
+
local scores, counts = score_terms(file, header.words, words, first, boundary, accepts, options.match)
|
|
512
|
+
local mark = string.char(96)
|
|
513
|
+
local literal = query:match('^%s*(.-)%s*$'):gsub('^' .. mark .. '+', ''):gsub(mark .. '+$', ''):lower()
|
|
514
|
+
local key = literal:gsub('\\', '/')
|
|
515
|
+
local entry = header.exact[key]
|
|
516
|
+
local exact = entry and read_block(file, entry, first, boundary) or {}
|
|
517
|
+
local literal_matches = boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
|
|
518
|
+
if options.match ~= 'all' or #words == 0 then
|
|
519
|
+
for id in pairs(exact) do if accepts(id) then scores[id] = scores[id] or 0 end end
|
|
520
|
+
end
|
|
521
|
+
boost_prefix(header, scores, words, options.prefix)
|
|
522
|
+
local results = {}
|
|
523
|
+
for rank_number, id in ipairs(rank(scores, exact, literal_matches, limit)) do
|
|
524
|
+
local location = header.locations[id]
|
|
525
|
+
results[rank_number] = {path = header.files[location[1]], symbol = location[2],
|
|
526
|
+
start_line = location[3], end_line = location[4], score = scores[id], kind = location[5]}
|
|
527
|
+
end
|
|
528
|
+
return results
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
function M.search(index, file, query, options)
|
|
532
|
+
assert(type(query) == 'string', 'BM25 query must be a string')
|
|
533
|
+
options = options or {}
|
|
534
|
+
assert(options.match == nil or options.match == 'any' or options.match == 'all',
|
|
535
|
+
'BM25 match must be any or all')
|
|
536
|
+
assert(options.prefix == nil or type(options.prefix) == 'string', 'BM25 prefix hint must be a string')
|
|
537
|
+
local limit = options.limit == nil and 20 or options.limit
|
|
538
|
+
assert(type(limit) == 'number' and limit >= 0 and limit < math.huge and limit % 1 == 0,
|
|
539
|
+
'BM25 limit must be a nonnegative integer')
|
|
540
|
+
if limit == 0 then return {} end
|
|
541
|
+
local input = assert(io.open(output_path(file), 'rb'))
|
|
542
|
+
local ok, result = pcall(search_file, index, input, query, options, limit)
|
|
543
|
+
input:close()
|
|
544
|
+
if not ok then error(result, 0) end
|
|
545
|
+
return result
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
return M
|