@codua/codeoutline-win32-x64 0.3.9 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,548 @@
1
+ -- Public API: build(index, file) and search(index, file, query, options).
2
+ -- Pass a refreshed CodeOutline index. The file is a lexical snapshot of that
3
+ -- project's source records and must be rebuilt when those records change.
4
+ local indexing = require('codeoutline.indexing.index')
5
+ local pool = require('codeoutline.indexing.parse_pool')
6
+ local collect = require('codeoutline.bm25.collect')
7
+ local control = require('codeoutline.shared.control')
8
+ local paths = require('codeoutline.shared.path')
9
+ local text_util = require('codeoutline.shared.text')
10
+ local snapshot = require('codeoutline.bm25.snapshot')
11
+
12
+ local M = {}
13
+ local TRAILER_SIZE = 80
14
+ local PASSAGE_BLOCK = 32
15
+ local GC_BATCH_BYTES = 512 * 1024
16
+ local GC_STEP_KB = 512
17
+ local cached_digest, cached_header
18
+
19
+ -- Drive GC by completed work so temporary batches can be reclaimed while the
20
+ -- live postings grow. Keep the collector settings of the calling Lua state.
21
+ local function build_gc()
22
+ local processed = 0
23
+ return function(bytes)
24
+ processed = processed + bytes
25
+ if processed >= GC_BATCH_BYTES then
26
+ processed = 0
27
+ collectgarbage('step', GC_STEP_KB)
28
+ end
29
+ end
30
+ end
31
+
32
+ local function plan(index, source_paths)
33
+ local jobs, files, locations = {}, {}, {}
34
+ for _, path in ipairs(source_paths) do
35
+ control.check()
36
+ local file = index.files[path]
37
+ local documents = {}
38
+ for _, node in ipairs(file.nodes) do
39
+ if node.kind == 'function' or node.kind == 'method' then
40
+ documents[#documents + 1] = {false, text_util.valid(node.qualified or node.name),
41
+ node.line, math.max(node.line, node.end_line or node.line), node.kind}
42
+ end
43
+ end
44
+ if #documents > 0 then
45
+ table.sort(documents, function(a, b)
46
+ if a[3] ~= b[3] then return a[3] < b[3] end
47
+ if a[2] ~= b[2] then return a[2] < b[2] end
48
+ if a[4] ~= b[4] then return a[4] < b[4] end
49
+ return a[5] < b[5]
50
+ end)
51
+ local file_id = #files + 1
52
+ files[file_id] = path
53
+ for _, doc in ipairs(documents) do
54
+ doc[1] = #locations + 1
55
+ locations[doc[1]] = {file_id, doc[2], doc[3], doc[4], doc[5]}
56
+ end
57
+ jobs[#jobs + 1] = {rel = path, abs = index:abs(path), size = file.size, crc = file.crc,
58
+ language = file.language, documents = documents, weight = file.size + indexing.FILE_WEIGHT}
59
+ end
60
+ end
61
+ return jobs, files, locations
62
+ end
63
+
64
+ local function collect_terms(index, jobs, spool, gc_step)
65
+ local totals, lengths, comments = {0, 0, 0, 0}, {}, 0
66
+ local function accept(_, result)
67
+ local encoded = result.terms
68
+ assert(spool:write(encoded))
69
+ lengths[#lengths + 1] = #encoded
70
+ comments = comments + result.comments
71
+ for i, length in ipairs(result.totals) do totals[i] = totals[i] + length end
72
+ gc_step(#encoded)
73
+ end
74
+ if #jobs >= indexing.PARALLEL_MIN_FILES and pool.ready() then
75
+ pool.run(jobs, index.max_bytes, accept, 'bm25_collect')
76
+ else
77
+ for _, entry in ipairs(jobs) do
78
+ accept(entry.rel, collect.file(entry.abs, entry.rel, entry.size, entry.crc, entry.language, entry.documents))
79
+ end
80
+ end
81
+ assert(#lengths == #jobs, 'incomplete BM25 source pass')
82
+ return totals, lengths, comments
83
+ end
84
+
85
+ local function read_at(file, offset, length)
86
+ assert(file:seek('set', offset) == offset, 'cannot seek BM25 index')
87
+ local data = assert(file:read(length), 'truncated BM25 index')
88
+ assert(#data == length, 'truncated BM25 index')
89
+ return data
90
+ end
91
+
92
+ local function merge_terms(spool, totals, lengths, count, write_passages, gc_step)
93
+ local postings, averages = {}, {}
94
+ for i, total in ipairs(totals) do averages[i] = count > 0 and total / count or 0 end
95
+ assert(spool:seek('set', 0) == 0)
96
+ for _, length in ipairs(lengths) do
97
+ control.check()
98
+ gc_step(length)
99
+ local data = assert(spool:read(length), 'truncated BM25 staging file')
100
+ assert(#data == length, 'truncated BM25 staging file')
101
+ local batch = cmsgpack.unpack(data)
102
+ if #batch > 0 then write_passages(batch) end
103
+ for _, doc in ipairs(batch) do
104
+ local id, fields = table.unpack(doc)
105
+ for i, field in ipairs(fields) do
106
+ if averages[i] > 0 then
107
+ local terms, length = table.unpack(field)
108
+ local normalization = 1.2 * (0.25 + 0.75 * length / averages[i])
109
+ local weight = collect.weights[i]
110
+ for word, frequency in pairs(terms) do
111
+ local matches = postings[word]
112
+ if not matches then matches = {}; postings[word] = matches end
113
+ local tf = weight * 2.2 * frequency / (frequency + normalization)
114
+ matches[id] = (matches[id] or 0) + tf
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
120
+ return postings
121
+ end
122
+
123
+ local function exact_keys(files, locations)
124
+ local exact = {}
125
+ for id, location in ipairs(locations) do
126
+ if id % 128 == 0 then control.check() end
127
+ local path = files[location[1]]:lower()
128
+ local name = location[2]:lower()
129
+ for _, key in ipairs({path, path:match('[^/]+$'), name, name:match('[^.:]+$') or name}) do
130
+ local matches = exact[key]
131
+ if not matches then matches = {}; exact[key] = matches end
132
+ matches[id] = true
133
+ end
134
+ end
135
+ return exact
136
+ end
137
+
138
+ local function write_index(output, spool, totals, lengths, header, gc_step)
139
+ local prefix = snapshot.prefix(header.version) .. header.stamp
140
+ assert(output:write(prefix))
141
+ local offset, count = #prefix, #header.locations
142
+ local function write_data(data)
143
+ local location = {offset, #data, xcompress.crc32(data)}
144
+ assert(output:write(data))
145
+ offset = offset + #data
146
+ return location
147
+ end
148
+ local function write_block(matches)
149
+ return write_data(cmsgpack.pack(matches))
150
+ end
151
+ local compressor = xcompress.new_compressor(1)
152
+ local function write_passages(batch)
153
+ local first_id = batch[1][1]
154
+ local blocks = {first_id}
155
+ header.passages[header.locations[first_id][1]] = blocks
156
+ for start = 1, #batch, PASSAGE_BLOCK do
157
+ control.check()
158
+ local passages = {}
159
+ for i = start, math.min(start + PASSAGE_BLOCK - 1, #batch) do
160
+ passages[#passages + 1] = {batch[i][3], batch[i][4]}
161
+ end
162
+ local data = cmsgpack.pack(passages)
163
+ local entry = write_data(assert(compressor:deflate(data)))
164
+ entry[4] = #data
165
+ blocks[#blocks + 1] = entry
166
+ end
167
+ end
168
+ local postings = merge_terms(spool, totals, lengths, count, write_passages, gc_step)
169
+ compressor:close()
170
+ local records = 0
171
+ for word, matches in pairs(postings) do
172
+ control.check()
173
+ local frequency = 0
174
+ for _ in pairs(matches) do frequency = frequency + 1 end
175
+ local entry = write_block(matches)
176
+ entry[4] = math.log(1 + (count - frequency + 0.5) / (frequency + 0.5))
177
+ header.words[word] = entry
178
+ records = records + frequency
179
+ postings[word] = nil
180
+ matches = nil
181
+ gc_step(entry[2])
182
+ end
183
+ postings = nil
184
+ local exact = exact_keys(header.files, header.locations)
185
+ for key, matches in pairs(exact) do
186
+ control.check()
187
+ local entry = write_block(matches)
188
+ header.exact[key] = entry
189
+ exact[key] = nil
190
+ matches = nil
191
+ gc_step(entry[2])
192
+ end
193
+ exact = nil
194
+ local encoded = cmsgpack.pack(header)
195
+ assert(output:write(encoded, xutils.sha256_hex(encoded), string.format('%016d', #encoded)))
196
+ return records, offset + #encoded + TRAILER_SIZE
197
+ end
198
+
199
+ local function output_path(file)
200
+ assert(type(file) == 'string' and file ~= '' and not file:find('%z'), 'BM25 index file is required')
201
+ return paths.normalize(file)
202
+ end
203
+
204
+ local function check_destination(file)
205
+ local existing = io.open(file, 'rb')
206
+ if not existing then return end
207
+ local magic = existing:read(#snapshot.magic)
208
+ existing:close()
209
+ assert(magic == snapshot.magic, 'refusing to replace a non-BM25 file: ' .. file)
210
+ end
211
+
212
+ function M.build(index, file)
213
+ file = output_path(file)
214
+ assert(not index.cache_path or paths.key(file) ~= paths.key(index.cache_path),
215
+ 'BM25 must use a separate file from the base index')
216
+ check_destination(file)
217
+ local parent = file:match('^(.*)/[^/]+$') or '.'
218
+ if parent == '' or parent:match('^%a:$') then parent = parent .. '/' end
219
+ assert(xutils.mkdir_p(parent))
220
+ local generation = index.generation
221
+ local stamp, source_paths = snapshot.stamp(index)
222
+ local jobs, files, locations = plan(index, source_paths)
223
+ local spool_name, staged, spool, output
224
+ local ok, result = pcall(function()
225
+ spool_name = assert(xutils.temp_file(parent))
226
+ spool = assert(io.open(spool_name, 'w+b'))
227
+ local gc_step = build_gc()
228
+ local totals, lengths, comments = collect_terms(index, jobs, spool, gc_step)
229
+ jobs = nil
230
+ staged = assert(xutils.temp_file(parent))
231
+ output = assert(io.open(staged, 'wb'))
232
+ local records, bytes = write_index(output, spool, totals, lengths, {version = snapshot.version,
233
+ stamp = stamp, files = files, locations = locations, words = {}, exact = {}, passages = {}}, gc_step)
234
+ assert(spool:close()); spool = nil
235
+ assert(output:close()); output = nil
236
+ control.check()
237
+ assert(index.generation == generation, 'base index changed during BM25 build')
238
+ check_destination(file)
239
+ assert(xutils.replace_file(staged, file))
240
+ staged = nil
241
+ return {files = #files, functions = #locations, comments = comments, postings = records, bytes = bytes}
242
+ end)
243
+ if spool then spool:close() end
244
+ if output then output:close() end
245
+ if spool_name then os.remove(spool_name) end
246
+ if staged then os.remove(staged) end
247
+ if not ok then error(result, 0) end
248
+ return result
249
+ end
250
+
251
+ local function open_header(file)
252
+ local size = assert(file:seek('end'))
253
+ local magic = snapshot.magic
254
+ assert(size > #magic + TRAILER_SIZE and read_at(file, 0, #magic) == magic, 'invalid BM25 index')
255
+ local trailer = read_at(file, size - TRAILER_SIZE, TRAILER_SIZE)
256
+ local digest, length = trailer:sub(1, 64), tonumber(trailer:sub(65))
257
+ assert(length and length % 1 == 0 and length > 0 and length <= size - #magic - TRAILER_SIZE,
258
+ 'invalid BM25 header length')
259
+ local offset = size - TRAILER_SIZE - length
260
+ if digest ~= cached_digest then
261
+ local encoded = read_at(file, offset, length)
262
+ assert(xutils.sha256_hex(encoded) == digest, 'corrupt BM25 header')
263
+ local header = cmsgpack.unpack(encoded)
264
+ assert(header.version == 1 or header.version == 2 or header.version == snapshot.version,
265
+ 'unsupported BM25 index version; rebuild it')
266
+ cached_digest, cached_header = digest, header
267
+ end
268
+ local first = #magic
269
+ if cached_header.version >= 2 then
270
+ local prefix = snapshot.prefix(cached_header.version) .. cached_header.stamp
271
+ assert(offset >= #prefix and read_at(file, 0, #prefix) == prefix, 'corrupt BM25 snapshot prefix')
272
+ first = #prefix
273
+ end
274
+ return cached_header, first, offset
275
+ end
276
+
277
+ local function read_data(file, entry, first, boundary)
278
+ local offset, length = entry[1], entry[2]
279
+ assert(offset >= first and length > 0 and offset + length <= boundary, 'invalid BM25 block range')
280
+ local encoded = read_at(file, offset, length)
281
+ assert(xcompress.crc32(encoded) == entry[3], 'corrupt BM25 postings')
282
+ return encoded
283
+ end
284
+
285
+ local function read_block(file, entry, first, boundary)
286
+ return cmsgpack.unpack(read_data(file, entry, first, boundary))
287
+ end
288
+
289
+ local function scope_key(index, path)
290
+ if path == nil or path == '' or path == '.' then return nil end
291
+ assert(type(path) == 'string', 'BM25 path filter must be a string')
292
+ path = paths.key(path):gsub('^%./', '')
293
+ if path:match('^/') or path:match('^%a:/') then
294
+ local root = paths.key(index.root)
295
+ assert(paths.contains(root, path), 'BM25 path filter is outside the project')
296
+ if root == path then return nil end
297
+ return path:sub(#root + (root:sub(-1) == '/' and 1 or 2))
298
+ end
299
+ return path
300
+ end
301
+
302
+ local function rank(scores, exact, literal, limit)
303
+ local function better(a, b)
304
+ if exact[a] ~= exact[b] then return exact[a] == true end
305
+ if literal[a] ~= literal[b] then return literal[a] == true end
306
+ if scores[a] ~= scores[b] then return scores[a] > scores[b] end
307
+ return a < b
308
+ end
309
+ -- The root of the bounded heap is the worst retained candidate.
310
+ local heap, visited = {}, 0
311
+ for id in pairs(scores) do
312
+ visited = visited + 1
313
+ if visited % 1024 == 0 then control.check() end
314
+ if #heap < limit then
315
+ local i = #heap + 1
316
+ while i > 1 do
317
+ local parent = math.floor(i / 2)
318
+ if not better(heap[parent], id) then break end
319
+ heap[i], i = heap[parent], parent
320
+ end
321
+ heap[i] = id
322
+ elseif better(id, heap[1]) then
323
+ local i, child = 1, 2
324
+ while child <= #heap do
325
+ if child < #heap and better(heap[child], heap[child + 1]) then child = child + 1 end
326
+ if not better(id, heap[child]) then break end
327
+ heap[i], i = heap[child], child
328
+ child = i * 2
329
+ end
330
+ heap[i] = id
331
+ end
332
+ end
333
+ table.sort(heap, better)
334
+ return heap
335
+ end
336
+
337
+ local function score_terms(file, entries, words, first, boundary, accepts, match)
338
+ local scores = {}
339
+ local counts = match ~= 'all' and #words > 1 and {} or nil
340
+ local rarest, seed
341
+ if match == 'all' and #words > 1 then
342
+ for _, word in ipairs(words) do
343
+ local entry = entries[word]
344
+ if not entry then return scores end
345
+ -- IDF decreases as the number of documents for a word increases.
346
+ if not rarest or entry[4] > entries[rarest][4] then rarest = word end
347
+ end
348
+ seed = read_block(file, entries[rarest], first, boundary)
349
+ local scanned = 0
350
+ for id in pairs(seed) do
351
+ scanned = scanned + 1
352
+ if scanned % 1024 == 0 then control.check() end
353
+ if accepts(id) then scores[id] = 0 end
354
+ end
355
+ end
356
+ -- Accumulate terms in the same order for union and intersection searches,
357
+ -- so a surviving document keeps the same BM25 score and tie-breaking.
358
+ for _, word in ipairs(words) do
359
+ control.check()
360
+ if rarest and not next(scores) then break end
361
+ local entry = entries[word]
362
+ if entry then
363
+ local matches = word == rarest and seed or read_block(file, entry, first, boundary)
364
+ local scanned = 0
365
+ if rarest then
366
+ for id, score in pairs(scores) do
367
+ scanned = scanned + 1
368
+ if scanned % 1024 == 0 then control.check() end
369
+ local tf = matches[id]
370
+ scores[id] = tf and (score + entry[4] * tf) or nil
371
+ end
372
+ if word == rarest then seed = nil end
373
+ else
374
+ for id, tf in pairs(matches) do
375
+ scanned = scanned + 1
376
+ if scanned % 1024 == 0 then control.check() end
377
+ if accepts(id) then
378
+ scores[id] = (scores[id] or 0) + entry[4] * tf
379
+ if counts then counts[id] = (counts[id] or 0) + 1 end
380
+ end
381
+ end
382
+ end
383
+ end
384
+ end
385
+ return scores, counts
386
+ end
387
+
388
+ local function contains_literal(text, literal, left_boundary, right_boundary)
389
+ if text == '' then return false end
390
+ text = text:lower()
391
+ local offset = 1
392
+ while true do
393
+ local first, last = text:find(literal, offset, true)
394
+ if not first then return false end
395
+ if (not left_boundary or first == 1 or not text:sub(first - 1, first - 1):match('[%w_$]'))
396
+ and (not right_boundary or not text:sub(last + 1, last + 1):match('[%w_$]')) then
397
+ return true
398
+ end
399
+ offset = first + 1
400
+ end
401
+ end
402
+
403
+ local function boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
404
+ local exact = {}
405
+ if not header.passages or #words == 0 or literal == '' or not next(scores) then return exact end
406
+ local strength = 0
407
+ for _, word in ipairs(words) do
408
+ local entry = header.words[word]
409
+ if not entry then return exact end
410
+ strength = strength + entry[4]
411
+ end
412
+ -- Group by block so each compressed passage is read at most once per query.
413
+ local groups, scanned = {}, 0
414
+ for id in pairs(scores) do
415
+ scanned = scanned + 1
416
+ if scanned % 128 == 0 then control.check() end
417
+ if not counts or counts[id] == #words then
418
+ local blocks = header.passages[header.locations[id][1]]
419
+ local offset = id - blocks[1]
420
+ local entry = blocks[2 + math.floor(offset / PASSAGE_BLOCK)]
421
+ local group = groups[entry]
422
+ if not group then group = {}; groups[entry] = group end
423
+ group[#group + 1] = id
424
+ end
425
+ end
426
+ if not next(groups) then return exact end
427
+ local decoder = xcompress.new_decompressor()
428
+ local left_boundary, right_boundary = literal:sub(1, 1):match('[%w_$]'), literal:sub(-1):match('[%w_$]')
429
+ for entry, ids in pairs(groups) do
430
+ control.check()
431
+ local data = read_data(file, entry, first, boundary)
432
+ local passages = cmsgpack.unpack(assert(decoder:deflate(data, entry[4]), 'corrupt BM25 passages'))
433
+ for _, id in ipairs(ids) do
434
+ local location = header.locations[id]
435
+ local offset = id - header.passages[location[1]][1]
436
+ local passage = passages[1 + offset % PASSAGE_BLOCK]
437
+ local best
438
+ for _, text in ipairs({location[2], header.files[location[1]], passage[1], passage[2]}) do
439
+ if not exact[id] and contains_literal(text, literal, left_boundary, right_boundary) then exact[id] = true end
440
+ if #words > 1 and best ~= 0 and text ~= '' then
441
+ local gap = collect.gap(text, terms, #words)
442
+ if gap and (not best or gap < best) then best = gap end
443
+ end
444
+ end
445
+ if best then scores[id] = scores[id] + strength / (1 + best) end
446
+ end
447
+ end
448
+ decoder:close()
449
+ return exact
450
+ end
451
+
452
+ -- Use the existing symbol and path tables. Prefix hints change ranking only.
453
+ local function boost_prefix(header, scores, words, prefix)
454
+ if not prefix or prefix == '' then return end
455
+ prefix = prefix:lower()
456
+ local strength = 0
457
+ for _, word in ipairs(words) do
458
+ local entry = header.words[word]
459
+ if entry then strength = strength + entry[4] end
460
+ end
461
+ strength = math.max(strength, 1)
462
+ local path_scores, scanned = {}, 0
463
+ local directory = '/' .. prefix .. '/'
464
+ for id, score in pairs(scores) do
465
+ scanned = scanned + 1
466
+ if scanned % 128 == 0 then control.check() end
467
+ local location = header.locations[id]
468
+ local file_id = location[1]
469
+ local weight = path_scores[file_id]
470
+ if not weight then
471
+ local path = header.files[file_id]:lower()
472
+ local stem = path:match('[^/]+$'):gsub('%.[^%.]+$', '')
473
+ if stem == prefix then weight = 4
474
+ elseif ('/' .. path):find(directory, 1, true) then weight = 3
475
+ elseif stem:sub(1, #prefix) == prefix and stem:sub(#prefix + 1, #prefix + 1):match('[_%.%-]') then
476
+ weight = 1
477
+ else weight = 0 end
478
+ path_scores[file_id] = weight
479
+ end
480
+ local owner = location[2]:match('([^.:]+)[.:]+[^.:]+$')
481
+ if owner and owner:lower() == prefix then weight = 5 end
482
+ if weight > 0 then scores[id] = score + weight * strength end
483
+ end
484
+ end
485
+
486
+ local function search_file(index, file, query, options, limit)
487
+ local header, first, boundary = open_header(file)
488
+ assert(header.stamp == snapshot.stamp(index), 'BM25 index is stale; rebuild it')
489
+ local scope, allowed = scope_key(index, options.path), {}
490
+ local function in_scope(id)
491
+ if not scope then return true end
492
+ local file_id = header.locations[id][1]
493
+ if allowed[file_id] == nil then allowed[file_id] = paths.contains(scope, header.files[file_id]) end
494
+ return allowed[file_id]
495
+ end
496
+ local accepts = in_scope
497
+ if options.accept then
498
+ local accepted = {}
499
+ accepts = function(id)
500
+ if accepted[id] == nil then
501
+ local location = header.locations[id]
502
+ accepted[id] = not not (in_scope(id)
503
+ and options.accept(header.files[location[1]], location[2], location[5]))
504
+ end
505
+ return accepted[id]
506
+ end
507
+ end
508
+ local words, terms = {}, collect.tokenize(query)
509
+ for token in pairs(terms) do words[#words + 1] = token end
510
+ table.sort(words)
511
+ local scores, counts = score_terms(file, header.words, words, first, boundary, accepts, options.match)
512
+ local mark = string.char(96)
513
+ local literal = query:match('^%s*(.-)%s*$'):gsub('^' .. mark .. '+', ''):gsub(mark .. '+$', ''):lower()
514
+ local key = literal:gsub('\\', '/')
515
+ local entry = header.exact[key]
516
+ local exact = entry and read_block(file, entry, first, boundary) or {}
517
+ local literal_matches = boost_candidates(file, header, first, boundary, scores, counts, words, terms, literal)
518
+ if options.match ~= 'all' or #words == 0 then
519
+ for id in pairs(exact) do if accepts(id) then scores[id] = scores[id] or 0 end end
520
+ end
521
+ boost_prefix(header, scores, words, options.prefix)
522
+ local results = {}
523
+ for rank_number, id in ipairs(rank(scores, exact, literal_matches, limit)) do
524
+ local location = header.locations[id]
525
+ results[rank_number] = {path = header.files[location[1]], symbol = location[2],
526
+ start_line = location[3], end_line = location[4], score = scores[id], kind = location[5]}
527
+ end
528
+ return results
529
+ end
530
+
531
+ function M.search(index, file, query, options)
532
+ assert(type(query) == 'string', 'BM25 query must be a string')
533
+ options = options or {}
534
+ assert(options.match == nil or options.match == 'any' or options.match == 'all',
535
+ 'BM25 match must be any or all')
536
+ assert(options.prefix == nil or type(options.prefix) == 'string', 'BM25 prefix hint must be a string')
537
+ local limit = options.limit == nil and 20 or options.limit
538
+ assert(type(limit) == 'number' and limit >= 0 and limit < math.huge and limit % 1 == 0,
539
+ 'BM25 limit must be a nonnegative integer')
540
+ if limit == 0 then return {} end
541
+ local input = assert(io.open(output_path(file), 'rb'))
542
+ local ok, result = pcall(search_file, index, input, query, options, limit)
543
+ input:close()
544
+ if not ok then error(result, 0) end
545
+ return result
546
+ end
547
+
548
+ return M
@@ -0,0 +1,45 @@
1
+ -- Internal source identity shared by readers and the background builder.
2
+ local indexing = require('codeoutline.indexing.index')
3
+ local paths = require('codeoutline.shared.path')
4
+ local control = require('codeoutline.shared.control')
5
+ local M = {magic = 'CODEOUTLINE-BM25\n', version = 3}
6
+
7
+ function M.prefix(version)
8
+ return M.magic .. version .. '\n'
9
+ end
10
+ local cache = setmetatable({}, {__mode = 'k'})
11
+
12
+ function M.stamp(index)
13
+ local cached = cache[index]
14
+ if cached and cached.generation == index.generation then return cached.stamp, cached.files end
15
+ local files, records = {}, {}
16
+ for path in pairs(index.files) do files[#files + 1] = path end
17
+ table.sort(files)
18
+ for i, path in ipairs(files) do
19
+ if i % 128 == 0 then control.check() end
20
+ local file = index.files[path]
21
+ records[i] = {path, file.size, file.crc, file.language, file.cpp_fragment or false}
22
+ end
23
+ local stamp = xutils.sha256_hex(cmsgpack.pack({indexing.VERSION, paths.key(index.root), records}))
24
+ cache[index] = {generation = index.generation, stamp = stamp, files = files}
25
+ return stamp, files
26
+ end
27
+
28
+ function M.current(path, stamp)
29
+ local file = io.open(path, 'rb')
30
+ if not file then return false end
31
+ local expected = M.prefix(M.version) .. stamp
32
+ local prefix = file:read(#expected)
33
+ file:close()
34
+ return prefix == expected
35
+ end
36
+
37
+ -- The background reader validates the disk snapshot against the foreground
38
+ -- index's identity without loading a second copy of its base records.
39
+ function M.reader(root, stamp)
40
+ local index = {root = root}
41
+ cache[index] = {stamp = stamp}
42
+ return index
43
+ end
44
+
45
+ return M
@@ -22,7 +22,7 @@ M.CHUNK_FILES = 32
22
22
  M.CHUNK_BYTES = 1048576
23
23
  M.IDLE_MS = 60000
24
24
 
25
- local threads, owner, shared = {}, nil, nil
25
+ local threads, owner, shared, abort_key = {}, nil, nil, nil
26
26
  local job, waiting, token = nil, false, 0
27
27
  local script_path, thread_count
28
28
  local outstanding, running, idle_since = {}, false, nil
@@ -36,9 +36,15 @@ function M.configure(script, count)
36
36
  script_path, thread_count = script, count or M.THREADS
37
37
  end
38
38
 
39
+ function M.configured_threads()
40
+ local count = tonumber(os.getenv('CODEOUTLINE_INDEX_THREADS') or '') or M.THREADS
41
+ return math.max(0, math.floor(count))
42
+ end
43
+
39
44
  local function start_threads()
40
45
  shared = assert(xshared.dict('codeoutline_control'))
41
46
  owner = xthread.current_id()
47
+ abort_key = 'parse_abort:' .. owner
42
48
  for i = 1, thread_count do
43
49
  local id = M.FIRST_ID + i - 1
44
50
  local ok, err = xthread.create_thread(id, 'PARSE' .. i, script_path)
@@ -51,7 +57,7 @@ end
51
57
 
52
58
  local function stop_threads(reason)
53
59
  local started, count, stopped = xtimer.now_ms(), #threads, 0
54
- if shared then shared:set('parse_abort', token) end
60
+ if shared then shared:set(abort_key, token) end
55
61
  for _, id in ipairs(threads) do
56
62
  local ok, err = xthread.shutdown_thread(id)
57
63
  if ok then stopped = stopped + 1
@@ -129,7 +135,7 @@ M.partition = partition
129
135
 
130
136
  -- jobs: { rel, abs, weight, size?, crc? }; on_result(rel, load_file result)
131
137
  -- runs in this thread, in arrival order. Must be called from the bound job.
132
- function M.run(jobs, max_bytes, on_result)
138
+ function M.run(jobs, max_bytes, on_result, operation)
133
139
  assert(M.ready(), 'parse pool is not available here')
134
140
  token = token + 1
135
141
  local current, pending = token, 0
@@ -142,14 +148,14 @@ function M.run(jobs, max_bytes, on_result)
142
148
  local chunk, bytes = {}, 0
143
149
  local function flush()
144
150
  if #chunk == 0 then return end
145
- local posted, why = xthread.post(threads[i], 'parse', owner, current, max_bytes, chunk)
151
+ local posted, why = xthread.post(threads[i], operation or 'parse', owner, current, max_bytes, chunk)
146
152
  assert(posted, 'cannot post to parse thread: ' .. tostring(why))
147
153
  pending = pending + 1
148
154
  outstanding[current] = (outstanding[current] or 0) + 1
149
155
  chunk, bytes = {}, 0
150
156
  end
151
157
  for _, j in ipairs(bucket) do
152
- chunk[#chunk + 1] = { j.rel, j.abs, j.size or false, j.crc or false, j.language or false }
158
+ chunk[#chunk + 1] = { j.rel, j.abs, j.size or false, j.crc or false, j.language or false, j.documents }
153
159
  bytes = bytes + j.weight
154
160
  if #chunk >= M.CHUNK_FILES or bytes >= M.CHUNK_BYTES then flush() end
155
161
  end
@@ -181,7 +187,7 @@ function M.run(jobs, max_bytes, on_result)
181
187
  xtimer.now_ms() - started)
182
188
  if not ok then
183
189
  -- Workers skip the rest of this pass; late results carry a stale token.
184
- shared:set('parse_abort', current)
190
+ shared:set(abort_key, current)
185
191
  error(err, 0)
186
192
  end
187
193
  end
@@ -50,7 +50,7 @@ if #config.roots == 0 then
50
50
  end
51
51
  local shared = assert(xshared.create('codeoutline_control', 65536, 256))
52
52
  local jobs, next_id, job_count, worker_ready = {}, 0, 0, false
53
- local stopping, indexing = false, false
53
+ local stopping, indexing, building_bm25 = false, false, false
54
54
  local endpoint, stdio, input = nil, nil, ''
55
55
  local updating = false
56
56
  local startup = xtimer.now_ms()
@@ -129,6 +129,9 @@ return {
129
129
  end
130
130
  os.remove(daemon.stop_file(config.port))
131
131
  end
132
+ local bm25 = require('codeoutline.bm25.automatic')
133
+ building_bm25 = xthread.create_thread(bm25.worker_id, 'BM25', scripts .. '/codeoutline/runtime/bm25_worker.lua')
134
+ shared:set('bm25_worker', building_bm25 and true or false)
132
135
  assert(xthread.create_thread(2, 'INDEX', scripts .. '/codeoutline/runtime/index_worker.lua'))
133
136
  indexing = true
134
137
  -- Launchers enable this for serve; the background service updates unless
@@ -264,6 +267,7 @@ return {
264
267
  shared:set('shutdown', true)
265
268
  if stdio then stdio:close() end
266
269
  if endpoint then endpoint.close() end
270
+ if building_bm25 then xthread.shutdown_thread(require('codeoutline.bm25.automatic').worker_id) end
267
271
  if indexing then xthread.shutdown_thread(2) end
268
272
  if updating then xthread.shutdown_thread(3) end
269
273
  xnet.uninit()