pikuri-fileindex 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +114 -0
- data/lib/pikuri/fileindex/extension.rb +36 -0
- data/lib/pikuri/fileindex/local_search.rb +402 -0
- data/lib/pikuri/fileindex/query.rb +50 -0
- data/lib/pikuri/fileindex/read.rb +231 -0
- data/lib/pikuri/fileindex/recoll.rb +372 -0
- data/lib/pikuri/fileindex/roots.rb +109 -0
- data/lib/pikuri/fileindex/search.rb +288 -0
- data/lib/pikuri-fileindex.rb +70 -0
- metadata +75 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'stringio'
|
|
4
|
+
|
|
5
|
+
module Pikuri
|
|
6
|
+
module Fileindex
|
|
7
|
+
# The +fileindex_read+ tool — read a file's full text *as stored in
|
|
8
|
+
# the desktop index*, line-paged like pikuri-workspace's +read+. The
|
|
9
|
+
# companion to {Search} (which locates + previews one match),
|
|
10
|
+
# sharing its swappable +backend:+ ({LocalSearch} / {Recoll}).
|
|
11
|
+
#
|
|
12
|
+
# It reads the index, not the file: the index already holds every
|
|
13
|
+
# indexed file's extracted plain text — PDFs/office docs included — so
|
|
14
|
+
# this returns them in full with no extractor dependency. The cost is
|
|
15
|
+
# that it's the *indexed snapshot* (can lag very recent edits, covers
|
|
16
|
+
# only files the index knows); a file just written is read from disk,
|
|
17
|
+
# with pikuri-workspace's +read+.
|
|
18
|
+
#
|
|
19
|
+
# +offset+ / +limit+ page exactly as in pikuri-workspace's +read+,
|
|
20
|
+
# windowed through {Pikuri::Extractor.extract_paged} (a +StringIO+ over
|
|
21
|
+
# the stored text) and rendered by {Pikuri::Tool::Result.page}, so the
|
|
22
|
+
# +cat -n+ prefix, trailer and caps are +read+'s own.
|
|
23
|
+
#
|
|
24
|
+
# +pattern+ turns the read into a +grep -n+ over the same text: only the
|
|
25
|
+
# matching lines, numbered as a read numbers them. The search hit names
|
|
26
|
+
# no line, so without it the model's only way to an answer deep in a
|
|
27
|
+
# long PDF is to page through all of it; with it, the model reads just
|
|
28
|
+
# around the lines that match. A regex, not the index's stemmed words:
|
|
29
|
+
# inside one file an exact match is what the model wants, and the
|
|
30
|
+
# index's own match positions (+fts:offsets+) came back as +(null):17+
|
|
31
|
+
# on LocalSearch 3.11, naming no property.
|
|
32
|
+
#
|
|
33
|
+
# A line longer than the page's per-line cap is broken at a space first
|
|
34
|
+
# ({.wrap}): localsearch keeps a PDF's line breaks and gives HTML a line
|
|
35
|
+
# per paragraph, but hands an office document over as one line, which
|
|
36
|
+
# the cap would cut after 2,000 characters, with no offset reaching the
|
|
37
|
+
# rest.
|
|
38
|
+
#
|
|
39
|
+
# Only a path inside the +roots:+, with nothing hidden below a root, is
|
|
40
|
+
# read; the path is the model's, so it is checked here even though the
|
|
41
|
+
# localsearch miner never indexes a hidden file (recoll does).
|
|
42
|
+
#
|
|
43
|
+
# Sharing: +P_stateless+ — the backend is a stateless module over a
|
|
44
|
+
# read-only index, so one instance serves any number of agents.
|
|
45
|
+
class Read < Pikuri::Tool
|
|
46
|
+
# @return [Integer] default +limit+ (lines per call); the shared
|
|
47
|
+
# page default, so it matches +read+.
|
|
48
|
+
DEFAULT_LIMIT = Pikuri::Extractor::PAGE_DEFAULT_LIMIT
|
|
49
|
+
|
|
50
|
+
# @return [Numeric] seconds one +pattern+ may spend matching, so a
|
|
51
|
+
# catastrophically backtracking regex fails instead of hanging the
|
|
52
|
+
# turn.
|
|
53
|
+
PATTERN_TIMEOUT = 2
|
|
54
|
+
|
|
55
|
+
# @return [String] opencode-shape description (summary + Usage).
|
|
56
|
+
DESCRIPTION = <<~DESC
|
|
57
|
+
Read a file's full text from the desktop search index, with line numbers.
|
|
58
|
+
|
|
59
|
+
Usage:
|
|
60
|
+
- The companion to fileindex_search: after it locates a file, read the file in full here.
|
|
61
|
+
- Reads the indexed text, so PDFs and office documents come back as their extracted text — no special handling needed.
|
|
62
|
+
- This is the indexed snapshot: it can lag very recent edits, and only the files fileindex_search can find are available. For a file just written, use the read tool instead.
|
|
63
|
+
- Output is line-numbered `cat -n` style; use `offset` and `limit` to page. When the response ends in `Use offset=N to continue`, call again with that offset.
|
|
64
|
+
- To find something in a long file without reading all of it, pass `pattern`: you get only the matching lines with their line numbers. Then read around a match with `offset` a few lines before it and a small `limit`.
|
|
65
|
+
DESC
|
|
66
|
+
|
|
67
|
+
# @param backend [Module] a ready file-index backend, from
|
|
68
|
+
# {Fileindex.pick}; this tool uses only its +read_text+.
|
|
69
|
+
# @param roots [Roots] the folders whose files may be read; their
|
|
70
|
+
# declaration grades the tool's legs
|
|
71
|
+
# @return [Read]
|
|
72
|
+
def initialize(backend:, roots:)
|
|
73
|
+
@backend = backend
|
|
74
|
+
super(
|
|
75
|
+
name: 'fileindex_read',
|
|
76
|
+
description: DESCRIPTION,
|
|
77
|
+
parameters: Parameters.build { |p|
|
|
78
|
+
p.required_string :path,
|
|
79
|
+
'Absolute path to the file (as returned by ' \
|
|
80
|
+
'fileindex_search), e.g. "/home/u/Documents/report.pdf".'
|
|
81
|
+
p.optional_integer :offset,
|
|
82
|
+
'Line number to start reading from (1-indexed). ' \
|
|
83
|
+
'Defaults to 1, e.g. 200.'
|
|
84
|
+
p.optional_integer :limit,
|
|
85
|
+
'Maximum number of lines to read. Defaults to ' \
|
|
86
|
+
"#{DEFAULT_LIMIT}, e.g. 500. With `pattern`, the most matching lines to return."
|
|
87
|
+
p.optional_string :pattern,
|
|
88
|
+
'Case-insensitive regular expression; returns only the lines that match, ' \
|
|
89
|
+
'with their line numbers, searching from `offset` on, e.g. "total|amount due".'
|
|
90
|
+
},
|
|
91
|
+
execute: lambda { |path:, offset: 1, limit: DEFAULT_LIMIT, pattern: nil|
|
|
92
|
+
Read.read(path: path, offset: offset, limit: limit, pattern: pattern, backend: backend, roots: roots)
|
|
93
|
+
},
|
|
94
|
+
trifecta_legs: roots.legs
|
|
95
|
+
)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @param path [String] absolute or +~+-relative, inside +roots+
|
|
99
|
+
# @param backend [Module] the file-index backend to read from
|
|
100
|
+
# @param roots [Roots] the folders whose files may be read
|
|
101
|
+
# @param offset [Integer] 1-indexed start line
|
|
102
|
+
# @param limit [Integer] max lines; with +pattern+, max matching lines
|
|
103
|
+
# @param pattern [String, nil] case-insensitive regex; when given, only
|
|
104
|
+
# the matching lines from +offset+ on are returned (see {.grep})
|
|
105
|
+
# @return [Pikuri::Tool::Result] the cat-n windowed text (+"Read 214
|
|
106
|
+
# lines"+), the matching lines (+"Found 6 lines"+), or
|
|
107
|
+
# {Pikuri::Tool::Result.error}
|
|
108
|
+
def self.read(path:, backend:, roots:, offset: 1, limit: DEFAULT_LIMIT, pattern: nil)
|
|
109
|
+
return Pikuri::Tool::Result.error("offset must be >= 1, got #{offset}") if offset < 1
|
|
110
|
+
return Pikuri::Tool::Result.error("limit must be >= 1, got #{limit}") if limit < 1
|
|
111
|
+
|
|
112
|
+
pattern = pattern.to_s
|
|
113
|
+
regex = nil
|
|
114
|
+
unless pattern.empty?
|
|
115
|
+
begin
|
|
116
|
+
regex = Regexp.new(pattern, Regexp::IGNORECASE, timeout: PATTERN_TIMEOUT)
|
|
117
|
+
rescue RegexpError => e
|
|
118
|
+
return Pikuri::Tool::Result.error("bad pattern: #{e.message}")
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
return Pikuri::Tool::Result.error("path must be absolute, got '#{path}'.") unless path.start_with?('/', '~')
|
|
123
|
+
|
|
124
|
+
begin
|
|
125
|
+
path = File.expand_path(path)
|
|
126
|
+
rescue ArgumentError => e # "~nosuchuser"
|
|
127
|
+
return Pikuri::Tool::Result.error("bad path '#{path}': #{e.message}")
|
|
128
|
+
end
|
|
129
|
+
real = roots.resolve(path)
|
|
130
|
+
unless real
|
|
131
|
+
return Pikuri::Tool::Result.error("#{path} is not inside the folders you can read " \
|
|
132
|
+
"(#{roots.dirs.join(', ')}), or is hidden.")
|
|
133
|
+
end
|
|
134
|
+
path = real
|
|
135
|
+
|
|
136
|
+
text = backend.read_text(path)
|
|
137
|
+
if text.nil?
|
|
138
|
+
return Pikuri::Tool::Result.error("#{path} is not in the index (PDFs and office docs are included). " \
|
|
139
|
+
'For a file just written, use the read tool.')
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
text = wrap(text)
|
|
143
|
+
return grep(text, regex, offset: offset, limit: limit) if regex
|
|
144
|
+
|
|
145
|
+
page = Pikuri::Extractor.extract_paged(
|
|
146
|
+
StringIO.new(text), content_type: 'text/plain', offset: offset, limit: limit
|
|
147
|
+
)
|
|
148
|
+
Pikuri::Tool::Result.page(page, empty_note: '(No indexed text for this file.)')
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# The lines of +text+ from +offset+ on that match +regex+, +cat -n+
|
|
152
|
+
# numbered as a plain read numbers them, stopping at +limit+ matches or
|
|
153
|
+
# {Pikuri::Extractor::PAGE_MAX_BYTES}, whichever comes first; a stop
|
|
154
|
+
# names the offset to continue from.
|
|
155
|
+
#
|
|
156
|
+
# @param text [String] the file's indexed text
|
|
157
|
+
# @param regex [Regexp]
|
|
158
|
+
# @param offset [Integer] 1-indexed first line searched
|
|
159
|
+
# @param limit [Integer] max matching lines
|
|
160
|
+
# @return [Pikuri::Tool::Result] the lines and a footer (+"Found 6
|
|
161
|
+
# lines"+), or {Pikuri::Tool::Result.error} when the pattern times out
|
|
162
|
+
def self.grep(text, regex, offset:, limit:)
|
|
163
|
+
rows = []
|
|
164
|
+
bytes = 0
|
|
165
|
+
stopped_at = nil
|
|
166
|
+
total = 0
|
|
167
|
+
text.each_line(chomp: true).with_index(1) do |line, number|
|
|
168
|
+
total = number
|
|
169
|
+
next if stopped_at || number < offset || !regex.match?(line)
|
|
170
|
+
|
|
171
|
+
row = format("%6d\t%s", number, line)
|
|
172
|
+
if rows.size == limit || bytes + row.bytesize > Pikuri::Extractor::PAGE_MAX_BYTES
|
|
173
|
+
stopped_at = number
|
|
174
|
+
next
|
|
175
|
+
end
|
|
176
|
+
rows << row
|
|
177
|
+
bytes += row.bytesize + 1
|
|
178
|
+
end
|
|
179
|
+
grep_result(rows, regex, total: total, offset: offset, stopped_at: stopped_at)
|
|
180
|
+
rescue Regexp::TimeoutError
|
|
181
|
+
Pikuri::Tool::Result.error("pattern /#{regex.source}/ took over #{PATTERN_TIMEOUT}s on this file; " \
|
|
182
|
+
'simplify it.')
|
|
183
|
+
end
|
|
184
|
+
private_class_method :grep
|
|
185
|
+
|
|
186
|
+
# @return [Pikuri::Tool::Result]
|
|
187
|
+
def self.grep_result(rows, regex, total:, offset:, stopped_at:)
|
|
188
|
+
where = offset > 1 ? " from line #{offset} on" : ''
|
|
189
|
+
if rows.empty?
|
|
190
|
+
return Pikuri::Tool::Result.new(
|
|
191
|
+
content: "No lines match /#{regex.source}/#{where} (the indexed text has " \
|
|
192
|
+
"#{Pikuri::Tool.pluralize(total, 'line')}).",
|
|
193
|
+
summary: 'Found 0 lines'
|
|
194
|
+
)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
footer = "Found #{Pikuri::Tool.pluralize(rows.size, 'matching line')}#{where} " \
|
|
198
|
+
"(the indexed text has #{Pikuri::Tool.pluralize(total, 'line')}). " \
|
|
199
|
+
'To read around one, call again without `pattern`, `offset` a few lines before it.'
|
|
200
|
+
if stopped_at
|
|
201
|
+
footer += " More lines match: call again with the same `pattern` and offset=#{stopped_at} for the rest."
|
|
202
|
+
end
|
|
203
|
+
Pikuri::Tool::Result.new(content: [rows.join("\n"), '', footer].join("\n"),
|
|
204
|
+
summary: "Found #{Pikuri::Tool.pluralize(rows.size, 'line')}")
|
|
205
|
+
end
|
|
206
|
+
private_class_method :grep_result
|
|
207
|
+
|
|
208
|
+
# @param text [String] a file's indexed text
|
|
209
|
+
# @return [String] +text+ with each line longer than
|
|
210
|
+
# {Pikuri::Extractor::PAGE_MAX_LINE_LENGTH} broken at its last
|
|
211
|
+
# whitespace within the cap, or at the cap when it has none; +text+
|
|
212
|
+
# itself when no line is that long
|
|
213
|
+
def self.wrap(text)
|
|
214
|
+
max = Pikuri::Extractor::PAGE_MAX_LINE_LENGTH
|
|
215
|
+
return text if text.each_line.none? { |line| line.chomp.length > max }
|
|
216
|
+
|
|
217
|
+
text.each_line(chomp: true).flat_map do |line|
|
|
218
|
+
pieces = []
|
|
219
|
+
while line.length > max
|
|
220
|
+
cut = line.rindex(/\s/, max)
|
|
221
|
+
cut = max if cut.nil? || cut.zero?
|
|
222
|
+
pieces << line[0, cut]
|
|
223
|
+
line = line[cut..].lstrip
|
|
224
|
+
end
|
|
225
|
+
pieces << line
|
|
226
|
+
end.join("\n")
|
|
227
|
+
end
|
|
228
|
+
private_class_method :wrap
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
end
|
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
module Fileindex
|
|
5
|
+
# Backend wrapper over the +recoll+ full-text index, via its headless
|
|
6
|
+
# +recollq+ CLI (Xapian-backed). The DE-independent sibling of
|
|
7
|
+
# {LocalSearch}: same seam shape ({.search} / {.read_text} /
|
|
8
|
+
# {.unavailable_reason}), interchangeable behind the +fileindex_search+ /
|
|
9
|
+
# +fileindex_read+ tools; {Fileindex.pick} takes it when localsearch
|
|
10
|
+
# isn't ready.
|
|
11
|
+
#
|
|
12
|
+
# Recall on prose ties +localsearch+, so recoll is *not* a better
|
|
13
|
+
# engine — it earns its place where +localsearch+ is structurally blind
|
|
14
|
+
# (source trees, +.git+ checkouts, non-GNOME desktops). Its blind spot
|
|
15
|
+
# is the inverse: recoll indexes exactly the *folders* the user
|
|
16
|
+
# configured (+topdirs+), source code included — everything outside them
|
|
17
|
+
# is invisible. The measured survey behind the pick-one-per-host order
|
|
18
|
+
# is in +pikuri-fileindex/design/architecture.md+.
|
|
19
|
+
#
|
|
20
|
+
# {.search} asks +recollq -F 'url abstract dbytes'+ for three base64-encoded
|
|
21
|
+
# fields per line (recoll's own recommended machine shape) so paths and
|
|
22
|
+
# snippets with spaces/brackets/newlines survive with no escaping
|
|
23
|
+
# guesswork. {.read_text} uses +recollq -d+, which dumps a document's
|
|
24
|
+
# stored/extracted text (PDFs/office docs included) after a header line.
|
|
25
|
+
module Recoll
|
|
26
|
+
# @return [Logger]
|
|
27
|
+
LOGGER = Pikuri.logger_for('Fileindex::Recoll')
|
|
28
|
+
|
|
29
|
+
# @return [String] the headless recoll query CLI.
|
|
30
|
+
BINARY = 'recollq'
|
|
31
|
+
|
|
32
|
+
# @return [String] the recoll indexer/config CLI (used only to locate
|
|
33
|
+
# the config; never invoked to *index* from here).
|
|
34
|
+
INDEXER = 'recollindex'
|
|
35
|
+
|
|
36
|
+
# @return [String] recoll config filename inside the config dir.
|
|
37
|
+
CONFIG_FILE = 'recoll.conf'
|
|
38
|
+
|
|
39
|
+
# recoll's compiled-in default when +topdirs+ is unset: the whole
|
|
40
|
+
# home directory (the sample config ships +topdirs = ~+ too).
|
|
41
|
+
#
|
|
42
|
+
# @return [String]
|
|
43
|
+
DEFAULT_TOPDIRS = '~'
|
|
44
|
+
|
|
45
|
+
# @return [String] the Xapian index directory name inside the config
|
|
46
|
+
# dir; missing until +recollindex+ first runs.
|
|
47
|
+
INDEX_DIR = 'xapiandb'
|
|
48
|
+
|
|
49
|
+
# @return [String] a query matching every document in the index.
|
|
50
|
+
ANY_DOCUMENT = 'mime:*'
|
|
51
|
+
|
|
52
|
+
# Raised when a recollq invocation exits non-zero — a recoverable
|
|
53
|
+
# failure the calling tool renders as +"Error: ..."+.
|
|
54
|
+
class CommandError < StandardError; end
|
|
55
|
+
|
|
56
|
+
# @return [String] short human name for this backend, used by the
|
|
57
|
+
# fileindex tools when prefixing an +"Error: ..."+ observation.
|
|
58
|
+
def self.label = 'recoll'
|
|
59
|
+
|
|
60
|
+
# Plain-English note of what this backend will *not* surface, appended
|
|
61
|
+
# to the fileindex tools' LLM description so the model knows when to
|
|
62
|
+
# fall back to a filesystem text/regex search. A *method*, not a
|
|
63
|
+
# constant like {LocalSearch::LIMITATIONS}, because the blind spot is
|
|
64
|
+
# host-specific: the user's +topdirs+, only readable at runtime
|
|
65
|
+
# ({.indexed_dirs}). Falls back to a generic caveat when the config
|
|
66
|
+
# can't be read.
|
|
67
|
+
#
|
|
68
|
+
# @return [String]
|
|
69
|
+
def self.limitations
|
|
70
|
+
dirs = indexed_dirs
|
|
71
|
+
return generic_limitation if dirs.empty?
|
|
72
|
+
|
|
73
|
+
listed = dirs.map { |dir| "- #{dir}" }.join("\n")
|
|
74
|
+
<<~LIMITS.chomp
|
|
75
|
+
Only files under these folders are indexed (anything elsewhere on disk is not in this index — search those files' text directly instead):
|
|
76
|
+
#{listed}
|
|
77
|
+
LIMITS
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Folders recoll is configured to index, read from +topdirs+ in the
|
|
81
|
+
# active config file. Resolution mirrors recoll itself: the config dir
|
|
82
|
+
# is +$RECOLL_CONFDIR+ or +~/.recoll+; +topdirs+ is a space-separated
|
|
83
|
+
# list (quoted entries may contain spaces, a trailing +\+ continues to
|
|
84
|
+
# the next line) living in the global section, i.e. above the first
|
|
85
|
+
# +[section]+ header. Entries are +~+- and +$VAR+-expanded to absolute
|
|
86
|
+
# paths. Defaults to the home dir ({DEFAULT_TOPDIRS}) when the config
|
|
87
|
+
# is absent or names no +topdirs+, matching recoll's own default.
|
|
88
|
+
#
|
|
89
|
+
# @return [Array<String>] absolute folder paths, or +[]+ if the config
|
|
90
|
+
# dir exists but the file is unreadable
|
|
91
|
+
def self.indexed_dirs
|
|
92
|
+
path = File.join(confdir, CONFIG_FILE)
|
|
93
|
+
raw = File.exist?(path) ? File.read(path) : nil
|
|
94
|
+
entries = raw ? parse_topdirs(raw) : [DEFAULT_TOPDIRS]
|
|
95
|
+
entries = [DEFAULT_TOPDIRS] if entries.empty?
|
|
96
|
+
entries.map { |entry| expand_entry(entry) }
|
|
97
|
+
rescue SystemCallError
|
|
98
|
+
[]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# The recoll configuration directory recoll would use: +$RECOLL_CONFDIR+
|
|
102
|
+
# if set and non-empty, else +~/.recoll+.
|
|
103
|
+
#
|
|
104
|
+
# @return [String]
|
|
105
|
+
def self.confdir
|
|
106
|
+
env = ENV['RECOLL_CONFDIR']
|
|
107
|
+
env && !env.empty? ? File.expand_path(env) : File.expand_path('~/.recoll')
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Run +recollq -F 'url abstract dbytes' -n <limit>+ and parse the results.
|
|
111
|
+
# The query becomes recoll's query language ({.language}), with the
|
|
112
|
+
# folders as one +(dir:"<a>" OR dir:"<b>")+ group appended; recoll
|
|
113
|
+
# matches a +dir:+ by whole path components (+/a/sp+ does not match
|
|
114
|
+
# +/a/sp ace+).
|
|
115
|
+
#
|
|
116
|
+
# The model's words are spliced into that language, so the folder group
|
|
117
|
+
# narrows the search but can't be relied on to bound it: the
|
|
118
|
+
# +fileindex_search+ tool drops any hit outside its roots.
|
|
119
|
+
#
|
|
120
|
+
# @param query [Query] a file matches every word of any group
|
|
121
|
+
# @param limit [Integer]
|
|
122
|
+
# @param folders [Array<String>] absolute folders to search under,
|
|
123
|
+
# subfolders included; at least one
|
|
124
|
+
# @return [Array<Hash>] +[{ path: String, snippet: String|nil,
|
|
125
|
+
# text_size: Integer|nil }, …]+, de-duplicated by path; +text_size+
|
|
126
|
+
# is the extracted text's bytes (+dbytes+), +nil+ when there is none
|
|
127
|
+
# @raise [CommandError] if recollq exits non-zero, or a folder contains
|
|
128
|
+
# a +"+ (recoll's query language cannot quote one)
|
|
129
|
+
def self.search(query:, limit:, folders:)
|
|
130
|
+
parse_search_output(query_fields('url abstract dbytes', limit, "#{language(query)} #{within(folders)}"))
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# @param query [Query]
|
|
134
|
+
# @return [String] each group's words in parentheses, the groups joined
|
|
135
|
+
# by +OR+: +(notice period) OR (tenant)+. recoll's +OR+ binds tighter
|
|
136
|
+
# than the implicit AND, so the folder group appended after it bounds
|
|
137
|
+
# the whole.
|
|
138
|
+
def self.language(query)
|
|
139
|
+
query.groups.map { |words| "(#{words.join(' ')})" }.join(' OR ')
|
|
140
|
+
end
|
|
141
|
+
private_class_method :language
|
|
142
|
+
|
|
143
|
+
# The path of every file {.search} would match, unranked and uncapped,
|
|
144
|
+
# so a caller can tell how many matched beyond its +limit+. recollq's
|
|
145
|
+
# own "N results" line can't stand in: it counts files the caller
|
|
146
|
+
# filters out, hidden ones included.
|
|
147
|
+
#
|
|
148
|
+
# @param query [Query]
|
|
149
|
+
# @param folders [Array<String>] as {.search}'s
|
|
150
|
+
# @return [Array<String>] absolute paths
|
|
151
|
+
# @raise [CommandError] as {.search}
|
|
152
|
+
def self.paths(query:, folders:)
|
|
153
|
+
parse_search_output(query_fields('url', 0, "#{language(query)} #{within(folders)}")).map { |hit| hit[:path] }
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# @param fields [String] recollq's +-F+ field list
|
|
157
|
+
# @param limit [Integer] max results; 0 for no limit
|
|
158
|
+
# @param query [String]
|
|
159
|
+
# @return [String] recollq's stdout
|
|
160
|
+
# @raise [CommandError] on a non-zero exit
|
|
161
|
+
def self.query_fields(fields, limit, query)
|
|
162
|
+
result = Pikuri::Subprocess.spawn(BINARY, '-F', fields, '-n', limit.to_s, query, chdir: '/').wait
|
|
163
|
+
return result.output if result.status.success?
|
|
164
|
+
|
|
165
|
+
stderr = result.output.strip
|
|
166
|
+
raise CommandError, stderr.empty? ? "exited #{result.status.exitstatus}" : stderr
|
|
167
|
+
end
|
|
168
|
+
private_class_method :query_fields
|
|
169
|
+
|
|
170
|
+
# Parse +recollq -F 'url abstract dbytes'+ output into hits. Each data
|
|
171
|
+
# line is three base64 tokens (url, abstract, text bytes); an empty field
|
|
172
|
+
# still prints a token. recollq's +"Recoll query: …"+ /
|
|
173
|
+
# +"N results"+ banner lines are skipped because their first token
|
|
174
|
+
# isn't valid base64 for a +file://+ URL.
|
|
175
|
+
#
|
|
176
|
+
# @param raw [String]
|
|
177
|
+
# @return [Array<Hash>] +[{ path:, snippet:, text_size: }, …]+ de-duped
|
|
178
|
+
# by path
|
|
179
|
+
def self.parse_search_output(raw)
|
|
180
|
+
hits = []
|
|
181
|
+
raw.each_line do |line|
|
|
182
|
+
url_b64, abstract_b64, dbytes_b64 = line.chomp.split(/ /)
|
|
183
|
+
url = decode_base64(url_b64)
|
|
184
|
+
next unless url&.start_with?('file://')
|
|
185
|
+
|
|
186
|
+
snippet = decode_base64(abstract_b64.to_s)&.strip
|
|
187
|
+
size = decode_base64(dbytes_b64.to_s).to_i
|
|
188
|
+
hits << { path: url.delete_prefix('file://'), snippet: (snippet unless snippet.to_s.empty?),
|
|
189
|
+
text_size: (size if size.positive?) }
|
|
190
|
+
end
|
|
191
|
+
hits.uniq { |hit| hit[:path] }
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Full stored/extracted text of an indexed file, via +recollq -d+
|
|
195
|
+
# scoped to the file's own name. Works for PDFs/office docs (extracted
|
|
196
|
+
# at index time). +nil+ when the path isn't in the index or has no
|
|
197
|
+
# stored text — the caller turns that into an LLM-facing message.
|
|
198
|
+
#
|
|
199
|
+
# @param path [String]
|
|
200
|
+
# @return [String, nil]
|
|
201
|
+
def self.read_text(path)
|
|
202
|
+
abs = File.expand_path(path)
|
|
203
|
+
result = Pikuri::Subprocess.spawn(BINARY, '-d', '-n', '50', "filename:#{File.basename(abs)}",
|
|
204
|
+
chdir: '/').wait
|
|
205
|
+
return nil unless result.status.success?
|
|
206
|
+
|
|
207
|
+
text = extract_dumped_text(result.output, abs)
|
|
208
|
+
text && !text.strip.empty? ? text : nil
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Pull the dumped body of the result whose header line matches +abs+
|
|
212
|
+
# out of +recollq -d+ output. Each result is a tab-delimited header
|
|
213
|
+
# line (+<mime>\t[<url>]\t[<title>]\t<n>\tbytes+) followed by its text,
|
|
214
|
+
# up to the next header line.
|
|
215
|
+
#
|
|
216
|
+
# @param raw [String]
|
|
217
|
+
# @param abs [String] absolute path to match against the header URL
|
|
218
|
+
# @return [String, nil]
|
|
219
|
+
def self.extract_dumped_text(raw, abs)
|
|
220
|
+
want = "file://#{abs}"
|
|
221
|
+
body = []
|
|
222
|
+
capturing = false
|
|
223
|
+
raw.each_line do |line|
|
|
224
|
+
url = line[%r{\t\[(file://[^\]]*)\]\t}, 1]
|
|
225
|
+
if url
|
|
226
|
+
break if capturing
|
|
227
|
+
|
|
228
|
+
capturing = (url == want)
|
|
229
|
+
elsif capturing
|
|
230
|
+
body << line
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
capturing || !body.empty? ? body.join.chomp : nil
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Why this index can't answer a search under +folders+ right now, for
|
|
237
|
+
# a human to act on; +nil+ when it can. A built index holding no
|
|
238
|
+
# document under any of them is not ready either: every search would
|
|
239
|
+
# come back empty. Some empty folders are normal (an empty Desktop), so
|
|
240
|
+
# they only log a warning naming each.
|
|
241
|
+
#
|
|
242
|
+
# Recoll.unavailable_reason(folders: ['/home/u/notes'])
|
|
243
|
+
# # => "no index in /home/u/.recoll yet. Build one with `recollindex`."
|
|
244
|
+
#
|
|
245
|
+
# @param folders [Array<String>] absolute real paths
|
|
246
|
+
# @return [String, nil]
|
|
247
|
+
def self.unavailable_reason(folders:)
|
|
248
|
+
quoted = folders.find { |folder| folder.include?('"') }
|
|
249
|
+
return "recoll can't search #{quoted}: its query language can't quote a double quote in a path." if quoted
|
|
250
|
+
|
|
251
|
+
empty = folders.reject { |folder| holds_document?(folder) }
|
|
252
|
+
if empty.size < folders.size
|
|
253
|
+
empty.each { |folder| LOGGER.warn("nothing under #{folder} is indexed; searches there find nothing") }
|
|
254
|
+
return nil
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
"nothing under #{folders.join(', ')} is in the index in #{confdir}. Check `topdirs` in " \
|
|
258
|
+
"#{File.join(confdir, CONFIG_FILE)}, then rerun `recollindex`."
|
|
259
|
+
rescue Errno::ENOENT
|
|
260
|
+
"the '#{BINARY}' command is missing. Install the 'recoll' package, then build an index " \
|
|
261
|
+
'with `recollindex`.'
|
|
262
|
+
rescue CommandError => e
|
|
263
|
+
return "no index in #{confdir} yet. Build one with `recollindex`." unless File.directory?(File.join(confdir, INDEX_DIR))
|
|
264
|
+
|
|
265
|
+
"the index in #{confdir} doesn't answer: #{e.message}"
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# @param folder [String]
|
|
269
|
+
# @return [Boolean] whether the index holds any document under +folder+
|
|
270
|
+
# @raise [CommandError] if recollq exits non-zero (message: its last
|
|
271
|
+
# line), or +folder+ contains a +"+
|
|
272
|
+
def self.holds_document?(folder)
|
|
273
|
+
result = Pikuri::Subprocess.spawn(BINARY, '-F', 'url', '-n', '1', "#{ANY_DOCUMENT} #{within([folder])}",
|
|
274
|
+
chdir: '/').wait
|
|
275
|
+
raise CommandError, result.output.strip.lines.last&.strip.to_s unless result.status.success?
|
|
276
|
+
|
|
277
|
+
result.output.each_line.any? { |line| decode_base64(line.strip)&.start_with?('file://') }
|
|
278
|
+
end
|
|
279
|
+
private_class_method :holds_document?
|
|
280
|
+
|
|
281
|
+
# @param folders [Array<String>]
|
|
282
|
+
# @return [String] +(dir:"<a>" OR dir:"<b>")+; positive +dir:+ clauses
|
|
283
|
+
# side by side would be ANDed
|
|
284
|
+
# @raise [CommandError] if a folder contains a +"+
|
|
285
|
+
def self.within(folders)
|
|
286
|
+
quoted = folders.find { |folder| folder.include?('"') }
|
|
287
|
+
raise CommandError, "can't search a folder whose path contains a double quote: #{quoted}" if quoted
|
|
288
|
+
|
|
289
|
+
"(#{folders.map { |folder| "dir:\"#{folder}\"" }.join(' OR ')})"
|
|
290
|
+
end
|
|
291
|
+
private_class_method :within
|
|
292
|
+
|
|
293
|
+
# Join backslash-continued lines, then read the +topdirs+ value from
|
|
294
|
+
# the global section (everything before the first +[section]+ header).
|
|
295
|
+
# The last assignment wins (ConfSimple semantics).
|
|
296
|
+
#
|
|
297
|
+
# @param raw [String]
|
|
298
|
+
# @return [Array<String>] raw (unexpanded) topdirs entries
|
|
299
|
+
def self.parse_topdirs(raw)
|
|
300
|
+
value = nil
|
|
301
|
+
joined_lines(raw).each do |line|
|
|
302
|
+
stripped = line.strip
|
|
303
|
+
break if stripped.start_with?('[') # first [section] ends the global scope
|
|
304
|
+
next if stripped.empty? || stripped.start_with?('#')
|
|
305
|
+
|
|
306
|
+
key, _, rest = stripped.partition('=')
|
|
307
|
+
value = rest if key.strip == 'topdirs'
|
|
308
|
+
end
|
|
309
|
+
value ? tokenize(value) : []
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# @param raw [String]
|
|
313
|
+
# @return [Array<String>] logical lines with trailing-backslash
|
|
314
|
+
# continuations folded into one
|
|
315
|
+
def self.joined_lines(raw)
|
|
316
|
+
lines = []
|
|
317
|
+
buffer = +''
|
|
318
|
+
raw.each_line(chomp: true) do |line|
|
|
319
|
+
if line.end_with?('\\')
|
|
320
|
+
buffer << line[0..-2] << ' '
|
|
321
|
+
else
|
|
322
|
+
lines << (buffer + line)
|
|
323
|
+
buffer = +''
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
lines << buffer unless buffer.empty?
|
|
327
|
+
lines
|
|
328
|
+
end
|
|
329
|
+
private_class_method :joined_lines
|
|
330
|
+
|
|
331
|
+
# Split a topdirs value into entries, honouring +"double quotes"+
|
|
332
|
+
# around paths that contain spaces.
|
|
333
|
+
#
|
|
334
|
+
# @param value [String]
|
|
335
|
+
# @return [Array<String>]
|
|
336
|
+
def self.tokenize(value)
|
|
337
|
+
value.scan(/"([^"]*)"|(\S+)/).map { |quoted, bare| quoted || bare }
|
|
338
|
+
end
|
|
339
|
+
private_class_method :tokenize
|
|
340
|
+
|
|
341
|
+
# Expand +$VAR+ / +${VAR}+ then +~+ to an absolute path; a relative
|
|
342
|
+
# entry resolves against the home dir (recoll's own base).
|
|
343
|
+
#
|
|
344
|
+
# @param entry [String]
|
|
345
|
+
# @return [String]
|
|
346
|
+
def self.expand_entry(entry)
|
|
347
|
+
env_expanded = entry.gsub(/\$\{(\w+)\}|\$(\w+)/) { ENV[::Regexp.last_match(1) || ::Regexp.last_match(2)] || '' }
|
|
348
|
+
File.expand_path(env_expanded, Dir.home)
|
|
349
|
+
end
|
|
350
|
+
private_class_method :expand_entry
|
|
351
|
+
|
|
352
|
+
# @param token [String, nil]
|
|
353
|
+
# @return [String, nil] decoded bytes tagged UTF-8, or +nil+ if +token+
|
|
354
|
+
# is blank or not valid strict base64
|
|
355
|
+
def self.decode_base64(token)
|
|
356
|
+
return nil if token.nil? || token.empty?
|
|
357
|
+
|
|
358
|
+
token.unpack1('m0').force_encoding('UTF-8') # m0: strict base64
|
|
359
|
+
rescue ArgumentError
|
|
360
|
+
nil
|
|
361
|
+
end
|
|
362
|
+
private_class_method :decode_base64
|
|
363
|
+
|
|
364
|
+
# @return [String]
|
|
365
|
+
def self.generic_limitation
|
|
366
|
+
'Only folders configured in recoll (topdirs) are indexed; files ' \
|
|
367
|
+
"outside them are not found — search those files' text directly instead."
|
|
368
|
+
end
|
|
369
|
+
private_class_method :generic_limitation
|
|
370
|
+
end
|
|
371
|
+
end
|
|
372
|
+
end
|