pikuri-fileindex 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,231 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'stringio'
4
+
5
+ module Pikuri
6
+ module Fileindex
7
+ # The +fileindex_read+ tool — read a file's full text *as stored in
8
+ # the desktop index*, line-paged like pikuri-workspace's +read+. The
9
+ # companion to {Search} (which locates + previews one match),
10
+ # sharing its swappable +backend:+ ({LocalSearch} / {Recoll}).
11
+ #
12
+ # It reads the index, not the file: the index already holds every
13
+ # indexed file's extracted plain text — PDFs/office docs included — so
14
+ # this returns them in full with no extractor dependency. The cost is
15
+ # that it's the *indexed snapshot* (can lag very recent edits, covers
16
+ # only files the index knows); a file just written is read from disk,
17
+ # with pikuri-workspace's +read+.
18
+ #
19
+ # +offset+ / +limit+ page exactly as in pikuri-workspace's +read+,
20
+ # windowed through {Pikuri::Extractor.extract_paged} (a +StringIO+ over
21
+ # the stored text) and rendered by {Pikuri::Tool::Result.page}, so the
22
+ # +cat -n+ prefix, trailer and caps are +read+'s own.
23
+ #
24
+ # +pattern+ turns the read into a +grep -n+ over the same text: only the
25
+ # matching lines, numbered as a read numbers them. The search hit names
26
+ # no line, so without it the model's only way to an answer deep in a
27
+ # long PDF is to page through all of it; with it, the model reads just
28
+ # around the lines that match. A regex, not the index's stemmed words:
29
+ # inside one file an exact match is what the model wants, and the
30
+ # index's own match positions (+fts:offsets+) came back as +(null):17+
31
+ # on LocalSearch 3.11, naming no property.
32
+ #
33
+ # A line longer than the page's per-line cap is broken at a space first
34
+ # ({.wrap}): localsearch keeps a PDF's line breaks and gives HTML a line
35
+ # per paragraph, but hands an office document over as one line, which
36
+ # the cap would cut after 2,000 characters, with no offset reaching the
37
+ # rest.
38
+ #
39
+ # Only a path inside the +roots:+, with nothing hidden below a root, is
40
+ # read; the path is the model's, so it is checked here even though the
41
+ # localsearch miner never indexes a hidden file (recoll does).
42
+ #
43
+ # Sharing: +P_stateless+ — the backend is a stateless module over a
44
+ # read-only index, so one instance serves any number of agents.
45
+ class Read < Pikuri::Tool
46
+ # @return [Integer] default +limit+ (lines per call); the shared
47
+ # page default, so it matches +read+.
48
+ DEFAULT_LIMIT = Pikuri::Extractor::PAGE_DEFAULT_LIMIT
49
+
50
+ # @return [Numeric] seconds one +pattern+ may spend matching, so a
51
+ # catastrophically backtracking regex fails instead of hanging the
52
+ # turn.
53
+ PATTERN_TIMEOUT = 2
54
+
55
+ # @return [String] opencode-shape description (summary + Usage).
56
+ DESCRIPTION = <<~DESC
57
+ Read a file's full text from the desktop search index, with line numbers.
58
+
59
+ Usage:
60
+ - The companion to fileindex_search: after it locates a file, read the file in full here.
61
+ - Reads the indexed text, so PDFs and office documents come back as their extracted text — no special handling needed.
62
+ - This is the indexed snapshot: it can lag very recent edits, and only the files fileindex_search can find are available. For a file just written, use the read tool instead.
63
+ - Output is line-numbered `cat -n` style; use `offset` and `limit` to page. When the response ends in `Use offset=N to continue`, call again with that offset.
64
+ - To find something in a long file without reading all of it, pass `pattern`: you get only the matching lines with their line numbers. Then read around a match with `offset` a few lines before it and a small `limit`.
65
+ DESC
66
+
67
+ # @param backend [Module] a ready file-index backend, from
68
+ # {Fileindex.pick}; this tool uses only its +read_text+.
69
+ # @param roots [Roots] the folders whose files may be read; their
70
+ # declaration grades the tool's legs
71
+ # @return [Read]
72
+ def initialize(backend:, roots:)
73
+ @backend = backend
74
+ super(
75
+ name: 'fileindex_read',
76
+ description: DESCRIPTION,
77
+ parameters: Parameters.build { |p|
78
+ p.required_string :path,
79
+ 'Absolute path to the file (as returned by ' \
80
+ 'fileindex_search), e.g. "/home/u/Documents/report.pdf".'
81
+ p.optional_integer :offset,
82
+ 'Line number to start reading from (1-indexed). ' \
83
+ 'Defaults to 1, e.g. 200.'
84
+ p.optional_integer :limit,
85
+ 'Maximum number of lines to read. Defaults to ' \
86
+ "#{DEFAULT_LIMIT}, e.g. 500. With `pattern`, the most matching lines to return."
87
+ p.optional_string :pattern,
88
+ 'Case-insensitive regular expression; returns only the lines that match, ' \
89
+ 'with their line numbers, searching from `offset` on, e.g. "total|amount due".'
90
+ },
91
+ execute: lambda { |path:, offset: 1, limit: DEFAULT_LIMIT, pattern: nil|
92
+ Read.read(path: path, offset: offset, limit: limit, pattern: pattern, backend: backend, roots: roots)
93
+ },
94
+ trifecta_legs: roots.legs
95
+ )
96
+ end
97
+
98
+ # @param path [String] absolute or +~+-relative, inside +roots+
99
+ # @param backend [Module] the file-index backend to read from
100
+ # @param roots [Roots] the folders whose files may be read
101
+ # @param offset [Integer] 1-indexed start line
102
+ # @param limit [Integer] max lines; with +pattern+, max matching lines
103
+ # @param pattern [String, nil] case-insensitive regex; when given, only
104
+ # the matching lines from +offset+ on are returned (see {.grep})
105
+ # @return [Pikuri::Tool::Result] the cat-n windowed text (+"Read 214
106
+ # lines"+), the matching lines (+"Found 6 lines"+), or
107
+ # {Pikuri::Tool::Result.error}
108
+ def self.read(path:, backend:, roots:, offset: 1, limit: DEFAULT_LIMIT, pattern: nil)
109
+ return Pikuri::Tool::Result.error("offset must be >= 1, got #{offset}") if offset < 1
110
+ return Pikuri::Tool::Result.error("limit must be >= 1, got #{limit}") if limit < 1
111
+
112
+ pattern = pattern.to_s
113
+ regex = nil
114
+ unless pattern.empty?
115
+ begin
116
+ regex = Regexp.new(pattern, Regexp::IGNORECASE, timeout: PATTERN_TIMEOUT)
117
+ rescue RegexpError => e
118
+ return Pikuri::Tool::Result.error("bad pattern: #{e.message}")
119
+ end
120
+ end
121
+
122
+ return Pikuri::Tool::Result.error("path must be absolute, got '#{path}'.") unless path.start_with?('/', '~')
123
+
124
+ begin
125
+ path = File.expand_path(path)
126
+ rescue ArgumentError => e # "~nosuchuser"
127
+ return Pikuri::Tool::Result.error("bad path '#{path}': #{e.message}")
128
+ end
129
+ real = roots.resolve(path)
130
+ unless real
131
+ return Pikuri::Tool::Result.error("#{path} is not inside the folders you can read " \
132
+ "(#{roots.dirs.join(', ')}), or is hidden.")
133
+ end
134
+ path = real
135
+
136
+ text = backend.read_text(path)
137
+ if text.nil?
138
+ return Pikuri::Tool::Result.error("#{path} is not in the index (PDFs and office docs are included). " \
139
+ 'For a file just written, use the read tool.')
140
+ end
141
+
142
+ text = wrap(text)
143
+ return grep(text, regex, offset: offset, limit: limit) if regex
144
+
145
+ page = Pikuri::Extractor.extract_paged(
146
+ StringIO.new(text), content_type: 'text/plain', offset: offset, limit: limit
147
+ )
148
+ Pikuri::Tool::Result.page(page, empty_note: '(No indexed text for this file.)')
149
+ end
150
+
151
+ # The lines of +text+ from +offset+ on that match +regex+, +cat -n+
152
+ # numbered as a plain read numbers them, stopping at +limit+ matches or
153
+ # {Pikuri::Extractor::PAGE_MAX_BYTES}, whichever comes first; a stop
154
+ # names the offset to continue from.
155
+ #
156
+ # @param text [String] the file's indexed text
157
+ # @param regex [Regexp]
158
+ # @param offset [Integer] 1-indexed first line searched
159
+ # @param limit [Integer] max matching lines
160
+ # @return [Pikuri::Tool::Result] the lines and a footer (+"Found 6
161
+ # lines"+), or {Pikuri::Tool::Result.error} when the pattern times out
162
+ def self.grep(text, regex, offset:, limit:)
163
+ rows = []
164
+ bytes = 0
165
+ stopped_at = nil
166
+ total = 0
167
+ text.each_line(chomp: true).with_index(1) do |line, number|
168
+ total = number
169
+ next if stopped_at || number < offset || !regex.match?(line)
170
+
171
+ row = format("%6d\t%s", number, line)
172
+ if rows.size == limit || bytes + row.bytesize > Pikuri::Extractor::PAGE_MAX_BYTES
173
+ stopped_at = number
174
+ next
175
+ end
176
+ rows << row
177
+ bytes += row.bytesize + 1
178
+ end
179
+ grep_result(rows, regex, total: total, offset: offset, stopped_at: stopped_at)
180
+ rescue Regexp::TimeoutError
181
+ Pikuri::Tool::Result.error("pattern /#{regex.source}/ took over #{PATTERN_TIMEOUT}s on this file; " \
182
+ 'simplify it.')
183
+ end
184
+ private_class_method :grep
185
+
186
+ # @return [Pikuri::Tool::Result]
187
+ def self.grep_result(rows, regex, total:, offset:, stopped_at:)
188
+ where = offset > 1 ? " from line #{offset} on" : ''
189
+ if rows.empty?
190
+ return Pikuri::Tool::Result.new(
191
+ content: "No lines match /#{regex.source}/#{where} (the indexed text has " \
192
+ "#{Pikuri::Tool.pluralize(total, 'line')}).",
193
+ summary: 'Found 0 lines'
194
+ )
195
+ end
196
+
197
+ footer = "Found #{Pikuri::Tool.pluralize(rows.size, 'matching line')}#{where} " \
198
+ "(the indexed text has #{Pikuri::Tool.pluralize(total, 'line')}). " \
199
+ 'To read around one, call again without `pattern`, `offset` a few lines before it.'
200
+ if stopped_at
201
+ footer += " More lines match: call again with the same `pattern` and offset=#{stopped_at} for the rest."
202
+ end
203
+ Pikuri::Tool::Result.new(content: [rows.join("\n"), '', footer].join("\n"),
204
+ summary: "Found #{Pikuri::Tool.pluralize(rows.size, 'line')}")
205
+ end
206
+ private_class_method :grep_result
207
+
208
+ # @param text [String] a file's indexed text
209
+ # @return [String] +text+ with each line longer than
210
+ # {Pikuri::Extractor::PAGE_MAX_LINE_LENGTH} broken at its last
211
+ # whitespace within the cap, or at the cap when it has none; +text+
212
+ # itself when no line is that long
213
+ def self.wrap(text)
214
+ max = Pikuri::Extractor::PAGE_MAX_LINE_LENGTH
215
+ return text if text.each_line.none? { |line| line.chomp.length > max }
216
+
217
+ text.each_line(chomp: true).flat_map do |line|
218
+ pieces = []
219
+ while line.length > max
220
+ cut = line.rindex(/\s/, max)
221
+ cut = max if cut.nil? || cut.zero?
222
+ pieces << line[0, cut]
223
+ line = line[cut..].lstrip
224
+ end
225
+ pieces << line
226
+ end.join("\n")
227
+ end
228
+ private_class_method :wrap
229
+ end
230
+ end
231
+ end
@@ -0,0 +1,372 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pikuri
4
+ module Fileindex
5
+ # Backend wrapper over the +recoll+ full-text index, via its headless
6
+ # +recollq+ CLI (Xapian-backed). The DE-independent sibling of
7
+ # {LocalSearch}: same seam shape ({.search} / {.read_text} /
8
+ # {.unavailable_reason}), interchangeable behind the +fileindex_search+ /
9
+ # +fileindex_read+ tools; {Fileindex.pick} takes it when localsearch
10
+ # isn't ready.
11
+ #
12
+ # Recall on prose ties +localsearch+, so recoll is *not* a better
13
+ # engine — it earns its place where +localsearch+ is structurally blind
14
+ # (source trees, +.git+ checkouts, non-GNOME desktops). Its blind spot
15
+ # is the inverse: recoll indexes exactly the *folders* the user
16
+ # configured (+topdirs+), source code included — everything outside them
17
+ # is invisible. The measured survey behind the pick-one-per-host order
18
+ # is in +pikuri-fileindex/design/architecture.md+.
19
+ #
20
+ # {.search} asks +recollq -F 'url abstract dbytes'+ for three base64-encoded
21
+ # fields per line (recoll's own recommended machine shape) so paths and
22
+ # snippets with spaces/brackets/newlines survive with no escaping
23
+ # guesswork. {.read_text} uses +recollq -d+, which dumps a document's
24
+ # stored/extracted text (PDFs/office docs included) after a header line.
25
+ module Recoll
26
+ # @return [Logger]
27
+ LOGGER = Pikuri.logger_for('Fileindex::Recoll')
28
+
29
+ # @return [String] the headless recoll query CLI.
30
+ BINARY = 'recollq'
31
+
32
+ # @return [String] the recoll indexer/config CLI (used only to locate
33
+ # the config; never invoked to *index* from here).
34
+ INDEXER = 'recollindex'
35
+
36
+ # @return [String] recoll config filename inside the config dir.
37
+ CONFIG_FILE = 'recoll.conf'
38
+
39
+ # recoll's compiled-in default when +topdirs+ is unset: the whole
40
+ # home directory (the sample config ships +topdirs = ~+ too).
41
+ #
42
+ # @return [String]
43
+ DEFAULT_TOPDIRS = '~'
44
+
45
+ # @return [String] the Xapian index directory name inside the config
46
+ # dir; missing until +recollindex+ first runs.
47
+ INDEX_DIR = 'xapiandb'
48
+
49
+ # @return [String] a query matching every document in the index.
50
+ ANY_DOCUMENT = 'mime:*'
51
+
52
+ # Raised when a recollq invocation exits non-zero — a recoverable
53
+ # failure the calling tool renders as +"Error: ..."+.
54
+ class CommandError < StandardError; end
55
+
56
+ # @return [String] short human name for this backend, used by the
57
+ # fileindex tools when prefixing an +"Error: ..."+ observation.
58
+ def self.label = 'recoll'
59
+
60
+ # Plain-English note of what this backend will *not* surface, appended
61
+ # to the fileindex tools' LLM description so the model knows when to
62
+ # fall back to a filesystem text/regex search. A *method*, not a
63
+ # constant like {LocalSearch::LIMITATIONS}, because the blind spot is
64
+ # host-specific: the user's +topdirs+, only readable at runtime
65
+ # ({.indexed_dirs}). Falls back to a generic caveat when the config
66
+ # can't be read.
67
+ #
68
+ # @return [String]
69
+ def self.limitations
70
+ dirs = indexed_dirs
71
+ return generic_limitation if dirs.empty?
72
+
73
+ listed = dirs.map { |dir| "- #{dir}" }.join("\n")
74
+ <<~LIMITS.chomp
75
+ Only files under these folders are indexed (anything elsewhere on disk is not in this index — search those files' text directly instead):
76
+ #{listed}
77
+ LIMITS
78
+ end
79
+
80
+ # Folders recoll is configured to index, read from +topdirs+ in the
81
+ # active config file. Resolution mirrors recoll itself: the config dir
82
+ # is +$RECOLL_CONFDIR+ or +~/.recoll+; +topdirs+ is a space-separated
83
+ # list (quoted entries may contain spaces, a trailing +\+ continues to
84
+ # the next line) living in the global section, i.e. above the first
85
+ # +[section]+ header. Entries are +~+- and +$VAR+-expanded to absolute
86
+ # paths. Defaults to the home dir ({DEFAULT_TOPDIRS}) when the config
87
+ # is absent or names no +topdirs+, matching recoll's own default.
88
+ #
89
+ # @return [Array<String>] absolute folder paths, or +[]+ if the config
90
+ # dir exists but the file is unreadable
91
+ def self.indexed_dirs
92
+ path = File.join(confdir, CONFIG_FILE)
93
+ raw = File.exist?(path) ? File.read(path) : nil
94
+ entries = raw ? parse_topdirs(raw) : [DEFAULT_TOPDIRS]
95
+ entries = [DEFAULT_TOPDIRS] if entries.empty?
96
+ entries.map { |entry| expand_entry(entry) }
97
+ rescue SystemCallError
98
+ []
99
+ end
100
+
101
+ # The recoll configuration directory recoll would use: +$RECOLL_CONFDIR+
102
+ # if set and non-empty, else +~/.recoll+.
103
+ #
104
+ # @return [String]
105
+ def self.confdir
106
+ env = ENV['RECOLL_CONFDIR']
107
+ env && !env.empty? ? File.expand_path(env) : File.expand_path('~/.recoll')
108
+ end
109
+
110
+ # Run +recollq -F 'url abstract dbytes' -n <limit>+ and parse the results.
111
+ # The query becomes recoll's query language ({.language}), with the
112
+ # folders as one +(dir:"<a>" OR dir:"<b>")+ group appended; recoll
113
+ # matches a +dir:+ by whole path components (+/a/sp+ does not match
114
+ # +/a/sp ace+).
115
+ #
116
+ # The model's words are spliced into that language, so the folder group
117
+ # narrows the search but can't be relied on to bound it: the
118
+ # +fileindex_search+ tool drops any hit outside its roots.
119
+ #
120
+ # @param query [Query] a file matches every word of any group
121
+ # @param limit [Integer]
122
+ # @param folders [Array<String>] absolute folders to search under,
123
+ # subfolders included; at least one
124
+ # @return [Array<Hash>] +[{ path: String, snippet: String|nil,
125
+ # text_size: Integer|nil }, …]+, de-duplicated by path; +text_size+
126
+ # is the extracted text's bytes (+dbytes+), +nil+ when there is none
127
+ # @raise [CommandError] if recollq exits non-zero, or a folder contains
128
+ # a +"+ (recoll's query language cannot quote one)
129
+ def self.search(query:, limit:, folders:)
130
+ parse_search_output(query_fields('url abstract dbytes', limit, "#{language(query)} #{within(folders)}"))
131
+ end
132
+
133
+ # @param query [Query]
134
+ # @return [String] each group's words in parentheses, the groups joined
135
+ # by +OR+: +(notice period) OR (tenant)+. recoll's +OR+ binds tighter
136
+ # than the implicit AND, so the folder group appended after it bounds
137
+ # the whole.
138
+ def self.language(query)
139
+ query.groups.map { |words| "(#{words.join(' ')})" }.join(' OR ')
140
+ end
141
+ private_class_method :language
142
+
143
+ # The path of every file {.search} would match, unranked and uncapped,
144
+ # so a caller can tell how many matched beyond its +limit+. recollq's
145
+ # own "N results" line can't stand in: it counts files the caller
146
+ # filters out, hidden ones included.
147
+ #
148
+ # @param query [Query]
149
+ # @param folders [Array<String>] as {.search}'s
150
+ # @return [Array<String>] absolute paths
151
+ # @raise [CommandError] as {.search}
152
+ def self.paths(query:, folders:)
153
+ parse_search_output(query_fields('url', 0, "#{language(query)} #{within(folders)}")).map { |hit| hit[:path] }
154
+ end
155
+
156
+ # @param fields [String] recollq's +-F+ field list
157
+ # @param limit [Integer] max results; 0 for no limit
158
+ # @param query [String]
159
+ # @return [String] recollq's stdout
160
+ # @raise [CommandError] on a non-zero exit
161
+ def self.query_fields(fields, limit, query)
162
+ result = Pikuri::Subprocess.spawn(BINARY, '-F', fields, '-n', limit.to_s, query, chdir: '/').wait
163
+ return result.output if result.status.success?
164
+
165
+ stderr = result.output.strip
166
+ raise CommandError, stderr.empty? ? "exited #{result.status.exitstatus}" : stderr
167
+ end
168
+ private_class_method :query_fields
169
+
170
+ # Parse +recollq -F 'url abstract dbytes'+ output into hits. Each data
171
+ # line is three base64 tokens (url, abstract, text bytes); an empty field
172
+ # still prints a token. recollq's +"Recoll query: …"+ /
173
+ # +"N results"+ banner lines are skipped because their first token
174
+ # isn't valid base64 for a +file://+ URL.
175
+ #
176
+ # @param raw [String]
177
+ # @return [Array<Hash>] +[{ path:, snippet:, text_size: }, …]+ de-duped
178
+ # by path
179
+ def self.parse_search_output(raw)
180
+ hits = []
181
+ raw.each_line do |line|
182
+ url_b64, abstract_b64, dbytes_b64 = line.chomp.split(/ /)
183
+ url = decode_base64(url_b64)
184
+ next unless url&.start_with?('file://')
185
+
186
+ snippet = decode_base64(abstract_b64.to_s)&.strip
187
+ size = decode_base64(dbytes_b64.to_s).to_i
188
+ hits << { path: url.delete_prefix('file://'), snippet: (snippet unless snippet.to_s.empty?),
189
+ text_size: (size if size.positive?) }
190
+ end
191
+ hits.uniq { |hit| hit[:path] }
192
+ end
193
+
194
+ # Full stored/extracted text of an indexed file, via +recollq -d+
195
+ # scoped to the file's own name. Works for PDFs/office docs (extracted
196
+ # at index time). +nil+ when the path isn't in the index or has no
197
+ # stored text — the caller turns that into an LLM-facing message.
198
+ #
199
+ # @param path [String]
200
+ # @return [String, nil]
201
+ def self.read_text(path)
202
+ abs = File.expand_path(path)
203
+ result = Pikuri::Subprocess.spawn(BINARY, '-d', '-n', '50', "filename:#{File.basename(abs)}",
204
+ chdir: '/').wait
205
+ return nil unless result.status.success?
206
+
207
+ text = extract_dumped_text(result.output, abs)
208
+ text && !text.strip.empty? ? text : nil
209
+ end
210
+
211
+ # Pull the dumped body of the result whose header line matches +abs+
212
+ # out of +recollq -d+ output. Each result is a tab-delimited header
213
+ # line (+<mime>\t[<url>]\t[<title>]\t<n>\tbytes+) followed by its text,
214
+ # up to the next header line.
215
+ #
216
+ # @param raw [String]
217
+ # @param abs [String] absolute path to match against the header URL
218
+ # @return [String, nil]
219
+ def self.extract_dumped_text(raw, abs)
220
+ want = "file://#{abs}"
221
+ body = []
222
+ capturing = false
223
+ raw.each_line do |line|
224
+ url = line[%r{\t\[(file://[^\]]*)\]\t}, 1]
225
+ if url
226
+ break if capturing
227
+
228
+ capturing = (url == want)
229
+ elsif capturing
230
+ body << line
231
+ end
232
+ end
233
+ capturing || !body.empty? ? body.join.chomp : nil
234
+ end
235
+
236
+ # Why this index can't answer a search under +folders+ right now, for
237
+ # a human to act on; +nil+ when it can. A built index holding no
238
+ # document under any of them is not ready either: every search would
239
+ # come back empty. Some empty folders are normal (an empty Desktop), so
240
+ # they only log a warning naming each.
241
+ #
242
+ # Recoll.unavailable_reason(folders: ['/home/u/notes'])
243
+ # # => "no index in /home/u/.recoll yet. Build one with `recollindex`."
244
+ #
245
+ # @param folders [Array<String>] absolute real paths
246
+ # @return [String, nil]
247
+ def self.unavailable_reason(folders:)
248
+ quoted = folders.find { |folder| folder.include?('"') }
249
+ return "recoll can't search #{quoted}: its query language can't quote a double quote in a path." if quoted
250
+
251
+ empty = folders.reject { |folder| holds_document?(folder) }
252
+ if empty.size < folders.size
253
+ empty.each { |folder| LOGGER.warn("nothing under #{folder} is indexed; searches there find nothing") }
254
+ return nil
255
+ end
256
+
257
+ "nothing under #{folders.join(', ')} is in the index in #{confdir}. Check `topdirs` in " \
258
+ "#{File.join(confdir, CONFIG_FILE)}, then rerun `recollindex`."
259
+ rescue Errno::ENOENT
260
+ "the '#{BINARY}' command is missing. Install the 'recoll' package, then build an index " \
261
+ 'with `recollindex`.'
262
+ rescue CommandError => e
263
+ return "no index in #{confdir} yet. Build one with `recollindex`." unless File.directory?(File.join(confdir, INDEX_DIR))
264
+
265
+ "the index in #{confdir} doesn't answer: #{e.message}"
266
+ end
267
+
268
+ # @param folder [String]
269
+ # @return [Boolean] whether the index holds any document under +folder+
270
+ # @raise [CommandError] if recollq exits non-zero (message: its last
271
+ # line), or +folder+ contains a +"+
272
+ def self.holds_document?(folder)
273
+ result = Pikuri::Subprocess.spawn(BINARY, '-F', 'url', '-n', '1', "#{ANY_DOCUMENT} #{within([folder])}",
274
+ chdir: '/').wait
275
+ raise CommandError, result.output.strip.lines.last&.strip.to_s unless result.status.success?
276
+
277
+ result.output.each_line.any? { |line| decode_base64(line.strip)&.start_with?('file://') }
278
+ end
279
+ private_class_method :holds_document?
280
+
281
+ # @param folders [Array<String>]
282
+ # @return [String] +(dir:"<a>" OR dir:"<b>")+; positive +dir:+ clauses
283
+ # side by side would be ANDed
284
+ # @raise [CommandError] if a folder contains a +"+
285
+ def self.within(folders)
286
+ quoted = folders.find { |folder| folder.include?('"') }
287
+ raise CommandError, "can't search a folder whose path contains a double quote: #{quoted}" if quoted
288
+
289
+ "(#{folders.map { |folder| "dir:\"#{folder}\"" }.join(' OR ')})"
290
+ end
291
+ private_class_method :within
292
+
293
+ # Join backslash-continued lines, then read the +topdirs+ value from
294
+ # the global section (everything before the first +[section]+ header).
295
+ # The last assignment wins (ConfSimple semantics).
296
+ #
297
+ # @param raw [String]
298
+ # @return [Array<String>] raw (unexpanded) topdirs entries
299
+ def self.parse_topdirs(raw)
300
+ value = nil
301
+ joined_lines(raw).each do |line|
302
+ stripped = line.strip
303
+ break if stripped.start_with?('[') # first [section] ends the global scope
304
+ next if stripped.empty? || stripped.start_with?('#')
305
+
306
+ key, _, rest = stripped.partition('=')
307
+ value = rest if key.strip == 'topdirs'
308
+ end
309
+ value ? tokenize(value) : []
310
+ end
311
+
312
+ # @param raw [String]
313
+ # @return [Array<String>] logical lines with trailing-backslash
314
+ # continuations folded into one
315
+ def self.joined_lines(raw)
316
+ lines = []
317
+ buffer = +''
318
+ raw.each_line(chomp: true) do |line|
319
+ if line.end_with?('\\')
320
+ buffer << line[0..-2] << ' '
321
+ else
322
+ lines << (buffer + line)
323
+ buffer = +''
324
+ end
325
+ end
326
+ lines << buffer unless buffer.empty?
327
+ lines
328
+ end
329
+ private_class_method :joined_lines
330
+
331
+ # Split a topdirs value into entries, honouring +"double quotes"+
332
+ # around paths that contain spaces.
333
+ #
334
+ # @param value [String]
335
+ # @return [Array<String>]
336
+ def self.tokenize(value)
337
+ value.scan(/"([^"]*)"|(\S+)/).map { |quoted, bare| quoted || bare }
338
+ end
339
+ private_class_method :tokenize
340
+
341
+ # Expand +$VAR+ / +${VAR}+ then +~+ to an absolute path; a relative
342
+ # entry resolves against the home dir (recoll's own base).
343
+ #
344
+ # @param entry [String]
345
+ # @return [String]
346
+ def self.expand_entry(entry)
347
+ env_expanded = entry.gsub(/\$\{(\w+)\}|\$(\w+)/) { ENV[::Regexp.last_match(1) || ::Regexp.last_match(2)] || '' }
348
+ File.expand_path(env_expanded, Dir.home)
349
+ end
350
+ private_class_method :expand_entry
351
+
352
+ # @param token [String, nil]
353
+ # @return [String, nil] decoded bytes tagged UTF-8, or +nil+ if +token+
354
+ # is blank or not valid strict base64
355
+ def self.decode_base64(token)
356
+ return nil if token.nil? || token.empty?
357
+
358
+ token.unpack1('m0').force_encoding('UTF-8') # m0: strict base64
359
+ rescue ArgumentError
360
+ nil
361
+ end
362
+ private_class_method :decode_base64
363
+
364
+ # @return [String]
365
+ def self.generic_limitation
366
+ 'Only folders configured in recoll (topdirs) are indexed; files ' \
367
+ "outside them are not found — search those files' text directly instead."
368
+ end
369
+ private_class_method :generic_limitation
370
+ end
371
+ end
372
+ end