pikuri-fileindex 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +114 -0
- data/lib/pikuri/fileindex/extension.rb +36 -0
- data/lib/pikuri/fileindex/local_search.rb +402 -0
- data/lib/pikuri/fileindex/query.rb +50 -0
- data/lib/pikuri/fileindex/read.rb +231 -0
- data/lib/pikuri/fileindex/recoll.rb +372 -0
- data/lib/pikuri/fileindex/roots.rb +109 -0
- data/lib/pikuri/fileindex/search.rb +288 -0
- data/lib/pikuri-fileindex.rb +70 -0
- metadata +75 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
module Fileindex
|
|
5
|
+
# The folders a host lets the file-index tools reach, and what the owner
|
|
6
|
+
# declares about them. The model's +folder+ can only narrow inside them:
|
|
7
|
+
#
|
|
8
|
+
# roots = Roots.new(dirs: %w[~/Desktop ~/Documents], private: true)
|
|
9
|
+
# roots.dirs # => ["/home/u/Desktop", "/home/u/Documents"]
|
|
10
|
+
# roots.resolve('~/Documents/tax/../2024.pdf') # => "/home/u/Documents/2024.pdf"
|
|
11
|
+
# roots.resolve('/home/u/work/plan.md') # => nil: outside every root
|
|
12
|
+
# roots.resolve('~/Documents/.secret/x.txt') # => nil: hidden below a root
|
|
13
|
+
# roots.resolve('~/Documents/link-to-work/a') # => nil: the symlink leads outside
|
|
14
|
+
# roots.legs.untrusted # => :hard, until the owner vouches
|
|
15
|
+
#
|
|
16
|
+
# A real boundary, not a hint: the stock localsearch index covers all of
|
|
17
|
+
# +$HOME+, so these roots are the only thing keeping +~/work+ or a
|
|
18
|
+
# Downloads file out of an agent that was given +~/notes+.
|
|
19
|
+
#
|
|
20
|
+
# Roots are real paths, because the index stores a file under its real
|
|
21
|
+
# path: each root is resolved through its symlinks at construction, and
|
|
22
|
+
# {#resolve} judges a path by where it leads.
|
|
23
|
+
#
|
|
24
|
+
# One declaration covers every root, not one per folder: each tool
|
|
25
|
+
# declares its legs once, so per-root grades would only union back to the
|
|
26
|
+
# least-trusted root's. Mixed folders take the declaration that fits all.
|
|
27
|
+
#
|
|
28
|
+
# Immutable.
|
|
29
|
+
Roots = Data.define(:dirs, :private, :trusted) do
|
|
30
|
+
# @param dirs [Array<String>] folders, absolute or +~+-relative; each
|
|
31
|
+
# must exist
|
|
32
|
+
# @param private [Boolean] the folders may hold data worth
|
|
33
|
+
# exfiltrating; the owner declares it, nothing inspects the files
|
|
34
|
+
# @param trusted [Boolean] every file under the folders is the user's
|
|
35
|
+
# own: nothing a stranger wrote lands there (so not Downloads, and
|
|
36
|
+
# not a folder an agent that reads mail writes into)
|
|
37
|
+
# @raise [ArgumentError] when +dirs+ is empty, or an entry is relative,
|
|
38
|
+
# missing or not a folder
|
|
39
|
+
def initialize(dirs:, private: false, trusted: false)
|
|
40
|
+
raise ArgumentError, 'roots: give at least one folder' if dirs.empty?
|
|
41
|
+
|
|
42
|
+
super(dirs: dirs.map { |dir| real_dir(dir.to_s) }.uniq.freeze, private: private, trusted: trusted)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# The real path of +path+ when it lies inside a root, or is one, with
|
|
46
|
+
# no hidden (dot-named) folder or file below that root; else +nil+.
|
|
47
|
+
# Symlinks resolve through the longest part of +path+ that exists, so a
|
|
48
|
+
# file not on disk (deleted since indexing, or never there) is judged
|
|
49
|
+
# by its folder. Hidden is judged only below the root, so a root under
|
|
50
|
+
# +~/.cache+ still covers its files.
|
|
51
|
+
#
|
|
52
|
+
# @param path [String] absolute or +~+-relative
|
|
53
|
+
# @return [String, nil]
|
|
54
|
+
def resolve(path)
|
|
55
|
+
real = real_path(File.expand_path(path))
|
|
56
|
+
real if dirs.any? do |dir|
|
|
57
|
+
rest = below(real, dir)
|
|
58
|
+
rest && rest.split('/').none? { |part| part.start_with?('.') }
|
|
59
|
+
end
|
|
60
|
+
rescue ArgumentError # "~nosuchuser"
|
|
61
|
+
nil
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @return [Pikuri::Tool::TrifectaLegs] the legs every tool returning
|
|
65
|
+
# these folders' contents declares; never egress, retrieval stays on
|
|
66
|
+
# the machine.
|
|
67
|
+
def legs
|
|
68
|
+
Pikuri::Tool::TrifectaLegs.new(private: private, untrusted: trusted ? :none : :hard,
|
|
69
|
+
egress_payload_review: :no_egress)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
# @param dir [String]
|
|
75
|
+
# @return [String] the folder's real path
|
|
76
|
+
# @raise [ArgumentError]
|
|
77
|
+
def real_dir(dir)
|
|
78
|
+
raise ArgumentError, "root must be an absolute path, got '#{dir}'" unless dir.start_with?('/', '~')
|
|
79
|
+
|
|
80
|
+
real = File.realpath(File.expand_path(dir))
|
|
81
|
+
raise ArgumentError, "root #{dir} is not a folder" unless File.directory?(real)
|
|
82
|
+
|
|
83
|
+
real
|
|
84
|
+
rescue Errno::ENOENT
|
|
85
|
+
raise ArgumentError, "root #{dir} does not exist"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# @param path [String] absolute, normalized
|
|
89
|
+
# @return [String] +path+ with every symlink in its existing part resolved
|
|
90
|
+
def real_path(path)
|
|
91
|
+
return File.realpath(path) if File.exist?(path)
|
|
92
|
+
|
|
93
|
+
parent = File.dirname(path)
|
|
94
|
+
parent == path ? path : File.join(real_path(parent), File.basename(path))
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @param path [String] absolute, normalized
|
|
98
|
+
# @param dir [String] a root
|
|
99
|
+
# @return [String, nil] +path+ relative to +dir+ (+""+ for +dir+
|
|
100
|
+
# itself), or +nil+ when +path+ is outside it
|
|
101
|
+
def below(path, dir)
|
|
102
|
+
return '' if path == dir
|
|
103
|
+
|
|
104
|
+
prefix = dir.end_with?('/') ? dir : "#{dir}/"
|
|
105
|
+
path.delete_prefix(prefix) if path.start_with?(prefix)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
module Fileindex
|
|
5
|
+
# The +fileindex_search+ tool — a fast, cheap keyword lookup over a
|
|
6
|
+
# desktop file index, returning matching paths each with the *single*
|
|
7
|
+
# snippet the index provides (or a short file's whole text, when the
|
|
8
|
+
# backend has it) and the size of its indexed text, so the
|
|
9
|
+
# model can tell a file it may read whole from one it should search
|
|
10
|
+
# with +fileindex_read+'s +pattern+. A {Pikuri::Tool} in the
|
|
11
|
+
# shape of pikuri-workspace's +grep+, shelling out through the backend.
|
|
12
|
+
#
|
|
13
|
+
# The +backend:+ is the stateless module {Fileindex.pick} returns —
|
|
14
|
+
# {LocalSearch} or {Recoll}. Backend-specific text (the blind-spot note,
|
|
15
|
+
# the error label) is read from the backend, so a new index arm ships
|
|
16
|
+
# its own without touching the tool.
|
|
17
|
+
#
|
|
18
|
+
# A file matches when it holds any of the query's words, or all the
|
|
19
|
+
# words of any quoted group (see {Query} for why); files holding every
|
|
20
|
+
# word come first, then those holding more of them.
|
|
21
|
+
#
|
|
22
|
+
# The +roots:+ bound it: the model's +folder+ must lie inside them, the
|
|
23
|
+
# backend searches only under them, and a hit outside them (or hidden
|
|
24
|
+
# below one) is dropped before the model sees it. That last filter is
|
|
25
|
+
# the guarantee; recoll's query language is spliced, so its own folder
|
|
26
|
+
# clause narrows but can't be trusted to bound.
|
|
27
|
+
#
|
|
28
|
+
# It's a locator, not a window: the index reports at most one match per
|
|
29
|
+
# file, so the tool previews one match each and tells the LLM to read
|
|
30
|
+
# the file for full content / other matches. It sees inside PDFs and
|
|
31
|
+
# office docs (indexed at extract time), and being an index lookup is
|
|
32
|
+
# cheap to run repeatedly.
|
|
33
|
+
#
|
|
34
|
+
# It says whether the hits are every match ("Found 8 files, all shown"
|
|
35
|
+
# or "Showing 10 of 37 matching files"), and for several words how many
|
|
36
|
+
# files hold all of them, counted after the roots filter, so the model
|
|
37
|
+
# can report that nothing else mentions a thing. No paging past
|
|
38
|
+
# {MAX_LIMIT}: the footer sends the model to narrow the query or +folder+
|
|
39
|
+
# instead.
|
|
40
|
+
#
|
|
41
|
+
# Sharing: +P_stateless+ — the backend is a stateless module and each call
|
|
42
|
+
# shells out to its own query, so one instance serves any number of agents.
|
|
43
|
+
class Search < Pikuri::Tool
|
|
44
|
+
# @return [Integer] default / max matching files returned.
|
|
45
|
+
DEFAULT_LIMIT = 10
|
|
46
|
+
MAX_LIMIT = 25
|
|
47
|
+
|
|
48
|
+
# @return [Integer] hard byte cap on combined output (the cap
|
|
49
|
+
# pikuri-workspace's +grep+ uses).
|
|
50
|
+
MAX_BYTES = 50 * 1024
|
|
51
|
+
|
|
52
|
+
# @return [String] human-readable form of {MAX_BYTES}.
|
|
53
|
+
MAX_BYTES_LABEL = "#{MAX_BYTES / 1024} KB"
|
|
54
|
+
|
|
55
|
+
# @return [String] how much text one +fileindex_read+ call returns,
|
|
56
|
+
# the yardstick a hit's text size is read against.
|
|
57
|
+
READ_PAGE_LABEL = Pikuri::Tool.format_bytes(Pikuri::Extractor::PAGE_MAX_BYTES)
|
|
58
|
+
|
|
59
|
+
# @return [String] opencode-shape description (summary + Usage),
|
|
60
|
+
# backend-neutral. The *runtime* tool description is this constant
|
|
61
|
+
# plus the roots and the backend's +#limitations+, appended at
|
|
62
|
+
# construction (see {#initialize}) — and it is that appended text, not
|
|
63
|
+
# this constant, that states *which folders are searched* and what the
|
|
64
|
+
# index misses there (recoll lists its +topdirs+; localsearch names
|
|
65
|
+
# source code and git checkouts). So keep coverage claims out of here.
|
|
66
|
+
DESCRIPTION = <<~DESC
|
|
67
|
+
Find files whose contents contain your words, using the desktop search index.
|
|
68
|
+
|
|
69
|
+
Usage:
|
|
70
|
+
- Use it when the user asks about something their own files may hold (a bill, a contract, a note, a saved web page): search before answering from general knowledge.
|
|
71
|
+
- Searches inside PDFs, Word/ODT/spreadsheet documents and plain text.
|
|
72
|
+
- A file matches when it contains any of your words (or their stems). Files with all of them come first, then those with more of them. So list the words the file may use, synonyms and other languages included: `rent lease vuokra`.
|
|
73
|
+
- Put words in double quotes to require all of them in the same file, in any order: `"notice period" tenant` finds files with both notice and period, or with tenant.
|
|
74
|
+
- Pass `folder` to search only under one folder (subfolders included) — the way to find a common word inside one project instead of across every folder you can search.
|
|
75
|
+
- The index returns at most ONE match per file and may be slightly stale — this is a locator, not the full story. A short file may come whole instead ("shown whole"). To read a file's full, current text, read it (see fileindex_read).
|
|
76
|
+
- Says how many files match in all: "Found 8 files, all shown" means no other file matches. When more match than are shown, narrow the query or `folder`.
|
|
77
|
+
- Each file shows the size of its indexed text. fileindex_read returns up to #{READ_PAGE_LABEL} per call, so a smaller file reads whole in one call; in a bigger one, find the lines you need with fileindex_read's `pattern` first.
|
|
78
|
+
- Cheap: it queries a prebuilt index rather than walking the filesystem, so it barely touches the disk of the machine the user is on — fine to run many times to narrow down.
|
|
79
|
+
- Output is truncated to #{MAX_BYTES_LABEL}; narrow the query if it ends in a truncation marker.
|
|
80
|
+
DESC
|
|
81
|
+
|
|
82
|
+
# @param backend [Module] a ready file-index backend, from
|
|
83
|
+
# {Fileindex.pick}; this tool uses its +search+, +paths+, +limitations+,
|
|
84
|
+
# +label+ and +CommandError+.
|
|
85
|
+
# @param roots [Roots] the folders searched; their declaration grades
|
|
86
|
+
# the tool's legs
|
|
87
|
+
# @return [Search]
|
|
88
|
+
def initialize(backend:, roots:)
|
|
89
|
+
@backend = backend
|
|
90
|
+
super(
|
|
91
|
+
name: 'fileindex_search',
|
|
92
|
+
description: "#{DESCRIPTION}\n#{Search.roots_note(roots)}\n#{backend.limitations}",
|
|
93
|
+
parameters: Parameters.build { |p|
|
|
94
|
+
p.required_string :query,
|
|
95
|
+
'Words to find in file contents; a file with any of them matches, ' \
|
|
96
|
+
'e.g. "tax return taxes 2024".'
|
|
97
|
+
p.optional_integer :limit,
|
|
98
|
+
"Max number of matching files to return " \
|
|
99
|
+
"(default #{DEFAULT_LIMIT}, max #{MAX_LIMIT}), e.g. 10."
|
|
100
|
+
p.optional_string :folder,
|
|
101
|
+
'Absolute path of a folder to search under, subfolders ' \
|
|
102
|
+
'included, inside the folders you can search; omit to search all of them, ' \
|
|
103
|
+
'e.g. "~/Documents/taxes".'
|
|
104
|
+
},
|
|
105
|
+
execute: lambda { |query:, limit: DEFAULT_LIMIT, folder: nil|
|
|
106
|
+
Search.search(query: query, limit: limit, folder: folder, backend: backend, roots: roots)
|
|
107
|
+
},
|
|
108
|
+
trifecta_legs: roots.legs
|
|
109
|
+
)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# @param roots [Roots]
|
|
113
|
+
# @return [String] the description lines naming the searched folders
|
|
114
|
+
def self.roots_note(roots)
|
|
115
|
+
"Searches only under these folders (subfolders included):\n#{roots.dirs.map { |dir| "- #{dir}" }.join("\n")}"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# @param query [String]
|
|
119
|
+
# @param limit [Integer]
|
|
120
|
+
# @param folder [String, nil] folder to search under, absolute or
|
|
121
|
+
# +~+-relative, inside +roots+; +nil+ or blank searches all of them
|
|
122
|
+
# @param backend [Module] the file-index backend to query
|
|
123
|
+
# @param roots [Roots] the folders this search may reach
|
|
124
|
+
# @return [Pikuri::Tool::Result] the hits and how many matched in all
|
|
125
|
+
# (summary +"Found 8 files"+, or +"Found 10 of 37 files"+), or
|
|
126
|
+
# {Pikuri::Tool::Result.error} for a bad +folder+ or a backend failure
|
|
127
|
+
def self.search(query:, backend:, roots:, limit: DEFAULT_LIMIT, folder: nil)
|
|
128
|
+
q = query.to_s.strip
|
|
129
|
+
parsed = Query.parse(q)
|
|
130
|
+
return Pikuri::Tool::Result.error('empty query.') if parsed.empty?
|
|
131
|
+
|
|
132
|
+
folder = folder.to_s.strip
|
|
133
|
+
folder = nil if folder.empty?
|
|
134
|
+
if folder
|
|
135
|
+
unless folder.start_with?('/', '~')
|
|
136
|
+
return Pikuri::Tool::Result.error("folder must be an absolute path, got '#{folder}'.")
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
begin
|
|
140
|
+
folder = File.expand_path(folder)
|
|
141
|
+
rescue ArgumentError => e # "~nosuchuser"
|
|
142
|
+
return Pikuri::Tool::Result.error("bad folder '#{folder}': #{e.message}")
|
|
143
|
+
end
|
|
144
|
+
# Resolved before any existence check, so nothing outside the roots
|
|
145
|
+
# can be probed for whether it exists.
|
|
146
|
+
real = roots.resolve(folder) or return outside_error(folder, roots)
|
|
147
|
+
folder = real
|
|
148
|
+
# A file as folder reads as "every match in this file", which the
|
|
149
|
+
# index can't answer: it keeps one snippet per file.
|
|
150
|
+
if File.file?(folder)
|
|
151
|
+
return Pikuri::Tool::Result.error("#{folder} is a file, not a folder. The index shows only one " \
|
|
152
|
+
'match per file; to see every match in this file, read its ' \
|
|
153
|
+
'full text with fileindex_read.')
|
|
154
|
+
end
|
|
155
|
+
return Pikuri::Tool::Result.error("no such folder: #{folder}") unless File.directory?(folder)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
limit = clamp(limit, 1, MAX_LIMIT)
|
|
159
|
+
folders = folder ? [folder] : roots.dirs
|
|
160
|
+
found = backend.search(query: parsed, limit: limit, folders: folders)
|
|
161
|
+
hits = found.select { |hit| roots.resolve(hit[:path]) }
|
|
162
|
+
total = count(backend, roots, query: parsed, folders: folders, hits: hits, full: found.size >= limit)
|
|
163
|
+
if total.zero?
|
|
164
|
+
return Pikuri::Tool::Result.new(content: no_match_message(q, folder, any: parsed.groups.size > 1),
|
|
165
|
+
summary: 'Found 0 files')
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
all = nil
|
|
169
|
+
if parsed.groups.size > 1
|
|
170
|
+
# Asked apart, so "all of your words, shown first" holds whatever
|
|
171
|
+
# order the backend ranks the any-word hits in.
|
|
172
|
+
all_found = backend.search(query: parsed.all_of, limit: limit, folders: folders)
|
|
173
|
+
all_hits = all_found.select { |hit| roots.resolve(hit[:path]) }
|
|
174
|
+
all = count(backend, roots, query: parsed.all_of, folders: folders, hits: all_hits,
|
|
175
|
+
full: all_found.size >= limit)
|
|
176
|
+
hits = (all_hits + hits).uniq { |hit| hit[:path] }.first(limit)
|
|
177
|
+
total = [total, hits.size].max
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
content, marker = head_truncate(hits.map { |hit| render_hit(hit) }.join("\n"))
|
|
181
|
+
Pikuri::Tool::Result.new(
|
|
182
|
+
content: [content.chomp, footer(hits.size, total, limit, all) + marker].reject(&:empty?).join("\n\n"),
|
|
183
|
+
summary: "Found #{hits.size == total ? '' : "#{hits.size} of "}#{Pikuri::Tool.pluralize(total, 'file')}"
|
|
184
|
+
)
|
|
185
|
+
rescue backend::CommandError => e
|
|
186
|
+
# Cap the error too — a raw subprocess error returned verbatim is
|
|
187
|
+
# the shape that overflows the model's context; share the success
|
|
188
|
+
# path's ceiling.
|
|
189
|
+
content, marker = head_truncate(e.message)
|
|
190
|
+
Pikuri::Tool::Result.error("#{backend.label}: #{content}#{marker}")
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# @param hit [Hash] +{ path:, snippet:, text_size:, text: }+ from the
|
|
194
|
+
# backend; +text+, a short file's whole text, is optional
|
|
195
|
+
# @return [String] +"<path> (<size> of text)"+, then +"\n <snippet>"+
|
|
196
|
+
# when there is one; for a hit with +text+, +"<path> (<size> of
|
|
197
|
+
# text, shown whole)"+ and the text on one line
|
|
198
|
+
def self.render_hit(hit)
|
|
199
|
+
size = hit[:text_size] ? "#{Pikuri::Tool.format_bytes(hit[:text_size])} of text" : 'no indexed text'
|
|
200
|
+
return "#{hit[:path]} (#{size}, shown whole)\n #{hit[:text].gsub(/\s+/, ' ').strip}" if hit[:text]
|
|
201
|
+
|
|
202
|
+
head = "#{hit[:path]} (#{size})"
|
|
203
|
+
hit[:snippet] && !hit[:snippet].empty? ? "#{head}\n #{hit[:snippet]}" : head
|
|
204
|
+
end
|
|
205
|
+
private_class_method :render_hit
|
|
206
|
+
|
|
207
|
+
# How many files match in all, counted as +hits+ were filtered, so the
|
|
208
|
+
# model can trust "all shown". A search that came back short of its
|
|
209
|
+
# limit already holds every match; only a full one costs a listing.
|
|
210
|
+
#
|
|
211
|
+
# @param full [Boolean] the backend answered as many hits as asked for
|
|
212
|
+
# @return [Integer]
|
|
213
|
+
def self.count(backend, roots, query:, folders:, hits:, full:)
|
|
214
|
+
return hits.size unless full
|
|
215
|
+
|
|
216
|
+
# max: a file indexed between the two queries can't shrink the count
|
|
217
|
+
# below what is on screen.
|
|
218
|
+
[backend.paths(query: query, folders: folders).count { |path| roots.resolve(path) }, hits.size].max
|
|
219
|
+
end
|
|
220
|
+
private_class_method :count
|
|
221
|
+
|
|
222
|
+
# @param shown [Integer]
|
|
223
|
+
# @param total [Integer]
|
|
224
|
+
# @param limit [Integer]
|
|
225
|
+
# @param all [Integer, nil] how many files hold every word; +nil+ for a
|
|
226
|
+
# one-group query, where that is every match
|
|
227
|
+
# @return [String] +"Found 8 files, all shown. 3 contain all of your
|
|
228
|
+
# words, shown first. …"+ or +"Showing 10 of 37 matching files, best
|
|
229
|
+
# first. To see the others, …"+
|
|
230
|
+
def self.footer(shown, total, limit, all)
|
|
231
|
+
per_file = 'One match shown per file, or the whole text of a short one; the index may be ' \
|
|
232
|
+
'slightly stale. Read a file to see its full text and any other matches.'
|
|
233
|
+
every = case all
|
|
234
|
+
when nil then ''
|
|
235
|
+
when 0 then ' None contains all of your words; these contain some of them.'
|
|
236
|
+
else " #{all == 1 ? '1 contains' : "#{all} contain"} all of your words, shown first."
|
|
237
|
+
end
|
|
238
|
+
return "Found #{Pikuri::Tool.pluralize(total, 'file')}, all shown.#{every} #{per_file}" if shown == total
|
|
239
|
+
|
|
240
|
+
raise_limit = limit < MAX_LIMIT ? ", or raise `limit` (max #{MAX_LIMIT})" : ''
|
|
241
|
+
"Showing #{shown} of #{total} matching files, best first.#{every} " \
|
|
242
|
+
"To see the others, narrow the query or `folder`#{raise_limit}. #{per_file}"
|
|
243
|
+
end
|
|
244
|
+
private_class_method :footer
|
|
245
|
+
|
|
246
|
+
# @return [Pikuri::Tool::Result]
|
|
247
|
+
def self.outside_error(folder, roots)
|
|
248
|
+
Pikuri::Tool::Result.error("#{folder} is not inside the folders you can search " \
|
|
249
|
+
"(#{roots.dirs.join(', ')}), or is hidden.")
|
|
250
|
+
end
|
|
251
|
+
private_class_method :outside_error
|
|
252
|
+
|
|
253
|
+
# @param any [Boolean] the query has several words, any of which would
|
|
254
|
+
# have matched
|
|
255
|
+
# @return [String]
|
|
256
|
+
def self.no_match_message(query, folder, any:)
|
|
257
|
+
where = folder ? " under #{folder}" : ''
|
|
258
|
+
"No files found matching #{any ? 'any of ' : ''}'#{query}'#{where}. (The index may also be slightly " \
|
|
259
|
+
'stale, or still building.)'
|
|
260
|
+
end
|
|
261
|
+
private_class_method :no_match_message
|
|
262
|
+
|
|
263
|
+
# Head-truncate to {MAX_BYTES} at the last newline boundary.
|
|
264
|
+
#
|
|
265
|
+
# @return [Array(String, String)] +[content, marker]+ (marker empty
|
|
266
|
+
# when no truncation)
|
|
267
|
+
def self.head_truncate(raw)
|
|
268
|
+
total = raw.bytesize
|
|
269
|
+
return [raw, ''] if total <= MAX_BYTES
|
|
270
|
+
|
|
271
|
+
head = raw.byteslice(0, MAX_BYTES)
|
|
272
|
+
last_nl = head.rindex("\n")
|
|
273
|
+
head = head.byteslice(0, last_nl) if last_nl
|
|
274
|
+
omitted = total - head.bytesize
|
|
275
|
+
[head, "\n\n... [#{omitted} bytes omitted; total was #{total} bytes; narrow the query] ..."]
|
|
276
|
+
end
|
|
277
|
+
private_class_method :head_truncate
|
|
278
|
+
|
|
279
|
+
# @return [Integer] +value+ clamped to +[lo, hi]+ (lo on bad input)
|
|
280
|
+
def self.clamp(value, lo, hi)
|
|
281
|
+
Integer(value).clamp(lo, hi)
|
|
282
|
+
rescue ArgumentError, TypeError
|
|
283
|
+
lo
|
|
284
|
+
end
|
|
285
|
+
private_class_method :clamp
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'pikuri-core'
|
|
4
|
+
|
|
5
|
+
# Entry file for the pikuri-fileindex gem: search and read your files through
|
|
6
|
+
# the index the desktop already keeps (GNOME's localsearch, or recoll), never
|
|
7
|
+
# an index of our own. Two tools, +fileindex_search+ ({Pikuri::Fileindex::Search})
|
|
8
|
+
# and +fileindex_read+ ({Pikuri::Fileindex::Read}), bounded by the folders the
|
|
9
|
+
# host names ({Pikuri::Fileindex::Roots}), over whichever backend
|
|
10
|
+
# {Pikuri::Fileindex.pick} finds ready; {Pikuri::Fileindex::Extension} wires
|
|
11
|
+
# both.
|
|
12
|
+
#
|
|
13
|
+
# The load-bearing posture: the desktop's extractor, not pikuri, parsed every
|
|
14
|
+
# office file and PDF the tools return, and the model's words reach the index
|
|
15
|
+
# as bound query parameters, never spliced into the query text. Threat model
|
|
16
|
+
# and the backend survey: +pikuri-fileindex/design/architecture.md+.
|
|
17
|
+
module Pikuri
|
|
18
|
+
module Fileindex
|
|
19
|
+
LOADER = Zeitwerk::Loader.new
|
|
20
|
+
LOADER.tag = 'pikuri-fileindex'
|
|
21
|
+
LOADER.push_dir(__dir__)
|
|
22
|
+
LOADER.ignore(File.expand_path('pikuri-fileindex.rb', __dir__))
|
|
23
|
+
LOADER.setup
|
|
24
|
+
LOADER.eager_load
|
|
25
|
+
|
|
26
|
+
# @return [Array<Module>] backends in preference order: localsearch needs
|
|
27
|
+
# no setup on GNOME; recoll only works once the user built an index.
|
|
28
|
+
BACKENDS = [LocalSearch, Recoll].freeze
|
|
29
|
+
|
|
30
|
+
# Raised by {.pick} when no backend is ready; the message names each
|
|
31
|
+
# one's reason and is meant for the user.
|
|
32
|
+
class Unavailable < StandardError; end
|
|
33
|
+
|
|
34
|
+
# The first backend ready to answer over +roots+, probing each once, in
|
|
35
|
+
# order, so a host never probes one itself. Probing spawns a subprocess
|
|
36
|
+
# per root, so a host picks once and hands the result to every agent:
|
|
37
|
+
#
|
|
38
|
+
# roots = Pikuri::Fileindex::Roots.new(dirs: %w[~/notes], private: true, trusted: true)
|
|
39
|
+
# backend = Pikuri::Fileindex.pick(roots) # => Pikuri::Fileindex::LocalSearch
|
|
40
|
+
# Pikuri::Agent.new(transport:, system_prompt:) do |c|
|
|
41
|
+
# c.add_extension Pikuri::Fileindex::Extension.new(roots:, backend:)
|
|
42
|
+
# end
|
|
43
|
+
#
|
|
44
|
+
# On a host with neither index ready, the {Unavailable} it raises reads:
|
|
45
|
+
#
|
|
46
|
+
# no file index is ready:
|
|
47
|
+
# - localsearch: the 'tinysparql' command is missing. Install …
|
|
48
|
+
# - recoll: no index in /home/u/.recoll yet. Build one with `recollindex`.
|
|
49
|
+
#
|
|
50
|
+
# A backend is a module answering +unavailable_reason(folders:)+,
|
|
51
|
+
# +search(query:, limit:, folders:)+ and +paths(query:, folders:)+ (a file
|
|
52
|
+
# matching any group of the {Query}), +read_text+, +limitations+ and
|
|
53
|
+
# +label+, with a +CommandError+: {LocalSearch} and {Recoll}.
|
|
54
|
+
#
|
|
55
|
+
# @param roots [Roots] the folders the agent will search
|
|
56
|
+
# @param backends [Array<Module>] candidates in preference order
|
|
57
|
+
# @return [Module] the first one whose +unavailable_reason+ is +nil+
|
|
58
|
+
# @raise [Unavailable] when none is ready
|
|
59
|
+
def self.pick(roots, backends: BACKENDS)
|
|
60
|
+
reasons = []
|
|
61
|
+
backends.each do |backend|
|
|
62
|
+
reason = backend.unavailable_reason(folders: roots.dirs)
|
|
63
|
+
return backend unless reason
|
|
64
|
+
|
|
65
|
+
reasons << "- #{backend.label}: #{reason}"
|
|
66
|
+
end
|
|
67
|
+
raise Unavailable, "no file index is ready:\n#{reasons.join("\n")}"
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: pikuri-fileindex
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.2.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Martin Vysny
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: pikuri-core
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - '='
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: 0.2.0
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - '='
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: 0.2.0
|
|
26
|
+
description: |
|
|
27
|
+
pikuri-fileindex gives a pikuri agent two tools, fileindex_search
|
|
28
|
+
and fileindex_read, over the file index the desktop already keeps:
|
|
29
|
+
GNOME's localsearch, queried in SPARQL through tinysparql, or
|
|
30
|
+
recoll. The index has already extracted the text of your PDFs and
|
|
31
|
+
office documents, so the agent reads them with no parser of its own.
|
|
32
|
+
|
|
33
|
+
Named for what it refuses to become: the owner of an index. It never
|
|
34
|
+
builds, configures or refreshes one; the desktop keeps it fresh.
|
|
35
|
+
email:
|
|
36
|
+
- martin@vysny.me
|
|
37
|
+
executables: []
|
|
38
|
+
extensions: []
|
|
39
|
+
extra_rdoc_files: []
|
|
40
|
+
files:
|
|
41
|
+
- README.md
|
|
42
|
+
- lib/pikuri-fileindex.rb
|
|
43
|
+
- lib/pikuri/fileindex/extension.rb
|
|
44
|
+
- lib/pikuri/fileindex/local_search.rb
|
|
45
|
+
- lib/pikuri/fileindex/query.rb
|
|
46
|
+
- lib/pikuri/fileindex/read.rb
|
|
47
|
+
- lib/pikuri/fileindex/recoll.rb
|
|
48
|
+
- lib/pikuri/fileindex/roots.rb
|
|
49
|
+
- lib/pikuri/fileindex/search.rb
|
|
50
|
+
homepage: https://github.com/mvysny/pikuri
|
|
51
|
+
licenses:
|
|
52
|
+
- MIT
|
|
53
|
+
metadata:
|
|
54
|
+
source_code_uri: https://github.com/mvysny/pikuri/tree/master
|
|
55
|
+
changelog_uri: https://github.com/mvysny/pikuri/blob/master/CHANGELOG.md
|
|
56
|
+
bug_tracker_uri: https://github.com/mvysny/pikuri/issues
|
|
57
|
+
rubygems_mfa_required: 'true'
|
|
58
|
+
rdoc_options: []
|
|
59
|
+
require_paths:
|
|
60
|
+
- lib
|
|
61
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
62
|
+
requirements:
|
|
63
|
+
- - ">="
|
|
64
|
+
- !ruby/object:Gem::Version
|
|
65
|
+
version: '3.3'
|
|
66
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
67
|
+
requirements:
|
|
68
|
+
- - ">="
|
|
69
|
+
- !ruby/object:Gem::Version
|
|
70
|
+
version: '0'
|
|
71
|
+
requirements: []
|
|
72
|
+
rubygems_version: 3.6.7
|
|
73
|
+
specification_version: 4
|
|
74
|
+
summary: Search your files through the desktop's own file index, for pikuri.
|
|
75
|
+
test_files: []
|