gemchat 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/AGENTS.md +354 -0
- data/CHANGELOG.md +5 -0
- data/CODE_OF_CONDUCT.md +10 -0
- data/LICENSE.txt +21 -0
- data/README.md +144 -0
- data/Rakefile +14 -0
- data/docs/plans/gemchat-local-bundle-index.md +1900 -0
- data/exe/gemchat +7 -0
- data/lib/gemchat/chunker.rb +120 -0
- data/lib/gemchat/cli.rb +731 -0
- data/lib/gemchat/embedder.rb +309 -0
- data/lib/gemchat/env.rb +27 -0
- data/lib/gemchat/errors.rb +10 -0
- data/lib/gemchat/hook.rb +113 -0
- data/lib/gemchat/hosted.rb +195 -0
- data/lib/gemchat/indexer.rb +114 -0
- data/lib/gemchat/init.rb +107 -0
- data/lib/gemchat/lockfile.rb +34 -0
- data/lib/gemchat/manifest.rb +100 -0
- data/lib/gemchat/markdown.rb +103 -0
- data/lib/gemchat/models.rb +241 -0
- data/lib/gemchat/paths.rb +33 -0
- data/lib/gemchat/plugin_index.rb +63 -0
- data/lib/gemchat/prose.rb +211 -0
- data/lib/gemchat/ri.rb +130 -0
- data/lib/gemchat/stopwords.rb +65 -0
- data/lib/gemchat/store.rb +555 -0
- data/lib/gemchat/symbols.rb +159 -0
- data/lib/gemchat/trust.rb +79 -0
- data/lib/gemchat/version.rb +5 -0
- data/lib/gemchat.rb +47 -0
- data/plugins.rb +74 -0
- data/sig/gemchat.rbs +4 -0
- data/skills/gemchat/SKILL.md +158 -0
- metadata +160 -0
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# The supervised child loads this file directly, so the base error has to be
|
|
4
|
+
# available without loading the whole gem.
|
|
5
|
+
require_relative "errors"
|
|
6
|
+
|
|
7
|
+
module Gemchat
|
|
8
|
+
# Turns text into a unit vector.
|
|
9
|
+
#
|
|
10
|
+
# The seam exists so nothing else in gemchat has to know how embeddings are
|
|
11
|
+
# made. Tests use a stub embedder and never load a model; `search` works with
|
|
12
|
+
# no embedder at all. That is not only for test speed — `gemchat search` is
|
|
13
|
+
# required to work with zero models downloaded, and BM25 must never become
|
|
14
|
+
# dependent on a 146MB file being present.
|
|
15
|
+
#
|
|
16
|
+
# Embedding runs in a **supervised subprocess**, and that is not an aesthetic
|
|
17
|
+
# choice. rllama 1.2.0's prebuilt llama.cpp aborts with SIGILL inside
|
|
18
|
+
# `llama_model_load_from_file` on this machine — measured intermittently, not
|
|
19
|
+
# deterministically: the same command that failed four times in a row later
|
|
20
|
+
# passed 20 of 20 with no configuration change, and `GGML_METAL_DISABLE=1`
|
|
21
|
+
# made no reliable difference. An illegal instruction cannot be rescued, so in
|
|
22
|
+
# this process there is no way to detect it, retry it, or degrade. A child we
|
|
23
|
+
# are willing to lose is the only place that failure can be handled at all.
|
|
24
|
+
module Embedder
|
|
25
|
+
class Error < Gemchat::Error; end
|
|
26
|
+
|
|
27
|
+
# Returned when no model is available, so callers can ask for vectors without
|
|
28
|
+
# branching on "is this a real embedder". Every method is a no-op rather than
|
|
29
|
+
# raising, because a missing model is a normal state, not an error.
|
|
30
|
+
class Null
|
|
31
|
+
def available? = false
|
|
32
|
+
def null? = true
|
|
33
|
+
def identity = nil
|
|
34
|
+
def dimensions = 0
|
|
35
|
+
def embed(_texts) = []
|
|
36
|
+
|
|
37
|
+
def close
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Wraps a loaded GGUF model. Used inside the child process.
|
|
42
|
+
#
|
|
43
|
+
# Named Gguf, not Rllama, on purpose: a class called `Embedder::Rllama`
|
|
44
|
+
# shadows the `rllama` gem's own `Rllama` for the rest of this lexical
|
|
45
|
+
# scope, so `Rllama.silence_log!` inside it resolves to
|
|
46
|
+
# Gemchat::Embedder::Rllama and raises NoMethodError. Every reference to the
|
|
47
|
+
# gem is root-scoped as `::Rllama` regardless.
|
|
48
|
+
class Gguf
|
|
49
|
+
def initialize(model_path:, model_name:)
|
|
50
|
+
require "rllama"
|
|
51
|
+
::Rllama.silence_log!
|
|
52
|
+
@model = ::Rllama::Model.new(model_path)
|
|
53
|
+
@identity = model_name
|
|
54
|
+
@dimensions = @model.n_embd
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
attr_reader :identity, :dimensions
|
|
58
|
+
|
|
59
|
+
def available? = true
|
|
60
|
+
def null? = false
|
|
61
|
+
|
|
62
|
+
# Returns unit vectors, so a dot product is a cosine similarity and no
|
|
63
|
+
# per-query magnitude normalisation is needed at search time.
|
|
64
|
+
def embed(texts)
|
|
65
|
+
list = Array(texts).map(&:to_s).reject { |t| t.strip.empty? }
|
|
66
|
+
return [] if list.empty?
|
|
67
|
+
|
|
68
|
+
@model.embed(list)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def close
|
|
72
|
+
@model&.close
|
|
73
|
+
@model = nil
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
module_function
|
|
78
|
+
|
|
79
|
+
# Never raises for a missing model: `search` depends on this returning Null
|
|
80
|
+
# rather than blowing up when no 146MB file is present.
|
|
81
|
+
def build(name = Models.default)
|
|
82
|
+
return Null.new unless Models.installed?(name)
|
|
83
|
+
|
|
84
|
+
Supervised.new(
|
|
85
|
+
model_path: Models.path(name),
|
|
86
|
+
identity: Models.identity(name),
|
|
87
|
+
prefix: Models.prefix(:document, name)
|
|
88
|
+
)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def available?(name = Models.default)
|
|
92
|
+
Models.installed?(name)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Embeds in a supervised child process, retrying once with the GPU backend
|
|
96
|
+
# disabled. Used directly by the CLI; `Supervised` is the object callers hold.
|
|
97
|
+
class Supervised
|
|
98
|
+
# A child is given a budget of input text and no more. llama.cpp sizes its
|
|
99
|
+
# embedding context from the input, and past a certain total it aborts the
|
|
100
|
+
# process rather than returning an error -- measured on this machine: a
|
|
101
|
+
# single 216-chunk batch died with SIGILL while 128 chunks in one batch
|
|
102
|
+
# succeeded, and where exactly it dies moved between runs.
|
|
103
|
+
#
|
|
104
|
+
# The ceiling is therefore treated as unknown rather than tuned. 16KB sits
|
|
105
|
+
# far below every failure point observed (45KB passed, 90KB did not), and
|
|
106
|
+
# `embed` halves a batch and retries when one fails anyway, so correctness
|
|
107
|
+
# does not depend on this number being right.
|
|
108
|
+
BATCH_BYTES = 16_000
|
|
109
|
+
|
|
110
|
+
# Per-item ceiling, and the reason it exists is a measurement rather than a
|
|
111
|
+
# spec. Real prose from the rake docs fails to embed as a single item at
|
|
112
|
+
# 1665 bytes and succeeds at 1634; the same bytes of repeated "word " run
|
|
113
|
+
# fine to 2500, so this is not a byte count and not a clean token count.
|
|
114
|
+
# Content-independent stripping made no difference either -- pure ASCII of
|
|
115
|
+
# the same text still failed -- which points at the native embedding path
|
|
116
|
+
# rather than at anything about the input.
|
|
117
|
+
#
|
|
118
|
+
# 1200 leaves margin under the observed cliff instead of sitting on it.
|
|
119
|
+
# The cost is real and worth stating: a long prose chunk is represented by
|
|
120
|
+
# its opening, and its tail is reachable through BM25 but not through
|
|
121
|
+
# `vsearch`. Prose is already split at the chunker, so this bites the
|
|
122
|
+
# largest sections only.
|
|
123
|
+
MAX_TEXT_BYTES = 1200
|
|
124
|
+
|
|
125
|
+
def initialize(model_path:, identity:, prefix: nil)
|
|
126
|
+
@model_path = model_path
|
|
127
|
+
@identity = identity
|
|
128
|
+
@prefix = prefix
|
|
129
|
+
@dimensions = nil
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
attr_reader :identity, :dimensions
|
|
133
|
+
|
|
134
|
+
def available? = true
|
|
135
|
+
def null? = false
|
|
136
|
+
|
|
137
|
+
# Returns [index, vector] pairs, not a bare vector list.
|
|
138
|
+
#
|
|
139
|
+
# Partial success is the normal case, not an edge case: a batch that cannot
|
|
140
|
+
# be embedded even after backoff is left pending, and those chunks must not
|
|
141
|
+
# be written with a neighbour's vector or dropped from the count. Pairing
|
|
142
|
+
# keeps the mapping to the caller's rows explicit so a caller can persist
|
|
143
|
+
# exactly what came back.
|
|
144
|
+
#
|
|
145
|
+
# `kind` records which side of the pair is being embedded. The prefix is
|
|
146
|
+
# fixed at construction, so this is documentation of intent rather than a
|
|
147
|
+
# switch -- documents and queries land in different regions for a prefixed
|
|
148
|
+
# model, and embedding a query with the document prefix is a silent
|
|
149
|
+
# retrieval-quality bug rather than a crash.
|
|
150
|
+
def embed(texts, kind: :document)
|
|
151
|
+
list = Array(texts).map { |t| clip(t.to_s) }.reject { |t| t.strip.empty? }
|
|
152
|
+
return [] if list.empty?
|
|
153
|
+
|
|
154
|
+
results = {}
|
|
155
|
+
failed = []
|
|
156
|
+
each_batch(list) { |batch, offset| collect(batch, offset, results, failed) }
|
|
157
|
+
|
|
158
|
+
unless failed.empty?
|
|
159
|
+
warn "left #{failed.size} chunk(s) pending: could not embed " \
|
|
160
|
+
"#{failed.first(3).join(", ")}#{", …" if failed.size > 3}"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
if results.empty?
|
|
164
|
+
raise Error, "no chunk could be embedded; this machine cannot run the model"
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
@dimensions ||= results.values.first.size
|
|
168
|
+
results.sort.map { |index, vector| [index, vector] }
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def close
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
private
|
|
175
|
+
|
|
176
|
+
# Trims to the per-item ceiling on a word boundary. The head is kept
|
|
177
|
+
# deliberately: a chunk's searchable text opens with its title, source
|
|
178
|
+
# type and signature, which is the part that should drive a match.
|
|
179
|
+
#
|
|
180
|
+
# `scrub` is not optional. A byte slice can land in the middle of a
|
|
181
|
+
# multi-byte character, and the very next `rindex(/\s/)` on the result
|
|
182
|
+
# raises `ArgumentError: invalid byte sequence in UTF-8`. Found by
|
|
183
|
+
# embedding 22k chunks from 120 real gems, not by the 236-chunk rake index
|
|
184
|
+
# that every other measurement here used -- so `gemchat embed` crashed on
|
|
185
|
+
# any bundle whose prose contained a character straddling the boundary.
|
|
186
|
+
def clip(text)
|
|
187
|
+
return text if text.bytesize <= MAX_TEXT_BYTES
|
|
188
|
+
|
|
189
|
+
cut = text.byteslice(0, MAX_TEXT_BYTES)
|
|
190
|
+
cut.force_encoding(text.encoding)
|
|
191
|
+
cut = cut.scrub("") unless cut.valid_encoding?
|
|
192
|
+
|
|
193
|
+
boundary = cut.rindex(/\s/)
|
|
194
|
+
(boundary ? cut[0...boundary] : cut).strip
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def each_batch(list)
|
|
198
|
+
batch = []
|
|
199
|
+
bytes = 0
|
|
200
|
+
start = 0
|
|
201
|
+
|
|
202
|
+
list.each_with_index do |text, i|
|
|
203
|
+
if batch.empty?
|
|
204
|
+
start = i
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
batch << text
|
|
208
|
+
bytes += text.bytesize
|
|
209
|
+
|
|
210
|
+
next unless bytes >= BATCH_BYTES
|
|
211
|
+
|
|
212
|
+
yield(batch, start)
|
|
213
|
+
batch = []
|
|
214
|
+
bytes = 0
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
yield(batch, start) unless batch.empty?
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Binary backoff: a batch that dies is retried as two halves, so a
|
|
221
|
+
# size-triggered abort costs one batch rather than the run.
|
|
222
|
+
def collect(texts, offset, results, failed)
|
|
223
|
+
vectors, dimensions, error = Embedder.attempt(
|
|
224
|
+
texts, model_path: @model_path, identity: @identity, prefix: @prefix
|
|
225
|
+
)
|
|
226
|
+
@dimensions ||= dimensions
|
|
227
|
+
|
|
228
|
+
if error.nil?
|
|
229
|
+
texts.each_index { |i| results[offset + i] = vectors[i] }
|
|
230
|
+
return
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
if texts.size == 1
|
|
234
|
+
failed << "##{offset + 1}"
|
|
235
|
+
warn " chunk ##{offset + 1} failed to embed: #{error}"
|
|
236
|
+
return
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
half = texts.size / 2
|
|
240
|
+
collect(texts.first(half), offset, results, failed)
|
|
241
|
+
collect(texts.drop(half), offset + half, results, failed)
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# One supervised attempt: plain backend, then Metal disabled.
|
|
246
|
+
# Returns [vectors, dimensions, error].
|
|
247
|
+
def attempt(texts, model_path:, identity:, prefix: nil)
|
|
248
|
+
[{}, {"GGML_METAL_DISABLE" => "1"}].each do |attempt_env|
|
|
249
|
+
vectors, dimensions, error = run_child(
|
|
250
|
+
texts, model_path:, identity:, env: attempt_env, prefix:
|
|
251
|
+
)
|
|
252
|
+
return [vectors, dimensions, nil] if error.nil?
|
|
253
|
+
|
|
254
|
+
@last_error = error
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
[nil, nil, @last_error]
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# The child body. Kept as one string because it has to run in a fresh
|
|
261
|
+
# process with no gemchat state, and because its correctness is easier to
|
|
262
|
+
# read here than split across two files.
|
|
263
|
+
def child_script
|
|
264
|
+
<<~RUBY
|
|
265
|
+
$stdout.sync = true
|
|
266
|
+
$LOAD_PATH.unshift(#{File.expand_path("..", __dir__).inspect})
|
|
267
|
+
require "json"
|
|
268
|
+
require "gemchat/embedder"
|
|
269
|
+
payload = JSON.parse($stdin.read)
|
|
270
|
+
embedder = Gemchat::Embedder::Gguf.new(
|
|
271
|
+
model_path: payload["model_path"], model_name: payload["identity"]
|
|
272
|
+
)
|
|
273
|
+
prefix = payload["prefix"].to_s
|
|
274
|
+
vectors = embedder.embed(payload["texts"].map { |t| prefix + t })
|
|
275
|
+
$stdout.write(JSON.generate({ "dimensions" => embedder.dimensions, "vectors" => vectors }))
|
|
276
|
+
embedder.close
|
|
277
|
+
RUBY
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def run_child(texts, model_path:, identity:, env:, prefix: nil)
|
|
281
|
+
out, status = Open3.capture2e(env, RbConfig.ruby, "-e", child_script,
|
|
282
|
+
stdin_data: JSON.generate({
|
|
283
|
+
"texts" => texts, "model_path" => model_path,
|
|
284
|
+
"identity" => identity, "prefix" => prefix.to_s
|
|
285
|
+
}))
|
|
286
|
+
unless status.success?
|
|
287
|
+
return [nil, nil, "exit #{status.exitstatus}#{detail(out)}"]
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
payload = JSON.parse(out)
|
|
291
|
+
dims = payload["dimensions"].to_i
|
|
292
|
+
vectors = payload["vectors"]
|
|
293
|
+
if vectors.empty? || vectors.first.size != dims
|
|
294
|
+
return [nil, nil, "child returned #{vectors.size} vectors of dim #{vectors.first&.size} (expected #{dims})"]
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
[vectors, dims, nil]
|
|
298
|
+
rescue JSON::ParserError => e
|
|
299
|
+
[nil, nil, "unparseable child output: #{e.message}"]
|
|
300
|
+
rescue => e
|
|
301
|
+
[nil, nil, "#{e.class}: #{e.message}"]
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def detail(out)
|
|
305
|
+
line = out.to_s.lines.reject { |l| l.strip.empty? }.last
|
|
306
|
+
line ? " (#{line.strip[0, 120]})" : ""
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
end
|
data/lib/gemchat/env.rb
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Gemchat
|
|
4
|
+
# Boolean environment parsing. This is the only thing we would have wanted
|
|
5
|
+
# from ActiveSupport, and depending on it would drag in 13 gems
|
|
6
|
+
# (concurrent-ruby, i18n, tzinfo, drb, connection_pool, ...) for a CLI that
|
|
7
|
+
# needs none of them. Plain Ruby instead.
|
|
8
|
+
module Env
|
|
9
|
+
TRUTHY = %w[1 true t yes y on].freeze
|
|
10
|
+
FALSEY = %w[0 false f no n off].freeze
|
|
11
|
+
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# Unset or unrecognised => default, so a typo never silently disables a
|
|
15
|
+
# feature or silently enables one.
|
|
16
|
+
def truthy?(key, default: false)
|
|
17
|
+
raw = ENV[key]
|
|
18
|
+
return default if raw.nil?
|
|
19
|
+
|
|
20
|
+
value = raw.strip.downcase
|
|
21
|
+
return true if TRUTHY.include?(value)
|
|
22
|
+
return false if FALSEY.include?(value)
|
|
23
|
+
|
|
24
|
+
default
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Gemchat
|
|
4
|
+
# Base class for every error gemchat raises deliberately, so the CLI can
|
|
5
|
+
# rescue one thing and print a message instead of a backtrace.
|
|
6
|
+
#
|
|
7
|
+
# Its own file because subclasses live in files that are required before the
|
|
8
|
+
# module body of lib/gemchat.rb has been evaluated.
|
|
9
|
+
class Error < StandardError; end
|
|
10
|
+
end
|
data/lib/gemchat/hook.rb
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Deliberately requires only what the hook needs, not all of gemchat. Bundler
|
|
4
|
+
# loads this from a fresh child process during `bundle install`, where eagerly
|
|
5
|
+
# pulling in the store and the indexer would mean opening SQLite on every install
|
|
6
|
+
# in a project that has gemchat as a plugin.
|
|
7
|
+
require "json"
|
|
8
|
+
require "rbconfig"
|
|
9
|
+
require_relative "paths"
|
|
10
|
+
require_relative "lockfile"
|
|
11
|
+
require_relative "manifest"
|
|
12
|
+
require_relative "trust"
|
|
13
|
+
|
|
14
|
+
module Gemchat
|
|
15
|
+
# The logic behind the Bundler plugin hook.
|
|
16
|
+
#
|
|
17
|
+
# Kept in lib/ rather than in plugins.rb on purpose. plugins.rb is the copy
|
|
18
|
+
# Bundler loads from the *plugin* install, and a dual `gem` + `plugin`
|
|
19
|
+
# declaration installs two physical copies with nothing pinning them together
|
|
20
|
+
# (§15.4). A file that contains only the `add_hook` registration and delegates
|
|
21
|
+
# here — reached through the bundled copy — has nothing to drift.
|
|
22
|
+
#
|
|
23
|
+
# Contract, all of it measured or tested in test/plugin_shim_test.rb:
|
|
24
|
+
# * never raises; a raising plugins.rb fails the whole install
|
|
25
|
+
# * silent unless there is genuinely something to say
|
|
26
|
+
# * at most one prefixed line
|
|
27
|
+
module Hook
|
|
28
|
+
HOST_GEM = "gemchat"
|
|
29
|
+
SUPPORTED = Gem::Requirement.new(">= 0.1.0")
|
|
30
|
+
PREFIX = "[gemchat] "
|
|
31
|
+
|
|
32
|
+
class << self
|
|
33
|
+
def auto_index
|
|
34
|
+
return if quiet_environment?
|
|
35
|
+
return report("#{HOST_GEM} is not in the bundle") unless host_spec
|
|
36
|
+
return report("skipped: outside supported range") unless SUPPORTED.satisfied_by?(host_spec.version)
|
|
37
|
+
|
|
38
|
+
project = Dir.pwd
|
|
39
|
+
return report("not trusted for #{project}; run `gemchat index` yourself") unless ::Gemchat::Trust.trusted?(project)
|
|
40
|
+
|
|
41
|
+
digest = ::Gemchat::Lockfile.digest(project)
|
|
42
|
+
return if digest.nil?
|
|
43
|
+
return if ::Gemchat::Manifest.new(project).up_to_date?(digest)
|
|
44
|
+
|
|
45
|
+
run_index(project)
|
|
46
|
+
rescue => e
|
|
47
|
+
report "auto-index skipped (#{e.class}: #{e.message})"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def host_spec
|
|
51
|
+
@host_spec ||= ::Bundler.load.specs.find { |s| s.name == HOST_GEM }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# The hook runs on CI and on deploy installs, where any output is noise and
|
|
55
|
+
# any work is wasted. Checked before the trust lookup because it is free.
|
|
56
|
+
#
|
|
57
|
+
# Deliberately not keyed on RAILS_ENV, which is how the Hyperdrive shim
|
|
58
|
+
# gates itself. gemchat is not a Rails tool, so in any plain Ruby project
|
|
59
|
+
# RAILS_ENV is unset and such a guard would silence the hook permanently —
|
|
60
|
+
# a no-op that looks installed and does nothing.
|
|
61
|
+
def quiet_environment?
|
|
62
|
+
!ENV["CI"].nil? || ENV["GEMCHAT_NO_AUTO_INDEX"].to_s == "1"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Shells out rather than indexing in-process. Two reasons: the hook must
|
|
66
|
+
# not hold Bundler's stdout or leave an open SQLite handle behind a bundle
|
|
67
|
+
# install, and a subprocess that dies cannot take the install with it.
|
|
68
|
+
def run_index(project)
|
|
69
|
+
summary = index_summary(project)
|
|
70
|
+
return if summary.nil?
|
|
71
|
+
|
|
72
|
+
if summary["changed"].to_i.positive?
|
|
73
|
+
report("indexed #{summary["changed"]} #{pluralise(summary["changed"], "gem")} " \
|
|
74
|
+
"(#{summary["chunks"]} chunks)")
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Runs the executable from the *bundled* gem, not whatever `gemchat` is on
|
|
79
|
+
# PATH. Under `bundle install` there may be no gemchat on PATH at all, and
|
|
80
|
+
# if there is one it may be a different version -- the same drift the shim
|
|
81
|
+
# design exists to prevent. A missing or unparseable result returns nil and
|
|
82
|
+
# the caller stays silent, rather than failing somebody's install.
|
|
83
|
+
def index_summary(project)
|
|
84
|
+
exe = File.join(host_spec.full_gem_path, "exe", "gemchat")
|
|
85
|
+
return nil unless File.executable?(exe)
|
|
86
|
+
|
|
87
|
+
raw = IO.popen(
|
|
88
|
+
{
|
|
89
|
+
"GEMCHAT_HOME" => ::Gemchat::Paths.root,
|
|
90
|
+
"GEMCHAT_CONFIG_HOME" => ::Gemchat::Paths.config_home
|
|
91
|
+
},
|
|
92
|
+
[RbConfig.ruby, exe, "index", "--json"],
|
|
93
|
+
chdir: project, err: [:child, :out]
|
|
94
|
+
) { |io| io.read.to_s }
|
|
95
|
+
|
|
96
|
+
JSON.parse(raw.lines.last.to_s)
|
|
97
|
+
rescue
|
|
98
|
+
# Never propagate. A failure here must not surface as a failed
|
|
99
|
+
# `bundle install`; the caller simply stays quiet and the index waits
|
|
100
|
+
# for the next explicit `gemchat index`.
|
|
101
|
+
nil
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def pluralise(count, word)
|
|
105
|
+
(count == 1) ? word : "#{word}s"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def report(message)
|
|
109
|
+
warn "#{PREFIX}#{message}"
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module Gemchat
|
|
8
|
+
# Client for gemchat.org's hosted index.
|
|
9
|
+
#
|
|
10
|
+
# This is the opt-in tier, and the rules around it are stricter than the code
|
|
11
|
+
# suggests. It is **never** a fallback: a local-first privacy tool that silently
|
|
12
|
+
# sent queries to a server would be a different product, and a user with a
|
|
13
|
+
# populated local index has no reason to expect their questions leaving the
|
|
14
|
+
# machine. So nothing here is ever reached unless the user asked, and a
|
|
15
|
+
# failure here stops rather than retrying locally.
|
|
16
|
+
#
|
|
17
|
+
# Speaks `POST /api/v1/query` rather than the MCP endpoint. MCP is for agents
|
|
18
|
+
# and returns prose; a CLI needs rows, and parsing formatted Markdown breaks
|
|
19
|
+
# silently the first time the server reformats a line.
|
|
20
|
+
class Hosted
|
|
21
|
+
DEFAULT_ENDPOINT = "https://gemchat.org/api/v1/query"
|
|
22
|
+
CREDENTIALS = "credentials"
|
|
23
|
+
TIMEOUT = 30
|
|
24
|
+
MAX_LIMIT = 20
|
|
25
|
+
|
|
26
|
+
class Error < Gemchat::Error; end
|
|
27
|
+
class Unauthorized < Error; end
|
|
28
|
+
|
|
29
|
+
# The server's search could not run, as distinct from a search that ran and
|
|
30
|
+
# matched nothing. Kept separate because the two mean opposite things: this
|
|
31
|
+
# is a broken or misconfigured server, and retrying or widening the query
|
|
32
|
+
# will not help.
|
|
33
|
+
class SearchUnavailable < Error; end
|
|
34
|
+
|
|
35
|
+
Result = Struct.new(:rows, :version_note, :missing, :notice, :scope)
|
|
36
|
+
|
|
37
|
+
def self.enabled?
|
|
38
|
+
endpoint = ENV["GEMCHAT_HOST"]
|
|
39
|
+
return true if endpoint && !endpoint.strip.empty?
|
|
40
|
+
|
|
41
|
+
!api_key.nil?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Where the key lives: an env var first so CI can set it without a file, then
|
|
45
|
+
# `~/.gemchat/credentials`, which is where §12.1 says it goes.
|
|
46
|
+
def self.api_key
|
|
47
|
+
from_env = ENV["GEMCHAT_API_KEY"]
|
|
48
|
+
return from_env unless from_env.nil? || from_env.strip.empty?
|
|
49
|
+
|
|
50
|
+
credentials_file
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def self.credentials_path
|
|
54
|
+
File.join(Gemchat.config_home, CREDENTIALS)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Reads the key, accepting the two shapes people actually write:
|
|
58
|
+
#
|
|
59
|
+
# GEMCHAT_API_KEY=abc… explicit, so the file can hold more later
|
|
60
|
+
# abc… bare, which is what a hand-written file usually is
|
|
61
|
+
#
|
|
62
|
+
# The bare form is not a nicety. A parser that only understands the first
|
|
63
|
+
# shape returns nil for a file that plainly contains a key, and the CLI then
|
|
64
|
+
# says "hosted backend not configured" -- which sends the user looking for a
|
|
65
|
+
# setting problem when the secret is sitting on disk. Failing to parse a
|
|
66
|
+
# credentials file must never look like there being no credentials.
|
|
67
|
+
#
|
|
68
|
+
# Anything else non-empty is returned as-is rather than discarded, so a
|
|
69
|
+
# malformed value produces a 401 from the server -- which names the real
|
|
70
|
+
# problem -- instead of silence. Only an empty or all-comment file is nil.
|
|
71
|
+
def self.credentials_file
|
|
72
|
+
path = credentials_path
|
|
73
|
+
return nil unless File.file?(path)
|
|
74
|
+
|
|
75
|
+
lines = File.read(path).lines.map(&:strip).reject { |line| line.empty? || line.start_with?("#") }
|
|
76
|
+
return nil if lines.empty?
|
|
77
|
+
|
|
78
|
+
assigned = lines.find { |line| line.match?(/\A(?:export\s+)?GEMCHAT_API_KEY\s*=/) }
|
|
79
|
+
if assigned
|
|
80
|
+
value = assigned.sub(/\A(?:export\s+)?GEMCHAT_API_KEY\s*=\s*/, "").strip
|
|
81
|
+
return value.delete_prefix("\"").delete_suffix("\"").delete_prefix("'").delete_suffix("'").strip
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
lines.first
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def initialize(key: nil, endpoint: nil)
|
|
88
|
+
@key = key || self.class.api_key
|
|
89
|
+
@endpoint = endpoint || ENV["GEMCHAT_HOST"]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
attr_reader :key, :endpoint
|
|
93
|
+
|
|
94
|
+
def configured?
|
|
95
|
+
!key.nil? && !key.to_s.strip.empty?
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def url
|
|
99
|
+
base = @endpoint.to_s.strip
|
|
100
|
+
return URI.parse(DEFAULT_ENDPOINT) if base.empty?
|
|
101
|
+
|
|
102
|
+
# GEMCHAT_HOST is documented as a host, so accept a bare host or a full URL
|
|
103
|
+
# and always land on the versioned path. Any path in the variable is
|
|
104
|
+
# ignored: §12.1 names a host, and letting a stray path through would send
|
|
105
|
+
# queries somewhere the user never indicated.
|
|
106
|
+
uri = base.include?("://") ? URI.parse(base) : URI.parse("https://#{base}")
|
|
107
|
+
uri.path = "/api/v1/query"
|
|
108
|
+
uri
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# `gem_names` and `gem_versions` come straight from the local lockfile.
|
|
112
|
+
# Passing the pins is what makes a hosted answer about the version that is
|
|
113
|
+
# actually installed rather than whatever the server happens to have indexed;
|
|
114
|
+
# the server reports any pin it could not apply, so nothing is silently wrong.
|
|
115
|
+
def query(text, limit: 5, gem_names: nil, gem_versions: nil, auto_index: false)
|
|
116
|
+
unless configured?
|
|
117
|
+
raise Unauthorized, "no API key; set GEMCHAT_API_KEY, or write GEMCHAT_API_KEY=... to #{self.class.credentials_path}"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
body = {
|
|
121
|
+
query: text,
|
|
122
|
+
limit: [limit.to_i, 1].max.clamp(1, MAX_LIMIT),
|
|
123
|
+
gem_names: Array(gem_names),
|
|
124
|
+
gem_versions: Array(gem_versions),
|
|
125
|
+
auto_index: auto_index
|
|
126
|
+
}.compact
|
|
127
|
+
|
|
128
|
+
response = post(url, body)
|
|
129
|
+
payload = parse(response)
|
|
130
|
+
|
|
131
|
+
raise Unauthorized, hosted_message(payload, "Unauthorized") if response.code == "401"
|
|
132
|
+
|
|
133
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
134
|
+
detail = hosted_message(payload, "")
|
|
135
|
+
|
|
136
|
+
# A server that could not run the search says so in a field of its own.
|
|
137
|
+
# Reporting it as a generic failure loses the one fact that separates a
|
|
138
|
+
# broken server from an empty index, and they call for opposite reactions:
|
|
139
|
+
# one is worth retrying later, the other means widen the query.
|
|
140
|
+
if payload.is_a?(Hash) && payload["error"] == "search_unavailable"
|
|
141
|
+
reason = payload["reason"].to_s.strip
|
|
142
|
+
raise SearchUnavailable,
|
|
143
|
+
"gemchat.org's search could not run#{" (#{reason})" unless reason.empty?}. " \
|
|
144
|
+
"This is not an empty result -- nothing was searched. `gemchat search` still works offline."
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
raise Error, "hosted query failed: HTTP #{response.code}#{" #{detail}" unless detail.to_s.empty?}"
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
Result.new(
|
|
151
|
+
rows: Array(payload["results"]),
|
|
152
|
+
version_note: payload["version_note"],
|
|
153
|
+
missing: Array(payload["missing"]),
|
|
154
|
+
notice: payload["notice"],
|
|
155
|
+
scope: payload["scope"].is_a?(Hash) ? payload["scope"] : nil
|
|
156
|
+
)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Public so tests can replace it and a future `--insecure-proxy` can wrap it.
|
|
160
|
+
# Everything above it is policy; this is the wire.
|
|
161
|
+
def post(uri, body)
|
|
162
|
+
request = Net::HTTP::Post.new(uri)
|
|
163
|
+
request["Authorization"] = "Bearer #{key}"
|
|
164
|
+
request["Content-Type"] = "application/json"
|
|
165
|
+
request["Accept"] = "application/json"
|
|
166
|
+
request.body = JSON.generate(body)
|
|
167
|
+
|
|
168
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: TIMEOUT, read_timeout: TIMEOUT) do |http|
|
|
169
|
+
http.request(request)
|
|
170
|
+
end
|
|
171
|
+
rescue Net::OpenTimeout, Net::ReadTimeout => e
|
|
172
|
+
raise Error, "hosted query timed out after #{TIMEOUT}s (#{e.class}). `gemchat search` still works offline."
|
|
173
|
+
rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
|
|
174
|
+
raise Error, "cannot reach #{uri.host} (#{e.class}). `gemchat search` still works offline."
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
private
|
|
178
|
+
|
|
179
|
+
def parse(response)
|
|
180
|
+
JSON.parse(response.body.to_s)
|
|
181
|
+
rescue JSON::ParserError
|
|
182
|
+
# A proxy or a login page can return 200 with HTML in it. Saying "invalid
|
|
183
|
+
# JSON" is technically true and practically useless.
|
|
184
|
+
raise Error, "hosted endpoint returned #{response.code} with a non-JSON body (#{response.body.to_s[0, 80].inspect})"
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# No ActiveSupport here: this gem is plain Ruby, so no `presence`.
|
|
188
|
+
def hosted_message(payload, fallback)
|
|
189
|
+
error = payload.is_a?(Hash) ? payload["error"].to_s.strip : ""
|
|
190
|
+
return error unless error.empty?
|
|
191
|
+
|
|
192
|
+
fallback.to_s.strip.empty? ? nil : fallback.to_s.strip
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|