rubik 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ATTRIBUTION.md +1 -1
- data/CHANGELOG.md +61 -1
- data/lib/rubik/ast/extractor.rb +2 -1
- data/lib/rubik/ast/treesitter_extractor.rb +2 -1
- data/lib/rubik/benchmark/vakanuvis_adoption.rb +2 -1
- data/lib/rubik/config/secret_policy.rb +6 -5
- data/lib/rubik/config/store.rb +2 -1
- data/lib/rubik/config.rb +2 -2
- data/lib/rubik/context/ignore_rules.rb +193 -0
- data/lib/rubik/context/scanner.rb +46 -161
- data/lib/rubik/llm/client.rb +6 -200
- data/lib/rubik/llm/connection_pool.rb +104 -0
- data/lib/rubik/llm/registry.rb +3 -2
- data/lib/rubik/llm/sse_parser.rb +110 -0
- data/lib/rubik/lsp/circuit_breaker.rb +98 -0
- data/lib/rubik/lsp/client_provisioner.rb +49 -0
- data/lib/rubik/lsp/manager.rb +55 -88
- data/lib/rubik/mcp/manager.rb +25 -13
- data/lib/rubik/orchestrator/auto_apply.rb +4 -3
- data/lib/rubik/orchestrator/goal_ledger/ledger.rb +12 -1
- data/lib/rubik/orchestrator/goal_ledger/schema_vocabulary.rb +38 -0
- data/lib/rubik/orchestrator/goal_ledger/xml_render.rb +6 -1
- data/lib/rubik/orchestrator/goal_ledger.rb +1 -0
- data/lib/rubik/orchestrator/lsp_feedback.rb +1 -1
- data/lib/rubik/orchestrator/step_executor.rb +2 -1
- data/lib/rubik/orchestrator/subagent/notices.rb +2 -2
- data/lib/rubik/orchestrator/subagent/transcript_formatter.rb +2 -1
- data/lib/rubik/platform/io_drain.rb +3 -1
- data/lib/rubik/plugin/storage.rb +8 -8
- data/lib/rubik/repl/config_wiring.rb +5 -3
- data/lib/rubik/repl/input_execution.rb +4 -3
- data/lib/rubik/repl/lifecycle.rb +1 -1
- data/lib/rubik/repl/tee_io.rb +2 -1
- data/lib/rubik/runtime/ast_service.rb +2 -1
- data/lib/rubik/runtime/overlay_storage.rb +17 -17
- data/lib/rubik/runtime/task_manager.rb +1 -1
- data/lib/rubik/sandbox/runner.rb +5 -4
- data/lib/rubik/security/audit_log.rb +2 -1
- data/lib/rubik/security/command_gate.rb +11 -152
- data/lib/rubik/security/guard.rb +1 -1
- data/lib/rubik/security/secret_store.rb +32 -3
- data/lib/rubik/security/segment_report.rb +161 -0
- data/lib/rubik/session/recorder.rb +2 -1
- data/lib/rubik/skills/manager.rb +2 -1
- data/lib/rubik/storage/transcript_indexer.rb +3 -2
- data/lib/rubik/storage/vakanuvis.rb +2 -1
- data/lib/rubik/tools/builtins/goal_ledger.rb +12 -1
- data/lib/rubik/tools/builtins/synthesis.rb +3 -3
- data/lib/rubik/tools/preflight_runner.rb +2 -1
- data/lib/rubik/tools/uri_resolver.rb +3 -2
- data/lib/rubik/ui/capability.rb +1 -0
- data/lib/rubik/version.rb +1 -1
- data/lib/rubik.rb +3 -3
- metadata +8 -1
|
@@ -4,10 +4,15 @@ require "digest"
|
|
|
4
4
|
require "pathname"
|
|
5
5
|
require_relative "../platform"
|
|
6
6
|
require_relative "git_lister"
|
|
7
|
+
require_relative "ignore_rules"
|
|
7
8
|
|
|
8
9
|
module Rubik
|
|
9
10
|
module Context
|
|
10
11
|
class Scanner
|
|
12
|
+
# Index language tags. Deliberately diverges from
|
|
13
|
+
# LSP::Manager::EXTENSION_MAP (which maps to LSP server ids, e.g.
|
|
14
|
+
# .cpp -> "c"); the two tables serve different consumers — keep
|
|
15
|
+
# them separate.
|
|
11
16
|
EXTENSION_MAP = {
|
|
12
17
|
".rb" => "ruby",
|
|
13
18
|
".py" => "python",
|
|
@@ -46,45 +51,14 @@ module Rubik
|
|
|
46
51
|
"package.json" => "javascript"
|
|
47
52
|
}.freeze
|
|
48
53
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
"**/.trash*/**",
|
|
58
|
-
".turbo/**",
|
|
59
|
-
".sst/**",
|
|
60
|
-
".next/**",
|
|
61
|
-
".nuxt/**",
|
|
62
|
-
"node_modules/**",
|
|
63
|
-
"vendor/**",
|
|
64
|
-
"tmp/**",
|
|
65
|
-
"dist/**",
|
|
66
|
-
"build/**",
|
|
67
|
-
".cache/**",
|
|
68
|
-
"coverage/**",
|
|
69
|
-
# Multi-GB local model blobs; never project sources.
|
|
70
|
-
".ollama/**",
|
|
71
|
-
# Agent-harness state: transcripts, sessions, caches.
|
|
72
|
-
".zcode/**"
|
|
73
|
-
].freeze
|
|
74
|
-
|
|
75
|
-
# Subtrees whose ignore pattern is "<path>/**" (optionally with a
|
|
76
|
-
# "**/" prefix) are pruned whole when the stem's components occur
|
|
77
|
-
# contiguously in a directory's path. NOTE: this is intentionally
|
|
78
|
-
# STRONGER than the legacy per-file filter — Ruby's fnmatch treats a
|
|
79
|
-
# trailing `**` as a single segment, so patterns like "node_modules/**"
|
|
80
|
-
# historically rejected only first-level entries while deeper files
|
|
81
|
-
# leaked into the index. The walker applies the pattern's plain meaning
|
|
82
|
-
# (whole subtree ignored), matching real gitignore semantics.
|
|
83
|
-
PRUNABLE_PATTERN = %r{\A(?:\*\*/)?(.+)/\*\*\z}
|
|
84
|
-
# Also hard-rejected inside `ignored?`; pruned before descending.
|
|
85
|
-
# ROOT-anchored only (comps.first) to match the rg-side `!/X/**`
|
|
86
|
-
# globs — a nested `src/AppData` stays indexed.
|
|
87
|
-
HARD_PRUNE_ROOTS = [".git", ".rubik", ".ua", ".turbo", ".sst", ".next", ".nuxt", "AppData"].freeze
|
|
54
|
+
# The "is this path in scope?" dialect (default ignores, prunable
|
|
55
|
+
# subtree patterns, hard-prune roots, glob->regex translation, the
|
|
56
|
+
# include dialect and the reserved-root predicate) lives in
|
|
57
|
+
# IgnoreRules; the Scanner is only a consumer of its predicates.
|
|
58
|
+
# Legacy aliases: external consumers (native_search's rg flags, tests)
|
|
59
|
+
# still read the tables off the Scanner.
|
|
60
|
+
DEFAULT_IGNORES = IgnoreRules::DEFAULT_IGNORES
|
|
61
|
+
HARD_PRUNE_ROOTS = IgnoreRules::HARD_PRUNE_ROOTS
|
|
88
62
|
|
|
89
63
|
# Outcome of the most recent scan. A partial index served as ground
|
|
90
64
|
# truth is worse than an honest flag: every rescue boundary that drops
|
|
@@ -121,15 +95,13 @@ module Rubik
|
|
|
121
95
|
files = {}
|
|
122
96
|
|
|
123
97
|
# 1. Scan primary root
|
|
124
|
-
|
|
125
|
-
files.merge!(scan_root(@root_dir, ".", primary_ignores, previous))
|
|
98
|
+
files.merge!(scan_root(@root_dir, ".", previous))
|
|
126
99
|
|
|
127
100
|
# 2. Scan additional roots
|
|
128
101
|
@additional_roots.each do |extra_root|
|
|
129
102
|
next unless File.directory?(extra_root)
|
|
130
103
|
|
|
131
|
-
|
|
132
|
-
files.merge!(scan_root(extra_root, relative_prefix(extra_root), extra_ignores, previous))
|
|
104
|
+
files.merge!(scan_root(extra_root, relative_prefix(extra_root), previous))
|
|
133
105
|
end
|
|
134
106
|
|
|
135
107
|
files
|
|
@@ -224,7 +196,7 @@ module Rubik
|
|
|
224
196
|
(stat.mtime.tv_sec * 1_000_000) + stat.mtime.tv_usec
|
|
225
197
|
end
|
|
226
198
|
|
|
227
|
-
def walk_dir(base_dir, rel_prefix, key_prefix,
|
|
199
|
+
def walk_dir(base_dir, rel_prefix, key_prefix, rules, previous)
|
|
228
200
|
current = rel_prefix.empty? ? base_dir : "#{base_dir}/#{rel_prefix}"
|
|
229
201
|
out = {}
|
|
230
202
|
|
|
@@ -250,9 +222,9 @@ module Rubik
|
|
|
250
222
|
end
|
|
251
223
|
|
|
252
224
|
if lst.directory? && !lst.symlink?
|
|
253
|
-
next if
|
|
225
|
+
next if prune_dir?(rel, rules)
|
|
254
226
|
|
|
255
|
-
out.merge!(walk_dir(base_dir, rel, key_prefix,
|
|
227
|
+
out.merge!(walk_dir(base_dir, rel, key_prefix, rules, previous))
|
|
256
228
|
else
|
|
257
229
|
# Symlinked FILES are still listed (File.stat follows the link),
|
|
258
230
|
# matching the old Dir.glob behavior; only directory descent is
|
|
@@ -264,8 +236,8 @@ module Rubik
|
|
|
264
236
|
nil
|
|
265
237
|
end
|
|
266
238
|
next unless st&.file?
|
|
267
|
-
next if
|
|
268
|
-
next unless
|
|
239
|
+
next if rules.ignore_file?(rel)
|
|
240
|
+
next unless rules.include_file?(rel)
|
|
269
241
|
|
|
270
242
|
rel_key = key_prefix == "." ? rel : "#{key_prefix}/#{rel}"
|
|
271
243
|
out[rel_key] = entry_for(rel_key, "#{base_dir}/#{rel}", previous, stat: st)
|
|
@@ -279,32 +251,17 @@ module Rubik
|
|
|
279
251
|
out
|
|
280
252
|
end
|
|
281
253
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
needles.any? { |needle| contiguous_components?(comps, needle) }
|
|
254
|
+
# The one directory-prune entry point the walker uses; split lives on
|
|
255
|
+
# the Scanner side so IgnoreRules#prune_dir? takes plain components.
|
|
256
|
+
def prune_dir?(rel, rules)
|
|
257
|
+
rules.prune_dir?(rel.split("/"))
|
|
287
258
|
end
|
|
288
259
|
|
|
289
|
-
#
|
|
290
|
-
#
|
|
291
|
-
#
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
ignore_patterns.filter_map do |pattern|
|
|
295
|
-
next unless (m = PRUNABLE_PATTERN.match(pattern))
|
|
296
|
-
|
|
297
|
-
m[1].split("/")
|
|
298
|
-
end
|
|
299
|
-
end
|
|
300
|
-
|
|
301
|
-
def contiguous_components?(haystack, needle)
|
|
302
|
-
return true if haystack == needle
|
|
303
|
-
return false if haystack.size < needle.size
|
|
304
|
-
|
|
305
|
-
(0..(haystack.size - needle.size)).any? do |offset|
|
|
306
|
-
haystack[offset, needle.size] == needle
|
|
307
|
-
end
|
|
260
|
+
# One IgnoreRules value object per scan root: the root's .gitignore is
|
|
261
|
+
# merged into the default/exclude tables and everything is compiled
|
|
262
|
+
# once, up front.
|
|
263
|
+
def rules_for(dir)
|
|
264
|
+
IgnoreRules.build(dir: dir, custom_excludes: @custom_excludes, custom_includes: @custom_includes)
|
|
308
265
|
end
|
|
309
266
|
|
|
310
267
|
# Key prefix for files found under an additional root, mirroring the
|
|
@@ -319,17 +276,17 @@ module Rubik
|
|
|
319
276
|
# One recursive directory walk with EARLY SUBTREE PRUNING: instead of
|
|
320
277
|
# enumerating ignored trees (.git/, node_modules/, vendor/ – thousands
|
|
321
278
|
# of entries on real projects) only to reject each file afterwards,
|
|
322
|
-
# descent into a matching subtree never happens.
|
|
323
|
-
#
|
|
324
|
-
#
|
|
325
|
-
#
|
|
326
|
-
def scan_root(base_dir, prefix,
|
|
327
|
-
|
|
279
|
+
# descent into a matching subtree never happens (rules.prune_dir?).
|
|
280
|
+
# `ignored?` semantics survive as rules.ignore_file? filtering every
|
|
281
|
+
# file that IS visited. `prefix` is the key namespace for this root
|
|
282
|
+
# ("." for the primary project root).
|
|
283
|
+
def scan_root(base_dir, prefix, previous)
|
|
284
|
+
rules = rules_for(base_dir)
|
|
328
285
|
if git_listing? && @custom_includes.empty? && (listed = GitLister.files(base_dir))
|
|
329
|
-
return index_listed(base_dir, prefix, listed,
|
|
286
|
+
return index_listed(base_dir, prefix, listed, rules, previous)
|
|
330
287
|
end
|
|
331
288
|
|
|
332
|
-
walk_dir(base_dir, "", prefix,
|
|
289
|
+
walk_dir(base_dir, "", prefix, rules, previous)
|
|
333
290
|
rescue StandardError => e
|
|
334
291
|
Rubik::Log.warn(:context, "root scan failed for #{base_dir} (#{e.class}: #{e.message}); root omitted from index")
|
|
335
292
|
mark_scan_aborted!("root scan failed for #{base_dir}: #{e.class}: #{e.message}")
|
|
@@ -341,26 +298,19 @@ module Rubik
|
|
|
341
298
|
# the repo's own gitignore: DEFAULT_IGNORES subtrees may be missing
|
|
342
299
|
# from it, and tracked files ignore gitignore entirely), ordered
|
|
343
300
|
# identically to walk_dir's depth-first-per-directory sort, and
|
|
344
|
-
# ancestor-pruned with the same whole-subtree semantics as the walker
|
|
301
|
+
# ancestor-pruned with the same whole-subtree semantics as the walker
|
|
302
|
+
# (rules.prune_dir?, memoized per ancestor directory below).
|
|
345
303
|
# Residual drift the OTHER way: gitignore features the walker's crude
|
|
346
304
|
# fnmatch translation lacks (negations, anchored patterns) can make the
|
|
347
305
|
# fast path list fewer files.
|
|
348
|
-
def index_listed(base_dir, key_prefix, rel_paths,
|
|
306
|
+
def index_listed(base_dir, key_prefix, rel_paths, rules, previous)
|
|
349
307
|
out = {}
|
|
350
308
|
# Verdict per ancestor directory, computed on first sight: walker
|
|
351
|
-
# order keeps siblings adjacent, so each distinct directory's
|
|
309
|
+
# order keeps siblings adjacent, so each distinct directory's prune
|
|
352
310
|
# status is evaluated once per scan instead of once per file per
|
|
353
|
-
# depth.
|
|
354
|
-
# the memo is exact.
|
|
355
|
-
dir_verdicts = Hash.new { |h, dir| h[dir] =
|
|
356
|
-
|
|
357
|
-
# PERF: filter ignore_patterns to exclude prunable subtree patterns that
|
|
358
|
-
# dir_verdicts already checked on ancestor directory segments.
|
|
359
|
-
file_patterns = ignore_patterns.reject do |pat|
|
|
360
|
-
m = PRUNABLE_PATTERN.match(pat)
|
|
361
|
-
m && !m[1].include?("*") && !m[1].include?("?") && !m[1].include?("[")
|
|
362
|
-
end
|
|
363
|
-
file_rules = compile_ignore_regexes(file_patterns)
|
|
311
|
+
# depth. rules.prune_dir? is a pure function of the directory's
|
|
312
|
+
# components, so the memo is exact.
|
|
313
|
+
dir_verdicts = Hash.new { |h, dir| h[dir] = rules.prune_dir?(dir.split("/")) }
|
|
364
314
|
|
|
365
315
|
order_like_walker(rel_paths).each do |rel|
|
|
366
316
|
# PERF: zero-allocation ancestor check using string slicing instead of Array#split and Array#join
|
|
@@ -374,9 +324,8 @@ module Rubik
|
|
|
374
324
|
idx += 1
|
|
375
325
|
end
|
|
376
326
|
next if ignored_dir
|
|
377
|
-
next if
|
|
378
|
-
next
|
|
379
|
-
next unless included?(rel, @custom_includes)
|
|
327
|
+
next if rules.listed_file_ignored?(rel)
|
|
328
|
+
next unless rules.include_file?(rel)
|
|
380
329
|
|
|
381
330
|
full = "#{base_dir}/#{rel}"
|
|
382
331
|
st = begin
|
|
@@ -413,70 +362,6 @@ module Rubik
|
|
|
413
362
|
def order_like_walker(paths)
|
|
414
363
|
paths.sort_by! { |p| p.tr("/", "\x00") }
|
|
415
364
|
end
|
|
416
|
-
|
|
417
|
-
def parse_gitignore(dir = @root_dir)
|
|
418
|
-
gitignore_path = File.join(dir, ".gitignore")
|
|
419
|
-
return [] unless File.exist?(gitignore_path)
|
|
420
|
-
|
|
421
|
-
patterns = File.readlines(gitignore_path, chomp: true).map(&:strip).reject do |line|
|
|
422
|
-
line.empty? || line.start_with?("#")
|
|
423
|
-
end
|
|
424
|
-
patterns.map do |pattern|
|
|
425
|
-
pattern.end_with?("/") ? "#{pattern}**" : pattern
|
|
426
|
-
end
|
|
427
|
-
rescue Errno::ENOENT, Errno::EACCES
|
|
428
|
-
[]
|
|
429
|
-
end
|
|
430
|
-
|
|
431
|
-
def ignored?(path, ignore_patterns)
|
|
432
|
-
return true if path == ".rubik" || path.start_with?(".rubik/")
|
|
433
|
-
return true if path == ".git" || path.start_with?(".git/")
|
|
434
|
-
|
|
435
|
-
# PERF: precompile simple globs to memoized Regexp objects to avoid
|
|
436
|
-
# repeated File.fnmatch? overhead across thousands of files; exotic
|
|
437
|
-
# globs keep the fnmatch semantics untouched.
|
|
438
|
-
compile_ignore_regexes(ignore_patterns).any? { |r| pattern_match?(r, path) }
|
|
439
|
-
end
|
|
440
|
-
|
|
441
|
-
# Translates a glob into a Regexp only where the mapping is exact: no
|
|
442
|
-
# character classes, no leading-slash anchoring, and `**` only ever a
|
|
443
|
-
# whole path segment (leading "**/", mid "/**/", trailing "/**").
|
|
444
|
-
# Anything else returns nil and pattern_match? falls back to
|
|
445
|
-
# File.fnmatch?, whose segment rules the Regexp form cannot express
|
|
446
|
-
# (fnmatch "a**b" never crosses "/" and "**/x" also matches bare "x").
|
|
447
|
-
def compile_ignore_regexes(ignore_patterns)
|
|
448
|
-
@compiled_ignore_regexes ||= {}
|
|
449
|
-
@compiled_ignore_regexes[ignore_patterns] ||= ignore_patterns.map do |pat|
|
|
450
|
-
simple_glob_regex(pat) || pat
|
|
451
|
-
end
|
|
452
|
-
end
|
|
453
|
-
|
|
454
|
-
def simple_glob_regex(pat)
|
|
455
|
-
return nil if pat.empty? || pat.start_with?("/") || pat.include?("[")
|
|
456
|
-
return nil unless pat.split("/").all? { |seg| !seg.include?("*") || seg == "**" || !seg.include?("**") }
|
|
457
|
-
|
|
458
|
-
src = Regexp.escape(pat)
|
|
459
|
-
src = src.sub(%r{\A\\\*\\\*/}, "") # leading "**/": the (?:.*/)? wrapper matches zero dirs too
|
|
460
|
-
src = src.gsub("/\\*\\*/", "/(?:.*/)?") # mid "/**/": zero or more directories
|
|
461
|
-
src = src.gsub("\\*\\*", ".*") # trailing "/**" or lone "**"
|
|
462
|
-
src = src.gsub("\\*", "[^/]*").gsub("\\?", "[^/]")
|
|
463
|
-
Regexp.new("\\A(?:.*/)?#{src}\\z")
|
|
464
|
-
end
|
|
465
|
-
|
|
466
|
-
def pattern_match?(pattern_or_regex, path)
|
|
467
|
-
if pattern_or_regex.is_a?(Regexp)
|
|
468
|
-
pattern_or_regex.match?(path)
|
|
469
|
-
else
|
|
470
|
-
File.fnmatch?(pattern_or_regex, path, File::FNM_PATHNAME | File::FNM_DOTMATCH) ||
|
|
471
|
-
File.fnmatch?("**/#{pattern_or_regex}", path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
|
|
472
|
-
end
|
|
473
|
-
end
|
|
474
|
-
|
|
475
|
-
def included?(path, patterns)
|
|
476
|
-
return true if patterns.nil? || patterns.empty?
|
|
477
|
-
|
|
478
|
-
patterns.any? { |pattern| pattern_match?(pattern, path) }
|
|
479
|
-
end
|
|
480
365
|
end
|
|
481
366
|
end
|
|
482
367
|
end
|
data/lib/rubik/llm/client.rb
CHANGED
|
@@ -5,6 +5,8 @@ require "json"
|
|
|
5
5
|
require "time"
|
|
6
6
|
require "uri"
|
|
7
7
|
require_relative "../platform"
|
|
8
|
+
require_relative "sse_parser"
|
|
9
|
+
require_relative "connection_pool"
|
|
8
10
|
|
|
9
11
|
module Rubik
|
|
10
12
|
module LLM
|
|
@@ -42,120 +44,6 @@ module Rubik
|
|
|
42
44
|
end
|
|
43
45
|
end
|
|
44
46
|
|
|
45
|
-
# Buffers raw bytes from a stream and emits complete events.
|
|
46
|
-
# Default delimiter is the SSE blank line (\n\n); pass
|
|
47
|
-
# `terminator: "\n"` to parse NDJSON streams (one JSON object per line).
|
|
48
|
-
# One event is a Hash like:
|
|
49
|
-
# { type: "message", data: "json-string" }
|
|
50
|
-
# In SSE mode, multi-line `data:` fields are joined with \n. In NDJSON
|
|
51
|
-
# mode (`terminator: "\n"`), the `type` field is always "message" and
|
|
52
|
-
# the entire line is the `data`. Incomplete events are held in the
|
|
53
|
-
# internal buffer until the next feed.
|
|
54
|
-
class SSEParser
|
|
55
|
-
SSE_EVENT_SEP = /\r\n\r\n|\n\n|\r\n\n|\n\r\n|\r\r/
|
|
56
|
-
SSE_TERMINATOR = "\n\n"
|
|
57
|
-
NDJSON_TERMINATOR = "\n"
|
|
58
|
-
|
|
59
|
-
def initialize(terminator: SSE_TERMINATOR)
|
|
60
|
-
@terminator = terminator
|
|
61
|
-
@sse_mode = @terminator == SSE_TERMINATOR || @terminator == SSE_EVENT_SEP
|
|
62
|
-
@buffer = String.new(encoding: Encoding::UTF_8)
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
# Feed raw bytes; returns an array of complete events parsed from this
|
|
66
|
-
# chunk. Partial events stay in the buffer.
|
|
67
|
-
def feed(chunk_bytes)
|
|
68
|
-
incoming = chunk_bytes.to_s.dup.force_encoding(Encoding::UTF_8)
|
|
69
|
-
incoming = incoming.scrub unless incoming.valid_encoding?
|
|
70
|
-
@buffer << incoming
|
|
71
|
-
events = []
|
|
72
|
-
|
|
73
|
-
if @sse_mode
|
|
74
|
-
while (m = @buffer.match(SSE_EVENT_SEP))
|
|
75
|
-
# Chunk boundary \r buffering: if match ends on \r at the very end of @buffer,
|
|
76
|
-
# wait for next feed to see if \n follows (e.g. \r\r could become \r\r\n).
|
|
77
|
-
break if m.end(0) == @buffer.bytesize && @buffer.end_with?("\r")
|
|
78
|
-
|
|
79
|
-
event_block = @buffer[0...m.begin(0)]
|
|
80
|
-
@buffer = @buffer[m.end(0)..] || String.new(encoding: Encoding::UTF_8)
|
|
81
|
-
parsed = parse_event(event_block)
|
|
82
|
-
events << parsed if parsed
|
|
83
|
-
end
|
|
84
|
-
else
|
|
85
|
-
while (idx = @buffer.index(@terminator))
|
|
86
|
-
event_block = @buffer[0...idx]
|
|
87
|
-
@buffer = @buffer[(idx + @terminator.length)..] || String.new(encoding: Encoding::UTF_8)
|
|
88
|
-
parsed = parse_event(event_block)
|
|
89
|
-
events << parsed if parsed
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
events
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
# Flush any trailing event (no trailing terminator) and reset.
|
|
97
|
-
def flush
|
|
98
|
-
return [] if @buffer.empty?
|
|
99
|
-
|
|
100
|
-
events = []
|
|
101
|
-
if @sse_mode
|
|
102
|
-
while (m = @buffer.match(SSE_EVENT_SEP))
|
|
103
|
-
event_block = @buffer[0...m.begin(0)]
|
|
104
|
-
@buffer = @buffer[m.end(0)..] || String.new(encoding: Encoding::UTF_8)
|
|
105
|
-
parsed = parse_event(event_block)
|
|
106
|
-
events << parsed if parsed
|
|
107
|
-
end
|
|
108
|
-
else
|
|
109
|
-
while (idx = @buffer.index(@terminator))
|
|
110
|
-
event_block = @buffer[0...idx]
|
|
111
|
-
@buffer = @buffer[(idx + @terminator.length)..] || String.new(encoding: Encoding::UTF_8)
|
|
112
|
-
parsed = parse_event(event_block)
|
|
113
|
-
events << parsed if parsed
|
|
114
|
-
end
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
unless @buffer.empty?
|
|
118
|
-
remaining = @buffer.dup
|
|
119
|
-
@buffer = String.new(encoding: Encoding::UTF_8)
|
|
120
|
-
parsed = parse_event(remaining)
|
|
121
|
-
events << parsed if parsed
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
events
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
private
|
|
128
|
-
|
|
129
|
-
def parse_event(block)
|
|
130
|
-
return nil if block.nil?
|
|
131
|
-
return { type: "message", data: block.to_s.chomp } unless @sse_mode
|
|
132
|
-
|
|
133
|
-
lines = block.to_s.split(/\r\n|\r|\n/)
|
|
134
|
-
return nil if lines.empty?
|
|
135
|
-
|
|
136
|
-
event_type = "message"
|
|
137
|
-
data_lines = []
|
|
138
|
-
has_field = false
|
|
139
|
-
|
|
140
|
-
lines.each do |line|
|
|
141
|
-
next if line.empty?
|
|
142
|
-
next if line.start_with?(":")
|
|
143
|
-
|
|
144
|
-
if (m = line.match(/\Aevent:\s*(.+)\z/))
|
|
145
|
-
event_type = m[1]
|
|
146
|
-
has_field = true
|
|
147
|
-
elsif (m = line.match(/\Adata:\s?(.*)\z/))
|
|
148
|
-
data_lines << m[1]
|
|
149
|
-
has_field = true
|
|
150
|
-
end
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
return nil unless has_field
|
|
154
|
-
|
|
155
|
-
{ type: event_type, data: data_lines.join("\n") }
|
|
156
|
-
end
|
|
157
|
-
end
|
|
158
|
-
|
|
159
47
|
class HTTPClient
|
|
160
48
|
# Errors that are safe to retry with an idempotent bounded backoff:
|
|
161
49
|
# connect-phase failures and transport resets. Net::ReadTimeout is
|
|
@@ -372,88 +260,12 @@ module Rubik
|
|
|
372
260
|
end
|
|
373
261
|
end
|
|
374
262
|
|
|
375
|
-
# Builds the real Net::HTTP transport. open_timeout is bounded LOW and
|
|
376
|
-
# independent of the (long) read timeout: Net::HTTP arms a Timeout thread
|
|
377
|
-
# around TCP connect on Ruby 3.3; a short arming window shrinks the chance
|
|
378
|
-
# that a stale Net::OpenTimeout is injected into unrelated main-thread
|
|
379
|
-
# code after an Interrupt escapes the connect region.
|
|
380
263
|
def self.default_http(uri, timeout:, open_timeout:)
|
|
381
|
-
|
|
382
|
-
http.use_ssl = (uri.scheme == "https")
|
|
383
|
-
http.open_timeout = open_timeout
|
|
384
|
-
http.read_timeout = timeout
|
|
385
|
-
http
|
|
264
|
+
ConnectionPool.default_http(uri, timeout: timeout, open_timeout: open_timeout)
|
|
386
265
|
end
|
|
387
266
|
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
# Opening a brand-new TCP(+TLS) connection per LLM round-trip costs
|
|
391
|
-
# ~100-300 ms each, and an agent turn issues many round-trips. Sessions
|
|
392
|
-
# here persist per (host, port, scheme); a mutex per key serializes
|
|
393
|
-
# concurrent use because a single Net::HTTP session is not thread-safe
|
|
394
|
-
# (same-endpoint calls wait for the current response to finish).
|
|
395
|
-
# Ruby's non-reentrant Mutex would deadlock on same-thread re-entry
|
|
396
|
-
# (nested provider call, Enumerator#next fibers) — those transparently
|
|
397
|
-
# fall back to a private single-use connection. Transport failures and
|
|
398
|
-
# interrupts drop the session so the next attempt dials fresh.
|
|
399
|
-
CONNECTION_IDLE_TIMEOUT = 300
|
|
400
|
-
|
|
401
|
-
# After any of these the socket's integrity is unknown (or the stream
|
|
402
|
-
# was cut mid-read by Ctrl+C): never hand the session back to the pool.
|
|
403
|
-
INTEGRITY_LOST_ERRORS = [
|
|
404
|
-
*RETRYABLE_ERRORS, Net::ReadTimeout, IOError,
|
|
405
|
-
OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Interrupt
|
|
406
|
-
].freeze
|
|
407
|
-
|
|
408
|
-
@pool_mutex = Mutex.new
|
|
409
|
-
@pool = {}
|
|
410
|
-
@pool_locks = {}
|
|
411
|
-
@pool_locks_guard = Mutex.new
|
|
412
|
-
|
|
413
|
-
def self.with_pooled_connection(uri, timeout:, open_timeout:)
|
|
414
|
-
key = [uri.host, uri.port, uri.scheme == "https"]
|
|
415
|
-
lock = @pool_locks_guard.synchronize { @pool_locks[key] ||= Mutex.new }
|
|
416
|
-
|
|
417
|
-
begin
|
|
418
|
-
lock.synchronize do
|
|
419
|
-
http = @pool_mutex.synchronize { @pool[key] }
|
|
420
|
-
if http&.started?
|
|
421
|
-
http.read_timeout = timeout
|
|
422
|
-
http.open_timeout = open_timeout
|
|
423
|
-
else
|
|
424
|
-
http = default_http(uri, timeout: timeout, open_timeout: open_timeout)
|
|
425
|
-
http.keep_alive_timeout = CONNECTION_IDLE_TIMEOUT
|
|
426
|
-
http.start
|
|
427
|
-
@pool_mutex.synchronize { @pool[key] = http }
|
|
428
|
-
end
|
|
429
|
-
|
|
430
|
-
begin
|
|
431
|
-
yield http
|
|
432
|
-
rescue *INTEGRITY_LOST_ERRORS
|
|
433
|
-
drop_pooled(key, http)
|
|
434
|
-
raise
|
|
435
|
-
end
|
|
436
|
-
end
|
|
437
|
-
rescue ThreadError
|
|
438
|
-
# Re-entrant call on the same thread (nested LLM call from inside a
|
|
439
|
-
# streaming callback, or a second Enumerator#next fiber) — the
|
|
440
|
-
# per-key mutex cannot be taken. Serve this call on a private,
|
|
441
|
-
# single-use connection instead of deadlocking.
|
|
442
|
-
http = default_http(uri, timeout: timeout, open_timeout: open_timeout)
|
|
443
|
-
http.start
|
|
444
|
-
begin
|
|
445
|
-
yield http
|
|
446
|
-
ensure
|
|
447
|
-
http.finish if http.started?
|
|
448
|
-
end
|
|
449
|
-
end
|
|
450
|
-
end
|
|
451
|
-
|
|
452
|
-
def self.drop_pooled(key, http)
|
|
453
|
-
@pool_mutex.synchronize { @pool.delete(key) if @pool[key].equal?(http) }
|
|
454
|
-
http.finish if http&.started?
|
|
455
|
-
rescue StandardError
|
|
456
|
-
nil
|
|
267
|
+
def self.with_pooled_connection(uri, timeout:, open_timeout:, &)
|
|
268
|
+
ConnectionPool.with_connection(uri, timeout: timeout, open_timeout: open_timeout, &)
|
|
457
269
|
end
|
|
458
270
|
|
|
459
271
|
def self.build_post_request(uri, headers, payload, accept: nil)
|
|
@@ -490,13 +302,7 @@ module Rubik
|
|
|
490
302
|
def self.post_stream_json(url_str, headers: {}, payload: {}, timeout: 120, open_timeout: 15,
|
|
491
303
|
idle_timeout: nil,
|
|
492
304
|
terminator: SSEParser::SSE_TERMINATOR, http_factory: nil, &)
|
|
493
|
-
|
|
494
|
-
uri = URI.parse(url_str.to_s)
|
|
495
|
-
rescue URI::InvalidURIError => e
|
|
496
|
-
raise ClientError, "Invalid endpoint URL '#{url_str}': #{e.message}"
|
|
497
|
-
end
|
|
498
|
-
|
|
499
|
-
raise ClientError, "Invalid endpoint URL '#{url_str}'. URL must start with http:// or https://" unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
|
|
305
|
+
uri = parse_and_validate_uri(url_str)
|
|
500
306
|
|
|
501
307
|
effective_timeout = idle_timeout || timeout
|
|
502
308
|
state = { surfaced: false }
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
|
|
5
|
+
module Rubik
|
|
6
|
+
module LLM
|
|
7
|
+
# ---- Persistent connection pool ---------------------------------------
|
|
8
|
+
#
|
|
9
|
+
# Opening a brand-new TCP(+TLS) connection per LLM round-trip costs
|
|
10
|
+
# ~100-300 ms each, and an agent turn issues many round-trips. Sessions
|
|
11
|
+
# here persist per (host, port, scheme); a mutex per key serializes
|
|
12
|
+
# concurrent use because a single Net::HTTP session is not thread-safe
|
|
13
|
+
# (same-endpoint calls wait for the current response to finish).
|
|
14
|
+
# Ruby's non-reentrant Mutex would deadlock on same-thread re-entry
|
|
15
|
+
# (nested provider call, Enumerator#next fibers) — those transparently
|
|
16
|
+
# fall back to a private single-use connection. Transport failures and
|
|
17
|
+
# interrupts drop the session so the next attempt dials fresh.
|
|
18
|
+
class ConnectionPool
|
|
19
|
+
CONNECTION_IDLE_TIMEOUT = 300
|
|
20
|
+
|
|
21
|
+
@pool_mutex = Mutex.new
|
|
22
|
+
@pool = {}
|
|
23
|
+
@pool_locks = {}
|
|
24
|
+
@pool_locks_guard = Mutex.new
|
|
25
|
+
@integrity_lost_errors = nil
|
|
26
|
+
|
|
27
|
+
class << self
|
|
28
|
+
# After any of these the socket's integrity is unknown (or the stream
|
|
29
|
+
# was cut mid-read by Ctrl+C): never hand the session back to the pool.
|
|
30
|
+
# Resolved lazily (not a load-time constant) because HTTPClient::
|
|
31
|
+
# RETRYABLE_ERRORS lives in client.rb, which requires this file before
|
|
32
|
+
# defining the HTTPClient class.
|
|
33
|
+
def integrity_lost_errors
|
|
34
|
+
@integrity_lost_errors ||= [
|
|
35
|
+
*HTTPClient::RETRYABLE_ERRORS, Net::ReadTimeout, IOError,
|
|
36
|
+
OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Interrupt
|
|
37
|
+
].freeze
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Yields an HTTP session for (host, port, scheme), reusing the pooled
|
|
41
|
+
# keep-alive session when one exists. Same-thread re-entry falls back
|
|
42
|
+
# to a private single-use connection (see the class comment).
|
|
43
|
+
def with_connection(uri, timeout:, open_timeout:)
|
|
44
|
+
key = [uri.host, uri.port, uri.scheme == "https"]
|
|
45
|
+
lock = @pool_locks_guard.synchronize { @pool_locks[key] ||= Mutex.new }
|
|
46
|
+
|
|
47
|
+
begin
|
|
48
|
+
lock.synchronize do
|
|
49
|
+
http = @pool_mutex.synchronize { @pool[key] }
|
|
50
|
+
if http&.started?
|
|
51
|
+
http.read_timeout = timeout
|
|
52
|
+
http.open_timeout = open_timeout
|
|
53
|
+
else
|
|
54
|
+
http = default_http(uri, timeout: timeout, open_timeout: open_timeout)
|
|
55
|
+
http.keep_alive_timeout = CONNECTION_IDLE_TIMEOUT
|
|
56
|
+
http.start
|
|
57
|
+
@pool_mutex.synchronize { @pool[key] = http }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
begin
|
|
61
|
+
yield http
|
|
62
|
+
rescue *integrity_lost_errors
|
|
63
|
+
drop(key, http)
|
|
64
|
+
raise
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
rescue ThreadError
|
|
68
|
+
# Re-entrant call on the same thread (nested LLM call from inside a
|
|
69
|
+
# streaming callback, or a second Enumerator#next fiber) — the
|
|
70
|
+
# per-key mutex cannot be taken. Serve this call on a private,
|
|
71
|
+
# single-use connection instead of deadlocking.
|
|
72
|
+
http = default_http(uri, timeout: timeout, open_timeout: open_timeout)
|
|
73
|
+
http.start
|
|
74
|
+
begin
|
|
75
|
+
yield http
|
|
76
|
+
ensure
|
|
77
|
+
http.finish if http.started?
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def drop(key, http)
|
|
83
|
+
@pool_mutex.synchronize { @pool.delete(key) if @pool[key].equal?(http) }
|
|
84
|
+
http.finish if http&.started?
|
|
85
|
+
rescue StandardError
|
|
86
|
+
nil
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Builds the real Net::HTTP transport. open_timeout is bounded LOW and
|
|
90
|
+
# independent of the (long) read timeout: Net::HTTP arms a Timeout thread
|
|
91
|
+
# around TCP connect on Ruby 3.3; a short arming window shrinks the chance
|
|
92
|
+
# that a stale Net::OpenTimeout is injected into unrelated main-thread
|
|
93
|
+
# code after an Interrupt escapes the connect region.
|
|
94
|
+
def default_http(uri, timeout:, open_timeout:)
|
|
95
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
96
|
+
http.use_ssl = (uri.scheme == "https")
|
|
97
|
+
http.open_timeout = open_timeout
|
|
98
|
+
http.read_timeout = timeout
|
|
99
|
+
http
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
data/lib/rubik/llm/registry.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../log"
|
|
3
4
|
require_relative "openai_compatible"
|
|
4
5
|
require_relative "../security/secret_store"
|
|
5
6
|
require_relative "../storage/layout"
|
|
@@ -266,8 +267,8 @@ module Rubik
|
|
|
266
267
|
# resolve it lazily per build so keyring updates apply on refresh.
|
|
267
268
|
api_key = Rubik::Security::SecretStore.resolve(p_spec["api_key"])
|
|
268
269
|
if api_key.nil? && Rubik::Security::SecretStore.reference?(p_spec["api_key"].to_s)
|
|
269
|
-
warn "[WARN] stored secret '#{p_spec['api_key'].delete_prefix(Rubik::Security::SecretStore::REFERENCE_PREFIX)}' " \
|
|
270
|
-
|
|
270
|
+
Rubik::Log.warn(:llm, "[WARN] stored secret '#{p_spec['api_key'].delete_prefix(Rubik::Security::SecretStore::REFERENCE_PREFIX)}' " \
|
|
271
|
+
"could not be retrieved via #{Rubik::Security::SecretStore.backend_name}")
|
|
271
272
|
end
|
|
272
273
|
|
|
273
274
|
case type
|