rubik 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ATTRIBUTION.md +1 -1
- data/CHANGELOG.md +125 -1
- data/lib/rubik/ast/cache_store.rb +33 -0
- data/lib/rubik/ast/graph.rb +2 -1
- data/lib/rubik/benchmark/runner.rb +14 -0
- data/lib/rubik/benchmark/vakanuvis_adoption.rb +439 -0
- data/lib/rubik/config/defaults.rb +7 -1
- data/lib/rubik/log.rb +4 -3
- data/lib/rubik/orchestrator/budget_manager.rb +2 -0
- data/lib/rubik/orchestrator/loop.rb +1 -0
- data/lib/rubik/orchestrator/shell_deps.rb +4 -0
- data/lib/rubik/orchestrator/step_executor.rb +3 -3
- data/lib/rubik/orchestrator/subagent/tool_dispatch.rb +31 -11
- data/lib/rubik/orchestrator/turn_executor/user_channel.rb +2 -0
- data/lib/rubik/prompt/doctrine.rb +10 -15
- data/lib/rubik/prompt/emergency_compactor.rb +7 -3
- data/lib/rubik/prompt/tool_teachings.rb +29 -35
- data/lib/rubik/repl/commands/agent_context.rb +1 -0
- data/lib/rubik/repl/commands/agent_execution.rb +7 -0
- data/lib/rubik/repl/commands/system.rb +2 -0
- data/lib/rubik/repl/input_execution.rb +5 -1
- data/lib/rubik/repl/turn_recorder.rb +3 -0
- data/lib/rubik/security/redactor.rb +13 -9
- data/lib/rubik/storage/vakanuvis/annal_writer.rb +51 -4
- data/lib/rubik/storage/vakanuvis/distill_gate.rb +14 -2
- data/lib/rubik/storage/vakanuvis/gate.rb +23 -15
- data/lib/rubik/storage/vakanuvis/harness.rb +149 -121
- data/lib/rubik/storage/vakanuvis/job.rb +25 -0
- data/lib/rubik/storage/vakanuvis/orientation.rb +277 -0
- data/lib/rubik/storage/vakanuvis/service.rb +5 -1
- data/lib/rubik/storage/vakanuvis.rb +20 -1
- data/lib/rubik/tools/builtins/vakanuvis.rb +13 -15
- data/lib/rubik/tools/truncator.rb +53 -0
- data/lib/rubik/version.rb +1 -1
- metadata +4 -1
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require_relative "../storage/vakanuvis"
|
|
6
|
+
require_relative "../tools/builtins/storage"
|
|
7
|
+
require_relative "../tools/builtins/vakanuvis"
|
|
8
|
+
|
|
9
|
+
module Rubik
|
|
10
|
+
module Benchmark
|
|
11
|
+
# Vakanüvis araç-benimseme senaryosu (issue #338): küçük modellerin
|
|
12
|
+
# vakanüvis'i ne sıklıkla, nasıl kullandığını ölçen taban çizgisi.
|
|
13
|
+
# "Bu düzeltme iğneyi hareket ettirdi mi?" sorusunun cevabı için
|
|
14
|
+
# prompt-doktrin / yönelim-enjeksiyonu / otomatik-damıtma düzeltmeleri
|
|
15
|
+
# ÖNCE bu ölçüm ister.
|
|
16
|
+
#
|
|
17
|
+
# Düzlem 1 (deterministik, CI-güvenli): tohum depo + ReplayDriver —
|
|
18
|
+
# üç davranış profili GERÇEK araç düzlemini çalıştırır (script'lenmiş
|
|
19
|
+
# araç çağrıları, sahte sonuçlar değil); metrik borusu ve recall
|
|
20
|
+
# çıktı biçimi regresyona girerse burada patlar.
|
|
21
|
+
# Düzlem 2 (canlı): LiveDriver provider.complete döngüsü — MODELS
|
|
22
|
+
# env'i ile model başına gerçek benimseme ölçümü.
|
|
23
|
+
#
|
|
24
|
+
# Sessizlik: canlı modeller maliyetli ve deterministik değildir — bu
|
|
25
|
+
# yüzden default Düzlem 1'dir; rake bench ana süitine asla canlı çağrı
|
|
26
|
+
# sızdırmaz (bench:vakanuvis_adoption MODELS=... açık istek ister).
|
|
27
|
+
module VakanuvisAdoption
|
|
28
|
+
# Tohum kartlar: TR/EN karışık, Keywords: satırlı/satırsız — issue
|
|
29
|
+
# senaryosunun ta kendisi. Görev-1 kartının ayırt edici değerleri
|
|
30
|
+
# (8 saniye adımı + 5 deneme) yanıt-kullanımını dürüstçe sayılabilir
|
|
31
|
+
# kılar: model hatırladığı karttan alıntıladıysa ikisi de görünür.
|
|
32
|
+
SEED_CARDS = [
|
|
33
|
+
{
|
|
34
|
+
topic: "Ağ geçidi yeniden deneme politikası",
|
|
35
|
+
body: "Ağ geçidi 429/503 yanıtında üstel geri çekilme uygular: 2s, 4s, 8s " \
|
|
36
|
+
"(en fazla 5 deneme). Devre kesici 30 ardışık hata sonra açılır.\n" \
|
|
37
|
+
"Keywords: gateway retry backoff üstel geri çekilme 429 503 circuit breaker exponent"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
topic: "Sandbox filesystem allowlist",
|
|
41
|
+
body: "The container sandbox mounts the worktree read-write and $HOME read-only; " \
|
|
42
|
+
"the allowlist file is sandbox.allow at the repo root.\n" \
|
|
43
|
+
"Keywords: sandbox allowlist container readonly mount worktree izin listesi"
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
topic: "Vakanüvis damıtma kapısı",
|
|
47
|
+
body: "Damıtma kapısı aynı hedefli işi yutar (dedupe); maksimum kuyruk 8 iş, " \
|
|
48
|
+
"hedef başına tek iz."
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
topic: "Fiber scheduler budget",
|
|
52
|
+
body: "The fiber scheduler caps concurrent subagent fibers at 12; overflow " \
|
|
53
|
+
"queues FIFO and never drops."
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
topic: "Goal sıra türetimi",
|
|
57
|
+
body: "Goal sırası mağazadaki en büyük annal.goal.N'den türetilir; duvar saati " \
|
|
58
|
+
"ASLA kullanılmaz — kronik MARKER-ZIKKIM-7 determinizmi budur."
|
|
59
|
+
}
|
|
60
|
+
].freeze
|
|
61
|
+
|
|
62
|
+
# Görev 1 yanıt izleri: karttan alıntı → ikisi de eşleşir; uydurma
|
|
63
|
+
# yanıt → en az biri (genelde ikisi) kaçar.
|
|
64
|
+
ANSWER_MARKER_BACKOFF = /8\s*(s\b|sn|sec|second|saniye)/i
|
|
65
|
+
ANSWER_MARKER_TRIES = /(5|beş|five)\s*(deneme|attempt|retr|try)/i
|
|
66
|
+
|
|
67
|
+
TASKS = [
|
|
68
|
+
{
|
|
69
|
+
id: "recall-tr-en",
|
|
70
|
+
prompt: "Bu sorunun cevabı rubik://storage kronikçisindeki bir hafıza kartında duruyor: " \
|
|
71
|
+
"ağ geçidi (gateway) 429/503 yanıtında yeniden deneme / geri çekilme politikası nedir? " \
|
|
72
|
+
"Kısa cevabını ver."
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
id: "durable-conclusion",
|
|
76
|
+
prompt: "We just concluded that the chronicler's digest gate deduplicates jobs for the " \
|
|
77
|
+
"same target. Persist this durable conclusion so future sessions can recall it, " \
|
|
78
|
+
"then confirm in one line."
|
|
79
|
+
}
|
|
80
|
+
].freeze
|
|
81
|
+
|
|
82
|
+
MAX_TURNS = 8
|
|
83
|
+
REPLAY_PROFILES = ["adopter", "wrong-path", "struggler"].freeze
|
|
84
|
+
|
|
85
|
+
module_function
|
|
86
|
+
|
|
87
|
+
# ---- mağaza + araç masası ---------------------------------------------
|
|
88
|
+
|
|
89
|
+
# Kronikçi yapılandırması: damıtıcı KAPALI (bench'te asla LLM doğumu —
|
|
90
|
+
# ölçtüğümüz şey maestro davranışı, damıtıcı maliyeti değil).
|
|
91
|
+
class QuietConfig
|
|
92
|
+
def fetch(key, default = nil)
|
|
93
|
+
return false if key == "subagents.vakanuvis.distiller"
|
|
94
|
+
|
|
95
|
+
default
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Deterministik tohum: aynı kart dizisi ⇒ aynı mağaza ⇒ aynı recall
|
|
100
|
+
# davranışı. Her model/profil ÖZGÜN kök dizinle başlar (record
|
|
101
|
+
# yan etkileri bir sonraki modelin ölçümünü kirletmesin).
|
|
102
|
+
def seed_store(root)
|
|
103
|
+
store = Rubik::Storage::Store.new(root: root, project: "vakanuvis-adoption-bench")
|
|
104
|
+
harness = Rubik::Storage::Vakanuvis::Harness.new(store: store, config: QuietConfig.new, session_slug: "bench-seed")
|
|
105
|
+
SEED_CARDS.each { |card| harness.record(topic: card[:topic], body: card[:body]) }
|
|
106
|
+
harness.shutdown
|
|
107
|
+
store
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# İki gerçek yerleşik araç: vakanüvis (doğru yol) + storage (yanlış
|
|
111
|
+
# yolun mümkün olması için masada DURMALI — yoksa yanlış-yol oranı
|
|
112
|
+
# ölçülemez, sahte sıfır doğardı).
|
|
113
|
+
class ToolBench
|
|
114
|
+
def initialize(store, harness)
|
|
115
|
+
@tools = {
|
|
116
|
+
"vakanuvis" => Rubik::Tools::VakanuvisTool.new(harness),
|
|
117
|
+
"storage" => Rubik::Tools::StorageTool.new(store)
|
|
118
|
+
}
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def execute(name, args)
|
|
122
|
+
tool = @tools[name.to_s]
|
|
123
|
+
return "Unknown tool '#{name}' (bench toolbelt: vakanuvis, storage)." unless tool
|
|
124
|
+
|
|
125
|
+
tool.execute(args || {})
|
|
126
|
+
rescue StandardError => e
|
|
127
|
+
"tool error: #{e.class}: #{e.message}"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def schemas = @tools.values.map(&:to_h)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# ---- sürücüler --------------------------------------------------------
|
|
134
|
+
# Sözleşme: run_task(task, tools) -> { turns:, answer:, calls:,
|
|
135
|
+
# first_recall_turn:, hit_cap: }. calls = [[name, args, result_text]]
|
|
136
|
+
# (sıralı). first_recall_turn = ilk recall'in 1-tabanlı turn'ü.
|
|
137
|
+
|
|
138
|
+
# Script'lenmiş davranış profilleri — scriptler GERÇEK araçlarla
|
|
139
|
+
# koşar (sonuç sahtelenmez); metrik borusunun deterministik kanıtı.
|
|
140
|
+
class ReplayDriver
|
|
141
|
+
attr_reader :profile
|
|
142
|
+
|
|
143
|
+
def initialize(profile)
|
|
144
|
+
@profile = profile.to_s
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def label = "scripted/#{@profile}"
|
|
148
|
+
|
|
149
|
+
def live? = false
|
|
150
|
+
|
|
151
|
+
def run_task(task, tools)
|
|
152
|
+
calls = []
|
|
153
|
+
answer = ""
|
|
154
|
+
turns = 0
|
|
155
|
+
first_recall_turn = nil
|
|
156
|
+
SCRIPTS.fetch(@profile).fetch(task[:id]).each do |step|
|
|
157
|
+
turns += 1
|
|
158
|
+
steps = step[:call] ? [step[:call]] : (step[:call_each] || [])
|
|
159
|
+
steps.each do |name, args|
|
|
160
|
+
first_recall_turn ||= turns if recall_step?(name, args)
|
|
161
|
+
calls << [name, args, tools.execute(name, args)]
|
|
162
|
+
end
|
|
163
|
+
answer = step[:answer] if step.key?(:answer)
|
|
164
|
+
end
|
|
165
|
+
{ turns: turns, answer: answer, calls: calls, first_recall_turn: first_recall_turn, hit_cap: false }
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
private
|
|
169
|
+
|
|
170
|
+
def recall_step?(name, args)
|
|
171
|
+
name == "vakanuvis" && (args["action"].nil? || args["action"] == "recall")
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Canlı sürücü: provider.complete döngüsü. Prompt görevi söyletir,
|
|
176
|
+
# araç seçimini ASLA emretmez — benimseme ölçümünün koşulu budur.
|
|
177
|
+
class LiveDriver
|
|
178
|
+
def initialize(provider, max_turns: MAX_TURNS)
|
|
179
|
+
@provider = provider
|
|
180
|
+
@max_turns = max_turns
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def label = "live/#{@provider.name}/#{@provider.model}"
|
|
184
|
+
|
|
185
|
+
def live? = true
|
|
186
|
+
|
|
187
|
+
def run_task(task, tools)
|
|
188
|
+
history = [{ "role" => "user", "content" => task[:prompt] }]
|
|
189
|
+
calls = []
|
|
190
|
+
turns = 0
|
|
191
|
+
first_recall_turn = nil
|
|
192
|
+
answer = ""
|
|
193
|
+
|
|
194
|
+
@max_turns.times do
|
|
195
|
+
turns += 1
|
|
196
|
+
res = @provider.complete(history, { tools: tools.schemas })
|
|
197
|
+
# Sağlayıcı sözleşmesi (inceleme turu 2): Response dışı bir
|
|
198
|
+
# dönüş, respond_to? zincirlerine girip sahte "araç
|
|
199
|
+
# benimsemedi / tur sınırına vurdu" veri noktası üretmesin —
|
|
200
|
+
# dürüstçe patla.
|
|
201
|
+
raise "provider returned #{res.inspect}, expected Rubik::LLM::Response" unless res.is_a?(Rubik::LLM::Response)
|
|
202
|
+
|
|
203
|
+
tool_calls = res.respond_to?(:tool_calls?) && res.tool_calls? ? res.tool_calls : []
|
|
204
|
+
if tool_calls.empty?
|
|
205
|
+
answer = res.respond_to?(:text) ? res.text.to_s : res.to_s
|
|
206
|
+
break
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
history << {
|
|
210
|
+
role: "assistant",
|
|
211
|
+
content: res.respond_to?(:text) ? res.text.to_s : "",
|
|
212
|
+
tool_calls: tool_calls.map { |tc| { "id" => tc["id"], "tool" => tc["tool"], "args" => tc["args"] } }
|
|
213
|
+
}
|
|
214
|
+
tool_calls.each do |tc|
|
|
215
|
+
args = tc["args"].is_a?(Hash) ? tc["args"] : {}
|
|
216
|
+
action = args["action"].to_s
|
|
217
|
+
first_recall_turn ||= turns if tc["tool"] == "vakanuvis" && (action.empty? || action == "recall")
|
|
218
|
+
result = tools.execute(tc["tool"], args)
|
|
219
|
+
calls << [tc["tool"], args, result]
|
|
220
|
+
history << { role: "tool", tool_call_id: tc["id"], tool_name: tc["tool"], content: result.to_s }
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
{ turns: turns, answer: answer, calls: calls, first_recall_turn: first_recall_turn, hit_cap: answer.empty? }
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# ---- metrikler --------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
Metrics = Struct.new(
|
|
231
|
+
:label, :recall_called, :first_recall_turn, :recall_calls, :avg_query_terms,
|
|
232
|
+
:keywords_line_used, :recall_empty_rate, :record_called, :wrong_path_puts,
|
|
233
|
+
:answer_used_recall, :hit_turn_cap, keyword_init: true
|
|
234
|
+
) do
|
|
235
|
+
def summary
|
|
236
|
+
"recall=#{recall_called ? "turn #{first_recall_turn}" : 'never'} calls=#{recall_calls} " \
|
|
237
|
+
"terms=#{avg_query_terms || '-'} kw=#{keywords_line_used ? 'yes' : 'no'} empty=#{recall_empty_rate || '-'}% " \
|
|
238
|
+
"record=#{record_called ? 'vakanuvis' : 'none'} wrong_put=#{wrong_path_puts} " \
|
|
239
|
+
"answer_used=#{answer_used_recall ? 'yes' : 'no'}"
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def metrics_for(label, task_runs)
|
|
244
|
+
recall_events = []
|
|
245
|
+
record_calls = 0
|
|
246
|
+
wrong_puts = 0
|
|
247
|
+
first_recall_turn = nil
|
|
248
|
+
answer_used = false
|
|
249
|
+
hit_cap = false
|
|
250
|
+
|
|
251
|
+
task_runs.each do |run|
|
|
252
|
+
run[:calls].each do |name, args, result|
|
|
253
|
+
if name == "vakanuvis"
|
|
254
|
+
action = args["action"].to_s
|
|
255
|
+
if action.empty? || action == "recall"
|
|
256
|
+
recall_events << [args["query"].to_s, result.to_s]
|
|
257
|
+
first_recall_turn ||= run[:first_recall_turn]
|
|
258
|
+
elsif action == "record"
|
|
259
|
+
record_calls += 1
|
|
260
|
+
end
|
|
261
|
+
elsif name == "storage" && args["action"].to_s == "put" && ["memory", "wiki"].include?(args["type"].to_s)
|
|
262
|
+
wrong_puts += 1 # kronikçi masada DURUYORKEN doğrudan put = yanlış yol
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
answer_used ||= answer_uses_recall?(run[:answer])
|
|
266
|
+
hit_cap ||= run[:hit_cap] == true
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
terms = recall_events.map { |query, _res| query.strip.split(/\s+/).reject(&:empty?).size }
|
|
270
|
+
Metrics.new(
|
|
271
|
+
label: label,
|
|
272
|
+
recall_called: !recall_events.empty?,
|
|
273
|
+
first_recall_turn: first_recall_turn,
|
|
274
|
+
recall_calls: recall_events.size,
|
|
275
|
+
avg_query_terms: terms.empty? ? nil : terms.sum.fdiv(terms.size).round(1),
|
|
276
|
+
keywords_line_used: recall_events.any? { |query, _res| query.include?("Keywords:") },
|
|
277
|
+
recall_empty_rate: recall_events.empty? ? nil : empty_rate(recall_events),
|
|
278
|
+
record_called: record_calls.positive?,
|
|
279
|
+
wrong_path_puts: wrong_puts,
|
|
280
|
+
answer_used_recall: answer_used,
|
|
281
|
+
hit_turn_cap: hit_cap
|
|
282
|
+
)
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# "Recalled N ..." başlığı dışındaki her şey (anahtar-AND boş + yalnız
|
|
286
|
+
# fallback izleri) boş sayılır — issue'nin "fallback dahil" tanımı.
|
|
287
|
+
def empty_rate(recall_events)
|
|
288
|
+
empty_count = recall_events.count { |_query, res| !res.start_with?("Recalled ") }
|
|
289
|
+
(empty_count.fdiv(recall_events.size) * 100).round(0)
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def answer_uses_recall?(text)
|
|
293
|
+
text = text.to_s
|
|
294
|
+
text.match?(ANSWER_MARKER_BACKOFF) && text.match?(ANSWER_MARKER_TRIES)
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# ---- koşu -------------------------------------------------------------
|
|
298
|
+
|
|
299
|
+
# Her sürücü özgün tohumlu mağaza alır (izolasyon: record yan
|
|
300
|
+
# etkileri sızamaz). Sürücü hatası TEK modeli tablodan düşürür —
|
|
301
|
+
# diğer modellerin ölçümü yaşamaya devam eder; düşen satır sınıfa
|
|
302
|
+
# göre ayrışır: RateLimitError → SkipRow (kota/ağ ölümü — geçici),
|
|
303
|
+
# diğer her şey → ErrorRow (kimlik doğrulama, bozuk JSON, determin-
|
|
304
|
+
# istik düzlemde script hatası... bunlar "skipped" kıyafetini taşımaya
|
|
305
|
+
# değmez). Deterministik düzlemde çöküş loud: diğer bench metotları
|
|
306
|
+
# gibi rake bench'i kırmalı — çökme "skipped" diye kaydedilmez.
|
|
307
|
+
def run(drivers, io: $stdout)
|
|
308
|
+
drivers.map do |driver|
|
|
309
|
+
root = Dir.mktmpdir("rubik_vakanuvis_adoption")
|
|
310
|
+
begin
|
|
311
|
+
store = seed_store(root)
|
|
312
|
+
harness = Rubik::Storage::Vakanuvis::Harness.new(store: store, config: QuietConfig.new, session_slug: "bench-#{driver.label}")
|
|
313
|
+
tools = ToolBench.new(store, harness)
|
|
314
|
+
runs = TASKS.map { |task| driver.run_task(task, tools) }
|
|
315
|
+
tool_errors = runs.sum { |r| r[:calls].count { |_n, _a, res| TOOL_ERROR_MARKERS.any? { |m| res.to_s.start_with?(m) } } }
|
|
316
|
+
if tool_errors.positive?
|
|
317
|
+
first_bad = runs.flat_map { |r| r[:calls] }.find { |_n, _a, res| TOOL_ERROR_MARKERS.any? { |m| res.to_s.start_with?(m) } }
|
|
318
|
+
raise "#{driver.label}: #{tool_errors} tool error(s) — the bench toolbelt itself failed; " \
|
|
319
|
+
"these numbers would read as model behavior. first: #{first_bad[2].to_s[0, 160]}"
|
|
320
|
+
end
|
|
321
|
+
harness.shutdown
|
|
322
|
+
metrics_for(driver.label, runs).tap { |m| io&.puts(" [vakanuvis-adoption] #{m.label}: #{m.summary}") }
|
|
323
|
+
rescue Rubik::LLM::RateLimitError => e
|
|
324
|
+
location = (e.backtrace || []).first
|
|
325
|
+
io&.puts(" [vakanuvis-adoption] #{driver.label}: SKIPPED (rate limited) — #{e.message} @ #{location}")
|
|
326
|
+
SkipRow.new(driver.label, e)
|
|
327
|
+
rescue Rubik::LLM::ClientError, StandardError => e
|
|
328
|
+
# Deterministik düzlem: asla satıra dönüşmesin (rake bench
|
|
329
|
+
# loud patlar). Canlı düzlem: tek bozuk sürücü kalan modellerin
|
|
330
|
+
# ölçümünü öldürmesin — dürüst ERROR satırı düşer.
|
|
331
|
+
raise unless driver.live?
|
|
332
|
+
|
|
333
|
+
location = (e.backtrace || []).first
|
|
334
|
+
io&.puts(" [vakanuvis-adoption] #{driver.label}: ERROR — #{e.class}: #{e.message} @ #{location}")
|
|
335
|
+
ErrorRow.new(driver.label, e)
|
|
336
|
+
ensure
|
|
337
|
+
begin
|
|
338
|
+
FileUtils.remove_entry(root)
|
|
339
|
+
rescue StandardError => e
|
|
340
|
+
# Windows EBUSY/ENOTEMPTY (AV tarayıcı, indeksleyici) cleanup'ı
|
|
341
|
+
# patlatabilir — cleanup hatası tabloyu ve işlenen hatayı
|
|
342
|
+
# MaskeleMEsin; tüm benchmark'ı da abort etmesin.
|
|
343
|
+
warn " [vakanuvis-adoption] cleanup failed for #{root}: #{e.class}: #{e.message}"
|
|
344
|
+
end
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# Kota/ağ ölümü yaşayan modelin tablo satırı: yok saymak değil —
|
|
350
|
+
# "ölçülemedi"yi tabloda göster (sessiz-satır-kaybı yok).
|
|
351
|
+
SkipRow = Struct.new(:label, :error) do
|
|
352
|
+
def summary = "skipped: #{error.class}: #{error.message.to_s[0, 120]}"
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# Kota DEĞİL — kimlik doğrulama, bozuk sağlayıcı yanıtı, kabuk...
|
|
356
|
+
# "skipped" yalan olur; tabloda error diye ayrışır.
|
|
357
|
+
ErrorRow = Struct.new(:label, :error) do
|
|
358
|
+
def summary = "error: #{error.class}: #{error.message.to_s[0, 120]}"
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# Bench tarafı araç hataları (ToolBench#execute'un kurtarma dizgileri):
|
|
362
|
+
# bunlar model davranışı DEĞİL — metrik borusuna girmeden yakalanır.
|
|
363
|
+
TOOL_ERROR_MARKERS = ["tool error: ", "Unknown tool "].freeze
|
|
364
|
+
|
|
365
|
+
# Metrik satırlarından runner sonuçları: hedef taban çizgisi (model
|
|
366
|
+
# puanı değil — senaryo KOŞTU mu). Tablo io'ya markdown düşer.
|
|
367
|
+
def metric_results(rows, io: $stdout)
|
|
368
|
+
io&.puts("\n#### Vakanüvis tool adoption (issue #338)\n")
|
|
369
|
+
io&.puts("| model | recall | first@ | calls | terms | kw:line | empty% | record | wrong put | answer used |")
|
|
370
|
+
io&.puts("|---|---|---|---|---|---|---|---|---|---|")
|
|
371
|
+
rows.map do |m|
|
|
372
|
+
if m.is_a?(SkipRow) || m.is_a?(ErrorRow)
|
|
373
|
+
# Ölçülemeyen model tabloda KALIR — satır kaybı sessiz değil;
|
|
374
|
+
# kota ölümü (skipped) ile gerçek hata (error) ayrışır.
|
|
375
|
+
status = m.is_a?(SkipRow) ? "skipped" : "error"
|
|
376
|
+
# 10 hücre: başlık 10 sütun — 11. hücreye düşen skip/error
|
|
377
|
+
# gerekçesi GFM tarafından SESSİZCE düşürülüyordu.
|
|
378
|
+
io&.puts("| #{m.label} | #{status} | - | - | - | - | - | - | - | (#{m.summary}) |")
|
|
379
|
+
next Rubik::Benchmark::MetricResult.new(
|
|
380
|
+
name: "Vakanüvis Tool Adoption — #{m.label}",
|
|
381
|
+
target: "baseline (issue #338)",
|
|
382
|
+
measured: m.summary,
|
|
383
|
+
passed: false
|
|
384
|
+
)
|
|
385
|
+
end
|
|
386
|
+
io&.puts("| #{m.label} | #{m.recall_called ? 'yes' : 'NO'} | #{m.first_recall_turn || '-'} | #{m.recall_calls} | " \
|
|
387
|
+
"#{m.avg_query_terms || '-'} | #{m.keywords_line_used ? 'yes' : 'no'} | #{m.recall_empty_rate || '-'}% | " \
|
|
388
|
+
"#{m.record_called ? 'vakanuvis' : 'none'} | #{m.wrong_path_puts} | #{m.answer_used_recall ? 'yes' : 'no'} |")
|
|
389
|
+
Rubik::Benchmark::MetricResult.new(
|
|
390
|
+
name: "Vakanüvis Tool Adoption — #{m.label}",
|
|
391
|
+
target: "baseline (issue #338)",
|
|
392
|
+
measured: m.summary,
|
|
393
|
+
passed: true
|
|
394
|
+
)
|
|
395
|
+
end
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
# Replay script'leri: her adım {call: [name, args]} | {call_each: [[name,
|
|
399
|
+
# args], ...]} | {answer: "..."} — ReplayDriver bunları GERÇEK
|
|
400
|
+
# araçlarla sırayla koşturur; yanıtlar profilin davranış taahhüdüdür.
|
|
401
|
+
SCRIPTS = {
|
|
402
|
+
"adopter" => {
|
|
403
|
+
"recall-tr-en" => [
|
|
404
|
+
{ call: ["vakanuvis", { "action" => "recall", "query" => "gateway retry backoff 429" }] },
|
|
405
|
+
{ answer: "Üstel geri çekilme: 2s, 4s, 8s — en fazla 5 deneme; 30 ardışık hata sonra devre kesici açılır." }
|
|
406
|
+
],
|
|
407
|
+
"durable-conclusion" => [
|
|
408
|
+
{ call: ["vakanuvis", { "action" => "record", "topic" => "digest gate dedupe",
|
|
409
|
+
"body" => "Same-target digest jobs are deduplicated; queue cap is 8 jobs, one in-flight per target.\n" \
|
|
410
|
+
"Keywords: digest dedupe gate queue target" }] },
|
|
411
|
+
{ answer: "Recorded as a memory card via the chronicler." }
|
|
412
|
+
]
|
|
413
|
+
},
|
|
414
|
+
"wrong-path" => {
|
|
415
|
+
"recall-tr-en" => [
|
|
416
|
+
{ answer: "Bilmiyorum; geçidi yeniden dene ve sonra tekrar sor." }
|
|
417
|
+
],
|
|
418
|
+
"durable-conclusion" => [
|
|
419
|
+
{ call_each: [["storage", { "action" => "put", "type" => "memory", "title" => "digest dedupe",
|
|
420
|
+
"body" => "Same-target digest jobs are deduplicated." }]] },
|
|
421
|
+
{ answer: "Persisted directly to storage." }
|
|
422
|
+
]
|
|
423
|
+
},
|
|
424
|
+
"struggler" => {
|
|
425
|
+
"recall-tr-en" => [
|
|
426
|
+
{ call: ["vakanuvis", { "action" => "recall", "query" => "gateway hangi durumlarda üstel bekleme uygular ve devre kesici kaç hata sonra açılır" }] },
|
|
427
|
+
{ call: ["vakanuvis", { "action" => "recall", "query" => "gateway retry backoff" }] },
|
|
428
|
+
{ answer: "2s, 4s, 8s üstel geri çekilme uygulanır (en fazla 5 deneme)." }
|
|
429
|
+
],
|
|
430
|
+
"durable-conclusion" => [
|
|
431
|
+
{ call: ["vakanuvis", { "action" => "record", "topic" => "digest gate dedupe",
|
|
432
|
+
"body" => "Same-target digest jobs are deduplicated." }] },
|
|
433
|
+
{ answer: "Saved via vakanuvis record." }
|
|
434
|
+
]
|
|
435
|
+
}
|
|
436
|
+
}.freeze
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
end
|
|
@@ -166,15 +166,21 @@ module Rubik
|
|
|
166
166
|
# annal + wiki derleyicisi. enabled: harness kur/kurulama; distiller:
|
|
167
167
|
# Düzlem 2 LLM doğumu (bütçe: flush_events x max_steps); model: nil =
|
|
168
168
|
# root provider; retention: supersede|purge; root_turns: kök-turn annal'ı.
|
|
169
|
+
# auto_digest: goal bitişinde/kapanışta dokunulan konuların hedefli damıtımı (issue #341).
|
|
169
170
|
"vakanuvis" => {
|
|
170
171
|
"enabled" => true,
|
|
171
172
|
"distiller" => true,
|
|
173
|
+
"auto_digest" => true,
|
|
172
174
|
"max_steps" => 8,
|
|
173
175
|
"flush_events" => 10,
|
|
174
176
|
"max_digest_jobs" => 100,
|
|
175
177
|
"model" => nil,
|
|
176
178
|
"retention" => "supersede",
|
|
177
|
-
"root_turns" => true
|
|
179
|
+
"root_turns" => true,
|
|
180
|
+
# Soğuk-başlangıç ve sıkıştırma sonrası yönelim enjeksiyonu (issue #340):
|
|
181
|
+
# sistem istemi şişirilmeden kök oturuma kısa mem.* özeti eklenir.
|
|
182
|
+
"orientation" => true,
|
|
183
|
+
"orientation_max_topics" => 10
|
|
178
184
|
},
|
|
179
185
|
# Pruner (repl/pruner): yarı deterministik KÖK pencere temizleyicisi,
|
|
180
186
|
# DEFAULT ON (kullanıcı kararı 2026-10-10; kill-switch: enabled=false).
|
data/lib/rubik/log.rb
CHANGED
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
# follows the repo's [rubik] prefix convention. Emit is error-isolated: a
|
|
7
7
|
# logger must never raise, least of all from inside a rescue body.
|
|
8
8
|
#
|
|
9
|
-
# warn/error always print
|
|
10
|
-
# RUBIK_LOG_LEVEL (default "warn")
|
|
11
|
-
#
|
|
9
|
+
# warn/error always print at the DEFAULT level. debug/info/warn are all
|
|
10
|
+
# filtered by RUBIK_LOG_LEVEL (default "warn") — an operator running at
|
|
11
|
+
# "error" also silences :warn, by design. The RUBIK_DEBUG-gated debug_warn
|
|
12
|
+
# stays a separate mechanism and is NOT migrated by this slice.
|
|
12
13
|
module Rubik
|
|
13
14
|
module Log
|
|
14
15
|
LEVELS = { debug: 0, info: 1, warn: 2, error: 3 }.freeze
|
|
@@ -57,6 +57,8 @@ module Rubik
|
|
|
57
57
|
max_output_tokens: @deps.max_output_tokens, config: context_cfg,
|
|
58
58
|
absorb: GoalLedger::ObservationProjection.directive_for(deps: @deps)
|
|
59
59
|
))
|
|
60
|
+
manager = @deps.runtime_manager if @deps.respond_to?(:runtime_manager)
|
|
61
|
+
Rubik::Storage::Vakanuvis::Orientation.inject_after_compaction!(@deps.history, manager, config: @deps.config)
|
|
60
62
|
end
|
|
61
63
|
|
|
62
64
|
def track_token_usage!(res)
|
|
@@ -177,6 +177,7 @@ module Rubik
|
|
|
177
177
|
end
|
|
178
178
|
end
|
|
179
179
|
|
|
180
|
+
Rubik::Storage::Vakanuvis::Orientation.inject_session_start!(@history, @runtime_manager, config: @config)
|
|
180
181
|
@history << { role: "user", content: formatted_task }
|
|
181
182
|
|
|
182
183
|
execute_steps(opts: opts, plan_only: plan_only)
|
|
@@ -71,6 +71,10 @@ module Rubik
|
|
|
71
71
|
@loop.emergency_compact_and_notify!(reason)
|
|
72
72
|
end
|
|
73
73
|
|
|
74
|
+
def runtime_manager
|
|
75
|
+
@loop.runtime_manager if @loop.respond_to?(:runtime_manager)
|
|
76
|
+
end
|
|
77
|
+
|
|
74
78
|
def respond_to_missing?(name, include_private = false)
|
|
75
79
|
(SURFACE.include?(name) && @loop.respond_to?(name, include_private)) || super
|
|
76
80
|
end
|
|
@@ -176,9 +176,9 @@ module Rubik
|
|
|
176
176
|
end
|
|
177
177
|
|
|
178
178
|
def normalize_raw_output(raw_output, tool_name, call_id, dur)
|
|
179
|
-
max_chars = @deps.shell&.config&.fetch("tools.max_output_chars",
|
|
180
|
-
max_lines = @deps.shell&.config&.fetch("tools.max_output_lines",
|
|
181
|
-
trunc_strat = @deps.shell&.config&.fetch("tools.truncation_strategy",
|
|
179
|
+
max_chars = @deps.shell&.config&.fetch("tools.max_output_chars", Rubik::Tools::Truncator::DEFAULT_MAX_CHARS) || Rubik::Tools::Truncator::DEFAULT_MAX_CHARS
|
|
180
|
+
max_lines = @deps.shell&.config&.fetch("tools.max_output_lines", Rubik::Tools::Truncator::DEFAULT_MAX_LINES) || Rubik::Tools::Truncator::DEFAULT_MAX_LINES
|
|
181
|
+
trunc_strat = Rubik::Tools::Truncator.resolve_strategy(@deps.shell&.config&.fetch("tools.truncation_strategy", Rubik::Tools::Truncator::DEFAULT_STRATEGY))
|
|
182
182
|
|
|
183
183
|
if raw_output.is_a?(Rubik::Tools::Result)
|
|
184
184
|
trunc_res = Rubik::Tools::Truncator.truncate(raw_output.output, max_chars: max_chars, max_lines: max_lines, strategy: trunc_strat)
|
|
@@ -17,41 +17,61 @@ module Rubik
|
|
|
17
17
|
return error_result(name, "Tool '#{name}' is not available in subagent toolset.") unless tool
|
|
18
18
|
|
|
19
19
|
raw = tool.execute(args)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
cfg = @config.respond_to?(:fetch) ? @config : nil
|
|
21
|
+
max_chars = (cfg&.fetch("tools.max_output_chars", Rubik::Tools::Truncator::DEFAULT_MAX_CHARS) || Rubik::Tools::Truncator::DEFAULT_MAX_CHARS).to_i
|
|
22
|
+
max_lines = (cfg&.fetch("tools.max_output_lines", Rubik::Tools::Truncator::DEFAULT_MAX_LINES) || Rubik::Tools::Truncator::DEFAULT_MAX_LINES).to_i
|
|
23
|
+
strategy = Rubik::Tools::Truncator.resolve_strategy(cfg&.fetch("tools.truncation_strategy", Rubik::Tools::Truncator::DEFAULT_STRATEGY))
|
|
24
|
+
normalize_tool_result(raw, name, max_chars: max_chars, max_lines: max_lines, strategy: strategy)
|
|
23
25
|
rescue StandardError => e
|
|
24
26
|
error_result(name, "Tool execution error: #{e.class}: #{e.message}", error: e.message)
|
|
25
27
|
end
|
|
26
28
|
|
|
27
29
|
private
|
|
28
30
|
|
|
29
|
-
def normalize_tool_result(raw, name, max_chars:, max_lines:)
|
|
31
|
+
def normalize_tool_result(raw, name, max_chars:, max_lines:, strategy: Rubik::Tools::Truncator::DEFAULT_STRATEGY)
|
|
30
32
|
if raw.is_a?(Rubik::Tools::Result)
|
|
31
|
-
trunc = Rubik::Tools::Truncator.truncate(raw.output, max_chars: max_chars, max_lines: max_lines)
|
|
33
|
+
trunc = Rubik::Tools::Truncator.truncate(raw.output, max_chars: max_chars, max_lines: max_lines, strategy: strategy)
|
|
32
34
|
# Truncation only shortens the display text — status, exit code,
|
|
33
35
|
# metadata and the original truncation flag all survive it.
|
|
36
|
+
first_line = raw.output.to_s.lines.first.to_s.strip
|
|
34
37
|
Rubik::Tools::Result.new(tool_name: raw.tool_name.to_s.empty? ? name : raw.tool_name,
|
|
35
|
-
status: raw.status,
|
|
38
|
+
status: raw.status,
|
|
39
|
+
error: raw.error || (raw.error? ? first_line : nil), exit_code: raw.exit_code,
|
|
36
40
|
output: trunc[:text], duration_ms: raw.duration_ms,
|
|
37
41
|
truncated: raw.truncated || trunc[:truncated], metadata: raw.metadata)
|
|
38
42
|
elsif raw.respond_to?(:status) && raw.respond_to?(:output)
|
|
39
43
|
out_str = raw.output.to_s
|
|
40
|
-
trunc = Rubik::Tools::Truncator.truncate(out_str, max_chars: max_chars, max_lines: max_lines)
|
|
41
|
-
|
|
42
|
-
|
|
44
|
+
trunc = Rubik::Tools::Truncator.truncate(out_str, max_chars: max_chars, max_lines: max_lines, strategy: strategy)
|
|
45
|
+
first_line = out_str.lines.first.to_s.strip
|
|
46
|
+
status = raw.status.to_sym
|
|
47
|
+
Rubik::Tools::Result.new(tool_name: name, status: status,
|
|
48
|
+
error: (raw.respond_to?(:error) && raw.error) || (status == :error ? first_line : nil),
|
|
49
|
+
output: trunc[:text], truncated: trunc[:truncated],
|
|
43
50
|
exit_code: raw.respond_to?(:exit_code) ? raw.exit_code : nil,
|
|
44
51
|
metadata: raw.respond_to?(:metadata) ? raw.metadata : {})
|
|
45
52
|
else
|
|
46
53
|
out_str = raw.to_s
|
|
47
|
-
trunc = Rubik::Tools::Truncator.truncate(out_str, max_chars: max_chars, max_lines: max_lines)
|
|
48
|
-
status =
|
|
54
|
+
trunc = Rubik::Tools::Truncator.truncate(out_str, max_chars: max_chars, max_lines: max_lines, strategy: strategy)
|
|
55
|
+
status = classify_raw_string(raw, out_str)
|
|
49
56
|
Rubik::Tools::Result.new(tool_name: name, status: status,
|
|
50
57
|
error: status == :error ? out_str.lines.first.to_s.strip : nil,
|
|
51
58
|
output: trunc[:text], truncated: trunc[:truncated])
|
|
52
59
|
end
|
|
53
60
|
end
|
|
54
61
|
|
|
62
|
+
# Root-parity string classification (arch scan card 5, review F1): the
|
|
63
|
+
# same lexicon as StepExecutor#normalize_raw_output's else branch —
|
|
64
|
+
# error?/failed? predicates, then the shared ERROR_PREFIXES table.
|
|
65
|
+
# Without this, a bare "Error: ..." string reads status=ok in subagent
|
|
66
|
+
# history while the root plane honestly reports :error.
|
|
67
|
+
def classify_raw_string(raw, out_str)
|
|
68
|
+
return :error if raw.respond_to?(:error?) && raw.error?
|
|
69
|
+
return :error if raw.respond_to?(:failed?) && raw.failed?
|
|
70
|
+
return :error if raw.is_a?(String) && Rubik::Orchestrator::StepExecutor::ERROR_PREFIXES.any? { |pfx| out_str.start_with?(pfx) }
|
|
71
|
+
|
|
72
|
+
:ok
|
|
73
|
+
end
|
|
74
|
+
|
|
55
75
|
def error_result(name, text, error: nil)
|
|
56
76
|
Rubik::Tools::Result.new(tool_name: name, status: :error, output: text, error: error || text)
|
|
57
77
|
end
|
|
@@ -103,6 +103,8 @@ module Rubik
|
|
|
103
103
|
|
|
104
104
|
# ── announce ──
|
|
105
105
|
def announce_compaction(reason, layer)
|
|
106
|
+
manager = @deps.runtime_manager if @deps.respond_to?(:runtime_manager)
|
|
107
|
+
Rubik::Storage::Vakanuvis::Orientation.inject_after_compaction!(@deps.history, manager, config: @deps.config)
|
|
106
108
|
@deps.on_step&.call(type: :emergency_compact, reason: reason, step: @deps.step_count,
|
|
107
109
|
max_steps: @deps.max_steps, layer: layer)
|
|
108
110
|
end
|