rubik 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +126 -1
- data/lib/rubik/benchmark/runner.rb +17 -0
- data/lib/rubik/benchmark/vakanuvis_adoption.rb +439 -0
- data/lib/rubik/config/defaults.rb +8 -0
- data/lib/rubik/context/index.rb +76 -42
- data/lib/rubik/context/root_heuristics.rb +93 -0
- data/lib/rubik/context/scanner.rb +33 -5
- data/lib/rubik/context.rb +1 -0
- data/lib/rubik/log.rb +4 -3
- data/lib/rubik/orchestrator/step_executor.rb +14 -4
- data/lib/rubik/orchestrator/subagent/tool_dispatch.rb +41 -12
- data/lib/rubik/orchestrator/tool_timeout.rb +193 -0
- data/lib/rubik/prompt/emergency_compactor.rb +7 -3
- data/lib/rubik/repl/command_context.rb +6 -0
- data/lib/rubik/repl/commands/agent_context.rb +17 -1
- data/lib/rubik/repl/commands/agent_execution.rb +7 -0
- data/lib/rubik/repl/composition.rb +2 -1
- data/lib/rubik/repl/input_execution.rb +5 -1
- data/lib/rubik/repl/lifecycle.rb +51 -1
- data/lib/rubik/storage/vakanuvis/annal_writer.rb +51 -4
- data/lib/rubik/storage/vakanuvis/distill_gate.rb +14 -2
- data/lib/rubik/storage/vakanuvis/gate.rb +16 -13
- data/lib/rubik/storage/vakanuvis/harness.rb +40 -15
- data/lib/rubik/storage/vakanuvis/job.rb +25 -0
- data/lib/rubik/storage/vakanuvis.rb +12 -0
- data/lib/rubik/tools/builtins/files.rb +24 -5
- data/lib/rubik/tools/builtins/grep.rb +54 -31
- data/lib/rubik/tools/index_messages.rb +38 -0
- data/lib/rubik/tools/native_search.rb +14 -5
- data/lib/rubik/tools/truncator.rb +53 -0
- data/lib/rubik/version.rb +1 -1
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e6db8019561c617d4b3006fc7caea11b4f780feb2a55bc200fc8ca4825040720
|
|
4
|
+
data.tar.gz: 0acc95f9ae8c299a2d253833c541954d92b1372c6af7c3c1af08ad9a08bb5c53
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6d2e351b5907c057bb29fe1b20d84dfef967afb8571116a147ca7e013149e6c5fe00bd5aa8cb2cccd08e4e7679184e02f126e8b2fab2f11cc5ee33f28d837648
|
|
7
|
+
data.tar.gz: b638835c922ad012b30e438f00460704d7f9aaf0d324d9de90267f1d10b6e0dd81b931b4d2491d312825dc0ed2539cd77880bfbddd83765e7e34fdbe27b14dfe
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,129 @@ commits, 2026-08-23 → 2026-10-09, see `git log`).
|
|
|
6
6
|
|
|
7
7
|
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
8
8
|
|
|
9
|
+
## [0.7.0] - 2026-11-03
|
|
10
|
+
|
|
11
|
+
A turn-stall-and-honesty minor: `list_files` and `grep_search` never
|
|
12
|
+
trigger a synchronous full scan again, partial context scans disclose
|
|
13
|
+
themselves instead of serving truncated truth as clean, huge session
|
|
14
|
+
roots get boot-time guidance instead of a silent minutes-long warm
|
|
15
|
+
walk, and a single stalling tool can no longer pin the whole turn —
|
|
16
|
+
the first dispatcher-level call budget lands as an off-by-default knob.
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- **Bounded tool execution (`tools.call_timeout`, `tools.call_timeout_overrides`)
|
|
21
|
+
(#355):** an off-by-default dispatcher budget applied at the tool-dispatch seam
|
|
22
|
+
on both orchestrator planes; a fired budget returns an honest error result (the
|
|
23
|
+
call is abandoned, not killed) and the turn continues, with the timeout visible
|
|
24
|
+
via `!dmesg -s tools`. Precedence: an explicit env disable
|
|
25
|
+
(`RUBIK_TOOLS_CALL_TIMEOUT=0`) wins over everything, per-tool overrides refine
|
|
26
|
+
the resolved global, and the env budget beats the config file. Knob unset means
|
|
27
|
+
byte-identical inline execution.
|
|
28
|
+
- **Huge-root guidance (`context.huge_root_notice`, `context.boot_scan_on_huge_root`)
|
|
29
|
+
(#356):** a session root that looks like a huge tree (the user home directory, a
|
|
30
|
+
filesystem root, or a root holding OS data directories) now prints one guidance
|
|
31
|
+
notice at boot and skips the boot-time warm scan; `!context scan` is the explicit
|
|
32
|
+
opt-in, and the knob restores the boot walk. Ordinary repositories boot with no
|
|
33
|
+
new output.
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- **Partial context scans are disclosed instead of served as truth (#357):** the
|
|
38
|
+
scanner records every dropped-subtree abort into a completeness verdict that
|
|
39
|
+
`Index#status_summary` publishes under the same lock as the files; `!context
|
|
40
|
+
scan` reports `finished PARTIALLY`, `!context status`/`stats` show the flag,
|
|
41
|
+
and `list_files`/`grep_search` append a PARTIAL note to answers served from a
|
|
42
|
+
truncated index (hits, no-matches, and ignored-directory verdicts alike).
|
|
43
|
+
Completed scans produce byte-identical output.
|
|
44
|
+
- **Tools never scan (#358):** an empty context index no longer triggers a
|
|
45
|
+
synchronous full walk inside `list_files`/`grep_search` — in a session rooted
|
|
46
|
+
at a large non-repo tree that walk ran for minutes inside the turn and, the
|
|
47
|
+
Ruby walker holding the GVL, stalled the whole REPL with it. An empty index
|
|
48
|
+
now answers honestly ("not ready, run `!context scan`"), the ignore-rules
|
|
49
|
+
explanation is reserved for indexes that were actually built, grep's
|
|
50
|
+
memory-only dirty-buffer searches disclose their scope on hits and no-hits
|
|
51
|
+
alike, AppData moves to the root-anchored hard-prune set (`.ollama`/`.zcode`
|
|
52
|
+
join the default ignores) with the rg prune globs gaining the any-depth forms
|
|
53
|
+
the Ruby side always applied, and `!context scan` warns on an empty result
|
|
54
|
+
instead of claiming success.
|
|
55
|
+
|
|
56
|
+
## [0.6.0] - 2026-11-02
|
|
57
|
+
|
|
58
|
+
A measurement-and-honesty minor: the planned vakanüvis fixes gain their
|
|
59
|
+
baseline with a tool-adoption bench scenario, goal suspensions finally
|
|
60
|
+
reach the chronicle through a typed digest contract, interleaved-goal
|
|
61
|
+
suspend rows stop mispairing their records, the annal dedupe can no
|
|
62
|
+
longer be poisoned by free-text `event=` markers, and a typo'd
|
|
63
|
+
truncation strategy warns instead of silently no-oping.
|
|
64
|
+
|
|
65
|
+
### Added
|
|
66
|
+
|
|
67
|
+
- **Goal suspension chronicling:** the chronicler now subscribes to
|
|
68
|
+
`:goal_suspended` (previously posted by `GoalState` but bus-orphaned —
|
|
69
|
+
the producer comment promised a consumption that never existed) and
|
|
70
|
+
appends a `goal.<n>.suspended` row to the session annal; the ordinal is
|
|
71
|
+
peeked rather than minted, so the eventual finish shares the same
|
|
72
|
+
`goal.<n>` record and no duplicate goal note is born from a suspension.
|
|
73
|
+
- **vakanüvis tool-adoption bench scenario (#338, #354):** a two-plane
|
|
74
|
+
measurement of whether models actually adopt recall/record — a
|
|
75
|
+
deterministic replay (default, runs in `rake bench` and CI) driving the
|
|
76
|
+
real tool plane through adopter/wrong-path/struggler profiles, and an
|
|
77
|
+
opt-in live plane (`rake bench:vakanuvis_adoption MODELS=...`) looping
|
|
78
|
+
real provider completions with both tools available; per-model metrics
|
|
79
|
+
cover recall adoption, Keywords usage, record-via-vakanüvis vs direct
|
|
80
|
+
storage put, answer grounding and turn-cap hits, with honest Skip
|
|
81
|
+
(quota) vs Error rows and a fresh seeded store per driver.
|
|
82
|
+
`:goal_suspended` (previously posted by `GoalState` but bus-orphaned —
|
|
83
|
+
the producer comment promised a consumption that never existed) and
|
|
84
|
+
appends a `goal.<n>.suspended` row to the session annal; the ordinal is
|
|
85
|
+
peeked rather than minted, so the eventual finish shares the same
|
|
86
|
+
`goal.<n>` record and no duplicate goal note is born from a suspension.
|
|
87
|
+
|
|
88
|
+
### Changed
|
|
89
|
+
|
|
90
|
+
- **Digest contract typed:** the digest job is one `DistillGate::Job`
|
|
91
|
+
Struct (the queue's two Hash literals collapse into a keyword-minted
|
|
92
|
+
shape), the orphaned `:goal_id` read that no site ever set becomes a
|
|
93
|
+
live `goal_ordinal` carried from goal-end auto-digest — the distiller
|
|
94
|
+
brief now cites the goal record for goal-tied digests — and the digest
|
|
95
|
+
receipt text is single-homed in the DistillGate rules module
|
|
96
|
+
(`receipt` / `duplicate_receipt` / `full_receipt`) instead of three
|
|
97
|
+
templates across two files.
|
|
98
|
+
- **Truncation strategy parity:** `tools.truncation_strategy` was honored
|
|
99
|
+
by exactly one of the Truncator call sites while the subagent tool
|
|
100
|
+
dispatch hard-coded `head_tail` under a comment claiming the two planes
|
|
101
|
+
cannot drift; both orchestrator planes now fetch the knob (fallback
|
|
102
|
+
`Truncator::DEFAULT_STRATEGY`), the restated 12_000/300 literals resolve
|
|
103
|
+
from `Truncator::DEFAULT_MAX_CHARS/LINES`, and a parity pin test turns
|
|
104
|
+
the prose invariant into a test invariant (classification, truncation
|
|
105
|
+
and the strategy knob are asserted equal across the two planes, oracle-
|
|
106
|
+
anchored against Truncator directly). The configuration docs now name
|
|
107
|
+
the honoring consumers.
|
|
108
|
+
|
|
109
|
+
### Fixed
|
|
110
|
+
|
|
111
|
+
- **Goal-identity suspend keys and honest logs (#350, #351):** suspend
|
|
112
|
+
rows key on goal identity (ledger id, else slug) plus steps and an
|
|
113
|
+
error digest instead of a peeked ordinal — interleaved goals no longer
|
|
114
|
+
mispair A's suspend row to B's finish and two suspensions sharing a
|
|
115
|
+
peek no longer dedupe into one row; `AnnalWriter.event_context` adds
|
|
116
|
+
status + goal so a dropped suspension and a dropped finish are
|
|
117
|
+
distinguishable in the logs; a `:duplicate` auto-digest admission warns
|
|
118
|
+
with the topic and unattached ordinal instead of silently succeeding;
|
|
119
|
+
the append dedupe anchors on line ends so free-text fields carrying
|
|
120
|
+
`(event=...)` cannot plant phantom keys that suppress genuine rows; the
|
|
121
|
+
chronicler worker gains a per-iteration last-ditch rescue with a
|
|
122
|
+
backtrace frame, the ledger id is stamped before `suspend!`, and a
|
|
123
|
+
spawn-disabled flush warns honestly instead of queueing in silence.
|
|
124
|
+
- **Unrecognized truncation strategy values warn once (#349):** a typo'd
|
|
125
|
+
`tools.truncation_strategy` (or `context.emergency_truncate_strategy`)
|
|
126
|
+
previously fell into the `head_tail` else branch with no signal in
|
|
127
|
+
either plane; `Truncator::STRATEGIES` is now the honest frontier, all
|
|
128
|
+
three config-fetch sites resolve through it, and each distinct unknown
|
|
129
|
+
value warns exactly once per knob+value pair with a UTF-8-scrubbed
|
|
130
|
+
message that can never turn tool results into errors.
|
|
131
|
+
|
|
9
132
|
## [0.5.0] - 2026-10-31
|
|
10
133
|
|
|
11
134
|
A vakanüvis minor: the memory system gains a harness-owned auto-digest
|
|
@@ -1007,7 +1130,9 @@ installation section.
|
|
|
1007
1130
|
- Context: serialized index mutations, snapshot-outside-lock reconcile,
|
|
1008
1131
|
streaming-first truncation.
|
|
1009
1132
|
|
|
1010
|
-
[Unreleased]: https://github.com/ozgurulukir/rubik/compare/v0.
|
|
1133
|
+
[Unreleased]: https://github.com/ozgurulukir/rubik/compare/v0.7.0...HEAD
|
|
1134
|
+
[0.7.0]: https://github.com/ozgurulukir/rubik/compare/v0.6.0...v0.7.0
|
|
1135
|
+
[0.6.0]: https://github.com/ozgurulukir/rubik/compare/v0.5.0...v0.6.0
|
|
1011
1136
|
[0.5.0]: https://github.com/ozgurulukir/rubik/compare/v0.4.2...v0.5.0
|
|
1012
1137
|
[0.4.2]: https://github.com/ozgurulukir/rubik/compare/v0.4.1...v0.4.2
|
|
1013
1138
|
[0.4.1]: https://github.com/ozgurulukir/rubik/compare/v0.4.0...v0.4.1
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require "benchmark"
|
|
4
4
|
require "tmpdir"
|
|
5
5
|
|
|
6
|
+
require_relative "vakanuvis_adoption"
|
|
7
|
+
|
|
6
8
|
module Rubik
|
|
7
9
|
module Benchmark
|
|
8
10
|
class MetricResult
|
|
@@ -38,9 +40,21 @@ module Rubik
|
|
|
38
40
|
results << bench_grep_tool
|
|
39
41
|
results << bench_cold_scan
|
|
40
42
|
results << bench_search_ranking
|
|
43
|
+
results.concat(bench_vakanuvis_adoption)
|
|
41
44
|
results
|
|
42
45
|
end
|
|
43
46
|
|
|
47
|
+
# issue #338: araç-benimseme taban çizgisi. Ana süit YALNIZ deterministik
|
|
48
|
+
# replay düzlemini koşar (canlı model çağrısı asla default olamaz);
|
|
49
|
+
# canlı ölçüm rake bench:vakanuvis_adoption MODELS=... ile açık istektir.
|
|
50
|
+
def bench_vakanuvis_adoption
|
|
51
|
+
rows = Rubik::Benchmark::VakanuvisAdoption.run(
|
|
52
|
+
Rubik::Benchmark::VakanuvisAdoption::REPLAY_PROFILES.map { |p| Rubik::Benchmark::VakanuvisAdoption::ReplayDriver.new(p) },
|
|
53
|
+
io: nil
|
|
54
|
+
)
|
|
55
|
+
Rubik::Benchmark::VakanuvisAdoption.metric_results(rows, io: nil)
|
|
56
|
+
end
|
|
57
|
+
|
|
44
58
|
def generate_markdown_report(results)
|
|
45
59
|
lines = []
|
|
46
60
|
lines << "| Metric | Target | Measured | Status |"
|
|
@@ -195,6 +209,9 @@ module Rubik
|
|
|
195
209
|
def grep_once(repo_dir, native:)
|
|
196
210
|
best_of(3) do
|
|
197
211
|
idx = Rubik::Context::Index.new(root_dir: repo_dir, native: native)
|
|
212
|
+
# Tools never scan; build the index up front (scan cost has its
|
|
213
|
+
# own scan_once benchmark).
|
|
214
|
+
idx.full_scan
|
|
198
215
|
tool = Rubik::Tools::GrepSearchTool.new(nil, idx, root_dir: repo_dir, config: NativeConfig.new(native))
|
|
199
216
|
::Benchmark.realtime { tool.execute({ "query" => "needle_rare_marker", "max_results" => 50 }) }
|
|
200
217
|
end
|
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require_relative "../storage/vakanuvis"
|
|
6
|
+
require_relative "../tools/builtins/storage"
|
|
7
|
+
require_relative "../tools/builtins/vakanuvis"
|
|
8
|
+
|
|
9
|
+
module Rubik
|
|
10
|
+
module Benchmark
|
|
11
|
+
# Vakanüvis araç-benimseme senaryosu (issue #338): küçük modellerin
|
|
12
|
+
# vakanüvis'i ne sıklıkla, nasıl kullandığını ölçen taban çizgisi.
|
|
13
|
+
# "Bu düzeltme iğneyi hareket ettirdi mi?" sorusunun cevabı için
|
|
14
|
+
# prompt-doktrin / yönelim-enjeksiyonu / otomatik-damıtma düzeltmeleri
|
|
15
|
+
# ÖNCE bu ölçüm ister.
|
|
16
|
+
#
|
|
17
|
+
# Düzlem 1 (deterministik, CI-güvenli): tohum depo + ReplayDriver —
|
|
18
|
+
# üç davranış profili GERÇEK araç düzlemini çalıştırır (script'lenmiş
|
|
19
|
+
# araç çağrıları, sahte sonuçlar değil); metrik borusu ve recall
|
|
20
|
+
# çıktı biçimi regresyona girerse burada patlar.
|
|
21
|
+
# Düzlem 2 (canlı): LiveDriver provider.complete döngüsü — MODELS
|
|
22
|
+
# env'i ile model başına gerçek benimseme ölçümü.
|
|
23
|
+
#
|
|
24
|
+
# Sessizlik: canlı modeller maliyetli ve deterministik değildir — bu
|
|
25
|
+
# yüzden default Düzlem 1'dir; rake bench ana süitine asla canlı çağrı
|
|
26
|
+
# sızdırmaz (bench:vakanuvis_adoption MODELS=... açık istek ister).
|
|
27
|
+
module VakanuvisAdoption
|
|
28
|
+
# Tohum kartlar: TR/EN karışık, Keywords: satırlı/satırsız — issue
|
|
29
|
+
# senaryosunun ta kendisi. Görev-1 kartının ayırt edici değerleri
|
|
30
|
+
# (8 saniye adımı + 5 deneme) yanıt-kullanımını dürüstçe sayılabilir
|
|
31
|
+
# kılar: model hatırladığı karttan alıntıladıysa ikisi de görünür.
|
|
32
|
+
SEED_CARDS = [
|
|
33
|
+
{
|
|
34
|
+
topic: "Ağ geçidi yeniden deneme politikası",
|
|
35
|
+
body: "Ağ geçidi 429/503 yanıtında üstel geri çekilme uygular: 2s, 4s, 8s " \
|
|
36
|
+
"(en fazla 5 deneme). Devre kesici 30 ardışık hata sonra açılır.\n" \
|
|
37
|
+
"Keywords: gateway retry backoff üstel geri çekilme 429 503 circuit breaker exponent"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
topic: "Sandbox filesystem allowlist",
|
|
41
|
+
body: "The container sandbox mounts the worktree read-write and $HOME read-only; " \
|
|
42
|
+
"the allowlist file is sandbox.allow at the repo root.\n" \
|
|
43
|
+
"Keywords: sandbox allowlist container readonly mount worktree izin listesi"
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
topic: "Vakanüvis damıtma kapısı",
|
|
47
|
+
body: "Damıtma kapısı aynı hedefli işi yutar (dedupe); maksimum kuyruk 8 iş, " \
|
|
48
|
+
"hedef başına tek iz."
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
topic: "Fiber scheduler budget",
|
|
52
|
+
body: "The fiber scheduler caps concurrent subagent fibers at 12; overflow " \
|
|
53
|
+
"queues FIFO and never drops."
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
topic: "Goal sıra türetimi",
|
|
57
|
+
body: "Goal sırası mağazadaki en büyük annal.goal.N'den türetilir; duvar saati " \
|
|
58
|
+
"ASLA kullanılmaz — kronik MARKER-ZIKKIM-7 determinizmi budur."
|
|
59
|
+
}
|
|
60
|
+
].freeze
|
|
61
|
+
|
|
62
|
+
# Görev 1 yanıt izleri: karttan alıntı → ikisi de eşleşir; uydurma
|
|
63
|
+
# yanıt → en az biri (genelde ikisi) kaçar.
|
|
64
|
+
ANSWER_MARKER_BACKOFF = /8\s*(s\b|sn|sec|second|saniye)/i
|
|
65
|
+
ANSWER_MARKER_TRIES = /(5|beş|five)\s*(deneme|attempt|retr|try)/i
|
|
66
|
+
|
|
67
|
+
TASKS = [
|
|
68
|
+
{
|
|
69
|
+
id: "recall-tr-en",
|
|
70
|
+
prompt: "Bu sorunun cevabı rubik://storage kronikçisindeki bir hafıza kartında duruyor: " \
|
|
71
|
+
"ağ geçidi (gateway) 429/503 yanıtında yeniden deneme / geri çekilme politikası nedir? " \
|
|
72
|
+
"Kısa cevabını ver."
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
id: "durable-conclusion",
|
|
76
|
+
prompt: "We just concluded that the chronicler's digest gate deduplicates jobs for the " \
|
|
77
|
+
"same target. Persist this durable conclusion so future sessions can recall it, " \
|
|
78
|
+
"then confirm in one line."
|
|
79
|
+
}
|
|
80
|
+
].freeze
|
|
81
|
+
|
|
82
|
+
MAX_TURNS = 8
|
|
83
|
+
REPLAY_PROFILES = ["adopter", "wrong-path", "struggler"].freeze
|
|
84
|
+
|
|
85
|
+
module_function
|
|
86
|
+
|
|
87
|
+
# ---- mağaza + araç masası ---------------------------------------------
|
|
88
|
+
|
|
89
|
+
# Kronikçi yapılandırması: damıtıcı KAPALI (bench'te asla LLM doğumu —
|
|
90
|
+
# ölçtüğümüz şey maestro davranışı, damıtıcı maliyeti değil).
|
|
91
|
+
class QuietConfig
|
|
92
|
+
def fetch(key, default = nil)
|
|
93
|
+
return false if key == "subagents.vakanuvis.distiller"
|
|
94
|
+
|
|
95
|
+
default
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Deterministik tohum: aynı kart dizisi ⇒ aynı mağaza ⇒ aynı recall
|
|
100
|
+
# davranışı. Her model/profil ÖZGÜN kök dizinle başlar (record
|
|
101
|
+
# yan etkileri bir sonraki modelin ölçümünü kirletmesin).
|
|
102
|
+
def seed_store(root)
|
|
103
|
+
store = Rubik::Storage::Store.new(root: root, project: "vakanuvis-adoption-bench")
|
|
104
|
+
harness = Rubik::Storage::Vakanuvis::Harness.new(store: store, config: QuietConfig.new, session_slug: "bench-seed")
|
|
105
|
+
SEED_CARDS.each { |card| harness.record(topic: card[:topic], body: card[:body]) }
|
|
106
|
+
harness.shutdown
|
|
107
|
+
store
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# İki gerçek yerleşik araç: vakanüvis (doğru yol) + storage (yanlış
|
|
111
|
+
# yolun mümkün olması için masada DURMALI — yoksa yanlış-yol oranı
|
|
112
|
+
# ölçülemez, sahte sıfır doğardı).
|
|
113
|
+
class ToolBench
|
|
114
|
+
def initialize(store, harness)
|
|
115
|
+
@tools = {
|
|
116
|
+
"vakanuvis" => Rubik::Tools::VakanuvisTool.new(harness),
|
|
117
|
+
"storage" => Rubik::Tools::StorageTool.new(store)
|
|
118
|
+
}
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def execute(name, args)
|
|
122
|
+
tool = @tools[name.to_s]
|
|
123
|
+
return "Unknown tool '#{name}' (bench toolbelt: vakanuvis, storage)." unless tool
|
|
124
|
+
|
|
125
|
+
tool.execute(args || {})
|
|
126
|
+
rescue StandardError => e
|
|
127
|
+
"tool error: #{e.class}: #{e.message}"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def schemas = @tools.values.map(&:to_h)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# ---- sürücüler --------------------------------------------------------
|
|
134
|
+
# Sözleşme: run_task(task, tools) -> { turns:, answer:, calls:,
|
|
135
|
+
# first_recall_turn:, hit_cap: }. calls = [[name, args, result_text]]
|
|
136
|
+
# (sıralı). first_recall_turn = ilk recall'in 1-tabanlı turn'ü.
|
|
137
|
+
|
|
138
|
+
# Script'lenmiş davranış profilleri — scriptler GERÇEK araçlarla
|
|
139
|
+
# koşar (sonuç sahtelenmez); metrik borusunun deterministik kanıtı.
|
|
140
|
+
class ReplayDriver
|
|
141
|
+
attr_reader :profile
|
|
142
|
+
|
|
143
|
+
def initialize(profile)
|
|
144
|
+
@profile = profile.to_s
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def label = "scripted/#{@profile}"
|
|
148
|
+
|
|
149
|
+
def live? = false
|
|
150
|
+
|
|
151
|
+
def run_task(task, tools)
|
|
152
|
+
calls = []
|
|
153
|
+
answer = ""
|
|
154
|
+
turns = 0
|
|
155
|
+
first_recall_turn = nil
|
|
156
|
+
SCRIPTS.fetch(@profile).fetch(task[:id]).each do |step|
|
|
157
|
+
turns += 1
|
|
158
|
+
steps = step[:call] ? [step[:call]] : (step[:call_each] || [])
|
|
159
|
+
steps.each do |name, args|
|
|
160
|
+
first_recall_turn ||= turns if recall_step?(name, args)
|
|
161
|
+
calls << [name, args, tools.execute(name, args)]
|
|
162
|
+
end
|
|
163
|
+
answer = step[:answer] if step.key?(:answer)
|
|
164
|
+
end
|
|
165
|
+
{ turns: turns, answer: answer, calls: calls, first_recall_turn: first_recall_turn, hit_cap: false }
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
private
|
|
169
|
+
|
|
170
|
+
def recall_step?(name, args)
|
|
171
|
+
name == "vakanuvis" && (args["action"].nil? || args["action"] == "recall")
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Canlı sürücü: provider.complete döngüsü. Prompt görevi söyletir,
|
|
176
|
+
# araç seçimini ASLA emretmez — benimseme ölçümünün koşulu budur.
|
|
177
|
+
class LiveDriver
|
|
178
|
+
def initialize(provider, max_turns: MAX_TURNS)
|
|
179
|
+
@provider = provider
|
|
180
|
+
@max_turns = max_turns
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def label = "live/#{@provider.name}/#{@provider.model}"
|
|
184
|
+
|
|
185
|
+
def live? = true
|
|
186
|
+
|
|
187
|
+
def run_task(task, tools)
|
|
188
|
+
history = [{ "role" => "user", "content" => task[:prompt] }]
|
|
189
|
+
calls = []
|
|
190
|
+
turns = 0
|
|
191
|
+
first_recall_turn = nil
|
|
192
|
+
answer = ""
|
|
193
|
+
|
|
194
|
+
@max_turns.times do
|
|
195
|
+
turns += 1
|
|
196
|
+
res = @provider.complete(history, { tools: tools.schemas })
|
|
197
|
+
# Sağlayıcı sözleşmesi (inceleme turu 2): Response dışı bir
|
|
198
|
+
# dönüş, respond_to? zincirlerine girip sahte "araç
|
|
199
|
+
# benimsemedi / tur sınırına vurdu" veri noktası üretmesin —
|
|
200
|
+
# dürüstçe patla.
|
|
201
|
+
raise "provider returned #{res.inspect}, expected Rubik::LLM::Response" unless res.is_a?(Rubik::LLM::Response)
|
|
202
|
+
|
|
203
|
+
tool_calls = res.respond_to?(:tool_calls?) && res.tool_calls? ? res.tool_calls : []
|
|
204
|
+
if tool_calls.empty?
|
|
205
|
+
answer = res.respond_to?(:text) ? res.text.to_s : res.to_s
|
|
206
|
+
break
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
history << {
|
|
210
|
+
role: "assistant",
|
|
211
|
+
content: res.respond_to?(:text) ? res.text.to_s : "",
|
|
212
|
+
tool_calls: tool_calls.map { |tc| { "id" => tc["id"], "tool" => tc["tool"], "args" => tc["args"] } }
|
|
213
|
+
}
|
|
214
|
+
tool_calls.each do |tc|
|
|
215
|
+
args = tc["args"].is_a?(Hash) ? tc["args"] : {}
|
|
216
|
+
action = args["action"].to_s
|
|
217
|
+
first_recall_turn ||= turns if tc["tool"] == "vakanuvis" && (action.empty? || action == "recall")
|
|
218
|
+
result = tools.execute(tc["tool"], args)
|
|
219
|
+
calls << [tc["tool"], args, result]
|
|
220
|
+
history << { role: "tool", tool_call_id: tc["id"], tool_name: tc["tool"], content: result.to_s }
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
{ turns: turns, answer: answer, calls: calls, first_recall_turn: first_recall_turn, hit_cap: answer.empty? }
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# ---- metrikler --------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
Metrics = Struct.new(
|
|
231
|
+
:label, :recall_called, :first_recall_turn, :recall_calls, :avg_query_terms,
|
|
232
|
+
:keywords_line_used, :recall_empty_rate, :record_called, :wrong_path_puts,
|
|
233
|
+
:answer_used_recall, :hit_turn_cap, keyword_init: true
|
|
234
|
+
) do
|
|
235
|
+
def summary
|
|
236
|
+
"recall=#{recall_called ? "turn #{first_recall_turn}" : 'never'} calls=#{recall_calls} " \
|
|
237
|
+
"terms=#{avg_query_terms || '-'} kw=#{keywords_line_used ? 'yes' : 'no'} empty=#{recall_empty_rate || '-'}% " \
|
|
238
|
+
"record=#{record_called ? 'vakanuvis' : 'none'} wrong_put=#{wrong_path_puts} " \
|
|
239
|
+
"answer_used=#{answer_used_recall ? 'yes' : 'no'}"
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def metrics_for(label, task_runs)
|
|
244
|
+
recall_events = []
|
|
245
|
+
record_calls = 0
|
|
246
|
+
wrong_puts = 0
|
|
247
|
+
first_recall_turn = nil
|
|
248
|
+
answer_used = false
|
|
249
|
+
hit_cap = false
|
|
250
|
+
|
|
251
|
+
task_runs.each do |run|
|
|
252
|
+
run[:calls].each do |name, args, result|
|
|
253
|
+
if name == "vakanuvis"
|
|
254
|
+
action = args["action"].to_s
|
|
255
|
+
if action.empty? || action == "recall"
|
|
256
|
+
recall_events << [args["query"].to_s, result.to_s]
|
|
257
|
+
first_recall_turn ||= run[:first_recall_turn]
|
|
258
|
+
elsif action == "record"
|
|
259
|
+
record_calls += 1
|
|
260
|
+
end
|
|
261
|
+
elsif name == "storage" && args["action"].to_s == "put" && ["memory", "wiki"].include?(args["type"].to_s)
|
|
262
|
+
wrong_puts += 1 # kronikçi masada DURUYORKEN doğrudan put = yanlış yol
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
answer_used ||= answer_uses_recall?(run[:answer])
|
|
266
|
+
hit_cap ||= run[:hit_cap] == true
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
terms = recall_events.map { |query, _res| query.strip.split(/\s+/).reject(&:empty?).size }
|
|
270
|
+
Metrics.new(
|
|
271
|
+
label: label,
|
|
272
|
+
recall_called: !recall_events.empty?,
|
|
273
|
+
first_recall_turn: first_recall_turn,
|
|
274
|
+
recall_calls: recall_events.size,
|
|
275
|
+
avg_query_terms: terms.empty? ? nil : terms.sum.fdiv(terms.size).round(1),
|
|
276
|
+
keywords_line_used: recall_events.any? { |query, _res| query.include?("Keywords:") },
|
|
277
|
+
recall_empty_rate: recall_events.empty? ? nil : empty_rate(recall_events),
|
|
278
|
+
record_called: record_calls.positive?,
|
|
279
|
+
wrong_path_puts: wrong_puts,
|
|
280
|
+
answer_used_recall: answer_used,
|
|
281
|
+
hit_turn_cap: hit_cap
|
|
282
|
+
)
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# "Recalled N ..." başlığı dışındaki her şey (anahtar-AND boş + yalnız
|
|
286
|
+
# fallback izleri) boş sayılır — issue'nin "fallback dahil" tanımı.
|
|
287
|
+
def empty_rate(recall_events)
|
|
288
|
+
empty_count = recall_events.count { |_query, res| !res.start_with?("Recalled ") }
|
|
289
|
+
(empty_count.fdiv(recall_events.size) * 100).round(0)
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def answer_uses_recall?(text)
|
|
293
|
+
text = text.to_s
|
|
294
|
+
text.match?(ANSWER_MARKER_BACKOFF) && text.match?(ANSWER_MARKER_TRIES)
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# ---- koşu -------------------------------------------------------------
|
|
298
|
+
|
|
299
|
+
# Her sürücü özgün tohumlu mağaza alır (izolasyon: record yan
|
|
300
|
+
# etkileri sızamaz). Sürücü hatası TEK modeli tablodan düşürür —
|
|
301
|
+
# diğer modellerin ölçümü yaşamaya devam eder; düşen satır sınıfa
|
|
302
|
+
# göre ayrışır: RateLimitError → SkipRow (kota/ağ ölümü — geçici),
|
|
303
|
+
# diğer her şey → ErrorRow (kimlik doğrulama, bozuk JSON, determin-
|
|
304
|
+
# istik düzlemde script hatası... bunlar "skipped" kıyafetini taşımaya
|
|
305
|
+
# değmez). Deterministik düzlemde çöküş loud: diğer bench metotları
|
|
306
|
+
# gibi rake bench'i kırmalı — çökme "skipped" diye kaydedilmez.
|
|
307
|
+
def run(drivers, io: $stdout)
|
|
308
|
+
drivers.map do |driver|
|
|
309
|
+
root = Dir.mktmpdir("rubik_vakanuvis_adoption")
|
|
310
|
+
begin
|
|
311
|
+
store = seed_store(root)
|
|
312
|
+
harness = Rubik::Storage::Vakanuvis::Harness.new(store: store, config: QuietConfig.new, session_slug: "bench-#{driver.label}")
|
|
313
|
+
tools = ToolBench.new(store, harness)
|
|
314
|
+
runs = TASKS.map { |task| driver.run_task(task, tools) }
|
|
315
|
+
tool_errors = runs.sum { |r| r[:calls].count { |_n, _a, res| TOOL_ERROR_MARKERS.any? { |m| res.to_s.start_with?(m) } } }
|
|
316
|
+
if tool_errors.positive?
|
|
317
|
+
first_bad = runs.flat_map { |r| r[:calls] }.find { |_n, _a, res| TOOL_ERROR_MARKERS.any? { |m| res.to_s.start_with?(m) } }
|
|
318
|
+
raise "#{driver.label}: #{tool_errors} tool error(s) — the bench toolbelt itself failed; " \
|
|
319
|
+
"these numbers would read as model behavior. first: #{first_bad[2].to_s[0, 160]}"
|
|
320
|
+
end
|
|
321
|
+
harness.shutdown
|
|
322
|
+
metrics_for(driver.label, runs).tap { |m| io&.puts(" [vakanuvis-adoption] #{m.label}: #{m.summary}") }
|
|
323
|
+
rescue Rubik::LLM::RateLimitError => e
|
|
324
|
+
location = (e.backtrace || []).first
|
|
325
|
+
io&.puts(" [vakanuvis-adoption] #{driver.label}: SKIPPED (rate limited) — #{e.message} @ #{location}")
|
|
326
|
+
SkipRow.new(driver.label, e)
|
|
327
|
+
rescue Rubik::LLM::ClientError, StandardError => e
|
|
328
|
+
# Deterministik düzlem: asla satıra dönüşmesin (rake bench
|
|
329
|
+
# loud patlar). Canlı düzlem: tek bozuk sürücü kalan modellerin
|
|
330
|
+
# ölçümünü öldürmesin — dürüst ERROR satırı düşer.
|
|
331
|
+
raise unless driver.live?
|
|
332
|
+
|
|
333
|
+
location = (e.backtrace || []).first
|
|
334
|
+
io&.puts(" [vakanuvis-adoption] #{driver.label}: ERROR — #{e.class}: #{e.message} @ #{location}")
|
|
335
|
+
ErrorRow.new(driver.label, e)
|
|
336
|
+
ensure
|
|
337
|
+
begin
|
|
338
|
+
FileUtils.remove_entry(root)
|
|
339
|
+
rescue StandardError => e
|
|
340
|
+
# Windows EBUSY/ENOTEMPTY (AV tarayıcı, indeksleyici) cleanup'ı
|
|
341
|
+
# patlatabilir — cleanup hatası tabloyu ve işlenen hatayı
|
|
342
|
+
# MaskeleMEsin; tüm benchmark'ı da abort etmesin.
|
|
343
|
+
warn " [vakanuvis-adoption] cleanup failed for #{root}: #{e.class}: #{e.message}"
|
|
344
|
+
end
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# Kota/ağ ölümü yaşayan modelin tablo satırı: yok saymak değil —
|
|
350
|
+
# "ölçülemedi"yi tabloda göster (sessiz-satır-kaybı yok).
|
|
351
|
+
SkipRow = Struct.new(:label, :error) do
|
|
352
|
+
def summary = "skipped: #{error.class}: #{error.message.to_s[0, 120]}"
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# Kota DEĞİL — kimlik doğrulama, bozuk sağlayıcı yanıtı, kabuk...
|
|
356
|
+
# "skipped" yalan olur; tabloda error diye ayrışır.
|
|
357
|
+
ErrorRow = Struct.new(:label, :error) do
|
|
358
|
+
def summary = "error: #{error.class}: #{error.message.to_s[0, 120]}"
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# Bench tarafı araç hataları (ToolBench#execute'un kurtarma dizgileri):
|
|
362
|
+
# bunlar model davranışı DEĞİL — metrik borusuna girmeden yakalanır.
|
|
363
|
+
TOOL_ERROR_MARKERS = ["tool error: ", "Unknown tool "].freeze
|
|
364
|
+
|
|
365
|
+
# Metrik satırlarından runner sonuçları: hedef taban çizgisi (model
|
|
366
|
+
# puanı değil — senaryo KOŞTU mu). Tablo io'ya markdown düşer.
|
|
367
|
+
def metric_results(rows, io: $stdout)
|
|
368
|
+
io&.puts("\n#### Vakanüvis tool adoption (issue #338)\n")
|
|
369
|
+
io&.puts("| model | recall | first@ | calls | terms | kw:line | empty% | record | wrong put | answer used |")
|
|
370
|
+
io&.puts("|---|---|---|---|---|---|---|---|---|---|")
|
|
371
|
+
rows.map do |m|
|
|
372
|
+
if m.is_a?(SkipRow) || m.is_a?(ErrorRow)
|
|
373
|
+
# Ölçülemeyen model tabloda KALIR — satır kaybı sessiz değil;
|
|
374
|
+
# kota ölümü (skipped) ile gerçek hata (error) ayrışır.
|
|
375
|
+
status = m.is_a?(SkipRow) ? "skipped" : "error"
|
|
376
|
+
# 10 hücre: başlık 10 sütun — 11. hücreye düşen skip/error
|
|
377
|
+
# gerekçesi GFM tarafından SESSİZCE düşürülüyordu.
|
|
378
|
+
io&.puts("| #{m.label} | #{status} | - | - | - | - | - | - | - | (#{m.summary}) |")
|
|
379
|
+
next Rubik::Benchmark::MetricResult.new(
|
|
380
|
+
name: "Vakanüvis Tool Adoption — #{m.label}",
|
|
381
|
+
target: "baseline (issue #338)",
|
|
382
|
+
measured: m.summary,
|
|
383
|
+
passed: false
|
|
384
|
+
)
|
|
385
|
+
end
|
|
386
|
+
io&.puts("| #{m.label} | #{m.recall_called ? 'yes' : 'NO'} | #{m.first_recall_turn || '-'} | #{m.recall_calls} | " \
|
|
387
|
+
"#{m.avg_query_terms || '-'} | #{m.keywords_line_used ? 'yes' : 'no'} | #{m.recall_empty_rate || '-'}% | " \
|
|
388
|
+
"#{m.record_called ? 'vakanuvis' : 'none'} | #{m.wrong_path_puts} | #{m.answer_used_recall ? 'yes' : 'no'} |")
|
|
389
|
+
Rubik::Benchmark::MetricResult.new(
|
|
390
|
+
name: "Vakanüvis Tool Adoption — #{m.label}",
|
|
391
|
+
target: "baseline (issue #338)",
|
|
392
|
+
measured: m.summary,
|
|
393
|
+
passed: true
|
|
394
|
+
)
|
|
395
|
+
end
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
# Replay script'leri: her adım {call: [name, args]} | {call_each: [[name,
|
|
399
|
+
# args], ...]} | {answer: "..."} — ReplayDriver bunları GERÇEK
|
|
400
|
+
# araçlarla sırayla koşturur; yanıtlar profilin davranış taahhüdüdür.
|
|
401
|
+
SCRIPTS = {
|
|
402
|
+
"adopter" => {
|
|
403
|
+
"recall-tr-en" => [
|
|
404
|
+
{ call: ["vakanuvis", { "action" => "recall", "query" => "gateway retry backoff 429" }] },
|
|
405
|
+
{ answer: "Üstel geri çekilme: 2s, 4s, 8s — en fazla 5 deneme; 30 ardışık hata sonra devre kesici açılır." }
|
|
406
|
+
],
|
|
407
|
+
"durable-conclusion" => [
|
|
408
|
+
{ call: ["vakanuvis", { "action" => "record", "topic" => "digest gate dedupe",
|
|
409
|
+
"body" => "Same-target digest jobs are deduplicated; queue cap is 8 jobs, one in-flight per target.\n" \
|
|
410
|
+
"Keywords: digest dedupe gate queue target" }] },
|
|
411
|
+
{ answer: "Recorded as a memory card via the chronicler." }
|
|
412
|
+
]
|
|
413
|
+
},
|
|
414
|
+
"wrong-path" => {
|
|
415
|
+
"recall-tr-en" => [
|
|
416
|
+
{ answer: "Bilmiyorum; geçidi yeniden dene ve sonra tekrar sor." }
|
|
417
|
+
],
|
|
418
|
+
"durable-conclusion" => [
|
|
419
|
+
{ call_each: [["storage", { "action" => "put", "type" => "memory", "title" => "digest dedupe",
|
|
420
|
+
"body" => "Same-target digest jobs are deduplicated." }]] },
|
|
421
|
+
{ answer: "Persisted directly to storage." }
|
|
422
|
+
]
|
|
423
|
+
},
|
|
424
|
+
"struggler" => {
|
|
425
|
+
"recall-tr-en" => [
|
|
426
|
+
{ call: ["vakanuvis", { "action" => "recall", "query" => "gateway hangi durumlarda üstel bekleme uygular ve devre kesici kaç hata sonra açılır" }] },
|
|
427
|
+
{ call: ["vakanuvis", { "action" => "recall", "query" => "gateway retry backoff" }] },
|
|
428
|
+
{ answer: "2s, 4s, 8s üstel geri çekilme uygulanır (en fazla 5 deneme)." }
|
|
429
|
+
],
|
|
430
|
+
"durable-conclusion" => [
|
|
431
|
+
{ call: ["vakanuvis", { "action" => "record", "topic" => "digest gate dedupe",
|
|
432
|
+
"body" => "Same-target digest jobs are deduplicated." }] },
|
|
433
|
+
{ answer: "Saved via vakanuvis record." }
|
|
434
|
+
]
|
|
435
|
+
}
|
|
436
|
+
}.freeze
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
end
|
|
@@ -45,6 +45,12 @@ module Rubik
|
|
|
45
45
|
"max_output_lines" => 300,
|
|
46
46
|
"truncation_strategy" => "head_tail",
|
|
47
47
|
"timeout_seconds" => 30,
|
|
48
|
+
# Dispatcher-level call budget in seconds; nil = off (inline execution,
|
|
49
|
+
# current behavior). RUBIK_TOOLS_CALL_TIMEOUT env overrides the file.
|
|
50
|
+
# A timed-out call is abandoned, not killed — bounds turn latency.
|
|
51
|
+
"call_timeout" => nil,
|
|
52
|
+
# Canonical tool name => seconds; 0/false disables for that tool.
|
|
53
|
+
"call_timeout_overrides" => {},
|
|
48
54
|
"native" => "auto"
|
|
49
55
|
},
|
|
50
56
|
"router" => { "mode" => "auto", "sticky" => true },
|
|
@@ -104,6 +110,8 @@ module Rubik
|
|
|
104
110
|
"tool_chronicle_max_entries" => 15, # Plan 85 D-2/D-3
|
|
105
111
|
"merge_turn_tool_history" => true, # Plan 85 D-7
|
|
106
112
|
"merge_turn_tool_max_entries" => 20, # Plan 85 D-7
|
|
113
|
+
"huge_root_notice" => true, # one guidance notice for likely-huge session roots (home dir, OS data)
|
|
114
|
+
"boot_scan_on_huge_root" => false, # false skips the boot warm walk on strong huge-root signals; !context scan opts in
|
|
107
115
|
"emergency_truncate_strategy" => "head_tail" # EmergencyCompactor hard-truncate layer: "head_tail" | "head_only" | "tail_only"
|
|
108
116
|
},
|
|
109
117
|
"ui" => {
|