mimic-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/NOTICE +16 -0
  4. data/README.md +175 -0
  5. data/exe/lllm +5 -0
  6. data/exe/llm +5 -0
  7. data/exe/mimic +5 -0
  8. data/lib/mimic/README.md +144 -0
  9. data/lib/mimic/agent/claude.rb +305 -0
  10. data/lib/mimic/agent/codex.rb +254 -0
  11. data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
  12. data/lib/mimic/agent/hermes.rb +107 -0
  13. data/lib/mimic/agent/muse.rb +106 -0
  14. data/lib/mimic/agent.rb +46 -0
  15. data/lib/mimic/cli.rb +159 -0
  16. data/lib/mimic/client/base.rb +44 -0
  17. data/lib/mimic/client/crush.rb +28 -0
  18. data/lib/mimic/client/goose.rb +52 -0
  19. data/lib/mimic/client/hermes.rb +21 -0
  20. data/lib/mimic/client/opencode.rb +29 -0
  21. data/lib/mimic/client/pi.rb +50 -0
  22. data/lib/mimic/client.rb +19 -0
  23. data/lib/mimic/engine/README.md +140 -0
  24. data/lib/mimic/engine/base.rb +138 -0
  25. data/lib/mimic/engine/chat_template.rb +207 -0
  26. data/lib/mimic/engine/ik_llama.rb +273 -0
  27. data/lib/mimic/engine/launcher.rb +25 -0
  28. data/lib/mimic/engine/llama.rb +124 -0
  29. data/lib/mimic/engine/llama_embed.rb +108 -0
  30. data/lib/mimic/engine/router.rb +57 -0
  31. data/lib/mimic/engine/server_process.rb +108 -0
  32. data/lib/mimic/engine.rb +5 -0
  33. data/lib/mimic/engine_cli/binary_options.rb +46 -0
  34. data/lib/mimic/engine_cli/builder.rb +98 -0
  35. data/lib/mimic/engine_cli/diffusion.rb +117 -0
  36. data/lib/mimic/engine_cli/llama_preset.rb +119 -0
  37. data/lib/mimic/engine_cli/registry.rb +94 -0
  38. data/lib/mimic/engine_cli/source.rb +110 -0
  39. data/lib/mimic/engine_cli/status.rb +161 -0
  40. data/lib/mimic/engine_cli/update.rb +68 -0
  41. data/lib/mimic/engine_cli.rb +7 -0
  42. data/lib/mimic/health/checks.rb +92 -0
  43. data/lib/mimic/health.rb +82 -0
  44. data/lib/mimic/llm_cli/base.rb +36 -0
  45. data/lib/mimic/llm_cli/configure.rb +84 -0
  46. data/lib/mimic/llm_cli/embed.rb +80 -0
  47. data/lib/mimic/llm_cli/engines.rb +44 -0
  48. data/lib/mimic/llm_cli/models.rb +31 -0
  49. data/lib/mimic/llm_cli/models_table.rb +119 -0
  50. data/lib/mimic/llm_cli/paths.rb +48 -0
  51. data/lib/mimic/llm_cli/root.rb +215 -0
  52. data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
  53. data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
  54. data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
  55. data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
  56. data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
  57. data/lib/mimic/llm_cli.rb +5 -0
  58. data/lib/mimic/llm_config/embed.rb +48 -0
  59. data/lib/mimic/llm_config/llama_settings.rb +60 -0
  60. data/lib/mimic/llm_config/model_settings.rb +84 -0
  61. data/lib/mimic/llm_config/sampling.rb +54 -0
  62. data/lib/mimic/llm_config/settings.rb +92 -0
  63. data/lib/mimic/llm_config.rb +132 -0
  64. data/lib/mimic/model/footprint.rb +303 -0
  65. data/lib/mimic/model/gguf.rb +204 -0
  66. data/lib/mimic/model/header.rb +169 -0
  67. data/lib/mimic/model/library.rb +132 -0
  68. data/lib/mimic/model/memory.rb +126 -0
  69. data/lib/mimic/model/store.rb +46 -0
  70. data/lib/mimic/model/upstream.rb +81 -0
  71. data/lib/mimic/model.rb +116 -0
  72. data/lib/mimic/paths.rb +45 -0
  73. data/lib/mimic/result.rb +8 -0
  74. data/lib/mimic/ui.rb +89 -0
  75. data/lib/mimic/util/fmt.rb +15 -0
  76. data/lib/mimic/util/hf.rb +55 -0
  77. data/lib/mimic/util/host.rb +95 -0
  78. data/lib/mimic/util/probes.rb +255 -0
  79. data/lib/mimic/version.rb +3 -0
  80. data/lib/mimic.rb +22 -0
  81. metadata +186 -0
@@ -0,0 +1,84 @@
1
+ require_relative "../llm_config"
2
+ require_relative "sampling"
3
+
4
+ module Mimic
5
+ module LlmConfig
6
+ # The model section: which GGUF is served and the knobs that belong to it
7
+ # rather than to the server. `file` is the Model behind the configured
8
+ # path -- rebuilt whenever the path changes, so nothing holds a Model for
9
+ # a GGUF that is no longer selected.
10
+ class ModelSettings
11
+ SECTION = "model".freeze
12
+
13
+ def initialize(cfg)
14
+ @cfg = cfg
15
+ end
16
+
17
+ def path = section["path"]
18
+ def context_length = section["context_length"]
19
+ def mtp = section.fetch("mtp", "auto")
20
+ def name = file.model_alias
21
+ # 'auto' rather than 'on': the engine detects thinking support from the
22
+ # chat template, so a model with no thinking mode is not forced into one
23
+ # it cannot render.
24
+ def reasoning = section.fetch("reasoning", "auto")
25
+
26
+ # The level picked in `llm setup`, or nil -- which leaves the chat
27
+ # template's own default in charge. Thinking off has no level.
28
+ def reasoning_effort
29
+ return nil if reasoning == "off"
30
+
31
+ @pending_effort_set ? @pending_effort : file.memory.effort
32
+ end
33
+
34
+ def sampling = Sampling.new(file.memory.sampling_for(reasoning: reasoning))
35
+
36
+ def file
37
+ @file = Model.new(path) if @file.nil? || @file.path.to_s != path
38
+ @file
39
+ end
40
+
41
+ # A remembered projector can be deleted or moved between runs, and
42
+ # llama-server refuses to boot on a missing --mmproj, so an entry that
43
+ # no longer resolves counts as no entry.
44
+ def mmproj_path
45
+ stored = @pending_mmproj_set ? @pending_mmproj : file.memory.mmproj
46
+ stored if stored && File.exist?(stored)
47
+ end
48
+
49
+ def path=(value)
50
+ writable["path"] = value
51
+ end
52
+
53
+ def context_length=(value)
54
+ writable["context_length"] = value
55
+ end
56
+
57
+ def reasoning=(value)
58
+ writable["reasoning"] = value
59
+ end
60
+
61
+ def mmproj_path=(value)
62
+ @pending_mmproj_set = true
63
+ @pending_mmproj = value
64
+ end
65
+
66
+ def reasoning_effort=(value)
67
+ @pending_effort_set = true
68
+ @pending_effort = value
69
+ end
70
+
71
+ def save
72
+ file.memory.mmproj = @pending_mmproj if @pending_mmproj_set
73
+ file.memory.effort = @pending_effort if @pending_effort_set
74
+ end
75
+
76
+ private
77
+
78
+ # Reads see an absent section as an empty one; only a write materialises
79
+ # it, so a hand-built config keeps the sections it actually carries.
80
+ def section = @cfg[SECTION] || {}
81
+ def writable = @cfg[SECTION] ||= {}
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,54 @@
1
+ module Mimic
2
+ module LlmConfig
3
+ class Sampling
4
+ FLAGS = {
5
+ "temperature" => "temp",
6
+ "top_p" => "top-p",
7
+ "top_k" => "top-k",
8
+ "min_p" => "min-p",
9
+ "presence_penalty" => "presence-penalty",
10
+ "repeat_penalty" => "repeat-penalty",
11
+ }.freeze
12
+ KEYS = FLAGS.keys.freeze
13
+
14
+ # llama-server's own defaults, so a key nothing overrides describes the
15
+ # same behaviour as passing no flag at all.
16
+ DEFAULTS = {
17
+ "temperature" => 0.8,
18
+ "top_p" => 0.95,
19
+ "top_k" => 40,
20
+ "min_p" => 0.05,
21
+ "presence_penalty" => 0.0,
22
+ "repeat_penalty" => 1.0,
23
+ }.freeze
24
+
25
+ # The publisher's own suggestion, carried in the GGUF header. Only
26
+ # these three are ever written there -- min_p and the two penalties
27
+ # appear in no GGUF and stay at the default.
28
+ HEADER_KEYS = {
29
+ "general.sampling.temp" => "temperature",
30
+ "general.sampling.top_p" => "top_p",
31
+ "general.sampling.top_k" => "top_k",
32
+ }.freeze
33
+
34
+ # GGUF stores these as float32, so a published 0.95 reads back as
35
+ # 0.949999988079071 and would reach the engine that way.
36
+ FLOAT_PRECISION = 6
37
+
38
+ def self.resolve(metadata)
39
+ suggested = HEADER_KEYS.filter_map do |header_key, key|
40
+ value = metadata[header_key]
41
+ [key, value.is_a?(Float) ? value.round(FLOAT_PRECISION) : value] unless value.nil?
42
+ end
43
+ DEFAULTS.merge(suggested.to_h)
44
+ end
45
+
46
+ def initialize(model_cfg)
47
+ @cfg = model_cfg
48
+ end
49
+
50
+ def to_h = FLAGS.to_h { |key, flag| [flag, @cfg[key]] }
51
+ def to_args = to_h.flat_map { |flag, value| ["--#{flag}", value.to_s] }
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,92 @@
1
+ require "psych"
2
+ require "pathname"
3
+ require_relative "../llm_config"
4
+ require_relative "model_settings"
5
+ require_relative "llama_settings"
6
+ require_relative "embed"
7
+
8
+ module Mimic
9
+ module LlmConfig
10
+ # The directories scanned for GGUF files: the builtin roots plus whatever
11
+ # `llm path add` has stored. Enumerating it yields every directory in
12
+ # scan order; add/remove report what they did as a symbol so the CLI can
13
+ # tell "already there" apart from "not yours to remove".
14
+ class SearchPaths
15
+ include Enumerable
16
+
17
+ def initialize(cfg)
18
+ @cfg = cfg
19
+ end
20
+
21
+ def each(&) = all.each(&)
22
+
23
+ def all
24
+ (BUILTIN_MODEL_DIRS + configured.map { |dir| Pathname.new(dir.to_s).expand_path }).uniq
25
+ end
26
+
27
+ def add(dir)
28
+ wanted = Pathname.new(dir.to_s).expand_path
29
+ return :builtin if BUILTIN_MODEL_DIRS.include?(wanted)
30
+ return :duplicate if configured.map { |p| Pathname.new(p).expand_path }.include?(wanted)
31
+
32
+ store(configured + [wanted.to_s])
33
+ :added
34
+ end
35
+
36
+ def remove(dir)
37
+ wanted = Pathname.new(dir.to_s).expand_path
38
+ kept = configured.reject { |p| Pathname.new(p).expand_path == wanted }
39
+ return BUILTIN_MODEL_DIRS.include?(wanted) ? :builtin : :missing if kept.size == configured.size
40
+
41
+ store(kept)
42
+ :removed
43
+ end
44
+
45
+ private
46
+
47
+ def configured = @cfg.dig("model", "search_paths") || []
48
+
49
+ def store(paths)
50
+ (@cfg["model"] ||= {})["search_paths"] = paths
51
+ end
52
+ end
53
+
54
+ class Settings
55
+ QUANT_MARKERS = %w[-Q4 -Q5 -Q6 -Q8 -IQ -F16 -BF16].freeze
56
+
57
+ def self.alias_for(path)
58
+ stem = Pathname.new(path).basename(".*").to_s
59
+ marker = QUANT_MARKERS.find { |m| stem.include?(m) }
60
+ marker ? stem.split(marker).first : stem
61
+ end
62
+
63
+ def initialize(cfg)
64
+ @cfg = cfg
65
+ end
66
+
67
+ def model = @model ||= ModelSettings.new(@cfg)
68
+ def llama = @llama ||= LlamaSettings.new(@cfg)
69
+
70
+ def save
71
+ LLM_CONFIG_DIR.mkpath
72
+ model.save
73
+ LLM_CONFIG_PATH.write(Psych.dump(@cfg.slice("model", "llama", "embed")))
74
+ end
75
+
76
+ def remember_ctx
77
+ model.file.memory.remember_ctx(model.context_length, llama.cache_type)
78
+ end
79
+
80
+ def remember_sampling
81
+ model.file.memory.remember_sampling(model.reasoning, model.file.memory.sampling_for(reasoning: model.reasoning))
82
+ end
83
+
84
+ def forget_sampling = model.file.memory.forget_sampling
85
+
86
+ def search_paths = SearchPaths.new(@cfg)
87
+ def embed = @embed ||= Embed.new(@cfg["embed"] ||= {}, host: llama.host)
88
+
89
+ def to_h = @cfg
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,132 @@
1
+ require "date"
2
+ require "psych"
3
+ require "json"
4
+ require "pathname"
5
+ require_relative "paths"
6
+ require_relative "model"
7
+ require_relative "llm_config/sampling"
8
+ require_relative "llm_config/settings"
9
+
10
+ # LLM engine config -- model + llama keys, separate from mimic service
11
+ # config. Owned by the `llm` CLI; the main `mimic` config reads from it for
12
+ # base URL, model alias, and context length (needed on every agent launch).
13
+ module Mimic
14
+ module LlmConfig
15
+ LLM_CONFIG_DIR = Paths::LLM_ROOT
16
+ LLM_CONFIG_PATH = LLM_CONFIG_DIR / "config.yaml"
17
+ MODEL_CTX_PATH = LLM_CONFIG_DIR / "model-ctx.json"
18
+ MODEL_MMPROJ_PATH = LLM_CONFIG_DIR / "model-mmproj.json"
19
+ MODEL_COMPUTE_PATH = LLM_CONFIG_DIR / "model-compute.json"
20
+ MODEL_SAMPLING_PATH = LLM_CONFIG_DIR / "model-sampling.json"
21
+ MODEL_EFFORT_PATH = LLM_CONFIG_DIR / "model-effort.json"
22
+ LEGACY_CONFIG = Paths::STATE_DIR / "llm_config.yaml"
23
+
24
+ BUILTIN_MODEL_DIRS = [
25
+ Pathname.new(Dir.home) / "hf-models",
26
+ LLM_CONFIG_DIR / "models",
27
+ Pathname.new(Dir.home) / ".lmstudio/models",
28
+ ].freeze
29
+
30
+ DEFAULTS = {
31
+ "model" => {
32
+ "path" => "",
33
+ "search_paths" => [],
34
+ "context_length" => 122_880,
35
+ "mtp" => "auto",
36
+ "reasoning" => "auto",
37
+ },
38
+ "llama" => {
39
+ "engine" => "llama.cpp",
40
+ "host" => "0.0.0.0",
41
+ "port" => 8080,
42
+ "flash_attn" => "on",
43
+ "cache_type" => "q8_0",
44
+ "n_slots" => 1,
45
+ "binary" => {},
46
+ },
47
+ "embed" => {
48
+ "path" => "",
49
+ "port" => 8081,
50
+ "context_length" => 0,
51
+ "batch_size" => 8192,
52
+ "mode" => "auto",
53
+ },
54
+ }.freeze
55
+
56
+ class << self
57
+ def merge(base, override)
58
+ out = Marshal.load(Marshal.dump(base))
59
+ override.each do |key, value|
60
+ out[key] = value.is_a?(Hash) && out[key].is_a?(Hash) ? merge(out[key], value) : value
61
+ end
62
+ out
63
+ end
64
+
65
+ def load
66
+ migrate_legacy
67
+ return Settings.new(merge(DEFAULTS, {})) unless LLM_CONFIG_PATH.exist?
68
+
69
+ data = Psych.safe_load(LLM_CONFIG_PATH.read, permitted_classes: [Symbol, Date, Time], aliases: true) || {}
70
+ migrate_stores(Settings.new(merge(DEFAULTS, data)))
71
+ end
72
+
73
+ def exists?
74
+ migrate_legacy
75
+ LLM_CONFIG_PATH.exist?
76
+ end
77
+
78
+ # ~/hf-models is a tree of per-file symlinks into their .lmstudio
79
+ # targets, and this store has to be keyed by the resolved target so a
80
+ # model reached through a different symlink still matches. expand_path
81
+ # alone leaves the symlink unresolved and splits the store into two
82
+ # disagreeing key sets; realdirpath covers the non-strict case for a
83
+ # path whose file is gone.
84
+ def resolve_path(path)
85
+ expanded = Pathname.new(path).expand_path
86
+ return expanded.realpath.to_s if expanded.exist?
87
+
88
+ expanded.realdirpath.to_s
89
+ rescue SystemCallError
90
+ expanded.to_s
91
+ end
92
+
93
+ private
94
+
95
+ # Relocate the engine config left behind when it lived under STATE_DIR.
96
+ def migrate_legacy
97
+ return if LLM_CONFIG_PATH.exist? || !LEGACY_CONFIG.file?
98
+
99
+ LLM_CONFIG_DIR.mkpath
100
+ LEGACY_CONFIG.rename(LLM_CONFIG_PATH)
101
+ end
102
+
103
+ # Relocate the fields that used to sit flat in config.yaml:
104
+ # model.mmproj_path (now per model, in Model::Memory) and a plain
105
+ # llama.binary string (now per engine). Both are relocated with the
106
+ # attribution they had when they were written -- the projector to the
107
+ # model that is configured, the binary to the engine that is configured
108
+ # -- and the old keys are dropped so nothing reads them again. The
109
+ # sampler keys are dropped outright: they resolve from the chosen
110
+ # model's own header now, so a stale hand-edited value would only
111
+ # contradict it. model.thinking was a boolean translated to the engines'
112
+ # reasoning value at read time; the value itself is stored now, so the
113
+ # translation happens once here instead.
114
+ def migrate_stores(settings)
115
+ cfg = settings.to_h
116
+ legacy_binary = cfg["llama"]["binary"] if cfg["llama"]["binary"].is_a?(String)
117
+ legacy_sampling = Sampling::KEYS.any? { |key| cfg["model"].key?(key) }
118
+ legacy_thinking = cfg["model"].key?("thinking")
119
+ return settings unless cfg["model"].key?("mmproj_path") || legacy_binary ||
120
+ legacy_sampling || legacy_thinking
121
+
122
+ projector = cfg["model"].delete("mmproj_path").to_s
123
+ settings.model.mmproj_path = projector unless projector.empty?
124
+ settings.llama.binary = legacy_binary if legacy_binary
125
+ settings.model.reasoning = cfg["model"].delete("thinking") ? "auto" : "off" if legacy_thinking
126
+ Sampling::KEYS.each { |key| cfg["model"].delete(key) }
127
+ settings.save
128
+ settings
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,303 @@
1
+ require_relative "header"
2
+ require_relative "../util/host"
3
+
4
+ # What one model at one context costs on the GPU -- weights + KV + compute
5
+ # buffer against the card's budget, so setup can reject a context length that
6
+ # would load but crash at runtime once the KV cache fills (llama.cpp
7
+ # over-commits ctx).
8
+ module Mimic
9
+ class Model
10
+ class Footprint
11
+ # bytes per element for KV cache quant types (block_bytes / block_elems)
12
+ CACHE_BPE = {
13
+ "f32" => 4.0,
14
+ "f16" => 2.0,
15
+ "bf16" => 2.0,
16
+ "q8_0" => 34 / 32.0,
17
+ "q5_1" => 24 / 32.0,
18
+ "q5_0" => 22 / 32.0,
19
+ "q4_1" => 20 / 32.0,
20
+ "q4_0" => 18 / 32.0,
21
+ }.freeze
22
+
23
+ # The compute buffer tracks ubatch, vocab and graph shape -- not card
24
+ # size -- so these are a guard, not a model. They scale with the card
25
+ # because a fixed 2 GiB is 8% of a 3090 but 17% of a 12 GiB 3060, and a
26
+ # smaller card runs the smaller ubatch/ctx that make the real buffer
27
+ # smaller anyway. ComputeStore replaces the guess after the first launch.
28
+ # A sliding-window layer's cache does not grow with context: llama.cpp
29
+ # sizes it at the window plus one ubatch, padded to 256 cells
30
+ # (llama-kv-cache-iswa.cpp:73). Both engine presets set
31
+ # --ubatch-size 1024, so that is what the window cache is priced at.
32
+ SWA_UBATCH = 1024
33
+ SWA_CELL_PAD = 256
34
+
35
+ # The recurrent state is f32 whatever --cache-type-k/v says.
36
+ RECURRENT_BPE = 4.0
37
+
38
+ COMPUTE_FRACTION = 0.06
39
+ COMPUTE_FLOOR_MIB = 1024
40
+ COMPUTE_CEILING_MIB = 2048
41
+ DESKTOP_RESERVE_MIB = 2048 # leave this much VRAM for the desktop/other apps
42
+
43
+ def self.compute_default(total_mib)
44
+ (total_mib * COMPUTE_FRACTION).round.clamp(COMPUTE_FLOOR_MIB, COMPUTE_CEILING_MIB)
45
+ end
46
+
47
+ def initialize(header, ctx:, cache_type:, weights_bytes:, compute_mib: nil, drafting: false, slots: 1,
48
+ reclaimable_mib: 0)
49
+ @header = header
50
+ @ctx = ctx
51
+ @cache_type = cache_type
52
+ @weights_bytes = weights_bytes
53
+ @compute_mib = compute_mib
54
+ @drafting = drafting
55
+ @slots = slots
56
+ @reclaimable_mib = reclaimable_mib
57
+ end
58
+
59
+ # False when the header carries no KV shape or no GPU answers -- every
60
+ # reader below is nil then, and callers print nothing rather than a
61
+ # number they made up.
62
+ def measurable? = !kv_per_token_mib.nil? && !total_mib.nil? && !@weights_bytes.nil?
63
+
64
+ def total_mib
65
+ return @total_mib if defined?(@total_mib)
66
+
67
+ @total_mib = Util::Host.gpu_total_mib
68
+ end
69
+
70
+ def weights_mib = @weights_bytes && @weights_bytes / 1024.0 / 1024.0
71
+ # Window layers are the fixed half: they hold the same cells at ctx 4096
72
+ # as at 262144, so only the full-attention layers scale.
73
+ def kv_mib = kv_per_token_mib && kv_per_token_mib * @ctx + kv_fixed_mib
74
+ def compute_mib = @compute_mib || (total_mib && self.class.compute_default(total_mib))
75
+
76
+ # A drafting engine builds a second context over the nextn block: its
77
+ # own KV for those layers plus a compute buffer the same size as the
78
+ # target's, since it runs the same graph shape at the same ubatch.
79
+ # Measured on Qwen3.6-35B-A3B at ctx 163840 (2026-08-23): 170 MiB KV
80
+ # and a second 978 MiB CUDA0 compute buffer beside the target's 978.
81
+ def draft_mib
82
+ return 0 unless @drafting && kv_per_token_mib
83
+
84
+ layers = target_attention_layers
85
+ return compute_mib if layers.nil? || layers.zero?
86
+
87
+ ((kv_per_token_mib * @ctx) / layers * @header.nextn_layers) + compute_mib
88
+ end
89
+ # What is actually available: the driver's free figure when it answers,
90
+ # since the desktop's draw is measurable and moves. DESKTOP_RESERVE_MIB
91
+ # is the fallback for a host where nvidia-smi cannot be read.
92
+ #
93
+ # `reclaimable_mib` is VRAM held by a server this launch replaces, which
94
+ # the free figure counts as unavailable even though it is about to be
95
+ # released. Other processes' VRAM stays subtracted -- only our own
96
+ # outgoing server comes back. The fallback path has no free figure to
97
+ # add to: the reserve is already the whole allowance.
98
+ def budget_mib
99
+ return @budget_mib if defined?(@budget_mib)
100
+
101
+ free = Util::Host.gpu_free_mib
102
+ @budget_mib = free ? free + @reclaimable_mib : (total_mib && total_mib - DESKTOP_RESERVE_MIB)
103
+ end
104
+
105
+ def need_mib
106
+ return nil unless measurable?
107
+
108
+ weights_mib + kv_mib + compute_mib + draft_mib
109
+ end
110
+
111
+ def fits? = measurable? && need_mib <= budget_mib
112
+
113
+ # What is left of the budget once this configuration is resident.
114
+ def headroom_mib = measurable? ? budget_mib - need_mib : nil
115
+
116
+ # The longest context the card can hold at this cache type, weights and
117
+ # compute buffer.
118
+ def max_ctx
119
+ return nil unless measurable?
120
+ return 0 if kv_per_token_mib.zero?
121
+
122
+ [0, ((budget_mib - weights_mib - compute_mib - draft_mib - kv_fixed_mib) / kv_per_token_mib).to_i].max
123
+ end
124
+
125
+ # How many layers' experts have to move to CPU to cover the deficit.
126
+ # 0 when nothing needs moving, nil when the model is dense (nothing to
127
+ # move) or the ceiling can't cover the gap.
128
+ def layers_to_offload(ceiling:)
129
+ return nil unless measurable?
130
+ return 0 if need_mib <= budget_mib
131
+
132
+ per_layer = @header.expert_bytes_per_layer
133
+ return nil if per_layer.nil? || per_layer.zero?
134
+
135
+ layers = ((need_mib - budget_mib) / (per_layer / 1024.0 / 1024.0)).ceil
136
+ layers <= ceiling ? layers : nil
137
+ end
138
+
139
+ # What a loaded instance holds beyond this estimate, in MiB, or nil when
140
+ # it can't be measured. The residual is what compute_default stands in
141
+ # for, so it is worth storing under the exact ctx and cache type it was
142
+ # taken at.
143
+ def measured_overhead_mib(pids:)
144
+ used = Util::Host.gpu_process_mib(pids)
145
+ return nil if !measurable? || used.nil?
146
+
147
+ overhead = used - need_mib
148
+ overhead.positive? ? overhead.round : nil
149
+ end
150
+
151
+ private
152
+
153
+ # The attention-layer count kv_mib was built from, so draft_mib can
154
+ # price one nextn block as a fraction of it.
155
+ def target_attention_layers
156
+ meta = @header.metadata
157
+ arch = meta["general.architecture"]
158
+ arch && attention_layers(meta, arch)
159
+ end
160
+
161
+ def kv_per_token_mib = kv_shape && mib(kv_shape.first)
162
+ def kv_fixed_mib = kv_shape ? mib(kv_shape.last) + recurrent_mib : 0
163
+
164
+ def mib(elems) = elems * CACHE_BPE.fetch(@cache_type, 2.0) / 1024 / 1024
165
+
166
+ # [elements per token, elements held whatever the context is]. Layers
167
+ # are walked one at a time because a model can mix widths: gemma-4
168
+ # gives its window layers 16 KV heads at head dim 256 and its
169
+ # full-attention layers 4 at 512, in a repeating pattern the header
170
+ # spells out per layer.
171
+ def kv_shape
172
+ return @kv_shape if defined?(@kv_shape)
173
+
174
+ @kv_shape = read_kv_shape
175
+ end
176
+
177
+ def read_kv_shape
178
+ meta = @header.metadata
179
+ arch = meta["general.architecture"]
180
+ return nil unless arch
181
+
182
+ n_layer = attention_layers(meta, arch)
183
+ k_dim = meta["#{arch}.attention.key_length"] || derived_head_dim(meta, arch)
184
+ # K and V are not always the same width -- gemma-4 ships both keys
185
+ # separately, and reading only key_length silently prices V wrong.
186
+ v_dim = meta["#{arch}.attention.value_length"] || k_dim
187
+ heads = heads_kv(meta, arch)
188
+ return nil unless n_layer && heads && k_dim && v_dim
189
+
190
+ swa = swa_pattern(meta, arch)
191
+ swa_dims = (meta["#{arch}.attention.key_length_swa"] || k_dim) +
192
+ (meta["#{arch}.attention.value_length_swa"] || v_dim)
193
+ cells = swa_cells(meta, arch)
194
+ per_token = 0
195
+ fixed = 0
196
+ cached_layers(meta, arch, n_layer).times do |il|
197
+ n_kv = heads.is_a?(Array) ? heads[il] : heads
198
+ next if n_kv.nil?
199
+
200
+ if swa[il]
201
+ fixed += n_kv * swa_dims * cells
202
+ else
203
+ per_token += n_kv * (k_dim + v_dim)
204
+ end
205
+ end
206
+ [per_token, fixed]
207
+ end
208
+
209
+ # Per layer in the newer headers, one number in the older ones. Absent
210
+ # means multi-head attention: llama.cpp defaults it to head_count
211
+ # (llama-model.cpp:1186).
212
+ def heads_kv(meta, arch)
213
+ meta["#{arch}.attention.head_count_kv"] || meta["#{arch}.attention.head_count"]
214
+ end
215
+
216
+ # Only the explicit per-layer pattern is honoured. Architectures whose
217
+ # window layout is a rule in llama.cpp rather than a key in the file
218
+ # (gemma-3's set_swa_pattern) read as all-full-attention here, which
219
+ # over-states their KV rather than inventing a layout.
220
+ def swa_pattern(meta, arch)
221
+ pattern = meta["#{arch}.attention.sliding_window_pattern"]
222
+ pattern.is_a?(Array) ? pattern.map { |flag| !flag.zero? } : []
223
+ end
224
+
225
+ def swa_cells(meta, arch)
226
+ window = meta["#{arch}.attention.sliding_window"].to_i
227
+ cells = [@ctx, window * @slots + SWA_UBATCH].min
228
+ -(-cells / SWA_CELL_PAD) * SWA_CELL_PAD
229
+ end
230
+
231
+ # Layers past this reuse an earlier layer's cache instead of holding
232
+ # their own (llama-model.cpp:2357). shared_kv_layers is how many of the
233
+ # stack does so.
234
+ def cached_layers(meta, arch, n_layer)
235
+ n_layer - meta["#{arch}.attention.shared_kv_layers"].to_i
236
+ end
237
+
238
+ # Hybrid attention (Qwen3.5/Next-style): only every Nth layer keeps a
239
+ # standard KV cache that scales with context -- the rest are
240
+ # linear/recurrent-attention layers with a small fixed-size state.
241
+ # block_count includes the MTP draft heads, which are not part of the
242
+ # target's layer stack -- counting them here inflates the KV estimate by
243
+ # a whole attention layer on any MTP-capable GGUF. draft_mib adds them
244
+ # back when the engine is drafting.
245
+ def attention_layers(meta, arch)
246
+ n_layer = meta["#{arch}.block_count"]
247
+ interval = meta["#{arch}.full_attention_interval"]
248
+ return n_layer unless n_layer && interval
249
+
250
+ -(-(n_layer - @header.nextn_layers) / interval) # ceil div
251
+ end
252
+
253
+ # A hybrid model's linear-attention layers hold an f32 recurrent state
254
+ # instead of a K/V pair. It never scales with context but it is real
255
+ # VRAM, and neither engine reports it as KV: ik folds it into the KV
256
+ # buffer while printing only K and V in `KV self size`, and llama.cpp
257
+ # allocates it as a separate recurrent buffer. Verified against ik on
258
+ # Qwen3.6-35B-A3B (62.81 MiB over 30 layers) and Qwen3.8-27B
259
+ # (149.63 over 48).
260
+ def recurrent_mib
261
+ meta = @header.metadata
262
+ arch = meta["general.architecture"]
263
+ state = recurrent_state_elems(meta, arch)
264
+ return 0 if state.zero?
265
+
266
+ layers = recurrent_layers(meta, arch)
267
+ layers * state * @slots * RECURRENT_BPE / 1024 / 1024
268
+ end
269
+
270
+ # The layers that are not full-attention ones: llama.cpp marks every
271
+ # layer whose 1-based index is not a multiple of the interval
272
+ # (llama-hparams.cpp:603), and the nextn heads are never recurrent.
273
+ def recurrent_layers(meta, arch)
274
+ n_layer = meta["#{arch}.block_count"]
275
+ return 0 unless n_layer && meta["#{arch}.full_attention_interval"]
276
+
277
+ (n_layer - @header.nextn_layers) - attention_layers(meta, arch)
278
+ end
279
+
280
+ # The gated-delta-net state: a convolution window over 2 key groups plus
281
+ # the value stream, and one square matrix per value head
282
+ # (llama-hparams.h:329). Only this layout is priced -- a plain Mamba
283
+ # state lands in the K/V caches instead and is already counted there.
284
+ def recurrent_state_elems(meta, arch)
285
+ groups = meta["#{arch}.ssm.group_count"].to_i
286
+ inner = meta["#{arch}.ssm.inner_size"].to_i
287
+ heads = meta["#{arch}.ssm.time_step_rank"].to_i
288
+ return 0 if groups.zero? || inner.zero? || heads.zero?
289
+
290
+ key_dim = meta["#{arch}.ssm.state_size"].to_i * groups
291
+ conv = [meta["#{arch}.ssm.conv_kernel"].to_i - 1, 0].max
292
+ head_dim = inner / heads
293
+ conv * (2 * key_dim + inner) + head_dim * head_dim * heads
294
+ end
295
+
296
+ def derived_head_dim(meta, arch)
297
+ emb = meta["#{arch}.embedding_length"]
298
+ n_head = meta["#{arch}.attention.head_count"]
299
+ emb && n_head && !n_head.zero? ? emb / n_head : nil
300
+ end
301
+ end
302
+ end
303
+ end