mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
require_relative "../llm_config"
|
|
2
|
+
require_relative "sampling"
|
|
3
|
+
|
|
4
|
+
module Mimic
|
|
5
|
+
module LlmConfig
|
|
6
|
+
# The model section: which GGUF is served and the knobs that belong to it
|
|
7
|
+
# rather than to the server. `file` is the Model behind the configured
|
|
8
|
+
# path -- rebuilt whenever the path changes, so nothing holds a Model for
|
|
9
|
+
# a GGUF that is no longer selected.
|
|
10
|
+
class ModelSettings
|
|
11
|
+
SECTION = "model".freeze
|
|
12
|
+
|
|
13
|
+
def initialize(cfg)
|
|
14
|
+
@cfg = cfg
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def path = section["path"]
|
|
18
|
+
def context_length = section["context_length"]
|
|
19
|
+
def mtp = section.fetch("mtp", "auto")
|
|
20
|
+
def name = file.model_alias
|
|
21
|
+
# 'auto' rather than 'on': the engine detects thinking support from the
|
|
22
|
+
# chat template, so a model with no thinking mode is not forced into one
|
|
23
|
+
# it cannot render.
|
|
24
|
+
def reasoning = section.fetch("reasoning", "auto")
|
|
25
|
+
|
|
26
|
+
# The level picked in `llm setup`, or nil -- which leaves the chat
|
|
27
|
+
# template's own default in charge. Thinking off has no level.
|
|
28
|
+
def reasoning_effort
|
|
29
|
+
return nil if reasoning == "off"
|
|
30
|
+
|
|
31
|
+
@pending_effort_set ? @pending_effort : file.memory.effort
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def sampling = Sampling.new(file.memory.sampling_for(reasoning: reasoning))
|
|
35
|
+
|
|
36
|
+
def file
|
|
37
|
+
@file = Model.new(path) if @file.nil? || @file.path.to_s != path
|
|
38
|
+
@file
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# A remembered projector can be deleted or moved between runs, and
|
|
42
|
+
# llama-server refuses to boot on a missing --mmproj, so an entry that
|
|
43
|
+
# no longer resolves counts as no entry.
|
|
44
|
+
def mmproj_path
|
|
45
|
+
stored = @pending_mmproj_set ? @pending_mmproj : file.memory.mmproj
|
|
46
|
+
stored if stored && File.exist?(stored)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def path=(value)
|
|
50
|
+
writable["path"] = value
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def context_length=(value)
|
|
54
|
+
writable["context_length"] = value
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def reasoning=(value)
|
|
58
|
+
writable["reasoning"] = value
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def mmproj_path=(value)
|
|
62
|
+
@pending_mmproj_set = true
|
|
63
|
+
@pending_mmproj = value
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def reasoning_effort=(value)
|
|
67
|
+
@pending_effort_set = true
|
|
68
|
+
@pending_effort = value
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def save
|
|
72
|
+
file.memory.mmproj = @pending_mmproj if @pending_mmproj_set
|
|
73
|
+
file.memory.effort = @pending_effort if @pending_effort_set
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
# Reads see an absent section as an empty one; only a write materialises
|
|
79
|
+
# it, so a hand-built config keeps the sections it actually carries.
|
|
80
|
+
def section = @cfg[SECTION] || {}
|
|
81
|
+
def writable = @cfg[SECTION] ||= {}
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
module Mimic
|
|
2
|
+
module LlmConfig
|
|
3
|
+
class Sampling
|
|
4
|
+
FLAGS = {
|
|
5
|
+
"temperature" => "temp",
|
|
6
|
+
"top_p" => "top-p",
|
|
7
|
+
"top_k" => "top-k",
|
|
8
|
+
"min_p" => "min-p",
|
|
9
|
+
"presence_penalty" => "presence-penalty",
|
|
10
|
+
"repeat_penalty" => "repeat-penalty",
|
|
11
|
+
}.freeze
|
|
12
|
+
KEYS = FLAGS.keys.freeze
|
|
13
|
+
|
|
14
|
+
# llama-server's own defaults, so a key nothing overrides describes the
|
|
15
|
+
# same behaviour as passing no flag at all.
|
|
16
|
+
DEFAULTS = {
|
|
17
|
+
"temperature" => 0.8,
|
|
18
|
+
"top_p" => 0.95,
|
|
19
|
+
"top_k" => 40,
|
|
20
|
+
"min_p" => 0.05,
|
|
21
|
+
"presence_penalty" => 0.0,
|
|
22
|
+
"repeat_penalty" => 1.0,
|
|
23
|
+
}.freeze
|
|
24
|
+
|
|
25
|
+
# The publisher's own suggestion, carried in the GGUF header. Only
|
|
26
|
+
# these three are ever written there -- min_p and the two penalties
|
|
27
|
+
# appear in no GGUF and stay at the default.
|
|
28
|
+
HEADER_KEYS = {
|
|
29
|
+
"general.sampling.temp" => "temperature",
|
|
30
|
+
"general.sampling.top_p" => "top_p",
|
|
31
|
+
"general.sampling.top_k" => "top_k",
|
|
32
|
+
}.freeze
|
|
33
|
+
|
|
34
|
+
# GGUF stores these as float32, so a published 0.95 reads back as
|
|
35
|
+
# 0.949999988079071 and would reach the engine that way.
|
|
36
|
+
FLOAT_PRECISION = 6
|
|
37
|
+
|
|
38
|
+
def self.resolve(metadata)
|
|
39
|
+
suggested = HEADER_KEYS.filter_map do |header_key, key|
|
|
40
|
+
value = metadata[header_key]
|
|
41
|
+
[key, value.is_a?(Float) ? value.round(FLOAT_PRECISION) : value] unless value.nil?
|
|
42
|
+
end
|
|
43
|
+
DEFAULTS.merge(suggested.to_h)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def initialize(model_cfg)
|
|
47
|
+
@cfg = model_cfg
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def to_h = FLAGS.to_h { |key, flag| [flag, @cfg[key]] }
|
|
51
|
+
def to_args = to_h.flat_map { |flag, value| ["--#{flag}", value.to_s] }
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
require "psych"
|
|
2
|
+
require "pathname"
|
|
3
|
+
require_relative "../llm_config"
|
|
4
|
+
require_relative "model_settings"
|
|
5
|
+
require_relative "llama_settings"
|
|
6
|
+
require_relative "embed"
|
|
7
|
+
|
|
8
|
+
module Mimic
|
|
9
|
+
module LlmConfig
|
|
10
|
+
# The directories scanned for GGUF files: the builtin roots plus whatever
|
|
11
|
+
# `llm path add` has stored. Enumerating it yields every directory in
|
|
12
|
+
# scan order; add/remove report what they did as a symbol so the CLI can
|
|
13
|
+
# tell "already there" apart from "not yours to remove".
|
|
14
|
+
class SearchPaths
|
|
15
|
+
include Enumerable
|
|
16
|
+
|
|
17
|
+
def initialize(cfg)
|
|
18
|
+
@cfg = cfg
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def each(&) = all.each(&)
|
|
22
|
+
|
|
23
|
+
def all
|
|
24
|
+
(BUILTIN_MODEL_DIRS + configured.map { |dir| Pathname.new(dir.to_s).expand_path }).uniq
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def add(dir)
|
|
28
|
+
wanted = Pathname.new(dir.to_s).expand_path
|
|
29
|
+
return :builtin if BUILTIN_MODEL_DIRS.include?(wanted)
|
|
30
|
+
return :duplicate if configured.map { |p| Pathname.new(p).expand_path }.include?(wanted)
|
|
31
|
+
|
|
32
|
+
store(configured + [wanted.to_s])
|
|
33
|
+
:added
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def remove(dir)
|
|
37
|
+
wanted = Pathname.new(dir.to_s).expand_path
|
|
38
|
+
kept = configured.reject { |p| Pathname.new(p).expand_path == wanted }
|
|
39
|
+
return BUILTIN_MODEL_DIRS.include?(wanted) ? :builtin : :missing if kept.size == configured.size
|
|
40
|
+
|
|
41
|
+
store(kept)
|
|
42
|
+
:removed
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def configured = @cfg.dig("model", "search_paths") || []
|
|
48
|
+
|
|
49
|
+
def store(paths)
|
|
50
|
+
(@cfg["model"] ||= {})["search_paths"] = paths
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
class Settings
|
|
55
|
+
QUANT_MARKERS = %w[-Q4 -Q5 -Q6 -Q8 -IQ -F16 -BF16].freeze
|
|
56
|
+
|
|
57
|
+
def self.alias_for(path)
|
|
58
|
+
stem = Pathname.new(path).basename(".*").to_s
|
|
59
|
+
marker = QUANT_MARKERS.find { |m| stem.include?(m) }
|
|
60
|
+
marker ? stem.split(marker).first : stem
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def initialize(cfg)
|
|
64
|
+
@cfg = cfg
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def model = @model ||= ModelSettings.new(@cfg)
|
|
68
|
+
def llama = @llama ||= LlamaSettings.new(@cfg)
|
|
69
|
+
|
|
70
|
+
def save
|
|
71
|
+
LLM_CONFIG_DIR.mkpath
|
|
72
|
+
model.save
|
|
73
|
+
LLM_CONFIG_PATH.write(Psych.dump(@cfg.slice("model", "llama", "embed")))
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def remember_ctx
|
|
77
|
+
model.file.memory.remember_ctx(model.context_length, llama.cache_type)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def remember_sampling
|
|
81
|
+
model.file.memory.remember_sampling(model.reasoning, model.file.memory.sampling_for(reasoning: model.reasoning))
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def forget_sampling = model.file.memory.forget_sampling
|
|
85
|
+
|
|
86
|
+
def search_paths = SearchPaths.new(@cfg)
|
|
87
|
+
def embed = @embed ||= Embed.new(@cfg["embed"] ||= {}, host: llama.host)
|
|
88
|
+
|
|
89
|
+
def to_h = @cfg
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
require "date"
|
|
2
|
+
require "psych"
|
|
3
|
+
require "json"
|
|
4
|
+
require "pathname"
|
|
5
|
+
require_relative "paths"
|
|
6
|
+
require_relative "model"
|
|
7
|
+
require_relative "llm_config/sampling"
|
|
8
|
+
require_relative "llm_config/settings"
|
|
9
|
+
|
|
10
|
+
# LLM engine config -- model + llama keys, separate from mimic service
|
|
11
|
+
# config. Owned by the `llm` CLI; the main `mimic` config reads from it for
|
|
12
|
+
# base URL, model alias, and context length (needed on every agent launch).
|
|
13
|
+
module Mimic
|
|
14
|
+
module LlmConfig
|
|
15
|
+
LLM_CONFIG_DIR = Paths::LLM_ROOT
|
|
16
|
+
LLM_CONFIG_PATH = LLM_CONFIG_DIR / "config.yaml"
|
|
17
|
+
MODEL_CTX_PATH = LLM_CONFIG_DIR / "model-ctx.json"
|
|
18
|
+
MODEL_MMPROJ_PATH = LLM_CONFIG_DIR / "model-mmproj.json"
|
|
19
|
+
MODEL_COMPUTE_PATH = LLM_CONFIG_DIR / "model-compute.json"
|
|
20
|
+
MODEL_SAMPLING_PATH = LLM_CONFIG_DIR / "model-sampling.json"
|
|
21
|
+
MODEL_EFFORT_PATH = LLM_CONFIG_DIR / "model-effort.json"
|
|
22
|
+
LEGACY_CONFIG = Paths::STATE_DIR / "llm_config.yaml"
|
|
23
|
+
|
|
24
|
+
BUILTIN_MODEL_DIRS = [
|
|
25
|
+
Pathname.new(Dir.home) / "hf-models",
|
|
26
|
+
LLM_CONFIG_DIR / "models",
|
|
27
|
+
Pathname.new(Dir.home) / ".lmstudio/models",
|
|
28
|
+
].freeze
|
|
29
|
+
|
|
30
|
+
DEFAULTS = {
|
|
31
|
+
"model" => {
|
|
32
|
+
"path" => "",
|
|
33
|
+
"search_paths" => [],
|
|
34
|
+
"context_length" => 122_880,
|
|
35
|
+
"mtp" => "auto",
|
|
36
|
+
"reasoning" => "auto",
|
|
37
|
+
},
|
|
38
|
+
"llama" => {
|
|
39
|
+
"engine" => "llama.cpp",
|
|
40
|
+
"host" => "0.0.0.0",
|
|
41
|
+
"port" => 8080,
|
|
42
|
+
"flash_attn" => "on",
|
|
43
|
+
"cache_type" => "q8_0",
|
|
44
|
+
"n_slots" => 1,
|
|
45
|
+
"binary" => {},
|
|
46
|
+
},
|
|
47
|
+
"embed" => {
|
|
48
|
+
"path" => "",
|
|
49
|
+
"port" => 8081,
|
|
50
|
+
"context_length" => 0,
|
|
51
|
+
"batch_size" => 8192,
|
|
52
|
+
"mode" => "auto",
|
|
53
|
+
},
|
|
54
|
+
}.freeze
|
|
55
|
+
|
|
56
|
+
class << self
|
|
57
|
+
def merge(base, override)
|
|
58
|
+
out = Marshal.load(Marshal.dump(base))
|
|
59
|
+
override.each do |key, value|
|
|
60
|
+
out[key] = value.is_a?(Hash) && out[key].is_a?(Hash) ? merge(out[key], value) : value
|
|
61
|
+
end
|
|
62
|
+
out
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def load
|
|
66
|
+
migrate_legacy
|
|
67
|
+
return Settings.new(merge(DEFAULTS, {})) unless LLM_CONFIG_PATH.exist?
|
|
68
|
+
|
|
69
|
+
data = Psych.safe_load(LLM_CONFIG_PATH.read, permitted_classes: [Symbol, Date, Time], aliases: true) || {}
|
|
70
|
+
migrate_stores(Settings.new(merge(DEFAULTS, data)))
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def exists?
|
|
74
|
+
migrate_legacy
|
|
75
|
+
LLM_CONFIG_PATH.exist?
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# ~/hf-models is a tree of per-file symlinks into their .lmstudio
|
|
79
|
+
# targets, and this store has to be keyed by the resolved target so a
|
|
80
|
+
# model reached through a different symlink still matches. expand_path
|
|
81
|
+
# alone leaves the symlink unresolved and splits the store into two
|
|
82
|
+
# disagreeing key sets; realdirpath covers the non-strict case for a
|
|
83
|
+
# path whose file is gone.
|
|
84
|
+
def resolve_path(path)
|
|
85
|
+
expanded = Pathname.new(path).expand_path
|
|
86
|
+
return expanded.realpath.to_s if expanded.exist?
|
|
87
|
+
|
|
88
|
+
expanded.realdirpath.to_s
|
|
89
|
+
rescue SystemCallError
|
|
90
|
+
expanded.to_s
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
private
|
|
94
|
+
|
|
95
|
+
# Relocate the engine config left behind when it lived under STATE_DIR.
|
|
96
|
+
def migrate_legacy
|
|
97
|
+
return if LLM_CONFIG_PATH.exist? || !LEGACY_CONFIG.file?
|
|
98
|
+
|
|
99
|
+
LLM_CONFIG_DIR.mkpath
|
|
100
|
+
LEGACY_CONFIG.rename(LLM_CONFIG_PATH)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Relocate the fields that used to sit flat in config.yaml:
|
|
104
|
+
# model.mmproj_path (now per model, in Model::Memory) and a plain
|
|
105
|
+
# llama.binary string (now per engine). Both are relocated with the
|
|
106
|
+
# attribution they had when they were written -- the projector to the
|
|
107
|
+
# model that is configured, the binary to the engine that is configured
|
|
108
|
+
# -- and the old keys are dropped so nothing reads them again. The
|
|
109
|
+
# sampler keys are dropped outright: they resolve from the chosen
|
|
110
|
+
# model's own header now, so a stale hand-edited value would only
|
|
111
|
+
# contradict it. model.thinking was a boolean translated to the engines'
|
|
112
|
+
# reasoning value at read time; the value itself is stored now, so the
|
|
113
|
+
# translation happens once here instead.
|
|
114
|
+
def migrate_stores(settings)
|
|
115
|
+
cfg = settings.to_h
|
|
116
|
+
legacy_binary = cfg["llama"]["binary"] if cfg["llama"]["binary"].is_a?(String)
|
|
117
|
+
legacy_sampling = Sampling::KEYS.any? { |key| cfg["model"].key?(key) }
|
|
118
|
+
legacy_thinking = cfg["model"].key?("thinking")
|
|
119
|
+
return settings unless cfg["model"].key?("mmproj_path") || legacy_binary ||
|
|
120
|
+
legacy_sampling || legacy_thinking
|
|
121
|
+
|
|
122
|
+
projector = cfg["model"].delete("mmproj_path").to_s
|
|
123
|
+
settings.model.mmproj_path = projector unless projector.empty?
|
|
124
|
+
settings.llama.binary = legacy_binary if legacy_binary
|
|
125
|
+
settings.model.reasoning = cfg["model"].delete("thinking") ? "auto" : "off" if legacy_thinking
|
|
126
|
+
Sampling::KEYS.each { |key| cfg["model"].delete(key) }
|
|
127
|
+
settings.save
|
|
128
|
+
settings
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
require_relative "header"
|
|
2
|
+
require_relative "../util/host"
|
|
3
|
+
|
|
4
|
+
# What one model at one context costs on the GPU -- weights + KV + compute
|
|
5
|
+
# buffer against the card's budget, so setup can reject a context length that
|
|
6
|
+
# would load but crash at runtime once the KV cache fills (llama.cpp
|
|
7
|
+
# over-commits ctx).
|
|
8
|
+
module Mimic
|
|
9
|
+
class Model
|
|
10
|
+
class Footprint
|
|
11
|
+
# bytes per element for KV cache quant types (block_bytes / block_elems)
|
|
12
|
+
CACHE_BPE = {
|
|
13
|
+
"f32" => 4.0,
|
|
14
|
+
"f16" => 2.0,
|
|
15
|
+
"bf16" => 2.0,
|
|
16
|
+
"q8_0" => 34 / 32.0,
|
|
17
|
+
"q5_1" => 24 / 32.0,
|
|
18
|
+
"q5_0" => 22 / 32.0,
|
|
19
|
+
"q4_1" => 20 / 32.0,
|
|
20
|
+
"q4_0" => 18 / 32.0,
|
|
21
|
+
}.freeze
|
|
22
|
+
|
|
23
|
+
# The compute buffer tracks ubatch, vocab and graph shape -- not card
|
|
24
|
+
# size -- so these are a guard, not a model. They scale with the card
|
|
25
|
+
# because a fixed 2 GiB is 8% of a 3090 but 17% of a 12 GiB 3060, and a
|
|
26
|
+
# smaller card runs the smaller ubatch/ctx that make the real buffer
|
|
27
|
+
# smaller anyway. ComputeStore replaces the guess after the first launch.
|
|
28
|
+
# A sliding-window layer's cache does not grow with context: llama.cpp
|
|
29
|
+
# sizes it at the window plus one ubatch, padded to 256 cells
|
|
30
|
+
# (llama-kv-cache-iswa.cpp:73). Both engine presets set
|
|
31
|
+
# --ubatch-size 1024, so that is what the window cache is priced at.
|
|
32
|
+
SWA_UBATCH = 1024
|
|
33
|
+
SWA_CELL_PAD = 256
|
|
34
|
+
|
|
35
|
+
# The recurrent state is f32 whatever --cache-type-k/v says.
|
|
36
|
+
RECURRENT_BPE = 4.0
|
|
37
|
+
|
|
38
|
+
COMPUTE_FRACTION = 0.06
|
|
39
|
+
COMPUTE_FLOOR_MIB = 1024
|
|
40
|
+
COMPUTE_CEILING_MIB = 2048
|
|
41
|
+
DESKTOP_RESERVE_MIB = 2048 # leave this much VRAM for the desktop/other apps
|
|
42
|
+
|
|
43
|
+
def self.compute_default(total_mib)
|
|
44
|
+
(total_mib * COMPUTE_FRACTION).round.clamp(COMPUTE_FLOOR_MIB, COMPUTE_CEILING_MIB)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def initialize(header, ctx:, cache_type:, weights_bytes:, compute_mib: nil, drafting: false, slots: 1,
|
|
48
|
+
reclaimable_mib: 0)
|
|
49
|
+
@header = header
|
|
50
|
+
@ctx = ctx
|
|
51
|
+
@cache_type = cache_type
|
|
52
|
+
@weights_bytes = weights_bytes
|
|
53
|
+
@compute_mib = compute_mib
|
|
54
|
+
@drafting = drafting
|
|
55
|
+
@slots = slots
|
|
56
|
+
@reclaimable_mib = reclaimable_mib
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# False when the header carries no KV shape or no GPU answers -- every
|
|
60
|
+
# reader below is nil then, and callers print nothing rather than a
|
|
61
|
+
# number they made up.
|
|
62
|
+
def measurable? = !kv_per_token_mib.nil? && !total_mib.nil? && !@weights_bytes.nil?
|
|
63
|
+
|
|
64
|
+
def total_mib
|
|
65
|
+
return @total_mib if defined?(@total_mib)
|
|
66
|
+
|
|
67
|
+
@total_mib = Util::Host.gpu_total_mib
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def weights_mib = @weights_bytes && @weights_bytes / 1024.0 / 1024.0
|
|
71
|
+
# Window layers are the fixed half: they hold the same cells at ctx 4096
|
|
72
|
+
# as at 262144, so only the full-attention layers scale.
|
|
73
|
+
def kv_mib = kv_per_token_mib && kv_per_token_mib * @ctx + kv_fixed_mib
|
|
74
|
+
def compute_mib = @compute_mib || (total_mib && self.class.compute_default(total_mib))
|
|
75
|
+
|
|
76
|
+
# A drafting engine builds a second context over the nextn block: its
|
|
77
|
+
# own KV for those layers plus a compute buffer the same size as the
|
|
78
|
+
# target's, since it runs the same graph shape at the same ubatch.
|
|
79
|
+
# Measured on Qwen3.6-35B-A3B at ctx 163840 (2026-08-23): 170 MiB KV
|
|
80
|
+
# and a second 978 MiB CUDA0 compute buffer beside the target's 978.
|
|
81
|
+
def draft_mib
|
|
82
|
+
return 0 unless @drafting && kv_per_token_mib
|
|
83
|
+
|
|
84
|
+
layers = target_attention_layers
|
|
85
|
+
return compute_mib if layers.nil? || layers.zero?
|
|
86
|
+
|
|
87
|
+
((kv_per_token_mib * @ctx) / layers * @header.nextn_layers) + compute_mib
|
|
88
|
+
end
|
|
89
|
+
# What is actually available: the driver's free figure when it answers,
|
|
90
|
+
# since the desktop's draw is measurable and moves. DESKTOP_RESERVE_MIB
|
|
91
|
+
# is the fallback for a host where nvidia-smi cannot be read.
|
|
92
|
+
#
|
|
93
|
+
# `reclaimable_mib` is VRAM held by a server this launch replaces, which
|
|
94
|
+
# the free figure counts as unavailable even though it is about to be
|
|
95
|
+
# released. Other processes' VRAM stays subtracted -- only our own
|
|
96
|
+
# outgoing server comes back. The fallback path has no free figure to
|
|
97
|
+
# add to: the reserve is already the whole allowance.
|
|
98
|
+
def budget_mib
|
|
99
|
+
return @budget_mib if defined?(@budget_mib)
|
|
100
|
+
|
|
101
|
+
free = Util::Host.gpu_free_mib
|
|
102
|
+
@budget_mib = free ? free + @reclaimable_mib : (total_mib && total_mib - DESKTOP_RESERVE_MIB)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def need_mib
|
|
106
|
+
return nil unless measurable?
|
|
107
|
+
|
|
108
|
+
weights_mib + kv_mib + compute_mib + draft_mib
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def fits? = measurable? && need_mib <= budget_mib
|
|
112
|
+
|
|
113
|
+
# What is left of the budget once this configuration is resident.
|
|
114
|
+
def headroom_mib = measurable? ? budget_mib - need_mib : nil
|
|
115
|
+
|
|
116
|
+
# The longest context the card can hold at this cache type, weights and
|
|
117
|
+
# compute buffer.
|
|
118
|
+
def max_ctx
|
|
119
|
+
return nil unless measurable?
|
|
120
|
+
return 0 if kv_per_token_mib.zero?
|
|
121
|
+
|
|
122
|
+
[0, ((budget_mib - weights_mib - compute_mib - draft_mib - kv_fixed_mib) / kv_per_token_mib).to_i].max
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# How many layers' experts have to move to CPU to cover the deficit.
|
|
126
|
+
# 0 when nothing needs moving, nil when the model is dense (nothing to
|
|
127
|
+
# move) or the ceiling can't cover the gap.
|
|
128
|
+
def layers_to_offload(ceiling:)
|
|
129
|
+
return nil unless measurable?
|
|
130
|
+
return 0 if need_mib <= budget_mib
|
|
131
|
+
|
|
132
|
+
per_layer = @header.expert_bytes_per_layer
|
|
133
|
+
return nil if per_layer.nil? || per_layer.zero?
|
|
134
|
+
|
|
135
|
+
layers = ((need_mib - budget_mib) / (per_layer / 1024.0 / 1024.0)).ceil
|
|
136
|
+
layers <= ceiling ? layers : nil
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# What a loaded instance holds beyond this estimate, in MiB, or nil when
|
|
140
|
+
# it can't be measured. The residual is what compute_default stands in
|
|
141
|
+
# for, so it is worth storing under the exact ctx and cache type it was
|
|
142
|
+
# taken at.
|
|
143
|
+
def measured_overhead_mib(pids:)
|
|
144
|
+
used = Util::Host.gpu_process_mib(pids)
|
|
145
|
+
return nil if !measurable? || used.nil?
|
|
146
|
+
|
|
147
|
+
overhead = used - need_mib
|
|
148
|
+
overhead.positive? ? overhead.round : nil
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
private
|
|
152
|
+
|
|
153
|
+
# The attention-layer count kv_mib was built from, so draft_mib can
|
|
154
|
+
# price one nextn block as a fraction of it.
|
|
155
|
+
def target_attention_layers
|
|
156
|
+
meta = @header.metadata
|
|
157
|
+
arch = meta["general.architecture"]
|
|
158
|
+
arch && attention_layers(meta, arch)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def kv_per_token_mib = kv_shape && mib(kv_shape.first)
|
|
162
|
+
def kv_fixed_mib = kv_shape ? mib(kv_shape.last) + recurrent_mib : 0
|
|
163
|
+
|
|
164
|
+
def mib(elems) = elems * CACHE_BPE.fetch(@cache_type, 2.0) / 1024 / 1024
|
|
165
|
+
|
|
166
|
+
# [elements per token, elements held whatever the context is]. Layers
|
|
167
|
+
# are walked one at a time because a model can mix widths: gemma-4
|
|
168
|
+
# gives its window layers 16 KV heads at head dim 256 and its
|
|
169
|
+
# full-attention layers 4 at 512, in a repeating pattern the header
|
|
170
|
+
# spells out per layer.
|
|
171
|
+
def kv_shape
|
|
172
|
+
return @kv_shape if defined?(@kv_shape)
|
|
173
|
+
|
|
174
|
+
@kv_shape = read_kv_shape
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def read_kv_shape
|
|
178
|
+
meta = @header.metadata
|
|
179
|
+
arch = meta["general.architecture"]
|
|
180
|
+
return nil unless arch
|
|
181
|
+
|
|
182
|
+
n_layer = attention_layers(meta, arch)
|
|
183
|
+
k_dim = meta["#{arch}.attention.key_length"] || derived_head_dim(meta, arch)
|
|
184
|
+
# K and V are not always the same width -- gemma-4 ships both keys
|
|
185
|
+
# separately, and reading only key_length silently prices V wrong.
|
|
186
|
+
v_dim = meta["#{arch}.attention.value_length"] || k_dim
|
|
187
|
+
heads = heads_kv(meta, arch)
|
|
188
|
+
return nil unless n_layer && heads && k_dim && v_dim
|
|
189
|
+
|
|
190
|
+
swa = swa_pattern(meta, arch)
|
|
191
|
+
swa_dims = (meta["#{arch}.attention.key_length_swa"] || k_dim) +
|
|
192
|
+
(meta["#{arch}.attention.value_length_swa"] || v_dim)
|
|
193
|
+
cells = swa_cells(meta, arch)
|
|
194
|
+
per_token = 0
|
|
195
|
+
fixed = 0
|
|
196
|
+
cached_layers(meta, arch, n_layer).times do |il|
|
|
197
|
+
n_kv = heads.is_a?(Array) ? heads[il] : heads
|
|
198
|
+
next if n_kv.nil?
|
|
199
|
+
|
|
200
|
+
if swa[il]
|
|
201
|
+
fixed += n_kv * swa_dims * cells
|
|
202
|
+
else
|
|
203
|
+
per_token += n_kv * (k_dim + v_dim)
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
[per_token, fixed]
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Per layer in the newer headers, one number in the older ones. Absent
|
|
210
|
+
# means multi-head attention: llama.cpp defaults it to head_count
|
|
211
|
+
# (llama-model.cpp:1186).
|
|
212
|
+
def heads_kv(meta, arch)
|
|
213
|
+
meta["#{arch}.attention.head_count_kv"] || meta["#{arch}.attention.head_count"]
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# Only the explicit per-layer pattern is honoured. Architectures whose
|
|
217
|
+
# window layout is a rule in llama.cpp rather than a key in the file
|
|
218
|
+
# (gemma-3's set_swa_pattern) read as all-full-attention here, which
|
|
219
|
+
# over-states their KV rather than inventing a layout.
|
|
220
|
+
def swa_pattern(meta, arch)
|
|
221
|
+
pattern = meta["#{arch}.attention.sliding_window_pattern"]
|
|
222
|
+
pattern.is_a?(Array) ? pattern.map { |flag| !flag.zero? } : []
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def swa_cells(meta, arch)
|
|
226
|
+
window = meta["#{arch}.attention.sliding_window"].to_i
|
|
227
|
+
cells = [@ctx, window * @slots + SWA_UBATCH].min
|
|
228
|
+
-(-cells / SWA_CELL_PAD) * SWA_CELL_PAD
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# Layers past this reuse an earlier layer's cache instead of holding
|
|
232
|
+
# their own (llama-model.cpp:2357). shared_kv_layers is how many of the
|
|
233
|
+
# stack does so.
|
|
234
|
+
def cached_layers(meta, arch, n_layer)
|
|
235
|
+
n_layer - meta["#{arch}.attention.shared_kv_layers"].to_i
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Hybrid attention (Qwen3.5/Next-style): only every Nth layer keeps a
|
|
239
|
+
# standard KV cache that scales with context -- the rest are
|
|
240
|
+
# linear/recurrent-attention layers with a small fixed-size state.
|
|
241
|
+
# block_count includes the MTP draft heads, which are not part of the
|
|
242
|
+
# target's layer stack -- counting them here inflates the KV estimate by
|
|
243
|
+
# a whole attention layer on any MTP-capable GGUF. draft_mib adds them
|
|
244
|
+
# back when the engine is drafting.
|
|
245
|
+
def attention_layers(meta, arch)
|
|
246
|
+
n_layer = meta["#{arch}.block_count"]
|
|
247
|
+
interval = meta["#{arch}.full_attention_interval"]
|
|
248
|
+
return n_layer unless n_layer && interval
|
|
249
|
+
|
|
250
|
+
-(-(n_layer - @header.nextn_layers) / interval) # ceil div
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# A hybrid model's linear-attention layers hold an f32 recurrent state
|
|
254
|
+
# instead of a K/V pair. It never scales with context but it is real
|
|
255
|
+
# VRAM, and neither engine reports it as KV: ik folds it into the KV
|
|
256
|
+
# buffer while printing only K and V in `KV self size`, and llama.cpp
|
|
257
|
+
# allocates it as a separate recurrent buffer. Verified against ik on
|
|
258
|
+
# Qwen3.6-35B-A3B (62.81 MiB over 30 layers) and Qwen3.8-27B
|
|
259
|
+
# (149.63 over 48).
|
|
260
|
+
def recurrent_mib
|
|
261
|
+
meta = @header.metadata
|
|
262
|
+
arch = meta["general.architecture"]
|
|
263
|
+
state = recurrent_state_elems(meta, arch)
|
|
264
|
+
return 0 if state.zero?
|
|
265
|
+
|
|
266
|
+
layers = recurrent_layers(meta, arch)
|
|
267
|
+
layers * state * @slots * RECURRENT_BPE / 1024 / 1024
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# The layers that are not full-attention ones: llama.cpp marks every
|
|
271
|
+
# layer whose 1-based index is not a multiple of the interval
|
|
272
|
+
# (llama-hparams.cpp:603), and the nextn heads are never recurrent.
|
|
273
|
+
def recurrent_layers(meta, arch)
|
|
274
|
+
n_layer = meta["#{arch}.block_count"]
|
|
275
|
+
return 0 unless n_layer && meta["#{arch}.full_attention_interval"]
|
|
276
|
+
|
|
277
|
+
(n_layer - @header.nextn_layers) - attention_layers(meta, arch)
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# The gated-delta-net state: a convolution window over 2 key groups plus
|
|
281
|
+
# the value stream, and one square matrix per value head
|
|
282
|
+
# (llama-hparams.h:329). Only this layout is priced -- a plain Mamba
|
|
283
|
+
# state lands in the K/V caches instead and is already counted there.
|
|
284
|
+
def recurrent_state_elems(meta, arch)
|
|
285
|
+
groups = meta["#{arch}.ssm.group_count"].to_i
|
|
286
|
+
inner = meta["#{arch}.ssm.inner_size"].to_i
|
|
287
|
+
heads = meta["#{arch}.ssm.time_step_rank"].to_i
|
|
288
|
+
return 0 if groups.zero? || inner.zero? || heads.zero?
|
|
289
|
+
|
|
290
|
+
key_dim = meta["#{arch}.ssm.state_size"].to_i * groups
|
|
291
|
+
conv = [meta["#{arch}.ssm.conv_kernel"].to_i - 1, 0].max
|
|
292
|
+
head_dim = inner / heads
|
|
293
|
+
conv * (2 * key_dim + inner) + head_dim * head_dim * heads
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def derived_head_dim(meta, arch)
|
|
297
|
+
emb = meta["#{arch}.embedding_length"]
|
|
298
|
+
n_head = meta["#{arch}.attention.head_count"]
|
|
299
|
+
emb && n_head && !n_head.zero? ? emb / n_head : nil
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|