mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require_relative "../paths"
|
|
3
|
+
require_relative "../model/header"
|
|
4
|
+
require_relative "../model"
|
|
5
|
+
require_relative "../util/probes"
|
|
6
|
+
|
|
7
|
+
module Mimic
|
|
8
|
+
# A model's embedded Jinja chat template, and the override mimic hands the
|
|
9
|
+
# server when that template would throw on real agent traffic.
|
|
10
|
+
#
|
|
11
|
+
# The Qwen-family templates ship `raise_exception` guards that llama.cpp's
|
|
12
|
+
# minja evaluates at render time: a request whose message list trips one
|
|
13
|
+
# comes back as a 500 rather than a completion. Claude Code trips
|
|
14
|
+
# `System message must be at the beginning` because llama.cpp's
|
|
15
|
+
# /v1/messages bridge emits system content the guard does not expect.
|
|
16
|
+
#
|
|
17
|
+
# Each rule replaces one guard with what the template should have rendered.
|
|
18
|
+
# `requires` names the constructs the replacement leans on -- a template
|
|
19
|
+
# that phrases its loop differently gets that guard left alone rather than
|
|
20
|
+
# rewritten wrong, and the hand-written override in CHAT_TEMPLATES_DIR
|
|
21
|
+
# stays the answer for it.
|
|
22
|
+
module Engine
|
|
23
|
+
class ChatTemplate
|
|
24
|
+
METADATA_KEY = "tokenizer.chat_template".freeze
|
|
25
|
+
RAISE_CALL = /\{\{-?\s*raise_exception\(\s*(['"])(.*?)\1\s*\)\s*-?\}\}/m
|
|
26
|
+
|
|
27
|
+
NOOP = "{{- '' }}".freeze
|
|
28
|
+
|
|
29
|
+
EFFORT_DEFAULT = /reasoning_effort\s*\|\s*default\(\s*['"]([\w-]+)['"]/
|
|
30
|
+
EFFORT_FALLBACK = /set\s+reasoning_effort\s*=\s*['"]([\w-]+)['"]/
|
|
31
|
+
EFFORT_CHOICES = /reasoning_effort\s+not\s+in\s+[(\[]([^)\]]*)[)\]]/
|
|
32
|
+
EFFORT_COMPARE = /reasoning_effort\s*==\s*['"]([\w-]+)['"]/
|
|
33
|
+
QUOTED = /['"]([\w-]+)['"]/
|
|
34
|
+
|
|
35
|
+
# gpt-oss's harmony template prints reasoning_effort without listing
|
|
36
|
+
# the values; these are the ones the format defines.
|
|
37
|
+
HARMONY_CHANNEL = "<|channel|>".freeze
|
|
38
|
+
HARMONY_EFFORTS = %w[low medium high].freeze
|
|
39
|
+
|
|
40
|
+
# llama.cpp's --reasoning auto renders the template and asks its format
|
|
41
|
+
# handler; these are the markers those handlers turn thinking on for.
|
|
42
|
+
THINKING_MARKERS = ["enable_thinking", "<think>", "[THINK]", HARMONY_CHANNEL].freeze
|
|
43
|
+
|
|
44
|
+
Rule = Struct.new(:match, :replacement, :requires, keyword_init: true)
|
|
45
|
+
|
|
46
|
+
RULES = [
|
|
47
|
+
Rule.new(
|
|
48
|
+
match: /system message must be at the beginning/i,
|
|
49
|
+
replacement: %q({{- '<|im_start|>system\n' + content + '<|im_end|>' + '\n' }}),
|
|
50
|
+
requires: ["<|im_start|>system", "set content = render_content("],
|
|
51
|
+
),
|
|
52
|
+
Rule.new(
|
|
53
|
+
match: /unexpected message role/i,
|
|
54
|
+
replacement: %q({{- '<|im_start|>user\n' + content + '<|im_end|>' + '\n' }}),
|
|
55
|
+
requires: ["<|im_start|>system", "set content = render_content("],
|
|
56
|
+
),
|
|
57
|
+
Rule.new(
|
|
58
|
+
match: /unexpected item type in content/i,
|
|
59
|
+
replacement: "{{- item.text if item.text is defined else '' }}",
|
|
60
|
+
requires: ["for item in content"],
|
|
61
|
+
),
|
|
62
|
+
Rule.new(
|
|
63
|
+
match: /unexpected content type/i,
|
|
64
|
+
replacement: "{{- content | string }}",
|
|
65
|
+
requires: [],
|
|
66
|
+
),
|
|
67
|
+
Rule.new(
|
|
68
|
+
match: /system message cannot contain|no messages provided|no user query found/i,
|
|
69
|
+
replacement: NOOP,
|
|
70
|
+
requires: [],
|
|
71
|
+
),
|
|
72
|
+
].freeze
|
|
73
|
+
|
|
74
|
+
# Renders the message shapes real agents send through the server's own
|
|
75
|
+
# /apply-template, which runs the same minja pass as a completion. A 500
|
|
76
|
+
# here is the template throwing -- the exact failure a request would hit,
|
|
77
|
+
# measured instead of predicted. Both engines expose the endpoint.
|
|
78
|
+
class Probe
|
|
79
|
+
SYSTEM = { role: "system", content: "You are a coding agent." }.freeze
|
|
80
|
+
USER = { role: "user", content: "list the files" }.freeze
|
|
81
|
+
|
|
82
|
+
FIXTURES = {
|
|
83
|
+
"a system message after the first turn" =>
|
|
84
|
+
[SYSTEM, USER, { role: "system", content: "<system-reminder>stay on task</system-reminder>" }],
|
|
85
|
+
"an assistant tool call and its response" =>
|
|
86
|
+
[SYSTEM, USER,
|
|
87
|
+
{ role: "assistant", content: "",
|
|
88
|
+
tool_calls: [{ type: "function",
|
|
89
|
+
function: { name: "Read", arguments: { path: "/tmp/x" } } }] },
|
|
90
|
+
{ role: "tool", content: "file body" }],
|
|
91
|
+
"content sent as typed blocks" =>
|
|
92
|
+
[SYSTEM, { role: "user", content: [{ type: "text", text: "and now?" }] }],
|
|
93
|
+
}.freeze
|
|
94
|
+
|
|
95
|
+
def initialize(base_url, model_alias:)
|
|
96
|
+
@url = "#{base_url}/apply-template"
|
|
97
|
+
@model_alias = model_alias
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def failures
|
|
101
|
+
FIXTURES.filter_map do |label, messages|
|
|
102
|
+
res = Util::Probes.post(@url, { model: @model_alias, messages: messages })
|
|
103
|
+
label if res && res.code.to_i >= 500
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def initialize(model_path, model_alias:)
|
|
109
|
+
@model_path = model_path.to_s
|
|
110
|
+
@model_alias = model_alias.to_s
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def source
|
|
114
|
+
return @source if defined?(@source)
|
|
115
|
+
|
|
116
|
+
# full: the template is written after the vocab arrays, and the
|
|
117
|
+
# default read stops at the first of them.
|
|
118
|
+
@source = Model::Header.new(@model_path).metadata(full: true)[METADATA_KEY]
|
|
119
|
+
rescue StandardError
|
|
120
|
+
@source = nil
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def guarded? = source.to_s.include?("raise_exception")
|
|
124
|
+
|
|
125
|
+
# The reasoning_effort values the served template accepts, in its own
|
|
126
|
+
# order: an override when one exists, the GGUF's template otherwise.
|
|
127
|
+
# Fewer than two means there is nothing to choose between.
|
|
128
|
+
def effort_levels
|
|
129
|
+
text = served_source.to_s
|
|
130
|
+
listed = text[EFFORT_CHOICES, 1]
|
|
131
|
+
levels = listed ? listed.scan(QUOTED).flatten : text.scan(EFFORT_COMPARE).flatten
|
|
132
|
+
levels = HARMONY_EFFORTS if levels.empty? && harmony?(text)
|
|
133
|
+
levels = (levels + [default_effort].compact).uniq
|
|
134
|
+
levels.size < 2 ? [] : levels
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def default_effort
|
|
138
|
+
text = served_source.to_s
|
|
139
|
+
text[EFFORT_DEFAULT, 1] || text[EFFORT_FALLBACK, 1]
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def thinks?
|
|
143
|
+
text = served_source.to_s
|
|
144
|
+
THINKING_MARKERS.any? { |marker| text.include?(marker) }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Hand-written first: a file the user wrote is a decision, and mimic's
|
|
148
|
+
# rules do not get to overrule it.
|
|
149
|
+
def override_path
|
|
150
|
+
[hand_written, generated].find(&:file?)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def install
|
|
154
|
+
return nil if hand_written.file?
|
|
155
|
+
|
|
156
|
+
text = sanitized
|
|
157
|
+
return nil if text.nil?
|
|
158
|
+
|
|
159
|
+
generated.dirname.mkpath
|
|
160
|
+
generated.write(text.end_with?("\n") ? text : "#{text}\n")
|
|
161
|
+
generated
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
private
|
|
165
|
+
|
|
166
|
+
# The rewritten template, or nil when no guard in it has a rule.
|
|
167
|
+
#
|
|
168
|
+
# Per guard, not per template: a guard mimic has no rule for is left
|
|
169
|
+
# standing exactly as the model shipped it, so the shapes that trip it
|
|
170
|
+
# keep failing the way they already do. Refusing the whole rewrite over
|
|
171
|
+
# one such guard is what would change behaviour -- it drops repairs that
|
|
172
|
+
# are correct and leaves a template that 500s on ordinary agent traffic.
|
|
173
|
+
def sanitized
|
|
174
|
+
return nil unless guarded?
|
|
175
|
+
|
|
176
|
+
rewrote = false
|
|
177
|
+
rewritten = source.gsub(RAISE_CALL) do
|
|
178
|
+
whole = Regexp.last_match(0)
|
|
179
|
+
rule = rule_for(Regexp.last_match(2))
|
|
180
|
+
next whole if rule.nil?
|
|
181
|
+
|
|
182
|
+
rewrote = true
|
|
183
|
+
rule.replacement
|
|
184
|
+
end
|
|
185
|
+
rewrote ? rewritten : nil
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def hand_written = Paths::CHAT_TEMPLATES_DIR / "#{@model_alias}.jinja"
|
|
189
|
+
def generated = Paths::GENERATED_TEMPLATES_DIR / "#{@model_alias}.jinja"
|
|
190
|
+
|
|
191
|
+
def served_source
|
|
192
|
+
override = override_path
|
|
193
|
+
override ? override.read : source
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def harmony?(text) = text.include?(HARMONY_CHANNEL) && text.include?("reasoning_effort")
|
|
197
|
+
|
|
198
|
+
def rule_for(message)
|
|
199
|
+
rule = RULES.find { |r| r.match.match?(message) }
|
|
200
|
+
return nil if rule.nil?
|
|
201
|
+
return nil unless rule.requires.all? { |needle| source.include?(needle) }
|
|
202
|
+
|
|
203
|
+
rule
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require_relative "../model"
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
module Engine
|
|
7
|
+
class IkLlama < Base
|
|
8
|
+
LAUNCH_RECORD = Paths::LLM_RUN_DIR / "ik-llama-launch.txt"
|
|
9
|
+
TRACE_LOG_BASE = Paths::LLM_RUN_DIR / "ik_llama"
|
|
10
|
+
|
|
11
|
+
# Layers offloaded to CPU on each retry after an OOM, and the ceiling
|
|
12
|
+
# we give up at -- generous enough for any current MoE GGUF's layer
|
|
13
|
+
# count.
|
|
14
|
+
CPU_MOE_STEP = 2
|
|
15
|
+
CPU_MOE_MAX = 28
|
|
16
|
+
|
|
17
|
+
OOM_MARKERS = ["cudaMalloc failed", "out of memory"].freeze
|
|
18
|
+
BIND_MARKER = "couldn't bind to server socket"
|
|
19
|
+
KV_OOM_MARKER = "failed to allocate buffer for kv cache"
|
|
20
|
+
|
|
21
|
+
# ik builds the MTP draft context against the target model after the
|
|
22
|
+
# weights and KV are already resident, and it is not in the footprint
|
|
23
|
+
# estimate -- so the stage can be what tips a load that would otherwise
|
|
24
|
+
# have fit. Recoverable: the same argv minus --spec-type starts.
|
|
25
|
+
MTP_OOM_MARKER = "failed to create MTP context"
|
|
26
|
+
|
|
27
|
+
# Every flag mimic sets, in argv order: `build_command` spells them from
|
|
28
|
+
# here and `ServerStatus` reads a launched argv back through the same
|
|
29
|
+
# table. Valueless switches stay out of it -- there is nothing to read.
|
|
30
|
+
ARG_FLAGS = {
|
|
31
|
+
model: "-m",
|
|
32
|
+
model_alias: "-a",
|
|
33
|
+
ctx: "-c",
|
|
34
|
+
flash_attn: "-fa",
|
|
35
|
+
cache_type: "-ctk",
|
|
36
|
+
cache_type_v: "-ctv",
|
|
37
|
+
n_gpu_layers: "-ngl",
|
|
38
|
+
ubatch: "-ub",
|
|
39
|
+
cram: "-cram",
|
|
40
|
+
n_slots: "-np",
|
|
41
|
+
host: "--host",
|
|
42
|
+
port: "--port",
|
|
43
|
+
log_file: "--log-file",
|
|
44
|
+
reasoning: "--reasoning",
|
|
45
|
+
chat_template_kwargs: "--chat-template-kwargs",
|
|
46
|
+
chat_template: "--chat-template-file",
|
|
47
|
+
mmproj: "--mmproj",
|
|
48
|
+
n_cpu_moe: "--n-cpu-moe",
|
|
49
|
+
spec_type: "--spec-type",
|
|
50
|
+
cuda: "-cuda",
|
|
51
|
+
}.freeze
|
|
52
|
+
|
|
53
|
+
# ik's server answers /v1/models without llama.cpp's router extensions,
|
|
54
|
+
# so status.args is not there to read -- the argv it was launched with
|
|
55
|
+
# is only on disk, in the record written beside the log.
|
|
56
|
+
def live_args
|
|
57
|
+
LAUNCH_RECORD.read.split
|
|
58
|
+
rescue SystemCallError
|
|
59
|
+
nil
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def render_preset(n_cpu_moe = 0)
|
|
63
|
+
ensure_chat_template
|
|
64
|
+
binary = find_binary || "llama-server"
|
|
65
|
+
Paths::LLM_RUN_DIR.mkpath
|
|
66
|
+
LAUNCH_RECORD.write("#{build_command(binary, n_cpu_moe).join(' ')}\n")
|
|
67
|
+
LAUNCH_RECORD
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def start(remember: true)
|
|
71
|
+
return Result.ok("ik_llama server already running") if running?
|
|
72
|
+
|
|
73
|
+
binary = find_binary
|
|
74
|
+
return Result.error("ik_llama llama-server not found -- run `mimic setup`") if binary.nil?
|
|
75
|
+
|
|
76
|
+
missing = model_missing
|
|
77
|
+
return Result.error(missing) if missing
|
|
78
|
+
|
|
79
|
+
model_path = @settings.model.path.to_s
|
|
80
|
+
@drafting = mtp_enabled?
|
|
81
|
+
if @drafting && drafting_costs_offload?(model_path)
|
|
82
|
+
puts "[ik_llama] MTP would push the experts off the card at this context -- serving undrafted instead"
|
|
83
|
+
@drafting = false
|
|
84
|
+
end
|
|
85
|
+
n_cpu_moe = [@settings.model.file.memory.n_cpu_moe || 0,
|
|
86
|
+
preflight_offload(model_path, drafting: @drafting)].max
|
|
87
|
+
loop do
|
|
88
|
+
return port_busy_error("ik_llama server") unless await_port_release
|
|
89
|
+
|
|
90
|
+
puts "[ik_llama] offloading MoE experts of first #{n_cpu_moe} layers to CPU" if n_cpu_moe.positive?
|
|
91
|
+
pid = launch(binary, n_cpu_moe)
|
|
92
|
+
if Util::Probes.wait_ready(models_url, alive: -> { Util::Probes.child_alive?(pid) }, label: "waiting for ik_llama server")
|
|
93
|
+
@settings.model.file.memory.remember_n_cpu_moe(n_cpu_moe)
|
|
94
|
+
@settings.remember_ctx if remember
|
|
95
|
+
@settings.remember_sampling if remember
|
|
96
|
+
record_compute(pid)
|
|
97
|
+
report_template_failures
|
|
98
|
+
return Result.ok("ik_llama server ready (pid #{pid})")
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
tail = process.log_tail
|
|
102
|
+
process.cleanup_failed(pid)
|
|
103
|
+
|
|
104
|
+
# Checked ahead of retry_blocker: this failure carries the same
|
|
105
|
+
# "out of memory" the blocker keys on, and it would answer with the
|
|
106
|
+
# offload verdict for a model whose experts are not the problem.
|
|
107
|
+
if @drafting && tail.include?(MTP_OOM_MARKER)
|
|
108
|
+
puts "[ik_llama] no VRAM left for the MTP draft context -- restarting without speculative decoding"
|
|
109
|
+
@drafting = false
|
|
110
|
+
next
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
blocker = retry_blocker(tail, n_cpu_moe)
|
|
114
|
+
if blocker
|
|
115
|
+
return Result.error("ik_llama server failed to start -- #{blocker}. See #{process.log_file}\n#{process.last_lines(tail)}")
|
|
116
|
+
end
|
|
117
|
+
ceiling = [CPU_MOE_MAX, meta_int(tail, "n_layer") || CPU_MOE_MAX].min
|
|
118
|
+
n_cpu_moe = [n_cpu_moe + CPU_MOE_STEP, ceiling].min
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
private
|
|
123
|
+
|
|
124
|
+
def meta_int(tail, key)
|
|
125
|
+
match = tail.match(/^llm_load_print_meta: #{Regexp.escape(key)}\s*= (\d+)$/)
|
|
126
|
+
match && match[1].to_i
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Why bumping --n-cpu-moe cannot rescue this failure, if it can't.
|
|
130
|
+
#
|
|
131
|
+
# The flag only relocates MoE expert tensors, so it is useless on a
|
|
132
|
+
# dense model and cannot shrink a KV cache. Retrying regardless just
|
|
133
|
+
# reloads the weights until the ceiling, which on a dense model means
|
|
134
|
+
# every attempt is byte-identical to the one that already failed.
|
|
135
|
+
def retry_blocker(tail, n_cpu_moe)
|
|
136
|
+
bind_failure = tail[/#{Regexp.escape(BIND_MARKER)}.*?port=(\d+)/m, 1]
|
|
137
|
+
return "port #{bind_failure} is still held by another process" if bind_failure
|
|
138
|
+
return "startup failed for a reason other than VRAM" unless OOM_MARKERS.any? { |m| tail.include?(m) }
|
|
139
|
+
return "this model is dense, so it has no experts to offload" if meta_int(tail, "n_expert") == 0
|
|
140
|
+
|
|
141
|
+
n_layer = meta_int(tail, "n_layer")
|
|
142
|
+
ceiling = [CPU_MOE_MAX, n_layer || CPU_MOE_MAX].min
|
|
143
|
+
return nil if n_cpu_moe < ceiling
|
|
144
|
+
|
|
145
|
+
return "the KV cache does not fit -- lower `model.context_length`" if tail.include?(KV_OOM_MARKER)
|
|
146
|
+
|
|
147
|
+
"every layer is already offloaded (ceiling #{ceiling})"
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# The retry loop below only reacts to a launch that fails outright, but
|
|
151
|
+
# an over-committed context loads and then dies once the KV cache
|
|
152
|
+
# fills -- so the cache stays pinned at whatever last started. Ask the
|
|
153
|
+
# estimate up front instead of waiting for a crash that never reaches
|
|
154
|
+
# the loop.
|
|
155
|
+
def preflight_offload(model_path, drafting: false)
|
|
156
|
+
model = @settings.model.file
|
|
157
|
+
ctx = @settings.model.context_length
|
|
158
|
+
cache_type = @settings.llama.cache_type
|
|
159
|
+
estimate = model.footprint(ctx, cache_type, compute_mib: model.memory.compute_mib(ctx, cache_type),
|
|
160
|
+
drafting: drafting, slots: @settings.llama.n_slots,
|
|
161
|
+
reclaimable_mib: process.gpu_mib,
|
|
162
|
+
mmproj_path: @settings.model.mmproj_path)
|
|
163
|
+
return 0 if !estimate.measurable? || estimate.fits?
|
|
164
|
+
|
|
165
|
+
ceiling = [CPU_MOE_MAX, Model::Header.new(model_path).block_count || CPU_MOE_MAX].min
|
|
166
|
+
estimate.layers_to_offload(ceiling: ceiling) || 0
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Drafting is only worth having when it is free of offload.
|
|
170
|
+
#
|
|
171
|
+
# MTP accepts ~87% of what it drafts and is worth +27-37% decode at a
|
|
172
|
+
# fixed offload, but its second context costs a nextn KV plus a whole
|
|
173
|
+
# duplicate compute buffer -- 1148 MiB on Qwen3.6-35B-A3B at ctx
|
|
174
|
+
# 163840. When that is what tips the model off the card, the experts it
|
|
175
|
+
# displaces cost more than the drafting returns: measured 2026-08-23 at
|
|
176
|
+
# that context, 82-88 tok/s undrafted with everything resident against
|
|
177
|
+
# 67-68 tok/s drafting with 16 layers of experts on the CPU.
|
|
178
|
+
def drafting_costs_offload?(model_path)
|
|
179
|
+
return true if preflight_offload(model_path, drafting: true) > preflight_offload(model_path, drafting: false)
|
|
180
|
+
|
|
181
|
+
# Fitting on paper is not enough. The draft context's compute buffer
|
|
182
|
+
# is one contiguous cudaMalloc the size of the target's, so a launch
|
|
183
|
+
# that clears the budget by less than that still dies with
|
|
184
|
+
# "allocating 978.00 MiB on device 0: cudaMalloc failed" -- observed
|
|
185
|
+
# at ctx 163840, where the estimate had ~600 MiB spare.
|
|
186
|
+
estimate = drafting_estimate
|
|
187
|
+
return false unless estimate&.measurable?
|
|
188
|
+
|
|
189
|
+
estimate.headroom_mib < estimate.compute_mib
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def drafting_estimate
|
|
193
|
+
ctx = @settings.model.context_length
|
|
194
|
+
cache_type = @settings.llama.cache_type
|
|
195
|
+
model = @settings.model.file
|
|
196
|
+
model.footprint(ctx, cache_type, compute_mib: model.memory.compute_mib(ctx, cache_type), drafting: true,
|
|
197
|
+
slots: @settings.llama.n_slots, reclaimable_mib: process.gpu_mib,
|
|
198
|
+
mmproj_path: @settings.model.mmproj_path)
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def build_command(binary, n_cpu_moe = 0)
|
|
202
|
+
flagged = launch_values(n_cpu_moe).flat_map do |key, value|
|
|
203
|
+
value.nil? ? [] : [ARG_FLAGS.fetch(key), value.to_s]
|
|
204
|
+
end
|
|
205
|
+
# ik defaults parallel tool calls off and, unlike mainline, never
|
|
206
|
+
# probes the chat template for the capability -- without this the
|
|
207
|
+
# grammar allows exactly one tool call and generation ends there.
|
|
208
|
+
#
|
|
209
|
+
# --swa-compress sizes a sliding-window layer's cache at the window,
|
|
210
|
+
# as mainline does by default. The footprint already prices it that
|
|
211
|
+
# way. Off, gemma-4-26B-A4B at ctx 60160 held a 6866 MiB KV cache
|
|
212
|
+
# that the estimate called ~1 GiB. ik ignores it, with a warning, on
|
|
213
|
+
# an arch it does not implement.
|
|
214
|
+
[binary, *flagged, "--jinja", "--metrics", "-ptcall", "--swa-compress", *@settings.model.sampling.to_args]
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def launch_values(n_cpu_moe)
|
|
218
|
+
{
|
|
219
|
+
model: @settings.model.path,
|
|
220
|
+
model_alias: model_alias,
|
|
221
|
+
ctx: @settings.model.context_length,
|
|
222
|
+
flash_attn: @settings.llama.flash_attn,
|
|
223
|
+
cache_type: @settings.llama.cache_type,
|
|
224
|
+
cache_type_v: @settings.llama.cache_type,
|
|
225
|
+
n_gpu_layers: 999,
|
|
226
|
+
ubatch: 1024,
|
|
227
|
+
cram: 2048,
|
|
228
|
+
n_slots: @settings.llama.n_slots,
|
|
229
|
+
host: @settings.llama.host,
|
|
230
|
+
port: @settings.llama.port,
|
|
231
|
+
# ik_llama's logger (llama.cpp's pre-revamp common/log.h) otherwise
|
|
232
|
+
# writes a bare "llama.log" into the cwd. Name it for the engine and
|
|
233
|
+
# put it beside the server log. Takes a path without the extension.
|
|
234
|
+
log_file: TRACE_LOG_BASE,
|
|
235
|
+
reasoning: @settings.model.reasoning,
|
|
236
|
+
# ik has no --reasoning-effort; the template reads the same variable
|
|
237
|
+
# from its kwargs.
|
|
238
|
+
chat_template_kwargs: effort_kwargs,
|
|
239
|
+
chat_template: chat_template_override,
|
|
240
|
+
mmproj: @settings.model.mmproj_path,
|
|
241
|
+
n_cpu_moe: (n_cpu_moe if n_cpu_moe.positive?),
|
|
242
|
+
# ik packs the stage's parameters into the value, where mainline
|
|
243
|
+
# spells the same thing as `draft-mtp` plus a separate
|
|
244
|
+
# --spec-draft-n-max. Its other stage types all want a draft model
|
|
245
|
+
# or a corpus; mtp is the only one the GGUF alone can feed.
|
|
246
|
+
spec_type: ("mtp:n_max=#{SPEC_DRAFT_N_MAX}" if drafting?),
|
|
247
|
+
# Workaround for ik_llama#1232 (open): ik caches one CUDA graph per
|
|
248
|
+
# distinct compute graph and never evicts, so VRAM climbs ~44 MiB/min
|
|
249
|
+
# under sustained load until the server dies mid-run. Measured here:
|
|
250
|
+
# +2440 MiB over 56 min with graphs on, +0 over 10 min with them off.
|
|
251
|
+
# The author suggested this flag and has not landed a real fix;
|
|
252
|
+
# mainline's LRU eviction (ggml-org/llama.cpp#21611) is not ported.
|
|
253
|
+
# Costs ~10% token generation -- drop this once ik evicts.
|
|
254
|
+
cuda: "graphs=0",
|
|
255
|
+
}
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# render_preset runs outside start (`llm status`, preset rendering), so
|
|
259
|
+
# the launch-time override has to fall back to the configured answer.
|
|
260
|
+
def drafting? = defined?(@drafting) ? @drafting : mtp_enabled?
|
|
261
|
+
|
|
262
|
+
def effort_kwargs
|
|
263
|
+
effort = @settings.model.reasoning_effort
|
|
264
|
+
JSON.generate("reasoning_effort" => effort) if effort
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def launch(binary, n_cpu_moe)
|
|
268
|
+
render_preset(n_cpu_moe)
|
|
269
|
+
launcher.spawn(build_command(binary, n_cpu_moe))
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
module Engine
|
|
5
|
+
class Launcher
|
|
6
|
+
def initialize(log_file:, pid_file:, chdir:)
|
|
7
|
+
@log_file = log_file
|
|
8
|
+
@pid_file = pid_file
|
|
9
|
+
@chdir = chdir
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def spawn(command)
|
|
13
|
+
@pid_file.dirname.mkpath
|
|
14
|
+
log = File.open(@log_file, "ab")
|
|
15
|
+
begin
|
|
16
|
+
pid = Process.spawn(*command, out: log, err: log, chdir: @chdir.to_s, pgroup: true)
|
|
17
|
+
ensure
|
|
18
|
+
log.close
|
|
19
|
+
end
|
|
20
|
+
@pid_file.write(pid.to_s)
|
|
21
|
+
pid
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
require_relative "base"
|
|
2
|
+
require_relative "../engine_cli/llama_preset"
|
|
3
|
+
|
|
4
|
+
module Mimic
|
|
5
|
+
module Engine
|
|
6
|
+
class Llama < Base
|
|
7
|
+
PRESET_FILE = Paths::LLM_RUN_DIR / "llama-models.ini"
|
|
8
|
+
|
|
9
|
+
# The launch surface is the preset file, not the argv, so these are the
|
|
10
|
+
# preset's own keys re-dashed into the flags the router expands them to
|
|
11
|
+
# in /v1/models. Only the three observed there are listed -- notably no
|
|
12
|
+
# n_cpu_moe, which is how `llm status` knows to leave the line off.
|
|
13
|
+
ARG_FLAGS = %i[ctx cache_type n_slots mmproj reasoning reasoning_effort].to_h { |key| [key, "--#{EngineCli::LlamaPreset::KEYS.fetch(key)}"] }.freeze
|
|
14
|
+
|
|
15
|
+
# argv of the actually-running instance, straight from the router's own
|
|
16
|
+
# /v1/models -- config.yaml can be stale (e.g. `llm parallel` overrides
|
|
17
|
+
# ctx/slots without persisting them).
|
|
18
|
+
def live_args
|
|
19
|
+
data = Util::Probes.http_json(models_url)
|
|
20
|
+
return nil if data.nil?
|
|
21
|
+
|
|
22
|
+
entry = (data["data"] || []).find { |e| e["id"] == model_alias && e.dig("status", "value") == "loaded" }
|
|
23
|
+
entry&.dig("status", "args")
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def render_preset
|
|
27
|
+
ensure_chat_template
|
|
28
|
+
drafter = mtp_drafter
|
|
29
|
+
preset = EngineCli::LlamaPreset.new(@settings, model_alias: model_alias, chat_template: chat_template_override,
|
|
30
|
+
mtp: mtp_enabled? || !drafter.nil?, mtp_drafter: drafter,
|
|
31
|
+
mmproj: @settings.model.mmproj_path, options: binary_options)
|
|
32
|
+
preset.write(PRESET_FILE)
|
|
33
|
+
report_dropped(preset.dropped)
|
|
34
|
+
PRESET_FILE
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def start(remember: true)
|
|
38
|
+
return Result.ok("llama-server already running") if running?
|
|
39
|
+
|
|
40
|
+
binary = find_binary
|
|
41
|
+
return Result.error("llama-server binary not found -- run `mimic setup`") if binary.nil?
|
|
42
|
+
|
|
43
|
+
missing = model_missing
|
|
44
|
+
return Result.error(missing) if missing
|
|
45
|
+
return port_busy_error("llama-server") unless await_port_release
|
|
46
|
+
|
|
47
|
+
render_preset
|
|
48
|
+
log_offset = process.log_size
|
|
49
|
+
pid = launcher.spawn(build_command(binary))
|
|
50
|
+
|
|
51
|
+
unless Util::Probes.wait_ready(models_url, alive: -> { Util::Probes.child_alive?(pid) }, label: "waiting for llama-server")
|
|
52
|
+
process.cleanup_failed(pid)
|
|
53
|
+
return Result.error("llama-server failed to start -- see #{process.log_file}\n#{process.last_lines(process.log_tail(log_offset))}")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
failure = []
|
|
57
|
+
instance_failed = lambda do
|
|
58
|
+
reason = instance_failure(log_offset)
|
|
59
|
+
failure << reason if reason
|
|
60
|
+
!reason.nil?
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
if Util::Probes.wait_until(attempts: 120, alive: -> { Util::Probes.child_alive?(pid) },
|
|
64
|
+
label: "loading model", abort: instance_failed) { model_loaded? }
|
|
65
|
+
@settings.remember_ctx if remember
|
|
66
|
+
@settings.remember_sampling if remember
|
|
67
|
+
record_compute(pid)
|
|
68
|
+
report_template_failures
|
|
69
|
+
return Result.ok("llama-server ready, model loaded (pid #{pid})")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
process.cleanup_failed(pid)
|
|
73
|
+
tail = process.last_lines(process.log_tail(log_offset))
|
|
74
|
+
return Result.error("#{failure.first} -- see #{process.log_file}\n#{tail}") unless failure.empty?
|
|
75
|
+
|
|
76
|
+
Result.error("model failed to load -- see #{process.log_file}\n#{tail}")
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
def binary_options
|
|
82
|
+
binary = find_binary
|
|
83
|
+
binary && EngineCli::BinaryOptions.new(binary)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def mtp_drafter = (@settings.model.file.drafter unless mtp_off?)
|
|
87
|
+
|
|
88
|
+
def report_dropped(dropped)
|
|
89
|
+
return if dropped.empty?
|
|
90
|
+
|
|
91
|
+
puts "[preset] #{@settings.llama.engine} does not carry " \
|
|
92
|
+
"#{dropped.map { |name| "--#{name}" }.join(', ')} -- left out of the preset"
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def build_command(binary)
|
|
96
|
+
[binary, "--models-preset", PRESET_FILE.to_s, "--host", @settings.llama.host,
|
|
97
|
+
"--port", @settings.llama.port.to_s, "--reuse-port", "--metrics"]
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def model_loaded?
|
|
101
|
+
data = Util::Probes.http_json("#{@settings.llama.base_url}/v1/models") || {}
|
|
102
|
+
(data["data"] || []).any? { |entry| entry["id"] == model_alias && entry.dig("status", "value") == "loaded" }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Why the router's model instance died, reading only this launch's log.
|
|
106
|
+
#
|
|
107
|
+
# The router survives its children, so a child that OOMs looks
|
|
108
|
+
# identical to one still loading: the poll that checks for it
|
|
109
|
+
# triggers another autoload, and the pair loop until the caller's
|
|
110
|
+
# attempts run out. The child's exit is only ever reported in the
|
|
111
|
+
# shared log.
|
|
112
|
+
def instance_failure(log_offset)
|
|
113
|
+
tail = process.log_tail(log_offset)
|
|
114
|
+
return nil unless tail.include?("instance name=#{model_alias} exited with status 1")
|
|
115
|
+
|
|
116
|
+
if tail.include?("cudaMalloc failed: out of memory") || tail.include?("failed to allocate")
|
|
117
|
+
return "#{model_alias} ran out of VRAM at #{@settings.model.context_length} ctx -- lower the context"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
"#{model_alias} instance exited during load"
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|