mimic-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/NOTICE +16 -0
  4. data/README.md +175 -0
  5. data/exe/lllm +5 -0
  6. data/exe/llm +5 -0
  7. data/exe/mimic +5 -0
  8. data/lib/mimic/README.md +144 -0
  9. data/lib/mimic/agent/claude.rb +305 -0
  10. data/lib/mimic/agent/codex.rb +254 -0
  11. data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
  12. data/lib/mimic/agent/hermes.rb +107 -0
  13. data/lib/mimic/agent/muse.rb +106 -0
  14. data/lib/mimic/agent.rb +46 -0
  15. data/lib/mimic/cli.rb +159 -0
  16. data/lib/mimic/client/base.rb +44 -0
  17. data/lib/mimic/client/crush.rb +28 -0
  18. data/lib/mimic/client/goose.rb +52 -0
  19. data/lib/mimic/client/hermes.rb +21 -0
  20. data/lib/mimic/client/opencode.rb +29 -0
  21. data/lib/mimic/client/pi.rb +50 -0
  22. data/lib/mimic/client.rb +19 -0
  23. data/lib/mimic/engine/README.md +140 -0
  24. data/lib/mimic/engine/base.rb +138 -0
  25. data/lib/mimic/engine/chat_template.rb +207 -0
  26. data/lib/mimic/engine/ik_llama.rb +273 -0
  27. data/lib/mimic/engine/launcher.rb +25 -0
  28. data/lib/mimic/engine/llama.rb +124 -0
  29. data/lib/mimic/engine/llama_embed.rb +108 -0
  30. data/lib/mimic/engine/router.rb +57 -0
  31. data/lib/mimic/engine/server_process.rb +108 -0
  32. data/lib/mimic/engine.rb +5 -0
  33. data/lib/mimic/engine_cli/binary_options.rb +46 -0
  34. data/lib/mimic/engine_cli/builder.rb +98 -0
  35. data/lib/mimic/engine_cli/diffusion.rb +117 -0
  36. data/lib/mimic/engine_cli/llama_preset.rb +119 -0
  37. data/lib/mimic/engine_cli/registry.rb +94 -0
  38. data/lib/mimic/engine_cli/source.rb +110 -0
  39. data/lib/mimic/engine_cli/status.rb +161 -0
  40. data/lib/mimic/engine_cli/update.rb +68 -0
  41. data/lib/mimic/engine_cli.rb +7 -0
  42. data/lib/mimic/health/checks.rb +92 -0
  43. data/lib/mimic/health.rb +82 -0
  44. data/lib/mimic/llm_cli/base.rb +36 -0
  45. data/lib/mimic/llm_cli/configure.rb +84 -0
  46. data/lib/mimic/llm_cli/embed.rb +80 -0
  47. data/lib/mimic/llm_cli/engines.rb +44 -0
  48. data/lib/mimic/llm_cli/models.rb +31 -0
  49. data/lib/mimic/llm_cli/models_table.rb +119 -0
  50. data/lib/mimic/llm_cli/paths.rb +48 -0
  51. data/lib/mimic/llm_cli/root.rb +215 -0
  52. data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
  53. data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
  54. data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
  55. data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
  56. data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
  57. data/lib/mimic/llm_cli.rb +5 -0
  58. data/lib/mimic/llm_config/embed.rb +48 -0
  59. data/lib/mimic/llm_config/llama_settings.rb +60 -0
  60. data/lib/mimic/llm_config/model_settings.rb +84 -0
  61. data/lib/mimic/llm_config/sampling.rb +54 -0
  62. data/lib/mimic/llm_config/settings.rb +92 -0
  63. data/lib/mimic/llm_config.rb +132 -0
  64. data/lib/mimic/model/footprint.rb +303 -0
  65. data/lib/mimic/model/gguf.rb +204 -0
  66. data/lib/mimic/model/header.rb +169 -0
  67. data/lib/mimic/model/library.rb +132 -0
  68. data/lib/mimic/model/memory.rb +126 -0
  69. data/lib/mimic/model/store.rb +46 -0
  70. data/lib/mimic/model/upstream.rb +81 -0
  71. data/lib/mimic/model.rb +116 -0
  72. data/lib/mimic/paths.rb +45 -0
  73. data/lib/mimic/result.rb +8 -0
  74. data/lib/mimic/ui.rb +89 -0
  75. data/lib/mimic/util/fmt.rb +15 -0
  76. data/lib/mimic/util/hf.rb +55 -0
  77. data/lib/mimic/util/host.rb +95 -0
  78. data/lib/mimic/util/probes.rb +255 -0
  79. data/lib/mimic/version.rb +3 -0
  80. data/lib/mimic.rb +22 -0
  81. metadata +186 -0
@@ -0,0 +1,207 @@
1
+ require "pathname"
2
+ require_relative "../paths"
3
+ require_relative "../model/header"
4
+ require_relative "../model"
5
+ require_relative "../util/probes"
6
+
7
+ module Mimic
8
+ # A model's embedded Jinja chat template, and the override mimic hands the
9
+ # server when that template would throw on real agent traffic.
10
+ #
11
+ # The Qwen-family templates ship `raise_exception` guards that llama.cpp's
12
+ # minja evaluates at render time: a request whose message list trips one
13
+ # comes back as a 500 rather than a completion. Claude Code trips
14
+ # `System message must be at the beginning` because llama.cpp's
15
+ # /v1/messages bridge emits system content the guard does not expect.
16
+ #
17
+ # Each rule replaces one guard with what the template should have rendered.
18
+ # `requires` names the constructs the replacement leans on -- a template
19
+ # that phrases its loop differently gets that guard left alone rather than
20
+ # rewritten wrong, and the hand-written override in CHAT_TEMPLATES_DIR
21
+ # stays the answer for it.
22
+ module Engine
23
+ class ChatTemplate
24
+ METADATA_KEY = "tokenizer.chat_template".freeze
25
+ RAISE_CALL = /\{\{-?\s*raise_exception\(\s*(['"])(.*?)\1\s*\)\s*-?\}\}/m
26
+
27
+ NOOP = "{{- '' }}".freeze
28
+
29
+ EFFORT_DEFAULT = /reasoning_effort\s*\|\s*default\(\s*['"]([\w-]+)['"]/
30
+ EFFORT_FALLBACK = /set\s+reasoning_effort\s*=\s*['"]([\w-]+)['"]/
31
+ EFFORT_CHOICES = /reasoning_effort\s+not\s+in\s+[(\[]([^)\]]*)[)\]]/
32
+ EFFORT_COMPARE = /reasoning_effort\s*==\s*['"]([\w-]+)['"]/
33
+ QUOTED = /['"]([\w-]+)['"]/
34
+
35
+ # gpt-oss's harmony template prints reasoning_effort without listing
36
+ # the values; these are the ones the format defines.
37
+ HARMONY_CHANNEL = "<|channel|>".freeze
38
+ HARMONY_EFFORTS = %w[low medium high].freeze
39
+
40
+ # llama.cpp's --reasoning auto renders the template and asks its format
41
+ # handler; these are the markers those handlers turn thinking on for.
42
+ THINKING_MARKERS = ["enable_thinking", "<think>", "[THINK]", HARMONY_CHANNEL].freeze
43
+
44
+ Rule = Struct.new(:match, :replacement, :requires, keyword_init: true)
45
+
46
+ RULES = [
47
+ Rule.new(
48
+ match: /system message must be at the beginning/i,
49
+ replacement: %q({{- '<|im_start|>system\n' + content + '<|im_end|>' + '\n' }}),
50
+ requires: ["<|im_start|>system", "set content = render_content("],
51
+ ),
52
+ Rule.new(
53
+ match: /unexpected message role/i,
54
+ replacement: %q({{- '<|im_start|>user\n' + content + '<|im_end|>' + '\n' }}),
55
+ requires: ["<|im_start|>system", "set content = render_content("],
56
+ ),
57
+ Rule.new(
58
+ match: /unexpected item type in content/i,
59
+ replacement: "{{- item.text if item.text is defined else '' }}",
60
+ requires: ["for item in content"],
61
+ ),
62
+ Rule.new(
63
+ match: /unexpected content type/i,
64
+ replacement: "{{- content | string }}",
65
+ requires: [],
66
+ ),
67
+ Rule.new(
68
+ match: /system message cannot contain|no messages provided|no user query found/i,
69
+ replacement: NOOP,
70
+ requires: [],
71
+ ),
72
+ ].freeze
73
+
74
+ # Renders the message shapes real agents send through the server's own
75
+ # /apply-template, which runs the same minja pass as a completion. A 500
76
+ # here is the template throwing -- the exact failure a request would hit,
77
+ # measured instead of predicted. Both engines expose the endpoint.
78
+ class Probe
79
+ SYSTEM = { role: "system", content: "You are a coding agent." }.freeze
80
+ USER = { role: "user", content: "list the files" }.freeze
81
+
82
+ FIXTURES = {
83
+ "a system message after the first turn" =>
84
+ [SYSTEM, USER, { role: "system", content: "<system-reminder>stay on task</system-reminder>" }],
85
+ "an assistant tool call and its response" =>
86
+ [SYSTEM, USER,
87
+ { role: "assistant", content: "",
88
+ tool_calls: [{ type: "function",
89
+ function: { name: "Read", arguments: { path: "/tmp/x" } } }] },
90
+ { role: "tool", content: "file body" }],
91
+ "content sent as typed blocks" =>
92
+ [SYSTEM, { role: "user", content: [{ type: "text", text: "and now?" }] }],
93
+ }.freeze
94
+
95
+ def initialize(base_url, model_alias:)
96
+ @url = "#{base_url}/apply-template"
97
+ @model_alias = model_alias
98
+ end
99
+
100
+ def failures
101
+ FIXTURES.filter_map do |label, messages|
102
+ res = Util::Probes.post(@url, { model: @model_alias, messages: messages })
103
+ label if res && res.code.to_i >= 500
104
+ end
105
+ end
106
+ end
107
+
108
+ def initialize(model_path, model_alias:)
109
+ @model_path = model_path.to_s
110
+ @model_alias = model_alias.to_s
111
+ end
112
+
113
+ def source
114
+ return @source if defined?(@source)
115
+
116
+ # full: the template is written after the vocab arrays, and the
117
+ # default read stops at the first of them.
118
+ @source = Model::Header.new(@model_path).metadata(full: true)[METADATA_KEY]
119
+ rescue StandardError
120
+ @source = nil
121
+ end
122
+
123
+ def guarded? = source.to_s.include?("raise_exception")
124
+
125
+ # The reasoning_effort values the served template accepts, in its own
126
+ # order: an override when one exists, the GGUF's template otherwise.
127
+ # Fewer than two means there is nothing to choose between.
128
+ def effort_levels
129
+ text = served_source.to_s
130
+ listed = text[EFFORT_CHOICES, 1]
131
+ levels = listed ? listed.scan(QUOTED).flatten : text.scan(EFFORT_COMPARE).flatten
132
+ levels = HARMONY_EFFORTS if levels.empty? && harmony?(text)
133
+ levels = (levels + [default_effort].compact).uniq
134
+ levels.size < 2 ? [] : levels
135
+ end
136
+
137
+ def default_effort
138
+ text = served_source.to_s
139
+ text[EFFORT_DEFAULT, 1] || text[EFFORT_FALLBACK, 1]
140
+ end
141
+
142
+ def thinks?
143
+ text = served_source.to_s
144
+ THINKING_MARKERS.any? { |marker| text.include?(marker) }
145
+ end
146
+
147
+ # Hand-written first: a file the user wrote is a decision, and mimic's
148
+ # rules do not get to overrule it.
149
+ def override_path
150
+ [hand_written, generated].find(&:file?)
151
+ end
152
+
153
+ def install
154
+ return nil if hand_written.file?
155
+
156
+ text = sanitized
157
+ return nil if text.nil?
158
+
159
+ generated.dirname.mkpath
160
+ generated.write(text.end_with?("\n") ? text : "#{text}\n")
161
+ generated
162
+ end
163
+
164
+ private
165
+
166
+ # The rewritten template, or nil when no guard in it has a rule.
167
+ #
168
+ # Per guard, not per template: a guard mimic has no rule for is left
169
+ # standing exactly as the model shipped it, so the shapes that trip it
170
+ # keep failing the way they already do. Refusing the whole rewrite over
171
+ # one such guard is what would change behaviour -- it drops repairs that
172
+ # are correct and leaves a template that 500s on ordinary agent traffic.
173
+ def sanitized
174
+ return nil unless guarded?
175
+
176
+ rewrote = false
177
+ rewritten = source.gsub(RAISE_CALL) do
178
+ whole = Regexp.last_match(0)
179
+ rule = rule_for(Regexp.last_match(2))
180
+ next whole if rule.nil?
181
+
182
+ rewrote = true
183
+ rule.replacement
184
+ end
185
+ rewrote ? rewritten : nil
186
+ end
187
+
188
+ def hand_written = Paths::CHAT_TEMPLATES_DIR / "#{@model_alias}.jinja"
189
+ def generated = Paths::GENERATED_TEMPLATES_DIR / "#{@model_alias}.jinja"
190
+
191
+ def served_source
192
+ override = override_path
193
+ override ? override.read : source
194
+ end
195
+
196
+ def harmony?(text) = text.include?(HARMONY_CHANNEL) && text.include?("reasoning_effort")
197
+
198
+ def rule_for(message)
199
+ rule = RULES.find { |r| r.match.match?(message) }
200
+ return nil if rule.nil?
201
+ return nil unless rule.requires.all? { |needle| source.include?(needle) }
202
+
203
+ rule
204
+ end
205
+ end
206
+ end
207
+ end
@@ -0,0 +1,273 @@
1
+ require "json"
2
+ require_relative "../model"
3
+ require_relative "base"
4
+
5
+ module Mimic
6
+ module Engine
7
+ class IkLlama < Base
8
+ LAUNCH_RECORD = Paths::LLM_RUN_DIR / "ik-llama-launch.txt"
9
+ TRACE_LOG_BASE = Paths::LLM_RUN_DIR / "ik_llama"
10
+
11
+ # Layers offloaded to CPU on each retry after an OOM, and the ceiling
12
+ # we give up at -- generous enough for any current MoE GGUF's layer
13
+ # count.
14
+ CPU_MOE_STEP = 2
15
+ CPU_MOE_MAX = 28
16
+
17
+ OOM_MARKERS = ["cudaMalloc failed", "out of memory"].freeze
18
+ BIND_MARKER = "couldn't bind to server socket"
19
+ KV_OOM_MARKER = "failed to allocate buffer for kv cache"
20
+
21
+ # ik builds the MTP draft context against the target model after the
22
+ # weights and KV are already resident, and it is not in the footprint
23
+ # estimate -- so the stage can be what tips a load that would otherwise
24
+ # have fit. Recoverable: the same argv minus --spec-type starts.
25
+ MTP_OOM_MARKER = "failed to create MTP context"
26
+
27
+ # Every flag mimic sets, in argv order: `build_command` spells them from
28
+ # here and `ServerStatus` reads a launched argv back through the same
29
+ # table. Valueless switches stay out of it -- there is nothing to read.
30
+ ARG_FLAGS = {
31
+ model: "-m",
32
+ model_alias: "-a",
33
+ ctx: "-c",
34
+ flash_attn: "-fa",
35
+ cache_type: "-ctk",
36
+ cache_type_v: "-ctv",
37
+ n_gpu_layers: "-ngl",
38
+ ubatch: "-ub",
39
+ cram: "-cram",
40
+ n_slots: "-np",
41
+ host: "--host",
42
+ port: "--port",
43
+ log_file: "--log-file",
44
+ reasoning: "--reasoning",
45
+ chat_template_kwargs: "--chat-template-kwargs",
46
+ chat_template: "--chat-template-file",
47
+ mmproj: "--mmproj",
48
+ n_cpu_moe: "--n-cpu-moe",
49
+ spec_type: "--spec-type",
50
+ cuda: "-cuda",
51
+ }.freeze
52
+
53
+ # ik's server answers /v1/models without llama.cpp's router extensions,
54
+ # so status.args is not there to read -- the argv it was launched with
55
+ # is only on disk, in the record written beside the log.
56
+ def live_args
57
+ LAUNCH_RECORD.read.split
58
+ rescue SystemCallError
59
+ nil
60
+ end
61
+
62
+ def render_preset(n_cpu_moe = 0)
63
+ ensure_chat_template
64
+ binary = find_binary || "llama-server"
65
+ Paths::LLM_RUN_DIR.mkpath
66
+ LAUNCH_RECORD.write("#{build_command(binary, n_cpu_moe).join(' ')}\n")
67
+ LAUNCH_RECORD
68
+ end
69
+
70
+ def start(remember: true)
71
+ return Result.ok("ik_llama server already running") if running?
72
+
73
+ binary = find_binary
74
+ return Result.error("ik_llama llama-server not found -- run `mimic setup`") if binary.nil?
75
+
76
+ missing = model_missing
77
+ return Result.error(missing) if missing
78
+
79
+ model_path = @settings.model.path.to_s
80
+ @drafting = mtp_enabled?
81
+ if @drafting && drafting_costs_offload?(model_path)
82
+ puts "[ik_llama] MTP would push the experts off the card at this context -- serving undrafted instead"
83
+ @drafting = false
84
+ end
85
+ n_cpu_moe = [@settings.model.file.memory.n_cpu_moe || 0,
86
+ preflight_offload(model_path, drafting: @drafting)].max
87
+ loop do
88
+ return port_busy_error("ik_llama server") unless await_port_release
89
+
90
+ puts "[ik_llama] offloading MoE experts of first #{n_cpu_moe} layers to CPU" if n_cpu_moe.positive?
91
+ pid = launch(binary, n_cpu_moe)
92
+ if Util::Probes.wait_ready(models_url, alive: -> { Util::Probes.child_alive?(pid) }, label: "waiting for ik_llama server")
93
+ @settings.model.file.memory.remember_n_cpu_moe(n_cpu_moe)
94
+ @settings.remember_ctx if remember
95
+ @settings.remember_sampling if remember
96
+ record_compute(pid)
97
+ report_template_failures
98
+ return Result.ok("ik_llama server ready (pid #{pid})")
99
+ end
100
+
101
+ tail = process.log_tail
102
+ process.cleanup_failed(pid)
103
+
104
+ # Checked ahead of retry_blocker: this failure carries the same
105
+ # "out of memory" the blocker keys on, and it would answer with the
106
+ # offload verdict for a model whose experts are not the problem.
107
+ if @drafting && tail.include?(MTP_OOM_MARKER)
108
+ puts "[ik_llama] no VRAM left for the MTP draft context -- restarting without speculative decoding"
109
+ @drafting = false
110
+ next
111
+ end
112
+
113
+ blocker = retry_blocker(tail, n_cpu_moe)
114
+ if blocker
115
+ return Result.error("ik_llama server failed to start -- #{blocker}. See #{process.log_file}\n#{process.last_lines(tail)}")
116
+ end
117
+ ceiling = [CPU_MOE_MAX, meta_int(tail, "n_layer") || CPU_MOE_MAX].min
118
+ n_cpu_moe = [n_cpu_moe + CPU_MOE_STEP, ceiling].min
119
+ end
120
+ end
121
+
122
+ private
123
+
124
+ def meta_int(tail, key)
125
+ match = tail.match(/^llm_load_print_meta: #{Regexp.escape(key)}\s*= (\d+)$/)
126
+ match && match[1].to_i
127
+ end
128
+
129
+ # Why bumping --n-cpu-moe cannot rescue this failure, if it can't.
130
+ #
131
+ # The flag only relocates MoE expert tensors, so it is useless on a
132
+ # dense model and cannot shrink a KV cache. Retrying regardless just
133
+ # reloads the weights until the ceiling, which on a dense model means
134
+ # every attempt is byte-identical to the one that already failed.
135
+ def retry_blocker(tail, n_cpu_moe)
136
+ bind_failure = tail[/#{Regexp.escape(BIND_MARKER)}.*?port=(\d+)/m, 1]
137
+ return "port #{bind_failure} is still held by another process" if bind_failure
138
+ return "startup failed for a reason other than VRAM" unless OOM_MARKERS.any? { |m| tail.include?(m) }
139
+ return "this model is dense, so it has no experts to offload" if meta_int(tail, "n_expert") == 0
140
+
141
+ n_layer = meta_int(tail, "n_layer")
142
+ ceiling = [CPU_MOE_MAX, n_layer || CPU_MOE_MAX].min
143
+ return nil if n_cpu_moe < ceiling
144
+
145
+ return "the KV cache does not fit -- lower `model.context_length`" if tail.include?(KV_OOM_MARKER)
146
+
147
+ "every layer is already offloaded (ceiling #{ceiling})"
148
+ end
149
+
150
+ # The retry loop below only reacts to a launch that fails outright, but
151
+ # an over-committed context loads and then dies once the KV cache
152
+ # fills -- so the cache stays pinned at whatever last started. Ask the
153
+ # estimate up front instead of waiting for a crash that never reaches
154
+ # the loop.
155
+ def preflight_offload(model_path, drafting: false)
156
+ model = @settings.model.file
157
+ ctx = @settings.model.context_length
158
+ cache_type = @settings.llama.cache_type
159
+ estimate = model.footprint(ctx, cache_type, compute_mib: model.memory.compute_mib(ctx, cache_type),
160
+ drafting: drafting, slots: @settings.llama.n_slots,
161
+ reclaimable_mib: process.gpu_mib,
162
+ mmproj_path: @settings.model.mmproj_path)
163
+ return 0 if !estimate.measurable? || estimate.fits?
164
+
165
+ ceiling = [CPU_MOE_MAX, Model::Header.new(model_path).block_count || CPU_MOE_MAX].min
166
+ estimate.layers_to_offload(ceiling: ceiling) || 0
167
+ end
168
+
169
+ # Drafting is only worth having when it is free of offload.
170
+ #
171
+ # MTP accepts ~87% of what it drafts and is worth +27-37% decode at a
172
+ # fixed offload, but its second context costs a nextn KV plus a whole
173
+ # duplicate compute buffer -- 1148 MiB on Qwen3.6-35B-A3B at ctx
174
+ # 163840. When that is what tips the model off the card, the experts it
175
+ # displaces cost more than the drafting returns: measured 2026-08-23 at
176
+ # that context, 82-88 tok/s undrafted with everything resident against
177
+ # 67-68 tok/s drafting with 16 layers of experts on the CPU.
178
+ def drafting_costs_offload?(model_path)
179
+ return true if preflight_offload(model_path, drafting: true) > preflight_offload(model_path, drafting: false)
180
+
181
+ # Fitting on paper is not enough. The draft context's compute buffer
182
+ # is one contiguous cudaMalloc the size of the target's, so a launch
183
+ # that clears the budget by less than that still dies with
184
+ # "allocating 978.00 MiB on device 0: cudaMalloc failed" -- observed
185
+ # at ctx 163840, where the estimate had ~600 MiB spare.
186
+ estimate = drafting_estimate
187
+ return false unless estimate&.measurable?
188
+
189
+ estimate.headroom_mib < estimate.compute_mib
190
+ end
191
+
192
+ def drafting_estimate
193
+ ctx = @settings.model.context_length
194
+ cache_type = @settings.llama.cache_type
195
+ model = @settings.model.file
196
+ model.footprint(ctx, cache_type, compute_mib: model.memory.compute_mib(ctx, cache_type), drafting: true,
197
+ slots: @settings.llama.n_slots, reclaimable_mib: process.gpu_mib,
198
+ mmproj_path: @settings.model.mmproj_path)
199
+ end
200
+
201
+ def build_command(binary, n_cpu_moe = 0)
202
+ flagged = launch_values(n_cpu_moe).flat_map do |key, value|
203
+ value.nil? ? [] : [ARG_FLAGS.fetch(key), value.to_s]
204
+ end
205
+ # ik defaults parallel tool calls off and, unlike mainline, never
206
+ # probes the chat template for the capability -- without this the
207
+ # grammar allows exactly one tool call and generation ends there.
208
+ #
209
+ # --swa-compress sizes a sliding-window layer's cache at the window,
210
+ # as mainline does by default. The footprint already prices it that
211
+ # way. Off, gemma-4-26B-A4B at ctx 60160 held a 6866 MiB KV cache
212
+ # that the estimate called ~1 GiB. ik ignores it, with a warning, on
213
+ # an arch it does not implement.
214
+ [binary, *flagged, "--jinja", "--metrics", "-ptcall", "--swa-compress", *@settings.model.sampling.to_args]
215
+ end
216
+
217
+ def launch_values(n_cpu_moe)
218
+ {
219
+ model: @settings.model.path,
220
+ model_alias: model_alias,
221
+ ctx: @settings.model.context_length,
222
+ flash_attn: @settings.llama.flash_attn,
223
+ cache_type: @settings.llama.cache_type,
224
+ cache_type_v: @settings.llama.cache_type,
225
+ n_gpu_layers: 999,
226
+ ubatch: 1024,
227
+ cram: 2048,
228
+ n_slots: @settings.llama.n_slots,
229
+ host: @settings.llama.host,
230
+ port: @settings.llama.port,
231
+ # ik_llama's logger (llama.cpp's pre-revamp common/log.h) otherwise
232
+ # writes a bare "llama.log" into the cwd. Name it for the engine and
233
+ # put it beside the server log. Takes a path without the extension.
234
+ log_file: TRACE_LOG_BASE,
235
+ reasoning: @settings.model.reasoning,
236
+ # ik has no --reasoning-effort; the template reads the same variable
237
+ # from its kwargs.
238
+ chat_template_kwargs: effort_kwargs,
239
+ chat_template: chat_template_override,
240
+ mmproj: @settings.model.mmproj_path,
241
+ n_cpu_moe: (n_cpu_moe if n_cpu_moe.positive?),
242
+ # ik packs the stage's parameters into the value, where mainline
243
+ # spells the same thing as `draft-mtp` plus a separate
244
+ # --spec-draft-n-max. Its other stage types all want a draft model
245
+ # or a corpus; mtp is the only one the GGUF alone can feed.
246
+ spec_type: ("mtp:n_max=#{SPEC_DRAFT_N_MAX}" if drafting?),
247
+ # Workaround for ik_llama#1232 (open): ik caches one CUDA graph per
248
+ # distinct compute graph and never evicts, so VRAM climbs ~44 MiB/min
249
+ # under sustained load until the server dies mid-run. Measured here:
250
+ # +2440 MiB over 56 min with graphs on, +0 over 10 min with them off.
251
+ # The author suggested this flag and has not landed a real fix;
252
+ # mainline's LRU eviction (ggml-org/llama.cpp#21611) is not ported.
253
+ # Costs ~10% token generation -- drop this once ik evicts.
254
+ cuda: "graphs=0",
255
+ }
256
+ end
257
+
258
+ # render_preset runs outside start (`llm status`, preset rendering), so
259
+ # the launch-time override has to fall back to the configured answer.
260
+ def drafting? = defined?(@drafting) ? @drafting : mtp_enabled?
261
+
262
+ def effort_kwargs
263
+ effort = @settings.model.reasoning_effort
264
+ JSON.generate("reasoning_effort" => effort) if effort
265
+ end
266
+
267
+ def launch(binary, n_cpu_moe)
268
+ render_preset(n_cpu_moe)
269
+ launcher.spawn(build_command(binary, n_cpu_moe))
270
+ end
271
+ end
272
+ end
273
+ end
@@ -0,0 +1,25 @@
1
+ require "pathname"
2
+
3
+ module Mimic
4
+ module Engine
5
+ class Launcher
6
+ def initialize(log_file:, pid_file:, chdir:)
7
+ @log_file = log_file
8
+ @pid_file = pid_file
9
+ @chdir = chdir
10
+ end
11
+
12
+ def spawn(command)
13
+ @pid_file.dirname.mkpath
14
+ log = File.open(@log_file, "ab")
15
+ begin
16
+ pid = Process.spawn(*command, out: log, err: log, chdir: @chdir.to_s, pgroup: true)
17
+ ensure
18
+ log.close
19
+ end
20
+ @pid_file.write(pid.to_s)
21
+ pid
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,124 @@
1
+ require_relative "base"
2
+ require_relative "../engine_cli/llama_preset"
3
+
4
+ module Mimic
5
+ module Engine
6
+ class Llama < Base
7
+ PRESET_FILE = Paths::LLM_RUN_DIR / "llama-models.ini"
8
+
9
+ # The launch surface is the preset file, not the argv, so these are the
10
+ # preset's own keys re-dashed into the flags the router expands them to
11
+ # in /v1/models. Only the three observed there are listed -- notably no
12
+ # n_cpu_moe, which is how `llm status` knows to leave the line off.
13
+ ARG_FLAGS = %i[ctx cache_type n_slots mmproj reasoning reasoning_effort].to_h { |key| [key, "--#{EngineCli::LlamaPreset::KEYS.fetch(key)}"] }.freeze
14
+
15
+ # argv of the actually-running instance, straight from the router's own
16
+ # /v1/models -- config.yaml can be stale (e.g. `llm parallel` overrides
17
+ # ctx/slots without persisting them).
18
+ def live_args
19
+ data = Util::Probes.http_json(models_url)
20
+ return nil if data.nil?
21
+
22
+ entry = (data["data"] || []).find { |e| e["id"] == model_alias && e.dig("status", "value") == "loaded" }
23
+ entry&.dig("status", "args")
24
+ end
25
+
26
+ def render_preset
27
+ ensure_chat_template
28
+ drafter = mtp_drafter
29
+ preset = EngineCli::LlamaPreset.new(@settings, model_alias: model_alias, chat_template: chat_template_override,
30
+ mtp: mtp_enabled? || !drafter.nil?, mtp_drafter: drafter,
31
+ mmproj: @settings.model.mmproj_path, options: binary_options)
32
+ preset.write(PRESET_FILE)
33
+ report_dropped(preset.dropped)
34
+ PRESET_FILE
35
+ end
36
+
37
+ def start(remember: true)
38
+ return Result.ok("llama-server already running") if running?
39
+
40
+ binary = find_binary
41
+ return Result.error("llama-server binary not found -- run `mimic setup`") if binary.nil?
42
+
43
+ missing = model_missing
44
+ return Result.error(missing) if missing
45
+ return port_busy_error("llama-server") unless await_port_release
46
+
47
+ render_preset
48
+ log_offset = process.log_size
49
+ pid = launcher.spawn(build_command(binary))
50
+
51
+ unless Util::Probes.wait_ready(models_url, alive: -> { Util::Probes.child_alive?(pid) }, label: "waiting for llama-server")
52
+ process.cleanup_failed(pid)
53
+ return Result.error("llama-server failed to start -- see #{process.log_file}\n#{process.last_lines(process.log_tail(log_offset))}")
54
+ end
55
+
56
+ failure = []
57
+ instance_failed = lambda do
58
+ reason = instance_failure(log_offset)
59
+ failure << reason if reason
60
+ !reason.nil?
61
+ end
62
+
63
+ if Util::Probes.wait_until(attempts: 120, alive: -> { Util::Probes.child_alive?(pid) },
64
+ label: "loading model", abort: instance_failed) { model_loaded? }
65
+ @settings.remember_ctx if remember
66
+ @settings.remember_sampling if remember
67
+ record_compute(pid)
68
+ report_template_failures
69
+ return Result.ok("llama-server ready, model loaded (pid #{pid})")
70
+ end
71
+
72
+ process.cleanup_failed(pid)
73
+ tail = process.last_lines(process.log_tail(log_offset))
74
+ return Result.error("#{failure.first} -- see #{process.log_file}\n#{tail}") unless failure.empty?
75
+
76
+ Result.error("model failed to load -- see #{process.log_file}\n#{tail}")
77
+ end
78
+
79
+ private
80
+
81
+ def binary_options
82
+ binary = find_binary
83
+ binary && EngineCli::BinaryOptions.new(binary)
84
+ end
85
+
86
+ def mtp_drafter = (@settings.model.file.drafter unless mtp_off?)
87
+
88
+ def report_dropped(dropped)
89
+ return if dropped.empty?
90
+
91
+ puts "[preset] #{@settings.llama.engine} does not carry " \
92
+ "#{dropped.map { |name| "--#{name}" }.join(', ')} -- left out of the preset"
93
+ end
94
+
95
+ def build_command(binary)
96
+ [binary, "--models-preset", PRESET_FILE.to_s, "--host", @settings.llama.host,
97
+ "--port", @settings.llama.port.to_s, "--reuse-port", "--metrics"]
98
+ end
99
+
100
+ def model_loaded?
101
+ data = Util::Probes.http_json("#{@settings.llama.base_url}/v1/models") || {}
102
+ (data["data"] || []).any? { |entry| entry["id"] == model_alias && entry.dig("status", "value") == "loaded" }
103
+ end
104
+
105
+ # Why the router's model instance died, reading only this launch's log.
106
+ #
107
+ # The router survives its children, so a child that OOMs looks
108
+ # identical to one still loading: the poll that checks for it
109
+ # triggers another autoload, and the pair loop until the caller's
110
+ # attempts run out. The child's exit is only ever reported in the
111
+ # shared log.
112
+ def instance_failure(log_offset)
113
+ tail = process.log_tail(log_offset)
114
+ return nil unless tail.include?("instance name=#{model_alias} exited with status 1")
115
+
116
+ if tail.include?("cudaMalloc failed: out of memory") || tail.include?("failed to allocate")
117
+ return "#{model_alias} ran out of VRAM at #{@settings.model.context_length} ctx -- lower the context"
118
+ end
119
+
120
+ "#{model_alias} instance exited during load"
121
+ end
122
+ end
123
+ end
124
+ end