mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
require_relative "../../ui"
|
|
2
|
+
require_relative "../../util/fmt"
|
|
3
|
+
require_relative "../../engine"
|
|
4
|
+
require_relative "../../model"
|
|
5
|
+
require_relative "../../model/footprint"
|
|
6
|
+
require_relative "../../model/header"
|
|
7
|
+
|
|
8
|
+
module Mimic
|
|
9
|
+
module LlmCli
|
|
10
|
+
module Wizard
|
|
11
|
+
# Estimate VRAM for a chosen context; warn + confirm if it would
|
|
12
|
+
# over-commit. llama.cpp accepts a context it cannot hold and dies once
|
|
13
|
+
# the KV cache fills, so the negotiation happens here rather than at
|
|
14
|
+
# launch.
|
|
15
|
+
class ContextPreflight
|
|
16
|
+
def initialize(config, prompt: Ui::Prompt.new)
|
|
17
|
+
@config = config
|
|
18
|
+
@prompt = prompt
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# True to accept (fits, un-checkable, offloaded, or user override),
|
|
22
|
+
# false to re-pick.
|
|
23
|
+
def ok?
|
|
24
|
+
est = estimate
|
|
25
|
+
return true unless est.measurable?
|
|
26
|
+
|
|
27
|
+
if est.fits?
|
|
28
|
+
puts "#{Ui.green('VRAM ok')} -- needs ~#{format('%.1f', est.need_mib / 1024.0)} GB of " \
|
|
29
|
+
"#{format('%.1f', est.total_mib / 1024.0)} GB (fits to ~#{Util::Fmt.commas(est.max_ctx)} tok)"
|
|
30
|
+
return true
|
|
31
|
+
end
|
|
32
|
+
return true if offloaded_to_fit?(est)
|
|
33
|
+
|
|
34
|
+
warn_over_commit(est)
|
|
35
|
+
@prompt.yes?("Use this over-committed context anyway?", default: false)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def estimate
|
|
41
|
+
model = @config.model.file
|
|
42
|
+
model.footprint(ctx, cache_type, compute_mib: model.memory.compute_mib(ctx, cache_type),
|
|
43
|
+
slots: @config.llama.n_slots, reclaimable_mib: Engine::ServerProcess.chat.gpu_mib,
|
|
44
|
+
mmproj_path: @config.model.mmproj_path)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def ctx = @config.model.context_length
|
|
48
|
+
def cache_type = @config.llama.cache_type
|
|
49
|
+
|
|
50
|
+
# A MoE model can trade VRAM for PCIe traffic by keeping some layers'
|
|
51
|
+
# experts on the CPU, so a deficit the ceiling can cover is a setting to
|
|
52
|
+
# write, not a warning to print. Only ik takes --n-cpu-moe; llama.cpp's
|
|
53
|
+
# preset pins n-gpu-layers=999 on purpose.
|
|
54
|
+
def offloaded_to_fit?(est)
|
|
55
|
+
return false unless @config.llama.ik?
|
|
56
|
+
|
|
57
|
+
layers = est.layers_to_offload(ceiling: offload_ceiling)
|
|
58
|
+
return false if layers.nil? || layers.zero?
|
|
59
|
+
|
|
60
|
+
@config.model.file.memory.remember_n_cpu_moe(layers)
|
|
61
|
+
puts "#{Ui.green("offloading #{layers} layers of MoE experts to CPU")} -- " \
|
|
62
|
+
"frees ~#{format('%.1f', (est.need_mib - est.budget_mib) / 1024.0)} GB, " \
|
|
63
|
+
"so #{Util::Fmt.commas(ctx)} tokens fits"
|
|
64
|
+
true
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def offload_ceiling
|
|
68
|
+
layers = Model::Header.new(@config.model.path).block_count
|
|
69
|
+
[Engine::IkLlama::CPU_MOE_MAX, layers || Engine::IkLlama::CPU_MOE_MAX].min
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def warn_over_commit(est)
|
|
73
|
+
puts "#{Ui.yellow('this context won\'t fully fit VRAM.')} Needs " \
|
|
74
|
+
"~#{format('%.1f', est.need_mib / 1024.0)} GB (weights #{format('%.1f', est.weights_mib / 1024.0)} + " \
|
|
75
|
+
"KV #{format('%.1f', est.kv_mib / 1024.0)} + compute), budget ~#{format('%.1f', est.budget_mib / 1024.0)} GB.\n" \
|
|
76
|
+
" It will load but crash at runtime once context passes " \
|
|
77
|
+
"~#{Util::Fmt.commas(est.max_ctx)} tokens. Pick <= that, or switch KV to q4_0."
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
require_relative "../../ui"
|
|
2
|
+
require_relative "../../llm_config"
|
|
3
|
+
require_relative "../../engine/llama_embed"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
module LlmCli
|
|
7
|
+
module Wizard
|
|
8
|
+
# Picks which embedding model `llm embed start` launches, and on which
|
|
9
|
+
# port. The endpoint it serves is not asked for -- Engine::LlamaEmbed reads it
|
|
10
|
+
# off the GGUF, and `mode` in config.yaml overrides that when the header is
|
|
11
|
+
# wrong.
|
|
12
|
+
class Embed
|
|
13
|
+
def initialize(config, models, prompt: Ui::Prompt.new)
|
|
14
|
+
@config = config
|
|
15
|
+
@embed = config.embed
|
|
16
|
+
@models = models
|
|
17
|
+
@prompt = prompt
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def run
|
|
21
|
+
Ui.banner("llm embed -- setup wizard")
|
|
22
|
+
return no_models if @models.empty?
|
|
23
|
+
|
|
24
|
+
choose_model
|
|
25
|
+
choose_port
|
|
26
|
+
choose_batch_size
|
|
27
|
+
@config.save
|
|
28
|
+
report
|
|
29
|
+
@config
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def choose_model
|
|
35
|
+
step(1, "Embedding model")
|
|
36
|
+
choices = @models.to_h { |model| [label(model), model.path.to_s] }
|
|
37
|
+
@embed.path = @prompt.select("Embedding model:", choices, default: choices.key(@embed.path))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def choose_port
|
|
41
|
+
step(2, "Port")
|
|
42
|
+
loop do
|
|
43
|
+
port = @prompt.ask("Port for the embedding server:", convert: :int, default: @embed.port)
|
|
44
|
+
if port.to_i == @config.llama.port.to_i
|
|
45
|
+
puts Ui.red("Port #{port} is where llama-server already serves the chat model.")
|
|
46
|
+
next
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
@embed.port = port.to_i
|
|
50
|
+
break
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Both llama-server batch flags come off this one answer: a pooled
|
|
55
|
+
# encoder cannot split a document across micro-batches, so the physical
|
|
56
|
+
# batch has to hold the longest document rather than sit at its default.
|
|
57
|
+
def choose_batch_size
|
|
58
|
+
step(3, "Batch size")
|
|
59
|
+
puts "Must be at least as long, in tokens, as the longest document you send."
|
|
60
|
+
@embed.batch_size = @prompt.ask("Batch size:", convert: :int, default: @embed.batch_size).to_i
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def label(model)
|
|
64
|
+
mode = Engine::LlamaEmbed.mode_for(model.path)
|
|
65
|
+
"#{model.path.dirname.basename}/#{model.path.basename} [#{mode}]"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def report
|
|
69
|
+
Ui.panel(
|
|
70
|
+
"Embedding model: #{File.basename(@embed.path)}\n" \
|
|
71
|
+
"Endpoint: #{@embed.base_url} (#{Engine::LlamaEmbed.new(@config).mode})\n" \
|
|
72
|
+
"Batch size: #{@embed.batch_size}\n" \
|
|
73
|
+
"Saved to #{LlmConfig::LLM_CONFIG_PATH}"
|
|
74
|
+
)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def no_models
|
|
78
|
+
puts "No embedding models found under the configured search paths."
|
|
79
|
+
puts "Add one with `llm path add <dir>` -- see `llm models`."
|
|
80
|
+
nil
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def step(index, title)
|
|
84
|
+
puts "\n#{Ui.bold_cyan("Step #{index}/3")} -- #{title}"
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
require_relative "../../ui"
|
|
2
|
+
require_relative "../../engine"
|
|
3
|
+
require_relative "../../model/footprint"
|
|
4
|
+
require_relative "../../model"
|
|
5
|
+
|
|
6
|
+
module Mimic
|
|
7
|
+
module LlmCli
|
|
8
|
+
module Wizard
|
|
9
|
+
class Moe
|
|
10
|
+
def initialize(scan, prompt: Ui::Prompt.new)
|
|
11
|
+
@scan = scan
|
|
12
|
+
@prompt = prompt
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def run
|
|
16
|
+
Ui.banner("llm -- MoE offload wizard")
|
|
17
|
+
paths = moe_model_paths
|
|
18
|
+
return puts "No MoE .gguf models found under the configured search paths." if paths.empty?
|
|
19
|
+
|
|
20
|
+
path = choose_model(paths)
|
|
21
|
+
case choose_action(path)
|
|
22
|
+
when :set then set_value(path)
|
|
23
|
+
when :clear then clear_value(path)
|
|
24
|
+
else puts "Cancelled."
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def moe_model_paths
|
|
31
|
+
@scan.each_with_object([]) do |model, found|
|
|
32
|
+
real = model.real.to_s
|
|
33
|
+
found << real if !found.include?(real) && Model::Header.new(real).moe?
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def choose_model(paths)
|
|
38
|
+
step(1, "Model")
|
|
39
|
+
choices = paths.to_h do |path|
|
|
40
|
+
cached = Model.new(path).memory.n_cpu_moe
|
|
41
|
+
["#{File.basename(path)}#{cached ? " (cached: #{cached})" : ''}", path]
|
|
42
|
+
end
|
|
43
|
+
@prompt.select("MoE model:", choices)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def choose_action(path)
|
|
47
|
+
step(2, "n_cpu_moe")
|
|
48
|
+
puts "Current cached value: #{Model.new(path).memory.n_cpu_moe || 'none (auto-tunes from 0 on next start)'}"
|
|
49
|
+
@prompt.select("What do you want to do?", [
|
|
50
|
+
{ name: "Set a value (0-#{ceiling(path)})", value: :set },
|
|
51
|
+
{ name: "Clear cached value", value: :clear },
|
|
52
|
+
{ name: "Cancel", value: :cancel },
|
|
53
|
+
])
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def set_value(path)
|
|
57
|
+
limit = ceiling(path)
|
|
58
|
+
value = @prompt.ask("Layers to offload to CPU (0-#{limit}):", convert: :int, default: Model.new(path).memory.n_cpu_moe || 0)
|
|
59
|
+
abort "n_cpu_moe must be between 0 and #{limit}" unless value.between?(0, limit)
|
|
60
|
+
|
|
61
|
+
Model.new(path).memory.remember_n_cpu_moe(value)
|
|
62
|
+
puts "#{Ui.green('Set')} #{File.basename(path)} n_cpu_moe=#{value}"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def clear_value(path)
|
|
66
|
+
abort "no cached value to clear for #{File.basename(path)}" if Model.new(path).memory.n_cpu_moe.nil?
|
|
67
|
+
|
|
68
|
+
Model.new(path).memory.forget_n_cpu_moe
|
|
69
|
+
puts "#{Ui.green('Cleared')} #{File.basename(path)}"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def ceiling(path)
|
|
73
|
+
@ceiling ||= Model::Header.new(path).block_count || Engine::IkLlama::CPU_MOE_MAX
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def step(index, title)
|
|
77
|
+
puts "\n#{Ui.bold_cyan("Step #{index}/2")} -- #{title}"
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
require_relative "../../ui"
|
|
2
|
+
require_relative "../../util/fmt"
|
|
3
|
+
require_relative "../../model/footprint"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
module LlmCli
|
|
7
|
+
module Wizard
|
|
8
|
+
# Pick an n_slots/ctx-per-request pair so the server can hold several
|
|
9
|
+
# requests at once. Total ctx-size is the product, and that is what has
|
|
10
|
+
# to fit in VRAM.
|
|
11
|
+
class Parallel
|
|
12
|
+
CTX_PRESETS = {
|
|
13
|
+
"2048 -- short prompts, max concurrency" => 2048,
|
|
14
|
+
"4096 -- humaneval-rb-sized prompts" => 4096,
|
|
15
|
+
"8192 -- moderate prompts" => 8192,
|
|
16
|
+
"16000 -- long prompts" => 16_000,
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
SLOT_CHOICES = [1, 2, 4, 8, 16, 32].freeze
|
|
20
|
+
|
|
21
|
+
def initialize(config, setup_wizard, prompt: Ui::Prompt.new)
|
|
22
|
+
@config = config
|
|
23
|
+
@setup_wizard = setup_wizard
|
|
24
|
+
@prompt = prompt
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def run
|
|
28
|
+
Ui.banner("llm -- parallel wizard")
|
|
29
|
+
|
|
30
|
+
step(1, "Model")
|
|
31
|
+
@config.model.path = @setup_wizard.select_model(@config.model.path)
|
|
32
|
+
|
|
33
|
+
step(2, "Context per request")
|
|
34
|
+
ctx_per_slot = choose_ctx_per_slot
|
|
35
|
+
|
|
36
|
+
step(3, "Slots")
|
|
37
|
+
n_slots = choose_slots(ctx_per_slot)
|
|
38
|
+
|
|
39
|
+
@config.model.context_length = ctx_per_slot * n_slots
|
|
40
|
+
@config.llama.n_slots = n_slots
|
|
41
|
+
@config
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def choose_ctx_per_slot
|
|
47
|
+
picked = @prompt.select("Context per request:", [*CTX_PRESETS.keys, "Custom..."])
|
|
48
|
+
return CTX_PRESETS.fetch(picked) unless picked == "Custom..."
|
|
49
|
+
|
|
50
|
+
Integer(@prompt.ask("Context per request (tokens):", default: "4096"))
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def choose_slots(ctx_per_slot)
|
|
54
|
+
puts "Total ctx-size = context-per-request x slots -- that's what has to fit in VRAM alongside the weights."
|
|
55
|
+
SLOT_CHOICES.each { |slots| puts fit_line(ctx_per_slot, slots) }
|
|
56
|
+
Integer(@prompt.select("n_slots:", SLOT_CHOICES.map(&:to_s), default: "8"))
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def fit_line(ctx_per_slot, slots)
|
|
60
|
+
total = ctx_per_slot * slots
|
|
61
|
+
model = @config.model.file
|
|
62
|
+
est = model.footprint(total, @config.llama.cache_type,
|
|
63
|
+
compute_mib: model.memory.compute_mib(total, @config.llama.cache_type),
|
|
64
|
+
slots: slots, reclaimable_mib: Engine::ServerProcess.chat.gpu_mib,
|
|
65
|
+
mmproj_path: @config.model.mmproj_path)
|
|
66
|
+
tag = !est.measurable? || est.fits? ? Ui.green("fits") : Ui.red("too big")
|
|
67
|
+
format(" %2d slots -> %10s total ctx %s", slots, Util::Fmt.commas(total), tag)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def step(index, title)
|
|
71
|
+
puts "\n#{Ui.bold_cyan("Step #{index}/3")} -- #{title}"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require "etc"
|
|
3
|
+
require_relative "../../ui"
|
|
4
|
+
require_relative "../../util/fmt"
|
|
5
|
+
require_relative "../../util/host"
|
|
6
|
+
require_relative "../../util/probes"
|
|
7
|
+
require_relative "../../engine_cli"
|
|
8
|
+
require_relative "../../engine"
|
|
9
|
+
require_relative "context_preflight"
|
|
10
|
+
require_relative "../../model/library"
|
|
11
|
+
require_relative "../../llm_config"
|
|
12
|
+
require_relative "../../model"
|
|
13
|
+
|
|
14
|
+
module Mimic
|
|
15
|
+
module LlmCli
|
|
16
|
+
module Wizard
|
|
17
|
+
class Setup
|
|
18
|
+
CONTEXT_PRESETS = {
|
|
19
|
+
"60000 / q8_0" => [60_000, "q8_0"],
|
|
20
|
+
"80000 / q8_0" => [80_000, "q8_0"],
|
|
21
|
+
"110000 / q8_0" => [110_000, "q8_0"],
|
|
22
|
+
"120000 / q8_0" => [120_000, "q8_0"],
|
|
23
|
+
"131072 / q8_0" => [131_072, "q8_0"],
|
|
24
|
+
"140000 / q8_0" => [140_000, "q8_0"],
|
|
25
|
+
"150000 / q8_0" => [150_000, "q8_0"],
|
|
26
|
+
"160000 / q8_0" => [160_000, "q8_0"],
|
|
27
|
+
"180000 / q8_0" => [180_000, "q8_0"],
|
|
28
|
+
"200000 / q8_0" => [200_000, "q8_0"],
|
|
29
|
+
"262144 / q4_0" => [262_144, "q4_0"],
|
|
30
|
+
}.freeze
|
|
31
|
+
|
|
32
|
+
ENGINE_BLURBS = {
|
|
33
|
+
"llama.cpp" => "upstream, router-mode preset",
|
|
34
|
+
"ik_llama.cpp" => "ikawrakow fork (IQK quants, fused MoE)",
|
|
35
|
+
"prism-v7-llama.cpp" => "current Prism fork (Bonsai 2 PQ2_0/PTQ1_0)",
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
attr_reader :config
|
|
39
|
+
|
|
40
|
+
def initialize(config, total: 8, cli_name: "llm", prompt: Ui::Prompt.new)
|
|
41
|
+
@config = config
|
|
42
|
+
@total = total
|
|
43
|
+
@cli_name = cli_name
|
|
44
|
+
@prompt = prompt
|
|
45
|
+
@step = 0
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def step(title)
|
|
49
|
+
@step += 1
|
|
50
|
+
puts "\n#{Ui.bold_cyan("Step #{@step}/#{@total}")} -- #{title}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def system_check
|
|
54
|
+
step("System check")
|
|
55
|
+
rows = [
|
|
56
|
+
["RAM", "#{Util::Host.total_mem_kb / 1024 / 1024} GB"],
|
|
57
|
+
["Free disk (home)", "#{Util::Host.free_home_bytes / (1024**3)} GB"],
|
|
58
|
+
["Ruby", RUBY_VERSION],
|
|
59
|
+
["GPU backend (detected)", Util::Host.backend],
|
|
60
|
+
["llama-server", Util::Host.llama_server || Ui.red("not found")],
|
|
61
|
+
["claude on PATH", Util::Probes.which("claude") || Ui.red("not found")],
|
|
62
|
+
]
|
|
63
|
+
rows.each { |label, value| puts format(" %-24s %s", label, value) }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def choose_engine
|
|
67
|
+
step("Inference engine")
|
|
68
|
+
choices = EngineCli::Registry.names.map do |name|
|
|
69
|
+
{ name: [name, ENGINE_BLURBS[name]].compact.join(" -- "), value: name }
|
|
70
|
+
end
|
|
71
|
+
current = choices.index { |choice| choice[:value] == @config.llama.engine } || 0
|
|
72
|
+
@config.llama.engine = @prompt.select("Inference engine:", choices, default: current + 1)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def ensure_binary(engine)
|
|
76
|
+
step("#{engine} binary")
|
|
77
|
+
@config.llama.binary = EngineCli::Builder.new(@config, cli_name: @cli_name, prompt: @prompt)
|
|
78
|
+
.find_or_build(engine)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def choose_model
|
|
82
|
+
step("Model")
|
|
83
|
+
@config.model.path = select_model(@config.model.path)
|
|
84
|
+
report_hidden_models
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def choose_mmproj
|
|
88
|
+
step("Multi-modal projector (optional)")
|
|
89
|
+
candidate = Model.projector_for(@config.model.path)
|
|
90
|
+
unless candidate
|
|
91
|
+
@config.model.mmproj_path = nil
|
|
92
|
+
return
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
accepted = @prompt.yes?("Use #{candidate.basename} as projector?", default: false)
|
|
96
|
+
@config.model.mmproj_path = accepted ? candidate.to_s : nil
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def choose_context
|
|
100
|
+
step("Context preset")
|
|
101
|
+
remembered = @config.model.file.memory.last_ctx
|
|
102
|
+
if remembered
|
|
103
|
+
@config.model.context_length, @config.llama.cache_type = remembered
|
|
104
|
+
unless CONTEXT_PRESETS.value?(remembered)
|
|
105
|
+
puts "#{Ui.green('Last good run for this model:')} #{Util::Fmt.commas(remembered[0])} / #{remembered[1]}"
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
preflight = ContextPreflight.new(@config, prompt: @prompt)
|
|
109
|
+
loop do
|
|
110
|
+
pick_context
|
|
111
|
+
break if preflight.ok?
|
|
112
|
+
end
|
|
113
|
+
@config
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def choose_reasoning
|
|
117
|
+
step("Reasoning")
|
|
118
|
+
wanted = @prompt.yes?("Let the model think before it answers?", default: true)
|
|
119
|
+
@config.model.reasoning = wanted ? "auto" : "off"
|
|
120
|
+
@config
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def choose_effort
|
|
124
|
+
step("Thinking level")
|
|
125
|
+
template = Engine::ChatTemplate.new(@config.model.path, model_alias: @config.model.name)
|
|
126
|
+
levels = template.effort_levels
|
|
127
|
+
if @config.model.reasoning == "off" || levels.empty?
|
|
128
|
+
@config.model.reasoning_effort = nil
|
|
129
|
+
puts " #{@config.model.reasoning == 'off' ? 'thinking is off' : 'the chat template has no levels'}"
|
|
130
|
+
return @config
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
@config.model.reasoning_effort = @prompt.select("How hard should the model think?", levels,
|
|
134
|
+
default: template.default_effort)
|
|
135
|
+
@config
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def select_model(current)
|
|
139
|
+
by_label = library.chat.to_h { |model| [model_label(model), model.path.to_s] }
|
|
140
|
+
manual = "Enter a path manually..."
|
|
141
|
+
answer = if by_label.any?
|
|
142
|
+
current_label = by_label.key(current)
|
|
143
|
+
@prompt.select("GGUF model file:", [*by_label.keys, manual], default: current_label)
|
|
144
|
+
else
|
|
145
|
+
manual
|
|
146
|
+
end
|
|
147
|
+
answer = if answer == manual
|
|
148
|
+
@prompt.ask("Path to .gguf file:", default: current)
|
|
149
|
+
else
|
|
150
|
+
by_label.fetch(answer, answer)
|
|
151
|
+
end
|
|
152
|
+
# A typed answer arrives literal -- no shell has expanded ~ for us,
|
|
153
|
+
# and File.file? does not either.
|
|
154
|
+
expanded = Pathname.new(answer.to_s).expand_path
|
|
155
|
+
if answer.to_s.empty? || !expanded.file?
|
|
156
|
+
puts Ui.red("Model file not found.")
|
|
157
|
+
raise SystemExit, 1
|
|
158
|
+
end
|
|
159
|
+
expanded.realpath.to_s
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
private
|
|
163
|
+
|
|
164
|
+
def library
|
|
165
|
+
@library ||= Model::Library.new(@config)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def report_hidden_models
|
|
169
|
+
return if library.hidden.empty?
|
|
170
|
+
|
|
171
|
+
library.hidden.group_by { |model| library.hidden_reason(model) }.each do |reason, models|
|
|
172
|
+
names = models.map { |model| model.path.basename.to_s }.join(", ")
|
|
173
|
+
puts Ui.yellow("hidden -- #{reason}: #{names}")
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def model_label(model)
|
|
178
|
+
tags = []
|
|
179
|
+
tags << "MoE" if Model::Header.new(model.real).moe?
|
|
180
|
+
tags << "vision" if model.vision?
|
|
181
|
+
suffix = tags.empty? ? "" : " [#{tags.join(', ')}]"
|
|
182
|
+
"#{model.path.dirname.basename}/#{model.path.basename}#{suffix}"
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def pick_context
|
|
186
|
+
presets = labelled_presets
|
|
187
|
+
current_preset = presets.find do |_, (ctx, kv)|
|
|
188
|
+
ctx == @config.model.context_length && kv == @config.llama.cache_type
|
|
189
|
+
end&.first
|
|
190
|
+
picked = @prompt.select("Context preset:", [*presets.keys, "Custom..."], default: current_preset)
|
|
191
|
+
if picked == "Custom..."
|
|
192
|
+
context = @prompt.ask("Context length (tokens):", default: @config.model.context_length.to_s)
|
|
193
|
+
@config.model.context_length = Integer(context)
|
|
194
|
+
@config.llama.cache_type = @prompt.select("KV cache type:", %w[q8_0 q4_0 f16], default: @config.llama.cache_type)
|
|
195
|
+
else
|
|
196
|
+
ctx, kv = presets.fetch(picked)
|
|
197
|
+
@config.model.context_length = ctx
|
|
198
|
+
@config.llama.cache_type = kv
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def labelled_presets
|
|
203
|
+
remembered = @config.model.file.memory.last_ctx
|
|
204
|
+
CONTEXT_PRESETS.to_h do |label, preset|
|
|
205
|
+
[preset == remembered ? "#{label} -- last success run" : label, preset]
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
require_relative "../llm_config"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
module LlmConfig
|
|
5
|
+
# The embedding server's profile: its own model and its own port, so it
|
|
6
|
+
# can serve /v1/embeddings beside the chat server rather than in place of
|
|
7
|
+
# it. `mode` picks the endpoint llama-server exposes -- "auto" reads the
|
|
8
|
+
# GGUF's pooling type, "embeddings" and "rerank" force it.
|
|
9
|
+
class Embed
|
|
10
|
+
MODES = %w[auto embeddings rerank].freeze
|
|
11
|
+
|
|
12
|
+
def initialize(cfg, host:)
|
|
13
|
+
@cfg = cfg
|
|
14
|
+
@host = host
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def path = @cfg["path"].to_s
|
|
18
|
+
def port = @cfg.fetch("port", 8081)
|
|
19
|
+
def context_length = @cfg.fetch("context_length", 0)
|
|
20
|
+
def batch_size = @cfg.fetch("batch_size", 8192)
|
|
21
|
+
def mode = @cfg.fetch("mode", "auto")
|
|
22
|
+
def host = @host
|
|
23
|
+
def configured? = !path.empty?
|
|
24
|
+
def model_alias = Settings.alias_for(path)
|
|
25
|
+
def base_url = "http://127.0.0.1:#{port}"
|
|
26
|
+
|
|
27
|
+
def path=(value)
|
|
28
|
+
@cfg["path"] = value
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def port=(value)
|
|
32
|
+
@cfg["port"] = value
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def context_length=(value)
|
|
36
|
+
@cfg["context_length"] = value
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def batch_size=(value)
|
|
40
|
+
@cfg["batch_size"] = value
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def mode=(value)
|
|
44
|
+
@cfg["mode"] = value
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
module Mimic
|
|
2
|
+
module LlmConfig
|
|
3
|
+
# The engine section: which llama-server build runs, where it binds, and
|
|
4
|
+
# the knobs that belong to the server rather than to the model.
|
|
5
|
+
class LlamaSettings
|
|
6
|
+
SECTION = "llama".freeze
|
|
7
|
+
|
|
8
|
+
def initialize(cfg)
|
|
9
|
+
@cfg = cfg
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def engine = section.fetch("engine", "llama.cpp")
|
|
13
|
+
def host = section["host"]
|
|
14
|
+
def port = section["port"]
|
|
15
|
+
def n_slots = section.fetch("n_slots", 1)
|
|
16
|
+
def flash_attn = section["flash_attn"]
|
|
17
|
+
def cache_type = section["cache_type"]
|
|
18
|
+
def ik? = engine == "ik_llama.cpp"
|
|
19
|
+
|
|
20
|
+
# `binary` is per-engine: llama.cpp and ik_llama.cpp are different
|
|
21
|
+
# binaries, so one flat field meant switching engines could hand the
|
|
22
|
+
# wrong engine the other's path. A pre-split config.yaml is upgraded to
|
|
23
|
+
# the per-engine hash by LlmConfig.load, so a String here only reaches
|
|
24
|
+
# a section built by hand.
|
|
25
|
+
def binary
|
|
26
|
+
raw = section["binary"]
|
|
27
|
+
raw.is_a?(Hash) ? raw[engine].to_s : raw.to_s
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Loopback URL for clients, not the server's bind address.
|
|
31
|
+
#
|
|
32
|
+
# `llama.host` is 0.0.0.0 so containers can reach the server; writing
|
|
33
|
+
# that into a client-facing URL would be wrong -- every client on this
|
|
34
|
+
# host (mimic claude/codex, hermes, status checks) dials 127.0.0.1.
|
|
35
|
+
def base_url = "http://127.0.0.1:#{port}"
|
|
36
|
+
|
|
37
|
+
def engine=(value)
|
|
38
|
+
writable["engine"] = value
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def cache_type=(value)
|
|
42
|
+
writable["cache_type"] = value
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def n_slots=(value)
|
|
46
|
+
writable["n_slots"] = value
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def binary=(value)
|
|
50
|
+
raw = section["binary"]
|
|
51
|
+
writable["binary"] = raw.is_a?(Hash) ? raw.merge(engine => value) : { engine => value }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def section = @cfg[SECTION] || {}
|
|
57
|
+
def writable = @cfg[SECTION] ||= {}
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|