mimic-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/NOTICE +16 -0
  4. data/README.md +175 -0
  5. data/exe/lllm +5 -0
  6. data/exe/llm +5 -0
  7. data/exe/mimic +5 -0
  8. data/lib/mimic/README.md +144 -0
  9. data/lib/mimic/agent/claude.rb +305 -0
  10. data/lib/mimic/agent/codex.rb +254 -0
  11. data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
  12. data/lib/mimic/agent/hermes.rb +107 -0
  13. data/lib/mimic/agent/muse.rb +106 -0
  14. data/lib/mimic/agent.rb +46 -0
  15. data/lib/mimic/cli.rb +159 -0
  16. data/lib/mimic/client/base.rb +44 -0
  17. data/lib/mimic/client/crush.rb +28 -0
  18. data/lib/mimic/client/goose.rb +52 -0
  19. data/lib/mimic/client/hermes.rb +21 -0
  20. data/lib/mimic/client/opencode.rb +29 -0
  21. data/lib/mimic/client/pi.rb +50 -0
  22. data/lib/mimic/client.rb +19 -0
  23. data/lib/mimic/engine/README.md +140 -0
  24. data/lib/mimic/engine/base.rb +138 -0
  25. data/lib/mimic/engine/chat_template.rb +207 -0
  26. data/lib/mimic/engine/ik_llama.rb +273 -0
  27. data/lib/mimic/engine/launcher.rb +25 -0
  28. data/lib/mimic/engine/llama.rb +124 -0
  29. data/lib/mimic/engine/llama_embed.rb +108 -0
  30. data/lib/mimic/engine/router.rb +57 -0
  31. data/lib/mimic/engine/server_process.rb +108 -0
  32. data/lib/mimic/engine.rb +5 -0
  33. data/lib/mimic/engine_cli/binary_options.rb +46 -0
  34. data/lib/mimic/engine_cli/builder.rb +98 -0
  35. data/lib/mimic/engine_cli/diffusion.rb +117 -0
  36. data/lib/mimic/engine_cli/llama_preset.rb +119 -0
  37. data/lib/mimic/engine_cli/registry.rb +94 -0
  38. data/lib/mimic/engine_cli/source.rb +110 -0
  39. data/lib/mimic/engine_cli/status.rb +161 -0
  40. data/lib/mimic/engine_cli/update.rb +68 -0
  41. data/lib/mimic/engine_cli.rb +7 -0
  42. data/lib/mimic/health/checks.rb +92 -0
  43. data/lib/mimic/health.rb +82 -0
  44. data/lib/mimic/llm_cli/base.rb +36 -0
  45. data/lib/mimic/llm_cli/configure.rb +84 -0
  46. data/lib/mimic/llm_cli/embed.rb +80 -0
  47. data/lib/mimic/llm_cli/engines.rb +44 -0
  48. data/lib/mimic/llm_cli/models.rb +31 -0
  49. data/lib/mimic/llm_cli/models_table.rb +119 -0
  50. data/lib/mimic/llm_cli/paths.rb +48 -0
  51. data/lib/mimic/llm_cli/root.rb +215 -0
  52. data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
  53. data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
  54. data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
  55. data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
  56. data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
  57. data/lib/mimic/llm_cli.rb +5 -0
  58. data/lib/mimic/llm_config/embed.rb +48 -0
  59. data/lib/mimic/llm_config/llama_settings.rb +60 -0
  60. data/lib/mimic/llm_config/model_settings.rb +84 -0
  61. data/lib/mimic/llm_config/sampling.rb +54 -0
  62. data/lib/mimic/llm_config/settings.rb +92 -0
  63. data/lib/mimic/llm_config.rb +132 -0
  64. data/lib/mimic/model/footprint.rb +303 -0
  65. data/lib/mimic/model/gguf.rb +204 -0
  66. data/lib/mimic/model/header.rb +169 -0
  67. data/lib/mimic/model/library.rb +132 -0
  68. data/lib/mimic/model/memory.rb +126 -0
  69. data/lib/mimic/model/store.rb +46 -0
  70. data/lib/mimic/model/upstream.rb +81 -0
  71. data/lib/mimic/model.rb +116 -0
  72. data/lib/mimic/paths.rb +45 -0
  73. data/lib/mimic/result.rb +8 -0
  74. data/lib/mimic/ui.rb +89 -0
  75. data/lib/mimic/util/fmt.rb +15 -0
  76. data/lib/mimic/util/hf.rb +55 -0
  77. data/lib/mimic/util/host.rb +95 -0
  78. data/lib/mimic/util/probes.rb +255 -0
  79. data/lib/mimic/version.rb +3 -0
  80. data/lib/mimic.rb +22 -0
  81. metadata +186 -0
@@ -0,0 +1,82 @@
1
+ require_relative "../../ui"
2
+ require_relative "../../util/fmt"
3
+ require_relative "../../engine"
4
+ require_relative "../../model"
5
+ require_relative "../../model/footprint"
6
+ require_relative "../../model/header"
7
+
8
+ module Mimic
9
+ module LlmCli
10
+ module Wizard
11
+ # Estimate VRAM for a chosen context; warn + confirm if it would
12
+ # over-commit. llama.cpp accepts a context it cannot hold and dies once
13
+ # the KV cache fills, so the negotiation happens here rather than at
14
+ # launch.
15
+ class ContextPreflight
16
+ def initialize(config, prompt: Ui::Prompt.new)
17
+ @config = config
18
+ @prompt = prompt
19
+ end
20
+
21
+ # True to accept (fits, un-checkable, offloaded, or user override),
22
+ # false to re-pick.
23
+ def ok?
24
+ est = estimate
25
+ return true unless est.measurable?
26
+
27
+ if est.fits?
28
+ puts "#{Ui.green('VRAM ok')} -- needs ~#{format('%.1f', est.need_mib / 1024.0)} GB of " \
29
+ "#{format('%.1f', est.total_mib / 1024.0)} GB (fits to ~#{Util::Fmt.commas(est.max_ctx)} tok)"
30
+ return true
31
+ end
32
+ return true if offloaded_to_fit?(est)
33
+
34
+ warn_over_commit(est)
35
+ @prompt.yes?("Use this over-committed context anyway?", default: false)
36
+ end
37
+
38
+ private
39
+
40
+ def estimate
41
+ model = @config.model.file
42
+ model.footprint(ctx, cache_type, compute_mib: model.memory.compute_mib(ctx, cache_type),
43
+ slots: @config.llama.n_slots, reclaimable_mib: Engine::ServerProcess.chat.gpu_mib,
44
+ mmproj_path: @config.model.mmproj_path)
45
+ end
46
+
47
+ def ctx = @config.model.context_length
48
+ def cache_type = @config.llama.cache_type
49
+
50
+ # A MoE model can trade VRAM for PCIe traffic by keeping some layers'
51
+ # experts on the CPU, so a deficit the ceiling can cover is a setting to
52
+ # write, not a warning to print. Only ik takes --n-cpu-moe; llama.cpp's
53
+ # preset pins n-gpu-layers=999 on purpose.
54
+ def offloaded_to_fit?(est)
55
+ return false unless @config.llama.ik?
56
+
57
+ layers = est.layers_to_offload(ceiling: offload_ceiling)
58
+ return false if layers.nil? || layers.zero?
59
+
60
+ @config.model.file.memory.remember_n_cpu_moe(layers)
61
+ puts "#{Ui.green("offloading #{layers} layers of MoE experts to CPU")} -- " \
62
+ "frees ~#{format('%.1f', (est.need_mib - est.budget_mib) / 1024.0)} GB, " \
63
+ "so #{Util::Fmt.commas(ctx)} tokens fits"
64
+ true
65
+ end
66
+
67
+ def offload_ceiling
68
+ layers = Model::Header.new(@config.model.path).block_count
69
+ [Engine::IkLlama::CPU_MOE_MAX, layers || Engine::IkLlama::CPU_MOE_MAX].min
70
+ end
71
+
72
+ def warn_over_commit(est)
73
+ puts "#{Ui.yellow('this context won\'t fully fit VRAM.')} Needs " \
74
+ "~#{format('%.1f', est.need_mib / 1024.0)} GB (weights #{format('%.1f', est.weights_mib / 1024.0)} + " \
75
+ "KV #{format('%.1f', est.kv_mib / 1024.0)} + compute), budget ~#{format('%.1f', est.budget_mib / 1024.0)} GB.\n" \
76
+ " It will load but crash at runtime once context passes " \
77
+ "~#{Util::Fmt.commas(est.max_ctx)} tokens. Pick <= that, or switch KV to q4_0."
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,89 @@
1
+ require_relative "../../ui"
2
+ require_relative "../../llm_config"
3
+ require_relative "../../engine/llama_embed"
4
+
5
+ module Mimic
6
+ module LlmCli
7
+ module Wizard
8
+ # Picks which embedding model `llm embed start` launches, and on which
9
+ # port. The endpoint it serves is not asked for -- Engine::LlamaEmbed reads it
10
+ # off the GGUF, and `mode` in config.yaml overrides that when the header is
11
+ # wrong.
12
+ class Embed
13
+ def initialize(config, models, prompt: Ui::Prompt.new)
14
+ @config = config
15
+ @embed = config.embed
16
+ @models = models
17
+ @prompt = prompt
18
+ end
19
+
20
+ def run
21
+ Ui.banner("llm embed -- setup wizard")
22
+ return no_models if @models.empty?
23
+
24
+ choose_model
25
+ choose_port
26
+ choose_batch_size
27
+ @config.save
28
+ report
29
+ @config
30
+ end
31
+
32
+ private
33
+
34
+ def choose_model
35
+ step(1, "Embedding model")
36
+ choices = @models.to_h { |model| [label(model), model.path.to_s] }
37
+ @embed.path = @prompt.select("Embedding model:", choices, default: choices.key(@embed.path))
38
+ end
39
+
40
+ def choose_port
41
+ step(2, "Port")
42
+ loop do
43
+ port = @prompt.ask("Port for the embedding server:", convert: :int, default: @embed.port)
44
+ if port.to_i == @config.llama.port.to_i
45
+ puts Ui.red("Port #{port} is where llama-server already serves the chat model.")
46
+ next
47
+ end
48
+
49
+ @embed.port = port.to_i
50
+ break
51
+ end
52
+ end
53
+
54
+ # Both llama-server batch flags come off this one answer: a pooled
55
+ # encoder cannot split a document across micro-batches, so the physical
56
+ # batch has to hold the longest document rather than sit at its default.
57
+ def choose_batch_size
58
+ step(3, "Batch size")
59
+ puts "Must be at least as long, in tokens, as the longest document you send."
60
+ @embed.batch_size = @prompt.ask("Batch size:", convert: :int, default: @embed.batch_size).to_i
61
+ end
62
+
63
+ def label(model)
64
+ mode = Engine::LlamaEmbed.mode_for(model.path)
65
+ "#{model.path.dirname.basename}/#{model.path.basename} [#{mode}]"
66
+ end
67
+
68
+ def report
69
+ Ui.panel(
70
+ "Embedding model: #{File.basename(@embed.path)}\n" \
71
+ "Endpoint: #{@embed.base_url} (#{Engine::LlamaEmbed.new(@config).mode})\n" \
72
+ "Batch size: #{@embed.batch_size}\n" \
73
+ "Saved to #{LlmConfig::LLM_CONFIG_PATH}"
74
+ )
75
+ end
76
+
77
+ def no_models
78
+ puts "No embedding models found under the configured search paths."
79
+ puts "Add one with `llm path add <dir>` -- see `llm models`."
80
+ nil
81
+ end
82
+
83
+ def step(index, title)
84
+ puts "\n#{Ui.bold_cyan("Step #{index}/3")} -- #{title}"
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,82 @@
1
+ require_relative "../../ui"
2
+ require_relative "../../engine"
3
+ require_relative "../../model/footprint"
4
+ require_relative "../../model"
5
+
6
+ module Mimic
7
+ module LlmCli
8
+ module Wizard
9
+ class Moe
10
+ def initialize(scan, prompt: Ui::Prompt.new)
11
+ @scan = scan
12
+ @prompt = prompt
13
+ end
14
+
15
+ def run
16
+ Ui.banner("llm -- MoE offload wizard")
17
+ paths = moe_model_paths
18
+ return puts "No MoE .gguf models found under the configured search paths." if paths.empty?
19
+
20
+ path = choose_model(paths)
21
+ case choose_action(path)
22
+ when :set then set_value(path)
23
+ when :clear then clear_value(path)
24
+ else puts "Cancelled."
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def moe_model_paths
31
+ @scan.each_with_object([]) do |model, found|
32
+ real = model.real.to_s
33
+ found << real if !found.include?(real) && Model::Header.new(real).moe?
34
+ end
35
+ end
36
+
37
+ def choose_model(paths)
38
+ step(1, "Model")
39
+ choices = paths.to_h do |path|
40
+ cached = Model.new(path).memory.n_cpu_moe
41
+ ["#{File.basename(path)}#{cached ? " (cached: #{cached})" : ''}", path]
42
+ end
43
+ @prompt.select("MoE model:", choices)
44
+ end
45
+
46
+ def choose_action(path)
47
+ step(2, "n_cpu_moe")
48
+ puts "Current cached value: #{Model.new(path).memory.n_cpu_moe || 'none (auto-tunes from 0 on next start)'}"
49
+ @prompt.select("What do you want to do?", [
50
+ { name: "Set a value (0-#{ceiling(path)})", value: :set },
51
+ { name: "Clear cached value", value: :clear },
52
+ { name: "Cancel", value: :cancel },
53
+ ])
54
+ end
55
+
56
+ def set_value(path)
57
+ limit = ceiling(path)
58
+ value = @prompt.ask("Layers to offload to CPU (0-#{limit}):", convert: :int, default: Model.new(path).memory.n_cpu_moe || 0)
59
+ abort "n_cpu_moe must be between 0 and #{limit}" unless value.between?(0, limit)
60
+
61
+ Model.new(path).memory.remember_n_cpu_moe(value)
62
+ puts "#{Ui.green('Set')} #{File.basename(path)} n_cpu_moe=#{value}"
63
+ end
64
+
65
+ def clear_value(path)
66
+ abort "no cached value to clear for #{File.basename(path)}" if Model.new(path).memory.n_cpu_moe.nil?
67
+
68
+ Model.new(path).memory.forget_n_cpu_moe
69
+ puts "#{Ui.green('Cleared')} #{File.basename(path)}"
70
+ end
71
+
72
+ def ceiling(path)
73
+ @ceiling ||= Model::Header.new(path).block_count || Engine::IkLlama::CPU_MOE_MAX
74
+ end
75
+
76
+ def step(index, title)
77
+ puts "\n#{Ui.bold_cyan("Step #{index}/2")} -- #{title}"
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,76 @@
1
+ require_relative "../../ui"
2
+ require_relative "../../util/fmt"
3
+ require_relative "../../model/footprint"
4
+
5
+ module Mimic
6
+ module LlmCli
7
+ module Wizard
8
+ # Pick an n_slots/ctx-per-request pair so the server can hold several
9
+ # requests at once. Total ctx-size is the product, and that is what has
10
+ # to fit in VRAM.
11
+ class Parallel
12
+ CTX_PRESETS = {
13
+ "2048 -- short prompts, max concurrency" => 2048,
14
+ "4096 -- humaneval-rb-sized prompts" => 4096,
15
+ "8192 -- moderate prompts" => 8192,
16
+ "16000 -- long prompts" => 16_000,
17
+ }.freeze
18
+
19
+ SLOT_CHOICES = [1, 2, 4, 8, 16, 32].freeze
20
+
21
+ def initialize(config, setup_wizard, prompt: Ui::Prompt.new)
22
+ @config = config
23
+ @setup_wizard = setup_wizard
24
+ @prompt = prompt
25
+ end
26
+
27
+ def run
28
+ Ui.banner("llm -- parallel wizard")
29
+
30
+ step(1, "Model")
31
+ @config.model.path = @setup_wizard.select_model(@config.model.path)
32
+
33
+ step(2, "Context per request")
34
+ ctx_per_slot = choose_ctx_per_slot
35
+
36
+ step(3, "Slots")
37
+ n_slots = choose_slots(ctx_per_slot)
38
+
39
+ @config.model.context_length = ctx_per_slot * n_slots
40
+ @config.llama.n_slots = n_slots
41
+ @config
42
+ end
43
+
44
+ private
45
+
46
+ def choose_ctx_per_slot
47
+ picked = @prompt.select("Context per request:", [*CTX_PRESETS.keys, "Custom..."])
48
+ return CTX_PRESETS.fetch(picked) unless picked == "Custom..."
49
+
50
+ Integer(@prompt.ask("Context per request (tokens):", default: "4096"))
51
+ end
52
+
53
+ def choose_slots(ctx_per_slot)
54
+ puts "Total ctx-size = context-per-request x slots -- that's what has to fit in VRAM alongside the weights."
55
+ SLOT_CHOICES.each { |slots| puts fit_line(ctx_per_slot, slots) }
56
+ Integer(@prompt.select("n_slots:", SLOT_CHOICES.map(&:to_s), default: "8"))
57
+ end
58
+
59
+ def fit_line(ctx_per_slot, slots)
60
+ total = ctx_per_slot * slots
61
+ model = @config.model.file
62
+ est = model.footprint(total, @config.llama.cache_type,
63
+ compute_mib: model.memory.compute_mib(total, @config.llama.cache_type),
64
+ slots: slots, reclaimable_mib: Engine::ServerProcess.chat.gpu_mib,
65
+ mmproj_path: @config.model.mmproj_path)
66
+ tag = !est.measurable? || est.fits? ? Ui.green("fits") : Ui.red("too big")
67
+ format(" %2d slots -> %10s total ctx %s", slots, Util::Fmt.commas(total), tag)
68
+ end
69
+
70
+ def step(index, title)
71
+ puts "\n#{Ui.bold_cyan("Step #{index}/3")} -- #{title}"
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,211 @@
1
+ require "pathname"
2
+ require "etc"
3
+ require_relative "../../ui"
4
+ require_relative "../../util/fmt"
5
+ require_relative "../../util/host"
6
+ require_relative "../../util/probes"
7
+ require_relative "../../engine_cli"
8
+ require_relative "../../engine"
9
+ require_relative "context_preflight"
10
+ require_relative "../../model/library"
11
+ require_relative "../../llm_config"
12
+ require_relative "../../model"
13
+
14
+ module Mimic
15
+ module LlmCli
16
+ module Wizard
17
+ class Setup
18
+ CONTEXT_PRESETS = {
19
+ "60000 / q8_0" => [60_000, "q8_0"],
20
+ "80000 / q8_0" => [80_000, "q8_0"],
21
+ "110000 / q8_0" => [110_000, "q8_0"],
22
+ "120000 / q8_0" => [120_000, "q8_0"],
23
+ "131072 / q8_0" => [131_072, "q8_0"],
24
+ "140000 / q8_0" => [140_000, "q8_0"],
25
+ "150000 / q8_0" => [150_000, "q8_0"],
26
+ "160000 / q8_0" => [160_000, "q8_0"],
27
+ "180000 / q8_0" => [180_000, "q8_0"],
28
+ "200000 / q8_0" => [200_000, "q8_0"],
29
+ "262144 / q4_0" => [262_144, "q4_0"],
30
+ }.freeze
31
+
32
+ ENGINE_BLURBS = {
33
+ "llama.cpp" => "upstream, router-mode preset",
34
+ "ik_llama.cpp" => "ikawrakow fork (IQK quants, fused MoE)",
35
+ "prism-v7-llama.cpp" => "current Prism fork (Bonsai 2 PQ2_0/PTQ1_0)",
36
+ }.freeze
37
+
38
+ attr_reader :config
39
+
40
+ def initialize(config, total: 8, cli_name: "llm", prompt: Ui::Prompt.new)
41
+ @config = config
42
+ @total = total
43
+ @cli_name = cli_name
44
+ @prompt = prompt
45
+ @step = 0
46
+ end
47
+
48
+ def step(title)
49
+ @step += 1
50
+ puts "\n#{Ui.bold_cyan("Step #{@step}/#{@total}")} -- #{title}"
51
+ end
52
+
53
+ def system_check
54
+ step("System check")
55
+ rows = [
56
+ ["RAM", "#{Util::Host.total_mem_kb / 1024 / 1024} GB"],
57
+ ["Free disk (home)", "#{Util::Host.free_home_bytes / (1024**3)} GB"],
58
+ ["Ruby", RUBY_VERSION],
59
+ ["GPU backend (detected)", Util::Host.backend],
60
+ ["llama-server", Util::Host.llama_server || Ui.red("not found")],
61
+ ["claude on PATH", Util::Probes.which("claude") || Ui.red("not found")],
62
+ ]
63
+ rows.each { |label, value| puts format(" %-24s %s", label, value) }
64
+ end
65
+
66
+ def choose_engine
67
+ step("Inference engine")
68
+ choices = EngineCli::Registry.names.map do |name|
69
+ { name: [name, ENGINE_BLURBS[name]].compact.join(" -- "), value: name }
70
+ end
71
+ current = choices.index { |choice| choice[:value] == @config.llama.engine } || 0
72
+ @config.llama.engine = @prompt.select("Inference engine:", choices, default: current + 1)
73
+ end
74
+
75
+ def ensure_binary(engine)
76
+ step("#{engine} binary")
77
+ @config.llama.binary = EngineCli::Builder.new(@config, cli_name: @cli_name, prompt: @prompt)
78
+ .find_or_build(engine)
79
+ end
80
+
81
+ def choose_model
82
+ step("Model")
83
+ @config.model.path = select_model(@config.model.path)
84
+ report_hidden_models
85
+ end
86
+
87
+ def choose_mmproj
88
+ step("Multi-modal projector (optional)")
89
+ candidate = Model.projector_for(@config.model.path)
90
+ unless candidate
91
+ @config.model.mmproj_path = nil
92
+ return
93
+ end
94
+
95
+ accepted = @prompt.yes?("Use #{candidate.basename} as projector?", default: false)
96
+ @config.model.mmproj_path = accepted ? candidate.to_s : nil
97
+ end
98
+
99
+ def choose_context
100
+ step("Context preset")
101
+ remembered = @config.model.file.memory.last_ctx
102
+ if remembered
103
+ @config.model.context_length, @config.llama.cache_type = remembered
104
+ unless CONTEXT_PRESETS.value?(remembered)
105
+ puts "#{Ui.green('Last good run for this model:')} #{Util::Fmt.commas(remembered[0])} / #{remembered[1]}"
106
+ end
107
+ end
108
+ preflight = ContextPreflight.new(@config, prompt: @prompt)
109
+ loop do
110
+ pick_context
111
+ break if preflight.ok?
112
+ end
113
+ @config
114
+ end
115
+
116
+ def choose_reasoning
117
+ step("Reasoning")
118
+ wanted = @prompt.yes?("Let the model think before it answers?", default: true)
119
+ @config.model.reasoning = wanted ? "auto" : "off"
120
+ @config
121
+ end
122
+
123
+ def choose_effort
124
+ step("Thinking level")
125
+ template = Engine::ChatTemplate.new(@config.model.path, model_alias: @config.model.name)
126
+ levels = template.effort_levels
127
+ if @config.model.reasoning == "off" || levels.empty?
128
+ @config.model.reasoning_effort = nil
129
+ puts " #{@config.model.reasoning == 'off' ? 'thinking is off' : 'the chat template has no levels'}"
130
+ return @config
131
+ end
132
+
133
+ @config.model.reasoning_effort = @prompt.select("How hard should the model think?", levels,
134
+ default: template.default_effort)
135
+ @config
136
+ end
137
+
138
+ def select_model(current)
139
+ by_label = library.chat.to_h { |model| [model_label(model), model.path.to_s] }
140
+ manual = "Enter a path manually..."
141
+ answer = if by_label.any?
142
+ current_label = by_label.key(current)
143
+ @prompt.select("GGUF model file:", [*by_label.keys, manual], default: current_label)
144
+ else
145
+ manual
146
+ end
147
+ answer = if answer == manual
148
+ @prompt.ask("Path to .gguf file:", default: current)
149
+ else
150
+ by_label.fetch(answer, answer)
151
+ end
152
+ # A typed answer arrives literal -- no shell has expanded ~ for us,
153
+ # and File.file? does not either.
154
+ expanded = Pathname.new(answer.to_s).expand_path
155
+ if answer.to_s.empty? || !expanded.file?
156
+ puts Ui.red("Model file not found.")
157
+ raise SystemExit, 1
158
+ end
159
+ expanded.realpath.to_s
160
+ end
161
+
162
+ private
163
+
164
+ def library
165
+ @library ||= Model::Library.new(@config)
166
+ end
167
+
168
+ def report_hidden_models
169
+ return if library.hidden.empty?
170
+
171
+ library.hidden.group_by { |model| library.hidden_reason(model) }.each do |reason, models|
172
+ names = models.map { |model| model.path.basename.to_s }.join(", ")
173
+ puts Ui.yellow("hidden -- #{reason}: #{names}")
174
+ end
175
+ end
176
+
177
+ def model_label(model)
178
+ tags = []
179
+ tags << "MoE" if Model::Header.new(model.real).moe?
180
+ tags << "vision" if model.vision?
181
+ suffix = tags.empty? ? "" : " [#{tags.join(', ')}]"
182
+ "#{model.path.dirname.basename}/#{model.path.basename}#{suffix}"
183
+ end
184
+
185
+ def pick_context
186
+ presets = labelled_presets
187
+ current_preset = presets.find do |_, (ctx, kv)|
188
+ ctx == @config.model.context_length && kv == @config.llama.cache_type
189
+ end&.first
190
+ picked = @prompt.select("Context preset:", [*presets.keys, "Custom..."], default: current_preset)
191
+ if picked == "Custom..."
192
+ context = @prompt.ask("Context length (tokens):", default: @config.model.context_length.to_s)
193
+ @config.model.context_length = Integer(context)
194
+ @config.llama.cache_type = @prompt.select("KV cache type:", %w[q8_0 q4_0 f16], default: @config.llama.cache_type)
195
+ else
196
+ ctx, kv = presets.fetch(picked)
197
+ @config.model.context_length = ctx
198
+ @config.llama.cache_type = kv
199
+ end
200
+ end
201
+
202
+ def labelled_presets
203
+ remembered = @config.model.file.memory.last_ctx
204
+ CONTEXT_PRESETS.to_h do |label, preset|
205
+ [preset == remembered ? "#{label} -- last success run" : label, preset]
206
+ end
207
+ end
208
+ end
209
+ end
210
+ end
211
+ end
@@ -0,0 +1,5 @@
1
+ require_relative "llm_cli/base"
2
+ require_relative "llm_cli/embed"
3
+ require_relative "llm_cli/paths"
4
+ require_relative "llm_cli/engines"
5
+ require_relative "llm_cli/root"
@@ -0,0 +1,48 @@
1
+ require_relative "../llm_config"
2
+
3
+ module Mimic
4
+ module LlmConfig
5
+ # The embedding server's profile: its own model and its own port, so it
6
+ # can serve /v1/embeddings beside the chat server rather than in place of
7
+ # it. `mode` picks the endpoint llama-server exposes -- "auto" reads the
8
+ # GGUF's pooling type, "embeddings" and "rerank" force it.
9
+ class Embed
10
+ MODES = %w[auto embeddings rerank].freeze
11
+
12
+ def initialize(cfg, host:)
13
+ @cfg = cfg
14
+ @host = host
15
+ end
16
+
17
+ def path = @cfg["path"].to_s
18
+ def port = @cfg.fetch("port", 8081)
19
+ def context_length = @cfg.fetch("context_length", 0)
20
+ def batch_size = @cfg.fetch("batch_size", 8192)
21
+ def mode = @cfg.fetch("mode", "auto")
22
+ def host = @host
23
+ def configured? = !path.empty?
24
+ def model_alias = Settings.alias_for(path)
25
+ def base_url = "http://127.0.0.1:#{port}"
26
+
27
+ def path=(value)
28
+ @cfg["path"] = value
29
+ end
30
+
31
+ def port=(value)
32
+ @cfg["port"] = value
33
+ end
34
+
35
+ def context_length=(value)
36
+ @cfg["context_length"] = value
37
+ end
38
+
39
+ def batch_size=(value)
40
+ @cfg["batch_size"] = value
41
+ end
42
+
43
+ def mode=(value)
44
+ @cfg["mode"] = value
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,60 @@
1
+ module Mimic
2
+ module LlmConfig
3
+ # The engine section: which llama-server build runs, where it binds, and
4
+ # the knobs that belong to the server rather than to the model.
5
+ class LlamaSettings
6
+ SECTION = "llama".freeze
7
+
8
+ def initialize(cfg)
9
+ @cfg = cfg
10
+ end
11
+
12
+ def engine = section.fetch("engine", "llama.cpp")
13
+ def host = section["host"]
14
+ def port = section["port"]
15
+ def n_slots = section.fetch("n_slots", 1)
16
+ def flash_attn = section["flash_attn"]
17
+ def cache_type = section["cache_type"]
18
+ def ik? = engine == "ik_llama.cpp"
19
+
20
+ # `binary` is per-engine: llama.cpp and ik_llama.cpp are different
21
+ # binaries, so one flat field meant switching engines could hand the
22
+ # wrong engine the other's path. A pre-split config.yaml is upgraded to
23
+ # the per-engine hash by LlmConfig.load, so a String here only reaches
24
+ # a section built by hand.
25
+ def binary
26
+ raw = section["binary"]
27
+ raw.is_a?(Hash) ? raw[engine].to_s : raw.to_s
28
+ end
29
+
30
+ # Loopback URL for clients, not the server's bind address.
31
+ #
32
+ # `llama.host` is 0.0.0.0 so containers can reach the server; writing
33
+ # that into a client-facing URL would be wrong -- every client on this
34
+ # host (mimic claude/codex, hermes, status checks) dials 127.0.0.1.
35
+ def base_url = "http://127.0.0.1:#{port}"
36
+
37
+ def engine=(value)
38
+ writable["engine"] = value
39
+ end
40
+
41
+ def cache_type=(value)
42
+ writable["cache_type"] = value
43
+ end
44
+
45
+ def n_slots=(value)
46
+ writable["n_slots"] = value
47
+ end
48
+
49
+ def binary=(value)
50
+ raw = section["binary"]
51
+ writable["binary"] = raw.is_a?(Hash) ? raw.merge(engine => value) : { engine => value }
52
+ end
53
+
54
+ private
55
+
56
+ def section = @cfg[SECTION] || {}
57
+ def writable = @cfg[SECTION] ||= {}
58
+ end
59
+ end
60
+ end