mimic-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/NOTICE +16 -0
  4. data/README.md +175 -0
  5. data/exe/lllm +5 -0
  6. data/exe/llm +5 -0
  7. data/exe/mimic +5 -0
  8. data/lib/mimic/README.md +144 -0
  9. data/lib/mimic/agent/claude.rb +305 -0
  10. data/lib/mimic/agent/codex.rb +254 -0
  11. data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
  12. data/lib/mimic/agent/hermes.rb +107 -0
  13. data/lib/mimic/agent/muse.rb +106 -0
  14. data/lib/mimic/agent.rb +46 -0
  15. data/lib/mimic/cli.rb +159 -0
  16. data/lib/mimic/client/base.rb +44 -0
  17. data/lib/mimic/client/crush.rb +28 -0
  18. data/lib/mimic/client/goose.rb +52 -0
  19. data/lib/mimic/client/hermes.rb +21 -0
  20. data/lib/mimic/client/opencode.rb +29 -0
  21. data/lib/mimic/client/pi.rb +50 -0
  22. data/lib/mimic/client.rb +19 -0
  23. data/lib/mimic/engine/README.md +140 -0
  24. data/lib/mimic/engine/base.rb +138 -0
  25. data/lib/mimic/engine/chat_template.rb +207 -0
  26. data/lib/mimic/engine/ik_llama.rb +273 -0
  27. data/lib/mimic/engine/launcher.rb +25 -0
  28. data/lib/mimic/engine/llama.rb +124 -0
  29. data/lib/mimic/engine/llama_embed.rb +108 -0
  30. data/lib/mimic/engine/router.rb +57 -0
  31. data/lib/mimic/engine/server_process.rb +108 -0
  32. data/lib/mimic/engine.rb +5 -0
  33. data/lib/mimic/engine_cli/binary_options.rb +46 -0
  34. data/lib/mimic/engine_cli/builder.rb +98 -0
  35. data/lib/mimic/engine_cli/diffusion.rb +117 -0
  36. data/lib/mimic/engine_cli/llama_preset.rb +119 -0
  37. data/lib/mimic/engine_cli/registry.rb +94 -0
  38. data/lib/mimic/engine_cli/source.rb +110 -0
  39. data/lib/mimic/engine_cli/status.rb +161 -0
  40. data/lib/mimic/engine_cli/update.rb +68 -0
  41. data/lib/mimic/engine_cli.rb +7 -0
  42. data/lib/mimic/health/checks.rb +92 -0
  43. data/lib/mimic/health.rb +82 -0
  44. data/lib/mimic/llm_cli/base.rb +36 -0
  45. data/lib/mimic/llm_cli/configure.rb +84 -0
  46. data/lib/mimic/llm_cli/embed.rb +80 -0
  47. data/lib/mimic/llm_cli/engines.rb +44 -0
  48. data/lib/mimic/llm_cli/models.rb +31 -0
  49. data/lib/mimic/llm_cli/models_table.rb +119 -0
  50. data/lib/mimic/llm_cli/paths.rb +48 -0
  51. data/lib/mimic/llm_cli/root.rb +215 -0
  52. data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
  53. data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
  54. data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
  55. data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
  56. data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
  57. data/lib/mimic/llm_cli.rb +5 -0
  58. data/lib/mimic/llm_config/embed.rb +48 -0
  59. data/lib/mimic/llm_config/llama_settings.rb +60 -0
  60. data/lib/mimic/llm_config/model_settings.rb +84 -0
  61. data/lib/mimic/llm_config/sampling.rb +54 -0
  62. data/lib/mimic/llm_config/settings.rb +92 -0
  63. data/lib/mimic/llm_config.rb +132 -0
  64. data/lib/mimic/model/footprint.rb +303 -0
  65. data/lib/mimic/model/gguf.rb +204 -0
  66. data/lib/mimic/model/header.rb +169 -0
  67. data/lib/mimic/model/library.rb +132 -0
  68. data/lib/mimic/model/memory.rb +126 -0
  69. data/lib/mimic/model/store.rb +46 -0
  70. data/lib/mimic/model/upstream.rb +81 -0
  71. data/lib/mimic/model.rb +116 -0
  72. data/lib/mimic/paths.rb +45 -0
  73. data/lib/mimic/result.rb +8 -0
  74. data/lib/mimic/ui.rb +89 -0
  75. data/lib/mimic/util/fmt.rb +15 -0
  76. data/lib/mimic/util/hf.rb +55 -0
  77. data/lib/mimic/util/host.rb +95 -0
  78. data/lib/mimic/util/probes.rb +255 -0
  79. data/lib/mimic/version.rb +3 -0
  80. data/lib/mimic.rb +22 -0
  81. metadata +186 -0
@@ -0,0 +1,84 @@
1
+ require_relative "../ui"
2
+ require_relative "../llm_config"
3
+ require_relative "../engine_cli"
4
+
5
+ # LLM config editor -- flat key=value editing for model + llama keys.
6
+ #
7
+ # Edits ~/.llm/config.yaml only.
8
+ module Mimic
9
+ module LlmCli
10
+ module Configure
11
+ FIELDS = {
12
+ "model.path" => ["Model GGUF path", :str],
13
+ "model.context_length" => ["Context length", :int],
14
+ "model.mtp" => ["MTP speculative decoding (auto/on/off)", :str],
15
+ "model.temperature" => ["Temperature", :float],
16
+ "model.top_p" => ["top_p", :float],
17
+ "model.top_k" => ["top_k", :int],
18
+ "llama.engine" => ["Inference engine (#{EngineCli::Registry.names.join('/')})", :str],
19
+ "llama.host" => ["Bind address (0.0.0.0 lets containers reach it; 127.0.0.1 is host-only)", :str],
20
+ "llama.port" => ["llama.cpp port", :int],
21
+ "llama.n_slots" => ["Number of llama.cpp slots (1 for local use)", :int],
22
+ }.freeze
23
+
24
+ class << self
25
+ def get(cfg, dotted)
26
+ section, key = dotted.split(".")
27
+ cfg[section][key]
28
+ end
29
+
30
+ def set(cfg, dotted, raw)
31
+ section, key = dotted.split(".")
32
+ cfg[section][key] = case FIELDS.fetch(dotted)[1]
33
+ when :int then Integer(raw, 10)
34
+ when :float then Float(raw)
35
+ else raw
36
+ end
37
+ end
38
+
39
+ def run
40
+ unless LlmConfig.exists?
41
+ puts Ui.yellow("No LLM config yet -- run `llm setup` first.")
42
+ return
43
+ end
44
+
45
+ settings = LlmConfig.load
46
+ cfg = settings.to_h
47
+ prompt = Ui::Prompt.new
48
+
49
+ loop do
50
+ picked = prompt.select("Edit which value?", menu_choices(cfg))
51
+ return if picked == "__quit__"
52
+ return save(settings) if picked == "__save__"
53
+
54
+ edit(cfg, picked, prompt)
55
+ end
56
+ end
57
+
58
+ private
59
+ def menu_choices(cfg)
60
+ choices = FIELDS.map do |dotted, (label, _)|
61
+ { name: format("%-45s = %s", label, get(cfg, dotted)), value: dotted }
62
+ end
63
+ choices << { name: "Save & exit", value: "__save__" }
64
+ choices << { name: "Exit without saving", value: "__quit__" }
65
+ end
66
+
67
+ def save(settings)
68
+ settings.save
69
+ EngineCli::Registry.for(settings).render_preset
70
+ puts "#{Ui.green('Saved')} #{LlmConfig::LLM_CONFIG_PATH}"
71
+ end
72
+
73
+ def edit(cfg, dotted, prompt)
74
+ raw = prompt.ask("#{FIELDS.fetch(dotted)[0]}:", default: get(cfg, dotted).to_s)
75
+ return if raw.nil?
76
+
77
+ set(cfg, dotted, raw)
78
+ rescue ArgumentError, TypeError
79
+ puts Ui.red("Invalid value, keeping previous.")
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,80 @@
1
+ require_relative "../ui"
2
+ require_relative "../util/fmt"
3
+ require_relative "base"
4
+ require_relative "wizard/embed"
5
+ require_relative "../llm_config"
6
+ require_relative "../model/library"
7
+ require_relative "../engine/llama_embed"
8
+
9
+ module Mimic
10
+ module LlmCli
11
+ # `llm embed` -- the embedding server, run beside the chat server rather
12
+ # than instead of it.
13
+ class Embed < Base
14
+ desc "setup", "pick the embedding model, its port and its batch size"
15
+ def setup
16
+ cfg = LlmConfig.load
17
+ return unless Wizard::Embed.new(cfg, embedding_models(cfg), prompt: @prompt).run
18
+ return unless @prompt.yes?("Start the embedding server now?", default: true)
19
+
20
+ start
21
+ end
22
+
23
+ desc "start", "start the embedding server"
24
+ def start
25
+ report(Engine::LlamaEmbed.new(LlmConfig.load).start)
26
+ end
27
+
28
+ desc "stop", "stop the embedding server"
29
+ def stop
30
+ puts "embedding server: #{Engine::ServerProcess.embed.stop ? 'stopped' : 'was not running'}"
31
+ end
32
+
33
+ desc "restart", "stop then start the embedding server"
34
+ def restart
35
+ stop
36
+ start
37
+ end
38
+
39
+ desc "status", "embedding server status"
40
+ def status
41
+ cfg = LlmConfig.load
42
+ embed = cfg.embed
43
+ return puts Ui.yellow("No embedding model configured -- run `llm embed setup`") unless embed.configured?
44
+
45
+ service = Engine::LlamaEmbed.new(cfg)
46
+ running = service.running?
47
+ puts running ? Ui.green("embedding server running") : Ui.yellow("embedding server not running")
48
+ puts "Model: #{File.basename(embed.path)}"
49
+ puts "Endpoint: #{embed.base_url}"
50
+ puts "Mode: #{service.mode}"
51
+ puts "Batch: #{embed.batch_size}"
52
+ end
53
+
54
+ desc "models", "list the embedding models under the search paths"
55
+ def models
56
+ cfg = LlmConfig.load
57
+ found = embedding_models(cfg)
58
+ return puts "No embedding models found under #{cfg.search_paths.to_a.join(', ')}" if found.empty?
59
+
60
+ rows = found.map do |model|
61
+ [
62
+ model.path.to_s == cfg.embed.path ? Ui.green("●") : "",
63
+ model.model_alias,
64
+ Engine::LlamaEmbed.mode_for(model.path),
65
+ Util::Fmt.human_size(model.real.size),
66
+ model.path.dirname.to_s.sub(/\A#{Regexp.escape(Dir.home)}/, "~"),
67
+ ]
68
+ end
69
+ Ui.table(header: ["", "Alias", "Mode", "Size", "Path"], rows: rows)
70
+ puts "\nSwitch with `llm embed setup` -- active model is marked ●"
71
+ end
72
+
73
+ private
74
+
75
+ def embedding_models(cfg)
76
+ Model::Library.new(cfg).embedding
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,44 @@
1
+ require_relative "base"
2
+ require_relative "../engine_cli"
3
+ require_relative "../result"
4
+
5
+ module Mimic
6
+ module LlmCli
7
+ # `llm engines` -- installed engine details and the update pass.
8
+ class Engines < Base
9
+ default_command :list
10
+
11
+ desc "list", "list all engines and their local source and binary details"
12
+ def list
13
+ active = LlmConfig.load.llama.engine if LlmConfig.exists?
14
+ EngineCli::Registry.names.each do |name|
15
+ registry = EngineCli::Registry
16
+ source = EngineCli::Source.local_status(name)
17
+ binary = registry.binaries(name).find { |path| path && File.file?(path) }
18
+
19
+ puts "#{name}#{' (active)' if name == active}"
20
+ puts " Source: #{registry.url(name)}"
21
+ puts " Branch: #{registry.branch(name)}"
22
+ puts " Checkout: #{registry.repo(name)}"
23
+ puts " Revision: #{source ? "#{source[:commit]} (#{source[:dirty] ? 'dirty' : 'clean'})" : 'not checked out'}"
24
+ puts " Binary: #{binary || 'missing'}"
25
+ puts
26
+ end
27
+ end
28
+
29
+ desc "update", "fetch and rebuild outdated llama.cpp, ik_llama.cpp, and Prism engines"
30
+ method_option :rollback, type: :string, enum: EngineCli::Registry.names,
31
+ desc: "rebuild an engine at its last-good commit"
32
+ def update
33
+ updater = EngineCli::Update.new
34
+ if options[:rollback]
35
+ failure = updater.rollback(options[:rollback])
36
+ report(failure) if failure
37
+ return
38
+ end
39
+
40
+ updater.run
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,31 @@
1
+ require_relative "base"
2
+ require_relative "models_table"
3
+ require_relative "../model/library"
4
+ require_relative "../llm_config"
5
+ require_relative "../util/hf"
6
+
7
+ module Mimic
8
+ module LlmCli
9
+ # `llm models` -- the board of every GGUF under the search paths.
10
+ class Models < Base
11
+ default_command :list
12
+
13
+ desc "list", "list available GGUF models"
14
+ def list
15
+ board.render
16
+ end
17
+
18
+ desc "check", "compare each local GGUF against the repo it was published from"
19
+ def check
20
+ board(client: Util::Hf.new).render
21
+ end
22
+
23
+ private
24
+
25
+ def board(client: nil)
26
+ cfg = LlmConfig.load
27
+ ModelsTable.new(cfg, Model::Library.new(cfg).chat_loadable_or_hidden, client: client)
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,119 @@
1
+ require_relative "../ui"
2
+ require "pathname"
3
+ require_relative "../util/fmt"
4
+ require_relative "../llm_config"
5
+ require_relative "../model"
6
+ require_relative "../paths"
7
+ require_relative "../model/footprint"
8
+ require_relative "../model/upstream"
9
+ require_relative "../util/hf"
10
+
11
+ module Mimic
12
+ # The `llm models` board: every GGUF under the configured search paths,
13
+ # with the per-model facts that decide whether it will actually run.
14
+ module LlmCli
15
+ class ModelsTable
16
+ HEADER = ["", "Alias", "Quant", "Size", "MoE", "MTP", "Vision", "n_cpu_moe", "Template", "Last Ctx", "Path"].freeze
17
+ ALIAS_WIDTH = 44
18
+ PATH_WIDTH = 20
19
+
20
+ def initialize(config, scan, client: nil)
21
+ @config = config
22
+ @scan = scan
23
+ @client = client
24
+ end
25
+
26
+ def render
27
+ rows = @scan.map { |model| row(model) }
28
+ return no_models if rows.empty?
29
+
30
+ Ui.table(header: header, rows: rows)
31
+ print_upstream_legend if @client
32
+ puts "\nSwitch with `llm setup` -- active model is marked ●"
33
+ print_search_paths
34
+ end
35
+
36
+ private
37
+
38
+ def header = @client ? HEADER + ["Upstream"] : HEADER
39
+
40
+ def row(model)
41
+ path = model.path
42
+ real = model.real
43
+ base_alias = model.model_alias
44
+ remembered = Model.new(real)
45
+ cells = [
46
+ real == active_real ? Ui.green("●") : "",
47
+ truncate(base_alias, ALIAS_WIDTH),
48
+ path.basename(".*").to_s.delete_prefix(base_alias).sub(/\A-/, ""),
49
+ Util::Fmt.human_size(real.size),
50
+ moe_mark(real),
51
+ model.supports_mtp? ? Ui.green("✓") : "",
52
+ model.vision? ? Ui.green("✓") : "",
53
+ remembered.memory.n_cpu_moe&.to_s || "",
54
+ template_for(base_alias).file? ? "✓" : "",
55
+ last_ctx(remembered),
56
+ parent_label(path),
57
+ ]
58
+ @client ? cells + [upstream_mark(model)] : cells
59
+ end
60
+
61
+ def print_upstream_legend
62
+ puts "\nUpstream compares file size against the repo the GGUF was published from."
63
+ puts " #{Ui.gray('?')} no source repo -- the header names none and the path is not LM Studio's, or huggingface.co was unreachable"
64
+ puts " #{Ui.yellow('gone')} the repo no longer publishes a file by that name"
65
+ end
66
+
67
+ def upstream_mark(model)
68
+ upstream = Model::Upstream.new(model, client: @client)
69
+ case upstream.verdict
70
+ when :current then Ui.green("current")
71
+ when :stale then Ui.red(["stale", upstream.updated_at].compact.join(" "))
72
+ when :gone then Ui.yellow("gone")
73
+ else Ui.gray("?")
74
+ end
75
+ end
76
+
77
+ def active_real
78
+ @active_real ||= begin
79
+ active = Pathname.new(@config.model.path).expand_path
80
+ active.exist? ? active.realpath : active
81
+ end
82
+ end
83
+
84
+ def moe_mark(real)
85
+ moe = Model::Header.new(real).moe?
86
+ return "" if moe.nil?
87
+
88
+ moe ? Ui.green("✓") : Ui.red("✗")
89
+ end
90
+
91
+ def template_for(base_alias) = Mimic::Paths::CHAT_TEMPLATES_DIR / "#{base_alias}.jinja"
92
+
93
+ def last_ctx(model)
94
+ recalled = model.memory.last_ctx&.first
95
+ recalled.nil? ? "" : Util::Fmt.commas(recalled)
96
+ end
97
+
98
+ def parent_label(path)
99
+ label = path.dirname.to_s.sub(/\A#{Regexp.escape(Dir.home)}/, "~")
100
+ label.length > PATH_WIDTH ? "…#{label[-(PATH_WIDTH - 1)..]}" : label
101
+ end
102
+
103
+ def truncate(text, width) = text.length > width ? "#{text[0, width - 1]}…" : text
104
+
105
+ def no_models
106
+ puts "No .gguf files found in #{@config.search_paths.to_a.join(', ')}"
107
+ print_search_paths
108
+ end
109
+
110
+ def print_search_paths
111
+ puts "\nSearched:"
112
+ @config.search_paths.each do |directory|
113
+ mark = directory.directory? ? Ui.green("✓") : Ui.red("✗")
114
+ puts " #{mark} #{directory}"
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,48 @@
1
+ require_relative "../ui"
2
+ require "pathname"
3
+ require_relative "base"
4
+ require_relative "../llm_config"
5
+
6
+ module Mimic
7
+ module LlmCli
8
+ # `llm path` -- the directories scanned for GGUF files.
9
+ class Paths < Base
10
+ desc "add DIR", "add a directory to the model search paths"
11
+ def add(dir)
12
+ target = Pathname.new(dir).expand_path
13
+ cfg = LlmConfig.load
14
+ case cfg.search_paths.add(target)
15
+ when :builtin then puts "#{target} is already searched by default."
16
+ when :duplicate then puts "#{target} is already a search path."
17
+ else
18
+ abort "not a directory: #{target}" unless target.directory?
19
+
20
+ cfg.save
21
+ puts "#{Ui.green('Added')} #{target}"
22
+ end
23
+ end
24
+
25
+ desc "rm DIR", "remove a directory from the model search paths"
26
+ def rm(dir)
27
+ target = Pathname.new(dir).expand_path
28
+ cfg = LlmConfig.load
29
+ status = cfg.search_paths.remove(target)
30
+ abort "#{target} is a builtin search path and cannot be removed" if status == :builtin
31
+ abort "#{target} is not a configured search path" if status == :missing
32
+
33
+ cfg.save
34
+ puts "#{Ui.green('Removed')} #{target}"
35
+ end
36
+
37
+ desc "list", "show every directory scanned for models"
38
+ def list
39
+ builtin = LlmConfig::BUILTIN_MODEL_DIRS
40
+ LlmConfig.load.search_paths.each do |directory|
41
+ mark = directory.directory? ? Ui.green("✓") : Ui.red("✗")
42
+ tag = builtin.include?(directory) ? Ui.gray(" (builtin)") : ""
43
+ puts " #{mark} #{directory}#{tag}"
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,215 @@
1
+ require_relative "../ui"
2
+ require_relative "base"
3
+ require_relative "embed"
4
+ require_relative "paths"
5
+ require_relative "engines"
6
+ require_relative "wizard/parallel"
7
+ require_relative "../engine_cli"
8
+ require_relative "../llm_config"
9
+ require_relative "models"
10
+ require_relative "wizard/moe"
11
+ require_relative "../model/library"
12
+ require_relative "../util/probes"
13
+ require_relative "../result"
14
+ require_relative "../engine_cli/status"
15
+ require_relative "wizard/setup"
16
+ require_relative "../client"
17
+ require_relative "../engine/llama_embed"
18
+
19
+ # LLM CLI -- owns llama.cpp / ik_llama.cpp management.
20
+ #
21
+ # Carved out of `mimic setup` / `start` / `restart`. Takes:
22
+ # start/stop/restart/status/engines + wizard steps 1-5 (system check, engine,
23
+ # binary, model, context preset).
24
+ module Mimic
25
+ module LlmCli
26
+ if Mimic::Paths.dev_build?
27
+ require_relative "dev/debug"
28
+ end
29
+
30
+ class Root < Base
31
+ desc "setup", "interactive LLM setup wizard"
32
+ def setup
33
+ wizard_run
34
+ end
35
+
36
+ desc "start", "start llama-server"
37
+ def start
38
+ report(EngineCli::Registry.for(require_config).start)
39
+ end
40
+
41
+ desc "stop", "stop llama-server"
42
+ def stop
43
+ stopped = Engine::ServerProcess.chat.stop(exclude: Engine::ServerProcess.embed.pids)
44
+ puts "llama-server: #{stopped ? 'stopped' : 'was not running'}"
45
+ end
46
+
47
+ desc "restart", "stop then start llama-server"
48
+ def restart
49
+ stop
50
+ return unless wait_for_release(LlmConfig.load, "restart")
51
+
52
+ start
53
+ end
54
+
55
+ desc "unload", "unload the chat model to free its VRAM; llama-server loads it again on the next request"
56
+ method_option :force, type: :boolean, desc: "unload even while the model is answering a request"
57
+ def unload
58
+ cfg = LlmConfig.load
59
+ return report(Result.error("ik_llama.cpp has no router -- use `llm stop`")) if cfg.llama.ik?
60
+
61
+ router = Engine::Router.new(cfg.llama.base_url)
62
+ children = router.running
63
+ return report(Result.error("llama-server is not answering on #{cfg.llama.base_url}")) if children.nil?
64
+ return puts("no model loaded") if children.empty?
65
+
66
+ report(unload_children(router, children))
67
+ end
68
+
69
+ desc "status", "llama-server status"
70
+ def status
71
+ EngineCli::Status.new(LlmConfig.load).print
72
+ end
73
+
74
+ desc "embed SUBCOMMAND", "run an embedding model beside llama-server"
75
+ subcommand "embed", Embed
76
+
77
+ desc "path SUBCOMMAND", "manage the directories scanned for models"
78
+ subcommand "path", Paths
79
+
80
+ desc "moe", "interactive wizard to view/set/clear a MoE model's cached --n-cpu-moe value"
81
+ def moe
82
+ Wizard::Moe.new(Model::Library.new(LlmConfig.load).all, prompt: @prompt).run
83
+ end
84
+
85
+ desc "models SUBCOMMAND", "list available GGUF models"
86
+ subcommand "models", Models
87
+
88
+ # `debug` is a development tool, not product: it restarts the engines
89
+ # back to back to shake out launch bugs. The gemspec leaves
90
+ # lib/mimic/llm_cli/dev out of the package, so this directory is absent
91
+ # in an installed gem and the command registers only in a checkout.
92
+ if Mimic::Paths.dev_build?
93
+ desc "debug SUBCOMMAND", "run local engine integration checks", hide: true
94
+ subcommand "debug", Dev::Debug
95
+ end
96
+
97
+ desc "engines SUBCOMMAND", "list and update the inference engines"
98
+ subcommand "engines", Engines
99
+
100
+ # `diffusion` (DiffusionGemma, unmerged llama.cpp PR #24423) is
101
+ # deliberately not exposed as a Thor command -- it's an experiment, not
102
+ # a working product (truncates mid-answer; --diffusion-blocks doesn't
103
+ # extend past block 1, root cause not yet found). EngineCli::Diffusion
104
+ # stays wired for whoever picks this back up.
105
+
106
+ desc "config", "edit LLM config values"
107
+ def config
108
+ require_relative "configure"
109
+ LlmCli::Configure.run
110
+ end
111
+
112
+ desc "sync", "push the active LLM config to #{Client::CLIENTS.join(', ')}"
113
+ def sync(cfg = nil)
114
+ cfg ||= LlmConfig.load
115
+ synced = Client.sync(cfg)
116
+ synced.each { |name| puts "#{Ui.green('synced')} #{name}" }
117
+ puts Ui.yellow("no SWE agent configs found") if synced.empty?
118
+ synced
119
+ end
120
+
121
+ desc "parallel", "restart with n_slots/ctx overrides so several tasks can run at once (not persisted)"
122
+ method_option :slots, type: :numeric, desc: "n_slots override"
123
+ method_option :ctx, type: :numeric, desc: "context_length override"
124
+ def parallel
125
+ cfg = parallel_config(require_config)
126
+
127
+ stop
128
+ return unless wait_for_release(cfg, "parallel start")
129
+
130
+ report(EngineCli::Registry.for(cfg).start(remember: false))
131
+ end
132
+
133
+ private
134
+
135
+ def require_config
136
+ unless LlmConfig.exists?
137
+ puts Ui.yellow("No LLM config yet -- running setup wizard first.")
138
+ return wizard_run
139
+ end
140
+ LlmConfig.load
141
+ end
142
+
143
+ def wizard
144
+ @wizard ||= Wizard::Setup.new(LlmConfig.load, prompt: @prompt)
145
+ end
146
+
147
+ def parallel_config(cfg)
148
+ return Wizard::Parallel.new(cfg, wizard, prompt: @prompt).run unless options[:slots] || options[:ctx]
149
+
150
+ cfg.llama.n_slots = options[:slots].to_i if options[:slots]
151
+ cfg.model.context_length = options[:ctx].to_i if options[:ctx]
152
+ cfg
153
+ end
154
+
155
+ def wizard_run
156
+ cfg = wizard.config
157
+ Ui.banner("llm -- setup wizard")
158
+
159
+ wizard.system_check
160
+ wizard.ensure_binary(wizard.choose_engine)
161
+ wizard.choose_model
162
+ wizard.choose_mmproj
163
+ wizard.choose_context
164
+ wizard.choose_reasoning
165
+ wizard.choose_effort
166
+
167
+ cfg.save
168
+ cfg.forget_sampling
169
+ EngineCli::Registry.for(cfg).render_preset
170
+ print_setup_summary(sync(cfg))
171
+
172
+ if @prompt.yes?("Start llama-server now?", default: true)
173
+ restart
174
+ else
175
+ puts "Next: `llm restart` to launch llama-server."
176
+ end
177
+ cfg
178
+ end
179
+
180
+ def print_setup_summary(synced)
181
+ Ui.panel(
182
+ "LLM config saved to #{LlmConfig::LLM_CONFIG_PATH}\n" \
183
+ "Preset written to #{Engine::Llama::PRESET_FILE}\n" \
184
+ "Agents synced: #{synced.empty? ? 'none found' : synced.join(', ')}",
185
+ title: "Setup complete"
186
+ )
187
+ end
188
+
189
+ def unload_children(router, children)
190
+ ids = children.map(&:id)
191
+ busy = children.find { |child| router.busy?(child) } unless options[:force]
192
+ return Result.error("#{busy.id} is answering a request -- rerun with --force to cut it off") if busy
193
+
194
+ refused = ids.reject { |id| router.unload(id) }
195
+ return Result.error("router refused to unload #{refused.join(', ')}") if refused.any?
196
+ return Result.error("#{ids.join(', ')} still running after 30 s") unless router.wait_stopped(ids)
197
+
198
+ Result.ok("unloaded #{ids.join(', ')} -- GPU free: #{Util::Host.gpu_free_mib || '?'} MiB")
199
+ end
200
+
201
+ def wait_for_release(cfg, action)
202
+ released = Util::Probes.wait_until(attempts: 30, delay: 1, label: "waiting for port to be released") { released?(cfg) }
203
+ return true if released
204
+
205
+ report(Result.error("llama-server still holds #{cfg.llama.host}:#{cfg.llama.port} -- #{action} aborted"))
206
+ false
207
+ end
208
+
209
+ def released?(cfg)
210
+ Util::Probes.find_pid_by_cmd("llama-server", exclude: Engine::ServerProcess.embed.pids).nil? &&
211
+ Util::Probes.port_free?(cfg.llama.host, cfg.llama.port.to_i)
212
+ end
213
+ end
214
+ end
215
+ end