mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
require_relative "../ui"
|
|
2
|
+
require_relative "../llm_config"
|
|
3
|
+
require_relative "../engine_cli"
|
|
4
|
+
|
|
5
|
+
# LLM config editor -- flat key=value editing for model + llama keys.
|
|
6
|
+
#
|
|
7
|
+
# Edits ~/.llm/config.yaml only.
|
|
8
|
+
module Mimic
|
|
9
|
+
module LlmCli
|
|
10
|
+
module Configure
|
|
11
|
+
FIELDS = {
|
|
12
|
+
"model.path" => ["Model GGUF path", :str],
|
|
13
|
+
"model.context_length" => ["Context length", :int],
|
|
14
|
+
"model.mtp" => ["MTP speculative decoding (auto/on/off)", :str],
|
|
15
|
+
"model.temperature" => ["Temperature", :float],
|
|
16
|
+
"model.top_p" => ["top_p", :float],
|
|
17
|
+
"model.top_k" => ["top_k", :int],
|
|
18
|
+
"llama.engine" => ["Inference engine (#{EngineCli::Registry.names.join('/')})", :str],
|
|
19
|
+
"llama.host" => ["Bind address (0.0.0.0 lets containers reach it; 127.0.0.1 is host-only)", :str],
|
|
20
|
+
"llama.port" => ["llama.cpp port", :int],
|
|
21
|
+
"llama.n_slots" => ["Number of llama.cpp slots (1 for local use)", :int],
|
|
22
|
+
}.freeze
|
|
23
|
+
|
|
24
|
+
class << self
|
|
25
|
+
def get(cfg, dotted)
|
|
26
|
+
section, key = dotted.split(".")
|
|
27
|
+
cfg[section][key]
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def set(cfg, dotted, raw)
|
|
31
|
+
section, key = dotted.split(".")
|
|
32
|
+
cfg[section][key] = case FIELDS.fetch(dotted)[1]
|
|
33
|
+
when :int then Integer(raw, 10)
|
|
34
|
+
when :float then Float(raw)
|
|
35
|
+
else raw
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def run
|
|
40
|
+
unless LlmConfig.exists?
|
|
41
|
+
puts Ui.yellow("No LLM config yet -- run `llm setup` first.")
|
|
42
|
+
return
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
settings = LlmConfig.load
|
|
46
|
+
cfg = settings.to_h
|
|
47
|
+
prompt = Ui::Prompt.new
|
|
48
|
+
|
|
49
|
+
loop do
|
|
50
|
+
picked = prompt.select("Edit which value?", menu_choices(cfg))
|
|
51
|
+
return if picked == "__quit__"
|
|
52
|
+
return save(settings) if picked == "__save__"
|
|
53
|
+
|
|
54
|
+
edit(cfg, picked, prompt)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
def menu_choices(cfg)
|
|
60
|
+
choices = FIELDS.map do |dotted, (label, _)|
|
|
61
|
+
{ name: format("%-45s = %s", label, get(cfg, dotted)), value: dotted }
|
|
62
|
+
end
|
|
63
|
+
choices << { name: "Save & exit", value: "__save__" }
|
|
64
|
+
choices << { name: "Exit without saving", value: "__quit__" }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def save(settings)
|
|
68
|
+
settings.save
|
|
69
|
+
EngineCli::Registry.for(settings).render_preset
|
|
70
|
+
puts "#{Ui.green('Saved')} #{LlmConfig::LLM_CONFIG_PATH}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def edit(cfg, dotted, prompt)
|
|
74
|
+
raw = prompt.ask("#{FIELDS.fetch(dotted)[0]}:", default: get(cfg, dotted).to_s)
|
|
75
|
+
return if raw.nil?
|
|
76
|
+
|
|
77
|
+
set(cfg, dotted, raw)
|
|
78
|
+
rescue ArgumentError, TypeError
|
|
79
|
+
puts Ui.red("Invalid value, keeping previous.")
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
require_relative "../ui"
|
|
2
|
+
require_relative "../util/fmt"
|
|
3
|
+
require_relative "base"
|
|
4
|
+
require_relative "wizard/embed"
|
|
5
|
+
require_relative "../llm_config"
|
|
6
|
+
require_relative "../model/library"
|
|
7
|
+
require_relative "../engine/llama_embed"
|
|
8
|
+
|
|
9
|
+
module Mimic
|
|
10
|
+
module LlmCli
|
|
11
|
+
# `llm embed` -- the embedding server, run beside the chat server rather
|
|
12
|
+
# than instead of it.
|
|
13
|
+
class Embed < Base
|
|
14
|
+
desc "setup", "pick the embedding model, its port and its batch size"
|
|
15
|
+
def setup
|
|
16
|
+
cfg = LlmConfig.load
|
|
17
|
+
return unless Wizard::Embed.new(cfg, embedding_models(cfg), prompt: @prompt).run
|
|
18
|
+
return unless @prompt.yes?("Start the embedding server now?", default: true)
|
|
19
|
+
|
|
20
|
+
start
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
desc "start", "start the embedding server"
|
|
24
|
+
def start
|
|
25
|
+
report(Engine::LlamaEmbed.new(LlmConfig.load).start)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
desc "stop", "stop the embedding server"
|
|
29
|
+
def stop
|
|
30
|
+
puts "embedding server: #{Engine::ServerProcess.embed.stop ? 'stopped' : 'was not running'}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
desc "restart", "stop then start the embedding server"
|
|
34
|
+
def restart
|
|
35
|
+
stop
|
|
36
|
+
start
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
desc "status", "embedding server status"
|
|
40
|
+
def status
|
|
41
|
+
cfg = LlmConfig.load
|
|
42
|
+
embed = cfg.embed
|
|
43
|
+
return puts Ui.yellow("No embedding model configured -- run `llm embed setup`") unless embed.configured?
|
|
44
|
+
|
|
45
|
+
service = Engine::LlamaEmbed.new(cfg)
|
|
46
|
+
running = service.running?
|
|
47
|
+
puts running ? Ui.green("embedding server running") : Ui.yellow("embedding server not running")
|
|
48
|
+
puts "Model: #{File.basename(embed.path)}"
|
|
49
|
+
puts "Endpoint: #{embed.base_url}"
|
|
50
|
+
puts "Mode: #{service.mode}"
|
|
51
|
+
puts "Batch: #{embed.batch_size}"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
desc "models", "list the embedding models under the search paths"
|
|
55
|
+
def models
|
|
56
|
+
cfg = LlmConfig.load
|
|
57
|
+
found = embedding_models(cfg)
|
|
58
|
+
return puts "No embedding models found under #{cfg.search_paths.to_a.join(', ')}" if found.empty?
|
|
59
|
+
|
|
60
|
+
rows = found.map do |model|
|
|
61
|
+
[
|
|
62
|
+
model.path.to_s == cfg.embed.path ? Ui.green("●") : "",
|
|
63
|
+
model.model_alias,
|
|
64
|
+
Engine::LlamaEmbed.mode_for(model.path),
|
|
65
|
+
Util::Fmt.human_size(model.real.size),
|
|
66
|
+
model.path.dirname.to_s.sub(/\A#{Regexp.escape(Dir.home)}/, "~"),
|
|
67
|
+
]
|
|
68
|
+
end
|
|
69
|
+
Ui.table(header: ["", "Alias", "Mode", "Size", "Path"], rows: rows)
|
|
70
|
+
puts "\nSwitch with `llm embed setup` -- active model is marked ●"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def embedding_models(cfg)
|
|
76
|
+
Model::Library.new(cfg).embedding
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
require_relative "base"
|
|
2
|
+
require_relative "../engine_cli"
|
|
3
|
+
require_relative "../result"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
module LlmCli
|
|
7
|
+
# `llm engines` -- installed engine details and the update pass.
|
|
8
|
+
class Engines < Base
|
|
9
|
+
default_command :list
|
|
10
|
+
|
|
11
|
+
desc "list", "list all engines and their local source and binary details"
|
|
12
|
+
def list
|
|
13
|
+
active = LlmConfig.load.llama.engine if LlmConfig.exists?
|
|
14
|
+
EngineCli::Registry.names.each do |name|
|
|
15
|
+
registry = EngineCli::Registry
|
|
16
|
+
source = EngineCli::Source.local_status(name)
|
|
17
|
+
binary = registry.binaries(name).find { |path| path && File.file?(path) }
|
|
18
|
+
|
|
19
|
+
puts "#{name}#{' (active)' if name == active}"
|
|
20
|
+
puts " Source: #{registry.url(name)}"
|
|
21
|
+
puts " Branch: #{registry.branch(name)}"
|
|
22
|
+
puts " Checkout: #{registry.repo(name)}"
|
|
23
|
+
puts " Revision: #{source ? "#{source[:commit]} (#{source[:dirty] ? 'dirty' : 'clean'})" : 'not checked out'}"
|
|
24
|
+
puts " Binary: #{binary || 'missing'}"
|
|
25
|
+
puts
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
desc "update", "fetch and rebuild outdated llama.cpp, ik_llama.cpp, and Prism engines"
|
|
30
|
+
method_option :rollback, type: :string, enum: EngineCli::Registry.names,
|
|
31
|
+
desc: "rebuild an engine at its last-good commit"
|
|
32
|
+
def update
|
|
33
|
+
updater = EngineCli::Update.new
|
|
34
|
+
if options[:rollback]
|
|
35
|
+
failure = updater.rollback(options[:rollback])
|
|
36
|
+
report(failure) if failure
|
|
37
|
+
return
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
updater.run
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
require_relative "base"
|
|
2
|
+
require_relative "models_table"
|
|
3
|
+
require_relative "../model/library"
|
|
4
|
+
require_relative "../llm_config"
|
|
5
|
+
require_relative "../util/hf"
|
|
6
|
+
|
|
7
|
+
module Mimic
|
|
8
|
+
module LlmCli
|
|
9
|
+
# `llm models` -- the board of every GGUF under the search paths.
|
|
10
|
+
class Models < Base
|
|
11
|
+
default_command :list
|
|
12
|
+
|
|
13
|
+
desc "list", "list available GGUF models"
|
|
14
|
+
def list
|
|
15
|
+
board.render
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
desc "check", "compare each local GGUF against the repo it was published from"
|
|
19
|
+
def check
|
|
20
|
+
board(client: Util::Hf.new).render
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def board(client: nil)
|
|
26
|
+
cfg = LlmConfig.load
|
|
27
|
+
ModelsTable.new(cfg, Model::Library.new(cfg).chat_loadable_or_hidden, client: client)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
require_relative "../ui"
|
|
2
|
+
require "pathname"
|
|
3
|
+
require_relative "../util/fmt"
|
|
4
|
+
require_relative "../llm_config"
|
|
5
|
+
require_relative "../model"
|
|
6
|
+
require_relative "../paths"
|
|
7
|
+
require_relative "../model/footprint"
|
|
8
|
+
require_relative "../model/upstream"
|
|
9
|
+
require_relative "../util/hf"
|
|
10
|
+
|
|
11
|
+
module Mimic
|
|
12
|
+
# The `llm models` board: every GGUF under the configured search paths,
|
|
13
|
+
# with the per-model facts that decide whether it will actually run.
|
|
14
|
+
module LlmCli
|
|
15
|
+
class ModelsTable
|
|
16
|
+
HEADER = ["", "Alias", "Quant", "Size", "MoE", "MTP", "Vision", "n_cpu_moe", "Template", "Last Ctx", "Path"].freeze
|
|
17
|
+
ALIAS_WIDTH = 44
|
|
18
|
+
PATH_WIDTH = 20
|
|
19
|
+
|
|
20
|
+
def initialize(config, scan, client: nil)
|
|
21
|
+
@config = config
|
|
22
|
+
@scan = scan
|
|
23
|
+
@client = client
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def render
|
|
27
|
+
rows = @scan.map { |model| row(model) }
|
|
28
|
+
return no_models if rows.empty?
|
|
29
|
+
|
|
30
|
+
Ui.table(header: header, rows: rows)
|
|
31
|
+
print_upstream_legend if @client
|
|
32
|
+
puts "\nSwitch with `llm setup` -- active model is marked ●"
|
|
33
|
+
print_search_paths
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def header = @client ? HEADER + ["Upstream"] : HEADER
|
|
39
|
+
|
|
40
|
+
def row(model)
|
|
41
|
+
path = model.path
|
|
42
|
+
real = model.real
|
|
43
|
+
base_alias = model.model_alias
|
|
44
|
+
remembered = Model.new(real)
|
|
45
|
+
cells = [
|
|
46
|
+
real == active_real ? Ui.green("●") : "",
|
|
47
|
+
truncate(base_alias, ALIAS_WIDTH),
|
|
48
|
+
path.basename(".*").to_s.delete_prefix(base_alias).sub(/\A-/, ""),
|
|
49
|
+
Util::Fmt.human_size(real.size),
|
|
50
|
+
moe_mark(real),
|
|
51
|
+
model.supports_mtp? ? Ui.green("✓") : "",
|
|
52
|
+
model.vision? ? Ui.green("✓") : "",
|
|
53
|
+
remembered.memory.n_cpu_moe&.to_s || "",
|
|
54
|
+
template_for(base_alias).file? ? "✓" : "",
|
|
55
|
+
last_ctx(remembered),
|
|
56
|
+
parent_label(path),
|
|
57
|
+
]
|
|
58
|
+
@client ? cells + [upstream_mark(model)] : cells
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def print_upstream_legend
|
|
62
|
+
puts "\nUpstream compares file size against the repo the GGUF was published from."
|
|
63
|
+
puts " #{Ui.gray('?')} no source repo -- the header names none and the path is not LM Studio's, or huggingface.co was unreachable"
|
|
64
|
+
puts " #{Ui.yellow('gone')} the repo no longer publishes a file by that name"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def upstream_mark(model)
|
|
68
|
+
upstream = Model::Upstream.new(model, client: @client)
|
|
69
|
+
case upstream.verdict
|
|
70
|
+
when :current then Ui.green("current")
|
|
71
|
+
when :stale then Ui.red(["stale", upstream.updated_at].compact.join(" "))
|
|
72
|
+
when :gone then Ui.yellow("gone")
|
|
73
|
+
else Ui.gray("?")
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def active_real
|
|
78
|
+
@active_real ||= begin
|
|
79
|
+
active = Pathname.new(@config.model.path).expand_path
|
|
80
|
+
active.exist? ? active.realpath : active
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def moe_mark(real)
|
|
85
|
+
moe = Model::Header.new(real).moe?
|
|
86
|
+
return "" if moe.nil?
|
|
87
|
+
|
|
88
|
+
moe ? Ui.green("✓") : Ui.red("✗")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def template_for(base_alias) = Mimic::Paths::CHAT_TEMPLATES_DIR / "#{base_alias}.jinja"
|
|
92
|
+
|
|
93
|
+
def last_ctx(model)
|
|
94
|
+
recalled = model.memory.last_ctx&.first
|
|
95
|
+
recalled.nil? ? "" : Util::Fmt.commas(recalled)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def parent_label(path)
|
|
99
|
+
label = path.dirname.to_s.sub(/\A#{Regexp.escape(Dir.home)}/, "~")
|
|
100
|
+
label.length > PATH_WIDTH ? "…#{label[-(PATH_WIDTH - 1)..]}" : label
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def truncate(text, width) = text.length > width ? "#{text[0, width - 1]}…" : text
|
|
104
|
+
|
|
105
|
+
def no_models
|
|
106
|
+
puts "No .gguf files found in #{@config.search_paths.to_a.join(', ')}"
|
|
107
|
+
print_search_paths
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def print_search_paths
|
|
111
|
+
puts "\nSearched:"
|
|
112
|
+
@config.search_paths.each do |directory|
|
|
113
|
+
mark = directory.directory? ? Ui.green("✓") : Ui.red("✗")
|
|
114
|
+
puts " #{mark} #{directory}"
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
require_relative "../ui"
|
|
2
|
+
require "pathname"
|
|
3
|
+
require_relative "base"
|
|
4
|
+
require_relative "../llm_config"
|
|
5
|
+
|
|
6
|
+
module Mimic
|
|
7
|
+
module LlmCli
|
|
8
|
+
# `llm path` -- the directories scanned for GGUF files.
|
|
9
|
+
class Paths < Base
|
|
10
|
+
desc "add DIR", "add a directory to the model search paths"
|
|
11
|
+
def add(dir)
|
|
12
|
+
target = Pathname.new(dir).expand_path
|
|
13
|
+
cfg = LlmConfig.load
|
|
14
|
+
case cfg.search_paths.add(target)
|
|
15
|
+
when :builtin then puts "#{target} is already searched by default."
|
|
16
|
+
when :duplicate then puts "#{target} is already a search path."
|
|
17
|
+
else
|
|
18
|
+
abort "not a directory: #{target}" unless target.directory?
|
|
19
|
+
|
|
20
|
+
cfg.save
|
|
21
|
+
puts "#{Ui.green('Added')} #{target}"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
desc "rm DIR", "remove a directory from the model search paths"
|
|
26
|
+
def rm(dir)
|
|
27
|
+
target = Pathname.new(dir).expand_path
|
|
28
|
+
cfg = LlmConfig.load
|
|
29
|
+
status = cfg.search_paths.remove(target)
|
|
30
|
+
abort "#{target} is a builtin search path and cannot be removed" if status == :builtin
|
|
31
|
+
abort "#{target} is not a configured search path" if status == :missing
|
|
32
|
+
|
|
33
|
+
cfg.save
|
|
34
|
+
puts "#{Ui.green('Removed')} #{target}"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
desc "list", "show every directory scanned for models"
|
|
38
|
+
def list
|
|
39
|
+
builtin = LlmConfig::BUILTIN_MODEL_DIRS
|
|
40
|
+
LlmConfig.load.search_paths.each do |directory|
|
|
41
|
+
mark = directory.directory? ? Ui.green("✓") : Ui.red("✗")
|
|
42
|
+
tag = builtin.include?(directory) ? Ui.gray(" (builtin)") : ""
|
|
43
|
+
puts " #{mark} #{directory}#{tag}"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
require_relative "../ui"
|
|
2
|
+
require_relative "base"
|
|
3
|
+
require_relative "embed"
|
|
4
|
+
require_relative "paths"
|
|
5
|
+
require_relative "engines"
|
|
6
|
+
require_relative "wizard/parallel"
|
|
7
|
+
require_relative "../engine_cli"
|
|
8
|
+
require_relative "../llm_config"
|
|
9
|
+
require_relative "models"
|
|
10
|
+
require_relative "wizard/moe"
|
|
11
|
+
require_relative "../model/library"
|
|
12
|
+
require_relative "../util/probes"
|
|
13
|
+
require_relative "../result"
|
|
14
|
+
require_relative "../engine_cli/status"
|
|
15
|
+
require_relative "wizard/setup"
|
|
16
|
+
require_relative "../client"
|
|
17
|
+
require_relative "../engine/llama_embed"
|
|
18
|
+
|
|
19
|
+
# LLM CLI -- owns llama.cpp / ik_llama.cpp management.
|
|
20
|
+
#
|
|
21
|
+
# Carved out of `mimic setup` / `start` / `restart`. Takes:
|
|
22
|
+
# start/stop/restart/status/engines + wizard steps 1-5 (system check, engine,
|
|
23
|
+
# binary, model, context preset).
|
|
24
|
+
module Mimic
|
|
25
|
+
module LlmCli
|
|
26
|
+
if Mimic::Paths.dev_build?
|
|
27
|
+
require_relative "dev/debug"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
class Root < Base
|
|
31
|
+
desc "setup", "interactive LLM setup wizard"
|
|
32
|
+
def setup
|
|
33
|
+
wizard_run
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
desc "start", "start llama-server"
|
|
37
|
+
def start
|
|
38
|
+
report(EngineCli::Registry.for(require_config).start)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
desc "stop", "stop llama-server"
|
|
42
|
+
def stop
|
|
43
|
+
stopped = Engine::ServerProcess.chat.stop(exclude: Engine::ServerProcess.embed.pids)
|
|
44
|
+
puts "llama-server: #{stopped ? 'stopped' : 'was not running'}"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
desc "restart", "stop then start llama-server"
|
|
48
|
+
def restart
|
|
49
|
+
stop
|
|
50
|
+
return unless wait_for_release(LlmConfig.load, "restart")
|
|
51
|
+
|
|
52
|
+
start
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
desc "unload", "unload the chat model to free its VRAM; llama-server loads it again on the next request"
|
|
56
|
+
method_option :force, type: :boolean, desc: "unload even while the model is answering a request"
|
|
57
|
+
def unload
|
|
58
|
+
cfg = LlmConfig.load
|
|
59
|
+
return report(Result.error("ik_llama.cpp has no router -- use `llm stop`")) if cfg.llama.ik?
|
|
60
|
+
|
|
61
|
+
router = Engine::Router.new(cfg.llama.base_url)
|
|
62
|
+
children = router.running
|
|
63
|
+
return report(Result.error("llama-server is not answering on #{cfg.llama.base_url}")) if children.nil?
|
|
64
|
+
return puts("no model loaded") if children.empty?
|
|
65
|
+
|
|
66
|
+
report(unload_children(router, children))
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
desc "status", "llama-server status"
|
|
70
|
+
def status
|
|
71
|
+
EngineCli::Status.new(LlmConfig.load).print
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
desc "embed SUBCOMMAND", "run an embedding model beside llama-server"
|
|
75
|
+
subcommand "embed", Embed
|
|
76
|
+
|
|
77
|
+
desc "path SUBCOMMAND", "manage the directories scanned for models"
|
|
78
|
+
subcommand "path", Paths
|
|
79
|
+
|
|
80
|
+
desc "moe", "interactive wizard to view/set/clear a MoE model's cached --n-cpu-moe value"
|
|
81
|
+
def moe
|
|
82
|
+
Wizard::Moe.new(Model::Library.new(LlmConfig.load).all, prompt: @prompt).run
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
desc "models SUBCOMMAND", "list available GGUF models"
|
|
86
|
+
subcommand "models", Models
|
|
87
|
+
|
|
88
|
+
# `debug` is a development tool, not product: it restarts the engines
|
|
89
|
+
# back to back to shake out launch bugs. The gemspec leaves
|
|
90
|
+
# lib/mimic/llm_cli/dev out of the package, so this directory is absent
|
|
91
|
+
# in an installed gem and the command registers only in a checkout.
|
|
92
|
+
if Mimic::Paths.dev_build?
|
|
93
|
+
desc "debug SUBCOMMAND", "run local engine integration checks", hide: true
|
|
94
|
+
subcommand "debug", Dev::Debug
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
desc "engines SUBCOMMAND", "list and update the inference engines"
|
|
98
|
+
subcommand "engines", Engines
|
|
99
|
+
|
|
100
|
+
# `diffusion` (DiffusionGemma, unmerged llama.cpp PR #24423) is
|
|
101
|
+
# deliberately not exposed as a Thor command -- it's an experiment, not
|
|
102
|
+
# a working product (truncates mid-answer; --diffusion-blocks doesn't
|
|
103
|
+
# extend past block 1, root cause not yet found). EngineCli::Diffusion
|
|
104
|
+
# stays wired for whoever picks this back up.
|
|
105
|
+
|
|
106
|
+
desc "config", "edit LLM config values"
|
|
107
|
+
def config
|
|
108
|
+
require_relative "configure"
|
|
109
|
+
LlmCli::Configure.run
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
desc "sync", "push the active LLM config to #{Client::CLIENTS.join(', ')}"
|
|
113
|
+
def sync(cfg = nil)
|
|
114
|
+
cfg ||= LlmConfig.load
|
|
115
|
+
synced = Client.sync(cfg)
|
|
116
|
+
synced.each { |name| puts "#{Ui.green('synced')} #{name}" }
|
|
117
|
+
puts Ui.yellow("no SWE agent configs found") if synced.empty?
|
|
118
|
+
synced
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
desc "parallel", "restart with n_slots/ctx overrides so several tasks can run at once (not persisted)"
|
|
122
|
+
method_option :slots, type: :numeric, desc: "n_slots override"
|
|
123
|
+
method_option :ctx, type: :numeric, desc: "context_length override"
|
|
124
|
+
def parallel
|
|
125
|
+
cfg = parallel_config(require_config)
|
|
126
|
+
|
|
127
|
+
stop
|
|
128
|
+
return unless wait_for_release(cfg, "parallel start")
|
|
129
|
+
|
|
130
|
+
report(EngineCli::Registry.for(cfg).start(remember: false))
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
private
|
|
134
|
+
|
|
135
|
+
def require_config
|
|
136
|
+
unless LlmConfig.exists?
|
|
137
|
+
puts Ui.yellow("No LLM config yet -- running setup wizard first.")
|
|
138
|
+
return wizard_run
|
|
139
|
+
end
|
|
140
|
+
LlmConfig.load
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def wizard
|
|
144
|
+
@wizard ||= Wizard::Setup.new(LlmConfig.load, prompt: @prompt)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def parallel_config(cfg)
|
|
148
|
+
return Wizard::Parallel.new(cfg, wizard, prompt: @prompt).run unless options[:slots] || options[:ctx]
|
|
149
|
+
|
|
150
|
+
cfg.llama.n_slots = options[:slots].to_i if options[:slots]
|
|
151
|
+
cfg.model.context_length = options[:ctx].to_i if options[:ctx]
|
|
152
|
+
cfg
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def wizard_run
|
|
156
|
+
cfg = wizard.config
|
|
157
|
+
Ui.banner("llm -- setup wizard")
|
|
158
|
+
|
|
159
|
+
wizard.system_check
|
|
160
|
+
wizard.ensure_binary(wizard.choose_engine)
|
|
161
|
+
wizard.choose_model
|
|
162
|
+
wizard.choose_mmproj
|
|
163
|
+
wizard.choose_context
|
|
164
|
+
wizard.choose_reasoning
|
|
165
|
+
wizard.choose_effort
|
|
166
|
+
|
|
167
|
+
cfg.save
|
|
168
|
+
cfg.forget_sampling
|
|
169
|
+
EngineCli::Registry.for(cfg).render_preset
|
|
170
|
+
print_setup_summary(sync(cfg))
|
|
171
|
+
|
|
172
|
+
if @prompt.yes?("Start llama-server now?", default: true)
|
|
173
|
+
restart
|
|
174
|
+
else
|
|
175
|
+
puts "Next: `llm restart` to launch llama-server."
|
|
176
|
+
end
|
|
177
|
+
cfg
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def print_setup_summary(synced)
|
|
181
|
+
Ui.panel(
|
|
182
|
+
"LLM config saved to #{LlmConfig::LLM_CONFIG_PATH}\n" \
|
|
183
|
+
"Preset written to #{Engine::Llama::PRESET_FILE}\n" \
|
|
184
|
+
"Agents synced: #{synced.empty? ? 'none found' : synced.join(', ')}",
|
|
185
|
+
title: "Setup complete"
|
|
186
|
+
)
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def unload_children(router, children)
|
|
190
|
+
ids = children.map(&:id)
|
|
191
|
+
busy = children.find { |child| router.busy?(child) } unless options[:force]
|
|
192
|
+
return Result.error("#{busy.id} is answering a request -- rerun with --force to cut it off") if busy
|
|
193
|
+
|
|
194
|
+
refused = ids.reject { |id| router.unload(id) }
|
|
195
|
+
return Result.error("router refused to unload #{refused.join(', ')}") if refused.any?
|
|
196
|
+
return Result.error("#{ids.join(', ')} still running after 30 s") unless router.wait_stopped(ids)
|
|
197
|
+
|
|
198
|
+
Result.ok("unloaded #{ids.join(', ')} -- GPU free: #{Util::Host.gpu_free_mib || '?'} MiB")
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def wait_for_release(cfg, action)
|
|
202
|
+
released = Util::Probes.wait_until(attempts: 30, delay: 1, label: "waiting for port to be released") { released?(cfg) }
|
|
203
|
+
return true if released
|
|
204
|
+
|
|
205
|
+
report(Result.error("llama-server still holds #{cfg.llama.host}:#{cfg.llama.port} -- #{action} aborted"))
|
|
206
|
+
false
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def released?(cfg)
|
|
210
|
+
Util::Probes.find_pid_by_cmd("llama-server", exclude: Engine::ServerProcess.embed.pids).nil? &&
|
|
211
|
+
Util::Probes.port_free?(cfg.llama.host, cfg.llama.port.to_i)
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|