mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
data/lib/mimic/cli.rb
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
require_relative "ui"
|
|
2
|
+
require_relative "llm_config"
|
|
3
|
+
require_relative "engine"
|
|
4
|
+
require_relative "health"
|
|
5
|
+
require_relative "agent/hermes"
|
|
6
|
+
require_relative "paths"
|
|
7
|
+
|
|
8
|
+
module Mimic
|
|
9
|
+
# mimic/claude/codex/hermes-lcc all take a REMAINDER of pass-through
|
|
10
|
+
# args after the subcommand name, which Thor's own flag parser would
|
|
11
|
+
# otherwise try (and fail) to interpret -- dispatch manually instead.
|
|
12
|
+
class Cli
|
|
13
|
+
Command = Struct.new(:name, :help, :menu, :handler, :dev)
|
|
14
|
+
|
|
15
|
+
COMMANDS = [
|
|
16
|
+
Command.new("verify", "check installs, running services and recent errors",
|
|
17
|
+
"Verify stack (installs, services, errors)", :verify),
|
|
18
|
+
Command.new("install-hermes", "install hermes-agent into ~/.hermes/venv",
|
|
19
|
+
"Install hermes-agent into an isolated venv", :install_hermes),
|
|
20
|
+
Command.new("hermes-lcc", "Hermes -> claude-code skill -> Claude Code -> llama.cpp",
|
|
21
|
+
"Launch Hermes + Claude Code bridge (claude-code skill direct to llama.cpp)", :hermes_lcc),
|
|
22
|
+
Command.new("claude", "Claude Code -> llama.cpp via Unsloth's start recipe",
|
|
23
|
+
"Launch Claude Code -- direct to llama.cpp (Unsloth start recipe)", :claude),
|
|
24
|
+
Command.new("codex", "Codex -> llama.cpp via Unsloth's config recipe",
|
|
25
|
+
"Launch Codex -- direct to llama.cpp (Unsloth config recipe)", :codex),
|
|
26
|
+
# `dev` holds muse back from an installed gem. It is wired correctly,
|
|
27
|
+
# but llama-server's /v1/responses stream omits `model` from
|
|
28
|
+
# response.created and muse aborts on the first event -- upstream
|
|
29
|
+
# llama.cpp PR #21174 is the fix, unmerged. Nothing an installed gem
|
|
30
|
+
# can do about it, so it is not offered there.
|
|
31
|
+
Command.new("muse", "Muse Code -> llama.cpp via its Responses base URL",
|
|
32
|
+
"Launch Muse Code -- direct to llama.cpp (Responses API)", :muse, true),
|
|
33
|
+
].freeze
|
|
34
|
+
|
|
35
|
+
# The menu leads with the launchers -- what someone opening it without a
|
|
36
|
+
# command almost always wants -- so it does not follow the help order.
|
|
37
|
+
MENU_ORDER = %w[hermes-lcc claude codex muse verify install-hermes].freeze
|
|
38
|
+
|
|
39
|
+
HELP_ARGS = %w[-h --help help].freeze
|
|
40
|
+
|
|
41
|
+
def initialize(argv)
|
|
42
|
+
@argv = argv
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def run
|
|
46
|
+
require_linux!
|
|
47
|
+
return menu unless command
|
|
48
|
+
return print_help if HELP_ARGS.include?(command)
|
|
49
|
+
|
|
50
|
+
entry = commands.find { |c| c.name == command }
|
|
51
|
+
return unknown_command unless entry
|
|
52
|
+
|
|
53
|
+
send(entry.handler, extra)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def menu
|
|
57
|
+
picked = Ui::Prompt.new.select("mimic-llm:", menu_entries.map(&:menu))
|
|
58
|
+
send(menu_entries.find { |c| c.menu == picked }.handler, [])
|
|
59
|
+
rescue Interrupt
|
|
60
|
+
nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
def command = @argv.first
|
|
66
|
+
def extra = @argv[1..] || []
|
|
67
|
+
|
|
68
|
+
# Every command this build offers: a dev-only one is absent in an
|
|
69
|
+
# installed gem and present in a checkout.
|
|
70
|
+
def commands = COMMANDS.reject { |c| c.dev && !Paths.dev_build? }
|
|
71
|
+
def menu_entries = MENU_ORDER.filter_map { |name| commands.find { |c| c.name == name } }
|
|
72
|
+
|
|
73
|
+
def hermes_lcc(extra)
|
|
74
|
+
cfg = require_config
|
|
75
|
+
require_llama_running!(cfg, "`mimic hermes-lcc`")
|
|
76
|
+
Agent::Hermes.launch(cfg, extra)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def claude(extra)
|
|
80
|
+
cfg = require_config
|
|
81
|
+
require_llama_running!(cfg, "`mimic claude`")
|
|
82
|
+
require_relative "agent"
|
|
83
|
+
Agent::Claude.new(cfg).launch(extra)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def codex(extra)
|
|
87
|
+
cfg = require_config
|
|
88
|
+
require_llama_running!(cfg, "`mimic codex`")
|
|
89
|
+
require_relative "agent"
|
|
90
|
+
Agent::Codex.new(cfg).launch(extra)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def muse(extra)
|
|
94
|
+
cfg = require_config
|
|
95
|
+
require_llama_running!(cfg, "`mimic muse`")
|
|
96
|
+
require_relative "agent"
|
|
97
|
+
Agent::Muse.new(cfg).launch(extra)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Single stack check: installs, running services, recent errors.
|
|
101
|
+
def verify(_extra)
|
|
102
|
+
raise SystemExit, 1 unless Health.run(require_config)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def install_hermes(_extra)
|
|
106
|
+
binary = Agent::Hermes.find_binary
|
|
107
|
+
return puts "#{Ui.green('hermes found:')} #{binary}" if binary
|
|
108
|
+
|
|
109
|
+
unless Ui::Prompt.new.yes?("hermes-agent not found. Install into an isolated venv?", default: true)
|
|
110
|
+
return puts Ui.yellow("Skipping Hermes -- `mimic hermes-lcc` will not work.")
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
puts "#{Ui.green('hermes installed:')} #{Agent::Hermes.install}"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def unknown_command
|
|
117
|
+
warn "unknown command: #{command}"
|
|
118
|
+
print_help
|
|
119
|
+
raise SystemExit, 1
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def require_linux!
|
|
123
|
+
abort "mimic-llm supports Linux only" unless RUBY_PLATFORM.include?("linux")
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def require_config
|
|
127
|
+
abort "no LLM config yet -- run `llm setup` first" unless LlmConfig.exists?
|
|
128
|
+
|
|
129
|
+
LlmConfig.load
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Check that llama-server is running; fail with a clear message if not.
|
|
133
|
+
def require_llama_running!(cfg, context)
|
|
134
|
+
return if Engine::Llama.new(cfg).running?
|
|
135
|
+
|
|
136
|
+
puts "#{Ui.red("#{context} requires llama-server to be running.")}\n" \
|
|
137
|
+
" Run `#{Ui.cyan('llm start')}` first."
|
|
138
|
+
raise SystemExit, 1
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def print_help
|
|
142
|
+
puts <<~HELP
|
|
143
|
+
usage: mimic [command] [args...]
|
|
144
|
+
|
|
145
|
+
Local Hermes, Claude Code and Codex stack
|
|
146
|
+
|
|
147
|
+
commands:
|
|
148
|
+
#{command_lines}
|
|
149
|
+
|
|
150
|
+
(no command) interactive menu
|
|
151
|
+
HELP
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def command_lines
|
|
155
|
+
width = commands.map { |c| c.name.length }.max + 1
|
|
156
|
+
commands.map { |c| " #{c.name.ljust(width)}#{c.help}" }.join("\n")
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "pathname"
|
|
3
|
+
|
|
4
|
+
module Mimic
|
|
5
|
+
module Client
|
|
6
|
+
# One SWE agent that reads the LLM config. Each subclass owns its config
|
|
7
|
+
# path and the shape that file wants; `sync` answers whether it wrote.
|
|
8
|
+
class Base
|
|
9
|
+
PROVIDER = "llamacpp".freeze
|
|
10
|
+
MAX_TOKENS = 8192
|
|
11
|
+
|
|
12
|
+
def initialize(settings)
|
|
13
|
+
@settings = settings
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def name = self.class.name.split("::").last.downcase
|
|
17
|
+
def config_path = self.class::CONFIG_PATH
|
|
18
|
+
def sync = raise(NotImplementedError)
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def model_alias = @settings.model.name
|
|
23
|
+
def ctx = @settings.model.context_length
|
|
24
|
+
def base_url = @settings.llama.base_url
|
|
25
|
+
|
|
26
|
+
def load_json(path = config_path)
|
|
27
|
+
text = path.read.gsub(%r{^\s*//.*$}, "")
|
|
28
|
+
JSON.parse(text)
|
|
29
|
+
rescue SystemCallError, JSON::ParserError
|
|
30
|
+
nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def write_json(path, data)
|
|
34
|
+
path.write("#{JSON.pretty_generate(data)}\n")
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def commented?(path)
|
|
38
|
+
path.read.match?(%r{^\s*//})
|
|
39
|
+
rescue SystemCallError
|
|
40
|
+
false
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
require_relative "base"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
module Client
|
|
5
|
+
class Crush < Base
|
|
6
|
+
CONFIG_PATH = Pathname.new(Dir.home) / ".config/crush/crush.json"
|
|
7
|
+
|
|
8
|
+
def sync
|
|
9
|
+
data = load_json
|
|
10
|
+
return false if data.nil?
|
|
11
|
+
|
|
12
|
+
provider = data.dig("providers", PROVIDER)
|
|
13
|
+
return false if provider.nil?
|
|
14
|
+
|
|
15
|
+
models = (provider["models"] ||= [])
|
|
16
|
+
entry = models.find { |model| model["id"] == model_alias }
|
|
17
|
+
if entry.nil?
|
|
18
|
+
entry = { "id" => model_alias, "name" => model_alias }
|
|
19
|
+
models.unshift(entry)
|
|
20
|
+
end
|
|
21
|
+
entry["context_window"] = ctx
|
|
22
|
+
entry["default_max_tokens"] = MAX_TOKENS
|
|
23
|
+
write_json(config_path, data)
|
|
24
|
+
true
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
require "psych"
|
|
2
|
+
require "date"
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
module Client
|
|
7
|
+
# Unlike the others, goose has no hand-made llamacpp block to find:
|
|
8
|
+
# providers live in their own file under custom_providers/, so this writes
|
|
9
|
+
# one. The config dir existing is what stands in for "goose is installed".
|
|
10
|
+
class Goose < Base
|
|
11
|
+
CONFIG_PATH = Pathname.new(Dir.home) / ".config/goose/config.yaml"
|
|
12
|
+
PROVIDER_FILE = Pathname.new(Dir.home) / ".config/goose/custom_providers/llamacpp.json"
|
|
13
|
+
|
|
14
|
+
def sync
|
|
15
|
+
return false unless config_path.file?
|
|
16
|
+
|
|
17
|
+
write_provider_file
|
|
18
|
+
data = load_yaml
|
|
19
|
+
return false if data.nil?
|
|
20
|
+
|
|
21
|
+
data["GOOSE_PROVIDER"] = PROVIDER
|
|
22
|
+
data["GOOSE_MODEL"] = model_alias
|
|
23
|
+
data["active_provider"] = PROVIDER
|
|
24
|
+
(data["providers"] ||= {})[PROVIDER] = { "enabled" => true, "model" => model_alias, "configured" => true }
|
|
25
|
+
config_path.write(Psych.dump(data))
|
|
26
|
+
true
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def write_provider_file
|
|
32
|
+
self.class::PROVIDER_FILE.dirname.mkpath
|
|
33
|
+
write_json(self.class::PROVIDER_FILE, {
|
|
34
|
+
"name" => PROVIDER,
|
|
35
|
+
"engine" => "openai",
|
|
36
|
+
"display_name" => "llama.cpp (local)",
|
|
37
|
+
"description" => "Local llama-server, OpenAI-compatible",
|
|
38
|
+
"base_url" => "#{base_url}/v1/chat/completions",
|
|
39
|
+
"models" => [{ "name" => model_alias, "context_limit" => ctx }],
|
|
40
|
+
"supports_streaming" => true,
|
|
41
|
+
"requires_auth" => false,
|
|
42
|
+
})
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def load_yaml
|
|
46
|
+
Psych.safe_load(config_path.read, permitted_classes: [Symbol, Date, Time], aliases: true) || {}
|
|
47
|
+
rescue SystemCallError, Psych::Exception
|
|
48
|
+
nil
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
require_relative "base"
|
|
2
|
+
require_relative "../agent/hermes"
|
|
3
|
+
|
|
4
|
+
module Mimic
|
|
5
|
+
module Client
|
|
6
|
+
# Hermes is a client of llama-server like any other: its managed keys are
|
|
7
|
+
# all LLM-derived (alias, base URL, context length), so it belongs on this
|
|
8
|
+
# roster rather than in the agent-side setup. The config itself lives at
|
|
9
|
+
# hermes' own path, which Agent::Hermes owns.
|
|
10
|
+
class Hermes < Base
|
|
11
|
+
def config_path = Agent::Hermes::CONFIG_PATH
|
|
12
|
+
|
|
13
|
+
def sync
|
|
14
|
+
return false unless config_path.dirname.directory?
|
|
15
|
+
|
|
16
|
+
Agent::Hermes.sync_config(@settings)
|
|
17
|
+
true
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
require_relative "base"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
module Client
|
|
5
|
+
class Opencode < Base
|
|
6
|
+
CONFIG_PATH = Pathname.new(Dir.home) / ".config/opencode/opencode.jsonc"
|
|
7
|
+
|
|
8
|
+
def sync
|
|
9
|
+
data = load_json
|
|
10
|
+
return false if data.nil?
|
|
11
|
+
|
|
12
|
+
if commented?(config_path)
|
|
13
|
+
warn "[opencode] #{config_path} contains comments; skipping write so they are not lost"
|
|
14
|
+
return false
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
provider = data.dig("provider", PROVIDER)
|
|
18
|
+
return false if provider.nil?
|
|
19
|
+
|
|
20
|
+
models = (provider["models"] ||= {})
|
|
21
|
+
entry = (models[model_alias] ||= { "name" => model_alias })
|
|
22
|
+
entry["limit"] = { "context" => ctx, "output" => MAX_TOKENS }
|
|
23
|
+
data["model"] = "#{PROVIDER}/#{model_alias}"
|
|
24
|
+
write_json(config_path, data)
|
|
25
|
+
true
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
require_relative "base"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
module Client
|
|
5
|
+
class Pi < Base
|
|
6
|
+
CONFIG_PATH = Pathname.new(Dir.home) / ".pi/agent/models.json"
|
|
7
|
+
SETTINGS_PATH = Pathname.new(Dir.home) / ".pi/agent/settings.json"
|
|
8
|
+
|
|
9
|
+
def sync
|
|
10
|
+
data = load_json
|
|
11
|
+
return false if data.nil?
|
|
12
|
+
|
|
13
|
+
provider = data.dig("providers", PROVIDER)
|
|
14
|
+
return false if provider.nil?
|
|
15
|
+
|
|
16
|
+
models = (provider["models"] ||= [])
|
|
17
|
+
entry = models.find { |model| model["id"] == model_alias }
|
|
18
|
+
if entry.nil?
|
|
19
|
+
entry = new_entry
|
|
20
|
+
models.unshift(entry)
|
|
21
|
+
end
|
|
22
|
+
entry["contextWindow"] = ctx
|
|
23
|
+
entry["maxTokens"] = MAX_TOKENS
|
|
24
|
+
write_json(config_path, data)
|
|
25
|
+
sync_settings
|
|
26
|
+
true
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def new_entry
|
|
32
|
+
{
|
|
33
|
+
"id" => model_alias,
|
|
34
|
+
"name" => "#{model_alias} (local llama.cpp)",
|
|
35
|
+
"input" => ["text"],
|
|
36
|
+
"cost" => { "input" => 0, "output" => 0, "cacheRead" => 0, "cacheWrite" => 0 },
|
|
37
|
+
}
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def sync_settings
|
|
41
|
+
settings = load_json(self.class::SETTINGS_PATH)
|
|
42
|
+
return if settings.nil?
|
|
43
|
+
|
|
44
|
+
settings["defaultProvider"] = PROVIDER
|
|
45
|
+
settings["defaultModel"] = model_alias
|
|
46
|
+
write_json(self.class::SETTINGS_PATH, settings)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
data/lib/mimic/client.rb
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
require_relative "client/hermes"
|
|
2
|
+
require_relative "client/goose"
|
|
3
|
+
require_relative "client/crush"
|
|
4
|
+
require_relative "client/opencode"
|
|
5
|
+
require_relative "client/pi"
|
|
6
|
+
|
|
7
|
+
module Mimic
|
|
8
|
+
module Client
|
|
9
|
+
# Every agent that reads the LLM config, in sync order. Single source for
|
|
10
|
+
# both the sync loop and the `llm sync` help text, so the two cannot drift
|
|
11
|
+
# apart.
|
|
12
|
+
ROSTER = [Hermes, Goose, Crush, Opencode, Pi].freeze
|
|
13
|
+
CLIENTS = ROSTER.map { |client| client.name.split("::").last.downcase }.freeze
|
|
14
|
+
|
|
15
|
+
def self.sync(settings)
|
|
16
|
+
ROSTER.map { |client| client.new(settings) }.select(&:sync).map(&:name)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# Mimic::Engine and Mimic::EngineCli
|
|
2
|
+
|
|
3
|
+
Two namespace modules, no behaviour on either.
|
|
4
|
+
|
|
5
|
+
`Engine` is the running AI-model engine and nothing else: a server process
|
|
6
|
+
speaking the OpenAI API. `EngineCli` is everything you do _to_ an engine from
|
|
7
|
+
the CLI — pick one, find its source, clone it, build it, update it, write its
|
|
8
|
+
launch preset.
|
|
9
|
+
|
|
10
|
+
| Constant | Role |
|
|
11
|
+
| ------------------------ | -------------------------------------------------------------------------------------------------------------- |
|
|
12
|
+
| `Engine::Base` | abstract; the model-side prep every engine does before spawning |
|
|
13
|
+
| `Engine::ServerProcess` | one running llama-server: pid file, log, stop and cleanup — `.chat` and `.embed` name the two |
|
|
14
|
+
| `Engine::Llama` | mainline llama.cpp, router mode (`--models-preset`) |
|
|
15
|
+
| `Engine::IkLlama` | ik_llama.cpp, direct argv + the CPU-MoE retry loop |
|
|
16
|
+
| `Engine::LlamaEmbed` | the embedding/rerank server — a second llama.cpp on a second port, picked by role not by config |
|
|
17
|
+
| `EngineCli::Registry` | the engine table (url, branch, build flags, binaries) + `.for(settings)` |
|
|
18
|
+
| `EngineCli::LlamaPreset` | writes the router preset ini |
|
|
19
|
+
| `EngineCli::Source` | checkout status, rebuild command, last-good commit |
|
|
20
|
+
| `EngineCli::Builder` | clone + cmake + compile when the box has no binary |
|
|
21
|
+
| `EngineCli::Update` | how far behind upstream each checkout is |
|
|
22
|
+
| `EngineCli::Status` | the `llm status` board — config profile corrected by the running instance's own argv |
|
|
23
|
+
|
|
24
|
+
Two collaborators live under `engine/` without being engines: `Launcher`, which
|
|
25
|
+
spawns the process, and `ChatTemplate`, which prepares the model's Jinja
|
|
26
|
+
template before the spawn and probes the live server after it. Nothing outside
|
|
27
|
+
`Base` uses either, which is why they stay here rather than in a namespace of
|
|
28
|
+
their own. `ServerProcess` is a third, but unlike those two it is reached from
|
|
29
|
+
outside — `llm stop` and `llm status` ask about the process without building an
|
|
30
|
+
engine.
|
|
31
|
+
|
|
32
|
+
The dependency runs both ways and that is deliberate: `Engine::Base` asks
|
|
33
|
+
`EngineCli::Registry` where its binary is, and `Registry.for` hands back an
|
|
34
|
+
`Engine::Llama` or `Engine::IkLlama`. `engine_cli.rb` requires `engine.rb`
|
|
35
|
+
first; the two cross-references are resolved at call time, not load time, so
|
|
36
|
+
the cycle never bites.
|
|
37
|
+
|
|
38
|
+
## Adding an engine
|
|
39
|
+
|
|
40
|
+
Add a row to `EngineCli::Registry::ENGINES`. That is the whole change if it is
|
|
41
|
+
an upstream fork that takes the same flags — no new class.
|
|
42
|
+
|
|
43
|
+
A new row needs a `binaries:` list, ordered. `:path` is `llama-server` on PATH,
|
|
44
|
+
`:build` is the engine's own checkout, `:local` is `~/.local/bin`. An engine
|
|
45
|
+
that omits `:path` cannot borrow another build; Prism stays isolated because
|
|
46
|
+
its ternary formats need its own fork.
|
|
47
|
+
|
|
48
|
+
Write a class only when the launch protocol differs — different argv, different
|
|
49
|
+
readiness check, a retry loop. Then it is `Engine::Whatever < Base` and
|
|
50
|
+
`Registry.for` picks it.
|
|
51
|
+
|
|
52
|
+
`Engine::LlamaEmbed` is the one subclass `Registry` must not know about: it is
|
|
53
|
+
selected by role (`llm embed start`), not by `llama.engine`, and `Registry.names`
|
|
54
|
+
drives the setup wizard's engine picker — a row there would offer the encoder as
|
|
55
|
+
a chat engine.
|
|
56
|
+
|
|
57
|
+
## Per-server log and pid file
|
|
58
|
+
|
|
59
|
+
`Engine::ServerProcess` is one running llama-server: its pid file, its log, and
|
|
60
|
+
how it is found and killed. It knows nothing about models, flags or engines.
|
|
61
|
+
`ServerProcess.chat` and `ServerProcess.embed` name the two that exist;
|
|
62
|
+
`Base#process` returns the first, `LlamaEmbed#process` the second.
|
|
63
|
+
|
|
64
|
+
`embed` sets `proc_scan_fallback?` to `false` — the `/proc` scan that rescues a
|
|
65
|
+
chat server whose pid file went missing would match the embedding server too,
|
|
66
|
+
since they are the same binary. `llm stop` excludes `ServerProcess.embed.pids`
|
|
67
|
+
for the same reason.
|
|
68
|
+
|
|
69
|
+
It is `ServerProcess`, not `Process`: a class named `Process` inside `Engine`
|
|
70
|
+
shadows core `Process` for every file in the namespace, and `Launcher`'s
|
|
71
|
+
`Process.spawn` resolves to it.
|
|
72
|
+
|
|
73
|
+
## Why the Prism engine has no class
|
|
74
|
+
|
|
75
|
+
It used to be `class PrismLlama < Llama` overriding exactly `find_binary`. A
|
|
76
|
+
subclass that overrides one method and adds no state is configuration wearing a
|
|
77
|
+
type. Prism uses a registry row.
|
|
78
|
+
|
|
79
|
+
Rails does the same with MariaDB — no `MariaDBAdapter`, just a `mariadb?`
|
|
80
|
+
predicate on the shared abstract adapter. If a fork ever needs behaviour rather
|
|
81
|
+
than a different binary path, add a predicate on `Base`, not a subclass.
|
|
82
|
+
|
|
83
|
+
## Where things moved (2026-08-15)
|
|
84
|
+
|
|
85
|
+
`Services::{Llama,IkLlama,PrismLlama,LlamaPreset,Engine,Engines}`,
|
|
86
|
+
`Mimic::EngineBuilder`, `Mimic::EngineUpdate` all collapsed into here.
|
|
87
|
+
`PrismLlama` and `EngineBuilder::ENGINE_REPOS` are gone — the repo-URL table and
|
|
88
|
+
the repo/branch/flags table were the same three engines keyed the same way, held
|
|
89
|
+
in sync by a spec that asserted their keys matched.
|
|
90
|
+
|
|
91
|
+
`Engine::Embed` joined this namespace in 2026-08-16 when `Services` was
|
|
92
|
+
dissolved: it is a second llama-server, so it belongs beside the chat engines.
|
|
93
|
+
`Engine::Launcher` came across with it — the spawn mechanism every engine uses.
|
|
94
|
+
|
|
95
|
+
One behaviour change came with the merge: `Builder#configure` now passes the
|
|
96
|
+
registry's per-engine flags. The old builder did not, so a first-time
|
|
97
|
+
ik_llama.cpp build was configured without `-DLLAMA_CURL=OFF
|
|
98
|
+
-DGGML_SCHED_MAX_COPIES=1` while every rebuild had them.
|
|
99
|
+
|
|
100
|
+
## Where things moved (2026-08-16)
|
|
101
|
+
|
|
102
|
+
The build/manage half split back out into `Mimic::EngineCli` under
|
|
103
|
+
`lib/mimic/engine_cli/`, so `Engine` holds only the model-running classes.
|
|
104
|
+
`Registry` and `LlamaPreset` went with it even though runtime touches both —
|
|
105
|
+
picking an engine and writing its launch file are management jobs the running
|
|
106
|
+
process merely consumes.
|
|
107
|
+
|
|
108
|
+
Three files left `Engine` in the same pass, sorted by who actually calls them
|
|
109
|
+
rather than by where they were first written:
|
|
110
|
+
|
|
111
|
+
- `Engine::Status` → `EngineCli::Status`. Its only consumer is `llm status`, and
|
|
112
|
+
rendering a board is something you do _to_ an engine from the CLI.
|
|
113
|
+
- `Engine::Schema` → `Model::Schema`, since split into `Model::Gguf` (the binary
|
|
114
|
+
reader) and `Model::Header` (the predicates over one parsed header).
|
|
115
|
+
- `Engine::Vram` → `Model::Vram`, since split into `Model::Footprint` (weights +
|
|
116
|
+
KV + compute against the budget, built by `Model#footprint`) and the two
|
|
117
|
+
`nvidia-smi` readers, which were machine facts rather than model facts and
|
|
118
|
+
went to `Util::Host`.
|
|
119
|
+
|
|
120
|
+
`Engine::Embed` became `Engine::LlamaEmbed < Base` in the same pass. It arrived
|
|
121
|
+
here depending on `Base` the awkward way — reaching in for `LOG_TAIL_BYTES`,
|
|
122
|
+
`cleanup_failed` and `last_lines`, and carrying its own copies of `log_tail`,
|
|
123
|
+
`launcher` and `first_existing`. Turning `LOG_FILE`/`PID_FILE` into overridable
|
|
124
|
+
singleton methods removed the reason those copies existed. It still overrides
|
|
125
|
+
`find_binary` (llama.cpp's own build only), `running?`/`models_url` (its own
|
|
126
|
+
port, `/health` for readiness) and `start` (no `remember:` to honour, so the
|
|
127
|
+
abstract signature on `Base` loosened to `start(**)`).
|
|
128
|
+
|
|
129
|
+
## Where things moved (2026-08-18)
|
|
130
|
+
|
|
131
|
+
`Base`'s class side became `Engine::ServerProcess`. `log_file`, `pid_file`,
|
|
132
|
+
`proc_scan_fallback?`, `log_tail`, `last_lines`, `stop` and `cleanup_failed`
|
|
133
|
+
were singleton methods on an abstract class, which made them one-per-subclass
|
|
134
|
+
by construction: `LlamaEmbed` existed partly to override three of them, and
|
|
135
|
+
`llm stop` reached the chat server by naming `Engine::Base` directly. They are
|
|
136
|
+
now instance state on an object with two named constructors.
|
|
137
|
+
|
|
138
|
+
`Base.supports_mtp?` went the other way, to `Model::Header#supports_mtp?` — it
|
|
139
|
+
reads GGUF bytes, which is model knowledge, not process knowledge. `Model`
|
|
140
|
+
delegates so callers holding a `Model` need not reach through to its header.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require_relative "chat_template"
|
|
3
|
+
require_relative "../llm_config"
|
|
4
|
+
require_relative "../paths"
|
|
5
|
+
require_relative "../util/probes"
|
|
6
|
+
require_relative "../result"
|
|
7
|
+
require_relative "../model/footprint"
|
|
8
|
+
require_relative "launcher"
|
|
9
|
+
require_relative "server_process"
|
|
10
|
+
require_relative "../engine_cli/registry"
|
|
11
|
+
|
|
12
|
+
module Mimic
|
|
13
|
+
module Engine
|
|
14
|
+
# What every engine shares: one llama-server process, one log, one pid
|
|
15
|
+
# file, and the model-side preparation each of them has to do before
|
|
16
|
+
# spawning it. Subclasses supply the argv and the readiness protocol.
|
|
17
|
+
class Base
|
|
18
|
+
PORT_RELEASE_ATTEMPTS = 60
|
|
19
|
+
|
|
20
|
+
# Tokens the MTP head drafts per verification pass, shared by every
|
|
21
|
+
# engine so a cross-engine tok/s comparison measures the engine, not
|
|
22
|
+
# the draft depth. 4 beat 2 by 15-19% tok/s on Gemma 4 31B (2026-10-24).
|
|
23
|
+
SPEC_DRAFT_N_MAX = 4
|
|
24
|
+
|
|
25
|
+
def initialize(settings)
|
|
26
|
+
@settings = settings
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def process = @process ||= Engine::ServerProcess.chat
|
|
30
|
+
|
|
31
|
+
def models_url = "#{@settings.llama.base_url}/v1/models"
|
|
32
|
+
def running? = Util::Probes.http_ok(models_url)
|
|
33
|
+
|
|
34
|
+
def find_binary
|
|
35
|
+
@find_binary ||= first_existing(@settings.llama.binary, EngineCli::Registry.binaries(@settings.llama.engine))
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def live_args = raise(NotImplementedError)
|
|
39
|
+
def render_preset = raise(NotImplementedError)
|
|
40
|
+
def start(**) = raise(NotImplementedError)
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def launcher
|
|
45
|
+
Engine::Launcher.new(log_file: process.log_file, pid_file: process.pid_file, chdir: Paths::LLM_RUN_DIR)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# A departing server stops answering HTTP the moment it starts unloading,
|
|
49
|
+
# but keeps its listening socket for the whole teardown -- so `running?`
|
|
50
|
+
# going false is not permission to bind, and spawning on it dies with
|
|
51
|
+
# "couldn't bind to server socket".
|
|
52
|
+
def await_port_release
|
|
53
|
+
host = @settings.llama.host
|
|
54
|
+
port = @settings.llama.port.to_i
|
|
55
|
+
return true if Util::Probes.port_free?(host, port)
|
|
56
|
+
|
|
57
|
+
Util::Probes.wait_until(attempts: PORT_RELEASE_ATTEMPTS, delay: 1,
|
|
58
|
+
label: "waiting for port #{port} to be released") do
|
|
59
|
+
Util::Probes.port_free?(host, port)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def port_busy_error(engine)
|
|
64
|
+
Result.error("#{engine} not started -- #{@settings.llama.host}:#{@settings.llama.port} " \
|
|
65
|
+
"is still held by another process")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def first_existing(*candidates)
|
|
69
|
+
candidates.flatten.compact.map(&:to_s).reject(&:empty?).find { |c| File.file?(c) }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def model_alias
|
|
73
|
+
@model_alias ||= @settings.model.name
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def chat_template
|
|
77
|
+
@chat_template ||= Engine::ChatTemplate.new(@settings.model.path, model_alias: model_alias)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def chat_template_override = chat_template.override_path
|
|
81
|
+
|
|
82
|
+
def ensure_chat_template
|
|
83
|
+
return if @chat_template_ensured
|
|
84
|
+
|
|
85
|
+
@chat_template_ensured = true
|
|
86
|
+
return unless chat_template.guarded?
|
|
87
|
+
|
|
88
|
+
installed = chat_template.install
|
|
89
|
+
return puts "[chat-template] #{model_alias} ships a template that can throw -- using #{installed}" if installed
|
|
90
|
+
return unless chat_template.override_path.nil?
|
|
91
|
+
|
|
92
|
+
puts "[chat-template] #{model_alias}'s template carries a guard mimic has no rule for -- " \
|
|
93
|
+
"requests may fail; hand-write #{Paths::CHAT_TEMPLATES_DIR}/#{model_alias}.jinja"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def report_template_failures
|
|
97
|
+
failures = Engine::ChatTemplate::Probe.new(@settings.llama.base_url, model_alias: model_alias).failures
|
|
98
|
+
return if failures.empty?
|
|
99
|
+
|
|
100
|
+
puts "[chat-template] #{model_alias} cannot render #{failures.join(', ')} -- " \
|
|
101
|
+
"those requests will fail; see #{Paths::CHAT_TEMPLATES_DIR}"
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# What this launch actually cost beyond the weights + KV estimate,
|
|
105
|
+
# kept so the next preflight budgets against a measurement instead of
|
|
106
|
+
# Model::Footprint.compute_default. Router mode loads the model in a child, so
|
|
107
|
+
# the descendants carry the VRAM the spawned pid does not.
|
|
108
|
+
def record_compute(pid)
|
|
109
|
+
pids = [pid, *Util::Probes.descendants(pid)]
|
|
110
|
+
estimate = @settings.model.file.footprint(@settings.model.context_length, @settings.llama.cache_type,
|
|
111
|
+
compute_mib: 0, slots: @settings.llama.n_slots,
|
|
112
|
+
mmproj_path: @settings.model.mmproj_path)
|
|
113
|
+
overhead = estimate.measured_overhead_mib(pids: pids)
|
|
114
|
+
return if overhead.nil?
|
|
115
|
+
|
|
116
|
+
@settings.model.file.memory.remember_compute(@settings.model.context_length, @settings.llama.cache_type,
|
|
117
|
+
overhead)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def mtp_enabled?
|
|
121
|
+
return true if %w[on true 1].include?(mtp_mode)
|
|
122
|
+
return false if mtp_off?
|
|
123
|
+
|
|
124
|
+
@settings.model.file.supports_mtp?
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def mtp_off? = %w[off false 0].include?(mtp_mode)
|
|
128
|
+
def mtp_mode = @settings.model.mtp.to_s.downcase
|
|
129
|
+
|
|
130
|
+
def model_missing
|
|
131
|
+
model_path = @settings.model.path
|
|
132
|
+
return nil unless model_path.to_s.empty? || !File.file?(model_path)
|
|
133
|
+
|
|
134
|
+
"GGUF model not found: #{model_path.to_s.empty? ? '<unset>' : model_path} -- run `mimic setup`"
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|