mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require_relative "../model/header"
|
|
3
|
+
require_relative "../llm_config"
|
|
4
|
+
require_relative "../model"
|
|
5
|
+
require_relative "../paths"
|
|
6
|
+
require_relative "../util/probes"
|
|
7
|
+
require_relative "../result"
|
|
8
|
+
require_relative "launcher"
|
|
9
|
+
require_relative "base"
|
|
10
|
+
require_relative "../engine_cli/registry"
|
|
11
|
+
|
|
12
|
+
module Mimic
|
|
13
|
+
module Engine
|
|
14
|
+
# The embedding server: a second llama-server, on a second port, holding a
|
|
15
|
+
# model that answers /v1/embeddings or /rerank and nothing else.
|
|
16
|
+
#
|
|
17
|
+
# It runs beside the chat server rather than through it -- the router
|
|
18
|
+
# preset loads one generative model per instance, and an embedding model
|
|
19
|
+
# needs process-wide flags (--embeddings sets pooling and restricts the
|
|
20
|
+
# server to that use case) that would apply to the chat model too.
|
|
21
|
+
#
|
|
22
|
+
# Unlike its siblings this one is selected by role, not by config: it must
|
|
23
|
+
# stay out of EngineCli::Registry, which dispatches on llama.engine and
|
|
24
|
+
# drives the setup wizard's engine picker.
|
|
25
|
+
class LlamaEmbed < Base
|
|
26
|
+
RERANK_NAME = /rerank/i
|
|
27
|
+
|
|
28
|
+
class << self
|
|
29
|
+
# Which endpoint a model wants, read from the GGUF. A reranker is an
|
|
30
|
+
# embedding model whose pooling is RANK, but the bge rerankers ship
|
|
31
|
+
# no pooling_type key at all -- for those the name is what is left.
|
|
32
|
+
def mode_for(path)
|
|
33
|
+
return "rerank" if Model::Header.new(path).pooling_type == Model::Header::POOLING_RANK
|
|
34
|
+
return "rerank" if RERANK_NAME.match?(File.basename(path.to_s))
|
|
35
|
+
|
|
36
|
+
"embeddings"
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def initialize(settings)
|
|
41
|
+
super
|
|
42
|
+
@embed = settings.embed
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def process = @process ||= Engine::ServerProcess.embed
|
|
46
|
+
|
|
47
|
+
def health_url = "#{@embed.base_url}/health"
|
|
48
|
+
def models_url = "#{@embed.base_url}/v1/models"
|
|
49
|
+
def running? = Util::Probes.http_ok(health_url)
|
|
50
|
+
def mode = @embed.mode == "auto" ? self.class.mode_for(@embed.path) : @embed.mode
|
|
51
|
+
|
|
52
|
+
# llama.cpp's own build, not whatever engine the chat server runs:
|
|
53
|
+
# ik_llama.cpp is the engine for CPU-MoE offload on a big generative
|
|
54
|
+
# model and has no reason to be serving a 600 MB encoder.
|
|
55
|
+
def find_binary
|
|
56
|
+
@find_binary ||= first_existing(EngineCli::Registry.builds("llama.cpp"),
|
|
57
|
+
Util::Probes.which("llama-server"), @settings.llama.binary)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def start
|
|
61
|
+
return Result.error("no embedding model configured -- run `llm embed setup`") unless @embed.configured?
|
|
62
|
+
|
|
63
|
+
if @embed.port.to_i == @settings.llama.port.to_i
|
|
64
|
+
return Result.error("embedding server cannot share the same port as llama-server (#{@embed.port})")
|
|
65
|
+
end
|
|
66
|
+
return Result.ok("embedding server already running") if running?
|
|
67
|
+
|
|
68
|
+
binary = find_binary
|
|
69
|
+
return Result.error("llama-server binary not found -- run `llm setup`") if binary.nil?
|
|
70
|
+
return Result.error("embedding model not found: #{@embed.path}") unless File.file?(@embed.path)
|
|
71
|
+
|
|
72
|
+
launch(binary)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def command(binary)
|
|
78
|
+
[
|
|
79
|
+
binary,
|
|
80
|
+
"-m", @embed.path,
|
|
81
|
+
"-a", @embed.model_alias,
|
|
82
|
+
"--host", @embed.host.to_s,
|
|
83
|
+
"--port", @embed.port.to_s,
|
|
84
|
+
"--ctx-size", @embed.context_length.to_s,
|
|
85
|
+
# A pooled embedding model is non-causal: a document has to fit in
|
|
86
|
+
# one ubatch or llama-server rejects it, so both batch sizes move
|
|
87
|
+
# together rather than leaving -ub at its 512 default.
|
|
88
|
+
"-b", @embed.batch_size.to_s,
|
|
89
|
+
"-ub", @embed.batch_size.to_s,
|
|
90
|
+
"-ngl", "999",
|
|
91
|
+
mode == "rerank" ? "--rerank" : "--embeddings",
|
|
92
|
+
]
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def launch(binary)
|
|
96
|
+
log_offset = process.log_size
|
|
97
|
+
pid = launcher.spawn(command(binary))
|
|
98
|
+
if Util::Probes.wait_ready(health_url, alive: -> { Util::Probes.child_alive?(pid) }, label: "waiting for embedding server")
|
|
99
|
+
return Result.ok("embedding server ready on #{@embed.base_url} (#{mode}, pid #{pid})")
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
process.cleanup_failed(pid)
|
|
103
|
+
Result.error("embedding server failed to start -- see #{process.log_file}\n" \
|
|
104
|
+
"#{process.last_lines(process.log_tail(log_offset))}")
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
require_relative "../util/probes"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
module Engine
|
|
5
|
+
# llama.cpp's router mode, seen from outside: which model children it is
|
|
6
|
+
# running, whether one is mid-request, and asking it to drop one. Unloading
|
|
7
|
+
# frees the child's VRAM while the router keeps the port, and the router
|
|
8
|
+
# loads the model again on the next request that names it.
|
|
9
|
+
class Router
|
|
10
|
+
RUNNING = %w[loaded loading sleeping].freeze
|
|
11
|
+
|
|
12
|
+
Child = Data.define(:id, :port)
|
|
13
|
+
|
|
14
|
+
def initialize(base_url)
|
|
15
|
+
@base_url = base_url
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# nil when the router does not answer, so "nothing loaded" and "no
|
|
19
|
+
# server" stay two different answers.
|
|
20
|
+
def running
|
|
21
|
+
data = Util::Probes.http_json("#{@base_url}/models", timeout: 3)
|
|
22
|
+
return nil if data.nil?
|
|
23
|
+
|
|
24
|
+
(data["data"] || []).filter_map do |entry|
|
|
25
|
+
next unless RUNNING.include?(entry.dig("status", "value"))
|
|
26
|
+
|
|
27
|
+
Child.new(entry["id"], child_port(entry.dig("status", "args") || []))
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def busy?(child)
|
|
32
|
+
slots = Util::Probes.http_json("http://127.0.0.1:#{child.port}/slots", timeout: 3)
|
|
33
|
+
Array(slots).any? { |slot| slot["is_processing"] }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def unload(id)
|
|
37
|
+
Util::Probes.post("#{@base_url}/models/unload", { "model" => id }).is_a?(Net::HTTPSuccess)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The router answers the unload at once and stops the child afterwards
|
|
41
|
+
# (a forced kill comes 10 s later), so the VRAM is only free once the
|
|
42
|
+
# child has left its list.
|
|
43
|
+
def wait_stopped(ids, attempts: 30)
|
|
44
|
+
Util::Probes.wait_until(attempts: attempts, delay: 1, label: "waiting for the model to unload") do
|
|
45
|
+
(running || []).none? { |child| ids.include?(child.id) }
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def child_port(args)
|
|
52
|
+
flag = args.index("--port")
|
|
53
|
+
flag && Integer(args[flag + 1], exception: false)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require "timeout"
|
|
3
|
+
require_relative "../paths"
|
|
4
|
+
require_relative "../util/probes"
|
|
5
|
+
require_relative "../util/host"
|
|
6
|
+
|
|
7
|
+
module Mimic
|
|
8
|
+
module Engine
|
|
9
|
+
# One llama-server process: where it records its pid, where it writes its
|
|
10
|
+
# log, and how it is found and killed. Nothing here knows about models,
|
|
11
|
+
# flags or engines -- the chat server and the embedding server are two
|
|
12
|
+
# instances of this, differing only in their paths and in whether a
|
|
13
|
+
# /proc scan may stand in for a missing pid file.
|
|
14
|
+
class ServerProcess
|
|
15
|
+
# The metadata block sits ~9 KB above a startup failure line, so the
|
|
16
|
+
# tail has to reach back far enough to still see n_expert / n_layer.
|
|
17
|
+
LOG_TAIL_BYTES = 64_000
|
|
18
|
+
|
|
19
|
+
BINARY = "llama-server".freeze
|
|
20
|
+
|
|
21
|
+
class << self
|
|
22
|
+
def chat
|
|
23
|
+
new(pid_file: Paths::PID_DIR / "llama-server.pid",
|
|
24
|
+
log_file: Paths::LLM_RUN_DIR / "llama-server.log",
|
|
25
|
+
proc_scan_fallback: true)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# The embedding server is one process with no children, so a scan for
|
|
29
|
+
# "llama-server" could only ever find the chat server.
|
|
30
|
+
def embed
|
|
31
|
+
new(pid_file: Paths::PID_DIR / "llama-embed.pid",
|
|
32
|
+
log_file: Paths::LLM_RUN_DIR / "llama-embed.log",
|
|
33
|
+
proc_scan_fallback: false)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
attr_reader :pid_file, :log_file
|
|
38
|
+
|
|
39
|
+
def initialize(pid_file:, log_file:, proc_scan_fallback:)
|
|
40
|
+
@pid_file = pid_file
|
|
41
|
+
@log_file = log_file
|
|
42
|
+
@proc_scan_fallback = proc_scan_fallback
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def proc_scan_fallback? = @proc_scan_fallback
|
|
46
|
+
|
|
47
|
+
# Both offsets are byte counts (`log_size` is a file size), so the
|
|
48
|
+
# slicing has to stay on the binary string. Decoding first turns them
|
|
49
|
+
# into character indexes, and one multi-byte character anywhere in a
|
|
50
|
+
# long log makes `offset` overshoot the end and the tail come back
|
|
51
|
+
# empty -- which is how a startup failure printed its own error line to
|
|
52
|
+
# the log and nothing to the caller.
|
|
53
|
+
def log_tail(offset = 0)
|
|
54
|
+
bytes = log_file.binread
|
|
55
|
+
bytes = bytes.byteslice(offset..) || "" if offset.positive?
|
|
56
|
+
bytes = bytes.byteslice(-LOG_TAIL_BYTES..) || bytes
|
|
57
|
+
bytes.force_encoding(Encoding::UTF_8).scrub
|
|
58
|
+
rescue SystemCallError
|
|
59
|
+
""
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def log_size = log_file.exist? ? log_file.size : 0
|
|
63
|
+
|
|
64
|
+
def last_lines(text, n = 4)
|
|
65
|
+
text.strip.lines.last(n).join
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def pid
|
|
69
|
+
pid_file.read.strip
|
|
70
|
+
rescue SystemCallError
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# This server's live pids, for the other server to leave alone.
|
|
75
|
+
def pids
|
|
76
|
+
live = Util::Probes.read_pid(pid_file, BINARY)
|
|
77
|
+
live ? [live, *Util::Probes.descendants(live)] : []
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# VRAM this server is holding, for a preflight that budgets a launch
|
|
81
|
+
# meant to replace it: the driver's free figure counts it as unavailable,
|
|
82
|
+
# but stopping this server hands it back.
|
|
83
|
+
def gpu_mib
|
|
84
|
+
live = pids
|
|
85
|
+
return 0 if live.empty?
|
|
86
|
+
|
|
87
|
+
Util::Host.gpu_process_mib(live) || 0
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def stop(exclude: [])
|
|
91
|
+
Util::Probes.stop_pid(pid_file, BINARY, exclude: exclude, fallback: proc_scan_fallback?)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def cleanup_failed(spawned_pid)
|
|
95
|
+
pid_file.unlink if pid_file.exist?
|
|
96
|
+
return unless Util::Probes.child_alive?(spawned_pid)
|
|
97
|
+
|
|
98
|
+
Process.kill("TERM", -spawned_pid)
|
|
99
|
+
Timeout.timeout(10) { Process.waitpid(spawned_pid) }
|
|
100
|
+
rescue Timeout::Error
|
|
101
|
+
Process.kill("KILL", -spawned_pid)
|
|
102
|
+
Process.waitpid(spawned_pid)
|
|
103
|
+
rescue Errno::ECHILD
|
|
104
|
+
nil
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
data/lib/mimic/engine.rb
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
require "open3"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
module EngineCli
|
|
5
|
+
# The long options one llama-server build actually carries, read from its
|
|
6
|
+
# own --completion-bash.
|
|
7
|
+
#
|
|
8
|
+
# The router rejects the whole preset file over a single key its arg table
|
|
9
|
+
# does not know ("option 'reasoning-effort' not recognized in preset"), and
|
|
10
|
+
# a fork can be months behind mainline's flag surface. Audit the preset
|
|
11
|
+
# against the binary that will read it, not mainline's current flag list.
|
|
12
|
+
#
|
|
13
|
+
# --help is not the source: its flag column wraps and its descriptions
|
|
14
|
+
# name other flags, so both directions of that parse are wrong.
|
|
15
|
+
class BinaryOptions
|
|
16
|
+
# set_preset_only() options are router-mode preset keys with no CLI
|
|
17
|
+
# form, so no completion list ever mentions them.
|
|
18
|
+
PRESET_ONLY = %w[load-on-startup stop-timeout dedup-cache-models].freeze
|
|
19
|
+
|
|
20
|
+
def initialize(binary)
|
|
21
|
+
@binary = binary.to_s
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# A build that answers nothing is not evidence against a key -- an
|
|
25
|
+
# unreadable flag list disables the audit rather than emptying the
|
|
26
|
+
# preset.
|
|
27
|
+
def supports?(name) = names.empty? || PRESET_ONLY.include?(name) || names.include?(name)
|
|
28
|
+
|
|
29
|
+
def names = @names ||= parse(completion_script)
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def completion_script
|
|
34
|
+
out, = Open3.capture2e(@binary, "--completion-bash")
|
|
35
|
+
out
|
|
36
|
+
rescue SystemCallError
|
|
37
|
+
""
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def parse(script)
|
|
41
|
+
opts = script[/opts="([^"]*)"/, 1] || ""
|
|
42
|
+
opts.split.grep(/\A--/).map { |flag| flag.delete_prefix("--") }
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
require_relative "../ui"
|
|
2
|
+
require "pathname"
|
|
3
|
+
require_relative "../util/host"
|
|
4
|
+
require_relative "../llm_config"
|
|
5
|
+
require_relative "../util/probes"
|
|
6
|
+
require_relative "registry"
|
|
7
|
+
|
|
8
|
+
module Mimic
|
|
9
|
+
module EngineCli
|
|
10
|
+
class Builder
|
|
11
|
+
BUILD_TOOLS = %w[git cmake cc c++].freeze
|
|
12
|
+
|
|
13
|
+
def initialize(config, cli_name:, prompt:)
|
|
14
|
+
@config = config
|
|
15
|
+
@cli_name = cli_name
|
|
16
|
+
@prompt = prompt
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def find_or_build(engine)
|
|
20
|
+
binary = Registry.for(@config).find_binary
|
|
21
|
+
if binary
|
|
22
|
+
puts "#{Ui.green("#{engine} llama-server found:")} #{binary}"
|
|
23
|
+
return binary
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
abort_with_install_guide(engine) if declined?(engine)
|
|
27
|
+
build(engine)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def build(engine)
|
|
31
|
+
src = Registry.repo(engine)
|
|
32
|
+
require_build_tools(engine)
|
|
33
|
+
clone(engine, src) unless (src / ".git").directory?
|
|
34
|
+
|
|
35
|
+
puts "Building #{engine} in #{src}..."
|
|
36
|
+
configure(engine, src)
|
|
37
|
+
compile(src)
|
|
38
|
+
built_binary(engine, src)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def declined?(engine)
|
|
44
|
+
action = @prompt.select(
|
|
45
|
+
"#{engine} llama-server not found. What now?",
|
|
46
|
+
["Build from source now (git + cmake, CUDA if nvcc present)", "I'll install it myself (abort setup)"]
|
|
47
|
+
)
|
|
48
|
+
action.start_with?("I'll")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def abort_with_install_guide(engine)
|
|
52
|
+
puts "Install llama-server and re-run `#{@cli_name} setup` " \
|
|
53
|
+
"(build guide: #{Registry.url(engine).delete_suffix('.git')})"
|
|
54
|
+
raise SystemExit, 1
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def require_build_tools(engine)
|
|
58
|
+
BUILD_TOOLS.each do |tool|
|
|
59
|
+
next if Util::Probes.which(tool)
|
|
60
|
+
|
|
61
|
+
fail_with("'#{tool}' is required to build #{engine}")
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def clone(engine, src)
|
|
66
|
+
puts "Cloning #{engine}..."
|
|
67
|
+
system("git", "clone", "--depth", "1", Registry.url(engine), src.to_s, exception: true)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def configure(engine, src)
|
|
71
|
+
return if system("cmake", "-S", src.to_s, "-B", (src / "build").to_s,
|
|
72
|
+
"-DCMAKE_BUILD_TYPE=Release", *Util::Host.gpu_build_flags, *Registry.flags(engine))
|
|
73
|
+
|
|
74
|
+
fail_with("cmake configure failed.")
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def compile(src)
|
|
78
|
+
return if system("cmake", "--build", (src / "build").to_s, "--config", "Release",
|
|
79
|
+
"-j#{Util::Host.build_jobs}", "--target", "llama-server")
|
|
80
|
+
|
|
81
|
+
fail_with("Build failed.")
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def built_binary(engine, src)
|
|
85
|
+
binary = src / "build" / "bin" / "llama-server"
|
|
86
|
+
fail_with("llama-server binary not found after build.") unless binary.file?
|
|
87
|
+
|
|
88
|
+
puts "#{Ui.green("#{engine} built:")} #{binary}"
|
|
89
|
+
binary.to_s
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def fail_with(message)
|
|
93
|
+
puts Ui.red(message)
|
|
94
|
+
raise SystemExit, 1
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
require_relative "../ui"
|
|
2
|
+
require "pathname"
|
|
3
|
+
require_relative "../util/host"
|
|
4
|
+
require_relative "../paths"
|
|
5
|
+
require_relative "../util/probes"
|
|
6
|
+
|
|
7
|
+
module Mimic
|
|
8
|
+
module EngineCli
|
|
9
|
+
# DiffusionGemma support (block-diffusion, not autoregressive) lives on
|
|
10
|
+
# an unmerged llama.cpp PR (ggml-org/llama.cpp#24423) and ships its own
|
|
11
|
+
# one-shot CLI, not a router-mode HTTP server -- llama-server cannot load
|
|
12
|
+
# this architecture at all ("unknown model architecture: 'diffusion-gemma'").
|
|
13
|
+
# This module owns a separate checkout under the same ~/.llm root the
|
|
14
|
+
# other two engines live in, tracking the PR head as a local branch since
|
|
15
|
+
# it has no real branch on origin.
|
|
16
|
+
module Diffusion
|
|
17
|
+
REPO = Paths::LLM_ROOT / "llama.cpp-diffusion"
|
|
18
|
+
UPSTREAM_URL = "https://github.com/ggml-org/llama.cpp.git"
|
|
19
|
+
PR_REF = "pull/24423/head"
|
|
20
|
+
LOCAL_BRANCH = "diffusion-gemma"
|
|
21
|
+
|
|
22
|
+
KNOWN_BINARIES = [
|
|
23
|
+
REPO / "build/bin/llama-diffusion-cli",
|
|
24
|
+
].freeze
|
|
25
|
+
|
|
26
|
+
# A block is a fixed 256-token canvas; one is barely enough for the
|
|
27
|
+
# model's own "thought" channel, let alone an answer -- the CLI's
|
|
28
|
+
# default of 1 truncates almost everything mid-thought.
|
|
29
|
+
DEFAULT_BLOCKS = 4
|
|
30
|
+
|
|
31
|
+
BUILD_TOOLS = %w[git cmake cc c++].freeze
|
|
32
|
+
|
|
33
|
+
class << self
|
|
34
|
+
# Kept alongside the engine builder rather than folded into it: the
|
|
35
|
+
# PR checkout tracks a branch that has no upstream equivalent, and
|
|
36
|
+
# the target is a one-shot CLI, not llama-server.
|
|
37
|
+
def build
|
|
38
|
+
require_build_tools()
|
|
39
|
+
fetch_pr_branch
|
|
40
|
+
puts "Building llama-diffusion-cli in #{REPO}..."
|
|
41
|
+
configure()
|
|
42
|
+
compile()
|
|
43
|
+
built_binary()
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def find_binary(cfg)
|
|
47
|
+
configured = cfg.dig("diffusion", "binary")
|
|
48
|
+
return configured if configured && !configured.empty? && File.file?(configured)
|
|
49
|
+
|
|
50
|
+
found = KNOWN_BINARIES.find(&:file?)
|
|
51
|
+
return found.to_s if found
|
|
52
|
+
|
|
53
|
+
Util::Probes.which("llama-diffusion-cli")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def build_command(binary, model_path, prompt, ctx: 4096, blocks: DEFAULT_BLOCKS)
|
|
57
|
+
[
|
|
58
|
+
binary,
|
|
59
|
+
"-m", model_path,
|
|
60
|
+
"-p", prompt,
|
|
61
|
+
"-ngl", "999",
|
|
62
|
+
"-c", ctx.to_s,
|
|
63
|
+
"--diffusion-blocks", blocks.to_s,
|
|
64
|
+
]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def require_build_tools
|
|
70
|
+
BUILD_TOOLS.each do |tool|
|
|
71
|
+
next if Util::Probes.which(tool)
|
|
72
|
+
|
|
73
|
+
fail_with("'#{tool}' is required to build llama-diffusion-cli")
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def fetch_pr_branch
|
|
78
|
+
unless (REPO / ".git").directory?
|
|
79
|
+
puts "Cloning llama.cpp (DiffusionGemma PR)..."
|
|
80
|
+
system("git", "clone", "--depth", "1", UPSTREAM_URL, REPO.to_s, exception: true)
|
|
81
|
+
end
|
|
82
|
+
puts "Fetching #{PR_REF}..."
|
|
83
|
+
system("git", "-C", REPO.to_s, "fetch", "origin", "#{PR_REF}:#{LOCAL_BRANCH}",
|
|
84
|
+
"--depth", "200", "--force", exception: true)
|
|
85
|
+
system("git", "-C", REPO.to_s, "checkout", LOCAL_BRANCH, exception: true)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def configure
|
|
89
|
+
return if system("cmake", "-B", (REPO / "build").to_s, REPO.to_s,
|
|
90
|
+
*Util::Host.gpu_build_flags, chdir: REPO.to_s)
|
|
91
|
+
|
|
92
|
+
fail_with("cmake configure failed.")
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def compile
|
|
96
|
+
return if system("cmake", "--build", (REPO / "build").to_s, "-j", Util::Host.build_jobs.to_s,
|
|
97
|
+
"--target", "llama-diffusion-cli", chdir: REPO.to_s)
|
|
98
|
+
|
|
99
|
+
fail_with("Build failed.")
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def built_binary
|
|
103
|
+
binary = REPO / "build" / "bin" / "llama-diffusion-cli"
|
|
104
|
+
fail_with("llama-diffusion-cli binary not found after build.") unless binary.file?
|
|
105
|
+
|
|
106
|
+
puts "#{Ui.green('llama-diffusion-cli built:')} #{binary}"
|
|
107
|
+
binary.to_s
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def fail_with(message)
|
|
111
|
+
puts Ui.red(message)
|
|
112
|
+
raise SystemExit, 1
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
require "etc"
|
|
2
|
+
require "pathname"
|
|
3
|
+
require_relative "../paths"
|
|
4
|
+
require_relative "binary_options"
|
|
5
|
+
|
|
6
|
+
module Mimic
|
|
7
|
+
module EngineCli
|
|
8
|
+
class LlamaPreset
|
|
9
|
+
# Preset keys are llama.cpp's long flag names without the dashes, so the
|
|
10
|
+
# same table names an entry here and, re-dashed, the flag the router
|
|
11
|
+
# reports it back as in /v1/models -- see Llama::ARG_FLAGS.
|
|
12
|
+
KEYS = {
|
|
13
|
+
model: "model",
|
|
14
|
+
ctx: "ctx-size",
|
|
15
|
+
n_gpu_layers: "n-gpu-layers",
|
|
16
|
+
flash_attn: "flash-attn",
|
|
17
|
+
ubatch: "ubatch-size",
|
|
18
|
+
cache_type: "cache-type-k",
|
|
19
|
+
cache_type_v: "cache-type-v",
|
|
20
|
+
threads: "threads",
|
|
21
|
+
n_slots: "parallel",
|
|
22
|
+
chat_template: "chat-template-file",
|
|
23
|
+
mmproj: "mmproj",
|
|
24
|
+
spec_draft_model: "spec-draft-model",
|
|
25
|
+
spec_type: "spec-type",
|
|
26
|
+
spec_draft_n_max: "spec-draft-n-max",
|
|
27
|
+
reasoning: "reasoning",
|
|
28
|
+
reasoning_effort: "reasoning-effort",
|
|
29
|
+
load_on_startup: "load-on-startup",
|
|
30
|
+
}.freeze
|
|
31
|
+
|
|
32
|
+
# `options` is the flag surface of the binary that will read this file.
|
|
33
|
+
# Nil means write every key unaudited -- the caller could not resolve a
|
|
34
|
+
# binary, so there is nothing to audit against.
|
|
35
|
+
def initialize(settings, model_alias:, chat_template: nil, mtp: false, mtp_drafter: nil, mmproj: nil,
|
|
36
|
+
options: nil)
|
|
37
|
+
@settings = settings
|
|
38
|
+
@model_alias = model_alias
|
|
39
|
+
@chat_template = chat_template
|
|
40
|
+
@mtp = mtp
|
|
41
|
+
@mtp_drafter = mtp_drafter
|
|
42
|
+
@mmproj = mmproj
|
|
43
|
+
@options = options
|
|
44
|
+
@dropped = []
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
attr_reader :dropped
|
|
48
|
+
|
|
49
|
+
def write(path)
|
|
50
|
+
@dropped = []
|
|
51
|
+
body = lines.join("\n")
|
|
52
|
+
path.dirname.mkpath
|
|
53
|
+
path.write("#{body}\n")
|
|
54
|
+
path
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# The router refuses to start on an unrecognized key rather than
|
|
60
|
+
# ignoring it, so a key this build has never heard of is left out --
|
|
61
|
+
# losing one setting beats losing the server.
|
|
62
|
+
def keep?(key)
|
|
63
|
+
return true if @options.nil? || @options.supports?(key)
|
|
64
|
+
|
|
65
|
+
@dropped << key
|
|
66
|
+
false
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# `version` must sit under the global `[*]` section: keys above the
|
|
70
|
+
# first section head become a preset literally named "default"
|
|
71
|
+
# (preset.cpp maps the empty section name to
|
|
72
|
+
# COMMON_PRESET_DEFAULT_NAME), and since the parser then skips
|
|
73
|
+
# `version` that preset carries no model. The router advertises it
|
|
74
|
+
# in /v1/models and, when a client picks it, spawns a model-less
|
|
75
|
+
# child that never finishes loading.
|
|
76
|
+
def lines
|
|
77
|
+
["[*]", "version = 1", "", "[#{@model_alias}]"] + entry_lines + sampling_lines
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def entry_lines
|
|
81
|
+
entries.filter_map do |key, value|
|
|
82
|
+
name = KEYS.fetch(key)
|
|
83
|
+
"#{name} = #{value}" unless value.nil? || !keep?(name)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def sampling_lines
|
|
88
|
+
@settings.model.sampling.to_h.filter_map { |flag, value| "#{flag} = #{value}" if keep?(flag.to_s) }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def entries
|
|
92
|
+
{
|
|
93
|
+
model: @settings.model.path,
|
|
94
|
+
ctx: @settings.model.context_length,
|
|
95
|
+
# All-GPU or nothing. Without this llama.cpp auto-fit silently
|
|
96
|
+
# moves tensors to CPU to make an oversized ctx boot, and the
|
|
97
|
+
# resulting PCIe traffic per ubatch costs more than the run is
|
|
98
|
+
# worth. ik is the engine for deliberate CPU-MoE offload; here a
|
|
99
|
+
# model that doesn't fit should fail at load.
|
|
100
|
+
n_gpu_layers: 999,
|
|
101
|
+
flash_attn: @settings.llama.flash_attn,
|
|
102
|
+
ubatch: 1024,
|
|
103
|
+
cache_type: @settings.llama.cache_type,
|
|
104
|
+
cache_type_v: @settings.llama.cache_type,
|
|
105
|
+
threads: Etc.nprocessors || 4,
|
|
106
|
+
n_slots: @settings.llama.n_slots,
|
|
107
|
+
chat_template: @chat_template,
|
|
108
|
+
mmproj: @mmproj,
|
|
109
|
+
spec_draft_model: (@mtp_drafter if @mtp),
|
|
110
|
+
spec_type: ("draft-mtp" if @mtp),
|
|
111
|
+
spec_draft_n_max: (Engine::Base::SPEC_DRAFT_N_MAX if @mtp),
|
|
112
|
+
reasoning: @settings.model.reasoning,
|
|
113
|
+
reasoning_effort: @settings.model.reasoning_effort,
|
|
114
|
+
load_on_startup: "true",
|
|
115
|
+
}
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|