mimic-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/NOTICE +16 -0
  4. data/README.md +175 -0
  5. data/exe/lllm +5 -0
  6. data/exe/llm +5 -0
  7. data/exe/mimic +5 -0
  8. data/lib/mimic/README.md +144 -0
  9. data/lib/mimic/agent/claude.rb +305 -0
  10. data/lib/mimic/agent/codex.rb +254 -0
  11. data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
  12. data/lib/mimic/agent/hermes.rb +107 -0
  13. data/lib/mimic/agent/muse.rb +106 -0
  14. data/lib/mimic/agent.rb +46 -0
  15. data/lib/mimic/cli.rb +159 -0
  16. data/lib/mimic/client/base.rb +44 -0
  17. data/lib/mimic/client/crush.rb +28 -0
  18. data/lib/mimic/client/goose.rb +52 -0
  19. data/lib/mimic/client/hermes.rb +21 -0
  20. data/lib/mimic/client/opencode.rb +29 -0
  21. data/lib/mimic/client/pi.rb +50 -0
  22. data/lib/mimic/client.rb +19 -0
  23. data/lib/mimic/engine/README.md +140 -0
  24. data/lib/mimic/engine/base.rb +138 -0
  25. data/lib/mimic/engine/chat_template.rb +207 -0
  26. data/lib/mimic/engine/ik_llama.rb +273 -0
  27. data/lib/mimic/engine/launcher.rb +25 -0
  28. data/lib/mimic/engine/llama.rb +124 -0
  29. data/lib/mimic/engine/llama_embed.rb +108 -0
  30. data/lib/mimic/engine/router.rb +57 -0
  31. data/lib/mimic/engine/server_process.rb +108 -0
  32. data/lib/mimic/engine.rb +5 -0
  33. data/lib/mimic/engine_cli/binary_options.rb +46 -0
  34. data/lib/mimic/engine_cli/builder.rb +98 -0
  35. data/lib/mimic/engine_cli/diffusion.rb +117 -0
  36. data/lib/mimic/engine_cli/llama_preset.rb +119 -0
  37. data/lib/mimic/engine_cli/registry.rb +94 -0
  38. data/lib/mimic/engine_cli/source.rb +110 -0
  39. data/lib/mimic/engine_cli/status.rb +161 -0
  40. data/lib/mimic/engine_cli/update.rb +68 -0
  41. data/lib/mimic/engine_cli.rb +7 -0
  42. data/lib/mimic/health/checks.rb +92 -0
  43. data/lib/mimic/health.rb +82 -0
  44. data/lib/mimic/llm_cli/base.rb +36 -0
  45. data/lib/mimic/llm_cli/configure.rb +84 -0
  46. data/lib/mimic/llm_cli/embed.rb +80 -0
  47. data/lib/mimic/llm_cli/engines.rb +44 -0
  48. data/lib/mimic/llm_cli/models.rb +31 -0
  49. data/lib/mimic/llm_cli/models_table.rb +119 -0
  50. data/lib/mimic/llm_cli/paths.rb +48 -0
  51. data/lib/mimic/llm_cli/root.rb +215 -0
  52. data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
  53. data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
  54. data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
  55. data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
  56. data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
  57. data/lib/mimic/llm_cli.rb +5 -0
  58. data/lib/mimic/llm_config/embed.rb +48 -0
  59. data/lib/mimic/llm_config/llama_settings.rb +60 -0
  60. data/lib/mimic/llm_config/model_settings.rb +84 -0
  61. data/lib/mimic/llm_config/sampling.rb +54 -0
  62. data/lib/mimic/llm_config/settings.rb +92 -0
  63. data/lib/mimic/llm_config.rb +132 -0
  64. data/lib/mimic/model/footprint.rb +303 -0
  65. data/lib/mimic/model/gguf.rb +204 -0
  66. data/lib/mimic/model/header.rb +169 -0
  67. data/lib/mimic/model/library.rb +132 -0
  68. data/lib/mimic/model/memory.rb +126 -0
  69. data/lib/mimic/model/store.rb +46 -0
  70. data/lib/mimic/model/upstream.rb +81 -0
  71. data/lib/mimic/model.rb +116 -0
  72. data/lib/mimic/paths.rb +45 -0
  73. data/lib/mimic/result.rb +8 -0
  74. data/lib/mimic/ui.rb +89 -0
  75. data/lib/mimic/util/fmt.rb +15 -0
  76. data/lib/mimic/util/hf.rb +55 -0
  77. data/lib/mimic/util/host.rb +95 -0
  78. data/lib/mimic/util/probes.rb +255 -0
  79. data/lib/mimic/version.rb +3 -0
  80. data/lib/mimic.rb +22 -0
  81. metadata +186 -0
@@ -0,0 +1,108 @@
1
+ require "pathname"
2
+ require_relative "../model/header"
3
+ require_relative "../llm_config"
4
+ require_relative "../model"
5
+ require_relative "../paths"
6
+ require_relative "../util/probes"
7
+ require_relative "../result"
8
+ require_relative "launcher"
9
+ require_relative "base"
10
+ require_relative "../engine_cli/registry"
11
+
12
+ module Mimic
13
+ module Engine
14
+ # The embedding server: a second llama-server, on a second port, holding a
15
+ # model that answers /v1/embeddings or /rerank and nothing else.
16
+ #
17
+ # It runs beside the chat server rather than through it -- the router
18
+ # preset loads one generative model per instance, and an embedding model
19
+ # needs process-wide flags (--embeddings sets pooling and restricts the
20
+ # server to that use case) that would apply to the chat model too.
21
+ #
22
+ # Unlike its siblings this one is selected by role, not by config: it must
23
+ # stay out of EngineCli::Registry, which dispatches on llama.engine and
24
+ # drives the setup wizard's engine picker.
25
+ class LlamaEmbed < Base
26
+ RERANK_NAME = /rerank/i
27
+
28
+ class << self
29
+ # Which endpoint a model wants, read from the GGUF. A reranker is an
30
+ # embedding model whose pooling is RANK, but the bge rerankers ship
31
+ # no pooling_type key at all -- for those the name is what is left.
32
+ def mode_for(path)
33
+ return "rerank" if Model::Header.new(path).pooling_type == Model::Header::POOLING_RANK
34
+ return "rerank" if RERANK_NAME.match?(File.basename(path.to_s))
35
+
36
+ "embeddings"
37
+ end
38
+ end
39
+
40
+ def initialize(settings)
41
+ super
42
+ @embed = settings.embed
43
+ end
44
+
45
+ def process = @process ||= Engine::ServerProcess.embed
46
+
47
+ def health_url = "#{@embed.base_url}/health"
48
+ def models_url = "#{@embed.base_url}/v1/models"
49
+ def running? = Util::Probes.http_ok(health_url)
50
+ def mode = @embed.mode == "auto" ? self.class.mode_for(@embed.path) : @embed.mode
51
+
52
+ # llama.cpp's own build, not whatever engine the chat server runs:
53
+ # ik_llama.cpp is the engine for CPU-MoE offload on a big generative
54
+ # model and has no reason to be serving a 600 MB encoder.
55
+ def find_binary
56
+ @find_binary ||= first_existing(EngineCli::Registry.builds("llama.cpp"),
57
+ Util::Probes.which("llama-server"), @settings.llama.binary)
58
+ end
59
+
60
+ def start
61
+ return Result.error("no embedding model configured -- run `llm embed setup`") unless @embed.configured?
62
+
63
+ if @embed.port.to_i == @settings.llama.port.to_i
64
+ return Result.error("embedding server cannot share the same port as llama-server (#{@embed.port})")
65
+ end
66
+ return Result.ok("embedding server already running") if running?
67
+
68
+ binary = find_binary
69
+ return Result.error("llama-server binary not found -- run `llm setup`") if binary.nil?
70
+ return Result.error("embedding model not found: #{@embed.path}") unless File.file?(@embed.path)
71
+
72
+ launch(binary)
73
+ end
74
+
75
+ private
76
+
77
+ def command(binary)
78
+ [
79
+ binary,
80
+ "-m", @embed.path,
81
+ "-a", @embed.model_alias,
82
+ "--host", @embed.host.to_s,
83
+ "--port", @embed.port.to_s,
84
+ "--ctx-size", @embed.context_length.to_s,
85
+ # A pooled embedding model is non-causal: a document has to fit in
86
+ # one ubatch or llama-server rejects it, so both batch sizes move
87
+ # together rather than leaving -ub at its 512 default.
88
+ "-b", @embed.batch_size.to_s,
89
+ "-ub", @embed.batch_size.to_s,
90
+ "-ngl", "999",
91
+ mode == "rerank" ? "--rerank" : "--embeddings",
92
+ ]
93
+ end
94
+
95
+ def launch(binary)
96
+ log_offset = process.log_size
97
+ pid = launcher.spawn(command(binary))
98
+ if Util::Probes.wait_ready(health_url, alive: -> { Util::Probes.child_alive?(pid) }, label: "waiting for embedding server")
99
+ return Result.ok("embedding server ready on #{@embed.base_url} (#{mode}, pid #{pid})")
100
+ end
101
+
102
+ process.cleanup_failed(pid)
103
+ Result.error("embedding server failed to start -- see #{process.log_file}\n" \
104
+ "#{process.last_lines(process.log_tail(log_offset))}")
105
+ end
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,57 @@
1
+ require_relative "../util/probes"
2
+
3
+ module Mimic
4
+ module Engine
5
+ # llama.cpp's router mode, seen from outside: which model children it is
6
+ # running, whether one is mid-request, and asking it to drop one. Unloading
7
+ # frees the child's VRAM while the router keeps the port, and the router
8
+ # loads the model again on the next request that names it.
9
+ class Router
10
+ RUNNING = %w[loaded loading sleeping].freeze
11
+
12
+ Child = Data.define(:id, :port)
13
+
14
+ def initialize(base_url)
15
+ @base_url = base_url
16
+ end
17
+
18
+ # nil when the router does not answer, so "nothing loaded" and "no
19
+ # server" stay two different answers.
20
+ def running
21
+ data = Util::Probes.http_json("#{@base_url}/models", timeout: 3)
22
+ return nil if data.nil?
23
+
24
+ (data["data"] || []).filter_map do |entry|
25
+ next unless RUNNING.include?(entry.dig("status", "value"))
26
+
27
+ Child.new(entry["id"], child_port(entry.dig("status", "args") || []))
28
+ end
29
+ end
30
+
31
+ def busy?(child)
32
+ slots = Util::Probes.http_json("http://127.0.0.1:#{child.port}/slots", timeout: 3)
33
+ Array(slots).any? { |slot| slot["is_processing"] }
34
+ end
35
+
36
+ def unload(id)
37
+ Util::Probes.post("#{@base_url}/models/unload", { "model" => id }).is_a?(Net::HTTPSuccess)
38
+ end
39
+
40
+ # The router answers the unload at once and stops the child afterwards
41
+ # (a forced kill comes 10 s later), so the VRAM is only free once the
42
+ # child has left its list.
43
+ def wait_stopped(ids, attempts: 30)
44
+ Util::Probes.wait_until(attempts: attempts, delay: 1, label: "waiting for the model to unload") do
45
+ (running || []).none? { |child| ids.include?(child.id) }
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def child_port(args)
52
+ flag = args.index("--port")
53
+ flag && Integer(args[flag + 1], exception: false)
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,108 @@
1
+ require "pathname"
2
+ require "timeout"
3
+ require_relative "../paths"
4
+ require_relative "../util/probes"
5
+ require_relative "../util/host"
6
+
7
+ module Mimic
8
+ module Engine
9
+ # One llama-server process: where it records its pid, where it writes its
10
+ # log, and how it is found and killed. Nothing here knows about models,
11
+ # flags or engines -- the chat server and the embedding server are two
12
+ # instances of this, differing only in their paths and in whether a
13
+ # /proc scan may stand in for a missing pid file.
14
+ class ServerProcess
15
+ # The metadata block sits ~9 KB above a startup failure line, so the
16
+ # tail has to reach back far enough to still see n_expert / n_layer.
17
+ LOG_TAIL_BYTES = 64_000
18
+
19
+ BINARY = "llama-server".freeze
20
+
21
+ class << self
22
+ def chat
23
+ new(pid_file: Paths::PID_DIR / "llama-server.pid",
24
+ log_file: Paths::LLM_RUN_DIR / "llama-server.log",
25
+ proc_scan_fallback: true)
26
+ end
27
+
28
+ # The embedding server is one process with no children, so a scan for
29
+ # "llama-server" could only ever find the chat server.
30
+ def embed
31
+ new(pid_file: Paths::PID_DIR / "llama-embed.pid",
32
+ log_file: Paths::LLM_RUN_DIR / "llama-embed.log",
33
+ proc_scan_fallback: false)
34
+ end
35
+ end
36
+
37
+ attr_reader :pid_file, :log_file
38
+
39
+ def initialize(pid_file:, log_file:, proc_scan_fallback:)
40
+ @pid_file = pid_file
41
+ @log_file = log_file
42
+ @proc_scan_fallback = proc_scan_fallback
43
+ end
44
+
45
+ def proc_scan_fallback? = @proc_scan_fallback
46
+
47
+ # Both offsets are byte counts (`log_size` is a file size), so the
48
+ # slicing has to stay on the binary string. Decoding first turns them
49
+ # into character indexes, and one multi-byte character anywhere in a
50
+ # long log makes `offset` overshoot the end and the tail come back
51
+ # empty -- which is how a startup failure printed its own error line to
52
+ # the log and nothing to the caller.
53
+ def log_tail(offset = 0)
54
+ bytes = log_file.binread
55
+ bytes = bytes.byteslice(offset..) || "" if offset.positive?
56
+ bytes = bytes.byteslice(-LOG_TAIL_BYTES..) || bytes
57
+ bytes.force_encoding(Encoding::UTF_8).scrub
58
+ rescue SystemCallError
59
+ ""
60
+ end
61
+
62
+ def log_size = log_file.exist? ? log_file.size : 0
63
+
64
+ def last_lines(text, n = 4)
65
+ text.strip.lines.last(n).join
66
+ end
67
+
68
+ def pid
69
+ pid_file.read.strip
70
+ rescue SystemCallError
71
+ nil
72
+ end
73
+
74
+ # This server's live pids, for the other server to leave alone.
75
+ def pids
76
+ live = Util::Probes.read_pid(pid_file, BINARY)
77
+ live ? [live, *Util::Probes.descendants(live)] : []
78
+ end
79
+
80
+ # VRAM this server is holding, for a preflight that budgets a launch
81
+ # meant to replace it: the driver's free figure counts it as unavailable,
82
+ # but stopping this server hands it back.
83
+ def gpu_mib
84
+ live = pids
85
+ return 0 if live.empty?
86
+
87
+ Util::Host.gpu_process_mib(live) || 0
88
+ end
89
+
90
+ def stop(exclude: [])
91
+ Util::Probes.stop_pid(pid_file, BINARY, exclude: exclude, fallback: proc_scan_fallback?)
92
+ end
93
+
94
+ def cleanup_failed(spawned_pid)
95
+ pid_file.unlink if pid_file.exist?
96
+ return unless Util::Probes.child_alive?(spawned_pid)
97
+
98
+ Process.kill("TERM", -spawned_pid)
99
+ Timeout.timeout(10) { Process.waitpid(spawned_pid) }
100
+ rescue Timeout::Error
101
+ Process.kill("KILL", -spawned_pid)
102
+ Process.waitpid(spawned_pid)
103
+ rescue Errno::ECHILD
104
+ nil
105
+ end
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,5 @@
1
+ require_relative "engine/server_process"
2
+ require_relative "engine/base"
3
+ require_relative "engine/llama"
4
+ require_relative "engine/ik_llama"
5
+ require_relative "engine/router"
@@ -0,0 +1,46 @@
1
+ require "open3"
2
+
3
+ module Mimic
4
+ module EngineCli
5
+ # The long options one llama-server build actually carries, read from its
6
+ # own --completion-bash.
7
+ #
8
+ # The router rejects the whole preset file over a single key its arg table
9
+ # does not know ("option 'reasoning-effort' not recognized in preset"), and
10
+ # a fork can be months behind mainline's flag surface. Audit the preset
11
+ # against the binary that will read it, not mainline's current flag list.
12
+ #
13
+ # --help is not the source: its flag column wraps and its descriptions
14
+ # name other flags, so both directions of that parse are wrong.
15
+ class BinaryOptions
16
+ # set_preset_only() options are router-mode preset keys with no CLI
17
+ # form, so no completion list ever mentions them.
18
+ PRESET_ONLY = %w[load-on-startup stop-timeout dedup-cache-models].freeze
19
+
20
+ def initialize(binary)
21
+ @binary = binary.to_s
22
+ end
23
+
24
+ # A build that answers nothing is not evidence against a key -- an
25
+ # unreadable flag list disables the audit rather than emptying the
26
+ # preset.
27
+ def supports?(name) = names.empty? || PRESET_ONLY.include?(name) || names.include?(name)
28
+
29
+ def names = @names ||= parse(completion_script)
30
+
31
+ private
32
+
33
+ def completion_script
34
+ out, = Open3.capture2e(@binary, "--completion-bash")
35
+ out
36
+ rescue SystemCallError
37
+ ""
38
+ end
39
+
40
+ def parse(script)
41
+ opts = script[/opts="([^"]*)"/, 1] || ""
42
+ opts.split.grep(/\A--/).map { |flag| flag.delete_prefix("--") }
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,98 @@
1
+ require_relative "../ui"
2
+ require "pathname"
3
+ require_relative "../util/host"
4
+ require_relative "../llm_config"
5
+ require_relative "../util/probes"
6
+ require_relative "registry"
7
+
8
+ module Mimic
9
+ module EngineCli
10
+ class Builder
11
+ BUILD_TOOLS = %w[git cmake cc c++].freeze
12
+
13
+ def initialize(config, cli_name:, prompt:)
14
+ @config = config
15
+ @cli_name = cli_name
16
+ @prompt = prompt
17
+ end
18
+
19
+ def find_or_build(engine)
20
+ binary = Registry.for(@config).find_binary
21
+ if binary
22
+ puts "#{Ui.green("#{engine} llama-server found:")} #{binary}"
23
+ return binary
24
+ end
25
+
26
+ abort_with_install_guide(engine) if declined?(engine)
27
+ build(engine)
28
+ end
29
+
30
+ def build(engine)
31
+ src = Registry.repo(engine)
32
+ require_build_tools(engine)
33
+ clone(engine, src) unless (src / ".git").directory?
34
+
35
+ puts "Building #{engine} in #{src}..."
36
+ configure(engine, src)
37
+ compile(src)
38
+ built_binary(engine, src)
39
+ end
40
+
41
+ private
42
+
43
+ def declined?(engine)
44
+ action = @prompt.select(
45
+ "#{engine} llama-server not found. What now?",
46
+ ["Build from source now (git + cmake, CUDA if nvcc present)", "I'll install it myself (abort setup)"]
47
+ )
48
+ action.start_with?("I'll")
49
+ end
50
+
51
+ def abort_with_install_guide(engine)
52
+ puts "Install llama-server and re-run `#{@cli_name} setup` " \
53
+ "(build guide: #{Registry.url(engine).delete_suffix('.git')})"
54
+ raise SystemExit, 1
55
+ end
56
+
57
+ def require_build_tools(engine)
58
+ BUILD_TOOLS.each do |tool|
59
+ next if Util::Probes.which(tool)
60
+
61
+ fail_with("'#{tool}' is required to build #{engine}")
62
+ end
63
+ end
64
+
65
+ def clone(engine, src)
66
+ puts "Cloning #{engine}..."
67
+ system("git", "clone", "--depth", "1", Registry.url(engine), src.to_s, exception: true)
68
+ end
69
+
70
+ def configure(engine, src)
71
+ return if system("cmake", "-S", src.to_s, "-B", (src / "build").to_s,
72
+ "-DCMAKE_BUILD_TYPE=Release", *Util::Host.gpu_build_flags, *Registry.flags(engine))
73
+
74
+ fail_with("cmake configure failed.")
75
+ end
76
+
77
+ def compile(src)
78
+ return if system("cmake", "--build", (src / "build").to_s, "--config", "Release",
79
+ "-j#{Util::Host.build_jobs}", "--target", "llama-server")
80
+
81
+ fail_with("Build failed.")
82
+ end
83
+
84
+ def built_binary(engine, src)
85
+ binary = src / "build" / "bin" / "llama-server"
86
+ fail_with("llama-server binary not found after build.") unless binary.file?
87
+
88
+ puts "#{Ui.green("#{engine} built:")} #{binary}"
89
+ binary.to_s
90
+ end
91
+
92
+ def fail_with(message)
93
+ puts Ui.red(message)
94
+ raise SystemExit, 1
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,117 @@
1
+ require_relative "../ui"
2
+ require "pathname"
3
+ require_relative "../util/host"
4
+ require_relative "../paths"
5
+ require_relative "../util/probes"
6
+
7
+ module Mimic
8
+ module EngineCli
9
+ # DiffusionGemma support (block-diffusion, not autoregressive) lives on
10
+ # an unmerged llama.cpp PR (ggml-org/llama.cpp#24423) and ships its own
11
+ # one-shot CLI, not a router-mode HTTP server -- llama-server cannot load
12
+ # this architecture at all ("unknown model architecture: 'diffusion-gemma'").
13
+ # This module owns a separate checkout under the same ~/.llm root the
14
+ # other two engines live in, tracking the PR head as a local branch since
15
+ # it has no real branch on origin.
16
+ module Diffusion
17
+ REPO = Paths::LLM_ROOT / "llama.cpp-diffusion"
18
+ UPSTREAM_URL = "https://github.com/ggml-org/llama.cpp.git"
19
+ PR_REF = "pull/24423/head"
20
+ LOCAL_BRANCH = "diffusion-gemma"
21
+
22
+ KNOWN_BINARIES = [
23
+ REPO / "build/bin/llama-diffusion-cli",
24
+ ].freeze
25
+
26
+ # A block is a fixed 256-token canvas; one is barely enough for the
27
+ # model's own "thought" channel, let alone an answer -- the CLI's
28
+ # default of 1 truncates almost everything mid-thought.
29
+ DEFAULT_BLOCKS = 4
30
+
31
+ BUILD_TOOLS = %w[git cmake cc c++].freeze
32
+
33
+ class << self
34
+ # Kept alongside the engine builder rather than folded into it: the
35
+ # PR checkout tracks a branch that has no upstream equivalent, and
36
+ # the target is a one-shot CLI, not llama-server.
37
+ def build
38
+ require_build_tools()
39
+ fetch_pr_branch
40
+ puts "Building llama-diffusion-cli in #{REPO}..."
41
+ configure()
42
+ compile()
43
+ built_binary()
44
+ end
45
+
46
+ def find_binary(cfg)
47
+ configured = cfg.dig("diffusion", "binary")
48
+ return configured if configured && !configured.empty? && File.file?(configured)
49
+
50
+ found = KNOWN_BINARIES.find(&:file?)
51
+ return found.to_s if found
52
+
53
+ Util::Probes.which("llama-diffusion-cli")
54
+ end
55
+
56
+ def build_command(binary, model_path, prompt, ctx: 4096, blocks: DEFAULT_BLOCKS)
57
+ [
58
+ binary,
59
+ "-m", model_path,
60
+ "-p", prompt,
61
+ "-ngl", "999",
62
+ "-c", ctx.to_s,
63
+ "--diffusion-blocks", blocks.to_s,
64
+ ]
65
+ end
66
+
67
+ private
68
+
69
+ def require_build_tools
70
+ BUILD_TOOLS.each do |tool|
71
+ next if Util::Probes.which(tool)
72
+
73
+ fail_with("'#{tool}' is required to build llama-diffusion-cli")
74
+ end
75
+ end
76
+
77
+ def fetch_pr_branch
78
+ unless (REPO / ".git").directory?
79
+ puts "Cloning llama.cpp (DiffusionGemma PR)..."
80
+ system("git", "clone", "--depth", "1", UPSTREAM_URL, REPO.to_s, exception: true)
81
+ end
82
+ puts "Fetching #{PR_REF}..."
83
+ system("git", "-C", REPO.to_s, "fetch", "origin", "#{PR_REF}:#{LOCAL_BRANCH}",
84
+ "--depth", "200", "--force", exception: true)
85
+ system("git", "-C", REPO.to_s, "checkout", LOCAL_BRANCH, exception: true)
86
+ end
87
+
88
+ def configure
89
+ return if system("cmake", "-B", (REPO / "build").to_s, REPO.to_s,
90
+ *Util::Host.gpu_build_flags, chdir: REPO.to_s)
91
+
92
+ fail_with("cmake configure failed.")
93
+ end
94
+
95
+ def compile
96
+ return if system("cmake", "--build", (REPO / "build").to_s, "-j", Util::Host.build_jobs.to_s,
97
+ "--target", "llama-diffusion-cli", chdir: REPO.to_s)
98
+
99
+ fail_with("Build failed.")
100
+ end
101
+
102
+ def built_binary
103
+ binary = REPO / "build" / "bin" / "llama-diffusion-cli"
104
+ fail_with("llama-diffusion-cli binary not found after build.") unless binary.file?
105
+
106
+ puts "#{Ui.green('llama-diffusion-cli built:')} #{binary}"
107
+ binary.to_s
108
+ end
109
+
110
+ def fail_with(message)
111
+ puts Ui.red(message)
112
+ raise SystemExit, 1
113
+ end
114
+ end
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,119 @@
1
+ require "etc"
2
+ require "pathname"
3
+ require_relative "../paths"
4
+ require_relative "binary_options"
5
+
6
+ module Mimic
7
+ module EngineCli
8
+ class LlamaPreset
9
+ # Preset keys are llama.cpp's long flag names without the dashes, so the
10
+ # same table names an entry here and, re-dashed, the flag the router
11
+ # reports it back as in /v1/models -- see Llama::ARG_FLAGS.
12
+ KEYS = {
13
+ model: "model",
14
+ ctx: "ctx-size",
15
+ n_gpu_layers: "n-gpu-layers",
16
+ flash_attn: "flash-attn",
17
+ ubatch: "ubatch-size",
18
+ cache_type: "cache-type-k",
19
+ cache_type_v: "cache-type-v",
20
+ threads: "threads",
21
+ n_slots: "parallel",
22
+ chat_template: "chat-template-file",
23
+ mmproj: "mmproj",
24
+ spec_draft_model: "spec-draft-model",
25
+ spec_type: "spec-type",
26
+ spec_draft_n_max: "spec-draft-n-max",
27
+ reasoning: "reasoning",
28
+ reasoning_effort: "reasoning-effort",
29
+ load_on_startup: "load-on-startup",
30
+ }.freeze
31
+
32
+ # `options` is the flag surface of the binary that will read this file.
33
+ # Nil means write every key unaudited -- the caller could not resolve a
34
+ # binary, so there is nothing to audit against.
35
+ def initialize(settings, model_alias:, chat_template: nil, mtp: false, mtp_drafter: nil, mmproj: nil,
36
+ options: nil)
37
+ @settings = settings
38
+ @model_alias = model_alias
39
+ @chat_template = chat_template
40
+ @mtp = mtp
41
+ @mtp_drafter = mtp_drafter
42
+ @mmproj = mmproj
43
+ @options = options
44
+ @dropped = []
45
+ end
46
+
47
+ attr_reader :dropped
48
+
49
+ def write(path)
50
+ @dropped = []
51
+ body = lines.join("\n")
52
+ path.dirname.mkpath
53
+ path.write("#{body}\n")
54
+ path
55
+ end
56
+
57
+ private
58
+
59
+ # The router refuses to start on an unrecognized key rather than
60
+ # ignoring it, so a key this build has never heard of is left out --
61
+ # losing one setting beats losing the server.
62
+ def keep?(key)
63
+ return true if @options.nil? || @options.supports?(key)
64
+
65
+ @dropped << key
66
+ false
67
+ end
68
+
69
+ # `version` must sit under the global `[*]` section: keys above the
70
+ # first section head become a preset literally named "default"
71
+ # (preset.cpp maps the empty section name to
72
+ # COMMON_PRESET_DEFAULT_NAME), and since the parser then skips
73
+ # `version` that preset carries no model. The router advertises it
74
+ # in /v1/models and, when a client picks it, spawns a model-less
75
+ # child that never finishes loading.
76
+ def lines
77
+ ["[*]", "version = 1", "", "[#{@model_alias}]"] + entry_lines + sampling_lines
78
+ end
79
+
80
+ def entry_lines
81
+ entries.filter_map do |key, value|
82
+ name = KEYS.fetch(key)
83
+ "#{name} = #{value}" unless value.nil? || !keep?(name)
84
+ end
85
+ end
86
+
87
+ def sampling_lines
88
+ @settings.model.sampling.to_h.filter_map { |flag, value| "#{flag} = #{value}" if keep?(flag.to_s) }
89
+ end
90
+
91
+ def entries
92
+ {
93
+ model: @settings.model.path,
94
+ ctx: @settings.model.context_length,
95
+ # All-GPU or nothing. Without this llama.cpp auto-fit silently
96
+ # moves tensors to CPU to make an oversized ctx boot, and the
97
+ # resulting PCIe traffic per ubatch costs more than the run is
98
+ # worth. ik is the engine for deliberate CPU-MoE offload; here a
99
+ # model that doesn't fit should fail at load.
100
+ n_gpu_layers: 999,
101
+ flash_attn: @settings.llama.flash_attn,
102
+ ubatch: 1024,
103
+ cache_type: @settings.llama.cache_type,
104
+ cache_type_v: @settings.llama.cache_type,
105
+ threads: Etc.nprocessors || 4,
106
+ n_slots: @settings.llama.n_slots,
107
+ chat_template: @chat_template,
108
+ mmproj: @mmproj,
109
+ spec_draft_model: (@mtp_drafter if @mtp),
110
+ spec_type: ("draft-mtp" if @mtp),
111
+ spec_draft_n_max: (Engine::Base::SPEC_DRAFT_N_MAX if @mtp),
112
+ reasoning: @settings.model.reasoning,
113
+ reasoning_effort: @settings.model.reasoning_effort,
114
+ load_on_startup: "true",
115
+ }
116
+ end
117
+ end
118
+ end
119
+ end