mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require_relative "../model"
|
|
3
|
+
require_relative "header"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
class Model
|
|
7
|
+
# What the repo this GGUF came from publishes today, against what is on
|
|
8
|
+
# disk.
|
|
9
|
+
#
|
|
10
|
+
# The comparison is byte size, not mtime: a local mtime is a download
|
|
11
|
+
# timestamp and survives nothing, while the repo's file size changes on
|
|
12
|
+
# every re-quantize. The blind spot is a re-quantize that lands on an
|
|
13
|
+
# identical byte count.
|
|
14
|
+
class Upstream
|
|
15
|
+
LMSTUDIO_ROOT = Pathname.new(Dir.home) / ".lmstudio" / "models"
|
|
16
|
+
|
|
17
|
+
def initialize(model, client:)
|
|
18
|
+
@model = model
|
|
19
|
+
@client = client
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def repo_id
|
|
23
|
+
return @repo_id if defined?(@repo_id)
|
|
24
|
+
|
|
25
|
+
@repo_id = from_lmstudio_path || from_header
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def verdict
|
|
29
|
+
return :unknown if repo_id.nil? || tree.nil?
|
|
30
|
+
return :gone if entry.nil?
|
|
31
|
+
|
|
32
|
+
entry["size"] == local_size ? :current : :stale
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def updated_at = entry&.dig("lastCommit", "date").to_s[0, 10].then { |date| date.empty? ? nil : date }
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def real
|
|
40
|
+
@real ||= @model.real
|
|
41
|
+
rescue SystemCallError
|
|
42
|
+
@real = @model.path
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def local_size = File.size(real)
|
|
46
|
+
|
|
47
|
+
# LM Studio lays its cache out as <root>/<owner>/<repo>/<file>, which is
|
|
48
|
+
# the repo id spelled out -- no header and no network needed.
|
|
49
|
+
def from_lmstudio_path
|
|
50
|
+
prefix = "#{LMSTUDIO_ROOT}/"
|
|
51
|
+
return nil unless real.to_s.start_with?(prefix)
|
|
52
|
+
|
|
53
|
+
parts = real.to_s.delete_prefix(prefix).split("/")
|
|
54
|
+
parts.length >= 3 ? parts[0, 2].join("/") : nil
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# general.repo_url carries the quantizer's org and stops there
|
|
58
|
+
# (`https://huggingface.co/unsloth`), so the repo half comes from the
|
|
59
|
+
# directory the file was downloaded into. Wrong guesses fail closed:
|
|
60
|
+
# the repo 404s and the verdict is :unknown.
|
|
61
|
+
def from_header
|
|
62
|
+
owner = Header.new(real).metadata["general.repo_url"].to_s[%r{huggingface\.co/([^/\s]+)}, 1]
|
|
63
|
+
owner && "#{owner}/#{real.dirname.basename}"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def tree
|
|
67
|
+
return @tree if defined?(@tree)
|
|
68
|
+
|
|
69
|
+
@tree = repo_id && @client.tree(repo_id)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def entry
|
|
73
|
+
return @entry if defined?(@entry)
|
|
74
|
+
|
|
75
|
+
@entry = tree&.find do |candidate|
|
|
76
|
+
candidate["type"] != "directory" && File.basename(candidate["path"].to_s) == real.basename.to_s
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
data/lib/mimic/model.rb
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require_relative "llm_config"
|
|
3
|
+
require_relative "paths"
|
|
4
|
+
require_relative "model/footprint"
|
|
5
|
+
require_relative "model/header"
|
|
6
|
+
require_relative "model/memory"
|
|
7
|
+
|
|
8
|
+
module Mimic
|
|
9
|
+
# A GGUF on disk together with the multimodal projector that belongs to it.
|
|
10
|
+
#
|
|
11
|
+
# llama.cpp ships a vision model as two files: the decoder, and a projector
|
|
12
|
+
# (the vision tower plus the adapter into the decoder's hidden dim) loaded
|
|
13
|
+
# with --mmproj. The projector cannot run on its own, so a scan pairs it
|
|
14
|
+
# onto its model rather than listing it as one.
|
|
15
|
+
class Model
|
|
16
|
+
class << self
|
|
17
|
+
# llama.cpp's own convert/clip.cpp mark a projector GGUF with
|
|
18
|
+
# general.type == "mmproj" -- naming (mmproj-prefixed, mmproj-suffixed,
|
|
19
|
+
# or neither) is convention, not a contract, so the header is the only
|
|
20
|
+
# reliable signal.
|
|
21
|
+
def projector?(path) = Header.new(path).metadata["general.type"] == "mmproj"
|
|
22
|
+
|
|
23
|
+
# Gemma 4 ships its MTP head as a separate GGUF loaded with
|
|
24
|
+
# --spec-draft-model; its architecture is the target's plus "-assistant".
|
|
25
|
+
def drafter?(path) = architecture(path).to_s.end_with?("-assistant")
|
|
26
|
+
|
|
27
|
+
def pair(entries)
|
|
28
|
+
entries.reject { |path, _scan_dir| projector?(path) || drafter?(path) }
|
|
29
|
+
.map { |path, scan_dir| new(path, scan_dir) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def projector_for(model_path) = sibling_for(model_path) { |candidate| projector?(candidate) }
|
|
33
|
+
|
|
34
|
+
def drafter_for(model_path)
|
|
35
|
+
target = architecture(model_path)
|
|
36
|
+
return nil if target.nil?
|
|
37
|
+
|
|
38
|
+
sibling_for(model_path) { |candidate| architecture(candidate) == "#{target}-assistant" }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def architecture(path) = Header.new(path).metadata["general.architecture"]
|
|
44
|
+
|
|
45
|
+
# The directory is the binding; stem only breaks a tie when a
|
|
46
|
+
# directory holds several matches. The glob has to run against the
|
|
47
|
+
# realpath directory: ~/hf-models is a tree of per-file symlinks, and
|
|
48
|
+
# a model's projector usually exists only in the target directory.
|
|
49
|
+
def sibling_for(model_path, &match)
|
|
50
|
+
real = Pathname.new(model_path.to_s).realpath
|
|
51
|
+
candidates = Pathname.glob(real.dirname / "*.gguf").select { |c| c.file? && match.call(c) }.sort
|
|
52
|
+
return nil if candidates.empty?
|
|
53
|
+
|
|
54
|
+
stem = LlmConfig::Settings.alias_for(model_path.to_s).downcase
|
|
55
|
+
candidates.find { |c| c.basename.to_s.downcase.include?(stem) } ||
|
|
56
|
+
(candidates.one? ? candidates.first : nil)
|
|
57
|
+
rescue SystemCallError
|
|
58
|
+
nil
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def initialize(path, scan_dir = nil)
|
|
63
|
+
@path = Pathname.new(path.to_s)
|
|
64
|
+
@scan_dir = scan_dir
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
attr_reader :path, :scan_dir
|
|
68
|
+
|
|
69
|
+
def real = @real ||= path.realpath
|
|
70
|
+
def model_alias = @model_alias ||= LlmConfig::Settings.alias_for(path.to_s)
|
|
71
|
+
def vision? = !projector.nil?
|
|
72
|
+
def embedding? = @embedding ||= header.embedding? == true
|
|
73
|
+
|
|
74
|
+
def projector
|
|
75
|
+
return @projector if defined?(@projector)
|
|
76
|
+
|
|
77
|
+
@projector = self.class.projector_for(path)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def drafter
|
|
81
|
+
return @drafter if defined?(@drafter)
|
|
82
|
+
|
|
83
|
+
@drafter = self.class.drafter_for(path)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def memory = @memory ||= Memory.new(path, header)
|
|
87
|
+
|
|
88
|
+
def supports_mtp? = header.supports_mtp?
|
|
89
|
+
def moe? = header.moe? == true
|
|
90
|
+
|
|
91
|
+
def footprint(ctx, cache_type, compute_mib: nil, drafting: false, slots: 1, reclaimable_mib: 0,
|
|
92
|
+
mmproj_path: projector)
|
|
93
|
+
Footprint.new(header, ctx: ctx, cache_type: cache_type, weights_bytes: weights_bytes(mmproj_path),
|
|
94
|
+
compute_mib: compute_mib, drafting: drafting, slots: slots,
|
|
95
|
+
reclaimable_mib: reclaimable_mib)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
def header = @header ||= Header.new(path)
|
|
101
|
+
|
|
102
|
+
def weights_bytes(mmproj_path)
|
|
103
|
+
File.size(path) + projector_bytes(mmproj_path)
|
|
104
|
+
rescue SystemCallError
|
|
105
|
+
nil
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# A vision model's projector is loaded onto the GPU with --mmproj, so it
|
|
109
|
+
# spends VRAM the decoder's own file size does not account for.
|
|
110
|
+
def projector_bytes(mmproj_path)
|
|
111
|
+
mmproj_path ? File.size(mmproj_path) : 0
|
|
112
|
+
rescue SystemCallError
|
|
113
|
+
0
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
data/lib/mimic/paths.rb
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
|
|
3
|
+
# Filesystem roots shared by the mimic and llm CLIs.
|
|
4
|
+
#
|
|
5
|
+
# The two CLIs own separate trees: LLM_ROOT is everything the inference
|
|
6
|
+
# engine touches, STATE_DIR is everything the agent side does. Neither
|
|
7
|
+
# engine claims a home directory of its own -- llama.cpp reads no config
|
|
8
|
+
# file and uses ~/.cache/llama.cpp only for -hf downloads, ik_llama.cpp has
|
|
9
|
+
# no convention at all -- so ~/.llm is ours to define.
|
|
10
|
+
module Mimic
|
|
11
|
+
module Paths
|
|
12
|
+
LLM_ROOT = Pathname.new(Dir.home) / ".llm"
|
|
13
|
+
LLM_RUN_DIR = LLM_ROOT / "run"
|
|
14
|
+
# Hand-maintained `<model-alias>.jinja` overrides. Present only for
|
|
15
|
+
# models whose shipped template llama.cpp cannot work with; the
|
|
16
|
+
# launchers pass one to the server when the file exists and ignore the
|
|
17
|
+
# directory otherwise.
|
|
18
|
+
CHAT_TEMPLATES_DIR = LLM_ROOT / "chat-templates"
|
|
19
|
+
# Sanitized copies mimic writes itself, kept apart from the
|
|
20
|
+
# hand-maintained ones so a regenerated file can never clobber a
|
|
21
|
+
# decision the user made.
|
|
22
|
+
GENERATED_TEMPLATES_DIR = CHAT_TEMPLATES_DIR / "generated"
|
|
23
|
+
PID_DIR = Pathname.new("/tmp/llm")
|
|
24
|
+
|
|
25
|
+
# The dev-only command tree is stripped from the published gem, so its
|
|
26
|
+
# presence on disk is what separates a checkout from an install. One
|
|
27
|
+
# marker rather than a flag per feature: anything held back from a
|
|
28
|
+
# release is held back on the same condition.
|
|
29
|
+
def self.dev_build? = (Pathname.new(__dir__) / "llm_cli/dev").directory?
|
|
30
|
+
|
|
31
|
+
STATE_DIR = Pathname.new(Dir.home) / ".mimic"
|
|
32
|
+
|
|
33
|
+
# Opt-in MCP servers for the agents mimic launches, in Claude Code's own
|
|
34
|
+
# `mcpServers` shape. Absent means mimic wires none -- it ships no server
|
|
35
|
+
# of its own and picks none for you.
|
|
36
|
+
MCP_SERVERS_PATH = STATE_DIR / "mcp-servers.json"
|
|
37
|
+
|
|
38
|
+
# Upstream nests agent homes as `auth_root()/agents/<name>`; agent.rb
|
|
39
|
+
# collapses that level, so homes land directly in AGENTS_DIR. Claude is
|
|
40
|
+
# the one agent whose home mimic passes in itself (CLAUDE_CONFIG_DIR);
|
|
41
|
+
# it reads AGENTS_DIR to sit beside the rest.
|
|
42
|
+
AGENTS_DIR = STATE_DIR / "agents"
|
|
43
|
+
CLAUDE_CONFIG_DIR = AGENTS_DIR / "claude"
|
|
44
|
+
end
|
|
45
|
+
end
|
data/lib/mimic/result.rb
ADDED
data/lib/mimic/ui.rb
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
require "cli/ui"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
# The only place that knows cli-ui exists. Everything else asks for a
|
|
5
|
+
# Ui::Prompt or calls Ui.banner/panel/table, so swapping the terminal
|
|
6
|
+
# library again is one file rather than every wizard.
|
|
7
|
+
module Ui
|
|
8
|
+
# cli-ui builds the "Choose with ↑ ↓ ⏎" instruction line inside
|
|
9
|
+
# ask_interactive and prints it through puts_question, so the one place
|
|
10
|
+
# to reach it is on the way out. ⏎ renders as an empty box in this
|
|
11
|
+
# terminal, and Space selects exactly as Return does, so both keys are
|
|
12
|
+
# named; the arrows render fine and are left alone.
|
|
13
|
+
module SpelledOutKeys
|
|
14
|
+
def puts_question(str) = super(str.gsub("⏎", "Enter/Space"))
|
|
15
|
+
end
|
|
16
|
+
CLI::UI::Prompt.singleton_class.prepend(SpelledOutKeys)
|
|
17
|
+
|
|
18
|
+
COLORS = %i[green red yellow cyan gray bold].freeze
|
|
19
|
+
|
|
20
|
+
class << self
|
|
21
|
+
def fmt(text) = CLI::UI.fmt(text)
|
|
22
|
+
|
|
23
|
+
COLORS.each do |color|
|
|
24
|
+
define_method(color) { |text| CLI::UI.fmt("{{#{color}:#{text}}}") }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def bold_cyan(text) = CLI::UI.fmt("{{bold:{{cyan:#{text}}}}}")
|
|
28
|
+
def bold_red(text) = CLI::UI.fmt("{{bold:{{red:#{text}}}}}")
|
|
29
|
+
|
|
30
|
+
def banner(title)
|
|
31
|
+
puts CLI::UI.fmt("{{bold:#{title}}}")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def panel(body, title: nil)
|
|
35
|
+
CLI::UI::Frame.open(title.to_s, color: :cyan) { puts body }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Not CLI::UI::Table: both of its entry points render through
|
|
39
|
+
# CLI::UI.puts, which hard-wraps at the terminal width and folds a wide
|
|
40
|
+
# model board into unreadable fragments. Padding is measured with
|
|
41
|
+
# printing_width so colour escapes do not count toward column widths.
|
|
42
|
+
def table(header:, rows:)
|
|
43
|
+
all = [header.map { |cell| bold(cell) }, *rows].map { |row| row.map(&:to_s) }
|
|
44
|
+
widths = all.transpose.map { |col| col.map { |cell| CLI::UI::ANSI.printing_width(cell) }.max }
|
|
45
|
+
all.each do |row|
|
|
46
|
+
padded = row.each_with_index.map { |cell, i| cell + (" " * (widths[i] - CLI::UI::ANSI.printing_width(cell))) }
|
|
47
|
+
puts padded.join(" ").rstrip
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# tty-prompt's shape, backed by cli-ui: choices may be an Array of
|
|
53
|
+
# labels, a Hash of label => value, or an Array of {name:, value:}, and
|
|
54
|
+
# the answer is always the value rather than the label.
|
|
55
|
+
class Prompt
|
|
56
|
+
def select(question, choices, default: nil)
|
|
57
|
+
pairs = normalize(choices)
|
|
58
|
+
labels = pairs.map(&:first)
|
|
59
|
+
picked = CLI::UI::Prompt.ask(question, options: labels, default: default_label(default, labels))
|
|
60
|
+
pairs.to_h.fetch(picked)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def ask(question, default: nil, convert: nil)
|
|
64
|
+
answer = CLI::UI::Prompt.ask(question, default: default&.to_s, allow_empty: !default.nil?)
|
|
65
|
+
convert == :int ? Integer(answer) : answer
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def yes?(question, default: true)
|
|
69
|
+
CLI::UI::Prompt.confirm(question, default: default)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def normalize(choices)
|
|
75
|
+
case choices
|
|
76
|
+
when Hash then choices.to_a
|
|
77
|
+
else choices.map { |choice| choice.is_a?(Hash) ? [choice.fetch(:name), choice.fetch(:value)] : [choice, choice] }
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# tty-prompt took either a label or a 1-based index; cli-ui only takes
|
|
82
|
+
# a label, and rejects one that is not among the options.
|
|
83
|
+
def default_label(default, labels)
|
|
84
|
+
label = default.is_a?(Integer) ? labels[default - 1] : default
|
|
85
|
+
labels.include?(label) ? label : nil
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
module Mimic
|
|
2
|
+
module Util
|
|
3
|
+
module Fmt
|
|
4
|
+
class << self
|
|
5
|
+
# "180000" -> "180,000"
|
|
6
|
+
def commas(n)
|
|
7
|
+
n.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\1,').reverse
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
# Decimal GB (1000^3), the unit model files are advertised in.
|
|
11
|
+
def human_size(bytes) = format("%.1f GB", bytes / 1000.0**3)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
require "net/http"
|
|
2
|
+
require "json"
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
module Util
|
|
7
|
+
# One instance per scan: a repo is fetched once and the answer -- including
|
|
8
|
+
# a failure -- is kept, so several quants out of the same repo cost one
|
|
9
|
+
# request. `expand=true` is what turns the listing into per-file
|
|
10
|
+
# `size`/`lastCommit`; without it the endpoint returns paths only.
|
|
11
|
+
#
|
|
12
|
+
# HF_ENDPOINT is huggingface_hub's own origin override, so a box already
|
|
13
|
+
# pointed at a mirror for downloads gets its API calls sent the same way.
|
|
14
|
+
class Hf
|
|
15
|
+
ORIGIN = "https://huggingface.co".freeze
|
|
16
|
+
TIMEOUT = 10
|
|
17
|
+
|
|
18
|
+
def initialize
|
|
19
|
+
@trees = {}
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def tree(repo_id)
|
|
23
|
+
return @trees[repo_id] if @trees.key?(repo_id)
|
|
24
|
+
|
|
25
|
+
@trees[repo_id] = fetch(repo_id)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
# Every failure mode here -- 404, DNS, TLS, timeout, truncated JSON --
|
|
31
|
+
# means the same thing to the caller and none of them may take down the
|
|
32
|
+
# rest of a scan, so the rescue is deliberately wide.
|
|
33
|
+
def fetch(repo_id)
|
|
34
|
+
response = get(URI("#{endpoint}/api/models/#{repo_id}/tree/main?expand=true&recursive=true"))
|
|
35
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
36
|
+
|
|
37
|
+
parsed = JSON.parse(response.body)
|
|
38
|
+
parsed.is_a?(Array) ? parsed : nil
|
|
39
|
+
rescue StandardError
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def endpoint
|
|
44
|
+
configured = ENV.fetch("HF_ENDPOINT", "").sub(%r{/+\z}, "")
|
|
45
|
+
configured.empty? ? ORIGIN : configured
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def get(uri)
|
|
49
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: TIMEOUT, read_timeout: TIMEOUT) do |http|
|
|
50
|
+
http.get(uri.request_uri)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
require "etc"
|
|
2
|
+
require "open3"
|
|
3
|
+
require "pathname"
|
|
4
|
+
require_relative "probes"
|
|
5
|
+
require_relative "../engine_cli/registry"
|
|
6
|
+
|
|
7
|
+
# Machine facts the wizard reports on and the build path has to respect.
|
|
8
|
+
module Mimic
|
|
9
|
+
module Util
|
|
10
|
+
module Host
|
|
11
|
+
class << self
|
|
12
|
+
def total_mem_kb
|
|
13
|
+
File.readlines("/proc/meminfo").each do |line|
|
|
14
|
+
return line.split[1].to_i if line.start_with?("MemTotal")
|
|
15
|
+
end
|
|
16
|
+
0
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def free_home_bytes
|
|
20
|
+
`df -B1 --output=avail #{Dir.home}`.lines[1].to_i
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def backend
|
|
24
|
+
return "cuda" if Probes.which("nvidia-smi")
|
|
25
|
+
return "hip" if amd?
|
|
26
|
+
|
|
27
|
+
"cpu"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Auto-detect GPU backend for cmake build flags. CPU is built-in by default.
|
|
31
|
+
def gpu_build_flags
|
|
32
|
+
return ["-DGGML_CUDA=on"] if Probes.which("nvcc")
|
|
33
|
+
return ["-DGGML_HIP=on"] if amd?
|
|
34
|
+
|
|
35
|
+
[]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Cap parallel compile jobs by RAM -- unbounded -j OOMs this box and logs out the desktop.
|
|
39
|
+
def build_jobs
|
|
40
|
+
mem_gb = total_mem_kb / 1024 / 1024
|
|
41
|
+
cpu = Etc.nprocessors || 4
|
|
42
|
+
[2, [cpu, [mem_gb / 5, 2].max].min].max
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def llama_server = EngineCli::Registry.installed_binary
|
|
46
|
+
|
|
47
|
+
def gpu_total_mib
|
|
48
|
+
out, status = Open3.capture2("nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits")
|
|
49
|
+
return nil unless status.success?
|
|
50
|
+
|
|
51
|
+
Integer(out.lines.first.to_s.strip)
|
|
52
|
+
rescue Errno::ENOENT, ArgumentError, TypeError
|
|
53
|
+
nil
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# VRAM the driver reports unused right now. Preferred over
|
|
57
|
+
# total-minus-a-constant: the desktop's real draw moves (2295 MiB
|
|
58
|
+
# measured against the 2048 the constant assumes), and that quarter
|
|
59
|
+
# gigabyte is the difference between an estimate that clears the
|
|
60
|
+
# budget and a launch that dies allocating its compute buffer.
|
|
61
|
+
def gpu_free_mib
|
|
62
|
+
out, status = Open3.capture2("nvidia-smi", "--query-gpu=memory.free", "--format=csv,noheader,nounits")
|
|
63
|
+
return nil unless status.success?
|
|
64
|
+
|
|
65
|
+
Integer(out.lines.first.to_s.strip)
|
|
66
|
+
rescue Errno::ENOENT, ArgumentError, TypeError
|
|
67
|
+
nil
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# VRAM held by the given processes. The engine that owns the CUDA
|
|
71
|
+
# context is not always the process we spawned -- llama.cpp's router
|
|
72
|
+
# loads each model in a child -- so the caller passes the whole
|
|
73
|
+
# descendant set and every match counts.
|
|
74
|
+
def gpu_process_mib(pids)
|
|
75
|
+
out, status = Open3.capture2("nvidia-smi", "--query-compute-apps=pid,used_memory",
|
|
76
|
+
"--format=csv,noheader,nounits")
|
|
77
|
+
return nil unless status.success?
|
|
78
|
+
|
|
79
|
+
wanted = pids.map(&:to_i)
|
|
80
|
+
total = out.lines.filter_map do |line|
|
|
81
|
+
pid, mib = line.split(",").map { |field| field.strip.to_i }
|
|
82
|
+
mib if wanted.include?(pid)
|
|
83
|
+
end.sum
|
|
84
|
+
total.positive? ? total : nil
|
|
85
|
+
rescue Errno::ENOENT, ArgumentError, TypeError
|
|
86
|
+
nil
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private
|
|
90
|
+
|
|
91
|
+
def amd? = Pathname.new("/dev/kfd").exist? || Probes.which("rocminfo")
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|