mimic-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/NOTICE +16 -0
- data/README.md +175 -0
- data/exe/lllm +5 -0
- data/exe/llm +5 -0
- data/exe/mimic +5 -0
- data/lib/mimic/README.md +144 -0
- data/lib/mimic/agent/claude.rb +305 -0
- data/lib/mimic/agent/codex.rb +254 -0
- data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
- data/lib/mimic/agent/hermes.rb +107 -0
- data/lib/mimic/agent/muse.rb +106 -0
- data/lib/mimic/agent.rb +46 -0
- data/lib/mimic/cli.rb +159 -0
- data/lib/mimic/client/base.rb +44 -0
- data/lib/mimic/client/crush.rb +28 -0
- data/lib/mimic/client/goose.rb +52 -0
- data/lib/mimic/client/hermes.rb +21 -0
- data/lib/mimic/client/opencode.rb +29 -0
- data/lib/mimic/client/pi.rb +50 -0
- data/lib/mimic/client.rb +19 -0
- data/lib/mimic/engine/README.md +140 -0
- data/lib/mimic/engine/base.rb +138 -0
- data/lib/mimic/engine/chat_template.rb +207 -0
- data/lib/mimic/engine/ik_llama.rb +273 -0
- data/lib/mimic/engine/launcher.rb +25 -0
- data/lib/mimic/engine/llama.rb +124 -0
- data/lib/mimic/engine/llama_embed.rb +108 -0
- data/lib/mimic/engine/router.rb +57 -0
- data/lib/mimic/engine/server_process.rb +108 -0
- data/lib/mimic/engine.rb +5 -0
- data/lib/mimic/engine_cli/binary_options.rb +46 -0
- data/lib/mimic/engine_cli/builder.rb +98 -0
- data/lib/mimic/engine_cli/diffusion.rb +117 -0
- data/lib/mimic/engine_cli/llama_preset.rb +119 -0
- data/lib/mimic/engine_cli/registry.rb +94 -0
- data/lib/mimic/engine_cli/source.rb +110 -0
- data/lib/mimic/engine_cli/status.rb +161 -0
- data/lib/mimic/engine_cli/update.rb +68 -0
- data/lib/mimic/engine_cli.rb +7 -0
- data/lib/mimic/health/checks.rb +92 -0
- data/lib/mimic/health.rb +82 -0
- data/lib/mimic/llm_cli/base.rb +36 -0
- data/lib/mimic/llm_cli/configure.rb +84 -0
- data/lib/mimic/llm_cli/embed.rb +80 -0
- data/lib/mimic/llm_cli/engines.rb +44 -0
- data/lib/mimic/llm_cli/models.rb +31 -0
- data/lib/mimic/llm_cli/models_table.rb +119 -0
- data/lib/mimic/llm_cli/paths.rb +48 -0
- data/lib/mimic/llm_cli/root.rb +215 -0
- data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
- data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
- data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
- data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
- data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
- data/lib/mimic/llm_cli.rb +5 -0
- data/lib/mimic/llm_config/embed.rb +48 -0
- data/lib/mimic/llm_config/llama_settings.rb +60 -0
- data/lib/mimic/llm_config/model_settings.rb +84 -0
- data/lib/mimic/llm_config/sampling.rb +54 -0
- data/lib/mimic/llm_config/settings.rb +92 -0
- data/lib/mimic/llm_config.rb +132 -0
- data/lib/mimic/model/footprint.rb +303 -0
- data/lib/mimic/model/gguf.rb +204 -0
- data/lib/mimic/model/header.rb +169 -0
- data/lib/mimic/model/library.rb +132 -0
- data/lib/mimic/model/memory.rb +126 -0
- data/lib/mimic/model/store.rb +46 -0
- data/lib/mimic/model/upstream.rb +81 -0
- data/lib/mimic/model.rb +116 -0
- data/lib/mimic/paths.rb +45 -0
- data/lib/mimic/result.rb +8 -0
- data/lib/mimic/ui.rb +89 -0
- data/lib/mimic/util/fmt.rb +15 -0
- data/lib/mimic/util/hf.rb +55 -0
- data/lib/mimic/util/host.rb +95 -0
- data/lib/mimic/util/probes.rb +255 -0
- data/lib/mimic/version.rb +3 -0
- data/lib/mimic.rb +22 -0
- metadata +186 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
module Mimic
|
|
2
|
+
class Model
|
|
3
|
+
# GGUF header reader -- the key/value metadata block and the tensor info
|
|
4
|
+
# block at the head of a model file. Tensor data is never touched; every
|
|
5
|
+
# question here is answered from the first few hundred KB.
|
|
6
|
+
#
|
|
7
|
+
# Raises for a file that is missing, truncated or not a GGUF at all;
|
|
8
|
+
# Model::Header is what turns those into nil.
|
|
9
|
+
class Gguf
|
|
10
|
+
MAGIC = "GGUF"
|
|
11
|
+
SCALAR_SIZE = { 0 => 1, 1 => 1, 2 => 2, 3 => 2, 4 => 4, 5 => 4, 6 => 4, 7 => 1, 10 => 8, 11 => 8, 12 => 8 }.freeze
|
|
12
|
+
SCALAR_FMT = {
|
|
13
|
+
0 => "C", 1 => "c", 2 => "S<", 3 => "s<", 4 => "L<", 5 => "l<",
|
|
14
|
+
6 => "e", 7 => "C", 10 => "Q<", 11 => "q<", 12 => "E",
|
|
15
|
+
}.freeze
|
|
16
|
+
STRING = 8
|
|
17
|
+
ARRAY = 9
|
|
18
|
+
|
|
19
|
+
# A string array this long is a vocab/merges table: megabytes of
|
|
20
|
+
# length-prefixed entries that can only be walked one at a time. No caller
|
|
21
|
+
# needs them, so the parse stops there and restarts in full only if the
|
|
22
|
+
# architecture block turned out to sit behind them.
|
|
23
|
+
BIG_STRING_ARRAY = 4096
|
|
24
|
+
CHUNK = 1 << 20
|
|
25
|
+
|
|
26
|
+
# Numeric arrays this short are per-layer shape (head_count_kv, the
|
|
27
|
+
# sliding-window pattern): one entry per block, kept because the KV
|
|
28
|
+
# estimate cannot be built without them. Anything longer is vocab-sized
|
|
29
|
+
# (token types, merge scores) and is skipped like the string tables.
|
|
30
|
+
BIG_SCALAR_ARRAY = 4096
|
|
31
|
+
|
|
32
|
+
def initialize(path)
|
|
33
|
+
@path = path.to_s
|
|
34
|
+
@metadata = {}
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Scalar, string and short-numeric-array KV pairs. Vocab-sized arrays
|
|
38
|
+
# are skipped.
|
|
39
|
+
def metadata(full: false)
|
|
40
|
+
@metadata[full] ||= read_metadata(full: full)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# [{name:, elements:, type:, offset:}, ...] from the tensor info block
|
|
44
|
+
# that follows the KV block.
|
|
45
|
+
def tensors
|
|
46
|
+
return @tensors if defined?(@tensors)
|
|
47
|
+
|
|
48
|
+
@tensors, @data_start = read_tensors
|
|
49
|
+
@tensors
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# {name => bytes}. Offsets are relative to the start of the data block,
|
|
53
|
+
# so the last tensor is bounded by the tail of the file rather than by a
|
|
54
|
+
# following offset.
|
|
55
|
+
def tensor_sizes
|
|
56
|
+
@tensor_sizes ||= sizes_from_gaps(tensors, File.size(@path) - @data_start)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def read_metadata(full:)
|
|
62
|
+
meta = {}
|
|
63
|
+
truncated = false
|
|
64
|
+
File.open(@path, "rb") do |f|
|
|
65
|
+
raise ArgumentError, "not a GGUF file" unless f.read(4) == MAGIC
|
|
66
|
+
|
|
67
|
+
u32(f) # version (v2/v3 both use u64 counts below)
|
|
68
|
+
u64(f) # tensor count
|
|
69
|
+
u64(f).times do
|
|
70
|
+
key = read_str(f)
|
|
71
|
+
vtype = u32(f)
|
|
72
|
+
if SCALAR_FMT.key?(vtype)
|
|
73
|
+
meta[key] = read_scalar(f, vtype)
|
|
74
|
+
elsif vtype == STRING
|
|
75
|
+
meta[key] = read_str(f)
|
|
76
|
+
elsif vtype == ARRAY
|
|
77
|
+
elem_type = u32(f)
|
|
78
|
+
count = u64(f)
|
|
79
|
+
if SCALAR_FMT.key?(elem_type) && count <= BIG_SCALAR_ARRAY
|
|
80
|
+
meta[key] = read_scalar_array(f, elem_type, count)
|
|
81
|
+
elsif !skip_array(f, elem_type, count, full: full)
|
|
82
|
+
truncated = true
|
|
83
|
+
break
|
|
84
|
+
end
|
|
85
|
+
else
|
|
86
|
+
raise ArgumentError, "unknown gguf type #{vtype}"
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
return meta unless truncated && !arch_block_read?(meta)
|
|
91
|
+
|
|
92
|
+
read_metadata(full: true)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# The KV walk has to be exhaustive here -- the early stop metadata takes
|
|
96
|
+
# on a vocab-sized array would leave the file positioned mid-header.
|
|
97
|
+
def read_tensors
|
|
98
|
+
File.open(@path, "rb") do |f|
|
|
99
|
+
raise ArgumentError, "not a GGUF file" unless f.read(4) == MAGIC
|
|
100
|
+
|
|
101
|
+
u32(f)
|
|
102
|
+
n_tensors = u64(f)
|
|
103
|
+
u64(f).times { read_str(f); skip_value(f, u32(f)) }
|
|
104
|
+
tensors = Array.new(n_tensors) do
|
|
105
|
+
name = read_str(f)
|
|
106
|
+
elements = Array.new(u32(f)) { u64(f) }.inject(1, :*)
|
|
107
|
+
{ name: name, elements: elements, type: u32(f), offset: u64(f) }
|
|
108
|
+
end
|
|
109
|
+
[tensors, f.pos]
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def sizes_from_gaps(tensors, data_bytes)
|
|
114
|
+
ordered = tensors.sort_by { |tensor| tensor[:offset] }
|
|
115
|
+
ordered.each_with_index.to_h do |tensor, i|
|
|
116
|
+
following = ordered[i + 1]
|
|
117
|
+
[tensor[:name], (following ? following[:offset] : data_bytes) - tensor[:offset]]
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def u32(f) = read_exactly(f, 4).unpack1("L<")
|
|
122
|
+
def u64(f) = read_exactly(f, 8).unpack1("Q<")
|
|
123
|
+
def read_str(f) = read_exactly(f, u64(f)).force_encoding("UTF-8").scrub
|
|
124
|
+
|
|
125
|
+
def read_scalar_array(f, elem_type, count)
|
|
126
|
+
read_exactly(f, SCALAR_SIZE.fetch(elem_type) * count).unpack("#{SCALAR_FMT.fetch(elem_type)}*")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def read_scalar(f, vtype)
|
|
130
|
+
read_exactly(f, SCALAR_SIZE.fetch(vtype)).unpack1(SCALAR_FMT.fetch(vtype))
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# A header cut mid-value leaves IO#read returning short or nil, which
|
|
134
|
+
# would surface as NoMethodError on the unpack that follows.
|
|
135
|
+
def read_exactly(f, bytes)
|
|
136
|
+
chunk = f.read(bytes)
|
|
137
|
+
raise EOFError, "truncated gguf header" if chunk.nil? || chunk.bytesize < bytes
|
|
138
|
+
|
|
139
|
+
chunk
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def arch_block_read?(meta)
|
|
143
|
+
arch = meta["general.architecture"]
|
|
144
|
+
!arch.nil? && meta.keys.any? { |key| key.start_with?("#{arch}.") }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def skip_string_array(f, count)
|
|
148
|
+
buf = "".b
|
|
149
|
+
off = 0
|
|
150
|
+
base = f.pos
|
|
151
|
+
count.times do
|
|
152
|
+
if buf.bytesize - off < 8
|
|
153
|
+
base += off
|
|
154
|
+
f.seek(base)
|
|
155
|
+
buf = f.read(CHUNK) || "".b
|
|
156
|
+
off = 0
|
|
157
|
+
raise EOFError, "truncated gguf string array" if buf.bytesize < 8
|
|
158
|
+
end
|
|
159
|
+
len = buf.unpack1("Q<", offset: off)
|
|
160
|
+
off += 8
|
|
161
|
+
if len > buf.bytesize - off
|
|
162
|
+
base += off + len
|
|
163
|
+
f.seek(base)
|
|
164
|
+
buf = "".b
|
|
165
|
+
off = 0
|
|
166
|
+
else
|
|
167
|
+
off += len
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
f.seek(base + off)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Returns false when the value was a vocab-sized string array and `full`
|
|
174
|
+
# is off -- see skip_array.
|
|
175
|
+
def skip_value(f, vtype, full: true)
|
|
176
|
+
if SCALAR_SIZE.key?(vtype)
|
|
177
|
+
f.seek(SCALAR_SIZE[vtype], IO::SEEK_CUR)
|
|
178
|
+
elsif vtype == STRING
|
|
179
|
+
f.seek(u64(f), IO::SEEK_CUR)
|
|
180
|
+
elsif vtype == ARRAY
|
|
181
|
+
return skip_array(f, u32(f), u64(f), full: full)
|
|
182
|
+
else
|
|
183
|
+
raise ArgumentError, "unknown gguf type #{vtype}"
|
|
184
|
+
end
|
|
185
|
+
true
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Returns false when the value was a vocab-sized string array and `full`
|
|
189
|
+
# is off -- the parse stops there.
|
|
190
|
+
def skip_array(f, elem_type, count, full: true)
|
|
191
|
+
if SCALAR_SIZE.key?(elem_type)
|
|
192
|
+
f.seek(SCALAR_SIZE[elem_type] * count, IO::SEEK_CUR)
|
|
193
|
+
elsif elem_type == STRING
|
|
194
|
+
return false if !full && count > BIG_STRING_ARRAY
|
|
195
|
+
|
|
196
|
+
skip_string_array(f, count)
|
|
197
|
+
else
|
|
198
|
+
raise ArgumentError, "nested array type #{elem_type}"
|
|
199
|
+
end
|
|
200
|
+
true
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
require_relative "gguf"
|
|
2
|
+
|
|
3
|
+
module Mimic
|
|
4
|
+
class Model
|
|
5
|
+
# Questions about one GGUF's header. Everything is answered off a single
|
|
6
|
+
# parse, and a file that is missing, truncated or not a GGUF answers nil
|
|
7
|
+
# rather than raising -- callers ask about models found by a directory
|
|
8
|
+
# scan, where an unreadable file is ordinary.
|
|
9
|
+
class Header
|
|
10
|
+
EXPERT_TENSOR = /\Ablk\.(\d+)\..*exps/
|
|
11
|
+
|
|
12
|
+
# Ternary (Bonsai) weights. ggml type 42 is the trap: mainline llama.cpp
|
|
13
|
+
# packs 64 weights into an 18-byte block, PrismML's fork 128 into 34, and
|
|
14
|
+
# neither reads the other's file -- under the same type id, so only the
|
|
15
|
+
# measured bytes-per-weight tells them apart. Ids above what any engine
|
|
16
|
+
# here defines are rejected unless they are current Prism ternary types.
|
|
17
|
+
TERNARY_TYPE = 42
|
|
18
|
+
PRISM_TYPES = { 142 => :pq2, 143 => :ptq1 }.freeze
|
|
19
|
+
MAX_COMMON_TYPE = 42
|
|
20
|
+
G128_BYTES_PER_WEIGHT = 34 / 128.0
|
|
21
|
+
G64_BYTES_PER_WEIGHT = 18 / 64.0
|
|
22
|
+
|
|
23
|
+
# Mainline llama.cpp's I-quants. ik_llama.cpp carries a dedicated CPU
|
|
24
|
+
# GEMM for exactly this family (iqk_gemm_iquants.cpp) plus interleaved
|
|
25
|
+
# _R4/_R8 repacks of it, which is why the pairing with MoE offload is
|
|
26
|
+
# ik's alone. ik's own IQx_K ids are absent on purpose: mainline cannot
|
|
27
|
+
# load those at all, so there is no engine choice left to make.
|
|
28
|
+
IQUANT_TYPES = [16, 17, 18, 19, 20, 21, 22, 23, 29].freeze
|
|
29
|
+
|
|
30
|
+
# Architectures that only ever ship as encoders. A decoder-shaped
|
|
31
|
+
# embedding model (Qwen3-Embedding and friends) keeps its chat
|
|
32
|
+
# architecture and is caught by the pooling/causal keys instead.
|
|
33
|
+
ENCODER_ARCHS = %w[bert nomic-bert nomic-bert-moe jina-bert-v2 modern-bert neo-bert xlm-roberta t5encoder].freeze
|
|
34
|
+
POOLING_RANK = 4
|
|
35
|
+
|
|
36
|
+
MTP_SCAN_BYTES = 8_000_000
|
|
37
|
+
|
|
38
|
+
def initialize(path)
|
|
39
|
+
@path = path.to_s
|
|
40
|
+
@gguf = Gguf.new(@path)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def moe? = arch_value { |meta, arch| !meta.fetch("#{arch}.expert_count", 0).zero? }
|
|
44
|
+
|
|
45
|
+
def embedding?
|
|
46
|
+
arch_value do |meta, arch|
|
|
47
|
+
meta["#{arch}.attention.causal"] == 0 ||
|
|
48
|
+
meta.key?("#{arch}.pooling_type") ||
|
|
49
|
+
ENCODER_ARCHS.include?(arch)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# MTP-capable GGUFs carry a *.nextn_predict_layers key. Read as raw
|
|
54
|
+
# bytes rather than through the parsed metadata: the key is
|
|
55
|
+
# architecture-prefixed and a model whose header will not parse at all
|
|
56
|
+
# still answers false instead of raising out of render_preset.
|
|
57
|
+
def supports_mtp?
|
|
58
|
+
File.open(@path, "rb") { |f| f.read(MTP_SCAN_BYTES).to_s.include?("nextn_predict_layers") }
|
|
59
|
+
rescue SystemCallError, IOError
|
|
60
|
+
false
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def pooling_type = arch_value { |meta, arch| meta["#{arch}.pooling_type"] }
|
|
64
|
+
def block_count = arch_value { |meta, arch| meta["#{arch}.block_count"] }
|
|
65
|
+
|
|
66
|
+
# How many of block_count's blocks are MTP draft heads rather than
|
|
67
|
+
# transformer layers. Counted out of the KV estimate unless the engine
|
|
68
|
+
# is actually drafting, and counted back in when it is -- a nextn block
|
|
69
|
+
# carries its own full attention.
|
|
70
|
+
def nextn_layers = arch_value { |meta, arch| meta["#{arch}.nextn_predict_layers"] } || 0
|
|
71
|
+
|
|
72
|
+
# Which engine's ternary packing this model carries: :g128 (PrismML
|
|
73
|
+
# v5 fork), :g64 (mainline llama.cpp), :pq2 or :ptq1 (current Prism
|
|
74
|
+
# fork), :unsupported (an unknown or mixed private type), or nil.
|
|
75
|
+
def ternary_packing
|
|
76
|
+
return @ternary_packing if defined?(@ternary_packing)
|
|
77
|
+
|
|
78
|
+
@ternary_packing = read_ternary_packing
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Whether the weights are predominantly a mainline I-quant. Judged on
|
|
82
|
+
# share of tensor bytes, not on any single tensor: a dynamic quant
|
|
83
|
+
# ("UD-IQ4_XS") mixes types per layer and its widest tensor is always
|
|
84
|
+
# the Q8_0/Q5_K token_embd or output, so the widest one names the file
|
|
85
|
+
# wrongly every time. Measured here: UD-IQ4_NL is 80% IQ by bytes,
|
|
86
|
+
# UD-IQ4_XS 68%, both behind a non-IQ widest tensor.
|
|
87
|
+
def iquant?
|
|
88
|
+
return @iquant if defined?(@iquant)
|
|
89
|
+
|
|
90
|
+
@iquant = read_iquant
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Average bytes of expert weights per MoE layer -- what moving one
|
|
94
|
+
# layer's experts to CPU with --n-cpu-moe frees on the GPU. nil for a
|
|
95
|
+
# dense model or an unreadable file.
|
|
96
|
+
#
|
|
97
|
+
# Sizes come from the gaps between consecutive tensor offsets rather
|
|
98
|
+
# than from shape x type, which would need the whole ggml block-size
|
|
99
|
+
# table to be correct for every quant.
|
|
100
|
+
def expert_bytes_per_layer
|
|
101
|
+
return @expert_bytes_per_layer if defined?(@expert_bytes_per_layer)
|
|
102
|
+
|
|
103
|
+
@expert_bytes_per_layer = read_expert_bytes_per_layer
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def metadata(full: false)
|
|
107
|
+
@metadata ||= {}
|
|
108
|
+
@metadata[full] ||= @gguf.metadata(full: full)
|
|
109
|
+
rescue Errno::ENOENT, ArgumentError, EOFError
|
|
110
|
+
@metadata[full] = {}
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
def read_ternary_packing
|
|
116
|
+
tensors = @gguf.tensors
|
|
117
|
+
return nil if tensors.empty?
|
|
118
|
+
return :unsupported if tensors.any? { |tensor| tensor[:type] > MAX_COMMON_TYPE && !PRISM_TYPES.key?(tensor[:type]) }
|
|
119
|
+
prism_packings = tensors.filter_map { |tensor| PRISM_TYPES[tensor[:type]] }.uniq
|
|
120
|
+
return :unsupported if prism_packings.size > 1
|
|
121
|
+
return prism_packings.first unless prism_packings.empty?
|
|
122
|
+
|
|
123
|
+
ternary = tensors.select { |tensor| tensor[:type] == TERNARY_TYPE }
|
|
124
|
+
return nil if ternary.empty?
|
|
125
|
+
|
|
126
|
+
# The gap to the next tensor includes GGUF's alignment padding, so
|
|
127
|
+
# the biggest tensor is the one whose bytes-per-weight is least
|
|
128
|
+
# distorted by it.
|
|
129
|
+
sizes = @gguf.tensor_sizes
|
|
130
|
+
widest = ternary.max_by { |tensor| sizes.fetch(tensor[:name]) }
|
|
131
|
+
per_weight = sizes.fetch(widest[:name]).to_f / widest[:elements]
|
|
132
|
+
(per_weight - G128_BYTES_PER_WEIGHT).abs < (per_weight - G64_BYTES_PER_WEIGHT).abs ? :g128 : :g64
|
|
133
|
+
rescue Errno::ENOENT, ArgumentError, EOFError
|
|
134
|
+
nil
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def read_iquant
|
|
138
|
+
sizes = @gguf.tensor_sizes
|
|
139
|
+
total = 0
|
|
140
|
+
iquant = 0
|
|
141
|
+
@gguf.tensors.each do |tensor|
|
|
142
|
+
bytes = sizes.fetch(tensor[:name])
|
|
143
|
+
total += bytes
|
|
144
|
+
iquant += bytes if IQUANT_TYPES.include?(tensor[:type])
|
|
145
|
+
end
|
|
146
|
+
return false if total.zero?
|
|
147
|
+
|
|
148
|
+
iquant * 2 > total
|
|
149
|
+
rescue Errno::ENOENT, ArgumentError, EOFError, KeyError
|
|
150
|
+
false
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def read_expert_bytes_per_layer
|
|
154
|
+
experts = @gguf.tensor_sizes.select { |name, _| name.match?(EXPERT_TENSOR) }
|
|
155
|
+
return nil if experts.empty?
|
|
156
|
+
|
|
157
|
+
layers = experts.keys.map { |name| name[EXPERT_TENSOR, 1] }.uniq.size
|
|
158
|
+
experts.values.sum / layers
|
|
159
|
+
rescue Errno::ENOENT, ArgumentError, EOFError
|
|
160
|
+
nil
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def arch_value
|
|
164
|
+
arch = metadata["general.architecture"]
|
|
165
|
+
arch && yield(metadata, arch)
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
require "pathname"
|
|
2
|
+
require_relative "../model"
|
|
3
|
+
require_relative "header"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
class Model
|
|
7
|
+
# Every GGUF under the configured search paths, sorted into what the
|
|
8
|
+
# configured engine can chat with, what `llm embed` owns, and what the
|
|
9
|
+
# engine's ternary packing rules leave out.
|
|
10
|
+
class Library
|
|
11
|
+
# A ternary GGUF is written for exactly one engine's block layout, and
|
|
12
|
+
# the wrong one either refuses it or reads garbage -- so the picker only
|
|
13
|
+
# offers the packing the chosen engine can read. ik_llama.cpp defines no
|
|
14
|
+
# Q2_0 at all, hence no entry.
|
|
15
|
+
ENGINE_TERNARY_PACKINGS = {
|
|
16
|
+
"llama.cpp" => %i[g64],
|
|
17
|
+
"prism-v7-llama.cpp" => %i[pq2 ptq1],
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
def initialize(config)
|
|
21
|
+
@config = config
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def all
|
|
25
|
+
@all ||= scan
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def chat = partitioned.first
|
|
29
|
+
|
|
30
|
+
# An embedding model answers /v1/embeddings and nothing else -- it has no
|
|
31
|
+
# chat template and no completions endpoint, so it belongs to `llm embed`
|
|
32
|
+
# rather than to the picker that chooses what the agents talk to.
|
|
33
|
+
def embedding = all.select(&:embedding?)
|
|
34
|
+
|
|
35
|
+
def chat_loadable_or_hidden = all.reject(&:embedding?)
|
|
36
|
+
|
|
37
|
+
def hidden = partitioned.last
|
|
38
|
+
|
|
39
|
+
# Why this model is not in the picker, for the caller that reports it.
|
|
40
|
+
# Kept next to `loadable?` so the two cannot drift into explaining
|
|
41
|
+
# different rules.
|
|
42
|
+
# Packing first: an engine that cannot read the weights at all outranks
|
|
43
|
+
# a judgement about which engine serves them faster.
|
|
44
|
+
def hidden_reason(model)
|
|
45
|
+
return "not a Prism Bonsai 2 ternary model" if prism_engine?
|
|
46
|
+
return "ternary weights #{@config.llama.engine} cannot read" unless packing_readable?(model)
|
|
47
|
+
return "MoE I-quant -- ik_llama.cpp has the kernels and the offload" if iquant_moe_under_llama?(model)
|
|
48
|
+
|
|
49
|
+
"dense -- llama.cpp serves these faster"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
def partitioned
|
|
55
|
+
@partitioned ||= chat_loadable_or_hidden.partition { |model| loadable?(model) }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The Prism engine exists only to serve its own ternary kernels --
|
|
59
|
+
# any ordinary GGUF it could also load belongs to llama.cpp/ik_llama.cpp
|
|
60
|
+
# instead, so its list narrows to its own packing types.
|
|
61
|
+
# ik_llama.cpp earns its place on MoE weights, where --n-cpu-moe puts
|
|
62
|
+
# experts on the CPU and nothing else here can. On a dense model that
|
|
63
|
+
# lever is inert and mainline wins outright -- measured on
|
|
64
|
+
# Qwen3.8-27B-UD-Q4_K_XL, 25.6-29.6 tok/s here against 50.1-51.2 on
|
|
65
|
+
# llama.cpp at the same ctx and quant, same quality score. Dense SWA
|
|
66
|
+
# models are worse still: ik has no windowed KV cache, so gemma-4-31B
|
|
67
|
+
# wants an order of magnitude more KV here than on mainline.
|
|
68
|
+
def dense_under_ik?(model)
|
|
69
|
+
@config.llama.engine == "ik_llama.cpp" && !model.moe?
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# A mainline I-quant on MoE weights is ik_llama.cpp's combination and
|
|
73
|
+
# mainline does not offer it. ik carries a dedicated CPU GEMM for that
|
|
74
|
+
# quant family (iqk_gemm_iquants.cpp, plus _R4/_R8 repacks) and
|
|
75
|
+
# --n-cpu-moe is what puts the weights in host RAM where those kernels
|
|
76
|
+
# run -- the two halves only engage together, and together they are what
|
|
77
|
+
# makes the big context reachable at all. Neither is on mainline: the
|
|
78
|
+
# IQ kernels are CPU-side and ik's CUDA path (iqk_mmvq.cu) does not
|
|
79
|
+
# touch the mainline IQ ids, so a fully resident run is unaffected.
|
|
80
|
+
def iquant_moe_under_llama?(model)
|
|
81
|
+
@config.llama.engine == "llama.cpp" && model.moe? && model.memory.iquant?
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def loadable?(model)
|
|
85
|
+
return false if dense_under_ik?(model)
|
|
86
|
+
return false if iquant_moe_under_llama?(model)
|
|
87
|
+
|
|
88
|
+
packing_readable?(model)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def packing_readable?(model)
|
|
92
|
+
packing = model.memory.ternary_packing
|
|
93
|
+
supported = ENGINE_TERNARY_PACKINGS[@config.llama.engine] || []
|
|
94
|
+
return supported.include?(packing) if prism_engine?
|
|
95
|
+
|
|
96
|
+
packing.nil? || supported.include?(packing)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def prism_engine? = @config.llama.engine == "prism-v7-llama.cpp"
|
|
100
|
+
|
|
101
|
+
def scan
|
|
102
|
+
found = []
|
|
103
|
+
seen = {}
|
|
104
|
+
@config.search_paths.each do |directory|
|
|
105
|
+
next unless directory.directory?
|
|
106
|
+
|
|
107
|
+
# Dir.glob returns each pattern's sorted matches in pattern order.
|
|
108
|
+
# `**` follows no symlinked folder, so its paths come first and win
|
|
109
|
+
# the realpath check; `*/**` adds what only a symlinked folder reaches.
|
|
110
|
+
in_directory = []
|
|
111
|
+
Dir.glob([directory / "**/*.gguf", directory / "*/**/*.gguf"]).uniq.each do |raw|
|
|
112
|
+
path = Pathname.new(raw)
|
|
113
|
+
|
|
114
|
+
# A dangling symlink still shows up in the glob; realpath is what
|
|
115
|
+
# proves the model is actually there.
|
|
116
|
+
real = begin
|
|
117
|
+
path.realpath
|
|
118
|
+
rescue SystemCallError
|
|
119
|
+
next
|
|
120
|
+
end
|
|
121
|
+
next if seen[real]
|
|
122
|
+
|
|
123
|
+
seen[real] = true
|
|
124
|
+
in_directory << [path, directory]
|
|
125
|
+
end
|
|
126
|
+
found.concat(in_directory.sort_by { |path, _| path.to_s })
|
|
127
|
+
end
|
|
128
|
+
Model.pair(found)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
require_relative "header"
|
|
2
|
+
require_relative "store"
|
|
3
|
+
require_relative "../llm_config"
|
|
4
|
+
|
|
5
|
+
module Mimic
|
|
6
|
+
class Model
|
|
7
|
+
# What the last good run of one model used: context, measured compute
|
|
8
|
+
# buffer, sampler values, MoE offload and projector. One sidecar
|
|
9
|
+
# JSON file per fact under ~/.llm, each keyed by resolved model path.
|
|
10
|
+
class Memory
|
|
11
|
+
N_CPU_MOE_PATH = Paths::LLM_ROOT / "ik-n-cpu-moe.json"
|
|
12
|
+
TERNARY_PACKING_PATH = Paths::LLM_ROOT / "model-packing.json"
|
|
13
|
+
|
|
14
|
+
def initialize(path, header = Header.new(path))
|
|
15
|
+
@path = path.to_s
|
|
16
|
+
@header = header
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def last_ctx
|
|
20
|
+
entry = ctx_store.fetch(@path)
|
|
21
|
+
entry && [entry["context_length"], entry["cache_type"]]
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def remember_ctx(context_length, cache_type)
|
|
25
|
+
ctx_store.write(@path, "context_length" => context_length, "cache_type" => cache_type)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def compute_mib(ctx, cache_type)
|
|
29
|
+
(compute_store.fetch(@path) || {})[compute_key(ctx, cache_type)]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def remember_compute(ctx, cache_type, mib)
|
|
33
|
+
entry = compute_store.fetch(@path) || {}
|
|
34
|
+
entry[compute_key(ctx, cache_type)] = mib
|
|
35
|
+
compute_store.write(@path, entry)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# What to launch with: the values the last successful run under this
|
|
39
|
+
# reasoning choice was given, or -- before there is one -- the publisher's
|
|
40
|
+
# suggestion from the header over the hardcoded defaults.
|
|
41
|
+
def sampling_for(reasoning:)
|
|
42
|
+
last_sampling(reasoning: reasoning) || resolved_sampling
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def last_sampling(reasoning:)
|
|
46
|
+
entry = sampling_store.fetch(@path)
|
|
47
|
+
entry["values"] if entry && entry["reasoning"] == reasoning
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def remember_sampling(reasoning, values)
|
|
51
|
+
sampling_store.write(@path, "reasoning" => reasoning, "values" => values)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def forget_sampling = sampling_store.delete(@path)
|
|
55
|
+
|
|
56
|
+
def n_cpu_moe = n_cpu_moe_store.fetch(@path)
|
|
57
|
+
def remember_n_cpu_moe(value) = n_cpu_moe_store.write(@path, value)
|
|
58
|
+
def forget_n_cpu_moe = n_cpu_moe_store.delete(@path)
|
|
59
|
+
|
|
60
|
+
# Both facts live in the tensor block, which sits behind the vocab
|
|
61
|
+
# array -- reaching it costs a full KV walk per model, and the setup
|
|
62
|
+
# picker asks for every model it lists. Cached together against size and
|
|
63
|
+
# mtime so a rebuilt or replaced file is re-read.
|
|
64
|
+
def ternary_packing = tensor_facts["packing"]&.to_sym
|
|
65
|
+
|
|
66
|
+
def iquant? = tensor_facts["iquant"] == true
|
|
67
|
+
|
|
68
|
+
def effort = effort_store.fetch(@path)
|
|
69
|
+
|
|
70
|
+
def effort=(value)
|
|
71
|
+
value.nil? ? effort_store.delete(@path) : effort_store.write(@path, value)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def mmproj = mmproj_store.fetch(@path)
|
|
75
|
+
|
|
76
|
+
def mmproj=(value)
|
|
77
|
+
value.nil? ? mmproj_store.delete(@path) : mmproj_store.write(@path, value)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def compute_key(ctx, cache_type) = "#{ctx}:#{cache_type}"
|
|
83
|
+
|
|
84
|
+
# An entry written before iquant existed carries no such key, so it is
|
|
85
|
+
# re-read rather than answered as false.
|
|
86
|
+
def tensor_facts
|
|
87
|
+
return @tensor_facts if defined?(@tensor_facts)
|
|
88
|
+
|
|
89
|
+
stamp = file_stamp
|
|
90
|
+
return @tensor_facts = read_tensor_facts if stamp.nil?
|
|
91
|
+
|
|
92
|
+
entry = packing_store.fetch(@path)
|
|
93
|
+
return @tensor_facts = entry if entry && entry.slice("size", "mtime") == stamp && entry.key?("iquant")
|
|
94
|
+
|
|
95
|
+
@tensor_facts = read_tensor_facts
|
|
96
|
+
packing_store.write(@path, stamp.merge(@tensor_facts))
|
|
97
|
+
@tensor_facts
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def read_tensor_facts
|
|
101
|
+
{ "packing" => @header.ternary_packing&.to_s, "iquant" => @header.iquant? }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def file_stamp
|
|
105
|
+
stat = File.stat(@path)
|
|
106
|
+
{ "size" => stat.size, "mtime" => stat.mtime.to_i }
|
|
107
|
+
rescue SystemCallError
|
|
108
|
+
nil
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def resolved_sampling
|
|
112
|
+
LlmConfig::Sampling.resolve(@header.metadata)
|
|
113
|
+
rescue ArgumentError, SystemCallError, IOError
|
|
114
|
+
LlmConfig::Sampling::DEFAULTS
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def ctx_store = Store.new(LlmConfig::MODEL_CTX_PATH)
|
|
118
|
+
def sampling_store = Store.new(LlmConfig::MODEL_SAMPLING_PATH)
|
|
119
|
+
def compute_store = Store.new(LlmConfig::MODEL_COMPUTE_PATH)
|
|
120
|
+
def n_cpu_moe_store = Store.new(N_CPU_MOE_PATH)
|
|
121
|
+
def packing_store = Store.new(TERNARY_PACKING_PATH)
|
|
122
|
+
def mmproj_store = Store.new(LlmConfig::MODEL_MMPROJ_PATH)
|
|
123
|
+
def effort_store = Store.new(LlmConfig::MODEL_EFFORT_PATH)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require_relative "../llm_config"
|
|
3
|
+
|
|
4
|
+
module Mimic
|
|
5
|
+
class Model
|
|
6
|
+
# Shared read/merge/write over a sidecar JSON file keyed by resolved
|
|
7
|
+
# model path. Value shape is the caller's business -- Store only owns the
|
|
8
|
+
# file and the key resolution.
|
|
9
|
+
class Store
|
|
10
|
+
def initialize(path)
|
|
11
|
+
@path = path
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def fetch(model_path) = read[key(model_path)]
|
|
15
|
+
|
|
16
|
+
def write(model_path, value)
|
|
17
|
+
known = read
|
|
18
|
+
known[key(model_path)] = value
|
|
19
|
+
persist(known)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def delete(model_path)
|
|
23
|
+
known = read
|
|
24
|
+
return unless known.key?(key(model_path))
|
|
25
|
+
|
|
26
|
+
known.delete(key(model_path))
|
|
27
|
+
persist(known)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def key(model_path) = LlmConfig.resolve_path(model_path)
|
|
33
|
+
|
|
34
|
+
def read
|
|
35
|
+
JSON.parse(@path.read)
|
|
36
|
+
rescue SystemCallError, JSON::ParserError
|
|
37
|
+
{}
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def persist(known)
|
|
41
|
+
@path.dirname.mkpath
|
|
42
|
+
@path.write(JSON.pretty_generate(known))
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|