mimic-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/NOTICE +16 -0
  4. data/README.md +175 -0
  5. data/exe/lllm +5 -0
  6. data/exe/llm +5 -0
  7. data/exe/mimic +5 -0
  8. data/lib/mimic/README.md +144 -0
  9. data/lib/mimic/agent/claude.rb +305 -0
  10. data/lib/mimic/agent/codex.rb +254 -0
  11. data/lib/mimic/agent/codex_fallback_prompt.md +275 -0
  12. data/lib/mimic/agent/hermes.rb +107 -0
  13. data/lib/mimic/agent/muse.rb +106 -0
  14. data/lib/mimic/agent.rb +46 -0
  15. data/lib/mimic/cli.rb +159 -0
  16. data/lib/mimic/client/base.rb +44 -0
  17. data/lib/mimic/client/crush.rb +28 -0
  18. data/lib/mimic/client/goose.rb +52 -0
  19. data/lib/mimic/client/hermes.rb +21 -0
  20. data/lib/mimic/client/opencode.rb +29 -0
  21. data/lib/mimic/client/pi.rb +50 -0
  22. data/lib/mimic/client.rb +19 -0
  23. data/lib/mimic/engine/README.md +140 -0
  24. data/lib/mimic/engine/base.rb +138 -0
  25. data/lib/mimic/engine/chat_template.rb +207 -0
  26. data/lib/mimic/engine/ik_llama.rb +273 -0
  27. data/lib/mimic/engine/launcher.rb +25 -0
  28. data/lib/mimic/engine/llama.rb +124 -0
  29. data/lib/mimic/engine/llama_embed.rb +108 -0
  30. data/lib/mimic/engine/router.rb +57 -0
  31. data/lib/mimic/engine/server_process.rb +108 -0
  32. data/lib/mimic/engine.rb +5 -0
  33. data/lib/mimic/engine_cli/binary_options.rb +46 -0
  34. data/lib/mimic/engine_cli/builder.rb +98 -0
  35. data/lib/mimic/engine_cli/diffusion.rb +117 -0
  36. data/lib/mimic/engine_cli/llama_preset.rb +119 -0
  37. data/lib/mimic/engine_cli/registry.rb +94 -0
  38. data/lib/mimic/engine_cli/source.rb +110 -0
  39. data/lib/mimic/engine_cli/status.rb +161 -0
  40. data/lib/mimic/engine_cli/update.rb +68 -0
  41. data/lib/mimic/engine_cli.rb +7 -0
  42. data/lib/mimic/health/checks.rb +92 -0
  43. data/lib/mimic/health.rb +82 -0
  44. data/lib/mimic/llm_cli/base.rb +36 -0
  45. data/lib/mimic/llm_cli/configure.rb +84 -0
  46. data/lib/mimic/llm_cli/embed.rb +80 -0
  47. data/lib/mimic/llm_cli/engines.rb +44 -0
  48. data/lib/mimic/llm_cli/models.rb +31 -0
  49. data/lib/mimic/llm_cli/models_table.rb +119 -0
  50. data/lib/mimic/llm_cli/paths.rb +48 -0
  51. data/lib/mimic/llm_cli/root.rb +215 -0
  52. data/lib/mimic/llm_cli/wizard/context_preflight.rb +82 -0
  53. data/lib/mimic/llm_cli/wizard/embed.rb +89 -0
  54. data/lib/mimic/llm_cli/wizard/moe.rb +82 -0
  55. data/lib/mimic/llm_cli/wizard/parallel.rb +76 -0
  56. data/lib/mimic/llm_cli/wizard/setup.rb +211 -0
  57. data/lib/mimic/llm_cli.rb +5 -0
  58. data/lib/mimic/llm_config/embed.rb +48 -0
  59. data/lib/mimic/llm_config/llama_settings.rb +60 -0
  60. data/lib/mimic/llm_config/model_settings.rb +84 -0
  61. data/lib/mimic/llm_config/sampling.rb +54 -0
  62. data/lib/mimic/llm_config/settings.rb +92 -0
  63. data/lib/mimic/llm_config.rb +132 -0
  64. data/lib/mimic/model/footprint.rb +303 -0
  65. data/lib/mimic/model/gguf.rb +204 -0
  66. data/lib/mimic/model/header.rb +169 -0
  67. data/lib/mimic/model/library.rb +132 -0
  68. data/lib/mimic/model/memory.rb +126 -0
  69. data/lib/mimic/model/store.rb +46 -0
  70. data/lib/mimic/model/upstream.rb +81 -0
  71. data/lib/mimic/model.rb +116 -0
  72. data/lib/mimic/paths.rb +45 -0
  73. data/lib/mimic/result.rb +8 -0
  74. data/lib/mimic/ui.rb +89 -0
  75. data/lib/mimic/util/fmt.rb +15 -0
  76. data/lib/mimic/util/hf.rb +55 -0
  77. data/lib/mimic/util/host.rb +95 -0
  78. data/lib/mimic/util/probes.rb +255 -0
  79. data/lib/mimic/version.rb +3 -0
  80. data/lib/mimic.rb +22 -0
  81. metadata +186 -0
@@ -0,0 +1,204 @@
1
+ module Mimic
2
+ class Model
3
+ # GGUF header reader -- the key/value metadata block and the tensor info
4
+ # block at the head of a model file. Tensor data is never touched; every
5
+ # question here is answered from the first few hundred KB.
6
+ #
7
+ # Raises for a file that is missing, truncated or not a GGUF at all;
8
+ # Model::Header is what turns those into nil.
9
+ class Gguf
10
+ MAGIC = "GGUF"
11
+ SCALAR_SIZE = { 0 => 1, 1 => 1, 2 => 2, 3 => 2, 4 => 4, 5 => 4, 6 => 4, 7 => 1, 10 => 8, 11 => 8, 12 => 8 }.freeze
12
+ SCALAR_FMT = {
13
+ 0 => "C", 1 => "c", 2 => "S<", 3 => "s<", 4 => "L<", 5 => "l<",
14
+ 6 => "e", 7 => "C", 10 => "Q<", 11 => "q<", 12 => "E",
15
+ }.freeze
16
+ STRING = 8
17
+ ARRAY = 9
18
+
19
+ # A string array this long is a vocab/merges table: megabytes of
20
+ # length-prefixed entries that can only be walked one at a time. No caller
21
+ # needs them, so the parse stops there and restarts in full only if the
22
+ # architecture block turned out to sit behind them.
23
+ BIG_STRING_ARRAY = 4096
24
+ CHUNK = 1 << 20
25
+
26
+ # Numeric arrays this short are per-layer shape (head_count_kv, the
27
+ # sliding-window pattern): one entry per block, kept because the KV
28
+ # estimate cannot be built without them. Anything longer is vocab-sized
29
+ # (token types, merge scores) and is skipped like the string tables.
30
+ BIG_SCALAR_ARRAY = 4096
31
+
32
+ def initialize(path)
33
+ @path = path.to_s
34
+ @metadata = {}
35
+ end
36
+
37
+ # Scalar, string and short-numeric-array KV pairs. Vocab-sized arrays
38
+ # are skipped.
39
+ def metadata(full: false)
40
+ @metadata[full] ||= read_metadata(full: full)
41
+ end
42
+
43
+ # [{name:, elements:, type:, offset:}, ...] from the tensor info block
44
+ # that follows the KV block.
45
+ def tensors
46
+ return @tensors if defined?(@tensors)
47
+
48
+ @tensors, @data_start = read_tensors
49
+ @tensors
50
+ end
51
+
52
+ # {name => bytes}. Offsets are relative to the start of the data block,
53
+ # so the last tensor is bounded by the tail of the file rather than by a
54
+ # following offset.
55
+ def tensor_sizes
56
+ @tensor_sizes ||= sizes_from_gaps(tensors, File.size(@path) - @data_start)
57
+ end
58
+
59
+ private
60
+
61
+ def read_metadata(full:)
62
+ meta = {}
63
+ truncated = false
64
+ File.open(@path, "rb") do |f|
65
+ raise ArgumentError, "not a GGUF file" unless f.read(4) == MAGIC
66
+
67
+ u32(f) # version (v2/v3 both use u64 counts below)
68
+ u64(f) # tensor count
69
+ u64(f).times do
70
+ key = read_str(f)
71
+ vtype = u32(f)
72
+ if SCALAR_FMT.key?(vtype)
73
+ meta[key] = read_scalar(f, vtype)
74
+ elsif vtype == STRING
75
+ meta[key] = read_str(f)
76
+ elsif vtype == ARRAY
77
+ elem_type = u32(f)
78
+ count = u64(f)
79
+ if SCALAR_FMT.key?(elem_type) && count <= BIG_SCALAR_ARRAY
80
+ meta[key] = read_scalar_array(f, elem_type, count)
81
+ elsif !skip_array(f, elem_type, count, full: full)
82
+ truncated = true
83
+ break
84
+ end
85
+ else
86
+ raise ArgumentError, "unknown gguf type #{vtype}"
87
+ end
88
+ end
89
+ end
90
+ return meta unless truncated && !arch_block_read?(meta)
91
+
92
+ read_metadata(full: true)
93
+ end
94
+
95
+ # The KV walk has to be exhaustive here -- the early stop metadata takes
96
+ # on a vocab-sized array would leave the file positioned mid-header.
97
+ def read_tensors
98
+ File.open(@path, "rb") do |f|
99
+ raise ArgumentError, "not a GGUF file" unless f.read(4) == MAGIC
100
+
101
+ u32(f)
102
+ n_tensors = u64(f)
103
+ u64(f).times { read_str(f); skip_value(f, u32(f)) }
104
+ tensors = Array.new(n_tensors) do
105
+ name = read_str(f)
106
+ elements = Array.new(u32(f)) { u64(f) }.inject(1, :*)
107
+ { name: name, elements: elements, type: u32(f), offset: u64(f) }
108
+ end
109
+ [tensors, f.pos]
110
+ end
111
+ end
112
+
113
+ def sizes_from_gaps(tensors, data_bytes)
114
+ ordered = tensors.sort_by { |tensor| tensor[:offset] }
115
+ ordered.each_with_index.to_h do |tensor, i|
116
+ following = ordered[i + 1]
117
+ [tensor[:name], (following ? following[:offset] : data_bytes) - tensor[:offset]]
118
+ end
119
+ end
120
+
121
+ def u32(f) = read_exactly(f, 4).unpack1("L<")
122
+ def u64(f) = read_exactly(f, 8).unpack1("Q<")
123
+ def read_str(f) = read_exactly(f, u64(f)).force_encoding("UTF-8").scrub
124
+
125
+ def read_scalar_array(f, elem_type, count)
126
+ read_exactly(f, SCALAR_SIZE.fetch(elem_type) * count).unpack("#{SCALAR_FMT.fetch(elem_type)}*")
127
+ end
128
+
129
+ def read_scalar(f, vtype)
130
+ read_exactly(f, SCALAR_SIZE.fetch(vtype)).unpack1(SCALAR_FMT.fetch(vtype))
131
+ end
132
+
133
+ # A header cut mid-value leaves IO#read returning short or nil, which
134
+ # would surface as NoMethodError on the unpack that follows.
135
+ def read_exactly(f, bytes)
136
+ chunk = f.read(bytes)
137
+ raise EOFError, "truncated gguf header" if chunk.nil? || chunk.bytesize < bytes
138
+
139
+ chunk
140
+ end
141
+
142
+ def arch_block_read?(meta)
143
+ arch = meta["general.architecture"]
144
+ !arch.nil? && meta.keys.any? { |key| key.start_with?("#{arch}.") }
145
+ end
146
+
147
+ def skip_string_array(f, count)
148
+ buf = "".b
149
+ off = 0
150
+ base = f.pos
151
+ count.times do
152
+ if buf.bytesize - off < 8
153
+ base += off
154
+ f.seek(base)
155
+ buf = f.read(CHUNK) || "".b
156
+ off = 0
157
+ raise EOFError, "truncated gguf string array" if buf.bytesize < 8
158
+ end
159
+ len = buf.unpack1("Q<", offset: off)
160
+ off += 8
161
+ if len > buf.bytesize - off
162
+ base += off + len
163
+ f.seek(base)
164
+ buf = "".b
165
+ off = 0
166
+ else
167
+ off += len
168
+ end
169
+ end
170
+ f.seek(base + off)
171
+ end
172
+
173
+ # Returns false when the value was a vocab-sized string array and `full`
174
+ # is off -- see skip_array.
175
+ def skip_value(f, vtype, full: true)
176
+ if SCALAR_SIZE.key?(vtype)
177
+ f.seek(SCALAR_SIZE[vtype], IO::SEEK_CUR)
178
+ elsif vtype == STRING
179
+ f.seek(u64(f), IO::SEEK_CUR)
180
+ elsif vtype == ARRAY
181
+ return skip_array(f, u32(f), u64(f), full: full)
182
+ else
183
+ raise ArgumentError, "unknown gguf type #{vtype}"
184
+ end
185
+ true
186
+ end
187
+
188
+ # Returns false when the value was a vocab-sized string array and `full`
189
+ # is off -- the parse stops there.
190
+ def skip_array(f, elem_type, count, full: true)
191
+ if SCALAR_SIZE.key?(elem_type)
192
+ f.seek(SCALAR_SIZE[elem_type] * count, IO::SEEK_CUR)
193
+ elsif elem_type == STRING
194
+ return false if !full && count > BIG_STRING_ARRAY
195
+
196
+ skip_string_array(f, count)
197
+ else
198
+ raise ArgumentError, "nested array type #{elem_type}"
199
+ end
200
+ true
201
+ end
202
+ end
203
+ end
204
+ end
@@ -0,0 +1,169 @@
1
+ require_relative "gguf"
2
+
3
+ module Mimic
4
+ class Model
5
+ # Questions about one GGUF's header. Everything is answered off a single
6
+ # parse, and a file that is missing, truncated or not a GGUF answers nil
7
+ # rather than raising -- callers ask about models found by a directory
8
+ # scan, where an unreadable file is ordinary.
9
+ class Header
10
+ EXPERT_TENSOR = /\Ablk\.(\d+)\..*exps/
11
+
12
+ # Ternary (Bonsai) weights. ggml type 42 is the trap: mainline llama.cpp
13
+ # packs 64 weights into an 18-byte block, PrismML's fork 128 into 34, and
14
+ # neither reads the other's file -- under the same type id, so only the
15
+ # measured bytes-per-weight tells them apart. Ids above what any engine
16
+ # here defines are rejected unless they are current Prism ternary types.
17
+ TERNARY_TYPE = 42
18
+ PRISM_TYPES = { 142 => :pq2, 143 => :ptq1 }.freeze
19
+ MAX_COMMON_TYPE = 42
20
+ G128_BYTES_PER_WEIGHT = 34 / 128.0
21
+ G64_BYTES_PER_WEIGHT = 18 / 64.0
22
+
23
+ # Mainline llama.cpp's I-quants. ik_llama.cpp carries a dedicated CPU
24
+ # GEMM for exactly this family (iqk_gemm_iquants.cpp) plus interleaved
25
+ # _R4/_R8 repacks of it, which is why the pairing with MoE offload is
26
+ # ik's alone. ik's own IQx_K ids are absent on purpose: mainline cannot
27
+ # load those at all, so there is no engine choice left to make.
28
+ IQUANT_TYPES = [16, 17, 18, 19, 20, 21, 22, 23, 29].freeze
29
+
30
+ # Architectures that only ever ship as encoders. A decoder-shaped
31
+ # embedding model (Qwen3-Embedding and friends) keeps its chat
32
+ # architecture and is caught by the pooling/causal keys instead.
33
+ ENCODER_ARCHS = %w[bert nomic-bert nomic-bert-moe jina-bert-v2 modern-bert neo-bert xlm-roberta t5encoder].freeze
34
+ POOLING_RANK = 4
35
+
36
+ MTP_SCAN_BYTES = 8_000_000
37
+
38
+ def initialize(path)
39
+ @path = path.to_s
40
+ @gguf = Gguf.new(@path)
41
+ end
42
+
43
+ def moe? = arch_value { |meta, arch| !meta.fetch("#{arch}.expert_count", 0).zero? }
44
+
45
+ def embedding?
46
+ arch_value do |meta, arch|
47
+ meta["#{arch}.attention.causal"] == 0 ||
48
+ meta.key?("#{arch}.pooling_type") ||
49
+ ENCODER_ARCHS.include?(arch)
50
+ end
51
+ end
52
+
53
+ # MTP-capable GGUFs carry a *.nextn_predict_layers key. Read as raw
54
+ # bytes rather than through the parsed metadata: the key is
55
+ # architecture-prefixed and a model whose header will not parse at all
56
+ # still answers false instead of raising out of render_preset.
57
+ def supports_mtp?
58
+ File.open(@path, "rb") { |f| f.read(MTP_SCAN_BYTES).to_s.include?("nextn_predict_layers") }
59
+ rescue SystemCallError, IOError
60
+ false
61
+ end
62
+
63
+ def pooling_type = arch_value { |meta, arch| meta["#{arch}.pooling_type"] }
64
+ def block_count = arch_value { |meta, arch| meta["#{arch}.block_count"] }
65
+
66
+ # How many of block_count's blocks are MTP draft heads rather than
67
+ # transformer layers. Counted out of the KV estimate unless the engine
68
+ # is actually drafting, and counted back in when it is -- a nextn block
69
+ # carries its own full attention.
70
+ def nextn_layers = arch_value { |meta, arch| meta["#{arch}.nextn_predict_layers"] } || 0
71
+
72
+ # Which engine's ternary packing this model carries: :g128 (PrismML
73
+ # v5 fork), :g64 (mainline llama.cpp), :pq2 or :ptq1 (current Prism
74
+ # fork), :unsupported (an unknown or mixed private type), or nil.
75
+ def ternary_packing
76
+ return @ternary_packing if defined?(@ternary_packing)
77
+
78
+ @ternary_packing = read_ternary_packing
79
+ end
80
+
81
+ # Whether the weights are predominantly a mainline I-quant. Judged on
82
+ # share of tensor bytes, not on any single tensor: a dynamic quant
83
+ # ("UD-IQ4_XS") mixes types per layer and its widest tensor is always
84
+ # the Q8_0/Q5_K token_embd or output, so the widest one names the file
85
+ # wrongly every time. Measured here: UD-IQ4_NL is 80% IQ by bytes,
86
+ # UD-IQ4_XS 68%, both behind a non-IQ widest tensor.
87
+ def iquant?
88
+ return @iquant if defined?(@iquant)
89
+
90
+ @iquant = read_iquant
91
+ end
92
+
93
+ # Average bytes of expert weights per MoE layer -- what moving one
94
+ # layer's experts to CPU with --n-cpu-moe frees on the GPU. nil for a
95
+ # dense model or an unreadable file.
96
+ #
97
+ # Sizes come from the gaps between consecutive tensor offsets rather
98
+ # than from shape x type, which would need the whole ggml block-size
99
+ # table to be correct for every quant.
100
+ def expert_bytes_per_layer
101
+ return @expert_bytes_per_layer if defined?(@expert_bytes_per_layer)
102
+
103
+ @expert_bytes_per_layer = read_expert_bytes_per_layer
104
+ end
105
+
106
+ def metadata(full: false)
107
+ @metadata ||= {}
108
+ @metadata[full] ||= @gguf.metadata(full: full)
109
+ rescue Errno::ENOENT, ArgumentError, EOFError
110
+ @metadata[full] = {}
111
+ end
112
+
113
+ private
114
+
115
+ def read_ternary_packing
116
+ tensors = @gguf.tensors
117
+ return nil if tensors.empty?
118
+ return :unsupported if tensors.any? { |tensor| tensor[:type] > MAX_COMMON_TYPE && !PRISM_TYPES.key?(tensor[:type]) }
119
+ prism_packings = tensors.filter_map { |tensor| PRISM_TYPES[tensor[:type]] }.uniq
120
+ return :unsupported if prism_packings.size > 1
121
+ return prism_packings.first unless prism_packings.empty?
122
+
123
+ ternary = tensors.select { |tensor| tensor[:type] == TERNARY_TYPE }
124
+ return nil if ternary.empty?
125
+
126
+ # The gap to the next tensor includes GGUF's alignment padding, so
127
+ # the biggest tensor is the one whose bytes-per-weight is least
128
+ # distorted by it.
129
+ sizes = @gguf.tensor_sizes
130
+ widest = ternary.max_by { |tensor| sizes.fetch(tensor[:name]) }
131
+ per_weight = sizes.fetch(widest[:name]).to_f / widest[:elements]
132
+ (per_weight - G128_BYTES_PER_WEIGHT).abs < (per_weight - G64_BYTES_PER_WEIGHT).abs ? :g128 : :g64
133
+ rescue Errno::ENOENT, ArgumentError, EOFError
134
+ nil
135
+ end
136
+
137
+ def read_iquant
138
+ sizes = @gguf.tensor_sizes
139
+ total = 0
140
+ iquant = 0
141
+ @gguf.tensors.each do |tensor|
142
+ bytes = sizes.fetch(tensor[:name])
143
+ total += bytes
144
+ iquant += bytes if IQUANT_TYPES.include?(tensor[:type])
145
+ end
146
+ return false if total.zero?
147
+
148
+ iquant * 2 > total
149
+ rescue Errno::ENOENT, ArgumentError, EOFError, KeyError
150
+ false
151
+ end
152
+
153
+ def read_expert_bytes_per_layer
154
+ experts = @gguf.tensor_sizes.select { |name, _| name.match?(EXPERT_TENSOR) }
155
+ return nil if experts.empty?
156
+
157
+ layers = experts.keys.map { |name| name[EXPERT_TENSOR, 1] }.uniq.size
158
+ experts.values.sum / layers
159
+ rescue Errno::ENOENT, ArgumentError, EOFError
160
+ nil
161
+ end
162
+
163
+ def arch_value
164
+ arch = metadata["general.architecture"]
165
+ arch && yield(metadata, arch)
166
+ end
167
+ end
168
+ end
169
+ end
@@ -0,0 +1,132 @@
1
+ require "pathname"
2
+ require_relative "../model"
3
+ require_relative "header"
4
+
5
+ module Mimic
6
+ class Model
7
+ # Every GGUF under the configured search paths, sorted into what the
8
+ # configured engine can chat with, what `llm embed` owns, and what the
9
+ # engine's ternary packing rules leave out.
10
+ class Library
11
+ # A ternary GGUF is written for exactly one engine's block layout, and
12
+ # the wrong one either refuses it or reads garbage -- so the picker only
13
+ # offers the packing the chosen engine can read. ik_llama.cpp defines no
14
+ # Q2_0 at all, hence no entry.
15
+ ENGINE_TERNARY_PACKINGS = {
16
+ "llama.cpp" => %i[g64],
17
+ "prism-v7-llama.cpp" => %i[pq2 ptq1],
18
+ }.freeze
19
+
20
+ def initialize(config)
21
+ @config = config
22
+ end
23
+
24
+ def all
25
+ @all ||= scan
26
+ end
27
+
28
+ def chat = partitioned.first
29
+
30
+ # An embedding model answers /v1/embeddings and nothing else -- it has no
31
+ # chat template and no completions endpoint, so it belongs to `llm embed`
32
+ # rather than to the picker that chooses what the agents talk to.
33
+ def embedding = all.select(&:embedding?)
34
+
35
+ def chat_loadable_or_hidden = all.reject(&:embedding?)
36
+
37
+ def hidden = partitioned.last
38
+
39
+ # Why this model is not in the picker, for the caller that reports it.
40
+ # Kept next to `loadable?` so the two cannot drift into explaining
41
+ # different rules.
42
+ # Packing first: an engine that cannot read the weights at all outranks
43
+ # a judgement about which engine serves them faster.
44
+ def hidden_reason(model)
45
+ return "not a Prism Bonsai 2 ternary model" if prism_engine?
46
+ return "ternary weights #{@config.llama.engine} cannot read" unless packing_readable?(model)
47
+ return "MoE I-quant -- ik_llama.cpp has the kernels and the offload" if iquant_moe_under_llama?(model)
48
+
49
+ "dense -- llama.cpp serves these faster"
50
+ end
51
+
52
+ private
53
+
54
+ def partitioned
55
+ @partitioned ||= chat_loadable_or_hidden.partition { |model| loadable?(model) }
56
+ end
57
+
58
+ # The Prism engine exists only to serve its own ternary kernels --
59
+ # any ordinary GGUF it could also load belongs to llama.cpp/ik_llama.cpp
60
+ # instead, so its list narrows to its own packing types.
61
+ # ik_llama.cpp earns its place on MoE weights, where --n-cpu-moe puts
62
+ # experts on the CPU and nothing else here can. On a dense model that
63
+ # lever is inert and mainline wins outright -- measured on
64
+ # Qwen3.8-27B-UD-Q4_K_XL, 25.6-29.6 tok/s here against 50.1-51.2 on
65
+ # llama.cpp at the same ctx and quant, same quality score. Dense SWA
66
+ # models are worse still: ik has no windowed KV cache, so gemma-4-31B
67
+ # wants an order of magnitude more KV here than on mainline.
68
+ def dense_under_ik?(model)
69
+ @config.llama.engine == "ik_llama.cpp" && !model.moe?
70
+ end
71
+
72
+ # A mainline I-quant on MoE weights is ik_llama.cpp's combination and
73
+ # mainline does not offer it. ik carries a dedicated CPU GEMM for that
74
+ # quant family (iqk_gemm_iquants.cpp, plus _R4/_R8 repacks) and
75
+ # --n-cpu-moe is what puts the weights in host RAM where those kernels
76
+ # run -- the two halves only engage together, and together they are what
77
+ # makes the big context reachable at all. Neither is on mainline: the
78
+ # IQ kernels are CPU-side and ik's CUDA path (iqk_mmvq.cu) does not
79
+ # touch the mainline IQ ids, so a fully resident run is unaffected.
80
+ def iquant_moe_under_llama?(model)
81
+ @config.llama.engine == "llama.cpp" && model.moe? && model.memory.iquant?
82
+ end
83
+
84
+ def loadable?(model)
85
+ return false if dense_under_ik?(model)
86
+ return false if iquant_moe_under_llama?(model)
87
+
88
+ packing_readable?(model)
89
+ end
90
+
91
+ def packing_readable?(model)
92
+ packing = model.memory.ternary_packing
93
+ supported = ENGINE_TERNARY_PACKINGS[@config.llama.engine] || []
94
+ return supported.include?(packing) if prism_engine?
95
+
96
+ packing.nil? || supported.include?(packing)
97
+ end
98
+
99
+ def prism_engine? = @config.llama.engine == "prism-v7-llama.cpp"
100
+
101
+ def scan
102
+ found = []
103
+ seen = {}
104
+ @config.search_paths.each do |directory|
105
+ next unless directory.directory?
106
+
107
+ # Dir.glob returns each pattern's sorted matches in pattern order.
108
+ # `**` follows no symlinked folder, so its paths come first and win
109
+ # the realpath check; `*/**` adds what only a symlinked folder reaches.
110
+ in_directory = []
111
+ Dir.glob([directory / "**/*.gguf", directory / "*/**/*.gguf"]).uniq.each do |raw|
112
+ path = Pathname.new(raw)
113
+
114
+ # A dangling symlink still shows up in the glob; realpath is what
115
+ # proves the model is actually there.
116
+ real = begin
117
+ path.realpath
118
+ rescue SystemCallError
119
+ next
120
+ end
121
+ next if seen[real]
122
+
123
+ seen[real] = true
124
+ in_directory << [path, directory]
125
+ end
126
+ found.concat(in_directory.sort_by { |path, _| path.to_s })
127
+ end
128
+ Model.pair(found)
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,126 @@
1
+ require_relative "header"
2
+ require_relative "store"
3
+ require_relative "../llm_config"
4
+
5
+ module Mimic
6
+ class Model
7
+ # What the last good run of one model used: context, measured compute
8
+ # buffer, sampler values, MoE offload and projector. One sidecar
9
+ # JSON file per fact under ~/.llm, each keyed by resolved model path.
10
+ class Memory
11
+ N_CPU_MOE_PATH = Paths::LLM_ROOT / "ik-n-cpu-moe.json"
12
+ TERNARY_PACKING_PATH = Paths::LLM_ROOT / "model-packing.json"
13
+
14
+ def initialize(path, header = Header.new(path))
15
+ @path = path.to_s
16
+ @header = header
17
+ end
18
+
19
+ def last_ctx
20
+ entry = ctx_store.fetch(@path)
21
+ entry && [entry["context_length"], entry["cache_type"]]
22
+ end
23
+
24
+ def remember_ctx(context_length, cache_type)
25
+ ctx_store.write(@path, "context_length" => context_length, "cache_type" => cache_type)
26
+ end
27
+
28
+ def compute_mib(ctx, cache_type)
29
+ (compute_store.fetch(@path) || {})[compute_key(ctx, cache_type)]
30
+ end
31
+
32
+ def remember_compute(ctx, cache_type, mib)
33
+ entry = compute_store.fetch(@path) || {}
34
+ entry[compute_key(ctx, cache_type)] = mib
35
+ compute_store.write(@path, entry)
36
+ end
37
+
38
+ # What to launch with: the values the last successful run under this
39
+ # reasoning choice was given, or -- before there is one -- the publisher's
40
+ # suggestion from the header over the hardcoded defaults.
41
+ def sampling_for(reasoning:)
42
+ last_sampling(reasoning: reasoning) || resolved_sampling
43
+ end
44
+
45
+ def last_sampling(reasoning:)
46
+ entry = sampling_store.fetch(@path)
47
+ entry["values"] if entry && entry["reasoning"] == reasoning
48
+ end
49
+
50
+ def remember_sampling(reasoning, values)
51
+ sampling_store.write(@path, "reasoning" => reasoning, "values" => values)
52
+ end
53
+
54
+ def forget_sampling = sampling_store.delete(@path)
55
+
56
+ def n_cpu_moe = n_cpu_moe_store.fetch(@path)
57
+ def remember_n_cpu_moe(value) = n_cpu_moe_store.write(@path, value)
58
+ def forget_n_cpu_moe = n_cpu_moe_store.delete(@path)
59
+
60
+ # Both facts live in the tensor block, which sits behind the vocab
61
+ # array -- reaching it costs a full KV walk per model, and the setup
62
+ # picker asks for every model it lists. Cached together against size and
63
+ # mtime so a rebuilt or replaced file is re-read.
64
+ def ternary_packing = tensor_facts["packing"]&.to_sym
65
+
66
+ def iquant? = tensor_facts["iquant"] == true
67
+
68
+ def effort = effort_store.fetch(@path)
69
+
70
+ def effort=(value)
71
+ value.nil? ? effort_store.delete(@path) : effort_store.write(@path, value)
72
+ end
73
+
74
+ def mmproj = mmproj_store.fetch(@path)
75
+
76
+ def mmproj=(value)
77
+ value.nil? ? mmproj_store.delete(@path) : mmproj_store.write(@path, value)
78
+ end
79
+
80
+ private
81
+
82
+ def compute_key(ctx, cache_type) = "#{ctx}:#{cache_type}"
83
+
84
+ # An entry written before iquant existed carries no such key, so it is
85
+ # re-read rather than answered as false.
86
+ def tensor_facts
87
+ return @tensor_facts if defined?(@tensor_facts)
88
+
89
+ stamp = file_stamp
90
+ return @tensor_facts = read_tensor_facts if stamp.nil?
91
+
92
+ entry = packing_store.fetch(@path)
93
+ return @tensor_facts = entry if entry && entry.slice("size", "mtime") == stamp && entry.key?("iquant")
94
+
95
+ @tensor_facts = read_tensor_facts
96
+ packing_store.write(@path, stamp.merge(@tensor_facts))
97
+ @tensor_facts
98
+ end
99
+
100
+ def read_tensor_facts
101
+ { "packing" => @header.ternary_packing&.to_s, "iquant" => @header.iquant? }
102
+ end
103
+
104
+ def file_stamp
105
+ stat = File.stat(@path)
106
+ { "size" => stat.size, "mtime" => stat.mtime.to_i }
107
+ rescue SystemCallError
108
+ nil
109
+ end
110
+
111
+ def resolved_sampling
112
+ LlmConfig::Sampling.resolve(@header.metadata)
113
+ rescue ArgumentError, SystemCallError, IOError
114
+ LlmConfig::Sampling::DEFAULTS
115
+ end
116
+
117
+ def ctx_store = Store.new(LlmConfig::MODEL_CTX_PATH)
118
+ def sampling_store = Store.new(LlmConfig::MODEL_SAMPLING_PATH)
119
+ def compute_store = Store.new(LlmConfig::MODEL_COMPUTE_PATH)
120
+ def n_cpu_moe_store = Store.new(N_CPU_MOE_PATH)
121
+ def packing_store = Store.new(TERNARY_PACKING_PATH)
122
+ def mmproj_store = Store.new(LlmConfig::MODEL_MMPROJ_PATH)
123
+ def effort_store = Store.new(LlmConfig::MODEL_EFFORT_PATH)
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,46 @@
1
+ require "json"
2
+ require_relative "../llm_config"
3
+
4
+ module Mimic
5
+ class Model
6
+ # Shared read/merge/write over a sidecar JSON file keyed by resolved
7
+ # model path. Value shape is the caller's business -- Store only owns the
8
+ # file and the key resolution.
9
+ class Store
10
+ def initialize(path)
11
+ @path = path
12
+ end
13
+
14
+ def fetch(model_path) = read[key(model_path)]
15
+
16
+ def write(model_path, value)
17
+ known = read
18
+ known[key(model_path)] = value
19
+ persist(known)
20
+ end
21
+
22
+ def delete(model_path)
23
+ known = read
24
+ return unless known.key?(key(model_path))
25
+
26
+ known.delete(key(model_path))
27
+ persist(known)
28
+ end
29
+
30
+ private
31
+
32
+ def key(model_path) = LlmConfig.resolve_path(model_path)
33
+
34
+ def read
35
+ JSON.parse(@path.read)
36
+ rescue SystemCallError, JSON::ParserError
37
+ {}
38
+ end
39
+
40
+ def persist(known)
41
+ @path.dirname.mkpath
42
+ @path.write(JSON.pretty_generate(known))
43
+ end
44
+ end
45
+ end
46
+ end