@qvac/inference 0.19.0 → 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/NOTICE +59 -35
  2. package/dist/api/delete-cache.d.ts +7 -1
  3. package/dist/api/delete-cache.js +11 -1
  4. package/dist/handlers/delete-cache.js +3 -0
  5. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  6. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  7. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +8 -3
  8. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +53 -13
  9. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +3 -0
  10. package/dist/resources/model-fit/assess.js +169 -68
  11. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  12. package/dist/resources/model-fit/estimators/llm.js +3 -1
  13. package/dist/resources/model-fit/floor.d.ts +31 -0
  14. package/dist/resources/model-fit/floor.js +65 -0
  15. package/dist/resources/native.js +5 -4
  16. package/dist/resources/types.d.ts +1 -3
  17. package/dist/schemas/assess-model-fit.d.ts +40 -0
  18. package/dist/schemas/assess-model-fit.js +32 -3
  19. package/dist/schemas/audio-gen.d.ts +3 -3
  20. package/dist/schemas/batch-completion-stream.d.ts +3 -0
  21. package/dist/schemas/bci.d.ts +1 -1
  22. package/dist/schemas/common.d.ts +33 -4
  23. package/dist/schemas/completion-event.d.ts +3 -0
  24. package/dist/schemas/completion-event.js +1 -0
  25. package/dist/schemas/completion-stream.d.ts +2 -0
  26. package/dist/schemas/load-model.d.ts +80 -20
  27. package/dist/schemas/model-ops.d.ts +3 -0
  28. package/dist/schemas/model-ops.js +6 -0
  29. package/dist/schemas/plugin.d.ts +11 -11
  30. package/dist/utils/addon-responses.d.ts +1 -0
  31. package/package.json +2 -2
@@ -2,6 +2,7 @@ import { getModelResourceProfile } from '../../models/registry/resource-profiles
2
2
  import { getGpuCalibration, getSharedGpuCalibration } from './calibration/index.js';
3
3
  import { estimateLlm } from './estimators/llm.js';
4
4
  import { estimateWhisper } from './estimators/whisper.js';
5
+ import { computeFloor, FLOOR_VERSION } from './floor.js';
5
6
  const GIB = 1024 * 1024 * 1024;
6
7
  /** Engines phase 1 can estimate, and which estimator owns each. */
7
8
  const ESTIMATORS = {
@@ -10,6 +11,12 @@ const ESTIMATORS = {
10
11
  'whispercpp-transcription': estimateWhisper
11
12
  };
12
13
  const MOBILE_PLATFORMS = ['android-arm64', 'ios-arm64'];
14
+ // Every allocation, GPU included, is system RAM: the system budget always bounds the load.
15
+ const UNIFIED_MEMORY_PLATFORMS = [
16
+ 'darwin-arm64',
17
+ 'android-arm64',
18
+ 'ios-arm64'
19
+ ];
13
20
  /**
14
21
  * Platforms whose fixture describes CPU-resident execution (`device: 'cpu'`),
15
22
  * so a reported GPU sends the host through `resolveGpuPlacement` instead.
@@ -98,7 +105,12 @@ export function assessModelFitFromResources(options) {
98
105
  else if (effectiveCalibration.measuredAt) {
99
106
  assumptions.push(calibrationAssumption(platform, effectiveCalibration));
100
107
  }
101
- const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode));
108
+ // Without coefficients, the computed floor still says something — but only
109
+ // where the model executes out of the memory the budget measures. A discrete
110
+ // card holds the weights in its own memory, so the system budget bounds
111
+ // nothing there and the floor has no budget to be compared against.
112
+ const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
113
+ const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode, floorApplies));
102
114
  for (const { result } of evaluated) {
103
115
  for (const assumption of result.assumptions ?? []) {
104
116
  if (!assumptions.includes(assumption))
@@ -106,19 +118,37 @@ export function assessModelFitFromResources(options) {
106
118
  }
107
119
  }
108
120
  const modelResults = evaluated.map(({ candidate, result }) => toModelResult(candidate, result, budget, deviceBudgets, alsoBoundBy));
109
- const estimates = evaluated
110
- .map(({ result }) => result)
111
- .filter((result) => {
121
+ const results = evaluated.map(({ result }) => result);
122
+ const estimates = results.filter((result) => {
112
123
  return result.kind === 'estimate';
113
124
  });
114
- const anyUnknown = estimates.length !== evaluated.length;
115
- const combined = anyUnknown ? undefined : aggregate(estimates, execution);
125
+ const anyUnknown = results.some((result) => result.kind === 'unknown');
126
+ const anyFloor = results.some((result) => result.kind === 'floor');
127
+ // A two-sided bound needs every candidate estimated; a floor needs every
128
+ // candidate to have at least a floor. One model with neither leaves the set
129
+ // with no evidence at all.
130
+ const combined = anyUnknown || anyFloor ? undefined : aggregate(estimates, execution);
131
+ const combinedFloor = anyUnknown ? undefined : aggregateFloor(results, execution);
132
+ // Absent when a candidate rests on nothing: the set then has no single kind
133
+ // of evidence to name, and an `unknown` with `evidence` present stays a
134
+ // near-miss or an uncalibrated floor rather than a missing model.
135
+ const evidence = anyUnknown
136
+ ? undefined
137
+ : anyFloor
138
+ ? 'computed-only'
139
+ : estimates.length > 0
140
+ ? 'calibration'
141
+ : undefined;
116
142
  if (anyUnknown) {
117
143
  reasons.push('at least one model could not be estimated, so the combined verdict is unknown');
118
144
  }
119
- const verdict = !budget || !combined
145
+ const verdict = !budget
120
146
  ? 'unknown'
121
- : verdictAgainst(combined, deviceBudgets ?? [budget], alsoBoundBy);
147
+ : combined
148
+ ? verdictAgainst(combined, deviceBudgets ?? [budget], alsoBoundBy)
149
+ : combinedFloor !== undefined
150
+ ? floorVerdict(combinedFloor, deviceBudgets ?? [budget])
151
+ : 'unknown';
122
152
  if (onDevice && combined && alsoBoundBy) {
123
153
  reasons.push('a GPU load is also paid for in system RAM, so every verdict is the more pessimistic of the GPU and system budgets');
124
154
  }
@@ -130,92 +160,110 @@ export function assessModelFitFromResources(options) {
130
160
  ? 'all models counted as resident with every working peak added'
131
161
  : 'all models counted as resident with only the largest working peak added');
132
162
  }
163
+ else if (budget && combinedFloor !== undefined) {
164
+ reasons.push(execution === 'concurrent'
165
+ ? 'all models counted as resident with their floors summed and every calibrated working peak added'
166
+ : 'all models counted as resident with their floors summed and only the largest calibrated working peak added');
167
+ }
168
+ if (anyFloor) {
169
+ reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
170
+ }
133
171
  return {
134
172
  verdict,
135
173
  basis,
136
174
  execution,
175
+ ...(evidence && { evidence }),
137
176
  ...(budget && { budget }),
138
177
  ...(combined && {
139
178
  estimate: { lowerBoundBytes: combined.lower, upperBoundBytes: combined.upper }
140
179
  }),
180
+ ...(anyFloor && combinedFloor !== undefined && { floorBytes: combinedFloor }),
141
181
  models: modelResults,
142
182
  reasons,
143
183
  assumptions
144
184
  };
145
185
  }
146
- function evaluate(candidate, platform, calibration, resources, resolveProfile, gpuMode) {
147
- if (!calibration) {
148
- return {
149
- candidate,
150
- result: {
151
- kind: 'unknown',
152
- estimatorVersion: 'none',
153
- reasons: [
154
- platform
155
- ? `no validated calibration for ${platform}`
156
- : 'the runtime platform is not one this assessment covers'
157
- ]
158
- }
159
- };
160
- }
161
- // A GPU is present and neither GPU mode engaged, so the only coefficients
162
- // left were measured with the offload disabled — not how this host runs.
163
- if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
164
- return {
165
- candidate,
166
- result: {
167
- kind: 'unknown',
168
- estimatorVersion: 'none',
169
- reasons: [
170
- 'a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against'
171
- ]
172
- }
173
- };
174
- }
186
+ /**
187
+ * Whether every allocation a load makes comes out of the memory the system (or
188
+ * process) budget measures — the precondition for the computed floor to be
189
+ * compared against that budget.
190
+ *
191
+ * True on unified-memory platforms whatever the collector reports, and
192
+ * elsewhere when no usable GPU is present or every usable GPU allocates out of
193
+ * system RAM. A card with its own memory, or a device whose class cannot be
194
+ * told (an AMD APU on linux), fails it: the weights may live where the budget
195
+ * cannot see them.
196
+ */
197
+ function boundBySystemMemory(resources, platform) {
198
+ if (platform !== undefined && UNIFIED_MEMORY_PLATFORMS.includes(platform))
199
+ return true;
200
+ const gpus = usableGpus(resources);
201
+ if (gpus.length === 0)
202
+ return true;
203
+ if (platform && gpus.some((gpu) => integratedIsIndistinguishable(gpu, platform)))
204
+ return false;
205
+ return gpus.every(allocatesFromSystemMemory);
206
+ }
207
+ /**
208
+ * One candidate's evaluation: the calibrated estimate when the coefficients
209
+ * describe this load, otherwise the computed floor where a floor can be
210
+ * compared at all, otherwise `unknown`. Whatever kept the estimate from
211
+ * forming stays in the reasons, so a floor verdict still says why it is only
212
+ * a floor.
213
+ */
214
+ function evaluate(candidate, platform, calibration, resources, resolveProfile, gpuMode, floorApplies) {
175
215
  const profile = resolveProfile(candidate.model.sha256Checksum);
176
216
  if (!profile) {
177
- return {
178
- candidate,
179
- result: {
180
- kind: 'unknown',
181
- estimatorVersion: 'none',
182
- reasons: ['no resource profile in the catalog for this checksum']
183
- }
184
- };
185
- }
186
- const estimator = ESTIMATORS[profile.engine];
187
- if (!estimator) {
188
- return {
189
- candidate,
190
- result: {
191
- kind: 'unknown',
192
- estimatorVersion: 'none',
193
- reasons: [`engine '${profile.engine}' has no estimator in this phase`]
194
- }
195
- };
217
+ return { candidate, result: unknown('no resource profile in the catalog for this checksum') };
196
218
  }
197
219
  const extra = extraArtifactBytes(candidate, resolveProfile);
198
220
  if (extra === undefined) {
199
221
  return {
200
222
  candidate,
201
- result: {
202
- kind: 'unknown',
203
- estimatorVersion: 'none',
204
- reasons: ['an entry in `artifacts` has no resource profile in the catalog']
205
- }
223
+ result: unknown('an entry in `artifacts` has no resource profile in the catalog')
206
224
  };
207
225
  }
226
+ const estimated = estimate(candidate, profile, extra, platform, calibration, resources, gpuMode);
227
+ if (estimated.kind === 'estimate' || !floorApplies)
228
+ return { candidate, result: estimated };
229
+ const floor = computeFloor({ profile, workload: candidate.workload, extraArtifactBytes: extra });
208
230
  return {
209
231
  candidate,
210
- result: estimator({
211
- profile,
212
- workload: candidate.workload,
213
- extraArtifactBytes: extra,
214
- calibration,
215
- hasGpu: hasGpu(resources)
216
- })
232
+ result: {
233
+ kind: 'floor',
234
+ bytes: floor.bytes,
235
+ reasons: [...estimated.reasons, ...floor.reasons],
236
+ assumptions: [...(estimated.assumptions ?? []), ...floor.assumptions]
237
+ }
217
238
  };
218
239
  }
240
+ function unknown(reason) {
241
+ return { kind: 'unknown', estimatorVersion: 'none', reasons: [reason] };
242
+ }
243
+ /** Runs the calibrated estimator for a candidate, or says why it cannot. */
244
+ function estimate(candidate, profile, extraArtifactBytes, platform, calibration, resources, gpuMode) {
245
+ if (!calibration) {
246
+ return unknown(platform
247
+ ? `no validated calibration for ${platform}`
248
+ : 'the runtime platform is not one this assessment covers');
249
+ }
250
+ // A GPU is present and neither GPU mode engaged, so the only coefficients
251
+ // left were measured with the offload disabled — not how this host runs.
252
+ if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
253
+ return unknown('a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against');
254
+ }
255
+ const estimator = ESTIMATORS[profile.engine];
256
+ if (!estimator) {
257
+ return unknown(`engine '${profile.engine}' has no estimator in this phase`);
258
+ }
259
+ return estimator({
260
+ profile,
261
+ workload: candidate.workload,
262
+ extraArtifactBytes,
263
+ calibration,
264
+ hasGpu: hasGpu(resources)
265
+ });
266
+ }
219
267
  /**
220
268
  * Sums the artifact bytes of companion constants.
221
269
  *
@@ -545,6 +593,40 @@ function aggregate(estimates, execution) {
545
593
  }
546
594
  return { lower: persistentLower + workingLower, upper: persistentUpper + workingUpper };
547
595
  }
596
+ /**
597
+ * The combined lower bound when at least one candidate has only a floor: every
598
+ * estimate contributes its own lower bound, every floor its bytes, aggregated
599
+ * the same way as `aggregate` — a floor has no working peak to add.
600
+ *
601
+ * @returns `undefined` when any candidate has no evidence at all.
602
+ */
603
+ function aggregateFloor(results, execution) {
604
+ let persistent = 0;
605
+ let working = 0;
606
+ for (const result of results) {
607
+ if (result.kind === 'unknown')
608
+ return undefined;
609
+ if (result.kind === 'floor') {
610
+ persistent += result.bytes;
611
+ continue;
612
+ }
613
+ persistent += result.persistent.lower;
614
+ working =
615
+ execution === 'concurrent'
616
+ ? working + result.working.lower
617
+ : Math.max(working, result.working.lower);
618
+ }
619
+ return persistent + working;
620
+ }
621
+ /**
622
+ * A floor can only refuse: over the largest candidate budget it is
623
+ * `likely-too-large`, and anything else is `unknown`, because the cost above
624
+ * the floor is unmeasured and could be anything.
625
+ */
626
+ function floorVerdict(floorBytes, candidates) {
627
+ const room = candidates.map((budget) => budget.availableAfterReserveBytes);
628
+ return floorBytes > Math.max(...room) ? 'likely-too-large' : 'unknown';
629
+ }
548
630
  function compare(estimate, budget) {
549
631
  if (estimate.lower > budget)
550
632
  return 'likely-too-large';
@@ -562,6 +644,24 @@ deviceBudgets, alsoBoundBy) {
562
644
  reasons: [...result.reasons]
563
645
  };
564
646
  }
647
+ if (result.kind === 'floor') {
648
+ const verdict = budget ? floorVerdict(result.bytes, deviceBudgets ?? [budget]) : 'unknown';
649
+ return {
650
+ name: candidate.model.name,
651
+ verdict,
652
+ evidence: 'computed-only',
653
+ floorBytes: result.bytes,
654
+ estimatorVersion: FLOOR_VERSION,
655
+ reasons: [
656
+ ...result.reasons,
657
+ !budget
658
+ ? 'no usable system-memory sample, so this model has no verdict'
659
+ : verdict === 'likely-too-large'
660
+ ? 'the computed floor alone exceeds the budget'
661
+ : 'the computed floor is within the budget, but without calibration the cost above it is unbounded, so a fit cannot be claimed'
662
+ ]
663
+ };
664
+ }
565
665
  const total = {
566
666
  lower: result.persistent.lower + result.working.lower,
567
667
  upper: result.persistent.upper + result.working.upper
@@ -569,6 +669,7 @@ deviceBudgets, alsoBoundBy) {
569
669
  return {
570
670
  name: candidate.model.name,
571
671
  verdict: budget ? verdictAgainst(total, deviceBudgets ?? [budget], alsoBoundBy) : 'unknown',
672
+ evidence: 'calibration',
572
673
  estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
573
674
  estimatorVersion: result.estimatorVersion,
574
675
  reasons: budget
@@ -1,6 +1,7 @@
1
1
  import type { GgufFacts } from '../../../schemas/model-resource-profile.js';
2
2
  import type { ByteRange, EstimatorInput, EstimatorResult } from '../types.js';
3
3
  export declare const LLM_ESTIMATOR_VERSION = "llm-v1";
4
+ export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
4
5
  /**
5
6
  * Estimates memory for a llama.cpp completion or embedding model from catalog
6
7
  * metadata alone.
@@ -19,6 +19,8 @@ const SSM_STATE_BYTES_PER_ELEMENT = 4;
19
19
  function disablesFlashAttention(architecture) {
20
20
  return architecture.startsWith('bitnet');
21
21
  }
22
+ // Shared with the computed floor so a mixed set carries one weights assumption, not two.
23
+ export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
22
24
  /**
23
25
  * Estimates memory for a llama.cpp completion or embedding model from catalog
24
26
  * metadata alone.
@@ -52,7 +54,7 @@ export function estimateLlm(input) {
52
54
  // The upper coefficient covers the allocator's copy-on-write and alignment
53
55
  // slack measured during calibration.
54
56
  const artifactBytes = profile.artifactBytes + extraArtifactBytes;
55
- assumptions.push('weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM');
57
+ assumptions.push(LLAMA_WEIGHTS_ASSUMPTION);
56
58
  if (extraArtifactBytes > 0) {
57
59
  assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
58
60
  }
@@ -0,0 +1,31 @@
1
+ import type { ModelFitWorkload } from '../../schemas/assess-model-fit.js';
2
+ import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
3
+ export declare const FLOOR_VERSION = "floor-v1";
4
+ export interface FloorInput {
5
+ profile: ModelResourceProfile;
6
+ workload: ModelFitWorkload;
7
+ /** Artifact bytes of companion constants the caller passed alongside. */
8
+ extraArtifactBytes: number;
9
+ }
10
+ /** A lower bound with no upper bound, and how it was assembled. */
11
+ export interface FloorResult {
12
+ bytes: number;
13
+ reasons: readonly string[];
14
+ assumptions: readonly string[];
15
+ }
16
+ /**
17
+ * The smallest resident footprint the catalog facts prove, with no coefficient
18
+ * measured anywhere: artifact bytes, plus the KV cache for a llama.cpp model at
19
+ * the cheapest element width the engine can default to.
20
+ *
21
+ * Every term a calibrated estimate adds on top — weight slack, engine overhead,
22
+ * compute buffers, a completion's working peak — is non-negative, so a floor
23
+ * with all of them at zero is unconditionally valid on any platform and any
24
+ * backend. It can therefore refuse a model (`likely-too-large` when even the
25
+ * floor is over budget) but never confirm one: how far above the floor the real
26
+ * load lands is exactly what calibration exists to measure.
27
+ *
28
+ * This is the zero-fetch gate. It reads nothing but the catalog and stays the
29
+ * fallback wherever no calibration or engine projection is available.
30
+ */
31
+ export declare function computeFloor(input: FloorInput): FloorResult;
@@ -0,0 +1,65 @@
1
+ import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheBytesForWidth, kvElementBytes } from './estimators/llm.js';
2
+ export const FLOOR_VERSION = 'floor-v1';
3
+ /**
4
+ * Engines whose load builds a KV cache the file sizes exactly, so the floor can
5
+ * count it. Every other engine's working memory is engine-owned and unsized
6
+ * here; the floor for those is the weights alone.
7
+ */
8
+ const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
9
+ /**
10
+ * The smallest resident footprint the catalog facts prove, with no coefficient
11
+ * measured anywhere: artifact bytes, plus the KV cache for a llama.cpp model at
12
+ * the cheapest element width the engine can default to.
13
+ *
14
+ * Every term a calibrated estimate adds on top — weight slack, engine overhead,
15
+ * compute buffers, a completion's working peak — is non-negative, so a floor
16
+ * with all of them at zero is unconditionally valid on any platform and any
17
+ * backend. It can therefore refuse a model (`likely-too-large` when even the
18
+ * floor is over budget) but never confirm one: how far above the floor the real
19
+ * load lands is exactly what calibration exists to measure.
20
+ *
21
+ * This is the zero-fetch gate. It reads nothing but the catalog and stays the
22
+ * fallback wherever no calibration or engine projection is available.
23
+ */
24
+ export function computeFloor(input) {
25
+ const { profile, workload, extraArtifactBytes } = input;
26
+ const reasons = [];
27
+ const assumptions = [
28
+ KV_CACHE_ENGINES.has(profile.engine)
29
+ ? LLAMA_WEIGHTS_ASSUMPTION
30
+ : 'weights are counted at full artifact size; whatever the engine maps or copies, the file has to be resident to run',
31
+ 'the floor omits engine overhead, compute buffers and working memory, all of which are non-negative, so it cannot overstate the cost — and says nothing about how much more the load needs'
32
+ ];
33
+ const artifactBytes = profile.artifactBytes + extraArtifactBytes;
34
+ if (extraArtifactBytes > 0) {
35
+ assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
36
+ }
37
+ if (!KV_CACHE_ENGINES.has(profile.engine)) {
38
+ reasons.push(`the floor for engine '${profile.engine}' counts weights only; its working memory is engine-owned and not sized here`);
39
+ return { bytes: artifactBytes, reasons, assumptions };
40
+ }
41
+ if (workload.kind !== 'llm') {
42
+ reasons.push(`workload kind '${workload.kind}' carries no context for a llama.cpp model, so the floor counts weights only`);
43
+ return { bytes: artifactBytes, reasons, assumptions };
44
+ }
45
+ const facts = profile.ggufFacts;
46
+ if (!facts) {
47
+ reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
48
+ return { bytes: artifactBytes, reasons, assumptions };
49
+ }
50
+ let contextTokens = workload.contextTokens;
51
+ if (contextTokens > facts.contextLength) {
52
+ assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
53
+ contextTokens = facts.contextLength;
54
+ }
55
+ // The narrowest default the engine can pick on any backend, so the floor
56
+ // holds whether the model lands on a GPU (q8_0 with flash attention) or on
57
+ // the CPU (f16). Architectures that disable flash attention stay f16 anyway.
58
+ const element = kvElementBytes(facts, true).bytes.lower;
59
+ const kv = kvCacheBytesForWidth(facts, contextTokens, element).lower;
60
+ assumptions.push(element < 2
61
+ ? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
62
+ : `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not expressible in a workload and would change these numbers');
63
+ reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
64
+ return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
65
+ }
@@ -14,6 +14,8 @@ const cpuArchitectures = cpuModule.status === 'fulfilled'
14
14
  const gpuTypes = gpuModule.status === 'fulfilled'
15
15
  ? supportedEnumValues(gpuModule.value.default.constants.gpuType, gpuModule.value.default.constants.gpuType.UNKNOWN)
16
16
  : [];
17
+ // `availableMemory()` reports the constrained memory on iOS.
18
+ const reportsProcessAllowance = os.platform() === 'ios';
17
19
  export const nativeResourceCollectorDependencies = {
18
20
  cpuArchitectures,
19
21
  gpuTypes,
@@ -38,12 +40,11 @@ export const nativeResourceCollectorDependencies = {
38
40
  // A throw here is a sampling failure, not a platform gap: let it reach the
39
41
  // collector so the metric reports `failed` rather than `unavailable`.
40
42
  const usage = os.memoryUsage();
41
- // No platform exposes the per-process allowance yet. On iOS this is
42
- // `os_proc_available_memory()` — the limit jetsam actually enforces — and
43
- // it needs a native source before the metric can report a value.
43
+ const available = reportsProcessAllowance ? os.availableMemory() : 0;
44
44
  return {
45
45
  usedBytes: usage && usage.rss > 0 ? usage.rss : undefined,
46
- availableBytes: undefined
46
+ // Zero states no headroom, which is not a budget.
47
+ availableBytes: available > 0 ? available : undefined
47
48
  };
48
49
  }
49
50
  };
@@ -55,9 +55,7 @@ export interface GPUInfoContext {
55
55
  *
56
56
  * @property usedBytes - The process's resident footprint (RSS).
57
57
  * @property availableBytes - How much more this process may allocate before
58
- * the OS intervenes. Only some platforms can state this — on iOS it is
59
- * `os_proc_available_memory()`, the number jetsam actually enforces.
60
- * `undefined` where no such per-process limit is exposed.
58
+ * the OS intervenes. `undefined` where the platform states no such limit.
61
59
  */
62
60
  export interface ProcessMemorySample {
63
61
  usedBytes: number | undefined;
@@ -120,6 +120,20 @@ export declare const modelFitBasisSchema: z.ZodEnum<{
120
120
  "device-memory": "device-memory";
121
121
  "device-budget": "device-budget";
122
122
  }>;
123
+ /**
124
+ * What kind of evidence a verdict rests on.
125
+ *
126
+ * - `calibration`: a two-sided estimate from coefficients measured on this
127
+ * platform. The only evidence that can support `likely-fits`.
128
+ * - `computed-only`: a floor computed from catalog facts alone — artifact bytes,
129
+ * plus the KV cache for llama.cpp models. It omits every engine cost that
130
+ * only a real load can tell, so it can refuse a model but never confirm one.
131
+ * This is what an uncalibrated platform, including Android and iOS, reports.
132
+ */
133
+ export declare const modelFitEvidenceSchema: z.ZodEnum<{
134
+ calibration: "calibration";
135
+ "computed-only": "computed-only";
136
+ }>;
123
137
  export declare const modelFitBudgetSchema: z.ZodObject<{
124
138
  totalBytes: z.ZodNumber;
125
139
  usedBytes: z.ZodNumber;
@@ -134,10 +148,15 @@ export declare const modelFitModelResultSchema: z.ZodObject<{
134
148
  "likely-fits": "likely-fits";
135
149
  "likely-too-large": "likely-too-large";
136
150
  }>;
151
+ evidence: z.ZodOptional<z.ZodEnum<{
152
+ calibration: "calibration";
153
+ "computed-only": "computed-only";
154
+ }>>;
137
155
  estimate: z.ZodOptional<z.ZodObject<{
138
156
  lowerBoundBytes: z.ZodNumber;
139
157
  upperBoundBytes: z.ZodNumber;
140
158
  }, z.core.$strip>>;
159
+ floorBytes: z.ZodOptional<z.ZodNumber>;
141
160
  estimatorVersion: z.ZodOptional<z.ZodString>;
142
161
  reasons: z.ZodArray<z.ZodString>;
143
162
  }, z.core.$strip>;
@@ -157,6 +176,10 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
157
176
  sequential: "sequential";
158
177
  concurrent: "concurrent";
159
178
  }>;
179
+ evidence: z.ZodOptional<z.ZodEnum<{
180
+ calibration: "calibration";
181
+ "computed-only": "computed-only";
182
+ }>>;
160
183
  budget: z.ZodOptional<z.ZodObject<{
161
184
  totalBytes: z.ZodNumber;
162
185
  usedBytes: z.ZodNumber;
@@ -168,6 +191,7 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
168
191
  lowerBoundBytes: z.ZodNumber;
169
192
  upperBoundBytes: z.ZodNumber;
170
193
  }, z.core.$strip>>;
194
+ floorBytes: z.ZodOptional<z.ZodNumber>;
171
195
  models: z.ZodArray<z.ZodObject<{
172
196
  name: z.ZodString;
173
197
  verdict: z.ZodEnum<{
@@ -175,10 +199,15 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
175
199
  "likely-fits": "likely-fits";
176
200
  "likely-too-large": "likely-too-large";
177
201
  }>;
202
+ evidence: z.ZodOptional<z.ZodEnum<{
203
+ calibration: "calibration";
204
+ "computed-only": "computed-only";
205
+ }>>;
178
206
  estimate: z.ZodOptional<z.ZodObject<{
179
207
  lowerBoundBytes: z.ZodNumber;
180
208
  upperBoundBytes: z.ZodNumber;
181
209
  }, z.core.$strip>>;
210
+ floorBytes: z.ZodOptional<z.ZodNumber>;
182
211
  estimatorVersion: z.ZodOptional<z.ZodString>;
183
212
  reasons: z.ZodArray<z.ZodString>;
184
213
  }, z.core.$strip>>;
@@ -228,6 +257,10 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
228
257
  sequential: "sequential";
229
258
  concurrent: "concurrent";
230
259
  }>;
260
+ evidence: z.ZodOptional<z.ZodEnum<{
261
+ calibration: "calibration";
262
+ "computed-only": "computed-only";
263
+ }>>;
231
264
  budget: z.ZodOptional<z.ZodObject<{
232
265
  totalBytes: z.ZodNumber;
233
266
  usedBytes: z.ZodNumber;
@@ -239,6 +272,7 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
239
272
  lowerBoundBytes: z.ZodNumber;
240
273
  upperBoundBytes: z.ZodNumber;
241
274
  }, z.core.$strip>>;
275
+ floorBytes: z.ZodOptional<z.ZodNumber>;
242
276
  models: z.ZodArray<z.ZodObject<{
243
277
  name: z.ZodString;
244
278
  verdict: z.ZodEnum<{
@@ -246,10 +280,15 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
246
280
  "likely-fits": "likely-fits";
247
281
  "likely-too-large": "likely-too-large";
248
282
  }>;
283
+ evidence: z.ZodOptional<z.ZodEnum<{
284
+ calibration: "calibration";
285
+ "computed-only": "computed-only";
286
+ }>>;
249
287
  estimate: z.ZodOptional<z.ZodObject<{
250
288
  lowerBoundBytes: z.ZodNumber;
251
289
  upperBoundBytes: z.ZodNumber;
252
290
  }, z.core.$strip>>;
291
+ floorBytes: z.ZodOptional<z.ZodNumber>;
253
292
  estimatorVersion: z.ZodOptional<z.ZodString>;
254
293
  reasons: z.ZodArray<z.ZodString>;
255
294
  }, z.core.$strip>>;
@@ -263,6 +302,7 @@ export type ModelFitWorkload = z.infer<typeof modelFitWorkloadSchema>;
263
302
  export type ModelFitCandidate = z.infer<typeof modelFitCandidateSchema>;
264
303
  export type ModelFitExecution = z.infer<typeof modelFitExecutionSchema>;
265
304
  export type ModelFitBasis = z.infer<typeof modelFitBasisSchema>;
305
+ export type ModelFitEvidence = z.infer<typeof modelFitEvidenceSchema>;
266
306
  export type ModelFitBudget = z.infer<typeof modelFitBudgetSchema>;
267
307
  export type ModelFitModelResult = z.infer<typeof modelFitModelResultSchema>;
268
308
  export type AssessModelFitInput = z.input<typeof assessModelFitInputSchema>;
@@ -108,6 +108,17 @@ export const modelFitBasisSchema = z.enum([
108
108
  'device-memory',
109
109
  'device-budget'
110
110
  ]);
111
+ /**
112
+ * What kind of evidence a verdict rests on.
113
+ *
114
+ * - `calibration`: a two-sided estimate from coefficients measured on this
115
+ * platform. The only evidence that can support `likely-fits`.
116
+ * - `computed-only`: a floor computed from catalog facts alone — artifact bytes,
117
+ * plus the KV cache for llama.cpp models. It omits every engine cost that
118
+ * only a real load can tell, so it can refuse a model but never confirm one.
119
+ * This is what an uncalibrated platform, including Android and iOS, reports.
120
+ */
121
+ export const modelFitEvidenceSchema = z.enum(['calibration', 'computed-only']);
111
122
  export const modelFitBudgetSchema = z.object({
112
123
  totalBytes: z
113
124
  .number()
@@ -126,19 +137,37 @@ export const modelFitBudgetSchema = z.object({
126
137
  export const modelFitModelResultSchema = z.object({
127
138
  name: z.string().describe('Catalog name of the assessed model.'),
128
139
  verdict: modelFitVerdictSchema,
129
- estimate: byteRangeSchema.optional().describe('Absent when this model assessed as `unknown`.'),
140
+ evidence: modelFitEvidenceSchema
141
+ .optional()
142
+ .describe('What the verdict rests on. Absent when nothing could be computed for this model, e.g. no catalog profile.'),
143
+ estimate: byteRangeSchema
144
+ .optional()
145
+ .describe('Two-sided bound from calibrated coefficients. Absent under computed-only evidence, or when this model assessed as `unknown` for want of any evidence.'),
146
+ floorBytes: z
147
+ .number()
148
+ .optional()
149
+ .describe('Under computed-only evidence: the smallest resident footprint the catalog facts prove — artifact bytes, plus the KV cache for llama.cpp. A floor only; the true cost is above it by an unmeasured amount.'),
130
150
  estimatorVersion: z
131
151
  .string()
132
152
  .optional()
133
- .describe('Estimator that produced the bounds, e.g. `llm-v1`.'),
153
+ .describe('Estimator that produced the bounds, e.g. `llm-v1`, or `floor-v1` for a computed floor.'),
134
154
  reasons: z.array(z.string()).describe('Why this model got this verdict.')
135
155
  });
136
156
  export const assessModelFitResultSchema = z.object({
137
157
  verdict: modelFitVerdictSchema.describe('Combined verdict across every candidate.'),
138
158
  basis: modelFitBasisSchema.describe('The evidence the budget was derived from — system RAM, the per-process ceiling on iOS, a discrete GPU’s own memory, or on Windows the GPU memory budget the OS grants this process. The two device bases also require the system-memory budget to hold.'),
139
159
  execution: modelFitExecutionSchema.describe('The declared execution mode this result assumed.'),
160
+ evidence: modelFitEvidenceSchema
161
+ .optional()
162
+ .describe('The weakest evidence among the candidates: `computed-only` as soon as one model has only a floor, since the combined verdict can then never be `likely-fits`. Absent whenever any candidate could not be assessed at all, so an `unknown` that carries `evidence` is a near-miss or an uncalibrated floor, never a missing model.'),
140
163
  budget: modelFitBudgetSchema.optional().describe('Absent when memory evidence was unusable.'),
141
- estimate: byteRangeSchema.optional().describe('Absent when the combined verdict is `unknown`.'),
164
+ estimate: byteRangeSchema
165
+ .optional()
166
+ .describe('Combined two-sided bound. Absent when the combined verdict is `unknown` for want of evidence, and under computed-only evidence, which has no upper bound.'),
167
+ floorBytes: z
168
+ .number()
169
+ .optional()
170
+ .describe('Under computed-only evidence: the combined floor across every candidate, aggregated under `execution`. Compared against the budget for `likely-too-large`; never enough for `likely-fits`.'),
142
171
  models: z.array(modelFitModelResultSchema).describe('Per-candidate verdicts, in input order.'),
143
172
  reasons: z.array(z.string()).describe('Why the combined verdict came out this way.'),
144
173
  assumptions: z