@qvac/inference 0.19.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +59 -35
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +8 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +53 -13
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +3 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/schemas/assess-model-fit.d.ts +40 -0
- package/dist/schemas/assess-model-fit.js +32 -3
- package/dist/schemas/audio-gen.d.ts +3 -3
- package/dist/schemas/batch-completion-stream.d.ts +3 -0
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +33 -4
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -0
- package/dist/schemas/completion-stream.d.ts +2 -0
- package/dist/schemas/load-model.d.ts +80 -20
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +11 -11
- package/dist/utils/addon-responses.d.ts +1 -0
- package/package.json +2 -2
|
@@ -2,6 +2,7 @@ import { getModelResourceProfile } from '../../models/registry/resource-profiles
|
|
|
2
2
|
import { getGpuCalibration, getSharedGpuCalibration } from './calibration/index.js';
|
|
3
3
|
import { estimateLlm } from './estimators/llm.js';
|
|
4
4
|
import { estimateWhisper } from './estimators/whisper.js';
|
|
5
|
+
import { computeFloor, FLOOR_VERSION } from './floor.js';
|
|
5
6
|
const GIB = 1024 * 1024 * 1024;
|
|
6
7
|
/** Engines phase 1 can estimate, and which estimator owns each. */
|
|
7
8
|
const ESTIMATORS = {
|
|
@@ -10,6 +11,12 @@ const ESTIMATORS = {
|
|
|
10
11
|
'whispercpp-transcription': estimateWhisper
|
|
11
12
|
};
|
|
12
13
|
const MOBILE_PLATFORMS = ['android-arm64', 'ios-arm64'];
|
|
14
|
+
// Every allocation, GPU included, is system RAM: the system budget always bounds the load.
|
|
15
|
+
const UNIFIED_MEMORY_PLATFORMS = [
|
|
16
|
+
'darwin-arm64',
|
|
17
|
+
'android-arm64',
|
|
18
|
+
'ios-arm64'
|
|
19
|
+
];
|
|
13
20
|
/**
|
|
14
21
|
* Platforms whose fixture describes CPU-resident execution (`device: 'cpu'`),
|
|
15
22
|
* so a reported GPU sends the host through `resolveGpuPlacement` instead.
|
|
@@ -98,7 +105,12 @@ export function assessModelFitFromResources(options) {
|
|
|
98
105
|
else if (effectiveCalibration.measuredAt) {
|
|
99
106
|
assumptions.push(calibrationAssumption(platform, effectiveCalibration));
|
|
100
107
|
}
|
|
101
|
-
|
|
108
|
+
// Without coefficients, the computed floor still says something — but only
|
|
109
|
+
// where the model executes out of the memory the budget measures. A discrete
|
|
110
|
+
// card holds the weights in its own memory, so the system budget bounds
|
|
111
|
+
// nothing there and the floor has no budget to be compared against.
|
|
112
|
+
const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
|
|
113
|
+
const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode, floorApplies));
|
|
102
114
|
for (const { result } of evaluated) {
|
|
103
115
|
for (const assumption of result.assumptions ?? []) {
|
|
104
116
|
if (!assumptions.includes(assumption))
|
|
@@ -106,19 +118,37 @@ export function assessModelFitFromResources(options) {
|
|
|
106
118
|
}
|
|
107
119
|
}
|
|
108
120
|
const modelResults = evaluated.map(({ candidate, result }) => toModelResult(candidate, result, budget, deviceBudgets, alsoBoundBy));
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
.filter((result) => {
|
|
121
|
+
const results = evaluated.map(({ result }) => result);
|
|
122
|
+
const estimates = results.filter((result) => {
|
|
112
123
|
return result.kind === 'estimate';
|
|
113
124
|
});
|
|
114
|
-
const anyUnknown =
|
|
115
|
-
const
|
|
125
|
+
const anyUnknown = results.some((result) => result.kind === 'unknown');
|
|
126
|
+
const anyFloor = results.some((result) => result.kind === 'floor');
|
|
127
|
+
// A two-sided bound needs every candidate estimated; a floor needs every
|
|
128
|
+
// candidate to have at least a floor. One model with neither leaves the set
|
|
129
|
+
// with no evidence at all.
|
|
130
|
+
const combined = anyUnknown || anyFloor ? undefined : aggregate(estimates, execution);
|
|
131
|
+
const combinedFloor = anyUnknown ? undefined : aggregateFloor(results, execution);
|
|
132
|
+
// Absent when a candidate rests on nothing: the set then has no single kind
|
|
133
|
+
// of evidence to name, and an `unknown` with `evidence` present stays a
|
|
134
|
+
// near-miss or an uncalibrated floor rather than a missing model.
|
|
135
|
+
const evidence = anyUnknown
|
|
136
|
+
? undefined
|
|
137
|
+
: anyFloor
|
|
138
|
+
? 'computed-only'
|
|
139
|
+
: estimates.length > 0
|
|
140
|
+
? 'calibration'
|
|
141
|
+
: undefined;
|
|
116
142
|
if (anyUnknown) {
|
|
117
143
|
reasons.push('at least one model could not be estimated, so the combined verdict is unknown');
|
|
118
144
|
}
|
|
119
|
-
const verdict = !budget
|
|
145
|
+
const verdict = !budget
|
|
120
146
|
? 'unknown'
|
|
121
|
-
:
|
|
147
|
+
: combined
|
|
148
|
+
? verdictAgainst(combined, deviceBudgets ?? [budget], alsoBoundBy)
|
|
149
|
+
: combinedFloor !== undefined
|
|
150
|
+
? floorVerdict(combinedFloor, deviceBudgets ?? [budget])
|
|
151
|
+
: 'unknown';
|
|
122
152
|
if (onDevice && combined && alsoBoundBy) {
|
|
123
153
|
reasons.push('a GPU load is also paid for in system RAM, so every verdict is the more pessimistic of the GPU and system budgets');
|
|
124
154
|
}
|
|
@@ -130,92 +160,110 @@ export function assessModelFitFromResources(options) {
|
|
|
130
160
|
? 'all models counted as resident with every working peak added'
|
|
131
161
|
: 'all models counted as resident with only the largest working peak added');
|
|
132
162
|
}
|
|
163
|
+
else if (budget && combinedFloor !== undefined) {
|
|
164
|
+
reasons.push(execution === 'concurrent'
|
|
165
|
+
? 'all models counted as resident with their floors summed and every calibrated working peak added'
|
|
166
|
+
: 'all models counted as resident with their floors summed and only the largest calibrated working peak added');
|
|
167
|
+
}
|
|
168
|
+
if (anyFloor) {
|
|
169
|
+
reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
|
|
170
|
+
}
|
|
133
171
|
return {
|
|
134
172
|
verdict,
|
|
135
173
|
basis,
|
|
136
174
|
execution,
|
|
175
|
+
...(evidence && { evidence }),
|
|
137
176
|
...(budget && { budget }),
|
|
138
177
|
...(combined && {
|
|
139
178
|
estimate: { lowerBoundBytes: combined.lower, upperBoundBytes: combined.upper }
|
|
140
179
|
}),
|
|
180
|
+
...(anyFloor && combinedFloor !== undefined && { floorBytes: combinedFloor }),
|
|
141
181
|
models: modelResults,
|
|
142
182
|
reasons,
|
|
143
183
|
assumptions
|
|
144
184
|
};
|
|
145
185
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
if (
|
|
164
|
-
return
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
186
|
+
/**
|
|
187
|
+
* Whether every allocation a load makes comes out of the memory the system (or
|
|
188
|
+
* process) budget measures — the precondition for the computed floor to be
|
|
189
|
+
* compared against that budget.
|
|
190
|
+
*
|
|
191
|
+
* True on unified-memory platforms whatever the collector reports, and
|
|
192
|
+
* elsewhere when no usable GPU is present or every usable GPU allocates out of
|
|
193
|
+
* system RAM. A card with its own memory, or a device whose class cannot be
|
|
194
|
+
* told (an AMD APU on linux), fails it: the weights may live where the budget
|
|
195
|
+
* cannot see them.
|
|
196
|
+
*/
|
|
197
|
+
function boundBySystemMemory(resources, platform) {
|
|
198
|
+
if (platform !== undefined && UNIFIED_MEMORY_PLATFORMS.includes(platform))
|
|
199
|
+
return true;
|
|
200
|
+
const gpus = usableGpus(resources);
|
|
201
|
+
if (gpus.length === 0)
|
|
202
|
+
return true;
|
|
203
|
+
if (platform && gpus.some((gpu) => integratedIsIndistinguishable(gpu, platform)))
|
|
204
|
+
return false;
|
|
205
|
+
return gpus.every(allocatesFromSystemMemory);
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* One candidate's evaluation: the calibrated estimate when the coefficients
|
|
209
|
+
* describe this load, otherwise the computed floor where a floor can be
|
|
210
|
+
* compared at all, otherwise `unknown`. Whatever kept the estimate from
|
|
211
|
+
* forming stays in the reasons, so a floor verdict still says why it is only
|
|
212
|
+
* a floor.
|
|
213
|
+
*/
|
|
214
|
+
function evaluate(candidate, platform, calibration, resources, resolveProfile, gpuMode, floorApplies) {
|
|
175
215
|
const profile = resolveProfile(candidate.model.sha256Checksum);
|
|
176
216
|
if (!profile) {
|
|
177
|
-
return {
|
|
178
|
-
candidate,
|
|
179
|
-
result: {
|
|
180
|
-
kind: 'unknown',
|
|
181
|
-
estimatorVersion: 'none',
|
|
182
|
-
reasons: ['no resource profile in the catalog for this checksum']
|
|
183
|
-
}
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
const estimator = ESTIMATORS[profile.engine];
|
|
187
|
-
if (!estimator) {
|
|
188
|
-
return {
|
|
189
|
-
candidate,
|
|
190
|
-
result: {
|
|
191
|
-
kind: 'unknown',
|
|
192
|
-
estimatorVersion: 'none',
|
|
193
|
-
reasons: [`engine '${profile.engine}' has no estimator in this phase`]
|
|
194
|
-
}
|
|
195
|
-
};
|
|
217
|
+
return { candidate, result: unknown('no resource profile in the catalog for this checksum') };
|
|
196
218
|
}
|
|
197
219
|
const extra = extraArtifactBytes(candidate, resolveProfile);
|
|
198
220
|
if (extra === undefined) {
|
|
199
221
|
return {
|
|
200
222
|
candidate,
|
|
201
|
-
result:
|
|
202
|
-
kind: 'unknown',
|
|
203
|
-
estimatorVersion: 'none',
|
|
204
|
-
reasons: ['an entry in `artifacts` has no resource profile in the catalog']
|
|
205
|
-
}
|
|
223
|
+
result: unknown('an entry in `artifacts` has no resource profile in the catalog')
|
|
206
224
|
};
|
|
207
225
|
}
|
|
226
|
+
const estimated = estimate(candidate, profile, extra, platform, calibration, resources, gpuMode);
|
|
227
|
+
if (estimated.kind === 'estimate' || !floorApplies)
|
|
228
|
+
return { candidate, result: estimated };
|
|
229
|
+
const floor = computeFloor({ profile, workload: candidate.workload, extraArtifactBytes: extra });
|
|
208
230
|
return {
|
|
209
231
|
candidate,
|
|
210
|
-
result:
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
})
|
|
232
|
+
result: {
|
|
233
|
+
kind: 'floor',
|
|
234
|
+
bytes: floor.bytes,
|
|
235
|
+
reasons: [...estimated.reasons, ...floor.reasons],
|
|
236
|
+
assumptions: [...(estimated.assumptions ?? []), ...floor.assumptions]
|
|
237
|
+
}
|
|
217
238
|
};
|
|
218
239
|
}
|
|
240
|
+
function unknown(reason) {
|
|
241
|
+
return { kind: 'unknown', estimatorVersion: 'none', reasons: [reason] };
|
|
242
|
+
}
|
|
243
|
+
/** Runs the calibrated estimator for a candidate, or says why it cannot. */
|
|
244
|
+
function estimate(candidate, profile, extraArtifactBytes, platform, calibration, resources, gpuMode) {
|
|
245
|
+
if (!calibration) {
|
|
246
|
+
return unknown(platform
|
|
247
|
+
? `no validated calibration for ${platform}`
|
|
248
|
+
: 'the runtime platform is not one this assessment covers');
|
|
249
|
+
}
|
|
250
|
+
// A GPU is present and neither GPU mode engaged, so the only coefficients
|
|
251
|
+
// left were measured with the offload disabled — not how this host runs.
|
|
252
|
+
if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
|
|
253
|
+
return unknown('a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against');
|
|
254
|
+
}
|
|
255
|
+
const estimator = ESTIMATORS[profile.engine];
|
|
256
|
+
if (!estimator) {
|
|
257
|
+
return unknown(`engine '${profile.engine}' has no estimator in this phase`);
|
|
258
|
+
}
|
|
259
|
+
return estimator({
|
|
260
|
+
profile,
|
|
261
|
+
workload: candidate.workload,
|
|
262
|
+
extraArtifactBytes,
|
|
263
|
+
calibration,
|
|
264
|
+
hasGpu: hasGpu(resources)
|
|
265
|
+
});
|
|
266
|
+
}
|
|
219
267
|
/**
|
|
220
268
|
* Sums the artifact bytes of companion constants.
|
|
221
269
|
*
|
|
@@ -545,6 +593,40 @@ function aggregate(estimates, execution) {
|
|
|
545
593
|
}
|
|
546
594
|
return { lower: persistentLower + workingLower, upper: persistentUpper + workingUpper };
|
|
547
595
|
}
|
|
596
|
+
/**
|
|
597
|
+
* The combined lower bound when at least one candidate has only a floor: every
|
|
598
|
+
* estimate contributes its own lower bound, every floor its bytes, aggregated
|
|
599
|
+
* the same way as `aggregate` — a floor has no working peak to add.
|
|
600
|
+
*
|
|
601
|
+
* @returns `undefined` when any candidate has no evidence at all.
|
|
602
|
+
*/
|
|
603
|
+
function aggregateFloor(results, execution) {
|
|
604
|
+
let persistent = 0;
|
|
605
|
+
let working = 0;
|
|
606
|
+
for (const result of results) {
|
|
607
|
+
if (result.kind === 'unknown')
|
|
608
|
+
return undefined;
|
|
609
|
+
if (result.kind === 'floor') {
|
|
610
|
+
persistent += result.bytes;
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
persistent += result.persistent.lower;
|
|
614
|
+
working =
|
|
615
|
+
execution === 'concurrent'
|
|
616
|
+
? working + result.working.lower
|
|
617
|
+
: Math.max(working, result.working.lower);
|
|
618
|
+
}
|
|
619
|
+
return persistent + working;
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* A floor can only refuse: over the largest candidate budget it is
|
|
623
|
+
* `likely-too-large`, and anything else is `unknown`, because the cost above
|
|
624
|
+
* the floor is unmeasured and could be anything.
|
|
625
|
+
*/
|
|
626
|
+
function floorVerdict(floorBytes, candidates) {
|
|
627
|
+
const room = candidates.map((budget) => budget.availableAfterReserveBytes);
|
|
628
|
+
return floorBytes > Math.max(...room) ? 'likely-too-large' : 'unknown';
|
|
629
|
+
}
|
|
548
630
|
function compare(estimate, budget) {
|
|
549
631
|
if (estimate.lower > budget)
|
|
550
632
|
return 'likely-too-large';
|
|
@@ -562,6 +644,24 @@ deviceBudgets, alsoBoundBy) {
|
|
|
562
644
|
reasons: [...result.reasons]
|
|
563
645
|
};
|
|
564
646
|
}
|
|
647
|
+
if (result.kind === 'floor') {
|
|
648
|
+
const verdict = budget ? floorVerdict(result.bytes, deviceBudgets ?? [budget]) : 'unknown';
|
|
649
|
+
return {
|
|
650
|
+
name: candidate.model.name,
|
|
651
|
+
verdict,
|
|
652
|
+
evidence: 'computed-only',
|
|
653
|
+
floorBytes: result.bytes,
|
|
654
|
+
estimatorVersion: FLOOR_VERSION,
|
|
655
|
+
reasons: [
|
|
656
|
+
...result.reasons,
|
|
657
|
+
!budget
|
|
658
|
+
? 'no usable system-memory sample, so this model has no verdict'
|
|
659
|
+
: verdict === 'likely-too-large'
|
|
660
|
+
? 'the computed floor alone exceeds the budget'
|
|
661
|
+
: 'the computed floor is within the budget, but without calibration the cost above it is unbounded, so a fit cannot be claimed'
|
|
662
|
+
]
|
|
663
|
+
};
|
|
664
|
+
}
|
|
565
665
|
const total = {
|
|
566
666
|
lower: result.persistent.lower + result.working.lower,
|
|
567
667
|
upper: result.persistent.upper + result.working.upper
|
|
@@ -569,6 +669,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
569
669
|
return {
|
|
570
670
|
name: candidate.model.name,
|
|
571
671
|
verdict: budget ? verdictAgainst(total, deviceBudgets ?? [budget], alsoBoundBy) : 'unknown',
|
|
672
|
+
evidence: 'calibration',
|
|
572
673
|
estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
|
|
573
674
|
estimatorVersion: result.estimatorVersion,
|
|
574
675
|
reasons: budget
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { GgufFacts } from '../../../schemas/model-resource-profile.js';
|
|
2
2
|
import type { ByteRange, EstimatorInput, EstimatorResult } from '../types.js';
|
|
3
3
|
export declare const LLM_ESTIMATOR_VERSION = "llm-v1";
|
|
4
|
+
export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
|
|
4
5
|
/**
|
|
5
6
|
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
6
7
|
* metadata alone.
|
|
@@ -19,6 +19,8 @@ const SSM_STATE_BYTES_PER_ELEMENT = 4;
|
|
|
19
19
|
function disablesFlashAttention(architecture) {
|
|
20
20
|
return architecture.startsWith('bitnet');
|
|
21
21
|
}
|
|
22
|
+
// Shared with the computed floor so a mixed set carries one weights assumption, not two.
|
|
23
|
+
export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
|
|
22
24
|
/**
|
|
23
25
|
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
24
26
|
* metadata alone.
|
|
@@ -52,7 +54,7 @@ export function estimateLlm(input) {
|
|
|
52
54
|
// The upper coefficient covers the allocator's copy-on-write and alignment
|
|
53
55
|
// slack measured during calibration.
|
|
54
56
|
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
55
|
-
assumptions.push(
|
|
57
|
+
assumptions.push(LLAMA_WEIGHTS_ASSUMPTION);
|
|
56
58
|
if (extraArtifactBytes > 0) {
|
|
57
59
|
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
58
60
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ModelFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
3
|
+
export declare const FLOOR_VERSION = "floor-v1";
|
|
4
|
+
export interface FloorInput {
|
|
5
|
+
profile: ModelResourceProfile;
|
|
6
|
+
workload: ModelFitWorkload;
|
|
7
|
+
/** Artifact bytes of companion constants the caller passed alongside. */
|
|
8
|
+
extraArtifactBytes: number;
|
|
9
|
+
}
|
|
10
|
+
/** A lower bound with no upper bound, and how it was assembled. */
|
|
11
|
+
export interface FloorResult {
|
|
12
|
+
bytes: number;
|
|
13
|
+
reasons: readonly string[];
|
|
14
|
+
assumptions: readonly string[];
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The smallest resident footprint the catalog facts prove, with no coefficient
|
|
18
|
+
* measured anywhere: artifact bytes, plus the KV cache for a llama.cpp model at
|
|
19
|
+
* the cheapest element width the engine can default to.
|
|
20
|
+
*
|
|
21
|
+
* Every term a calibrated estimate adds on top — weight slack, engine overhead,
|
|
22
|
+
* compute buffers, a completion's working peak — is non-negative, so a floor
|
|
23
|
+
* with all of them at zero is unconditionally valid on any platform and any
|
|
24
|
+
* backend. It can therefore refuse a model (`likely-too-large` when even the
|
|
25
|
+
* floor is over budget) but never confirm one: how far above the floor the real
|
|
26
|
+
* load lands is exactly what calibration exists to measure.
|
|
27
|
+
*
|
|
28
|
+
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
29
|
+
* fallback wherever no calibration or engine projection is available.
|
|
30
|
+
*/
|
|
31
|
+
export declare function computeFloor(input: FloorInput): FloorResult;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheBytesForWidth, kvElementBytes } from './estimators/llm.js';
|
|
2
|
+
export const FLOOR_VERSION = 'floor-v1';
|
|
3
|
+
/**
|
|
4
|
+
* Engines whose load builds a KV cache the file sizes exactly, so the floor can
|
|
5
|
+
* count it. Every other engine's working memory is engine-owned and unsized
|
|
6
|
+
* here; the floor for those is the weights alone.
|
|
7
|
+
*/
|
|
8
|
+
const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
|
|
9
|
+
/**
|
|
10
|
+
* The smallest resident footprint the catalog facts prove, with no coefficient
|
|
11
|
+
* measured anywhere: artifact bytes, plus the KV cache for a llama.cpp model at
|
|
12
|
+
* the cheapest element width the engine can default to.
|
|
13
|
+
*
|
|
14
|
+
* Every term a calibrated estimate adds on top — weight slack, engine overhead,
|
|
15
|
+
* compute buffers, a completion's working peak — is non-negative, so a floor
|
|
16
|
+
* with all of them at zero is unconditionally valid on any platform and any
|
|
17
|
+
* backend. It can therefore refuse a model (`likely-too-large` when even the
|
|
18
|
+
* floor is over budget) but never confirm one: how far above the floor the real
|
|
19
|
+
* load lands is exactly what calibration exists to measure.
|
|
20
|
+
*
|
|
21
|
+
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
22
|
+
* fallback wherever no calibration or engine projection is available.
|
|
23
|
+
*/
|
|
24
|
+
export function computeFloor(input) {
|
|
25
|
+
const { profile, workload, extraArtifactBytes } = input;
|
|
26
|
+
const reasons = [];
|
|
27
|
+
const assumptions = [
|
|
28
|
+
KV_CACHE_ENGINES.has(profile.engine)
|
|
29
|
+
? LLAMA_WEIGHTS_ASSUMPTION
|
|
30
|
+
: 'weights are counted at full artifact size; whatever the engine maps or copies, the file has to be resident to run',
|
|
31
|
+
'the floor omits engine overhead, compute buffers and working memory, all of which are non-negative, so it cannot overstate the cost — and says nothing about how much more the load needs'
|
|
32
|
+
];
|
|
33
|
+
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
34
|
+
if (extraArtifactBytes > 0) {
|
|
35
|
+
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
36
|
+
}
|
|
37
|
+
if (!KV_CACHE_ENGINES.has(profile.engine)) {
|
|
38
|
+
reasons.push(`the floor for engine '${profile.engine}' counts weights only; its working memory is engine-owned and not sized here`);
|
|
39
|
+
return { bytes: artifactBytes, reasons, assumptions };
|
|
40
|
+
}
|
|
41
|
+
if (workload.kind !== 'llm') {
|
|
42
|
+
reasons.push(`workload kind '${workload.kind}' carries no context for a llama.cpp model, so the floor counts weights only`);
|
|
43
|
+
return { bytes: artifactBytes, reasons, assumptions };
|
|
44
|
+
}
|
|
45
|
+
const facts = profile.ggufFacts;
|
|
46
|
+
if (!facts) {
|
|
47
|
+
reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
|
|
48
|
+
return { bytes: artifactBytes, reasons, assumptions };
|
|
49
|
+
}
|
|
50
|
+
let contextTokens = workload.contextTokens;
|
|
51
|
+
if (contextTokens > facts.contextLength) {
|
|
52
|
+
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
53
|
+
contextTokens = facts.contextLength;
|
|
54
|
+
}
|
|
55
|
+
// The narrowest default the engine can pick on any backend, so the floor
|
|
56
|
+
// holds whether the model lands on a GPU (q8_0 with flash attention) or on
|
|
57
|
+
// the CPU (f16). Architectures that disable flash attention stay f16 anyway.
|
|
58
|
+
const element = kvElementBytes(facts, true).bytes.lower;
|
|
59
|
+
const kv = kvCacheBytesForWidth(facts, contextTokens, element).lower;
|
|
60
|
+
assumptions.push(element < 2
|
|
61
|
+
? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
|
|
62
|
+
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not expressible in a workload and would change these numbers');
|
|
63
|
+
reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
|
|
64
|
+
return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
|
|
65
|
+
}
|
package/dist/resources/native.js
CHANGED
|
@@ -14,6 +14,8 @@ const cpuArchitectures = cpuModule.status === 'fulfilled'
|
|
|
14
14
|
const gpuTypes = gpuModule.status === 'fulfilled'
|
|
15
15
|
? supportedEnumValues(gpuModule.value.default.constants.gpuType, gpuModule.value.default.constants.gpuType.UNKNOWN)
|
|
16
16
|
: [];
|
|
17
|
+
// `availableMemory()` reports the constrained memory on iOS.
|
|
18
|
+
const reportsProcessAllowance = os.platform() === 'ios';
|
|
17
19
|
export const nativeResourceCollectorDependencies = {
|
|
18
20
|
cpuArchitectures,
|
|
19
21
|
gpuTypes,
|
|
@@ -38,12 +40,11 @@ export const nativeResourceCollectorDependencies = {
|
|
|
38
40
|
// A throw here is a sampling failure, not a platform gap: let it reach the
|
|
39
41
|
// collector so the metric reports `failed` rather than `unavailable`.
|
|
40
42
|
const usage = os.memoryUsage();
|
|
41
|
-
|
|
42
|
-
// `os_proc_available_memory()` — the limit jetsam actually enforces — and
|
|
43
|
-
// it needs a native source before the metric can report a value.
|
|
43
|
+
const available = reportsProcessAllowance ? os.availableMemory() : 0;
|
|
44
44
|
return {
|
|
45
45
|
usedBytes: usage && usage.rss > 0 ? usage.rss : undefined,
|
|
46
|
-
|
|
46
|
+
// Zero states no headroom, which is not a budget.
|
|
47
|
+
availableBytes: available > 0 ? available : undefined
|
|
47
48
|
};
|
|
48
49
|
}
|
|
49
50
|
};
|
|
@@ -55,9 +55,7 @@ export interface GPUInfoContext {
|
|
|
55
55
|
*
|
|
56
56
|
* @property usedBytes - The process's resident footprint (RSS).
|
|
57
57
|
* @property availableBytes - How much more this process may allocate before
|
|
58
|
-
* the OS intervenes.
|
|
59
|
-
* `os_proc_available_memory()`, the number jetsam actually enforces.
|
|
60
|
-
* `undefined` where no such per-process limit is exposed.
|
|
58
|
+
* the OS intervenes. `undefined` where the platform states no such limit.
|
|
61
59
|
*/
|
|
62
60
|
export interface ProcessMemorySample {
|
|
63
61
|
usedBytes: number | undefined;
|
|
@@ -120,6 +120,20 @@ export declare const modelFitBasisSchema: z.ZodEnum<{
|
|
|
120
120
|
"device-memory": "device-memory";
|
|
121
121
|
"device-budget": "device-budget";
|
|
122
122
|
}>;
|
|
123
|
+
/**
|
|
124
|
+
* What kind of evidence a verdict rests on.
|
|
125
|
+
*
|
|
126
|
+
* - `calibration`: a two-sided estimate from coefficients measured on this
|
|
127
|
+
* platform. The only evidence that can support `likely-fits`.
|
|
128
|
+
* - `computed-only`: a floor computed from catalog facts alone — artifact bytes,
|
|
129
|
+
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
130
|
+
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
131
|
+
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
132
|
+
*/
|
|
133
|
+
export declare const modelFitEvidenceSchema: z.ZodEnum<{
|
|
134
|
+
calibration: "calibration";
|
|
135
|
+
"computed-only": "computed-only";
|
|
136
|
+
}>;
|
|
123
137
|
export declare const modelFitBudgetSchema: z.ZodObject<{
|
|
124
138
|
totalBytes: z.ZodNumber;
|
|
125
139
|
usedBytes: z.ZodNumber;
|
|
@@ -134,10 +148,15 @@ export declare const modelFitModelResultSchema: z.ZodObject<{
|
|
|
134
148
|
"likely-fits": "likely-fits";
|
|
135
149
|
"likely-too-large": "likely-too-large";
|
|
136
150
|
}>;
|
|
151
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
152
|
+
calibration: "calibration";
|
|
153
|
+
"computed-only": "computed-only";
|
|
154
|
+
}>>;
|
|
137
155
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
138
156
|
lowerBoundBytes: z.ZodNumber;
|
|
139
157
|
upperBoundBytes: z.ZodNumber;
|
|
140
158
|
}, z.core.$strip>>;
|
|
159
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
141
160
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
142
161
|
reasons: z.ZodArray<z.ZodString>;
|
|
143
162
|
}, z.core.$strip>;
|
|
@@ -157,6 +176,10 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
157
176
|
sequential: "sequential";
|
|
158
177
|
concurrent: "concurrent";
|
|
159
178
|
}>;
|
|
179
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
180
|
+
calibration: "calibration";
|
|
181
|
+
"computed-only": "computed-only";
|
|
182
|
+
}>>;
|
|
160
183
|
budget: z.ZodOptional<z.ZodObject<{
|
|
161
184
|
totalBytes: z.ZodNumber;
|
|
162
185
|
usedBytes: z.ZodNumber;
|
|
@@ -168,6 +191,7 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
168
191
|
lowerBoundBytes: z.ZodNumber;
|
|
169
192
|
upperBoundBytes: z.ZodNumber;
|
|
170
193
|
}, z.core.$strip>>;
|
|
194
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
171
195
|
models: z.ZodArray<z.ZodObject<{
|
|
172
196
|
name: z.ZodString;
|
|
173
197
|
verdict: z.ZodEnum<{
|
|
@@ -175,10 +199,15 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
175
199
|
"likely-fits": "likely-fits";
|
|
176
200
|
"likely-too-large": "likely-too-large";
|
|
177
201
|
}>;
|
|
202
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
203
|
+
calibration: "calibration";
|
|
204
|
+
"computed-only": "computed-only";
|
|
205
|
+
}>>;
|
|
178
206
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
179
207
|
lowerBoundBytes: z.ZodNumber;
|
|
180
208
|
upperBoundBytes: z.ZodNumber;
|
|
181
209
|
}, z.core.$strip>>;
|
|
210
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
182
211
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
183
212
|
reasons: z.ZodArray<z.ZodString>;
|
|
184
213
|
}, z.core.$strip>>;
|
|
@@ -228,6 +257,10 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
228
257
|
sequential: "sequential";
|
|
229
258
|
concurrent: "concurrent";
|
|
230
259
|
}>;
|
|
260
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
261
|
+
calibration: "calibration";
|
|
262
|
+
"computed-only": "computed-only";
|
|
263
|
+
}>>;
|
|
231
264
|
budget: z.ZodOptional<z.ZodObject<{
|
|
232
265
|
totalBytes: z.ZodNumber;
|
|
233
266
|
usedBytes: z.ZodNumber;
|
|
@@ -239,6 +272,7 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
239
272
|
lowerBoundBytes: z.ZodNumber;
|
|
240
273
|
upperBoundBytes: z.ZodNumber;
|
|
241
274
|
}, z.core.$strip>>;
|
|
275
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
242
276
|
models: z.ZodArray<z.ZodObject<{
|
|
243
277
|
name: z.ZodString;
|
|
244
278
|
verdict: z.ZodEnum<{
|
|
@@ -246,10 +280,15 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
246
280
|
"likely-fits": "likely-fits";
|
|
247
281
|
"likely-too-large": "likely-too-large";
|
|
248
282
|
}>;
|
|
283
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
284
|
+
calibration: "calibration";
|
|
285
|
+
"computed-only": "computed-only";
|
|
286
|
+
}>>;
|
|
249
287
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
250
288
|
lowerBoundBytes: z.ZodNumber;
|
|
251
289
|
upperBoundBytes: z.ZodNumber;
|
|
252
290
|
}, z.core.$strip>>;
|
|
291
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
253
292
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
254
293
|
reasons: z.ZodArray<z.ZodString>;
|
|
255
294
|
}, z.core.$strip>>;
|
|
@@ -263,6 +302,7 @@ export type ModelFitWorkload = z.infer<typeof modelFitWorkloadSchema>;
|
|
|
263
302
|
export type ModelFitCandidate = z.infer<typeof modelFitCandidateSchema>;
|
|
264
303
|
export type ModelFitExecution = z.infer<typeof modelFitExecutionSchema>;
|
|
265
304
|
export type ModelFitBasis = z.infer<typeof modelFitBasisSchema>;
|
|
305
|
+
export type ModelFitEvidence = z.infer<typeof modelFitEvidenceSchema>;
|
|
266
306
|
export type ModelFitBudget = z.infer<typeof modelFitBudgetSchema>;
|
|
267
307
|
export type ModelFitModelResult = z.infer<typeof modelFitModelResultSchema>;
|
|
268
308
|
export type AssessModelFitInput = z.input<typeof assessModelFitInputSchema>;
|
|
@@ -108,6 +108,17 @@ export const modelFitBasisSchema = z.enum([
|
|
|
108
108
|
'device-memory',
|
|
109
109
|
'device-budget'
|
|
110
110
|
]);
|
|
111
|
+
/**
|
|
112
|
+
* What kind of evidence a verdict rests on.
|
|
113
|
+
*
|
|
114
|
+
* - `calibration`: a two-sided estimate from coefficients measured on this
|
|
115
|
+
* platform. The only evidence that can support `likely-fits`.
|
|
116
|
+
* - `computed-only`: a floor computed from catalog facts alone — artifact bytes,
|
|
117
|
+
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
118
|
+
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
119
|
+
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
120
|
+
*/
|
|
121
|
+
export const modelFitEvidenceSchema = z.enum(['calibration', 'computed-only']);
|
|
111
122
|
export const modelFitBudgetSchema = z.object({
|
|
112
123
|
totalBytes: z
|
|
113
124
|
.number()
|
|
@@ -126,19 +137,37 @@ export const modelFitBudgetSchema = z.object({
|
|
|
126
137
|
export const modelFitModelResultSchema = z.object({
|
|
127
138
|
name: z.string().describe('Catalog name of the assessed model.'),
|
|
128
139
|
verdict: modelFitVerdictSchema,
|
|
129
|
-
|
|
140
|
+
evidence: modelFitEvidenceSchema
|
|
141
|
+
.optional()
|
|
142
|
+
.describe('What the verdict rests on. Absent when nothing could be computed for this model, e.g. no catalog profile.'),
|
|
143
|
+
estimate: byteRangeSchema
|
|
144
|
+
.optional()
|
|
145
|
+
.describe('Two-sided bound from calibrated coefficients. Absent under computed-only evidence, or when this model assessed as `unknown` for want of any evidence.'),
|
|
146
|
+
floorBytes: z
|
|
147
|
+
.number()
|
|
148
|
+
.optional()
|
|
149
|
+
.describe('Under computed-only evidence: the smallest resident footprint the catalog facts prove — artifact bytes, plus the KV cache for llama.cpp. A floor only; the true cost is above it by an unmeasured amount.'),
|
|
130
150
|
estimatorVersion: z
|
|
131
151
|
.string()
|
|
132
152
|
.optional()
|
|
133
|
-
.describe('Estimator that produced the bounds, e.g. `llm-v1
|
|
153
|
+
.describe('Estimator that produced the bounds, e.g. `llm-v1`, or `floor-v1` for a computed floor.'),
|
|
134
154
|
reasons: z.array(z.string()).describe('Why this model got this verdict.')
|
|
135
155
|
});
|
|
136
156
|
export const assessModelFitResultSchema = z.object({
|
|
137
157
|
verdict: modelFitVerdictSchema.describe('Combined verdict across every candidate.'),
|
|
138
158
|
basis: modelFitBasisSchema.describe('The evidence the budget was derived from — system RAM, the per-process ceiling on iOS, a discrete GPU’s own memory, or on Windows the GPU memory budget the OS grants this process. The two device bases also require the system-memory budget to hold.'),
|
|
139
159
|
execution: modelFitExecutionSchema.describe('The declared execution mode this result assumed.'),
|
|
160
|
+
evidence: modelFitEvidenceSchema
|
|
161
|
+
.optional()
|
|
162
|
+
.describe('The weakest evidence among the candidates: `computed-only` as soon as one model has only a floor, since the combined verdict can then never be `likely-fits`. Absent whenever any candidate could not be assessed at all, so an `unknown` that carries `evidence` is a near-miss or an uncalibrated floor, never a missing model.'),
|
|
140
163
|
budget: modelFitBudgetSchema.optional().describe('Absent when memory evidence was unusable.'),
|
|
141
|
-
estimate: byteRangeSchema
|
|
164
|
+
estimate: byteRangeSchema
|
|
165
|
+
.optional()
|
|
166
|
+
.describe('Combined two-sided bound. Absent when the combined verdict is `unknown` for want of evidence, and under computed-only evidence, which has no upper bound.'),
|
|
167
|
+
floorBytes: z
|
|
168
|
+
.number()
|
|
169
|
+
.optional()
|
|
170
|
+
.describe('Under computed-only evidence: the combined floor across every candidate, aggregated under `execution`. Compared against the budget for `likely-too-large`; never enough for `likely-fits`.'),
|
|
142
171
|
models: z.array(modelFitModelResultSchema).describe('Per-candidate verdicts, in input order.'),
|
|
143
172
|
reasons: z.array(z.string()).describe('Why the combined verdict came out this way.'),
|
|
144
173
|
assumptions: z
|