@qvac/harness 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -1
- package/README.md +15 -2
- package/dist-lib/index.d.ts +3 -0
- package/dist-lib/index.js +29 -17
- package/dist-lib/lib/models/model-handle.d.ts +5 -1
- package/dist-lib/lib/models/model-handle.js +8 -3
- package/dist-lib/lib/models/model-list.d.ts +5 -3
- package/dist-lib/lib/models/model-pool.d.ts +4 -1
- package/dist-lib/lib/models/model-pool.js +33 -10
- package/dist-lib/lib/serve.js +1 -1
- package/dist-lib/schema/harness.js +2 -1
- package/dist-lib/spec/hrpc/messages.js +7 -5
- package/dist-lib/spec/hyperschema/index.js +7 -5
- package/dist-lib/spec/hyperschema/schema.json +6 -1
- package/dist-lib/spec/hyperschema/types.d.ts +2 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,46 @@ While the major version is `0`, a breaking change bumps the minor version.
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [0.7.1] - 2026-09-30
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- `list()` rows carry `isLoading`: true from the moment a load of that model
|
|
16
|
+
starts, whoever asked for it, until it is ready or the load fails. It is read
|
|
17
|
+
off the pool, so a host shows a model as loading without counting loads
|
|
18
|
+
itself. A model loading again under other settings while a copy of it is
|
|
19
|
+
resident reads `isLoaded` and `isLoading` both true. Chat models read it
|
|
20
|
+
today: image, music and translation models load for each generation, and
|
|
21
|
+
speech, transcription and embedding pools key by source, so those rows read
|
|
22
|
+
`false` for now. An optional field on the wire; a row from an older harness
|
|
23
|
+
reads `false`. (#143)
|
|
24
|
+
|
|
25
|
+
```js
|
|
26
|
+
const loading = (await harness.list({ cachedOnly: true })).filter((row) => row.isLoading)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### Changed
|
|
30
|
+
|
|
31
|
+
- The `model` event carries `catalogChanged` at every edge a `list()` re-read
|
|
32
|
+
would show: the `loading` that starts a load and the `ready` or `error` that
|
|
33
|
+
ends it, beside `unloaded`. Progress ticks still do not. A host that re-reads
|
|
34
|
+
the list on `catalogChanged` sees a model turn loading, then loaded, with no
|
|
35
|
+
bookkeeping of its own. The served event stream never drops a frame that
|
|
36
|
+
carries it, so two `ready` frames of one model under two keys both cross. (#143)
|
|
37
|
+
- `ModelInfo` requires `isLoading`, and `listModels`, `customModelRows` and
|
|
38
|
+
`mergeCustomRows` return `CatalogRow`, the row before the pools are asked. A
|
|
39
|
+
typed fake `list()` adds the field. (#143)
|
|
40
|
+
- A custom model's row reads `isLoaded` only once its load has landed; it used
|
|
41
|
+
to read `true` from the first attempt. (#143)
|
|
42
|
+
|
|
43
|
+
### Fixed
|
|
44
|
+
|
|
45
|
+
- A load that ends without the model in memory emits `model` with `error` and
|
|
46
|
+
the reason: it failed, or a suspend or close cancelled it. It used to end in
|
|
47
|
+
silence, so a host could not tell a failed load from one still running. The
|
|
48
|
+
`error` also refreshes the cached `isCached`, since a load may have downloaded
|
|
49
|
+
the weights before it failed. (#143)
|
|
50
|
+
|
|
11
51
|
## [0.7.0] - 2026-09-29
|
|
12
52
|
|
|
13
53
|
### Added
|
|
@@ -979,7 +1019,8 @@ Tagged but never published to npm; these changes first shipped in 0.2.12.
|
|
|
979
1019
|
- GPU stats are read through `bare-gpu-info`, so every dependency installs from
|
|
980
1020
|
the public npm registry. (#12)
|
|
981
1021
|
|
|
982
|
-
[Unreleased]: https://github.com/tetherto/qvac-harness/compare/v0.7.
|
|
1022
|
+
[Unreleased]: https://github.com/tetherto/qvac-harness/compare/v0.7.1...HEAD
|
|
1023
|
+
[0.7.1]: https://github.com/tetherto/qvac-harness/compare/v0.7.0...v0.7.1
|
|
983
1024
|
[0.7.0]: https://github.com/tetherto/qvac-harness/compare/v0.6.0...v0.7.0
|
|
984
1025
|
[0.6.0]: https://github.com/tetherto/qvac-harness/compare/v0.5.0...v0.6.0
|
|
985
1026
|
[0.5.0]: https://github.com/tetherto/qvac-harness/compare/v0.4.0...v0.5.0
|
package/README.md
CHANGED
|
@@ -165,6 +165,7 @@ const custom = models.filter((model) => model.registrySource === 'custom')
|
|
|
165
165
|
// registrySource: 'custom',
|
|
166
166
|
// isCached: false,
|
|
167
167
|
// isLoaded: false,
|
|
168
|
+
// isLoading: false,
|
|
168
169
|
// supportsImages: false,
|
|
169
170
|
// expectedSize: 1868506272,
|
|
170
171
|
// label: 'Nanbeige 4.2 3B',
|
|
@@ -234,8 +235,20 @@ if (big.reason === 'memory') showNotice('Unload a model first')
|
|
|
234
235
|
`unloadModel` waits for any request holding the model to finish, then unloads
|
|
235
236
|
every copy of it. `unloaded` is false when a request still held it after the
|
|
236
237
|
wait, so a caller that wants the model gone at once stops those requests first.
|
|
237
|
-
Both change `isLoaded` in the next `list()
|
|
238
|
-
|
|
238
|
+
Both change `isLoaded` in the next `list()`. `isLoading` is true on a row from
|
|
239
|
+
the moment a load of its model starts, whoever asked for it (a request,
|
|
240
|
+
`warmModel` or `loadModel`), until the model is ready or the load fails; it is
|
|
241
|
+
read off the pool, so a host keeps no count of its own. A model loading again
|
|
242
|
+
under other settings while a copy of it is resident reads `isLoaded: true` and
|
|
243
|
+
`isLoading: true` at once. The `model` event carries `catalogChanged` at each
|
|
244
|
+
edge a `list()` re-read would show: the `loading` that starts a load, the
|
|
245
|
+
`ready` or `error` that ends it, and `unloaded`. Progress ticks between them do
|
|
246
|
+
not. A load that ends without the model in memory reports
|
|
247
|
+
`error` with the reason in `message`: it failed, or a suspend or close cancelled it.
|
|
248
|
+
An image, music or translation model loads for each generation and never reads
|
|
249
|
+
`isLoading: true`; a speech, transcription or embedding model loads under its
|
|
250
|
+
source rather than its row name, so its row reads `false` for now. A load or an
|
|
251
|
+
unload counts as a call in flight for
|
|
239
252
|
[`setModelIdleTimeout`](#freeing-memory-when-idle), and one that lands during an
|
|
240
253
|
idle unload waits for it, so a load never loads a second copy and an unload
|
|
241
254
|
answers once the memory is free.
|
package/dist-lib/index.d.ts
CHANGED
|
@@ -218,6 +218,9 @@ export declare class Harness extends ReadyResource {
|
|
|
218
218
|
describe(): Promise<HarnessInfo>;
|
|
219
219
|
diagnostics(): HarnessDiagnostics;
|
|
220
220
|
private _loadedModels;
|
|
221
|
+
private _loadingModels;
|
|
222
|
+
private _settledModels;
|
|
223
|
+
private _pools;
|
|
221
224
|
private _loadPluginSdk;
|
|
222
225
|
private _loadDiffusionSdk;
|
|
223
226
|
private _slmTranslate;
|
package/dist-lib/index.js
CHANGED
|
@@ -24,7 +24,7 @@ import { AbortedError, raceAbort } from './lib/helpers/race-abort.js';
|
|
|
24
24
|
import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, curatedModelAsset, infraModelsFor, modelAddonOf, modelNameToCapabilityProfile, musicModelSet, residentSrc, setCustomModels } from './lib/models/model-catalog.js';
|
|
25
25
|
import { createModelStatusCache, customModelRows, invalidateModelStatusCache, mergeCustomRows, listModels } from './lib/models/model-list.js';
|
|
26
26
|
import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js';
|
|
27
|
-
import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
|
|
27
|
+
import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, loadingModels, settledModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
|
|
28
28
|
import { defaultContextSize } from './lib/models/context.js';
|
|
29
29
|
import { resolveMainGpu } from './lib/models/gpu-selection.js';
|
|
30
30
|
import { createIdleReaper } from './lib/models/idle-unload.js';
|
|
@@ -558,13 +558,22 @@ export class Harness extends ReadyResource {
|
|
|
558
558
|
};
|
|
559
559
|
}
|
|
560
560
|
_loadedModels() {
|
|
561
|
-
|
|
561
|
+
return this._pools().flatMap((pool) => loadedModels(pool));
|
|
562
|
+
}
|
|
563
|
+
_loadingModels() {
|
|
564
|
+
return new Set(this._pools().flatMap((pool) => loadingModels(pool)));
|
|
565
|
+
}
|
|
566
|
+
_settledModels() {
|
|
567
|
+
return new Set(this._pools().flatMap((pool) => settledModels(pool)));
|
|
568
|
+
}
|
|
569
|
+
_pools() {
|
|
570
|
+
return [
|
|
571
|
+
this._models,
|
|
562
572
|
this._embedModels,
|
|
563
573
|
...this._retiredEmbedModels,
|
|
564
574
|
this._ttsModels,
|
|
565
575
|
this._transcriptionModels
|
|
566
576
|
];
|
|
567
|
-
return [...loadedModels(this._models), ...pools.flatMap((pool) => loadedModels(pool))];
|
|
568
577
|
}
|
|
569
578
|
_loadPluginSdk(current, store, importPlugin) {
|
|
570
579
|
let pending = current();
|
|
@@ -667,8 +676,7 @@ export class Harness extends ReadyResource {
|
|
|
667
676
|
});
|
|
668
677
|
}
|
|
669
678
|
_notifyModel(state) {
|
|
670
|
-
|
|
671
|
-
if (catalogChanged) {
|
|
679
|
+
if (state.status === 'ready' || state.status === 'unloaded' || state.status === 'error') {
|
|
672
680
|
invalidateModelStatusCache(this._modelStatus);
|
|
673
681
|
}
|
|
674
682
|
if (state.status === 'unloaded') {
|
|
@@ -678,7 +686,11 @@ export class Harness extends ReadyResource {
|
|
|
678
686
|
}
|
|
679
687
|
}
|
|
680
688
|
}
|
|
681
|
-
this.emit('model', {
|
|
689
|
+
this.emit('model', {
|
|
690
|
+
...state,
|
|
691
|
+
catalogChanged: state.catalogChanged === true,
|
|
692
|
+
loaded: this._loadedModels()
|
|
693
|
+
});
|
|
682
694
|
}
|
|
683
695
|
async _collectRetiredEmbedPool(pool) {
|
|
684
696
|
if (!this._retiredEmbedModels.has(pool) || pool.idle !== pool.size)
|
|
@@ -1055,17 +1067,17 @@ export class Harness extends ReadyResource {
|
|
|
1055
1067
|
if (!this.opened)
|
|
1056
1068
|
await this.ready();
|
|
1057
1069
|
const sdk = await this._sdk.load();
|
|
1058
|
-
const
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1070
|
+
const rows = await listModels(sdk, options, {
|
|
1071
|
+
cache: this._modelStatus,
|
|
1072
|
+
projectionFor: (name) => this._projectionFor(name)
|
|
1073
|
+
});
|
|
1074
|
+
const loaded = this._settledModels();
|
|
1075
|
+
const loading = this._loadingModels();
|
|
1076
|
+
const custom = await customModelRows(options, { modelsDir: this._modelsDir, loaded });
|
|
1077
|
+
return mergeCustomRows(rows, custom).map((row) => ({
|
|
1078
|
+
...row,
|
|
1079
|
+
isLoading: loading.has(row.name)
|
|
1080
|
+
}));
|
|
1069
1081
|
}
|
|
1070
1082
|
async deleteModel(name) {
|
|
1071
1083
|
if (!this.opened)
|
|
@@ -5,13 +5,17 @@ export interface ModelResource {
|
|
|
5
5
|
}
|
|
6
6
|
export interface ModelHandleOptions {
|
|
7
7
|
load(): Promise<ModelResource>;
|
|
8
|
+
onLoaded?(): void;
|
|
9
|
+
onFailed?(err: unknown): void;
|
|
8
10
|
}
|
|
9
11
|
export declare class ModelHandle {
|
|
10
12
|
private readonly _load;
|
|
11
13
|
private readonly _peek;
|
|
14
|
+
private readonly _onLoaded;
|
|
15
|
+
private readonly _onFailed;
|
|
12
16
|
private _tracked;
|
|
13
17
|
private _loading;
|
|
14
|
-
constructor({ load }: ModelHandleOptions);
|
|
18
|
+
constructor({ load, onLoaded, onFailed }: ModelHandleOptions);
|
|
15
19
|
load(): Promise<ModelResource>;
|
|
16
20
|
get loading(): boolean;
|
|
17
21
|
peek(): Promise<ModelResource> | null;
|
|
@@ -2,12 +2,16 @@ import { retryMemo } from '../helpers/retry-memo.js';
|
|
|
2
2
|
export class ModelHandle {
|
|
3
3
|
_load;
|
|
4
4
|
_peek;
|
|
5
|
+
_onLoaded;
|
|
6
|
+
_onFailed;
|
|
5
7
|
_tracked = null;
|
|
6
8
|
_loading = false;
|
|
7
|
-
constructor({ load }) {
|
|
9
|
+
constructor({ load, onLoaded, onFailed }) {
|
|
8
10
|
const memo = retryMemo(load);
|
|
9
11
|
this._load = memo;
|
|
10
12
|
this._peek = memo.peek;
|
|
13
|
+
this._onLoaded = onLoaded;
|
|
14
|
+
this._onFailed = onFailed;
|
|
11
15
|
this.load().catch(noop);
|
|
12
16
|
}
|
|
13
17
|
load() {
|
|
@@ -15,12 +19,13 @@ export class ModelHandle {
|
|
|
15
19
|
if (attempt !== this._tracked) {
|
|
16
20
|
this._tracked = attempt;
|
|
17
21
|
this._loading = true;
|
|
18
|
-
const
|
|
22
|
+
const settle = (report) => {
|
|
19
23
|
if (this._tracked === attempt) {
|
|
20
24
|
this._loading = false;
|
|
25
|
+
report();
|
|
21
26
|
}
|
|
22
27
|
};
|
|
23
|
-
attempt.then(
|
|
28
|
+
attempt.then(() => settle(() => this._onLoaded?.()), (err) => settle(() => this._onFailed?.(err)));
|
|
24
29
|
}
|
|
25
30
|
return attempt;
|
|
26
31
|
}
|
|
@@ -16,12 +16,14 @@ export interface ModelInfo {
|
|
|
16
16
|
registrySource: string;
|
|
17
17
|
isCached: boolean;
|
|
18
18
|
isLoaded: boolean;
|
|
19
|
+
isLoading: boolean;
|
|
19
20
|
supportsImages: boolean;
|
|
20
21
|
expectedSize: number;
|
|
21
22
|
actualSize?: number;
|
|
22
23
|
label?: string;
|
|
23
24
|
taskType: ModelTaskType;
|
|
24
25
|
}
|
|
26
|
+
export type CatalogRow = Omit<ModelInfo, 'isLoading'>;
|
|
25
27
|
export interface ListModelsOptions {
|
|
26
28
|
type?: ModelType;
|
|
27
29
|
cachedOnly?: boolean;
|
|
@@ -45,11 +47,11 @@ export interface ListModelsRuntime {
|
|
|
45
47
|
cache?: ModelStatusCache | undefined;
|
|
46
48
|
projectionFor?: ((name: string) => string | undefined) | undefined;
|
|
47
49
|
}
|
|
48
|
-
export declare function listModels(sdk: ModelListSdk, { type, cachedOnly, fullRegistry }?: ListModelsOptions, { cache, projectionFor }?: ListModelsRuntime): Promise<
|
|
50
|
+
export declare function listModels(sdk: ModelListSdk, { type, cachedOnly, fullRegistry }?: ListModelsOptions, { cache, projectionFor }?: ListModelsRuntime): Promise<CatalogRow[]>;
|
|
49
51
|
export interface CustomModelRowsContext {
|
|
50
52
|
readonly modelsDir?: string;
|
|
51
53
|
readonly loaded: ReadonlySet<string>;
|
|
52
54
|
}
|
|
53
|
-
export declare function customModelRows({ type, cachedOnly }: ListModelsOptions, { modelsDir, loaded }: CustomModelRowsContext): Promise<
|
|
54
|
-
export declare function mergeCustomRows(catalog: ReadonlyArray<
|
|
55
|
+
export declare function customModelRows({ type, cachedOnly }: ListModelsOptions, { modelsDir, loaded }: CustomModelRowsContext): Promise<CatalogRow[]>;
|
|
56
|
+
export declare function mergeCustomRows(catalog: ReadonlyArray<CatalogRow>, custom: ReadonlyArray<CatalogRow>): CatalogRow[];
|
|
55
57
|
export {};
|
|
@@ -5,8 +5,9 @@ import { type ModelAddon } from './model-source.ts';
|
|
|
5
5
|
import type { GpuFaultState } from './sdk-faults.ts';
|
|
6
6
|
type SdkModule = typeof import('@qvac/sdk');
|
|
7
7
|
export interface ModelState {
|
|
8
|
-
status: 'loading' | 'downloading' | 'downloaded' | 'ready' | 'unloading' | 'unloaded';
|
|
8
|
+
status: 'loading' | 'downloading' | 'downloaded' | 'ready' | 'error' | 'unloading' | 'unloaded';
|
|
9
9
|
model: string;
|
|
10
|
+
catalogChanged?: boolean;
|
|
10
11
|
[key: string]: unknown;
|
|
11
12
|
}
|
|
12
13
|
export type NotifyFn = (state: ModelState) => void;
|
|
@@ -78,6 +79,8 @@ type PoolLike = ModelPuddle;
|
|
|
78
79
|
export declare function modelOf(key: string): string;
|
|
79
80
|
export declare function releaseModel(pool: PoolLike, key: string): void;
|
|
80
81
|
export declare function loadedModels(pool: PoolLike): LoadedModel[];
|
|
82
|
+
export declare function settledModels(pool: PoolLike): string[];
|
|
83
|
+
export declare function loadingModels(pool: PoolLike): string[];
|
|
81
84
|
export interface ResidentLease {
|
|
82
85
|
model: string;
|
|
83
86
|
modelId: string;
|
|
@@ -10,6 +10,7 @@ import { hlog } from '../log.js';
|
|
|
10
10
|
import { RUNAWAY_STOP_SEQUENCES } from '../helpers/control-tokens.js';
|
|
11
11
|
import { raceTimeout } from '../helpers/race-timeout.js';
|
|
12
12
|
import { resolveModelSrc } from './model-source.js';
|
|
13
|
+
import { errorMessage } from '../guards.js';
|
|
13
14
|
export function createLoadTracker() {
|
|
14
15
|
const inflight = new Set();
|
|
15
16
|
let draining = null;
|
|
@@ -101,6 +102,34 @@ export function loadedModels(pool) {
|
|
|
101
102
|
}
|
|
102
103
|
return out;
|
|
103
104
|
}
|
|
105
|
+
export function settledModels(pool) {
|
|
106
|
+
const out = [];
|
|
107
|
+
for (const entry of pool.resources.values()) {
|
|
108
|
+
if (entry.value.peek() !== null && !entry.value.loading) {
|
|
109
|
+
out.push(modelOf(entry.id));
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
export function loadingModels(pool) {
|
|
115
|
+
const out = [];
|
|
116
|
+
for (const entry of pool.resources.values()) {
|
|
117
|
+
if (entry.value.loading) {
|
|
118
|
+
out.push(modelOf(entry.id));
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
function reportedHandle(notify, model, load) {
|
|
124
|
+
return new ModelHandle({
|
|
125
|
+
load: () => {
|
|
126
|
+
notify({ status: 'loading', model, catalogChanged: true });
|
|
127
|
+
return load();
|
|
128
|
+
},
|
|
129
|
+
onLoaded: () => notify({ status: 'ready', model, catalogChanged: true }),
|
|
130
|
+
onFailed: (err) => notify({ status: 'error', model, message: errorMessage(err), catalogChanged: true })
|
|
131
|
+
});
|
|
132
|
+
}
|
|
104
133
|
export async function leaseResidentModel(pool, wanted = () => true) {
|
|
105
134
|
for (const entry of pool.resources.values()) {
|
|
106
135
|
const key = entry.id;
|
|
@@ -292,7 +321,6 @@ export function loadClockGuard(model, sdk, { now = () => Date.now(), monoMs = mo
|
|
|
292
321
|
export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = createLoadTracker() } = {}) {
|
|
293
322
|
const load = async (model, opts) => {
|
|
294
323
|
const sdk = await loadSdk();
|
|
295
|
-
notify({ status: 'loading', model });
|
|
296
324
|
const modelSrc = await resolveModelSrc(sdk, model);
|
|
297
325
|
const resolved = resolveLoadOptions(opts, deviceConfig);
|
|
298
326
|
const ctxSize = resolved.ctxSize || defaultContextSize(modelSrc, customContextLength(model));
|
|
@@ -332,11 +360,10 @@ export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = cr
|
|
|
332
360
|
onProgress: (progress) => notify(progressState(model, progress))
|
|
333
361
|
})));
|
|
334
362
|
hlog('llm loaded', { model, loadMs: Date.now() - loadStarted });
|
|
335
|
-
notify({ status: 'ready', model });
|
|
336
363
|
return { model, modelId, ctxSize };
|
|
337
364
|
};
|
|
338
365
|
return new ModelPuddle({
|
|
339
|
-
open: (key, model = key, opts = {}) =>
|
|
366
|
+
open: (key, model = key, opts = {}) => reportedHandle(notify, model, () => load(model, opts)),
|
|
340
367
|
gc: unloadOnGc(loadSdk, notify),
|
|
341
368
|
...NO_IDLE_GC
|
|
342
369
|
});
|
|
@@ -344,7 +371,6 @@ export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = cr
|
|
|
344
371
|
export function createEmbedPool(loadSdk, notify, { faults, batchSize = DEFAULT_EMBED_BATCH_SIZE, openclCacheDir, loads = createLoadTracker() }) {
|
|
345
372
|
const load = async (model) => {
|
|
346
373
|
const sdk = await loadSdk();
|
|
347
|
-
notify({ status: 'loading', model });
|
|
348
374
|
const modelSrc = await resolveModelSrc(sdk, model, 'embeddings');
|
|
349
375
|
const openclCache = faults.device !== 'cpu' && openclCacheDir ? openclCacheDir() : undefined;
|
|
350
376
|
const modelId = await loadClockGuard(model, sdk).race(loads.track(sdk.loadModel({
|
|
@@ -358,11 +384,10 @@ export function createEmbedPool(loadSdk, notify, { faults, batchSize = DEFAULT_E
|
|
|
358
384
|
},
|
|
359
385
|
onProgress: (progress) => notify(progressState(model, progress))
|
|
360
386
|
})));
|
|
361
|
-
notify({ status: 'ready', model });
|
|
362
387
|
return { model, modelId };
|
|
363
388
|
};
|
|
364
389
|
return new ModelPuddle({
|
|
365
|
-
open: (key, model = key) =>
|
|
390
|
+
open: (key, model = key) => reportedHandle(notify, model, () => load(model)),
|
|
366
391
|
gc: unloadOnGc(loadSdk, notify),
|
|
367
392
|
...NO_IDLE_GC
|
|
368
393
|
});
|
|
@@ -372,7 +397,6 @@ export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig
|
|
|
372
397
|
let gpuRefused = false;
|
|
373
398
|
const load = async (model) => {
|
|
374
399
|
const sdk = await loadSdk();
|
|
375
|
-
notify({ status: 'loading', model });
|
|
376
400
|
const modelSrc = await resolveModelSrc(sdk, model, addon);
|
|
377
401
|
const attempt = () => {
|
|
378
402
|
const config = gpuRefused ? { ...modelConfig, useGPU: false } : modelConfig;
|
|
@@ -391,11 +415,10 @@ export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig
|
|
|
391
415
|
gpuRefused = true;
|
|
392
416
|
return attempt();
|
|
393
417
|
}));
|
|
394
|
-
notify({ status: 'ready', model });
|
|
395
418
|
return { model, modelId };
|
|
396
419
|
};
|
|
397
420
|
return new ModelPuddle({
|
|
398
|
-
open: (key, model = key) =>
|
|
421
|
+
open: (key, model = key) => reportedHandle(notify, model, () => load(model)),
|
|
399
422
|
gc: unloadOnGc(loadSdk, notify),
|
|
400
423
|
...NO_IDLE_GC
|
|
401
424
|
});
|
|
@@ -423,7 +446,7 @@ function unloadOnGc(loadSdk, notify) {
|
|
|
423
446
|
catch {
|
|
424
447
|
continue;
|
|
425
448
|
}
|
|
426
|
-
notify({ status: 'unloaded', model });
|
|
449
|
+
notify({ status: 'unloaded', model, catalogChanged: true });
|
|
427
450
|
}
|
|
428
451
|
};
|
|
429
452
|
}
|
package/dist-lib/lib/serve.js
CHANGED
|
@@ -339,7 +339,8 @@ export function registerHarnessTypes(ns) {
|
|
|
339
339
|
{ name: 'supportsImages', type: 'bool', required: true },
|
|
340
340
|
{ name: 'expectedSize', type: 'uint', required: true },
|
|
341
341
|
{ name: 'actualSize', type: 'uint' },
|
|
342
|
-
{ name: 'label', type: 'string' }
|
|
342
|
+
{ name: 'label', type: 'string' },
|
|
343
|
+
{ name: 'isLoading', type: 'bool' }
|
|
343
344
|
]
|
|
344
345
|
});
|
|
345
346
|
ns.register({
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// This file is autogenerated by the hyperschema compiler
|
|
2
|
-
// Schema Version:
|
|
2
|
+
// Schema Version: 18
|
|
3
3
|
/* eslint-disable camelcase */
|
|
4
4
|
/* eslint-disable quotes */
|
|
5
5
|
/* eslint-disable space-before-function-paren */
|
|
6
6
|
|
|
7
7
|
import { c } from 'hyperschema/runtime'
|
|
8
8
|
|
|
9
|
-
const VERSION =
|
|
9
|
+
const VERSION = 18
|
|
10
10
|
|
|
11
11
|
// eslint-disable-next-line no-unused-vars
|
|
12
12
|
let version = VERSION
|
|
@@ -1610,7 +1610,7 @@ const encoding31 = {
|
|
|
1610
1610
|
c.string.preencode(state, m.type)
|
|
1611
1611
|
c.string.preencode(state, m.registryPath)
|
|
1612
1612
|
c.string.preencode(state, m.registrySource)
|
|
1613
|
-
state.end++ // max flag is
|
|
1613
|
+
state.end++ // max flag is 32 so always one byte
|
|
1614
1614
|
c.uint.preencode(state, m.expectedSize)
|
|
1615
1615
|
|
|
1616
1616
|
if (m.actualSize) c.uint.preencode(state, m.actualSize)
|
|
@@ -1622,7 +1622,8 @@ const encoding31 = {
|
|
|
1622
1622
|
(m.isLoaded ? 2 : 0) |
|
|
1623
1623
|
(m.supportsImages ? 4 : 0) |
|
|
1624
1624
|
(m.actualSize ? 8 : 0) |
|
|
1625
|
-
((version >= 15 && m.label) ? 16 : 0)
|
|
1625
|
+
((version >= 15 && m.label) ? 16 : 0) |
|
|
1626
|
+
((version >= 18 && m.isLoading) ? 32 : 0)
|
|
1626
1627
|
|
|
1627
1628
|
c.string.encode(state, m.name)
|
|
1628
1629
|
c.string.encode(state, m.type)
|
|
@@ -1651,7 +1652,8 @@ const encoding31 = {
|
|
|
1651
1652
|
supportsImages: (flags & 4) !== 0,
|
|
1652
1653
|
expectedSize: c.uint.decode(state),
|
|
1653
1654
|
actualSize: (flags & 8) !== 0 ? c.uint.decode(state) : 0,
|
|
1654
|
-
label: (version >= 15 && (flags & 16) !== 0) ? c.string.decode(state) : null
|
|
1655
|
+
label: (version >= 15 && (flags & 16) !== 0) ? c.string.decode(state) : null,
|
|
1656
|
+
isLoading: (version >= 18 && (flags & 32) !== 0)
|
|
1655
1657
|
}
|
|
1656
1658
|
}
|
|
1657
1659
|
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// This file is autogenerated by the hyperschema compiler
|
|
2
|
-
// Schema Version:
|
|
2
|
+
// Schema Version: 18
|
|
3
3
|
/* eslint-disable camelcase */
|
|
4
4
|
/* eslint-disable quotes */
|
|
5
5
|
/* eslint-disable space-before-function-paren */
|
|
6
6
|
|
|
7
7
|
import { c } from 'hyperschema/runtime'
|
|
8
8
|
|
|
9
|
-
const VERSION =
|
|
9
|
+
const VERSION = 18
|
|
10
10
|
|
|
11
11
|
// eslint-disable-next-line no-unused-vars
|
|
12
12
|
let version = VERSION
|
|
@@ -1610,7 +1610,7 @@ const encoding31 = {
|
|
|
1610
1610
|
c.string.preencode(state, m.type)
|
|
1611
1611
|
c.string.preencode(state, m.registryPath)
|
|
1612
1612
|
c.string.preencode(state, m.registrySource)
|
|
1613
|
-
state.end++ // max flag is
|
|
1613
|
+
state.end++ // max flag is 32 so always one byte
|
|
1614
1614
|
c.uint.preencode(state, m.expectedSize)
|
|
1615
1615
|
|
|
1616
1616
|
if (m.actualSize) c.uint.preencode(state, m.actualSize)
|
|
@@ -1622,7 +1622,8 @@ const encoding31 = {
|
|
|
1622
1622
|
(m.isLoaded ? 2 : 0) |
|
|
1623
1623
|
(m.supportsImages ? 4 : 0) |
|
|
1624
1624
|
(m.actualSize ? 8 : 0) |
|
|
1625
|
-
((version >= 15 && m.label) ? 16 : 0)
|
|
1625
|
+
((version >= 15 && m.label) ? 16 : 0) |
|
|
1626
|
+
((version >= 18 && m.isLoading) ? 32 : 0)
|
|
1626
1627
|
|
|
1627
1628
|
c.string.encode(state, m.name)
|
|
1628
1629
|
c.string.encode(state, m.type)
|
|
@@ -1651,7 +1652,8 @@ const encoding31 = {
|
|
|
1651
1652
|
supportsImages: (flags & 4) !== 0,
|
|
1652
1653
|
expectedSize: c.uint.decode(state),
|
|
1653
1654
|
actualSize: (flags & 8) !== 0 ? c.uint.decode(state) : 0,
|
|
1654
|
-
label: (version >= 15 && (flags & 16) !== 0) ? c.string.decode(state) : null
|
|
1655
|
+
label: (version >= 15 && (flags & 16) !== 0) ? c.string.decode(state) : null,
|
|
1656
|
+
isLoading: (version >= 18 && (flags & 32) !== 0)
|
|
1655
1657
|
}
|
|
1656
1658
|
}
|
|
1657
1659
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version":
|
|
2
|
+
"version": 18,
|
|
3
3
|
"schema": [
|
|
4
4
|
{
|
|
5
5
|
"name": "empty",
|
|
@@ -1341,6 +1341,11 @@
|
|
|
1341
1341
|
"name": "label",
|
|
1342
1342
|
"type": "string",
|
|
1343
1343
|
"version": 15
|
|
1344
|
+
},
|
|
1345
|
+
{
|
|
1346
|
+
"name": "isLoading",
|
|
1347
|
+
"type": "bool",
|
|
1348
|
+
"version": 18
|
|
1344
1349
|
}
|
|
1345
1350
|
]
|
|
1346
1351
|
},
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is autogenerated by hyperschema-ts
|
|
2
|
-
// Schema version:
|
|
2
|
+
// Schema version: 18
|
|
3
3
|
// Do not edit manually.
|
|
4
4
|
|
|
5
5
|
export interface HarnessEmpty {
|
|
@@ -257,6 +257,7 @@ export interface HarnessModelInfo {
|
|
|
257
257
|
expectedSize: number
|
|
258
258
|
actualSize?: number
|
|
259
259
|
label?: string | null
|
|
260
|
+
isLoading?: boolean
|
|
260
261
|
}
|
|
261
262
|
|
|
262
263
|
export interface HarnessListModelsRequest {
|