@telora/daemon 0.18.61 → 0.18.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-info.json +2 -2
- package/dist/config.d.ts +16 -3
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +69 -13
- package/dist/config.js.map +1 -1
- package/dist/discovery-poll.d.ts.map +1 -1
- package/dist/discovery-poll.js +14 -2
- package/dist/discovery-poll.js.map +1 -1
- package/dist/focus-executor.d.ts +26 -3
- package/dist/focus-executor.d.ts.map +1 -1
- package/dist/focus-executor.js +79 -0
- package/dist/focus-executor.js.map +1 -1
- package/dist/git-fetch.d.ts +123 -0
- package/dist/git-fetch.d.ts.map +1 -0
- package/dist/git-fetch.js +164 -0
- package/dist/git-fetch.js.map +1 -0
- package/dist/git-state-detector.d.ts.map +1 -1
- package/dist/git-state-detector.js +63 -26
- package/dist/git-state-detector.js.map +1 -1
- package/dist/host-control/capability.d.ts +31 -0
- package/dist/host-control/capability.d.ts.map +1 -0
- package/dist/host-control/capability.js +50 -0
- package/dist/host-control/capability.js.map +1 -0
- package/dist/host-control/catalog.d.ts +48 -0
- package/dist/host-control/catalog.d.ts.map +1 -0
- package/dist/host-control/catalog.js +149 -0
- package/dist/host-control/catalog.js.map +1 -0
- package/dist/host-control/client.d.ts +76 -0
- package/dist/host-control/client.d.ts.map +1 -0
- package/dist/host-control/client.js +203 -0
- package/dist/host-control/client.js.map +1 -0
- package/dist/host-control/index.d.ts +18 -0
- package/dist/host-control/index.d.ts.map +1 -0
- package/dist/host-control/index.js +14 -0
- package/dist/host-control/index.js.map +1 -0
- package/dist/host-control/load.d.ts +136 -0
- package/dist/host-control/load.d.ts.map +1 -0
- package/dist/host-control/load.js +241 -0
- package/dist/host-control/load.js.map +1 -0
- package/dist/host-control/sizing.d.ts +207 -0
- package/dist/host-control/sizing.d.ts.map +1 -0
- package/dist/host-control/sizing.js +464 -0
- package/dist/host-control/sizing.js.map +1 -0
- package/dist/host-control/types.d.ts +174 -0
- package/dist/host-control/types.d.ts.map +1 -0
- package/dist/host-control/types.js +16 -0
- package/dist/host-control/types.js.map +1 -0
- package/dist/prd-controller.d.ts +10 -12
- package/dist/prd-controller.d.ts.map +1 -1
- package/dist/prd-controller.js +24 -54
- package/dist/prd-controller.js.map +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Load / unload orchestration for the host-control plane (Local model host
|
|
3
|
+
* control, D2).
|
|
4
|
+
*
|
|
5
|
+
* Per-pass model selection (D1) records which local model a coding vs review
|
|
6
|
+
* pass wants, but the daemon had no LOAD path -- so the selection could not
|
|
7
|
+
* drive an actual model switch. This module closes that gap: it loads / unloads
|
|
8
|
+
* the chosen per-pass model around each pass as a SEQUENTIAL swap on a single
|
|
9
|
+
* GPU (one model resident at a time -- loading evicts the prior), so a distinct
|
|
10
|
+
* coding-vs-review model can run end-to-end with NO manual swap in the studio.
|
|
11
|
+
*
|
|
12
|
+
* The control endpoints live under `/v1` (reachable from the stored baseUrl
|
|
13
|
+
* directly, which already ends in `/v1`): `POST /v1/load`, `POST /v1/unload`,
|
|
14
|
+
* `GET /v1/status`. Live JSON shapes are host-specific; `/v1/status` in
|
|
15
|
+
* particular is NOT pinned, so `normalizeStatus` is DEFENSIVE (tolerant of
|
|
16
|
+
* reasonable field names) and must be reconciled per the runbook.
|
|
17
|
+
*
|
|
18
|
+
* The maxSeqLength a load uses is an INJECTED input (an option, never hardcoded
|
|
19
|
+
* inside `ensureLoaded`) so Delivery 3 can replace the fixed `DEFAULT_MAX_SEQ_LENGTH`
|
|
20
|
+
* floor with a computed safe value without re-architecting this module.
|
|
21
|
+
*/
|
|
22
|
+
import type { HostStatus, LoadModelParams, SwapDecision } from './types.js';
|
|
23
|
+
/**
|
|
24
|
+
* The control surface lives under `/v1` -- the SAME segment the stored baseUrl
|
|
25
|
+
* already ends in. We append these to the baseUrl directly (NOT the host root):
|
|
26
|
+
* a baseUrl of `http://host:8888/v1` yields `http://host:8888/v1/load`. A
|
|
27
|
+
* baseUrl without a trailing `/v1` still composes a sane path.
|
|
28
|
+
*/
|
|
29
|
+
export declare const HOST_CONTROL_LOAD_PATH = "/load";
|
|
30
|
+
export declare const HOST_CONTROL_UNLOAD_PATH = "/unload";
|
|
31
|
+
export declare const HOST_CONTROL_STATUS_PATH = "/status";
|
|
32
|
+
/**
|
|
33
|
+
* Default max sequence length for a load. Sized at the live-proven 16384, which
|
|
34
|
+
* clears codex's ~8k turn-1 context floor with headroom. NOTE: Delivery 3 will
|
|
35
|
+
* REPLACE this fixed value with a computed safe value derived from the model +
|
|
36
|
+
* available VRAM; `loadModel`/`ensureLoaded` take maxSeqLength as an INJECTED
|
|
37
|
+
* input so D3 can pass a computed value without touching this module's shape.
|
|
38
|
+
*/
|
|
39
|
+
export declare const DEFAULT_MAX_SEQ_LENGTH = 16384;
|
|
40
|
+
/**
|
|
41
|
+
* How long `ensureLoaded` will poll `/v1/status` for the target to become ready
|
|
42
|
+
* after issuing a load before giving up with a typed timeout. A cold load on a
|
|
43
|
+
* 12GB GPU has real latency; this is the bounded ceiling, not the expected wait.
|
|
44
|
+
*/
|
|
45
|
+
export declare const HOST_CONTROL_LOAD_READY_TIMEOUT_MS = 180000;
|
|
46
|
+
/** Interval between `/v1/status` readiness polls while waiting for a load. */
|
|
47
|
+
export declare const HOST_CONTROL_LOAD_POLL_INTERVAL_MS = 2000;
|
|
48
|
+
/** Address-free category for a load that never reached ready within the budget. */
|
|
49
|
+
export declare const HOST_CONTROL_ERROR_LOAD_TIMEOUT = "Host load did not reach ready in time";
|
|
50
|
+
/**
|
|
51
|
+
* Normalize a `/v1/status` payload into a typed `HostStatus`. DEFENSIVE: the
|
|
52
|
+
* live shape is NOT pinned, so we tolerate the loaded-model id under any of
|
|
53
|
+
* `model`/`id`/`loaded_model`/... and readiness under `status`/`state`/`ready`
|
|
54
|
+
* (string token OR boolean). A payload that proves nothing loaded yields
|
|
55
|
+
* `{ loadedModelId: null, ready: false }`. Pure + exported for unit testing.
|
|
56
|
+
*
|
|
57
|
+
* Runbook flag: confirm the live `/v1/status` field names + readiness vocabulary
|
|
58
|
+
* and tighten `STATUS_*_KEYS` / `READY_TOKENS` once observed.
|
|
59
|
+
*/
|
|
60
|
+
export declare function normalizeStatus(payload: unknown): HostStatus;
|
|
61
|
+
/**
|
|
62
|
+
* Issue a model load (`POST /v1/load`). Maps camelCase app params to the host's
|
|
63
|
+
* snake_case wire field names (`model_path`, `max_seq_length`, `cache_type_kv`,
|
|
64
|
+
* `gpu_ids`) at THIS boundary only. `maxSeqLength` is a required INJECTED input
|
|
65
|
+
* so D3 can pass a computed value. Throws `HostControlError` on any failure.
|
|
66
|
+
*/
|
|
67
|
+
export declare function loadModel(baseUrl: string, params: LoadModelParams, apiKey?: string | null): Promise<void>;
|
|
68
|
+
/** Issue an unload (`POST /v1/unload`) -- evicts whatever is currently resident. */
|
|
69
|
+
export declare function unloadModel(baseUrl: string, apiKey?: string | null): Promise<void>;
|
|
70
|
+
/** Read current host status (`GET /v1/status`), normalized to a typed shape. */
|
|
71
|
+
export declare function getStatus(baseUrl: string, apiKey?: string | null): Promise<HostStatus>;
|
|
72
|
+
/**
|
|
73
|
+
* Resolve a chosen local_models `model` id to the host's `model_path` by reading
|
|
74
|
+
* the loadable catalog (D1) and matching by id. Returns null when the host is
|
|
75
|
+
* not controllable, the id is not in the catalog, or the matching entry carries
|
|
76
|
+
* no modelPath (a floor entry). The caller treats null as "cannot load" and
|
|
77
|
+
* skips the swap.
|
|
78
|
+
*/
|
|
79
|
+
export declare function resolveModelPath(baseUrl: string, modelId: string, apiKey?: string | null): Promise<string | null>;
|
|
80
|
+
/**
|
|
81
|
+
* PURE swap decision over the current vs target resident model. Extracted from
|
|
82
|
+
* `ensureLoaded` so the core single-GPU swap logic is unit-testable with NO HTTP:
|
|
83
|
+
*
|
|
84
|
+
* - target already resident + ready -> 'noop'
|
|
85
|
+
* - a DIFFERENT model resident -> 'evict_then_load' (single GPU -- evict first)
|
|
86
|
+
* - nothing resident -> 'load'
|
|
87
|
+
*
|
|
88
|
+
* `current.modelId` is the currently-loaded id (null = nothing resident);
|
|
89
|
+
* `current.ready` gates the noop (a resident-but-not-ready target still needs a
|
|
90
|
+
* poll, but no reload -- treated as 'load' here so `ensureLoaded` re-issues +
|
|
91
|
+
* polls; the host treats a load of the resident model as a fast no-op/confirm).
|
|
92
|
+
*/
|
|
93
|
+
export declare function decideSwap(input: {
|
|
94
|
+
current: {
|
|
95
|
+
modelId: string | null;
|
|
96
|
+
ready: boolean;
|
|
97
|
+
};
|
|
98
|
+
target: string;
|
|
99
|
+
}): SwapDecision;
|
|
100
|
+
/** Options for `ensureLoaded`. `maxSeqLength` is injected (D3 computes it). */
|
|
101
|
+
export interface EnsureLoadedOptions {
|
|
102
|
+
/** Context window to load with. Defaults to `DEFAULT_MAX_SEQ_LENGTH`. */
|
|
103
|
+
maxSeqLength?: number;
|
|
104
|
+
/** Optional KV cache dtype passed through to the load. */
|
|
105
|
+
cacheTypeKv?: string;
|
|
106
|
+
/** Optional GPU id selection passed through to the load. */
|
|
107
|
+
gpuIds?: number[];
|
|
108
|
+
/** Override the ready-poll budget (ms). Defaults to the module constant. */
|
|
109
|
+
readyTimeoutMs?: number;
|
|
110
|
+
/** Override the ready-poll interval (ms). Defaults to the module constant. */
|
|
111
|
+
pollIntervalMs?: number;
|
|
112
|
+
/**
|
|
113
|
+
* Injectable sleep, so the timeout/poll path is fast + deterministic in tests.
|
|
114
|
+
* Defaults to a real `setTimeout` wait.
|
|
115
|
+
*/
|
|
116
|
+
sleep?: (ms: number) => Promise<void>;
|
|
117
|
+
/** Injectable clock for the bounded-poll deadline (defaults to `Date.now`). */
|
|
118
|
+
now?: () => number;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Orchestrate the SEQUENTIAL single-GPU swap so the target model is resident +
|
|
122
|
+
* ready before the caller spawns against it:
|
|
123
|
+
*
|
|
124
|
+
* 1. read status; if the target is already resident + ready -> no-op return.
|
|
125
|
+
* 2. else: if a DIFFERENT model is resident, unload it first (single GPU --
|
|
126
|
+
* one resident at a time). Resolve the target id -> model_path, then load
|
|
127
|
+
* with the injected maxSeqLength.
|
|
128
|
+
* 3. poll `/v1/status` until the target is ready, or throw a typed timeout
|
|
129
|
+
* (`HOST_CONTROL_ERROR_LOAD_TIMEOUT`) once the bounded budget elapses.
|
|
130
|
+
*
|
|
131
|
+
* Throws `HostControlError` on an unresolvable path, a load/unload failure, or
|
|
132
|
+
* the ready timeout. The caller (focus-executor) turns any throw into "skip the
|
|
133
|
+
* spawn this tick" -- it must NOT spawn against a not-yet-loaded model.
|
|
134
|
+
*/
|
|
135
|
+
export declare function ensureLoaded(baseUrl: string, modelId: string, opts?: EnsureLoadedOptions, apiKey?: string | null): Promise<void>;
|
|
136
|
+
//# sourceMappingURL=load.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../src/host-control/load.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAc,YAAY,EAAE,MAAM,YAAY,CAAC;AAExF;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,UAAU,CAAC;AAC9C,eAAO,MAAM,wBAAwB,YAAY,CAAC;AAClD,eAAO,MAAM,wBAAwB,YAAY,CAAC;AAElD;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,QAAQ,CAAC;AAE5C;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,SAAU,CAAC;AAE1D,8EAA8E;AAC9E,eAAO,MAAM,kCAAkC,OAAQ,CAAC;AAExD,mFAAmF;AACnF,eAAO,MAAM,+BAA+B,0CAA0C,CAAC;AAoCvF;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,UAAU,CA4B5D;AAED;;;;;GAKG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,eAAe,EACvB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,oFAAoF;AACpF,wBAAsB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAExF;AAED,gFAAgF;AAChF,wBAAsB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAG5F;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE;IAChC,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC;IACpD,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,YAAY,CAWf;AAED,+EAA+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,yEAAyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,+EAA+E;IAC/E,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,mBAAwB,EAC9B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,OAAO,CAAC,IAAI,CAAC,CAqDf"}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Load / unload orchestration for the host-control plane (Local model host
|
|
3
|
+
* control, D2).
|
|
4
|
+
*
|
|
5
|
+
* Per-pass model selection (D1) records which local model a coding vs review
|
|
6
|
+
* pass wants, but the daemon had no LOAD path -- so the selection could not
|
|
7
|
+
* drive an actual model switch. This module closes that gap: it loads / unloads
|
|
8
|
+
* the chosen per-pass model around each pass as a SEQUENTIAL swap on a single
|
|
9
|
+
* GPU (one model resident at a time -- loading evicts the prior), so a distinct
|
|
10
|
+
* coding-vs-review model can run end-to-end with NO manual swap in the studio.
|
|
11
|
+
*
|
|
12
|
+
* The control endpoints live under `/v1` (reachable from the stored baseUrl
|
|
13
|
+
* directly, which already ends in `/v1`): `POST /v1/load`, `POST /v1/unload`,
|
|
14
|
+
* `GET /v1/status`. Live JSON shapes are host-specific; `/v1/status` in
|
|
15
|
+
* particular is NOT pinned, so `normalizeStatus` is DEFENSIVE (tolerant of
|
|
16
|
+
* reasonable field names) and must be reconciled per the runbook.
|
|
17
|
+
*
|
|
18
|
+
* The maxSeqLength a load uses is an INJECTED input (an option, never hardcoded
|
|
19
|
+
* inside `ensureLoaded`) so Delivery 3 can replace the fixed `DEFAULT_MAX_SEQ_LENGTH`
|
|
20
|
+
* floor with a computed safe value without re-architecting this module.
|
|
21
|
+
*/
|
|
22
|
+
import { hostGet, hostPost, HostControlError, HOST_CONTROL_ERROR_UNREACHABLE } from './client.js';
|
|
23
|
+
import { readLoadableCatalog } from './catalog.js';
|
|
24
|
+
/**
|
|
25
|
+
* The control surface lives under `/v1` -- the SAME segment the stored baseUrl
|
|
26
|
+
* already ends in. We append these to the baseUrl directly (NOT the host root):
|
|
27
|
+
* a baseUrl of `http://host:8888/v1` yields `http://host:8888/v1/load`. A
|
|
28
|
+
* baseUrl without a trailing `/v1` still composes a sane path.
|
|
29
|
+
*/
|
|
30
|
+
export const HOST_CONTROL_LOAD_PATH = '/load';
|
|
31
|
+
export const HOST_CONTROL_UNLOAD_PATH = '/unload';
|
|
32
|
+
export const HOST_CONTROL_STATUS_PATH = '/status';
|
|
33
|
+
/**
|
|
34
|
+
* Default max sequence length for a load. Sized at the live-proven 16384, which
|
|
35
|
+
* clears codex's ~8k turn-1 context floor with headroom. NOTE: Delivery 3 will
|
|
36
|
+
* REPLACE this fixed value with a computed safe value derived from the model +
|
|
37
|
+
* available VRAM; `loadModel`/`ensureLoaded` take maxSeqLength as an INJECTED
|
|
38
|
+
* input so D3 can pass a computed value without touching this module's shape.
|
|
39
|
+
*/
|
|
40
|
+
export const DEFAULT_MAX_SEQ_LENGTH = 16384;
|
|
41
|
+
/**
|
|
42
|
+
* How long `ensureLoaded` will poll `/v1/status` for the target to become ready
|
|
43
|
+
* after issuing a load before giving up with a typed timeout. A cold load on a
|
|
44
|
+
* 12GB GPU has real latency; this is the bounded ceiling, not the expected wait.
|
|
45
|
+
*/
|
|
46
|
+
export const HOST_CONTROL_LOAD_READY_TIMEOUT_MS = 180_000;
|
|
47
|
+
/** Interval between `/v1/status` readiness polls while waiting for a load. */
|
|
48
|
+
export const HOST_CONTROL_LOAD_POLL_INTERVAL_MS = 2_000;
|
|
49
|
+
/** Address-free category for a load that never reached ready within the budget. */
|
|
50
|
+
export const HOST_CONTROL_ERROR_LOAD_TIMEOUT = 'Host load did not reach ready in time';
|
|
51
|
+
/**
|
|
52
|
+
* Join a baseUrl and a control-surface sub-path under `/v1`. The baseUrl already
|
|
53
|
+
* carries the `/v1` segment (it is the OpenAI-compatible base), so we append the
|
|
54
|
+
* sub-path directly, trimming a trailing slash so we never double up.
|
|
55
|
+
*/
|
|
56
|
+
function controlUrl(baseUrl, subPath) {
|
|
57
|
+
return `${baseUrl.replace(/\/+$/, '')}${subPath}`;
|
|
58
|
+
}
|
|
59
|
+
/** Read the first string-valued key present on an object, trimmed; null if none. */
|
|
60
|
+
function firstStringOrNull(obj, keys) {
|
|
61
|
+
for (const key of keys) {
|
|
62
|
+
const v = obj[key];
|
|
63
|
+
if (typeof v === 'string') {
|
|
64
|
+
const trimmed = v.trim();
|
|
65
|
+
if (trimmed)
|
|
66
|
+
return trimmed;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
/** Candidate keys carrying the currently-loaded model id on a `/v1/status` body. */
|
|
72
|
+
const STATUS_LOADED_KEYS = ['model', 'id', 'loaded_model', 'loadedModel', 'model_id'];
|
|
73
|
+
/** Candidate keys carrying a readiness signal on a `/v1/status` body. */
|
|
74
|
+
const STATUS_READY_KEYS = ['status', 'state', 'ready'];
|
|
75
|
+
/**
|
|
76
|
+
* String readiness tokens that mean "the loaded model is up and serving". The
|
|
77
|
+
* host's exact vocabulary is not pinned -- accept the common set. Reconcile
|
|
78
|
+
* against the live host per the runbook.
|
|
79
|
+
*/
|
|
80
|
+
const READY_TOKENS = new Set(['ready', 'loaded', 'running', 'ok', 'up', 'serving', 'idle']);
|
|
81
|
+
/**
|
|
82
|
+
* Normalize a `/v1/status` payload into a typed `HostStatus`. DEFENSIVE: the
|
|
83
|
+
* live shape is NOT pinned, so we tolerate the loaded-model id under any of
|
|
84
|
+
* `model`/`id`/`loaded_model`/... and readiness under `status`/`state`/`ready`
|
|
85
|
+
* (string token OR boolean). A payload that proves nothing loaded yields
|
|
86
|
+
* `{ loadedModelId: null, ready: false }`. Pure + exported for unit testing.
|
|
87
|
+
*
|
|
88
|
+
* Runbook flag: confirm the live `/v1/status` field names + readiness vocabulary
|
|
89
|
+
* and tighten `STATUS_*_KEYS` / `READY_TOKENS` once observed.
|
|
90
|
+
*/
|
|
91
|
+
export function normalizeStatus(payload) {
|
|
92
|
+
if (!payload || typeof payload !== 'object') {
|
|
93
|
+
return { loadedModelId: null, ready: false };
|
|
94
|
+
}
|
|
95
|
+
const obj = payload;
|
|
96
|
+
const loadedModelId = firstStringOrNull(obj, STATUS_LOADED_KEYS);
|
|
97
|
+
// Readiness can arrive as a boolean (`ready: true`) or a string token
|
|
98
|
+
// (`status: "ready"` / `state: "running"`). Accept either, defensively.
|
|
99
|
+
let ready = false;
|
|
100
|
+
for (const key of STATUS_READY_KEYS) {
|
|
101
|
+
const v = obj[key];
|
|
102
|
+
if (typeof v === 'boolean') {
|
|
103
|
+
if (v)
|
|
104
|
+
ready = true;
|
|
105
|
+
// An explicit `ready: false` is authoritative -- stop looking.
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
108
|
+
if (typeof v === 'string' && READY_TOKENS.has(v.trim().toLowerCase())) {
|
|
109
|
+
ready = true;
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// A model id with no contradicting readiness signal is treated as ready only
|
|
114
|
+
// when a readiness key actually said so; absent any readiness key but with a
|
|
115
|
+
// loaded id, be conservative and report not-ready (let the poller confirm).
|
|
116
|
+
return { loadedModelId, ready };
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Issue a model load (`POST /v1/load`). Maps camelCase app params to the host's
|
|
120
|
+
* snake_case wire field names (`model_path`, `max_seq_length`, `cache_type_kv`,
|
|
121
|
+
* `gpu_ids`) at THIS boundary only. `maxSeqLength` is a required INJECTED input
|
|
122
|
+
* so D3 can pass a computed value. Throws `HostControlError` on any failure.
|
|
123
|
+
*/
|
|
124
|
+
export async function loadModel(baseUrl, params, apiKey) {
|
|
125
|
+
const body = {
|
|
126
|
+
model_path: params.modelPath,
|
|
127
|
+
max_seq_length: params.maxSeqLength,
|
|
128
|
+
};
|
|
129
|
+
if (params.cacheTypeKv !== undefined)
|
|
130
|
+
body.cache_type_kv = params.cacheTypeKv;
|
|
131
|
+
if (params.gpuIds !== undefined)
|
|
132
|
+
body.gpu_ids = params.gpuIds;
|
|
133
|
+
await hostPost(controlUrl(baseUrl, HOST_CONTROL_LOAD_PATH), body, { apiKey });
|
|
134
|
+
}
|
|
135
|
+
/** Issue an unload (`POST /v1/unload`) -- evicts whatever is currently resident. */
|
|
136
|
+
export async function unloadModel(baseUrl, apiKey) {
|
|
137
|
+
await hostPost(controlUrl(baseUrl, HOST_CONTROL_UNLOAD_PATH), {}, { apiKey });
|
|
138
|
+
}
|
|
139
|
+
/** Read current host status (`GET /v1/status`), normalized to a typed shape. */
|
|
140
|
+
export async function getStatus(baseUrl, apiKey) {
|
|
141
|
+
const payload = await hostGet(controlUrl(baseUrl, HOST_CONTROL_STATUS_PATH), { apiKey });
|
|
142
|
+
return normalizeStatus(payload);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Resolve a chosen local_models `model` id to the host's `model_path` by reading
|
|
146
|
+
* the loadable catalog (D1) and matching by id. Returns null when the host is
|
|
147
|
+
* not controllable, the id is not in the catalog, or the matching entry carries
|
|
148
|
+
* no modelPath (a floor entry). The caller treats null as "cannot load" and
|
|
149
|
+
* skips the swap.
|
|
150
|
+
*/
|
|
151
|
+
export async function resolveModelPath(baseUrl, modelId, apiKey) {
|
|
152
|
+
const catalog = await readLoadableCatalog(baseUrl, apiKey);
|
|
153
|
+
const match = catalog.models.find((m) => m.id === modelId);
|
|
154
|
+
return match?.modelPath ?? null;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* PURE swap decision over the current vs target resident model. Extracted from
|
|
158
|
+
* `ensureLoaded` so the core single-GPU swap logic is unit-testable with NO HTTP:
|
|
159
|
+
*
|
|
160
|
+
* - target already resident + ready -> 'noop'
|
|
161
|
+
* - a DIFFERENT model resident -> 'evict_then_load' (single GPU -- evict first)
|
|
162
|
+
* - nothing resident -> 'load'
|
|
163
|
+
*
|
|
164
|
+
* `current.modelId` is the currently-loaded id (null = nothing resident);
|
|
165
|
+
* `current.ready` gates the noop (a resident-but-not-ready target still needs a
|
|
166
|
+
* poll, but no reload -- treated as 'load' here so `ensureLoaded` re-issues +
|
|
167
|
+
* polls; the host treats a load of the resident model as a fast no-op/confirm).
|
|
168
|
+
*/
|
|
169
|
+
export function decideSwap(input) {
|
|
170
|
+
const { current, target } = input;
|
|
171
|
+
let action;
|
|
172
|
+
if (current.modelId === target) {
|
|
173
|
+
action = current.ready ? 'noop' : 'load';
|
|
174
|
+
}
|
|
175
|
+
else if (current.modelId !== null) {
|
|
176
|
+
action = 'evict_then_load';
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
action = 'load';
|
|
180
|
+
}
|
|
181
|
+
return { action };
|
|
182
|
+
}
|
|
183
|
+
function defaultSleep(ms) {
|
|
184
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Orchestrate the SEQUENTIAL single-GPU swap so the target model is resident +
|
|
188
|
+
* ready before the caller spawns against it:
|
|
189
|
+
*
|
|
190
|
+
* 1. read status; if the target is already resident + ready -> no-op return.
|
|
191
|
+
* 2. else: if a DIFFERENT model is resident, unload it first (single GPU --
|
|
192
|
+
* one resident at a time). Resolve the target id -> model_path, then load
|
|
193
|
+
* with the injected maxSeqLength.
|
|
194
|
+
* 3. poll `/v1/status` until the target is ready, or throw a typed timeout
|
|
195
|
+
* (`HOST_CONTROL_ERROR_LOAD_TIMEOUT`) once the bounded budget elapses.
|
|
196
|
+
*
|
|
197
|
+
* Throws `HostControlError` on an unresolvable path, a load/unload failure, or
|
|
198
|
+
* the ready timeout. The caller (focus-executor) turns any throw into "skip the
|
|
199
|
+
* spawn this tick" -- it must NOT spawn against a not-yet-loaded model.
|
|
200
|
+
*/
|
|
201
|
+
export async function ensureLoaded(baseUrl, modelId, opts = {}, apiKey) {
|
|
202
|
+
const maxSeqLength = opts.maxSeqLength ?? DEFAULT_MAX_SEQ_LENGTH;
|
|
203
|
+
const readyTimeoutMs = opts.readyTimeoutMs ?? HOST_CONTROL_LOAD_READY_TIMEOUT_MS;
|
|
204
|
+
const pollIntervalMs = opts.pollIntervalMs ?? HOST_CONTROL_LOAD_POLL_INTERVAL_MS;
|
|
205
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
206
|
+
const now = opts.now ?? Date.now;
|
|
207
|
+
const status = await getStatus(baseUrl, apiKey);
|
|
208
|
+
const decision = decideSwap({
|
|
209
|
+
current: { modelId: status.loadedModelId, ready: status.ready },
|
|
210
|
+
target: modelId,
|
|
211
|
+
});
|
|
212
|
+
if (decision.action === 'noop')
|
|
213
|
+
return;
|
|
214
|
+
if (decision.action === 'evict_then_load') {
|
|
215
|
+
await unloadModel(baseUrl, apiKey);
|
|
216
|
+
}
|
|
217
|
+
const modelPath = await resolveModelPath(baseUrl, modelId, apiKey);
|
|
218
|
+
if (!modelPath) {
|
|
219
|
+
// No load path -> cannot load this model. Address-free category so the
|
|
220
|
+
// caller logs daemon-side and skips the spawn (re-attempts next poll).
|
|
221
|
+
throw new HostControlError(`No host model_path for model id "${modelId}"`, HOST_CONTROL_ERROR_UNREACHABLE);
|
|
222
|
+
}
|
|
223
|
+
await loadModel(baseUrl, {
|
|
224
|
+
modelPath,
|
|
225
|
+
maxSeqLength,
|
|
226
|
+
...(opts.cacheTypeKv !== undefined ? { cacheTypeKv: opts.cacheTypeKv } : {}),
|
|
227
|
+
...(opts.gpuIds !== undefined ? { gpuIds: opts.gpuIds } : {}),
|
|
228
|
+
}, apiKey);
|
|
229
|
+
// Poll for readiness up to the bounded budget. A cold load has real latency.
|
|
230
|
+
const deadline = now() + readyTimeoutMs;
|
|
231
|
+
for (;;) {
|
|
232
|
+
const polled = await getStatus(baseUrl, apiKey);
|
|
233
|
+
if (polled.loadedModelId === modelId && polled.ready)
|
|
234
|
+
return;
|
|
235
|
+
if (now() >= deadline) {
|
|
236
|
+
throw new HostControlError(`Model "${modelId}" did not reach ready within ${readyTimeoutMs}ms`, HOST_CONTROL_ERROR_LOAD_TIMEOUT);
|
|
237
|
+
}
|
|
238
|
+
await sleep(pollIntervalMs);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
//# sourceMappingURL=load.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/host-control/load.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,8BAA8B,EAAE,MAAM,aAAa,CAAC;AAClG,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAGnD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,OAAO,CAAC;AAC9C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAClD,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,OAAO,CAAC;AAE1D,8EAA8E;AAC9E,MAAM,CAAC,MAAM,kCAAkC,GAAG,KAAK,CAAC;AAExD,mFAAmF;AACnF,MAAM,CAAC,MAAM,+BAA+B,GAAG,uCAAuC,CAAC;AAEvF;;;;GAIG;AACH,SAAS,UAAU,CAAC,OAAe,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC;AACpD,CAAC;AAED,oFAAoF;AACpF,SAAS,iBAAiB,CAAC,GAA4B,EAAE,IAAuB;IAC9E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACzB,IAAI,OAAO;gBAAE,OAAO,OAAO,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oFAAoF;AACpF,MAAM,kBAAkB,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,UAAU,CAAU,CAAC;AAE/F,yEAAyE;AACzE,MAAM,iBAAiB,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAU,CAAC;AAEhE;;;;GAIG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAE5F;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAAC,OAAgB;IAC9C,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC5C,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC/C,CAAC;IACD,MAAM,GAAG,GAAG,OAAkC,CAAC;IAE/C,MAAM,aAAa,GAAG,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;IAEjE,sEAAsE;IACtE,wEAAwE;IACxE,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,IAAI,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC;gBAAE,KAAK,GAAG,IAAI,CAAC;YACpB,+DAA+D;YAC/D,MAAM;QACR,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACtE,KAAK,GAAG,IAAI,CAAC;YACb,MAAM;QACR,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,6EAA6E;IAC7E,4EAA4E;IAC5E,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AAClC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,MAAuB,EACvB,MAAsB;IAEtB,MAAM,IAAI,GAA4B;QACpC,UAAU,EAAE,MAAM,CAAC,SAAS;QAC5B,cAAc,EAAE,MAAM,CAAC,YAAY;KACpC,CAAC;IACF,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS;QAAE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC;IAC9E,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;IAC9D,MAAM,QAAQ,CAAU,UAAU,CAAC,OAAO,EAAE,sBAAsB,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;AACzF,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAe,EAAE,MAAsB;IACvE,MAAM,QAAQ,CAAU,UAAU,CAAC,OAAO,EAAE,wBAAwB,CAAC,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;AACzF,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,MAAsB;IACrE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAU,UAAU,CAAC,OAAO,EAAE,wBAAwB,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAClG,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,OAAe,EACf,MAAsB;IAEtB,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;IAC3D,OAAO,KAAK,EAAE,SAAS,IAAI,IAAI,CAAC;AAClC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAC,KAG1B;IACC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAClC,IAAI,MAAkB,CAAC;IACvB,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAC/B,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3C,CAAC;SAAM,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,GAAG,iBAAiB,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,CAAC;AACpB,CAAC;AAuBD,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAe,EACf,OAAe,EACf,OAA4B,EAAE,EAC9B,MAAsB;IAEtB,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,sBAAsB,CAAC;IACjE,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,kCAAkC,CAAC;IACjF,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,kCAAkC,CAAC;IACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,YAAY,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IAEjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,UAAU,CAAC;QAC1B,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE;QAC/D,MAAM,EAAE,OAAO;KAChB,CAAC,CAAC;IAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM;QAAE,OAAO;IAEvC,IAAI,QAAQ,CAAC,MAAM,KAAK,iBAAiB,EAAE,CAAC;QAC1C,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACnE,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,uEAAuE;QACvE,uEAAuE;QACvE,MAAM,IAAI,gBAAgB,CACxB,oCAAoC,OAAO,GAAG,EAC9C,8BAA8B,CAC/B,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,CACb,OAAO,EACP;QACE,SAAS;QACT,YAAY;QACZ,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,EACD,MAAM,CACP,CAAC;IAEF,6EAA6E;IAC7E,MAAM,QAAQ,GAAG,GAAG,EAAE,GAAG,cAAc,CAAC;IACxC,SAAS,CAAC;QACR,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,IAAI,MAAM,CAAC,KAAK;YAAE,OAAO;QAC7D,IAAI,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,gBAAgB,CACxB,UAAU,OAAO,gCAAgC,cAAc,IAAI,EACnE,+BAA+B,CAChC,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC,cAAc,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hardware-aware context sizing for the host-control plane (Local model host
|
|
3
|
+
* control, D3).
|
|
4
|
+
*
|
|
5
|
+
* The problem this closes: `max_seq_length` (the load-time context window) was a
|
|
6
|
+
* MANUAL, hardware-blind input. Set it too SMALL and codex's turn-1 prompt does
|
|
7
|
+
* not fit -- the load silently fails turn-1 with an instant `response.failed`
|
|
8
|
+
* (live: a 4096-ctx load failed; a manual reload to >=16k fixed it). Set it too
|
|
9
|
+
* LARGE and the KV-cache (which scales with context) OOMs a near-full GPU (live:
|
|
10
|
+
* a 12GB card sat ~93% used with a model resident). The safe value is bounded
|
|
11
|
+
* BELOW by codex's ~8k turn-1 floor and ABOVE by remaining VRAM headroom.
|
|
12
|
+
*
|
|
13
|
+
* This module adds two things:
|
|
14
|
+
* 1. defensive client reads over the host's hardware + kv-cache endpoints
|
|
15
|
+
* (`getHardware`, `getKvCacheEstimate`) -- the live JSON shapes are NOT
|
|
16
|
+
* pinned, so the normalizers tolerate reasonable field-name variants and
|
|
17
|
+
* are flagged for runbook reconciliation;
|
|
18
|
+
* 2. a PURE `computeSafeMaxSeqLength` (no HTTP) -- the heart: pick the largest
|
|
19
|
+
* context that clears the codex floor without exceeding VRAM headroom,
|
|
20
|
+
* reaching for optional KV-cache quantization when it lets a context fit.
|
|
21
|
+
*
|
|
22
|
+
* The hardware/kv-cache endpoints live under the host ROOT (NOT `/v1`), so we
|
|
23
|
+
* derive the root with `hostRootFromBaseUrl` (strips a trailing `/v1`). All
|
|
24
|
+
* errors are address-free (TEL-5): the load path treats any read failure as
|
|
25
|
+
* "no sizing data" and falls back to the fixed floor default, never breaking a
|
|
26
|
+
* swap that previously worked.
|
|
27
|
+
*/
|
|
28
|
+
import type { HostHardware, KvCacheEstimate, KvCacheEstimateParams, SafeMaxSeqInput, SafeMaxSeqResult } from './types.js';
|
|
29
|
+
/**
|
|
30
|
+
* The control-surface path (under the host ROOT, not `/v1`) for GPU hardware
|
|
31
|
+
* info: GPU name + total/used VRAM. Reconcile field names per the runbook.
|
|
32
|
+
*/
|
|
33
|
+
export declare const HOST_CONTROL_HARDWARE_PATH = "/api/system/hardware";
|
|
34
|
+
/**
|
|
35
|
+
* The control-surface path (under the host ROOT, not `/v1`) for a KV-cache fit
|
|
36
|
+
* prediction at a candidate context. Reconcile field names per the runbook.
|
|
37
|
+
*/
|
|
38
|
+
export declare const HOST_CONTROL_KV_CACHE_PATH = "/api/models/kv-cache-estimate";
|
|
39
|
+
/**
|
|
40
|
+
* Codex's turn-1 context floor. A load BELOW this silently fails the very first
|
|
41
|
+
* codex turn (instant `response.failed`) -- so we treat it as a HARD lower
|
|
42
|
+
* bound and never return a `maxSeqLength` under it, even when VRAM headroom is
|
|
43
|
+
* tight (we signal quant-required/marginal instead of silently going lower).
|
|
44
|
+
*
|
|
45
|
+
* The live datapoint: a 4096-ctx load failed turn-1; a reload to >=16384
|
|
46
|
+
* worked. The exact floor is ~8k; 8192 is the documented conservative pin.
|
|
47
|
+
*/
|
|
48
|
+
export declare const CODEX_TURN1_FLOOR = 8192;
|
|
49
|
+
/**
|
|
50
|
+
* VRAM safety margin (MB) held back from raw headroom before sizing the KV
|
|
51
|
+
* cache. A near-full GPU (live: 12GB card ~93% used with a model resident)
|
|
52
|
+
* leaves little slack; this guards against the estimate being slightly optimistic,
|
|
53
|
+
* driver/allocator overhead, and fragmentation. Conservative on purpose --
|
|
54
|
+
* an OOM costs a whole swap, a slightly-smaller context costs nothing.
|
|
55
|
+
*/
|
|
56
|
+
export declare const VRAM_SAFETY_MARGIN_MB = 512;
|
|
57
|
+
/**
|
|
58
|
+
* Per-token KV-cache cost heuristic (MB per 1 token of context) used as the
|
|
59
|
+
* FALLBACK growth model when the host's kv-cache-estimate endpoint is not
|
|
60
|
+
* consulted (the pure path takes only static inputs). This is intentionally a
|
|
61
|
+
* coarse upper-ish estimate for a ~7-8B class model at fp16 KV; the actual
|
|
62
|
+
* footprint is model-architecture-specific. The host's `getKvCacheEstimate`
|
|
63
|
+
* endpoint, when reachable, gives a far better per-context number and SHOULD be
|
|
64
|
+
* preferred by the orchestrator; this constant only bounds the pure computation
|
|
65
|
+
* when no live estimate is supplied.
|
|
66
|
+
*
|
|
67
|
+
* Derivation sketch (documented so the runbook can refine it): KV bytes ~=
|
|
68
|
+
* 2 (K and V) * layers * 2 (kv heads grouping varies) * head_dim * bytes/elem.
|
|
69
|
+
* For a 7B-class model at fp16 this lands on the order of ~0.12-0.20 MB/token;
|
|
70
|
+
* we pin a conservative 0.16 MB/token. Quantized KV (int8/fp8) roughly halves
|
|
71
|
+
* this -- see `KV_QUANT_FOOTPRINT_FACTOR`.
|
|
72
|
+
*/
|
|
73
|
+
export declare const KV_CACHE_MB_PER_TOKEN = 0.16;
|
|
74
|
+
/**
|
|
75
|
+
* Footprint multiplier when KV-cache quantization is requested. int8/fp8 KV
|
|
76
|
+
* roughly halves the per-token cost vs fp16. Conservative 0.5 -- a real host
|
|
77
|
+
* may do better; the kv-cache-estimate endpoint, when consulted, supersedes
|
|
78
|
+
* this heuristic.
|
|
79
|
+
*/
|
|
80
|
+
export declare const KV_QUANT_FOOTPRINT_FACTOR = 0.5;
|
|
81
|
+
/**
|
|
82
|
+
* The KV-cache dtype token sent as `cache_type_kv` on the load wire when the
|
|
83
|
+
* sizer elects to quantize to fit a context. The exact accepted token is
|
|
84
|
+
* host-specific; `q8_0` is the common llama.cpp-family int8 KV token. Flag for
|
|
85
|
+
* the runbook to confirm against the live host's accepted values.
|
|
86
|
+
*/
|
|
87
|
+
export declare const KV_QUANT_CACHE_TYPE = "q8_0";
|
|
88
|
+
/**
|
|
89
|
+
* Hard ceiling on the context we will ever request, independent of headroom,
|
|
90
|
+
* when no native-context bound is known. A sane upper clamp so a wildly large
|
|
91
|
+
* estimate cannot produce an absurd request. Native context (when known) clamps
|
|
92
|
+
* tighter than this.
|
|
93
|
+
*/
|
|
94
|
+
export declare const MAX_SEQ_LENGTH_CEILING = 131072;
|
|
95
|
+
/**
|
|
96
|
+
* Normalize a `/api/system/hardware` payload into a typed `HostHardware`.
|
|
97
|
+
* DEFENSIVE: the live shape is NOT pinned. Tolerates MB- or bytes-named VRAM
|
|
98
|
+
* fields, a flat or a `{ gpu: {...} }` / `{ gpus: [...] }` nesting, and missing
|
|
99
|
+
* fields (-> null). Pure + exported for unit testing.
|
|
100
|
+
*
|
|
101
|
+
* Runbook flag: confirm the live `/api/system/hardware` field names + units
|
|
102
|
+
* (MB vs bytes) and tighten `HW_*_KEYS` once observed.
|
|
103
|
+
*/
|
|
104
|
+
export declare function normalizeHardware(payload: unknown): HostHardware;
|
|
105
|
+
/**
|
|
106
|
+
* Normalize a `/api/models/kv-cache-estimate` payload into a typed
|
|
107
|
+
* `KvCacheEstimate`. DEFENSIVE: footprint under MB- or bytes-named fields, an
|
|
108
|
+
* optional boolean fit verdict, both null/undefined when absent. Pure + exported.
|
|
109
|
+
*
|
|
110
|
+
* Runbook flag: confirm the live estimate field names + whether the endpoint
|
|
111
|
+
* echoes the candidate context; tighten `KV_*_KEYS` once observed.
|
|
112
|
+
*/
|
|
113
|
+
export declare function normalizeKvCacheEstimate(payload: unknown): KvCacheEstimate;
|
|
114
|
+
/**
|
|
115
|
+
* Read the host's GPU + VRAM info (`GET <root>/api/system/hardware`), normalized
|
|
116
|
+
* to a typed `HostHardware`. The hardware endpoint lives under the host ROOT,
|
|
117
|
+
* NOT `/v1`, so we derive the root from the stored baseUrl. Throws
|
|
118
|
+
* `HostControlError` (address-free) on any transport failure -- the caller
|
|
119
|
+
* fail-safes to the floor default.
|
|
120
|
+
*/
|
|
121
|
+
export declare function getHardware(baseUrl: string, apiKey?: string | null): Promise<HostHardware>;
|
|
122
|
+
/**
|
|
123
|
+
* Read a KV-cache fit prediction for a candidate context
|
|
124
|
+
* (`GET <root>/api/models/kv-cache-estimate`). The candidate context (and an
|
|
125
|
+
* optional model path + kv quant) are passed as query params; the live host's
|
|
126
|
+
* exact param names are NOT pinned, so we send the common-sense set and the
|
|
127
|
+
* normalizer is defensive about the response. Throws `HostControlError` on any
|
|
128
|
+
* transport failure -- the caller treats it as "no estimate" and uses the pure
|
|
129
|
+
* heuristic instead.
|
|
130
|
+
*
|
|
131
|
+
* Runbook flag: confirm the live query-param names (`max_seq_length`,
|
|
132
|
+
* `model_path`, `cache_type_kv`) the endpoint expects.
|
|
133
|
+
*/
|
|
134
|
+
export declare function getKvCacheEstimate(baseUrl: string, params: KvCacheEstimateParams, apiKey?: string | null): Promise<KvCacheEstimate>;
|
|
135
|
+
/**
|
|
136
|
+
* PURE: compute the safe `max_seq_length` (and whether to quantize the KV cache)
|
|
137
|
+
* for a load, given VRAM totals/usage, the resident model size, and the model's
|
|
138
|
+
* native context. NO HTTP -- the orchestrator gathers the inputs and (optionally)
|
|
139
|
+
* validates the result against the live kv-cache endpoint.
|
|
140
|
+
*
|
|
141
|
+
* The arithmetic, with the live constraints in mind:
|
|
142
|
+
*
|
|
143
|
+
* - headroom = vramTotal - vramUsed - VRAM_SAFETY_MARGIN_MB. This is the VRAM
|
|
144
|
+
* available for the KV cache AFTER the model weights are resident. (Live: a
|
|
145
|
+
* 12GB card sat ~93% used with a model loaded -- headroom is small, so the
|
|
146
|
+
* KV cache, which scales with context, is the binding constraint.) When
|
|
147
|
+
* `vramUsed` already accounts for the weights, `modelSizeMb` is informational;
|
|
148
|
+
* when it does not, we subtract `modelSizeMb` too (see below).
|
|
149
|
+
*
|
|
150
|
+
* - The KV cache grows ~linearly with context (`KV_CACHE_MB_PER_TOKEN`). We pick
|
|
151
|
+
* the LARGEST context whose KV cache fits headroom, clamped to the model's
|
|
152
|
+
* native context (`nativeContext`) and the global ceiling.
|
|
153
|
+
*
|
|
154
|
+
* - HARD LOWER BOUND (codex floor): a context below `CODEX_TURN1_FLOOR` silently
|
|
155
|
+
* fails turn-1, so we NEVER return below it. If headroom can't fit the floor
|
|
156
|
+
* at fp16 KV, we try quantization (`cacheTypeKv`), which ~halves the footprint.
|
|
157
|
+
* If even quantized KV can't fit the floor, we return the floor ANYWAY with
|
|
158
|
+
* `quantRequired`/`marginal` set -- going below is not an option; better to
|
|
159
|
+
* attempt the floor (and let the load fail loudly / the operator add VRAM)
|
|
160
|
+
* than to silently load a too-small context that fails turn-1 with no signal.
|
|
161
|
+
*
|
|
162
|
+
* - When inputs are missing/degenerate (no VRAM data, zero/negative headroom),
|
|
163
|
+
* we degrade SAFELY to the floor default -- never throw, never go below floor.
|
|
164
|
+
*/
|
|
165
|
+
export declare function computeSafeMaxSeqLength(input: SafeMaxSeqInput): SafeMaxSeqResult;
|
|
166
|
+
/**
|
|
167
|
+
* The sizing inputs the orchestrator could not read from the host but the caller
|
|
168
|
+
* may know (from the D1 catalog entry or registry). Both optional -- when a field
|
|
169
|
+
* is unknown the sizer degrades safely (native unknown -> ceiling clamp; model
|
|
170
|
+
* size unknown -> no weights reservation).
|
|
171
|
+
*/
|
|
172
|
+
export interface SafeLoadSizingHints {
|
|
173
|
+
/** The candidate model's weights size in MB, when the caller knows it. */
|
|
174
|
+
modelSizeMb?: number | null;
|
|
175
|
+
/** The model's native context window, when the caller knows it. */
|
|
176
|
+
nativeContext?: number | null;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Resolve the safe `EnsureLoadedOptions` sizing bits (`maxSeqLength` + optional
|
|
180
|
+
* `cacheTypeKv`) for a load by reading the host's hardware, running the PURE
|
|
181
|
+
* `computeSafeMaxSeqLength`, and OPTIONALLY validating the result against the
|
|
182
|
+
* live kv-cache-estimate endpoint. This is the seam the focus-executor calls in
|
|
183
|
+
* place of D2's fixed `DEFAULT_MAX_SEQ_LENGTH`.
|
|
184
|
+
*
|
|
185
|
+
* FAIL-SAFE: if the hardware endpoint is missing / unreachable / returns no VRAM
|
|
186
|
+
* totals, we DO NOT compute against headroom -- we return the fixed
|
|
187
|
+
* `DEFAULT_MAX_SEQ_LENGTH` (>= the codex floor) exactly as D2 did, and report the
|
|
188
|
+
* fallback so the caller logs it. A sizing failure must NEVER break a swap that
|
|
189
|
+
* previously worked with the fixed default.
|
|
190
|
+
*
|
|
191
|
+
* The kv-cache-estimate validation is best-effort: if it predicts the computed
|
|
192
|
+
* context will NOT fit (and we are not already quantizing), we re-run the sizer
|
|
193
|
+
* forcing quantization; if the endpoint is unreachable we keep the pure result.
|
|
194
|
+
* The estimate is never allowed to push the context BELOW the floor.
|
|
195
|
+
*
|
|
196
|
+
* Returns `{ maxSeqLength, cacheTypeKv?, source, marginal?, quantRequired? }`.
|
|
197
|
+
* `source` is `'computed'` when hardware drove the value, `'fallback'` when we
|
|
198
|
+
* fell back to the fixed default -- the caller logs which path it took.
|
|
199
|
+
*/
|
|
200
|
+
export declare function resolveSafeLoadOptions(baseUrl: string, modelPath: string | null, hints?: SafeLoadSizingHints, apiKey?: string | null): Promise<{
|
|
201
|
+
maxSeqLength: number;
|
|
202
|
+
cacheTypeKv?: string;
|
|
203
|
+
source: 'computed' | 'fallback';
|
|
204
|
+
marginal?: boolean;
|
|
205
|
+
quantRequired?: boolean;
|
|
206
|
+
}>;
|
|
207
|
+
//# sourceMappingURL=sizing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sizing.d.ts","sourceRoot":"","sources":["../../src/host-control/sizing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,qBAAqB,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE1H;;;GAGG;AACH,eAAO,MAAM,0BAA0B,yBAAyB,CAAC;AAEjE;;;GAGG;AACH,eAAO,MAAM,0BAA0B,kCAAkC,CAAC;AAE1E;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAEtC;;;;;;GAMG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,SAAS,CAAC;AAE1C;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,SAAS,CAAC;AA0F7C;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,OAAO,GAAG,YAAY,CAUhE;AASD;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,CAmB1E;AAID;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAIhG;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,qBAAqB,EAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,OAAO,CAAC,eAAe,CAAC,CAY1B;AAmCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,eAAe,GAAG,gBAAgB,CAsEhF;AAID;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAClC,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,mEAAmE;IACnE,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,GAAE,mBAAwB,EAC/B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,OAAO,CAAC;IACT,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,UAAU,CAAC;IAChC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC,CAyDD"}
|