@tokenbuddy/tokenbuddy 1.0.9 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/buyer-store.d.ts +13 -0
- package/dist/src/buyer-store.d.ts.map +1 -1
- package/dist/src/buyer-store.js +21 -2
- package/dist/src/buyer-store.js.map +1 -1
- package/dist/src/cli.d.ts.map +1 -1
- package/dist/src/cli.js +54 -0
- package/dist/src/cli.js.map +1 -1
- package/dist/src/credit-tracker.d.ts +118 -0
- package/dist/src/credit-tracker.d.ts.map +1 -0
- package/dist/src/credit-tracker.js +220 -0
- package/dist/src/credit-tracker.js.map +1 -0
- package/dist/src/daemon.d.ts +49 -4
- package/dist/src/daemon.d.ts.map +1 -1
- package/dist/src/daemon.js +541 -405
- package/dist/src/daemon.js.map +1 -1
- package/dist/src/model-index.d.ts +86 -0
- package/dist/src/model-index.d.ts.map +1 -0
- package/dist/src/model-index.js +214 -0
- package/dist/src/model-index.js.map +1 -0
- package/dist/src/prewarm-cache.d.ts +149 -0
- package/dist/src/prewarm-cache.d.ts.map +1 -0
- package/dist/src/prewarm-cache.js +288 -0
- package/dist/src/prewarm-cache.js.map +1 -0
- package/dist/src/prewarm-scheduler.d.ts +150 -0
- package/dist/src/prewarm-scheduler.d.ts.map +1 -0
- package/dist/src/prewarm-scheduler.js +484 -0
- package/dist/src/prewarm-scheduler.js.map +1 -0
- package/dist/src/provider-install.d.ts.map +1 -1
- package/dist/src/provider-install.js +10 -0
- package/dist/src/provider-install.js.map +1 -1
- package/dist/src/route-failover.d.ts +96 -0
- package/dist/src/route-failover.d.ts.map +1 -0
- package/dist/src/route-failover.js +177 -0
- package/dist/src/route-failover.js.map +1 -0
- package/dist/src/seller-catalog.d.ts +26 -0
- package/dist/src/seller-catalog.d.ts.map +1 -1
- package/dist/src/seller-catalog.js +40 -0
- package/dist/src/seller-catalog.js.map +1 -1
- package/dist/src/seller-pool.d.ts +127 -0
- package/dist/src/seller-pool.d.ts.map +1 -0
- package/dist/src/seller-pool.js +243 -0
- package/dist/src/seller-pool.js.map +1 -0
- package/dist/src/stream-failover.d.ts +78 -0
- package/dist/src/stream-failover.d.ts.map +1 -0
- package/dist/src/stream-failover.js +93 -0
- package/dist/src/stream-failover.js.map +1 -0
- package/package.json +1 -1
- package/src/buyer-store.ts +32 -2
- package/src/cli.ts +61 -0
- package/src/credit-tracker.test.ts +165 -0
- package/src/credit-tracker.ts +269 -0
- package/src/daemon.ts +569 -445
- package/src/model-index.test.ts +184 -0
- package/src/model-index.ts +266 -0
- package/src/prewarm-cache.test.ts +281 -0
- package/src/prewarm-cache.ts +373 -0
- package/src/prewarm-scheduler.test.ts +367 -0
- package/src/prewarm-scheduler.ts +581 -0
- package/src/provider-install.ts +10 -0
- package/src/route-failover.test.ts +193 -0
- package/src/route-failover.ts +233 -0
- package/src/seller-catalog-413.test.ts +61 -0
- package/src/seller-catalog.ts +47 -0
- package/src/seller-pool.test.ts +231 -0
- package/src/seller-pool.ts +333 -0
- package/src/stream-failover.test.ts +52 -0
- package/src/stream-failover.ts +129 -0
- package/src/thousand-seller.test.ts +151 -0
- package/tests/daemon-413-fallback.test.ts +92 -0
- package/tests/e2e.test.ts +3 -2
- package/tests/tokenbuddy.test.ts +70 -11
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default TTL for a successfully warmed entry. 10 minutes is the v1.2 starting
|
|
3
|
+
* point; see buyer-driven-fallback-design.md §18.13 for the trade-off. The
|
|
4
|
+
* cache constructor accepts an override so tests and the future PR-E config
|
|
5
|
+
* loader can change this without re-architecting.
|
|
6
|
+
*/
|
|
7
|
+
export declare const DEFAULT_PREWARM_TTL_MS: number;
|
|
8
|
+
export type PrewarmState = "warming" | "warm" | "stale" | "empty";
|
|
9
|
+
export interface PrewarmCandidate {
|
|
10
|
+
sellerId: string;
|
|
11
|
+
url: string;
|
|
12
|
+
healthScore: number;
|
|
13
|
+
lastSuccessAt: number;
|
|
14
|
+
lastFailAt: number;
|
|
15
|
+
avgLatencyMs: number;
|
|
16
|
+
}
|
|
17
|
+
export interface PrewarmEntry {
|
|
18
|
+
modelId: string;
|
|
19
|
+
protocol: string;
|
|
20
|
+
paymentMethod: string;
|
|
21
|
+
state: PrewarmState;
|
|
22
|
+
candidates: PrewarmCandidate[];
|
|
23
|
+
warmedAt: number;
|
|
24
|
+
ttlMs: number;
|
|
25
|
+
consecutiveWarmingFailures: number;
|
|
26
|
+
lastInFlightAt?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface PrewarmCandidateInput {
|
|
29
|
+
sellerId: string;
|
|
30
|
+
url: string;
|
|
31
|
+
healthScore?: number;
|
|
32
|
+
lastSuccessAt?: number;
|
|
33
|
+
lastFailAt?: number;
|
|
34
|
+
avgLatencyMs?: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Build the cache key for a (model, protocol, payment) triple. The colon
|
|
38
|
+
* separator is reserved at the model-id level because `RegistrySeller.models`
|
|
39
|
+
* entries are trimmed but not colon-escaped. v1.2 forbids `:` inside model
|
|
40
|
+
* ids so this format is collision-free.
|
|
41
|
+
*/
|
|
42
|
+
export declare function prewarmKey(modelId: string, protocol: string, paymentMethod: string): string;
|
|
43
|
+
interface PrewarmCacheOptions {
|
|
44
|
+
defaultTtlMs?: number;
|
|
45
|
+
now?: () => number;
|
|
46
|
+
}
|
|
47
|
+
export declare class PrewarmCache {
|
|
48
|
+
private readonly entries;
|
|
49
|
+
private readonly defaultTtlMs;
|
|
50
|
+
private readonly now;
|
|
51
|
+
constructor(options?: PrewarmCacheOptions);
|
|
52
|
+
/**
|
|
53
|
+
* Read an entry without mutating state. Returns `undefined` when the key is
|
|
54
|
+
* unknown; the caller decides whether "absent" should be treated as a miss
|
|
55
|
+
* (i.e. trigger a fresh prewarm) or as a known empty model.
|
|
56
|
+
*/
|
|
57
|
+
get(modelId: string, protocol: string, paymentMethod: string): PrewarmEntry | undefined;
|
|
58
|
+
/**
|
|
59
|
+
* Look up an entry and return a `Freshness` descriptor. This is the cheap
|
|
60
|
+
* path used on every inference request to decide whether a prewarm is
|
|
61
|
+
* still authoritative, expiring soon, or already stale.
|
|
62
|
+
*/
|
|
63
|
+
freshness(modelId: string, protocol: string, paymentMethod: string): PrewarmFreshness;
|
|
64
|
+
/**
|
|
65
|
+
* Mark a (model, protocol, payment) triple as currently being warmed. If an
|
|
66
|
+
* existing warm entry is present it is kept untouched (the new probe
|
|
67
|
+
* supersedes it on commit) and the previous state is reported to the
|
|
68
|
+
* caller via the returned descriptor.
|
|
69
|
+
*/
|
|
70
|
+
beginWarming(modelId: string, protocol: string, paymentMethod: string, ttlMs?: number): PrewarmBeginResult;
|
|
71
|
+
/**
|
|
72
|
+
* Commit a successful warm. The entry's `warmedAt` is reset to the current
|
|
73
|
+
* time so the TTL window starts fresh, and any prior stale candidates are
|
|
74
|
+
* replaced with the new probe results. The previous candidate set is
|
|
75
|
+
* returned for caller-side telemetry (e.g. detecting churn).
|
|
76
|
+
*/
|
|
77
|
+
commitWarm(input: {
|
|
78
|
+
modelId: string;
|
|
79
|
+
protocol: string;
|
|
80
|
+
paymentMethod: string;
|
|
81
|
+
candidates: PrewarmCandidateInput[];
|
|
82
|
+
ttlMs?: number;
|
|
83
|
+
}): PrewarmCommitResult;
|
|
84
|
+
/**
|
|
85
|
+
* Mark a warm as failed. Consecutive failures are tracked so the scheduler
|
|
86
|
+
* can apply exponential backoff and so `tb doctor` can surface persistently
|
|
87
|
+
* broken models.
|
|
88
|
+
*/
|
|
89
|
+
recordFailure(modelId: string, protocol: string, paymentMethod: string, errorMessage?: string): PrewarmEntry | undefined;
|
|
90
|
+
/**
|
|
91
|
+
* Invalidate every entry that references the given seller. Used when the
|
|
92
|
+
* registry signals a seller is gone (grace period expires) or when a hard
|
|
93
|
+
* failure (e.g. 5xx storm) should drop the seller from the cache
|
|
94
|
+
* immediately.
|
|
95
|
+
*/
|
|
96
|
+
invalidateSeller(sellerId: string): number;
|
|
97
|
+
/**
|
|
98
|
+
* Invalidate a specific cache key. Used by `tb doctor --refresh <model>`
|
|
99
|
+
* and by the registry loop when a model is removed from the focus set.
|
|
100
|
+
*/
|
|
101
|
+
invalidateKey(modelId: string, protocol: string, paymentMethod: string): boolean;
|
|
102
|
+
/**
|
|
103
|
+
* Drop every entry whose TTL has expired. Returns the number of removed
|
|
104
|
+
* entries so the caller can log it.
|
|
105
|
+
*/
|
|
106
|
+
evictExpired(now?: number): number;
|
|
107
|
+
/**
|
|
108
|
+
* Returns `true` when the entry's TTL is within `withinMs` of expiring. The
|
|
109
|
+
* scheduler uses this to schedule idle-cycle prewarms just-in-time rather
|
|
110
|
+
* than at fixed wall-clock intervals.
|
|
111
|
+
*/
|
|
112
|
+
isExpiringSoon(modelId: string, protocol: string, paymentMethod: string, withinMs: number, now?: number): boolean;
|
|
113
|
+
/**
|
|
114
|
+
* Snapshot all entries for diagnostics. Returns a deep-copy of the values
|
|
115
|
+
* so callers can serialize without risking mutation of cache state.
|
|
116
|
+
*/
|
|
117
|
+
snapshot(): PrewarmEntry[];
|
|
118
|
+
/**
|
|
119
|
+
* List every cached key, decoded back into its (model, protocol, payment)
|
|
120
|
+
* triple. Used by `tb doctor` to render the prewarm table.
|
|
121
|
+
*/
|
|
122
|
+
keys(): Array<{
|
|
123
|
+
modelId: string;
|
|
124
|
+
protocol: string;
|
|
125
|
+
paymentMethod: string;
|
|
126
|
+
}>;
|
|
127
|
+
size(): number;
|
|
128
|
+
clear(): void;
|
|
129
|
+
}
|
|
130
|
+
export interface PrewarmFreshness {
|
|
131
|
+
present: boolean;
|
|
132
|
+
expired: boolean;
|
|
133
|
+
expiringSoon: boolean;
|
|
134
|
+
remainingMs?: number;
|
|
135
|
+
state: PrewarmState;
|
|
136
|
+
entry?: PrewarmEntry;
|
|
137
|
+
}
|
|
138
|
+
export interface PrewarmBeginResult {
|
|
139
|
+
key: string;
|
|
140
|
+
entry: PrewarmEntry;
|
|
141
|
+
hadPrevious: boolean;
|
|
142
|
+
}
|
|
143
|
+
export interface PrewarmCommitResult {
|
|
144
|
+
key: string;
|
|
145
|
+
entry: PrewarmEntry;
|
|
146
|
+
replacedSellers: string[];
|
|
147
|
+
}
|
|
148
|
+
export {};
|
|
149
|
+
//# sourceMappingURL=prewarm-cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prewarm-cache.d.ts","sourceRoot":"","sources":["../../src/prewarm-cache.ts"],"names":[],"mappings":"AAIA;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AAErD,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,0BAA0B,EAAE,MAAM,CAAC;IACnC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,MAAM,CAE3F;AAcD,UAAU,mBAAmB;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;gBAEvB,OAAO,GAAE,mBAAwB;IAK7C;;;;OAIG;IACH,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIvF;;;;OAIG;IACH,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,gBAAgB;IAmBrF;;;;;OAKG;IACH,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,kBAAkB;IA0B1G;;;;;OAKG;IACH,UAAU,CAAC,KAAK,EAAE;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,qBAAqB,EAAE,CAAC;QACpC,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,mBAAmB;IAwCvB;;;;OAIG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAuBxH;;;;;OAKG;IACH,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAsB1C;;;OAGG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO;IAIhF;;;OAGG;IACH,YAAY,CAAC,GAAG,GAAE,MAAmB,GAAG,MAAM;IAc9C;;;;OAIG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,GAAE,MAAmB,GAAG,OAAO;IAS7H;;;OAGG;IACH,QAAQ,IAAI,YAAY,EAAE;IAO1B;;;OAGG;IACH,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IAW3E,IAAI,IAAI,MAAM;IAId,KAAK,IAAI,IAAI;CAGd;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,YAAY,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,YAAY,CAAC;IACpB,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B"}
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import { createModuleLogger } from "@tokenbuddy/logging";
|
|
2
|
+
const logger = createModuleLogger("tb-proxyd:prewarm-cache");
|
|
3
|
+
/**
|
|
4
|
+
* Default TTL for a successfully warmed entry. 10 minutes is the v1.2 starting
|
|
5
|
+
* point; see buyer-driven-fallback-design.md §18.13 for the trade-off. The
|
|
6
|
+
* cache constructor accepts an override so tests and the future PR-E config
|
|
7
|
+
* loader can change this without re-architecting.
|
|
8
|
+
*/
|
|
9
|
+
export const DEFAULT_PREWARM_TTL_MS = 10 * 60 * 1000;
|
|
10
|
+
/**
|
|
11
|
+
* Build the cache key for a (model, protocol, payment) triple. The colon
|
|
12
|
+
* separator is reserved at the model-id level because `RegistrySeller.models`
|
|
13
|
+
* entries are trimmed but not colon-escaped. v1.2 forbids `:` inside model
|
|
14
|
+
* ids so this format is collision-free.
|
|
15
|
+
*/
|
|
16
|
+
export function prewarmKey(modelId, protocol, paymentMethod) {
|
|
17
|
+
return `${modelId.trim().toLowerCase()}\u0001${protocol.trim().toLowerCase()}\u0001${paymentMethod.trim().toLowerCase()}`;
|
|
18
|
+
}
|
|
19
|
+
function parseKey(key) {
|
|
20
|
+
const parts = key.split("\u0001");
|
|
21
|
+
if (parts.length !== 3) {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
const [modelId, protocol, paymentMethod] = parts;
|
|
25
|
+
if (!modelId || !protocol || !paymentMethod) {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
return { modelId, protocol, paymentMethod };
|
|
29
|
+
}
|
|
30
|
+
export class PrewarmCache {
|
|
31
|
+
entries = new Map();
|
|
32
|
+
defaultTtlMs;
|
|
33
|
+
now;
|
|
34
|
+
constructor(options = {}) {
|
|
35
|
+
this.defaultTtlMs = options.defaultTtlMs ?? DEFAULT_PREWARM_TTL_MS;
|
|
36
|
+
this.now = options.now ?? Date.now;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Read an entry without mutating state. Returns `undefined` when the key is
|
|
40
|
+
* unknown; the caller decides whether "absent" should be treated as a miss
|
|
41
|
+
* (i.e. trigger a fresh prewarm) or as a known empty model.
|
|
42
|
+
*/
|
|
43
|
+
get(modelId, protocol, paymentMethod) {
|
|
44
|
+
return this.entries.get(prewarmKey(modelId, protocol, paymentMethod));
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Look up an entry and return a `Freshness` descriptor. This is the cheap
|
|
48
|
+
* path used on every inference request to decide whether a prewarm is
|
|
49
|
+
* still authoritative, expiring soon, or already stale.
|
|
50
|
+
*/
|
|
51
|
+
freshness(modelId, protocol, paymentMethod) {
|
|
52
|
+
const entry = this.get(modelId, protocol, paymentMethod);
|
|
53
|
+
if (!entry) {
|
|
54
|
+
return { present: false, expired: true, expiringSoon: true, state: "empty" };
|
|
55
|
+
}
|
|
56
|
+
const now = this.now();
|
|
57
|
+
const ageMs = now - entry.warmedAt;
|
|
58
|
+
const expired = ageMs >= entry.ttlMs;
|
|
59
|
+
const remainingMs = Math.max(0, entry.ttlMs - ageMs);
|
|
60
|
+
return {
|
|
61
|
+
present: true,
|
|
62
|
+
expired,
|
|
63
|
+
expiringSoon: !expired && remainingMs <= entry.ttlMs * 0.1,
|
|
64
|
+
remainingMs,
|
|
65
|
+
state: expired ? "stale" : entry.state,
|
|
66
|
+
entry
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Mark a (model, protocol, payment) triple as currently being warmed. If an
|
|
71
|
+
* existing warm entry is present it is kept untouched (the new probe
|
|
72
|
+
* supersedes it on commit) and the previous state is reported to the
|
|
73
|
+
* caller via the returned descriptor.
|
|
74
|
+
*/
|
|
75
|
+
beginWarming(modelId, protocol, paymentMethod, ttlMs) {
|
|
76
|
+
const key = prewarmKey(modelId, protocol, paymentMethod);
|
|
77
|
+
const previous = this.entries.get(key);
|
|
78
|
+
const now = this.now();
|
|
79
|
+
const entry = {
|
|
80
|
+
modelId,
|
|
81
|
+
protocol,
|
|
82
|
+
paymentMethod,
|
|
83
|
+
state: "warming",
|
|
84
|
+
candidates: previous?.candidates ?? [],
|
|
85
|
+
warmedAt: previous?.warmedAt ?? now,
|
|
86
|
+
ttlMs: ttlMs ?? previous?.ttlMs ?? this.defaultTtlMs,
|
|
87
|
+
consecutiveWarmingFailures: previous?.consecutiveWarmingFailures ?? 0,
|
|
88
|
+
lastInFlightAt: now
|
|
89
|
+
};
|
|
90
|
+
this.entries.set(key, entry);
|
|
91
|
+
logger.debug("prewarm.cache.warming_started", "prewarm probe in flight", {
|
|
92
|
+
modelId,
|
|
93
|
+
protocol,
|
|
94
|
+
paymentMethod,
|
|
95
|
+
ttlMs: entry.ttlMs,
|
|
96
|
+
previousState: previous?.state
|
|
97
|
+
});
|
|
98
|
+
return { key, entry, hadPrevious: Boolean(previous) };
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Commit a successful warm. The entry's `warmedAt` is reset to the current
|
|
102
|
+
* time so the TTL window starts fresh, and any prior stale candidates are
|
|
103
|
+
* replaced with the new probe results. The previous candidate set is
|
|
104
|
+
* returned for caller-side telemetry (e.g. detecting churn).
|
|
105
|
+
*/
|
|
106
|
+
commitWarm(input) {
|
|
107
|
+
const key = prewarmKey(input.modelId, input.protocol, input.paymentMethod);
|
|
108
|
+
const previous = this.entries.get(key);
|
|
109
|
+
const now = this.now();
|
|
110
|
+
const next = {
|
|
111
|
+
modelId: input.modelId,
|
|
112
|
+
protocol: input.protocol,
|
|
113
|
+
paymentMethod: input.paymentMethod,
|
|
114
|
+
state: input.candidates.length > 0 ? "warm" : "empty",
|
|
115
|
+
candidates: input.candidates.map(toCandidate),
|
|
116
|
+
warmedAt: now,
|
|
117
|
+
ttlMs: input.ttlMs ?? previous?.ttlMs ?? this.defaultTtlMs,
|
|
118
|
+
consecutiveWarmingFailures: 0,
|
|
119
|
+
lastInFlightAt: now
|
|
120
|
+
};
|
|
121
|
+
this.entries.set(key, next);
|
|
122
|
+
if (input.candidates.length === 0) {
|
|
123
|
+
logger.warn("prewarm.cache.commit_empty", "prewarm commit returned no candidates", {
|
|
124
|
+
modelId: input.modelId,
|
|
125
|
+
protocol: input.protocol,
|
|
126
|
+
paymentMethod: input.paymentMethod
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
logger.info("prewarm.cache.committed", "prewarm commit updated candidates", {
|
|
131
|
+
modelId: input.modelId,
|
|
132
|
+
protocol: input.protocol,
|
|
133
|
+
paymentMethod: input.paymentMethod,
|
|
134
|
+
candidateCount: next.candidates.length,
|
|
135
|
+
ttlMs: next.ttlMs
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
return {
|
|
139
|
+
key,
|
|
140
|
+
entry: next,
|
|
141
|
+
replacedSellers: previous?.candidates.map((c) => c.sellerId) ?? []
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Mark a warm as failed. Consecutive failures are tracked so the scheduler
|
|
146
|
+
* can apply exponential backoff and so `tb doctor` can surface persistently
|
|
147
|
+
* broken models.
|
|
148
|
+
*/
|
|
149
|
+
recordFailure(modelId, protocol, paymentMethod, errorMessage) {
|
|
150
|
+
const key = prewarmKey(modelId, protocol, paymentMethod);
|
|
151
|
+
const previous = this.entries.get(key);
|
|
152
|
+
if (!previous) {
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
const next = {
|
|
156
|
+
...previous,
|
|
157
|
+
state: "stale",
|
|
158
|
+
consecutiveWarmingFailures: previous.consecutiveWarmingFailures + 1,
|
|
159
|
+
lastInFlightAt: this.now()
|
|
160
|
+
};
|
|
161
|
+
this.entries.set(key, next);
|
|
162
|
+
logger.warn("prewarm.cache.failure_recorded", "prewarm commit failed; entry marked stale", {
|
|
163
|
+
modelId,
|
|
164
|
+
protocol,
|
|
165
|
+
paymentMethod,
|
|
166
|
+
consecutiveFailures: next.consecutiveWarmingFailures,
|
|
167
|
+
errorMessage
|
|
168
|
+
});
|
|
169
|
+
return next;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Invalidate every entry that references the given seller. Used when the
|
|
173
|
+
* registry signals a seller is gone (grace period expires) or when a hard
|
|
174
|
+
* failure (e.g. 5xx storm) should drop the seller from the cache
|
|
175
|
+
* immediately.
|
|
176
|
+
*/
|
|
177
|
+
invalidateSeller(sellerId) {
|
|
178
|
+
let removed = 0;
|
|
179
|
+
for (const [key, entry] of this.entries.entries()) {
|
|
180
|
+
const filtered = entry.candidates.filter((candidate) => candidate.sellerId !== sellerId);
|
|
181
|
+
if (filtered.length !== entry.candidates.length) {
|
|
182
|
+
removed += 1;
|
|
183
|
+
this.entries.set(key, {
|
|
184
|
+
...entry,
|
|
185
|
+
candidates: filtered,
|
|
186
|
+
state: filtered.length > 0 ? entry.state : "empty"
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
if (removed > 0) {
|
|
191
|
+
logger.info("prewarm.cache.seller_invalidated", "seller dropped from all prewarm entries", {
|
|
192
|
+
sellerId,
|
|
193
|
+
entriesAffected: removed
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
return removed;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Invalidate a specific cache key. Used by `tb doctor --refresh <model>`
|
|
200
|
+
* and by the registry loop when a model is removed from the focus set.
|
|
201
|
+
*/
|
|
202
|
+
invalidateKey(modelId, protocol, paymentMethod) {
|
|
203
|
+
return this.entries.delete(prewarmKey(modelId, protocol, paymentMethod));
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Drop every entry whose TTL has expired. Returns the number of removed
|
|
207
|
+
* entries so the caller can log it.
|
|
208
|
+
*/
|
|
209
|
+
evictExpired(now = this.now()) {
|
|
210
|
+
let removed = 0;
|
|
211
|
+
for (const [key, entry] of this.entries.entries()) {
|
|
212
|
+
if (now - entry.warmedAt >= entry.ttlMs) {
|
|
213
|
+
this.entries.delete(key);
|
|
214
|
+
removed += 1;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
if (removed > 0) {
|
|
218
|
+
logger.info("prewarm.cache.evicted", "expired prewarm entries evicted", { removed });
|
|
219
|
+
}
|
|
220
|
+
return removed;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Returns `true` when the entry's TTL is within `withinMs` of expiring. The
|
|
224
|
+
* scheduler uses this to schedule idle-cycle prewarms just-in-time rather
|
|
225
|
+
* than at fixed wall-clock intervals.
|
|
226
|
+
*/
|
|
227
|
+
isExpiringSoon(modelId, protocol, paymentMethod, withinMs, now = this.now()) {
|
|
228
|
+
const entry = this.get(modelId, protocol, paymentMethod);
|
|
229
|
+
if (!entry) {
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
const age = now - entry.warmedAt;
|
|
233
|
+
return age >= entry.ttlMs - withinMs && age < entry.ttlMs;
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Snapshot all entries for diagnostics. Returns a deep-copy of the values
|
|
237
|
+
* so callers can serialize without risking mutation of cache state.
|
|
238
|
+
*/
|
|
239
|
+
snapshot() {
|
|
240
|
+
return Array.from(this.entries.values()).map((entry) => ({
|
|
241
|
+
...entry,
|
|
242
|
+
candidates: entry.candidates.map((candidate) => ({ ...candidate }))
|
|
243
|
+
}));
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* List every cached key, decoded back into its (model, protocol, payment)
|
|
247
|
+
* triple. Used by `tb doctor` to render the prewarm table.
|
|
248
|
+
*/
|
|
249
|
+
keys() {
|
|
250
|
+
const out = [];
|
|
251
|
+
for (const key of this.entries.keys()) {
|
|
252
|
+
const parsed = parseKey(key);
|
|
253
|
+
if (parsed) {
|
|
254
|
+
out.push(parsed);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return out;
|
|
258
|
+
}
|
|
259
|
+
size() {
|
|
260
|
+
return this.entries.size;
|
|
261
|
+
}
|
|
262
|
+
clear() {
|
|
263
|
+
this.entries.clear();
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
function toCandidate(input) {
|
|
267
|
+
return {
|
|
268
|
+
sellerId: input.sellerId,
|
|
269
|
+
url: input.url,
|
|
270
|
+
healthScore: clampScore(input.healthScore ?? 50),
|
|
271
|
+
lastSuccessAt: input.lastSuccessAt ?? 0,
|
|
272
|
+
lastFailAt: input.lastFailAt ?? 0,
|
|
273
|
+
avgLatencyMs: Math.max(0, input.avgLatencyMs ?? 0)
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
function clampScore(score) {
|
|
277
|
+
if (!Number.isFinite(score)) {
|
|
278
|
+
return 50;
|
|
279
|
+
}
|
|
280
|
+
if (score < 0) {
|
|
281
|
+
return 0;
|
|
282
|
+
}
|
|
283
|
+
if (score > 100) {
|
|
284
|
+
return 100;
|
|
285
|
+
}
|
|
286
|
+
return score;
|
|
287
|
+
}
|
|
288
|
+
//# sourceMappingURL=prewarm-cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prewarm-cache.js","sourceRoot":"","sources":["../../src/prewarm-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,MAAM,GAAG,kBAAkB,CAAC,yBAAyB,CAAC,CAAC;AAE7D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAkCrD;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,OAAe,EAAE,QAAgB,EAAE,aAAqB;IACjF,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,SAAS,aAAa,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;AAC5H,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,GAAG,KAAK,CAAC;IACjD,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,aAAa,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC;AAC9C,CAAC;AAOD,MAAM,OAAO,YAAY;IACN,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC1C,YAAY,CAAS;IACrB,GAAG,CAAe;IAEnC,YAAY,UAA+B,EAAE;QAC3C,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,sBAAsB,CAAC;QACnE,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,OAAe,EAAE,QAAgB,EAAE,aAAqB;QAC1D,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;IACxE,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,OAAe,EAAE,QAAgB,EAAE,aAAqB;QAChE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC/E,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC;QACnC,MAAM,OAAO,GAAG,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;QACrC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;QACrD,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO;YACP,YAAY,EAAE,CAAC,OAAO,IAAI,WAAW,IAAI,KAAK,CAAC,KAAK,GAAG,GAAG;YAC1D,WAAW;YACX,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK;YACtC,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,OAAe,EAAE,QAAgB,EAAE,aAAqB,EAAE,KAAc;QACnF,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAiB;YAC1B,OAAO;YACP,QAAQ;YACR,aAAa;YACb,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,EAAE;YACtC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,IAAI,GAAG;YACnC,KAAK,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,IAAI,CAAC,YAAY;YACpD,0BAA0B,EAAE,QAAQ,EAAE,0BAA0B,IAAI,CAAC;YACrE,cAAc,EAAE,GAAG;SACpB,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,yBAAyB,EAAE;YACvE,OAAO;YACP,QAAQ;YACR,aAAa;YACb,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,aAAa,EAAE,QAAQ,EAAE,KAAK;SAC/B,CAAC,CAAC;QACH,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxD,CAAC;IAED;;;;;OAKG;IACH,UAAU,CAAC,KAMV;QACC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAiB;YACzB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;YACrD,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC;YAC7C,QAAQ,EAAE,GAAG;YACb,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,IAAI,CAAC,YAAY;YAC1D,0BAA0B,EAAE,CAAC;YAC7B,cAAc,EAAE,GAAG;SACpB,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE5B,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,uCAAuC,EAAE;gBACjF,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,aAAa,EAAE,KAAK,CAAC,aAAa;aACnC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE,mCAAmC,EAAE;gBAC1E,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;gBACtC,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,GAAG;YACH,KAAK,EAAE,IAAI;YACX,eAAe,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE;SACnE,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,aAAa,CAAC,OAAe,EAAE,QAAgB,EAAE,aAAqB,EAAE,YAAqB;QAC3F,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,IAAI,GAAiB;YACzB,GAAG,QAAQ;YACX,KAAK,EAAE,OAAO;YACd,0BAA0B,EAAE,QAAQ,CAAC,0BAA0B,GAAG,CAAC;YACnE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;SAC3B,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE,2CAA2C,EAAE;YACzF,OAAO;YACP,QAAQ;YACR,aAAa;YACb,mBAAmB,EAAE,IAAI,CAAC,0BAA0B;YACpD,YAAY;SACb,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,QAAgB;QAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;YACzF,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;gBAChD,OAAO,IAAI,CAAC,CAAC;gBACb,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;oBACpB,GAAG,KAAK;oBACR,UAAU,EAAE,QAAQ;oBACpB,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;iBACnD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE,yCAAyC,EAAE;gBACzF,QAAQ;gBACR,eAAe,EAAE,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,OAAe,EAAE,QAAgB,EAAE,aAAqB;QACpE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;QACnC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,IAAI,GAAG,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACzB,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,iCAAiC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACvF,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,OAAe,EAAE,QAAgB,EAAE,aAAqB,EAAE,QAAgB,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;QACjH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC;QACjC,OAAO,GAAG,IAAI,KAAK,CAAC,KAAK,GAAG,QAAQ,IAAI,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC;IAC5D,CAAC;IAED;;;OAGG;IACH,QAAQ;QACN,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACvD,GAAG,KAAK;YACR,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;SACpE,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,IAAI;QACF,MAAM,GAAG,GAAwE,EAAE,CAAC;QACpF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,MAAM,EAAE,CAAC;gBACX,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF;AAuBD,SAAS,WAAW,CAAC,KAA4B;IAC/C,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,WAAW,EAAE,UAAU,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;QAChD,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;QACvC,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,CAAC;QACjC,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;QAChB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import type { RegistrySeller } from "./seller-catalog.js";
|
|
2
|
+
import type { ModelIndex } from "./model-index.js";
|
|
3
|
+
import type { PrewarmCache } from "./prewarm-cache.js";
|
|
4
|
+
export type PrewarmReason = "startup" | "lazy" | "idle" | "explicit";
|
|
5
|
+
export interface ProbeResult {
|
|
6
|
+
ok: boolean;
|
|
7
|
+
latencyMs: number;
|
|
8
|
+
httpStatus?: number;
|
|
9
|
+
errorMessage?: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The probe function used by the scheduler. Decoupled so the scheduler can
|
|
13
|
+
* be unit-tested without spinning up HTTP servers. The default
|
|
14
|
+
* implementation in `health-probe.ts` (PR-2/PR-3) calls
|
|
15
|
+
* `GET <seller.url>/healthz` with a 3s `AbortSignal.timeout`. Probers must
|
|
16
|
+
* observe the provided `AbortSignal` and reject when it aborts so the
|
|
17
|
+
* scheduler can short-circuit in-flight probes on `stop()`.
|
|
18
|
+
*/
|
|
19
|
+
export type SellerProber = (seller: RegistrySeller, signal: AbortSignal) => Promise<ProbeResult>;
|
|
20
|
+
export interface PrewarmSchedulerOptions {
|
|
21
|
+
modelIndex: ModelIndex;
|
|
22
|
+
cache: PrewarmCache;
|
|
23
|
+
prober: SellerProber;
|
|
24
|
+
concurrency?: number;
|
|
25
|
+
perSellerMinIntervalMs?: number;
|
|
26
|
+
maxPrewarmPerMinute?: number;
|
|
27
|
+
idleIntervalMs?: number;
|
|
28
|
+
startupJitterMinMs?: number;
|
|
29
|
+
startupJitterMaxMs?: number;
|
|
30
|
+
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
31
|
+
random?: () => number;
|
|
32
|
+
now?: () => number;
|
|
33
|
+
protocol?: string;
|
|
34
|
+
paymentMethod?: string;
|
|
35
|
+
}
|
|
36
|
+
interface PrewarmTask {
|
|
37
|
+
id: number;
|
|
38
|
+
modelId: string;
|
|
39
|
+
reason: PrewarmReason;
|
|
40
|
+
protocol: string;
|
|
41
|
+
paymentMethod: string;
|
|
42
|
+
enqueuedAt: number;
|
|
43
|
+
sellerIds: string[];
|
|
44
|
+
startedAt?: number;
|
|
45
|
+
completedAt?: number;
|
|
46
|
+
status: "queued" | "running" | "succeeded" | "failed" | "canceled" | "rate_limited";
|
|
47
|
+
errorMessage?: string;
|
|
48
|
+
}
|
|
49
|
+
export interface PrewarmSchedulerStats {
|
|
50
|
+
queueDepth: number;
|
|
51
|
+
inFlight: number;
|
|
52
|
+
totalScheduled: number;
|
|
53
|
+
totalSucceeded: number;
|
|
54
|
+
totalFailed: number;
|
|
55
|
+
totalRateLimited: number;
|
|
56
|
+
recentProbesInLastMinute: number;
|
|
57
|
+
concurrency: number;
|
|
58
|
+
maxPrewarmPerMinute: number;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Background scheduler that warms up sellers for a (model, protocol,
|
|
62
|
+
* payment) triple on demand. The scheduler owns:
|
|
63
|
+
* - queue management with bounded concurrency (default 4)
|
|
64
|
+
* - per-seller rate limiting (default 30s between probes to the same
|
|
65
|
+
* seller, even across different models)
|
|
66
|
+
* - global rate limiting (default 30 probes/minute)
|
|
67
|
+
* - jitter on startup and between probes to avoid thundering herds
|
|
68
|
+
*
|
|
69
|
+
* The scheduler does NOT own HTTP I/O; that lives in the injected
|
|
70
|
+
* `prober` so tests can swap in a deterministic stub.
|
|
71
|
+
*/
|
|
72
|
+
export declare class PrewarmScheduler {
|
|
73
|
+
private readonly modelIndex;
|
|
74
|
+
private readonly cache;
|
|
75
|
+
private readonly prober;
|
|
76
|
+
private readonly concurrency;
|
|
77
|
+
private readonly perSellerMinIntervalMs;
|
|
78
|
+
private readonly maxPrewarmPerMinute;
|
|
79
|
+
private readonly idleIntervalMs;
|
|
80
|
+
private readonly startupJitterMinMs;
|
|
81
|
+
private readonly startupJitterMaxMs;
|
|
82
|
+
private readonly sleep;
|
|
83
|
+
private readonly random;
|
|
84
|
+
private readonly now;
|
|
85
|
+
private readonly protocol;
|
|
86
|
+
private readonly paymentMethod;
|
|
87
|
+
private readonly queue;
|
|
88
|
+
private inFlight;
|
|
89
|
+
private recentProbes;
|
|
90
|
+
private lastProbeAtBySeller;
|
|
91
|
+
private nextTaskId;
|
|
92
|
+
private totalScheduled;
|
|
93
|
+
private totalSucceeded;
|
|
94
|
+
private totalFailed;
|
|
95
|
+
private totalRateLimited;
|
|
96
|
+
private abortController;
|
|
97
|
+
private idleLoopPromise;
|
|
98
|
+
constructor(options: PrewarmSchedulerOptions);
|
|
99
|
+
/**
|
|
100
|
+
* Start the background idle loop. Safe to call once per scheduler
|
|
101
|
+
* instance; subsequent calls are no-ops. The idle loop probes any cached
|
|
102
|
+
* entry whose TTL is within 10% of expiry (`isExpiringSoon`).
|
|
103
|
+
*/
|
|
104
|
+
start(): void;
|
|
105
|
+
/**
|
|
106
|
+
* Cancel the idle loop and any pending tasks. Existing `inFlight` probes
|
|
107
|
+
* are not aborted (the prober owns its own timeout) but will not be
|
|
108
|
+
* dispatched to the cache.
|
|
109
|
+
*/
|
|
110
|
+
stop(): Promise<void>;
|
|
111
|
+
/**
|
|
112
|
+
* Enqueue a prewarm for a (model, protocol, payment) triple. The
|
|
113
|
+
* `reason` controls how aggressively the scheduler resolves candidates
|
|
114
|
+
* (e.g. `startup` defers; `lazy` waits on the returned promise). The
|
|
115
|
+
* returned promise resolves with the final task status once the queue
|
|
116
|
+
* drains or the scheduler is stopped.
|
|
117
|
+
*/
|
|
118
|
+
schedulePrewarm(input: {
|
|
119
|
+
modelId: string;
|
|
120
|
+
reason: PrewarmReason;
|
|
121
|
+
protocol?: string;
|
|
122
|
+
paymentMethod?: string;
|
|
123
|
+
blockOnFirst?: boolean;
|
|
124
|
+
}): Promise<PrewarmTask>;
|
|
125
|
+
/**
|
|
126
|
+
* Run a one-shot sweep that probes every focus-set model. Used by the
|
|
127
|
+
* `tb doctor --prewarm` explicit trigger and by the startup hook after
|
|
128
|
+
* the configured jitter window. Resolves once every scheduled task has
|
|
129
|
+
* reached a terminal state.
|
|
130
|
+
*/
|
|
131
|
+
runStartupPrewarm(modelIds: string[]): Promise<void>;
|
|
132
|
+
/**
|
|
133
|
+
* Force a sweep of any cache key whose TTL is about to expire. Returns
|
|
134
|
+
* the number of tasks that were enqueued. Intended to be called from
|
|
135
|
+
* the registry loop's heartbeat (replaces the v1 "all sellers" probe
|
|
136
|
+
* cycle with "only the ones we are about to forget").
|
|
137
|
+
*/
|
|
138
|
+
tickIdle(): number;
|
|
139
|
+
stats(): PrewarmSchedulerStats;
|
|
140
|
+
private jitterMs;
|
|
141
|
+
private runIdleLoop;
|
|
142
|
+
private dispatch;
|
|
143
|
+
private runTask;
|
|
144
|
+
private isOverBudget;
|
|
145
|
+
private recentProbesInLastMinute;
|
|
146
|
+
private recordProbeAttempt;
|
|
147
|
+
private isSellerRateLimited;
|
|
148
|
+
}
|
|
149
|
+
export {};
|
|
150
|
+
//# sourceMappingURL=prewarm-scheduler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prewarm-scheduler.d.ts","sourceRoot":"","sources":["../../src/prewarm-scheduler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAoB,MAAM,oBAAoB,CAAC;AAIzE,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAErE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAEjG,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,UAAU,CAAC;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IAErB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IAEnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,aAAa,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,UAAU,GAAG,cAAc,CAAC;IACpF,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,wBAAwB,EAAE,MAAM,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IAEtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsD;IAC5E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IAEnD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,UAAU,CAAK;IAEvB,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,gBAAgB,CAAK;IAE7B,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,eAAe,CAA8B;gBAEzC,OAAO,EAAE,uBAAuB;IAiB5C;;;;OAIG;IACH,KAAK,IAAI,IAAI;IAQb;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB3B;;;;;;OAMG;IACH,eAAe,CAAC,KAAK,EAAE;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,aAAa,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GAAG,OAAO,CAAC,WAAW,CAAC;IA4CxB;;;;;OAKG;IACG,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAW1D;;;;;OAKG;IACH,QAAQ,IAAI,MAAM;IAkBlB,KAAK,IAAI,qBAAqB;IAiB9B,OAAO,CAAC,QAAQ;YAKF,WAAW;YAoBX,QAAQ;YAuDR,OAAO;IA4IrB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,wBAAwB;IAQhC,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,mBAAmB;CAO5B"}
|