@goodandready/dsh-key-rotation 0.8.77 → 0.8.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/lib/client-src/00-header.js +40 -0
- package/lib/client-src/10-locales.js +585 -0
- package/lib/client-src/20-hooks.js +55 -0
- package/lib/client-src/30-utils.js +185 -0
- package/lib/client-src/40-components.js +346 -0
- package/lib/client-src/50-card-main.js +647 -0
- package/lib/client-src/51-card-pool-mutators.js +501 -0
- package/lib/client-src/52-card-pool-rows.js +590 -0
- package/lib/client-src/53-card-dialogs-and-render.js +568 -0
- package/lib/client-src/60-header-chip.js +137 -0
- package/lib/client-src/90-apply.js +69 -0
- package/lib/index.js +3 -0
- package/lib/model-catalog.js +296 -0
- package/lib/pool-selection.js +36 -13
- package/lib/quota-runway.js +222 -0
- package/lib/stream-metrics.js +13 -5
- package/lib/transports/anthropic.js +125 -0
- package/lib/transports/index.js +23 -0
- package/lib/transports/openai.js +93 -0
- package/lib/transports/sse.js +160 -0
- package/lib/transports/tokens.js +124 -0
- package/package.json +15 -10
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// (issue 201, #499) header chip: one dot + counts for all pools. Green = all healthy,
|
|
2
|
+
// amber = some keys cooling, red = a pool fully exhausted. Click opens quick actions popover.
|
|
3
|
+
function KeyRotationHeaderChip(props) {
|
|
4
|
+
const t = resolveT(props);
|
|
5
|
+
const [snap, setSnap] = React.useState(null);
|
|
6
|
+
const [open, setOpen] = React.useState(false);
|
|
7
|
+
const [, setTick] = React.useState(0);
|
|
8
|
+
const [busy, setBusy] = React.useState(false);
|
|
9
|
+
const ref = React.useRef(null);
|
|
10
|
+
|
|
11
|
+
const load = () => {
|
|
12
|
+
if (typeof document !== 'undefined' && document.visibilityState !== 'visible') return;
|
|
13
|
+
fetch('/dsh-key-rotation/health', {
|
|
14
|
+
headers: { accept: 'application/json' },
|
|
15
|
+
credentials: 'same-origin',
|
|
16
|
+
signal: (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') ? AbortSignal.timeout(8000) : undefined
|
|
17
|
+
})
|
|
18
|
+
.then((r) => (r.ok ? r.json() : null))
|
|
19
|
+
.then((d) => { if (d) setSnap(d); })
|
|
20
|
+
.catch((e) => { void e; });
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
React.useEffect(() => {
|
|
24
|
+
load();
|
|
25
|
+
const id = setInterval(load, 5000);
|
|
26
|
+
const onVis = () => { if (typeof document !== 'undefined' && document.visibilityState === 'visible') load(); };
|
|
27
|
+
if (typeof document !== 'undefined') document.addEventListener('visibilitychange', onVis);
|
|
28
|
+
return () => {
|
|
29
|
+
clearInterval(id);
|
|
30
|
+
if (typeof document !== 'undefined') document.removeEventListener('visibilitychange', onVis);
|
|
31
|
+
};
|
|
32
|
+
}, []);
|
|
33
|
+
|
|
34
|
+
React.useEffect(() => {
|
|
35
|
+
if (!props?.settingsSource?.subscribe) return;
|
|
36
|
+
return props.settingsSource.subscribe(() => {
|
|
37
|
+
setTick((c) => c + 1);
|
|
38
|
+
});
|
|
39
|
+
}, [props?.settingsSource]);
|
|
40
|
+
|
|
41
|
+
React.useEffect(() => {
|
|
42
|
+
if (!open) return;
|
|
43
|
+
const onDocClick = (e) => {
|
|
44
|
+
if (ref.current && !ref.current.contains(e.target)) setOpen(false);
|
|
45
|
+
};
|
|
46
|
+
document.addEventListener('click', onDocClick);
|
|
47
|
+
return () => document.removeEventListener('click', onDocClick);
|
|
48
|
+
}, [open]);
|
|
49
|
+
|
|
50
|
+
const runAction = (action, provider, keyRef) => {
|
|
51
|
+
setBusy(true);
|
|
52
|
+
fetch('/dsh-key-rotation/quick-action', {
|
|
53
|
+
method: 'POST',
|
|
54
|
+
headers: { 'content-type': 'application/json', accept: 'application/json' },
|
|
55
|
+
body: JSON.stringify({ action, provider, ref: keyRef }),
|
|
56
|
+
credentials: 'same-origin',
|
|
57
|
+
signal: (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') ? AbortSignal.timeout(5000) : undefined,
|
|
58
|
+
})
|
|
59
|
+
.then((r) => r.json())
|
|
60
|
+
.then(() => { load(); })
|
|
61
|
+
.catch((err) => { void err; })
|
|
62
|
+
.finally(() => { setBusy(false); });
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
let settingsVal = null;
|
|
66
|
+
try {
|
|
67
|
+
const sSnap = props?.settingsSource?.getSnapshot?.()?.service?.getSnapshot?.();
|
|
68
|
+
settingsVal = sSnap?.value;
|
|
69
|
+
} catch (e) {
|
|
70
|
+
void e;
|
|
71
|
+
}
|
|
72
|
+
if (settingsVal && settingsVal.showChatHeaderChip === false) return null;
|
|
73
|
+
if (snap && snap.showChatHeaderChip === false) return null;
|
|
74
|
+
|
|
75
|
+
const poolsObj = (snap && snap.pools) || {};
|
|
76
|
+
const poolEntries = Object.entries(poolsObj);
|
|
77
|
+
const total = poolEntries.reduce((a, [, p]) => a + (p.total || 0), 0);
|
|
78
|
+
const healthy = poolEntries.reduce((a, [, p]) => a + (p.healthy || 0), 0);
|
|
79
|
+
const anyExhausted = poolEntries.some(([, p]) => p.exhausted);
|
|
80
|
+
const color = !poolEntries.length ? 'var(--dsw-alias-label-tertiary)' : anyExhausted ? 'var(--dsw-alias-state-error-primary)' : healthy < total ? 'var(--dsw-alias-state-warn-primary)' : 'var(--dsw-alias-state-success-primary)';
|
|
81
|
+
const label = poolEntries.length ? `${healthy}/${total} rot` : 'rot';
|
|
82
|
+
|
|
83
|
+
return h('div', { ref, style: { position: 'relative', display: 'inline-flex' } },
|
|
84
|
+
h('button', {
|
|
85
|
+
type: 'button',
|
|
86
|
+
className: 'krot-header-chip',
|
|
87
|
+
title: t('title'),
|
|
88
|
+
onClick: () => setOpen((v) => !v),
|
|
89
|
+
},
|
|
90
|
+
h('span', { style: { width: '8px', height: '8px', borderRadius: '50%', background: color, flex: 'none', boxShadow: `0 0 6px ${color}` } }),
|
|
91
|
+
label,
|
|
92
|
+
h('span', { style: { fontSize: '9px', opacity: 0.6 } }, open ? '▲' : '▼')
|
|
93
|
+
),
|
|
94
|
+
open ? h('div', { className: 'krot-popover', style: { minWidth: '220px' } },
|
|
95
|
+
h('div', { className: 'krot-pop-title' }, t('headerPoolsTitle')),
|
|
96
|
+
!poolEntries.length ? h('div', { style: { fontSize: '12px', color: 'var(--dsw-alias-label-tertiary)' } }, t('noActivePools')) : null,
|
|
97
|
+
poolEntries.map(([name, p]) => {
|
|
98
|
+
const c = p.exhausted ? 'var(--dsw-alias-state-error-primary)' : (p.healthy < p.total ? 'var(--dsw-alias-state-warn-primary)' : 'var(--dsw-alias-state-success-primary)');
|
|
99
|
+
const provName = p.providerName || name;
|
|
100
|
+
const tail = p.lastUsedTail;
|
|
101
|
+
return h('div', { key: name, className: 'krot-pop-row', style: { display: 'flex', flexDirection: 'column', gap: '4px', padding: '6px 0', borderBottom: '1px solid var(--dsw-alias-border-subtle)' } },
|
|
102
|
+
h('div', { style: { display: 'flex', justifyContent: 'space-between', alignItems: 'center' } },
|
|
103
|
+
h('div', { className: 'krot-pop-name' },
|
|
104
|
+
h('span', { style: { width: '7px', height: '7px', borderRadius: '50%', background: c, flex: 'none' } }),
|
|
105
|
+
name,
|
|
106
|
+
tail ? h('span', { style: { marginLeft: '6px', fontSize: '11px', color: 'var(--dsw-alias-label-tertiary)', fontFeatureSettings: '"tnum"', fontFamily: 'monospace' } }, `...${tail}`) : null
|
|
107
|
+
),
|
|
108
|
+
h('div', { className: 'krot-pop-count', style: { color: c, fontWeight: 600 } }, `${p.healthy}/${p.total}`)
|
|
109
|
+
),
|
|
110
|
+
h('div', { style: { display: 'flex', gap: '4px', marginTop: '2px' } },
|
|
111
|
+
h('button', {
|
|
112
|
+
type: 'button',
|
|
113
|
+
disabled: busy,
|
|
114
|
+
onClick: (e) => { e.stopPropagation(); runAction('skip', provName, p.lastUsed); },
|
|
115
|
+
title: t('quickSkipTitle'),
|
|
116
|
+
style: { padding: '2px 6px', fontSize: '11px', borderRadius: '4px', border: '1px solid var(--dsw-alias-border-subtle)', background: 'transparent', cursor: 'pointer', color: 'inherit' }
|
|
117
|
+
}, `⏭️ ${t('quickSkip')}`),
|
|
118
|
+
h('button', {
|
|
119
|
+
type: 'button',
|
|
120
|
+
disabled: busy,
|
|
121
|
+
onClick: (e) => { e.stopPropagation(); runAction('reset', provName, p.lastUsed); },
|
|
122
|
+
title: t('quickResetTitle'),
|
|
123
|
+
style: { padding: '2px 6px', fontSize: '11px', borderRadius: '4px', border: '1px solid var(--dsw-alias-border-subtle)', background: 'transparent', cursor: 'pointer', color: 'inherit' }
|
|
124
|
+
}, `🔄 ${t('quickReset')}`),
|
|
125
|
+
h('button', {
|
|
126
|
+
type: 'button',
|
|
127
|
+
disabled: busy || !p.lastUsed,
|
|
128
|
+
onClick: (e) => { e.stopPropagation(); runAction('pause', provName, p.lastUsed); },
|
|
129
|
+
title: t('quickPauseTitle'),
|
|
130
|
+
style: { padding: '2px 6px', fontSize: '11px', borderRadius: '4px', border: '1px solid var(--dsw-alias-border-subtle)', background: 'transparent', cursor: 'pointer', color: 'inherit' }
|
|
131
|
+
}, `⏸️ ${t('quickPause')}`)
|
|
132
|
+
)
|
|
133
|
+
);
|
|
134
|
+
})
|
|
135
|
+
) : null
|
|
136
|
+
);
|
|
137
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
function apply(ctx) {
|
|
2
|
+
const settingsSource = createSettingsSource(ctx);
|
|
3
|
+
ctx.effect(() => ctx.locale.register(NS, { en, zh }), 'dsh-key-rotation: dictionaries');
|
|
4
|
+
// Header chip (#201): status dot in the session header utilities slot,
|
|
5
|
+
// same slot dsh-gitea / dsh-subscriptions use for their header widgets.
|
|
6
|
+
ctx.effect(() => {
|
|
7
|
+
if (!ctx.slots) return;
|
|
8
|
+
try {
|
|
9
|
+
ctx.slots.inject('conversation.session.header.utilities', () =>
|
|
10
|
+
ctx.slots.register(
|
|
11
|
+
{ name: 'conversation.session.header.utilities', id: 'dsh-key-rotation-header-chip', order: 6, locale: NS, inject: () => ({ ctx, settingsSource }) },
|
|
12
|
+
KeyRotationHeaderChip,
|
|
13
|
+
));
|
|
14
|
+
} catch { /* slot not available in this build */ }
|
|
15
|
+
}, 'dsh-key-rotation: header chip');
|
|
16
|
+
|
|
17
|
+
function useLocale() {
|
|
18
|
+
return useActiveLocale(ctx);
|
|
19
|
+
}
|
|
20
|
+
// Collapsible card in Settings -> Plugins -> Plugin settings
|
|
21
|
+
// (settings.plugin.item), matching Model Sync / Spendmeter / Vision Bridge.
|
|
22
|
+
// key MUST equal the settings namespace (NS), else the tab silently skips it.
|
|
23
|
+
|
|
24
|
+
function KeyRotationCard(props) {
|
|
25
|
+
const locale = useLocale();
|
|
26
|
+
const t = resolveT(props);
|
|
27
|
+
const page = !!(props && props.view === 'page');
|
|
28
|
+
const [open, setOpen] = React.useState(!!page);
|
|
29
|
+
// Row seat (plugins.row.config): the host page draws title/icon/crumb and the
|
|
30
|
+
// padding, so the summary is a one-liner and the page drops our card chrome.
|
|
31
|
+
if (props && props.view === 'summary') {
|
|
32
|
+
return h('span', { className: 'krot-card-description' }, t('subtitle'));
|
|
33
|
+
}
|
|
34
|
+
return h(page ? 'div' : 'div', { className: page ? 'krot-page' : 'krot-card' + (open ? ' krot-card-open' : '') },
|
|
35
|
+
h('button', { type: 'button', className: 'krot-card-header', style: page ? { display: 'none' } : undefined, 'aria-expanded': page ? true : open, onClick: () => setOpen((v) => !v) },
|
|
36
|
+
h('span', { className: 'krot-card-head-text' },
|
|
37
|
+
h('span', { className: 'krot-card-name' }, t('title')),
|
|
38
|
+
h('span', { className: 'krot-card-description' }, t('subtitle'))),
|
|
39
|
+
h('span', { className: 'krot-card-chevron' + (open ? ' krot-card-chevron-open' : ''), 'aria-hidden': 'true' },
|
|
40
|
+
h('svg', { width: 14, height: 14, viewBox: '0 0 14 14', fill: 'none', stroke: 'currentColor', strokeWidth: 1.5, style: { display: 'block' } },
|
|
41
|
+
h('path', { d: 'M3.5 5.25L7 8.75L10.5 5.25' })))),
|
|
42
|
+
(page || open) ? h('div', { className: 'krot-card-body' }, h(KeyRotationErrorBoundary, null, h(KeyRotationSection, { ...props, locale }))) : null);
|
|
43
|
+
}
|
|
44
|
+
// Register each available seat independently. A missing legacy seat must
|
|
45
|
+
// not prevent the current bundle or row seat from registering.
|
|
46
|
+
for (const [name, key] of [
|
|
47
|
+
['plugins.bundle.config', PKG],
|
|
48
|
+
['plugins.row.config', ROW_CONFIG_KEY],
|
|
49
|
+
['settings.plugin.item', NS],
|
|
50
|
+
]) {
|
|
51
|
+
try {
|
|
52
|
+
ctx.slots.inject(name, () => ctx.slots.register(
|
|
53
|
+
{ name, key, locale: NS, inject: () => ({ ctx, settingsSource }) }, KeyRotationCard,
|
|
54
|
+
));
|
|
55
|
+
} catch (error) {
|
|
56
|
+
bestEffort('slots.register.log', () => { console.error('[dsh-key-rotation] settings seat unavailable', name, error); });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
module.exports = { apply, inject: ['slots', 'locale'] };
|
|
62
|
+
if (typeof globalThis !== 'undefined') {
|
|
63
|
+
globalThis.__dshKeyRotationClientInternals = {
|
|
64
|
+
quotaMapOf, withQuotas, modelEntriesOf, isValidModelId, formatCountdown, validatePoolDraft,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
return module.exports;
|
|
68
|
+
},
|
|
69
|
+
});
|
package/lib/index.js
CHANGED
|
@@ -60,6 +60,9 @@ export { notifySwitch, notifyExhaustion, notifyEarlyWarning };
|
|
|
60
60
|
export { classifyResponseError } from './errors.js';
|
|
61
61
|
export { SessionRouter, defaultSessionRouter, unstickFailedKey } from './session-router.js';
|
|
62
62
|
export { isContentChunk } from './rotate.js';
|
|
63
|
+
export { ModelCatalog, defaultModelCatalog } from './model-catalog.js';
|
|
64
|
+
export { QuotaRunway, defaultQuotaRunway, estimateRunway } from './quota-runway.js';
|
|
65
|
+
export * as transports from './transports/index.js';
|
|
63
66
|
|
|
64
67
|
const NS = 'dsh-key-rotation';
|
|
65
68
|
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
// lib/model-catalog.js — Centralized model catalog with Cache Stampede protection (#550).
|
|
2
|
+
// Deduplicates concurrent in-flight requests and provides TTL-based caching and model capability normalization.
|
|
3
|
+
|
|
4
|
+
export const DEFAULT_CATALOG_TTL_MS = 60 * 60 * 1000; // 1 hour
|
|
5
|
+
export const DEFAULT_CATALOG_TIMEOUT_MS = 15_000; // 15 seconds
|
|
6
|
+
|
|
7
|
+
export const DEFAULT_CONTEXT_WINDOW = 128_000;
|
|
8
|
+
export const DEFAULT_MAX_TOKENS = 8_192;
|
|
9
|
+
|
|
10
|
+
export const KNOWN_MODELS_FALLBACK = {
|
|
11
|
+
'deepseek-chat': {
|
|
12
|
+
id: 'deepseek-chat',
|
|
13
|
+
name: 'DeepSeek Chat',
|
|
14
|
+
contextWindow: 64_000,
|
|
15
|
+
maxTokens: 8_000,
|
|
16
|
+
supportsReasoning: false,
|
|
17
|
+
supportsTools: true,
|
|
18
|
+
supportsVision: false,
|
|
19
|
+
},
|
|
20
|
+
'deepseek-reasoner': {
|
|
21
|
+
id: 'deepseek-reasoner',
|
|
22
|
+
name: 'DeepSeek Reasoner',
|
|
23
|
+
contextWindow: 64_000,
|
|
24
|
+
maxTokens: 8_000,
|
|
25
|
+
supportsReasoning: true,
|
|
26
|
+
supportsTools: false,
|
|
27
|
+
supportsVision: false,
|
|
28
|
+
},
|
|
29
|
+
'gpt-4o': {
|
|
30
|
+
id: 'gpt-4o',
|
|
31
|
+
name: 'GPT-4o',
|
|
32
|
+
contextWindow: 128_000,
|
|
33
|
+
maxTokens: 16_384,
|
|
34
|
+
supportsReasoning: false,
|
|
35
|
+
supportsTools: true,
|
|
36
|
+
supportsVision: true,
|
|
37
|
+
},
|
|
38
|
+
'gpt-4o-mini': {
|
|
39
|
+
id: 'gpt-4o-mini',
|
|
40
|
+
name: 'GPT-4o mini',
|
|
41
|
+
contextWindow: 128_000,
|
|
42
|
+
maxTokens: 16_384,
|
|
43
|
+
supportsReasoning: false,
|
|
44
|
+
supportsTools: true,
|
|
45
|
+
supportsVision: true,
|
|
46
|
+
},
|
|
47
|
+
'claude-3-5-sonnet-20241022': {
|
|
48
|
+
id: 'claude-3-5-sonnet-20241022',
|
|
49
|
+
name: 'Claude 3.5 Sonnet',
|
|
50
|
+
contextWindow: 200_000,
|
|
51
|
+
maxTokens: 8_192,
|
|
52
|
+
supportsReasoning: false,
|
|
53
|
+
supportsTools: true,
|
|
54
|
+
supportsVision: true,
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Normalizes raw model object from provider /models endpoint.
|
|
60
|
+
* @param {object|string} raw
|
|
61
|
+
* @returns {object}
|
|
62
|
+
*/
|
|
63
|
+
export function normalizeModelMeta(raw) {
|
|
64
|
+
if (typeof raw === 'string') {
|
|
65
|
+
const fallback = KNOWN_MODELS_FALLBACK[raw];
|
|
66
|
+
return fallback ? { ...fallback } : {
|
|
67
|
+
id: raw,
|
|
68
|
+
name: raw,
|
|
69
|
+
contextWindow: DEFAULT_CONTEXT_WINDOW,
|
|
70
|
+
maxTokens: DEFAULT_MAX_TOKENS,
|
|
71
|
+
supportsReasoning: raw.includes('reason') || raw.includes('r1') || raw.includes('thought'),
|
|
72
|
+
supportsTools: true,
|
|
73
|
+
supportsVision: raw.includes('vision') || raw.includes('4o') || raw.includes('vl'),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const id = String(raw?.id || raw?.slug || raw?.name || '');
|
|
78
|
+
const fallback = KNOWN_MODELS_FALLBACK[id];
|
|
79
|
+
|
|
80
|
+
const name = String(raw?.name || raw?.display_name || id);
|
|
81
|
+
const contextWindow =
|
|
82
|
+
Number(raw?.context_window || raw?.max_context_window || raw?.contextWindow) ||
|
|
83
|
+
fallback?.contextWindow ||
|
|
84
|
+
DEFAULT_CONTEXT_WINDOW;
|
|
85
|
+
const maxTokens =
|
|
86
|
+
Number(raw?.max_tokens || raw?.max_output_tokens || raw?.maxTokens) ||
|
|
87
|
+
fallback?.maxTokens ||
|
|
88
|
+
DEFAULT_MAX_TOKENS;
|
|
89
|
+
|
|
90
|
+
const caps = Array.isArray(raw?.capabilities) ? raw.capabilities : [];
|
|
91
|
+
const reasoningLevels = Array.isArray(raw?.supported_reasoning_levels) ? raw.supported_reasoning_levels : [];
|
|
92
|
+
|
|
93
|
+
const supportsReasoning =
|
|
94
|
+
Boolean(raw?.supportsReasoning) ||
|
|
95
|
+
caps.includes('thinking') ||
|
|
96
|
+
caps.includes('reasoning') ||
|
|
97
|
+
reasoningLevels.length > 0 ||
|
|
98
|
+
id.includes('reason') ||
|
|
99
|
+
id.includes('r1') ||
|
|
100
|
+
fallback?.supportsReasoning ||
|
|
101
|
+
false;
|
|
102
|
+
|
|
103
|
+
const supportsTools =
|
|
104
|
+
raw?.supportsTools !== undefined
|
|
105
|
+
? Boolean(raw.supportsTools)
|
|
106
|
+
: caps.includes('tools') || fallback?.supportsTools || true;
|
|
107
|
+
|
|
108
|
+
const inputModalities = Array.isArray(raw?.input_modalities) ? raw.input_modalities : [];
|
|
109
|
+
const supportsVision =
|
|
110
|
+
Boolean(raw?.supportsVision) ||
|
|
111
|
+
inputModalities.includes('image') ||
|
|
112
|
+
id.includes('vision') ||
|
|
113
|
+
id.includes('4o') ||
|
|
114
|
+
id.includes('vl') ||
|
|
115
|
+
fallback?.supportsVision ||
|
|
116
|
+
false;
|
|
117
|
+
|
|
118
|
+
return {
|
|
119
|
+
id,
|
|
120
|
+
name,
|
|
121
|
+
contextWindow,
|
|
122
|
+
maxTokens,
|
|
123
|
+
supportsReasoning,
|
|
124
|
+
supportsTools,
|
|
125
|
+
supportsVision,
|
|
126
|
+
raw,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export class ModelCatalog {
|
|
131
|
+
/**
|
|
132
|
+
* @param {object} [options]
|
|
133
|
+
* @param {number} [options.ttlMs]
|
|
134
|
+
* @param {number} [options.timeoutMs]
|
|
135
|
+
* @param {typeof fetch} [options.fetchImpl]
|
|
136
|
+
*/
|
|
137
|
+
constructor(options = {}) {
|
|
138
|
+
this.ttlMs = options.ttlMs ?? DEFAULT_CATALOG_TTL_MS;
|
|
139
|
+
this.timeoutMs = options.timeoutMs ?? DEFAULT_CATALOG_TIMEOUT_MS;
|
|
140
|
+
this.fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
141
|
+
/** @type {Map<string, { at: number, models: Array<object> }>} */
|
|
142
|
+
this.cache = new Map();
|
|
143
|
+
/** @type {Map<string, Promise<Array<object>>>} In-flight deduplication map */
|
|
144
|
+
this.inflight = new Map();
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
_buildKey(endpoint, apiKey) {
|
|
148
|
+
const keyHint = apiKey ? apiKey.slice(-6) : 'anonymous';
|
|
149
|
+
return `${endpoint || 'default'}|${keyHint}`;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Gets list of models for an endpoint.
|
|
154
|
+
* If a request for this endpoint is already in flight, reuses the promise (anti-stampede).
|
|
155
|
+
*
|
|
156
|
+
* @param {object} options
|
|
157
|
+
* @param {string} options.endpoint Base URL or /models URL
|
|
158
|
+
* @param {string} [options.apiKey]
|
|
159
|
+
* @param {boolean} [options.forceRefresh]
|
|
160
|
+
* @param {typeof fetch} [options.fetchImpl]
|
|
161
|
+
* @returns {Promise<Array<object>>}
|
|
162
|
+
*/
|
|
163
|
+
async getModels(options = {}) {
|
|
164
|
+
const { endpoint, apiKey, forceRefresh = false } = options;
|
|
165
|
+
const fetchFn = options.fetchImpl || this.fetchImpl;
|
|
166
|
+
const cacheKey = this._buildKey(endpoint, apiKey);
|
|
167
|
+
const now = Date.now();
|
|
168
|
+
|
|
169
|
+
// 1. Check TTL cache if not forcing refresh
|
|
170
|
+
if (!forceRefresh) {
|
|
171
|
+
const cached = this.cache.get(cacheKey);
|
|
172
|
+
if (cached && (now - cached.at < this.ttlMs)) {
|
|
173
|
+
return cached.models;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// 2. Check in-flight deduplication map (Cache Stampede protection)
|
|
178
|
+
const existingPromise = this.inflight.get(cacheKey);
|
|
179
|
+
if (existingPromise) {
|
|
180
|
+
return existingPromise;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// 3. Initiate single fetch and record in inflight map
|
|
184
|
+
const fetchPromise = (async () => {
|
|
185
|
+
try {
|
|
186
|
+
const models = await this._fetchModels(endpoint, apiKey, fetchFn);
|
|
187
|
+
this.cache.set(cacheKey, { at: Date.now(), models });
|
|
188
|
+
return models;
|
|
189
|
+
} catch (err) {
|
|
190
|
+
// On error, if we have stale cache, serve stale cache
|
|
191
|
+
const stale = this.cache.get(cacheKey);
|
|
192
|
+
if (stale && Array.isArray(stale.models) && stale.models.length > 0) {
|
|
193
|
+
return stale.models;
|
|
194
|
+
}
|
|
195
|
+
// Fallback to known models
|
|
196
|
+
return Object.values(KNOWN_MODELS_FALLBACK).map(normalizeModelMeta);
|
|
197
|
+
} finally {
|
|
198
|
+
this.inflight.delete(cacheKey);
|
|
199
|
+
}
|
|
200
|
+
})();
|
|
201
|
+
|
|
202
|
+
this.inflight.set(cacheKey, fetchPromise);
|
|
203
|
+
return fetchPromise;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Gets metadata for a specific model ID.
|
|
208
|
+
* @param {string} modelId
|
|
209
|
+
* @param {object} [options]
|
|
210
|
+
* @returns {Promise<object|null>}
|
|
211
|
+
*/
|
|
212
|
+
async getModel(modelId, options = {}) {
|
|
213
|
+
if (!modelId) return null;
|
|
214
|
+
const models = await this.getModels(options);
|
|
215
|
+
const found = models.find((m) => m.id === modelId);
|
|
216
|
+
if (found) return found;
|
|
217
|
+
return normalizeModelMeta(modelId);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async _fetchModels(endpoint, apiKey, fetchFn) {
|
|
221
|
+
if (!endpoint || typeof fetchFn !== 'function') {
|
|
222
|
+
return Object.values(KNOWN_MODELS_FALLBACK).map(normalizeModelMeta);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
let url = endpoint;
|
|
226
|
+
if (!url.endsWith('/models') && !url.includes('/models?')) {
|
|
227
|
+
url = url.replace(/\/+$/, '') + '/models';
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const headers = {
|
|
231
|
+
accept: 'application/json',
|
|
232
|
+
'user-agent': 'dsh-key-rotation/model-catalog',
|
|
233
|
+
};
|
|
234
|
+
if (apiKey) {
|
|
235
|
+
headers.authorization = `Bearer ${apiKey}`;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const controller = new AbortController();
|
|
239
|
+
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
240
|
+
|
|
241
|
+
try {
|
|
242
|
+
const resp = await fetchFn(url, {
|
|
243
|
+
headers,
|
|
244
|
+
signal: controller.signal,
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
if (!resp.ok) {
|
|
248
|
+
throw new Error(`Catalog fetch failed with status ${resp.status}`);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const body = await resp.json();
|
|
252
|
+
let rawList = [];
|
|
253
|
+
|
|
254
|
+
if (Array.isArray(body?.data)) {
|
|
255
|
+
rawList = body.data;
|
|
256
|
+
} else if (Array.isArray(body?.models)) {
|
|
257
|
+
rawList = body.models;
|
|
258
|
+
} else if (body?.models && typeof body.models === 'object') {
|
|
259
|
+
rawList = Object.entries(body.models).map(([k, v]) => ({ id: k, ...(typeof v === 'object' ? v : {}) }));
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
if (rawList.length === 0) {
|
|
263
|
+
return Object.values(KNOWN_MODELS_FALLBACK).map(normalizeModelMeta);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
return rawList.map(normalizeModelMeta);
|
|
267
|
+
} finally {
|
|
268
|
+
clearTimeout(timer);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
invalidate(endpoint, apiKey) {
|
|
273
|
+
const key = this._buildKey(endpoint, apiKey);
|
|
274
|
+
this.cache.delete(key);
|
|
275
|
+
this.inflight.delete(key);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
clear() {
|
|
279
|
+
this.cache.clear();
|
|
280
|
+
this.inflight.clear();
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
getStatus(endpoint, apiKey) {
|
|
284
|
+
const key = this._buildKey(endpoint, apiKey);
|
|
285
|
+
const entry = this.cache.get(key);
|
|
286
|
+
const isInflight = this.inflight.has(key);
|
|
287
|
+
return {
|
|
288
|
+
cached: Boolean(entry),
|
|
289
|
+
count: entry?.models?.length ?? 0,
|
|
290
|
+
fetchedAt: entry?.at ?? null,
|
|
291
|
+
inflight: isInflight,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export const defaultModelCatalog = new ModelCatalog();
|
package/lib/pool-selection.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512, #548).
|
|
1
|
+
// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512, #548, #549).
|
|
2
2
|
// Extracted from lib/pool.js to enforce < 600 line modularity standard.
|
|
3
3
|
|
|
4
4
|
import {
|
|
@@ -8,11 +8,18 @@ import {
|
|
|
8
8
|
SESSION_TTL_MS,
|
|
9
9
|
} from './session-router.js';
|
|
10
10
|
|
|
11
|
+
import {
|
|
12
|
+
QuotaRunway,
|
|
13
|
+
defaultQuotaRunway,
|
|
14
|
+
} from './quota-runway.js';
|
|
15
|
+
|
|
11
16
|
export {
|
|
12
17
|
SessionRouter,
|
|
13
18
|
defaultSessionRouter,
|
|
14
19
|
MAX_SESSIONS,
|
|
15
20
|
SESSION_TTL_MS,
|
|
21
|
+
QuotaRunway,
|
|
22
|
+
defaultQuotaRunway,
|
|
16
23
|
};
|
|
17
24
|
|
|
18
25
|
export const MAX_SESSION_STICKY_ENTRIES = MAX_SESSIONS;
|
|
@@ -103,11 +110,13 @@ export function selectPool(modelPoolByProvider, providerToPool, provider, model)
|
|
|
103
110
|
|
|
104
111
|
/**
|
|
105
112
|
* Sort candidate refs according to pool routing strategy.
|
|
113
|
+
* Preemptively deprioritizes keys projected to exhaust their quota before reset (#549).
|
|
106
114
|
* Returns a new array; does not mutate input.
|
|
107
115
|
*/
|
|
108
116
|
export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
|
|
109
117
|
if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
|
|
110
|
-
|
|
118
|
+
let list = refs.slice();
|
|
119
|
+
|
|
111
120
|
if (strategy === 'sticky') {
|
|
112
121
|
const router = getOrCreateSessionRouter(deps.pool);
|
|
113
122
|
let chosenKey = null;
|
|
@@ -133,19 +142,16 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
|
|
|
133
142
|
list.unshift(last);
|
|
134
143
|
}
|
|
135
144
|
}
|
|
136
|
-
|
|
137
|
-
}
|
|
138
|
-
if (strategy === 'tier-priority') {
|
|
145
|
+
} else if (strategy === 'tier-priority') {
|
|
139
146
|
const tiers = deps.tiers || (deps.pool && deps.pool.tiers) || {};
|
|
140
|
-
|
|
147
|
+
list = list.sort((a, b) => {
|
|
141
148
|
const ta = (tiers[a] && Number(tiers[a]) >= 1) ? Number(tiers[a]) : 1;
|
|
142
149
|
const tb = (tiers[b] && Number(tiers[b]) >= 1) ? Number(tiers[b]) : 1;
|
|
143
150
|
return ta - tb;
|
|
144
151
|
});
|
|
145
|
-
}
|
|
146
|
-
if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
|
|
152
|
+
} else if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
|
|
147
153
|
const weights = deps.weights || (deps.pool && deps.pool.weightsMap) || {};
|
|
148
|
-
|
|
154
|
+
list = list.sort((a, b) => {
|
|
149
155
|
const ca = deps.concurrencyTracker.getActive(a) ?? 0;
|
|
150
156
|
const cb = deps.concurrencyTracker.getActive(b) ?? 0;
|
|
151
157
|
const wa = (weights[a] && Number(weights[a]) > 0) ? Number(weights[a]) : 1;
|
|
@@ -155,9 +161,8 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
|
|
|
155
161
|
if (scoreA !== scoreB) return scoreA - scoreB;
|
|
156
162
|
return ca - cb;
|
|
157
163
|
});
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
return list.sort((a, b) => {
|
|
164
|
+
} else if (strategy === 'lowest-latency' && deps.latencyHistogram && typeof deps.latencyHistogram.snapshot === 'function') {
|
|
165
|
+
list = list.sort((a, b) => {
|
|
161
166
|
const sa = deps.latencyHistogram.snapshot(a);
|
|
162
167
|
const sb = deps.latencyHistogram.snapshot(b);
|
|
163
168
|
const la = (sa && Number.isFinite(sa.p95)) ? sa.p95 : ((sa && Number.isFinite(sa.avg)) ? sa.avg : 500);
|
|
@@ -165,6 +170,25 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
|
|
|
165
170
|
return la - lb;
|
|
166
171
|
});
|
|
167
172
|
}
|
|
173
|
+
|
|
174
|
+
// Preemptively deprioritize keys projected to exhaust before quota window reset (#549)
|
|
175
|
+
const runway = deps.quotaRunway ?? deps.pool?.state?.quotaRunway;
|
|
176
|
+
if (runway && typeof runway.isExhaustingBeforeReset === 'function' && list.length > 1) {
|
|
177
|
+
const now = typeof deps.now === 'function' ? deps.now() : (Number(deps.now) || Date.now());
|
|
178
|
+
const healthy = [];
|
|
179
|
+
const exhausting = [];
|
|
180
|
+
for (const ref of list) {
|
|
181
|
+
if (runway.isExhaustingBeforeReset(ref, now)) {
|
|
182
|
+
exhausting.push(ref);
|
|
183
|
+
} else {
|
|
184
|
+
healthy.push(ref);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (healthy.length > 0 && exhausting.length > 0) {
|
|
188
|
+
return healthy.concat(exhausting);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
168
192
|
return list;
|
|
169
193
|
}
|
|
170
194
|
|
|
@@ -221,4 +245,3 @@ export function budgetVerdict(spend, budget) {
|
|
|
221
245
|
const ratio = spend / budget;
|
|
222
246
|
return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
|
|
223
247
|
}
|
|
224
|
-
|