@goodandready/dsh-key-rotation 0.8.86 → 0.8.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/lib/cache-affinity.js +237 -0
- package/lib/index.js +14 -0
- package/lib/pool-builder.js +28 -0
- package/lib/resolver.js +9 -0
- package/lib/rotate.js +12 -1
- package/lib/stream-metrics.js +6 -0
- package/lib/subagent-partitions.js +163 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.88 (2026-11-03)
|
|
4
|
+
|
|
5
|
+
### Subagent Concurrency Partitions & Task Isolation (Block 15 Release)
|
|
6
|
+
- **Subagent Partitions Module (Issue #575)**: Introduced `src/subagent-partitions.js` to detect child tasks and background subagents (`extractSubagentContext`) and partition pool capacity (`partitionPoolKeys`).
|
|
7
|
+
- **Primary Chat Immunity**: Protects main user conversations from rate limit spikes and concurrency starvation caused by parallel background tasks by reserving dedicated subagent keys (`subagentReservedKeysCount`) or applying isolated partitioning (`subagentPartitionStrategy: 'isolated' | 'shared-burst'`).
|
|
8
|
+
- **Per-Key Subagent Concurrency Caps**: Implemented `subagentMaxConcurrencyPerKey` (default: 2) to cap parallel subagent calls per key, preventing sudden RPM/TPM exhaustion.
|
|
9
|
+
- **Config & Schemastery Support**: Added `subagentPartitionsEnabled`, `subagentReservedKeysCount`, `subagentPartitionStrategy`, and `subagentMaxConcurrencyPerKey` across Schemastery configuration, pool builders, and TypeScript types.
|
|
10
|
+
|
|
11
|
+
## 0.8.87 (2026-11-03)
|
|
12
|
+
|
|
13
|
+
### Cache-Affinity Router for Hot Prompt Cache (Block 14 Release)
|
|
14
|
+
- **Cache-Affinity Module (Issue #573)**: Introduced `src/cache-affinity.js` with deterministic FNV-1a prompt prefix hashing (`extractPromptPrefixKey`) and LRU-managed `CacheAffinityRouter`.
|
|
15
|
+
- **Hot Prompt Cache Affinity**: Binds stable system prompts and context prefixes to the provider key that has already warmed the upstream prompt cache, maintaining high hit-rates (>80%) across conversation sessions and multi-step tool calls.
|
|
16
|
+
- **Synergistic Proactive Routing**: Cooperates with `applyPreemptiveYield` to ensure that hot-cache keys are prioritized only when they have safe token/request headroom, preventing cache optimization from triggering 429 rate limit failures.
|
|
17
|
+
- **Config & Schemastery Support**: Added `cacheAffinityEnabled`, `cacheAffinityTtlMs` (default: 5 minutes), and `cacheAffinityPrefixChars` (default: 2048 chars) across Schemastery configuration, pool builders, and TypeScript types.
|
|
18
|
+
|
|
3
19
|
## 0.8.86 (2026-11-03)
|
|
4
20
|
|
|
5
21
|
### Pre-emptive Token & Request Yield before 429 (Block 13 Release)
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
// src/cache-affinity.js — Cache-Affinity Router for hot prompt cache (#573).
|
|
2
|
+
// Pure module: zero runtime dependencies, strictly < 600 lines.
|
|
3
|
+
|
|
4
|
+
export const DEFAULT_CACHE_AFFINITY_TTL_MS = 300000; // 5 minutes
|
|
5
|
+
export const DEFAULT_CACHE_AFFINITY_PREFIX_CHARS = 2048;
|
|
6
|
+
export const MIN_CACHE_PREFIX_CHARS = 32;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* 32-bit FNV-1a hash algorithm for fast, deterministic string hashing.
|
|
10
|
+
* @param {string} str
|
|
11
|
+
* @returns {string} Hexadecimal hash representation
|
|
12
|
+
*/
|
|
13
|
+
export function fnv1a(str) {
|
|
14
|
+
if (typeof str !== 'string') return '0';
|
|
15
|
+
let hash = 0x811c9dc5;
|
|
16
|
+
for (let i = 0; i < str.length; i++) {
|
|
17
|
+
hash ^= str.charCodeAt(i);
|
|
18
|
+
hash = Math.imul(hash, 0x01000193);
|
|
19
|
+
}
|
|
20
|
+
return (hash >>> 0).toString(16);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Extracts and hashes a deterministic prompt prefix key from request options.
|
|
25
|
+
* Inspects system prompt, conversation messages, prompt string, or contents.
|
|
26
|
+
* @param {object} options - Request options (messages, prompt, system, model)
|
|
27
|
+
* @param {number} [maxChars=2048]
|
|
28
|
+
* @returns {string|null} Cache prefix key or null if insufficient content
|
|
29
|
+
*/
|
|
30
|
+
export function extractPromptPrefixKey(options, maxChars = DEFAULT_CACHE_AFFINITY_PREFIX_CHARS) {
|
|
31
|
+
if (!options || typeof options !== 'object') return null;
|
|
32
|
+
|
|
33
|
+
let text = '';
|
|
34
|
+
|
|
35
|
+
// 1. Explicit system prompt option
|
|
36
|
+
if (typeof options.system === 'string' && options.system.trim()) {
|
|
37
|
+
text += options.system.trim() + '\n';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// 2. Messages array (OpenAI / Anthropic format)
|
|
41
|
+
if (Array.isArray(options.messages) && options.messages.length > 0) {
|
|
42
|
+
for (const msg of options.messages) {
|
|
43
|
+
if (!msg) continue;
|
|
44
|
+
const role = String(msg.role ?? '').toLowerCase();
|
|
45
|
+
let content = '';
|
|
46
|
+
|
|
47
|
+
if (typeof msg.content === 'string') {
|
|
48
|
+
content = msg.content;
|
|
49
|
+
} else if (Array.isArray(msg.content)) {
|
|
50
|
+
// Multipart content (e.g. [{ type: 'text', text: '...' }])
|
|
51
|
+
content = msg.content
|
|
52
|
+
.map(part => (typeof part === 'string' ? part : (part?.text ?? '')))
|
|
53
|
+
.filter(Boolean)
|
|
54
|
+
.join('\n');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (content.trim()) {
|
|
58
|
+
text += `[${role}] ${content.trim()}\n`;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Stop once we have reached or exceeded maxChars
|
|
62
|
+
if (text.length >= maxChars) break;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// 3. Raw prompt string
|
|
67
|
+
if (!text && typeof options.prompt === 'string' && options.prompt.trim()) {
|
|
68
|
+
text = options.prompt.trim();
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// 4. Contents array (Gemini format)
|
|
72
|
+
if (!text && Array.isArray(options.contents) && options.contents.length > 0) {
|
|
73
|
+
for (const item of options.contents) {
|
|
74
|
+
const parts = item?.parts ?? [];
|
|
75
|
+
for (const p of parts) {
|
|
76
|
+
if (typeof p?.text === 'string' && p.text.trim()) {
|
|
77
|
+
text += p.text.trim() + '\n';
|
|
78
|
+
if (text.length >= maxChars) break;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
if (text.length >= maxChars) break;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
text = text.trim();
|
|
86
|
+
if (text.length < MIN_CACHE_PREFIX_CHARS) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const slice = text.slice(0, maxChars);
|
|
91
|
+
const hash = fnv1a(slice);
|
|
92
|
+
const model = options.model ? String(options.model).trim() : '';
|
|
93
|
+
|
|
94
|
+
return model ? `${model}:${hash}` : hash;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* CacheAffinityRouter tracks which key has warmed the upstream prompt cache.
|
|
99
|
+
*/
|
|
100
|
+
export class CacheAffinityRouter {
|
|
101
|
+
constructor(options = {}) {
|
|
102
|
+
this.ttlMs = options.ttlMs ?? DEFAULT_CACHE_AFFINITY_TTL_MS;
|
|
103
|
+
this.maxEntries = options.maxEntries ?? 1000;
|
|
104
|
+
this.entries = new Map();
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
resolveAffinity(cacheKey, candidateRefs, now = Date.now()) {
|
|
108
|
+
if (!cacheKey || !this.entries.has(cacheKey) || !Array.isArray(candidateRefs) || candidateRefs.length === 0) {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const entry = this.entries.get(cacheKey);
|
|
113
|
+
if (!entry) return null;
|
|
114
|
+
|
|
115
|
+
if (now >= entry.expiresAt) {
|
|
116
|
+
this.entries.delete(cacheKey);
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (!candidateRefs.includes(entry.ref)) {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
entry.hits = (entry.hits || 0) + 1;
|
|
125
|
+
return {
|
|
126
|
+
ref: entry.ref,
|
|
127
|
+
hits: entry.hits,
|
|
128
|
+
warmedAt: entry.warmedAt,
|
|
129
|
+
expiresAt: entry.expiresAt,
|
|
130
|
+
ttlRemainingMs: Math.max(0, entry.expiresAt - now),
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
recordAffinity(cacheKey, ref, options = {}) {
|
|
135
|
+
if (!cacheKey || !ref) return;
|
|
136
|
+
const now = options.now ?? Date.now();
|
|
137
|
+
const ttlMs = options.ttlMs ?? this.ttlMs;
|
|
138
|
+
|
|
139
|
+
if (this.entries.size >= this.maxEntries && !this.entries.has(cacheKey)) {
|
|
140
|
+
const oldestKey = this.entries.keys().next().value;
|
|
141
|
+
if (oldestKey) this.entries.delete(oldestKey);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const existing = this.entries.get(cacheKey);
|
|
145
|
+
const hits = existing ? (existing.hits || 0) : 0;
|
|
146
|
+
|
|
147
|
+
this.entries.set(cacheKey, {
|
|
148
|
+
ref,
|
|
149
|
+
warmedAt: existing?.warmedAt ?? now,
|
|
150
|
+
lastSeenAt: now,
|
|
151
|
+
expiresAt: now + ttlMs,
|
|
152
|
+
hits,
|
|
153
|
+
model: options.model,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
deleteAffinity(cacheKey) {
|
|
158
|
+
if (cacheKey) this.entries.delete(cacheKey);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
clear() {
|
|
162
|
+
this.entries.clear();
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
get size() {
|
|
166
|
+
return this.entries.size;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Obtains or lazily creates a CacheAffinityRouter on a pool state.
|
|
172
|
+
* @param {object} pool
|
|
173
|
+
* @returns {CacheAffinityRouter}
|
|
174
|
+
*/
|
|
175
|
+
export function getOrCreateCacheAffinityRouter(pool) {
|
|
176
|
+
if (!pool) return new CacheAffinityRouter();
|
|
177
|
+
if (!pool.state) pool.state = {};
|
|
178
|
+
if (!pool.state.cacheAffinityRouter || !(pool.state.cacheAffinityRouter instanceof CacheAffinityRouter)) {
|
|
179
|
+
pool.state.cacheAffinityRouter = new CacheAffinityRouter({
|
|
180
|
+
ttlMs: pool.cacheAffinityTtlMs ?? DEFAULT_CACHE_AFFINITY_TTL_MS,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
return pool.state.cacheAffinityRouter;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Applies prompt cache affinity ordering to candidate attempt list.
|
|
188
|
+
* @param {string[]} attemptList
|
|
189
|
+
* @param {object} pool
|
|
190
|
+
* @param {object} [options]
|
|
191
|
+
* @returns {string[]}
|
|
192
|
+
*/
|
|
193
|
+
export function applyCacheAffinity(attemptList, pool, options = {}) {
|
|
194
|
+
if (!Array.isArray(attemptList) || attemptList.length <= 1 || !pool) {
|
|
195
|
+
return (attemptList ?? []).slice();
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const enabled = options.enabled ?? pool.cacheAffinityEnabled ?? true;
|
|
199
|
+
if (!enabled) return attemptList.slice();
|
|
200
|
+
|
|
201
|
+
const cacheKey = options.cacheKey;
|
|
202
|
+
if (!cacheKey) return attemptList.slice();
|
|
203
|
+
|
|
204
|
+
// If a pinned key is configured and already first, keep pinned priority
|
|
205
|
+
if (pool.pinnedRef && attemptList[0] === pool.pinnedRef && options.preservePinned !== false) {
|
|
206
|
+
return attemptList.slice();
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const router = getOrCreateCacheAffinityRouter(pool);
|
|
210
|
+
const now = options.now ?? Date.now();
|
|
211
|
+
const affinity = router.resolveAffinity(cacheKey, attemptList, now);
|
|
212
|
+
|
|
213
|
+
if (!affinity || !affinity.ref) {
|
|
214
|
+
return attemptList.slice();
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const targetRef = affinity.ref;
|
|
218
|
+
const list = attemptList.slice();
|
|
219
|
+
const idx = list.indexOf(targetRef);
|
|
220
|
+
|
|
221
|
+
if (idx > 0) {
|
|
222
|
+
list.splice(idx, 1);
|
|
223
|
+
list.unshift(targetRef);
|
|
224
|
+
|
|
225
|
+
if (options.trace && typeof options.trace.addStep === 'function') {
|
|
226
|
+
options.trace.addStep(targetRef, 'PRIORITIZED', 'CACHE_AFFINITY', `Prompt cache hit affinity (key ${targetRef}, hits: ${affinity.hits})`);
|
|
227
|
+
}
|
|
228
|
+
if (options.pushEvent && typeof options.pushEvent === 'function') {
|
|
229
|
+
options.pushEvent(pool, targetRef, 'CACHE_AFFINITY_HIT', { cacheKey, hits: affinity.hits });
|
|
230
|
+
}
|
|
231
|
+
if (pool.state) {
|
|
232
|
+
pool.state.cacheAffinityHits = (pool.state.cacheAffinityHits ?? 0) + 1;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
return list;
|
|
237
|
+
}
|
package/lib/index.js
CHANGED
|
@@ -143,6 +143,13 @@ const SettingsSchema = Schema.object({
|
|
|
143
143
|
preemptiveRotationEnabled: Schema.boolean().default(true),
|
|
144
144
|
preemptiveSafetyMarginTokens: Schema.number().default(4000),
|
|
145
145
|
preemptiveSafetyMarginRequests: Schema.number().default(2),
|
|
146
|
+
cacheAffinityEnabled: Schema.boolean().default(true),
|
|
147
|
+
cacheAffinityTtlMs: Schema.number().default(300000),
|
|
148
|
+
cacheAffinityPrefixChars: Schema.number().default(2048),
|
|
149
|
+
subagentPartitionsEnabled: Schema.boolean().default(true),
|
|
150
|
+
subagentReservedKeysCount: Schema.number().default(0),
|
|
151
|
+
subagentPartitionStrategy: Schema.union(['shared-burst', 'isolated']).default('shared-burst'),
|
|
152
|
+
subagentMaxConcurrencyPerKey: Schema.number().default(2),
|
|
146
153
|
providers: Schema.array(Schema.object({
|
|
147
154
|
provider: Schema.string().required(),
|
|
148
155
|
keys: Schema.array(Schema.string()).default([]),
|
|
@@ -178,6 +185,13 @@ const SettingsSchema = Schema.object({
|
|
|
178
185
|
preemptiveRotationEnabled: Schema.boolean(),
|
|
179
186
|
preemptiveSafetyMarginTokens: Schema.number(),
|
|
180
187
|
preemptiveSafetyMarginRequests: Schema.number(),
|
|
188
|
+
cacheAffinityEnabled: Schema.boolean(),
|
|
189
|
+
cacheAffinityTtlMs: Schema.number(),
|
|
190
|
+
cacheAffinityPrefixChars: Schema.number(),
|
|
191
|
+
subagentPartitionsEnabled: Schema.boolean(),
|
|
192
|
+
subagentReservedKeysCount: Schema.number(),
|
|
193
|
+
subagentPartitionStrategy: Schema.union(['shared-burst', 'isolated']),
|
|
194
|
+
subagentMaxConcurrencyPerKey: Schema.number(),
|
|
181
195
|
})).default([...DEFAULT_PROVIDERS]),
|
|
182
196
|
});
|
|
183
197
|
|
package/lib/pool-builder.js
CHANGED
|
@@ -47,6 +47,13 @@ export function buildPoolItem({
|
|
|
47
47
|
preemptiveRotationEnabled = true,
|
|
48
48
|
preemptiveSafetyMarginTokens = 4000,
|
|
49
49
|
preemptiveSafetyMarginRequests = 2,
|
|
50
|
+
cacheAffinityEnabled = true,
|
|
51
|
+
cacheAffinityTtlMs = 300000,
|
|
52
|
+
cacheAffinityPrefixChars = 2048,
|
|
53
|
+
subagentPartitionsEnabled = true,
|
|
54
|
+
subagentReservedKeysCount = 0,
|
|
55
|
+
subagentPartitionStrategy = 'shared-burst',
|
|
56
|
+
subagentMaxConcurrencyPerKey = 2,
|
|
50
57
|
}) {
|
|
51
58
|
const seen = new Set();
|
|
52
59
|
const indexed = (Array.isArray(keys) ? keys : []).map((key, index) => ({ ref: typeof key === 'string' ? key.trim() : '', index }))
|
|
@@ -163,6 +170,13 @@ const weightsMap = {};
|
|
|
163
170
|
preemptiveRotationEnabled: preemptiveRotationEnabled ?? true,
|
|
164
171
|
preemptiveSafetyMarginTokens: typeof preemptiveSafetyMarginTokens === 'number' && Number.isFinite(preemptiveSafetyMarginTokens) && preemptiveSafetyMarginTokens >= 0 ? Math.floor(preemptiveSafetyMarginTokens) : 4000,
|
|
165
172
|
preemptiveSafetyMarginRequests: typeof preemptiveSafetyMarginRequests === 'number' && Number.isFinite(preemptiveSafetyMarginRequests) && preemptiveSafetyMarginRequests >= 0 ? Math.floor(preemptiveSafetyMarginRequests) : 2,
|
|
173
|
+
cacheAffinityEnabled: cacheAffinityEnabled ?? true,
|
|
174
|
+
cacheAffinityTtlMs: typeof cacheAffinityTtlMs === 'number' && Number.isFinite(cacheAffinityTtlMs) && cacheAffinityTtlMs > 0 ? Math.floor(cacheAffinityTtlMs) : 300000,
|
|
175
|
+
cacheAffinityPrefixChars: typeof cacheAffinityPrefixChars === 'number' && Number.isFinite(cacheAffinityPrefixChars) && cacheAffinityPrefixChars > 0 ? Math.floor(cacheAffinityPrefixChars) : 2048,
|
|
176
|
+
subagentPartitionsEnabled: subagentPartitionsEnabled ?? true,
|
|
177
|
+
subagentReservedKeysCount: typeof subagentReservedKeysCount === 'number' && Number.isFinite(subagentReservedKeysCount) && subagentReservedKeysCount >= 0 ? Math.floor(subagentReservedKeysCount) : 0,
|
|
178
|
+
subagentPartitionStrategy: subagentPartitionStrategy === 'isolated' ? 'isolated' : 'shared-burst',
|
|
179
|
+
subagentMaxConcurrencyPerKey: typeof subagentMaxConcurrencyPerKey === 'number' && Number.isFinite(subagentMaxConcurrencyPerKey) && subagentMaxConcurrencyPerKey > 0 ? Math.floor(subagentMaxConcurrencyPerKey) : 2,
|
|
166
180
|
preemptiveRotationEnabled: preemptiveRotationEnabled ?? true,
|
|
167
181
|
preemptiveSafetyMarginTokens: typeof preemptiveSafetyMarginTokens === 'number' && Number.isFinite(preemptiveSafetyMarginTokens) && preemptiveSafetyMarginTokens >= 0 ? Math.floor(preemptiveSafetyMarginTokens) : 4000,
|
|
168
182
|
preemptiveSafetyMarginRequests: typeof preemptiveSafetyMarginRequests === 'number' && Number.isFinite(preemptiveSafetyMarginRequests) && preemptiveSafetyMarginRequests >= 0 ? Math.floor(preemptiveSafetyMarginRequests) : 2,
|
|
@@ -219,6 +233,13 @@ export function buildPools({ cfg = {}, poolState = new Map(), defaultState = ini
|
|
|
219
233
|
preemptiveRotationEnabled: p.preemptiveRotationEnabled ?? cfg.preemptiveRotationEnabled ?? true,
|
|
220
234
|
preemptiveSafetyMarginTokens: p.preemptiveSafetyMarginTokens ?? cfg.preemptiveSafetyMarginTokens ?? 4000,
|
|
221
235
|
preemptiveSafetyMarginRequests: p.preemptiveSafetyMarginRequests ?? cfg.preemptiveSafetyMarginRequests ?? 2,
|
|
236
|
+
cacheAffinityEnabled: p.cacheAffinityEnabled ?? cfg.cacheAffinityEnabled ?? true,
|
|
237
|
+
cacheAffinityTtlMs: p.cacheAffinityTtlMs ?? cfg.cacheAffinityTtlMs ?? 300000,
|
|
238
|
+
cacheAffinityPrefixChars: p.cacheAffinityPrefixChars ?? cfg.cacheAffinityPrefixChars ?? 2048,
|
|
239
|
+
subagentPartitionsEnabled: p.subagentPartitionsEnabled ?? cfg.subagentPartitionsEnabled ?? true,
|
|
240
|
+
subagentReservedKeysCount: p.subagentReservedKeysCount ?? cfg.subagentReservedKeysCount ?? 0,
|
|
241
|
+
subagentPartitionStrategy: p.subagentPartitionStrategy ?? cfg.subagentPartitionStrategy ?? 'shared-burst',
|
|
242
|
+
subagentMaxConcurrencyPerKey: p.subagentMaxConcurrencyPerKey ?? cfg.subagentMaxConcurrencyPerKey ?? 2,
|
|
222
243
|
preemptiveRotationEnabled: p.preemptiveRotationEnabled ?? cfg.preemptiveRotationEnabled ?? true,
|
|
223
244
|
preemptiveSafetyMarginTokens: p.preemptiveSafetyMarginTokens ?? cfg.preemptiveSafetyMarginTokens ?? 4000,
|
|
224
245
|
preemptiveSafetyMarginRequests: p.preemptiveSafetyMarginRequests ?? cfg.preemptiveSafetyMarginRequests ?? 2,
|
|
@@ -260,6 +281,13 @@ export function buildPools({ cfg = {}, poolState = new Map(), defaultState = ini
|
|
|
260
281
|
preemptiveRotationEnabled: mp.preemptiveRotationEnabled ?? p.preemptiveRotationEnabled ?? cfg.preemptiveRotationEnabled ?? true,
|
|
261
282
|
preemptiveSafetyMarginTokens: mp.preemptiveSafetyMarginTokens ?? p.preemptiveSafetyMarginTokens ?? cfg.preemptiveSafetyMarginTokens ?? 4000,
|
|
262
283
|
preemptiveSafetyMarginRequests: mp.preemptiveSafetyMarginRequests ?? p.preemptiveSafetyMarginRequests ?? cfg.preemptiveSafetyMarginRequests ?? 2,
|
|
284
|
+
cacheAffinityEnabled: mp.cacheAffinityEnabled ?? p.cacheAffinityEnabled ?? cfg.cacheAffinityEnabled ?? true,
|
|
285
|
+
cacheAffinityTtlMs: mp.cacheAffinityTtlMs ?? p.cacheAffinityTtlMs ?? cfg.cacheAffinityTtlMs ?? 300000,
|
|
286
|
+
cacheAffinityPrefixChars: mp.cacheAffinityPrefixChars ?? p.cacheAffinityPrefixChars ?? cfg.cacheAffinityPrefixChars ?? 2048,
|
|
287
|
+
subagentPartitionsEnabled: mp.subagentPartitionsEnabled ?? p.subagentPartitionsEnabled ?? cfg.subagentPartitionsEnabled ?? true,
|
|
288
|
+
subagentReservedKeysCount: mp.subagentReservedKeysCount ?? p.subagentReservedKeysCount ?? cfg.subagentReservedKeysCount ?? 0,
|
|
289
|
+
subagentPartitionStrategy: mp.subagentPartitionStrategy ?? p.subagentPartitionStrategy ?? cfg.subagentPartitionStrategy ?? 'shared-burst',
|
|
290
|
+
subagentMaxConcurrencyPerKey: mp.subagentMaxConcurrencyPerKey ?? p.subagentMaxConcurrencyPerKey ?? cfg.subagentMaxConcurrencyPerKey ?? 2,
|
|
263
291
|
preemptiveRotationEnabled: mp.preemptiveRotationEnabled ?? p.preemptiveRotationEnabled ?? cfg.preemptiveRotationEnabled ?? true,
|
|
264
292
|
preemptiveSafetyMarginTokens: mp.preemptiveSafetyMarginTokens ?? p.preemptiveSafetyMarginTokens ?? cfg.preemptiveSafetyMarginTokens ?? 4000,
|
|
265
293
|
preemptiveSafetyMarginRequests: mp.preemptiveSafetyMarginRequests ?? p.preemptiveSafetyMarginRequests ?? cfg.preemptiveSafetyMarginRequests ?? 2,
|
package/lib/resolver.js
CHANGED
|
@@ -12,6 +12,8 @@ import { bucketAllow, bucketRetryMs, tpmAllow, tpmRetryMs } from './bucket.js';
|
|
|
12
12
|
import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, isKeyBroken, costForDay, costForWeek, budgetVerdict, getSessionSticky, setSessionSticky } from './pool.js';
|
|
13
13
|
import { decryptSecret } from './crypto-storage.js';
|
|
14
14
|
import { applyPreemptiveYield } from './preemptive-yield.js';
|
|
15
|
+
import { extractPromptPrefixKey, applyCacheAffinity } from './cache-affinity.js';
|
|
16
|
+
import { partitionPoolKeys } from './subagent-partitions.js';
|
|
15
17
|
import { resolveRefPool, isQuotaManagedRequest, providerOfBase, basePoolForBase } from './pool-index.js';
|
|
16
18
|
import {
|
|
17
19
|
isModelQuotaAvailable,
|
|
@@ -133,6 +135,13 @@ export function createResolver({
|
|
|
133
135
|
lastUsed: pool.state.lastUsed,
|
|
134
136
|
});
|
|
135
137
|
}
|
|
138
|
+
// Issue #573: Cache-Affinity Router
|
|
139
|
+
const cacheKey = extractPromptPrefixKey(resolveOpts, pool.cacheAffinityPrefixChars);
|
|
140
|
+
if (cacheKey) {
|
|
141
|
+
list = applyCacheAffinity(list, pool, { cacheKey, now: Date.now() });
|
|
142
|
+
}
|
|
143
|
+
// Issue #575: Subagent Concurrency Partitions
|
|
144
|
+
list = partitionPoolKeys(list, pool, { reqOptions: resolveOpts, concurrencyTracker, now: Date.now() });
|
|
136
145
|
// Issue #571: Pre-emptive Token & Request Yield before 429
|
|
137
146
|
list = applyPreemptiveYield(list, pool, { now: Date.now() });
|
|
138
147
|
// Fail closed: when this request is managed by a model pool that has local
|
package/lib/rotate.js
CHANGED
|
@@ -28,6 +28,8 @@ import { LOCAL_POOL_EXHAUSTED_CODE, resolveKeyDirectly } from './resolver.js';
|
|
|
28
28
|
import { recordStreamSuccessMetrics, preserveDispatchContext } from './stream-metrics.js';
|
|
29
29
|
import { withStreamWatchdog, StreamStallError } from './stream-watchdog.js';
|
|
30
30
|
import { applyPreemptiveYield } from './preemptive-yield.js';
|
|
31
|
+
import { extractPromptPrefixKey, applyCacheAffinity } from './cache-affinity.js';
|
|
32
|
+
import { partitionPoolKeys } from './subagent-partitions.js';
|
|
31
33
|
import { penalizeKey } from './rotate-penalty.js';
|
|
32
34
|
import { recordPromptCache } from './prompt-cache.js';
|
|
33
35
|
import { recordRequestOutcome, recordFailureCategory } from './ops-analytics.js';
|
|
@@ -109,7 +111,8 @@ export function createRotate(deps) {
|
|
|
109
111
|
const breaker = runtime0.circuitBreakerEnabled === false ? null : circuitBreaker;
|
|
110
112
|
let activePermit = null;
|
|
111
113
|
const sessionId = options?.sessionId ?? options?.session ?? options?.metadata?.sessionId ?? options?.chatId ?? options?.conversationId;
|
|
112
|
-
const
|
|
114
|
+
const cacheKey = extractPromptPrefixKey(options, pool.cacheAffinityPrefixChars);
|
|
115
|
+
const reqStore = { pool, pickedRef: undefined, concurrencyRef: undefined, startMs: now(), sessionId, model: options.model, cacheKey };
|
|
113
116
|
setRotateStartMs(reqStore.startMs);
|
|
114
117
|
const wallNow = Date.now();
|
|
115
118
|
const quotaManaged = hasModelQuotaConfig(pool);
|
|
@@ -204,6 +207,13 @@ export function createRotate(deps) {
|
|
|
204
207
|
}
|
|
205
208
|
}
|
|
206
209
|
|
|
210
|
+
// Issue #573: Cache-Affinity Router
|
|
211
|
+
if (reqStore.cacheKey) {
|
|
212
|
+
attemptList = applyCacheAffinity(attemptList, pool, { cacheKey: reqStore.cacheKey, now: wallNow, trace, pushEvent });
|
|
213
|
+
}
|
|
214
|
+
// Issue #575: Subagent Concurrency Partitions
|
|
215
|
+
attemptList = partitionPoolKeys(attemptList, pool, { reqOptions: options, concurrencyTracker, now: wallNow, trace });
|
|
216
|
+
|
|
207
217
|
// Issue #571: Pre-emptive Token & Request Yield before 429
|
|
208
218
|
attemptList = applyPreemptiveYield(attemptList, pool, {
|
|
209
219
|
now: wallNow,
|
|
@@ -448,6 +458,7 @@ export function createRotate(deps) {
|
|
|
448
458
|
rawHdrs,
|
|
449
459
|
pushEvent,
|
|
450
460
|
quotaStore,
|
|
461
|
+
cacheKey: reqStore?.cacheKey,
|
|
451
462
|
});
|
|
452
463
|
}
|
|
453
464
|
recordLatency(pool, reqStore);
|
package/lib/stream-metrics.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
import { recordSuccess, setSessionSticky, recordFailure, extractRateLimit, isRateLimited } from './pool.js';
|
|
5
5
|
import { recordPromptCache } from './prompt-cache.js';
|
|
6
|
+
import { getOrCreateCacheAffinityRouter } from './cache-affinity.js';
|
|
6
7
|
import { recordRequestOutcome } from './ops-analytics.js';
|
|
7
8
|
import { tpmRecord } from './bucket.js';
|
|
8
9
|
import { extractUsageTokens, consumeModelTokens } from './model-quota.js';
|
|
@@ -49,6 +50,7 @@ export function recordStreamSuccessMetrics(pool, activeRef, options, chunk, deps
|
|
|
49
50
|
rawHdrs,
|
|
50
51
|
pushEvent,
|
|
51
52
|
quotaStore,
|
|
53
|
+
cacheKey,
|
|
52
54
|
} = deps;
|
|
53
55
|
|
|
54
56
|
const nowMs = typeof now === 'function' ? now() : Date.now();
|
|
@@ -61,6 +63,10 @@ export function recordStreamSuccessMetrics(pool, activeRef, options, chunk, deps
|
|
|
61
63
|
// 2. Prompt cache and analytics outcome
|
|
62
64
|
const usagePayload = chunk?.usage ?? chunk?.metadata?.usage;
|
|
63
65
|
recordPromptCache(pool, activeRef, options.model, usagePayload, wallNow);
|
|
66
|
+
if (cacheKey) {
|
|
67
|
+
const affinityRouter = getOrCreateCacheAffinityRouter(pool);
|
|
68
|
+
affinityRouter.recordAffinity(cacheKey, activeRef, { now: wallNow, model: options.model });
|
|
69
|
+
}
|
|
64
70
|
recordRequestOutcome(pool, {
|
|
65
71
|
ref: activeRef,
|
|
66
72
|
success: true,
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// src/subagent-partitions.js — Subagent Concurrency Partitions & Isolation (#575).
|
|
2
|
+
// Pure module: zero runtime dependencies, strictly < 600 lines.
|
|
3
|
+
|
|
4
|
+
export const DEFAULT_SUBAGENT_MAX_CONCURRENCY_PER_KEY = 2;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Extracts subagent or background task context from request options and metadata.
|
|
8
|
+
* @param {object} options
|
|
9
|
+
* @returns {{ isSubagent: boolean, subagentId?: string, parentSessionId?: string }}
|
|
10
|
+
*/
|
|
11
|
+
export function extractSubagentContext(options) {
|
|
12
|
+
if (!options || typeof options !== 'object') return { isSubagent: false };
|
|
13
|
+
const meta = options.metadata || options.meta || {};
|
|
14
|
+
const parentSessionId = typeof meta.parentSessionId === 'string' && meta.parentSessionId.trim() ? meta.parentSessionId.trim() : undefined;
|
|
15
|
+
|
|
16
|
+
let isSubagent = false;
|
|
17
|
+
let subagentId = undefined;
|
|
18
|
+
|
|
19
|
+
if (options.isSubagent === true || meta.isSubagent === true) {
|
|
20
|
+
isSubagent = true;
|
|
21
|
+
subagentId = options.subagentId || meta.subagentId;
|
|
22
|
+
} else if (typeof options.subagentId === 'string' && options.subagentId.trim()) {
|
|
23
|
+
isSubagent = true;
|
|
24
|
+
subagentId = options.subagentId.trim();
|
|
25
|
+
} else if (typeof meta.subagentId === 'string' && meta.subagentId.trim()) {
|
|
26
|
+
isSubagent = true;
|
|
27
|
+
subagentId = meta.subagentId.trim();
|
|
28
|
+
} else if (typeof options.subagent === 'string' && options.subagent.trim()) {
|
|
29
|
+
isSubagent = true;
|
|
30
|
+
subagentId = options.subagent.trim();
|
|
31
|
+
} else if (options.subagent === true) {
|
|
32
|
+
isSubagent = true;
|
|
33
|
+
} else if (parentSessionId) {
|
|
34
|
+
isSubagent = true;
|
|
35
|
+
subagentId = meta.childSessionId;
|
|
36
|
+
} else {
|
|
37
|
+
// Check sessionId prefixes
|
|
38
|
+
const sid = String(options.sessionId || options.session || options.chatId || options.conversationId || '').trim();
|
|
39
|
+
if (
|
|
40
|
+
sid.startsWith('subagent:') ||
|
|
41
|
+
sid.startsWith('subagent-') ||
|
|
42
|
+
sid.startsWith('subtask:') ||
|
|
43
|
+
sid.startsWith('subtask-') ||
|
|
44
|
+
sid.startsWith('agent:sub:')
|
|
45
|
+
) {
|
|
46
|
+
isSubagent = true;
|
|
47
|
+
subagentId = sid;
|
|
48
|
+
} else {
|
|
49
|
+
// Check tags
|
|
50
|
+
const tags = Array.isArray(options.tags) ? options.tags : (Array.isArray(meta.tags) ? meta.tags : []);
|
|
51
|
+
if (tags.some(t => typeof t === 'string' && (t === 'subagent' || t === 'subtask' || t === 'background'))) {
|
|
52
|
+
isSubagent = true;
|
|
53
|
+
subagentId = options.subagentId || meta.subagentId;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (isSubagent) {
|
|
59
|
+
const res = { isSubagent: true };
|
|
60
|
+
if (subagentId) res.subagentId = subagentId;
|
|
61
|
+
if (parentSessionId) res.parentSessionId = parentSessionId;
|
|
62
|
+
return res;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return { isSubagent: false };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Partitions or reorders candidate keys according to subagent isolation rules.
|
|
70
|
+
* @param {string[]} attemptList
|
|
71
|
+
* @param {object} pool
|
|
72
|
+
* @param {object} [options]
|
|
73
|
+
* @returns {string[]}
|
|
74
|
+
*/
|
|
75
|
+
export function partitionPoolKeys(attemptList, pool, options = {}) {
|
|
76
|
+
if (!Array.isArray(attemptList) || attemptList.length <= 1 || !pool) {
|
|
77
|
+
return (attemptList ?? []).slice();
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const enabled = options.enabled ?? pool.subagentPartitionsEnabled ?? true;
|
|
81
|
+
if (!enabled) return attemptList.slice();
|
|
82
|
+
|
|
83
|
+
const subCtx = options.subagentContext ?? extractSubagentContext(options.reqOptions);
|
|
84
|
+
const isSubagent = Boolean(subCtx.isSubagent);
|
|
85
|
+
const reservedCount = pool.subagentReservedKeysCount ?? 0;
|
|
86
|
+
const strategy = pool.subagentPartitionStrategy ?? 'shared-burst';
|
|
87
|
+
const labels = pool.labels || {};
|
|
88
|
+
|
|
89
|
+
const explicitSubagentRefs = new Set();
|
|
90
|
+
const explicitPrimaryRefs = new Set();
|
|
91
|
+
|
|
92
|
+
for (const [ref, label] of Object.entries(labels)) {
|
|
93
|
+
const lbl = String(label).toLowerCase();
|
|
94
|
+
if (lbl === 'subagent' || lbl === 'subtask' || lbl === 'worker') explicitSubagentRefs.add(ref);
|
|
95
|
+
if (lbl === 'primary' || lbl === 'main' || lbl === 'user') explicitPrimaryRefs.add(ref);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// If reservedCount > 0, reserve the last reservedCount keys of pool.refs if not already labelled primary
|
|
99
|
+
if (reservedCount > 0 && pool.refs && pool.refs.length > reservedCount) {
|
|
100
|
+
const reservedSlice = pool.refs.slice(pool.refs.length - reservedCount);
|
|
101
|
+
for (const r of reservedSlice) {
|
|
102
|
+
if (!explicitPrimaryRefs.has(r)) explicitSubagentRefs.add(r);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const hasPartitioning = explicitSubagentRefs.size > 0;
|
|
107
|
+
let list = attemptList.slice();
|
|
108
|
+
|
|
109
|
+
if (isSubagent) {
|
|
110
|
+
// SUBAGENT REQUEST
|
|
111
|
+
if (hasPartitioning) {
|
|
112
|
+
const dedicated = list.filter(r => explicitSubagentRefs.has(r));
|
|
113
|
+
const shared = list.filter(r => !explicitSubagentRefs.has(r));
|
|
114
|
+
|
|
115
|
+
if (strategy === 'isolated') {
|
|
116
|
+
if (dedicated.length > 0) list = dedicated;
|
|
117
|
+
} else {
|
|
118
|
+
list = dedicated.concat(shared);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
} else {
|
|
122
|
+
// PRIMARY / USER REQUEST
|
|
123
|
+
if (hasPartitioning) {
|
|
124
|
+
const primary = list.filter(r => !explicitSubagentRefs.has(r));
|
|
125
|
+
const subOnly = list.filter(r => explicitSubagentRefs.has(r));
|
|
126
|
+
|
|
127
|
+
if (primary.length > 0) {
|
|
128
|
+
list = strategy === 'isolated' ? primary : primary.concat(subOnly);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Apply subagent per-key concurrency cap
|
|
134
|
+
const tracker = options.concurrencyTracker;
|
|
135
|
+
const maxSubConcurrency = pool.subagentMaxConcurrencyPerKey ?? DEFAULT_SUBAGENT_MAX_CONCURRENCY_PER_KEY;
|
|
136
|
+
if (isSubagent && tracker && typeof tracker.getActive === 'function' && maxSubConcurrency > 0) {
|
|
137
|
+
const now = options.now ?? Date.now();
|
|
138
|
+
const withinCap = [];
|
|
139
|
+
const saturated = [];
|
|
140
|
+
for (const r of list) {
|
|
141
|
+
const active = tracker.getActive(r, now) ?? 0;
|
|
142
|
+
if (active >= maxSubConcurrency) {
|
|
143
|
+
saturated.push(r);
|
|
144
|
+
} else {
|
|
145
|
+
withinCap.push(r);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
if (withinCap.length > 0 && saturated.length > 0) {
|
|
149
|
+
list = withinCap.concat(saturated);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (options.trace && typeof options.trace.addStep === 'function' && list[0] !== attemptList[0]) {
|
|
154
|
+
options.trace.addStep(
|
|
155
|
+
list[0],
|
|
156
|
+
'PRIORITIZED',
|
|
157
|
+
'SUBAGENT_PARTITION',
|
|
158
|
+
isSubagent ? 'Subagent partition key allocated' : 'Primary chat partition prioritized'
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return list;
|
|
163
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.88",
|
|
4
4
|
"packageManager": "pnpm@10.33.2",
|
|
5
5
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
6
6
|
"keywords": [
|