@danypops/vehicle-core 0.18.5 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approvals/approval.d.ts +14 -0
- package/dist/approvals/approval.js +14 -0
- package/dist/resource-pool/bounded-resource-pool.d.ts +19 -12
- package/dist/resource-pool/bounded-resource-pool.js +106 -52
- package/dist/resource-pool/errors.d.ts +6 -4
- package/dist/resource-pool/errors.js +10 -6
- package/package.json +1 -1
- package/src/approvals/approval.ts +15 -0
- package/src/resource-pool/bounded-resource-pool.ts +119 -51
- package/src/resource-pool/errors.ts +6 -6
|
@@ -13,6 +13,20 @@ export declare const DEFAULT_APPROVAL_EFFECTS: readonly VehicleEffect[];
|
|
|
13
13
|
* loop point of the gate entirely.
|
|
14
14
|
*/
|
|
15
15
|
export declare const VEHICLE_APPROVAL_RESOLVE_OPERATION_NAME = "vehicle.approval.resolve";
|
|
16
|
+
/**
|
|
17
|
+
* The name VehicleRegistry.configureApprovals() registers its own read-only status-lookup
|
|
18
|
+
* operation under. Unlike vehicle.approval.resolve, this one is deliberately left projectable
|
|
19
|
+
* as an ordinary Pi/model-callable tool: reading a requestId's own already-made decision (who
|
|
20
|
+
* decided, when, and any comment) can never let a caller grant its own pending request -- it
|
|
21
|
+
* only ever surfaces a decision a human (or other authority) already recorded. Exists because a
|
|
22
|
+
* gated invoke()'s own local HITL prompt and the eventual human decision can genuinely happen on
|
|
23
|
+
* two different calls (a caller-side timeout, a UI presented asynchronously outside the
|
|
24
|
+
* originating call's own lifetime, a session restart in between) -- without this, a decision
|
|
25
|
+
* (and any comment explaining it) made after the originating call already gave up is
|
|
26
|
+
* unrecoverable, since vehicle.approval.resolve's own vehicleApprovalResolvedEvent is transient
|
|
27
|
+
* pub/sub with no replay for a caller that wasn't subscribed at that exact moment.
|
|
28
|
+
*/
|
|
29
|
+
export declare const VEHICLE_APPROVAL_STATUS_OPERATION_NAME = "vehicle.approval.status";
|
|
16
30
|
/** How long a request stays resolvable before it lapses and must be re-requested. */
|
|
17
31
|
export declare const DEFAULT_APPROVAL_TIMEOUT_MS: number;
|
|
18
32
|
/** Emitted (as a Vehicle Event) the moment a gated-effect invoke() has no valid capability -- durable-first, before any interactive prompt is attempted. */
|
|
@@ -37,6 +37,20 @@ export const DEFAULT_APPROVAL_EFFECTS = ["destructive", "open-world"];
|
|
|
37
37
|
* loop point of the gate entirely.
|
|
38
38
|
*/
|
|
39
39
|
export const VEHICLE_APPROVAL_RESOLVE_OPERATION_NAME = "vehicle.approval.resolve";
|
|
40
|
+
/**
|
|
41
|
+
* The name VehicleRegistry.configureApprovals() registers its own read-only status-lookup
|
|
42
|
+
* operation under. Unlike vehicle.approval.resolve, this one is deliberately left projectable
|
|
43
|
+
* as an ordinary Pi/model-callable tool: reading a requestId's own already-made decision (who
|
|
44
|
+
* decided, when, and any comment) can never let a caller grant its own pending request -- it
|
|
45
|
+
* only ever surfaces a decision a human (or other authority) already recorded. Exists because a
|
|
46
|
+
* gated invoke()'s own local HITL prompt and the eventual human decision can genuinely happen on
|
|
47
|
+
* two different calls (a caller-side timeout, a UI presented asynchronously outside the
|
|
48
|
+
* originating call's own lifetime, a session restart in between) -- without this, a decision
|
|
49
|
+
* (and any comment explaining it) made after the originating call already gave up is
|
|
50
|
+
* unrecoverable, since vehicle.approval.resolve's own vehicleApprovalResolvedEvent is transient
|
|
51
|
+
* pub/sub with no replay for a caller that wasn't subscribed at that exact moment.
|
|
52
|
+
*/
|
|
53
|
+
export const VEHICLE_APPROVAL_STATUS_OPERATION_NAME = "vehicle.approval.status";
|
|
40
54
|
/** How long a request stays resolvable before it lapses and must be re-requested. */
|
|
41
55
|
export const DEFAULT_APPROVAL_TIMEOUT_MS = 5 * 60_000;
|
|
42
56
|
const requestedPayloadSchema = defineVehicleSchema({
|
|
@@ -9,9 +9,8 @@ export interface PooledResource {
|
|
|
9
9
|
}
|
|
10
10
|
/**
|
|
11
11
|
* Distinguishes an interactive human/agent-facing request from a self-scheduled background one.
|
|
12
|
-
* Foreground
|
|
13
|
-
*
|
|
14
|
-
* that never opts in gets today's exact unreserved behavior.
|
|
12
|
+
* Foreground keeps the full ceiling and remains fail-fast unless its explicit queue timeout is
|
|
13
|
+
* enabled; background can be held below the reserved-slot ceiling. Defaults to "foreground".
|
|
15
14
|
*/
|
|
16
15
|
export type ResourceWorkKind = "foreground" | "background";
|
|
17
16
|
export type ResourcePoolEvent = {
|
|
@@ -33,7 +32,9 @@ export interface ResourcePoolStatus<Status = unknown> {
|
|
|
33
32
|
readonly absoluteMaxActive: number;
|
|
34
33
|
readonly byPartition: Readonly<Record<string, number>>;
|
|
35
34
|
readonly resources?: Status;
|
|
36
|
-
/** How many
|
|
35
|
+
/** How many opt-in foreground admissions are waiting for a lease to free. */
|
|
36
|
+
readonly waitingForegroundAdmissions: number;
|
|
37
|
+
/** How many background admissions are waiting for room below their effective ceiling. */
|
|
37
38
|
readonly waitingBackgroundAdmissions: number;
|
|
38
39
|
}
|
|
39
40
|
export interface PoolLease<Value> extends AsyncDisposable {
|
|
@@ -52,6 +53,10 @@ export interface ResourcePoolOptions<Status = unknown> {
|
|
|
52
53
|
readonly backgroundAdmissionQueueTimeoutMs?: number;
|
|
53
54
|
/** How many background admissions may be simultaneously waiting before a new one fails fast with ResourceAdmissionQueueFull instead of growing the wait queue further. Default 8. */
|
|
54
55
|
readonly maxQueuedBackgroundAdmissions?: number;
|
|
56
|
+
/** Opt-in foreground wait bound. Zero (default) preserves fail-fast ResourceCapacityExceeded behavior. */
|
|
57
|
+
readonly foregroundAdmissionQueueTimeoutMs?: number;
|
|
58
|
+
/** How many opted-in foreground admissions may wait simultaneously. Default 8. */
|
|
59
|
+
readonly maxQueuedForegroundAdmissions?: number;
|
|
55
60
|
/** Fed one (partitionKey, costHandle) pair per active entry with a costHandle on calibrateCosts(). */
|
|
56
61
|
readonly costRecorder?: {
|
|
57
62
|
recordSample(partitionKey: string, costHandle: unknown): void;
|
|
@@ -80,7 +85,11 @@ export declare class BoundedResourcePool<OwnerKey extends string, Resource exten
|
|
|
80
85
|
private readonly reservedForegroundSlots;
|
|
81
86
|
private readonly backgroundAdmissionQueueTimeoutMs;
|
|
82
87
|
private readonly maxQueuedBackgroundAdmissions;
|
|
88
|
+
private readonly foregroundAdmissionQueueTimeoutMs;
|
|
89
|
+
private readonly maxQueuedForegroundAdmissions;
|
|
83
90
|
private readonly admissionWaiters;
|
|
91
|
+
private admissionGeneration;
|
|
92
|
+
private queuedForegroundAdmissions;
|
|
84
93
|
private queuedBackgroundAdmissions;
|
|
85
94
|
private readonly waitingCounts;
|
|
86
95
|
constructor(options?: ResourcePoolOptions<Status>);
|
|
@@ -90,16 +99,14 @@ export declare class BoundedResourcePool<OwnerKey extends string, Resource exten
|
|
|
90
99
|
private activePartitions;
|
|
91
100
|
private leastRecentlyUsedIdle;
|
|
92
101
|
private evict;
|
|
93
|
-
/** Wakes
|
|
102
|
+
/** Wakes one waiter per newly available opportunity, preferring foreground and preserving FIFO within each class. */
|
|
94
103
|
private notifyAdmissionWaiters;
|
|
95
|
-
/** True while at least one background admission for this owner is
|
|
104
|
+
/** True while at least one background admission for this owner is waiting. */
|
|
96
105
|
waitingForAdmission(ownerKey: OwnerKey): boolean;
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
* included, which is the exact starvation this exists to prevent.
|
|
102
|
-
*/
|
|
106
|
+
private waitTimeoutMs;
|
|
107
|
+
private canWait;
|
|
108
|
+
private capacityUnavailable;
|
|
109
|
+
/** Waits outside the serialized admission lock, with one total deadline across false wakes. */
|
|
103
110
|
private waitForAdmissionRoom;
|
|
104
111
|
private admit;
|
|
105
112
|
private serialized;
|
|
@@ -5,14 +5,22 @@ const DEFAULT_MAX_ACTIVE = 3;
|
|
|
5
5
|
const DEFAULT_ABSOLUTE_MAX_ACTIVE = 32;
|
|
6
6
|
const DEFAULT_BACKGROUND_ADMISSION_QUEUE_TIMEOUT_MS = 10_000;
|
|
7
7
|
const DEFAULT_MAX_QUEUED_BACKGROUND_ADMISSIONS = 8;
|
|
8
|
+
const DEFAULT_FOREGROUND_ADMISSION_QUEUE_TIMEOUT_MS = 0;
|
|
9
|
+
const DEFAULT_MAX_QUEUED_FOREGROUND_ADMISSIONS = 8;
|
|
8
10
|
/**
|
|
9
11
|
* Internal signal only: admit() throws this to tell acquire() "release the serialized lock and
|
|
10
|
-
* wait outside it" -- never surfaced to a caller.
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* the starvation this exists to prevent.
|
|
12
|
+
* wait outside it" -- never surfaced to a caller. A bounded admission wait can take seconds;
|
|
13
|
+
* holding admissionTail (the single global admission mutex) for that span would block every other
|
|
14
|
+
* request and create the starvation this mechanism exists to prevent.
|
|
14
15
|
*/
|
|
15
|
-
class
|
|
16
|
+
class NeedsAdmissionWait extends Error {
|
|
17
|
+
workKind;
|
|
18
|
+
observedGeneration;
|
|
19
|
+
constructor(workKind, observedGeneration) {
|
|
20
|
+
super();
|
|
21
|
+
this.workKind = workKind;
|
|
22
|
+
this.observedGeneration = observedGeneration;
|
|
23
|
+
}
|
|
16
24
|
}
|
|
17
25
|
/**
|
|
18
26
|
* Owns the bounded lifecycle of pooled, expensive, stateful resources partitioned by an
|
|
@@ -36,7 +44,11 @@ export class BoundedResourcePool {
|
|
|
36
44
|
reservedForegroundSlots;
|
|
37
45
|
backgroundAdmissionQueueTimeoutMs;
|
|
38
46
|
maxQueuedBackgroundAdmissions;
|
|
39
|
-
|
|
47
|
+
foregroundAdmissionQueueTimeoutMs;
|
|
48
|
+
maxQueuedForegroundAdmissions;
|
|
49
|
+
admissionWaiters = [];
|
|
50
|
+
admissionGeneration = 0;
|
|
51
|
+
queuedForegroundAdmissions = 0;
|
|
40
52
|
queuedBackgroundAdmissions = 0;
|
|
41
53
|
waitingCounts = new Map();
|
|
42
54
|
constructor(options = {}) {
|
|
@@ -67,6 +79,14 @@ export class BoundedResourcePool {
|
|
|
67
79
|
if (!Number.isSafeInteger(this.maxQueuedBackgroundAdmissions) || this.maxQueuedBackgroundAdmissions < 1) {
|
|
68
80
|
throw new TypeError("maxQueuedBackgroundAdmissions must be a positive safe integer");
|
|
69
81
|
}
|
|
82
|
+
this.foregroundAdmissionQueueTimeoutMs = options.foregroundAdmissionQueueTimeoutMs ?? DEFAULT_FOREGROUND_ADMISSION_QUEUE_TIMEOUT_MS;
|
|
83
|
+
if (!Number.isSafeInteger(this.foregroundAdmissionQueueTimeoutMs) || this.foregroundAdmissionQueueTimeoutMs < 0) {
|
|
84
|
+
throw new TypeError("foregroundAdmissionQueueTimeoutMs must be a non-negative safe integer");
|
|
85
|
+
}
|
|
86
|
+
this.maxQueuedForegroundAdmissions = options.maxQueuedForegroundAdmissions ?? DEFAULT_MAX_QUEUED_FOREGROUND_ADMISSIONS;
|
|
87
|
+
if (!Number.isSafeInteger(this.maxQueuedForegroundAdmissions) || this.maxQueuedForegroundAdmissions < 1) {
|
|
88
|
+
throw new TypeError("maxQueuedForegroundAdmissions must be a positive safe integer");
|
|
89
|
+
}
|
|
70
90
|
}
|
|
71
91
|
key(ownerKey, partitionKey) {
|
|
72
92
|
return `${ownerKey}:${partitionKey}`;
|
|
@@ -116,73 +136,103 @@ export class BoundedResourcePool {
|
|
|
116
136
|
this.options.observe?.({ kind, partitionKey: entry[1].partitionKey });
|
|
117
137
|
this.notifyAdmissionWaiters();
|
|
118
138
|
}
|
|
119
|
-
/** Wakes
|
|
139
|
+
/** Wakes one waiter per newly available opportunity, preferring foreground and preserving FIFO within each class. */
|
|
120
140
|
notifyAdmissionWaiters() {
|
|
121
|
-
|
|
141
|
+
this.admissionGeneration++;
|
|
142
|
+
if (this.admissionWaiters.length === 0)
|
|
122
143
|
return;
|
|
123
|
-
const
|
|
124
|
-
this.admissionWaiters.
|
|
125
|
-
|
|
126
|
-
waiter();
|
|
144
|
+
const foregroundIndex = this.admissionWaiters.findIndex((waiter) => waiter.workKind === "foreground");
|
|
145
|
+
const [waiter] = this.admissionWaiters.splice(foregroundIndex === -1 ? 0 : foregroundIndex, 1);
|
|
146
|
+
waiter?.wake();
|
|
127
147
|
}
|
|
128
|
-
/** True while at least one background admission for this owner is
|
|
148
|
+
/** True while at least one background admission for this owner is waiting. */
|
|
129
149
|
waitingForAdmission(ownerKey) {
|
|
130
150
|
return (this.waitingCounts.get(ownerKey) ?? 0) > 0;
|
|
131
151
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
if (this.
|
|
140
|
-
throw new
|
|
152
|
+
waitTimeoutMs(workKind) {
|
|
153
|
+
return workKind === "foreground" ? this.foregroundAdmissionQueueTimeoutMs : this.backgroundAdmissionQueueTimeoutMs;
|
|
154
|
+
}
|
|
155
|
+
canWait(workKind) {
|
|
156
|
+
return workKind === "background" || this.foregroundAdmissionQueueTimeoutMs > 0;
|
|
157
|
+
}
|
|
158
|
+
capacityUnavailable(workKind, partitionKey, maxActive, partitionLimit) {
|
|
159
|
+
if (this.canWait(workKind))
|
|
160
|
+
throw new NeedsAdmissionWait(workKind, this.admissionGeneration);
|
|
161
|
+
throw new ResourceCapacityExceeded(partitionKey, maxActive, partitionLimit);
|
|
162
|
+
}
|
|
163
|
+
/** Waits outside the serialized admission lock, with one total deadline across false wakes. */
|
|
164
|
+
async waitForAdmissionRoom(partitionKey, ownerKey, workKind, deadline, observedGeneration) {
|
|
165
|
+
// A lease may complete after admit() releases the mutex but before this waiter is registered.
|
|
166
|
+
// Retry immediately when that happened rather than sleeping through a lost wake-up.
|
|
167
|
+
if (this.admissionGeneration !== observedGeneration)
|
|
168
|
+
return;
|
|
169
|
+
const maxQueued = workKind === "foreground" ? this.maxQueuedForegroundAdmissions : this.maxQueuedBackgroundAdmissions;
|
|
170
|
+
const queued = workKind === "foreground" ? this.queuedForegroundAdmissions : this.queuedBackgroundAdmissions;
|
|
171
|
+
if (queued >= maxQueued)
|
|
172
|
+
throw new ResourceAdmissionQueueFull(partitionKey, maxQueued, workKind);
|
|
173
|
+
const timeoutMs = this.waitTimeoutMs(workKind);
|
|
174
|
+
const remainingMs = deadline - Date.now();
|
|
175
|
+
if (remainingMs <= 0)
|
|
176
|
+
throw new ResourceAdmissionQueueTimedOut(partitionKey, timeoutMs, workKind);
|
|
177
|
+
if (workKind === "foreground")
|
|
178
|
+
this.queuedForegroundAdmissions++;
|
|
179
|
+
else {
|
|
180
|
+
this.queuedBackgroundAdmissions++;
|
|
181
|
+
this.waitingCounts.set(ownerKey, (this.waitingCounts.get(ownerKey) ?? 0) + 1);
|
|
141
182
|
}
|
|
142
|
-
this.queuedBackgroundAdmissions++;
|
|
143
|
-
this.waitingCounts.set(ownerKey, (this.waitingCounts.get(ownerKey) ?? 0) + 1);
|
|
144
183
|
try {
|
|
145
184
|
const gotSignal = await new Promise((resolve) => {
|
|
146
185
|
let settled = false;
|
|
186
|
+
let timer;
|
|
187
|
+
const waiter = {
|
|
188
|
+
workKind,
|
|
189
|
+
wake: () => finish(true),
|
|
190
|
+
};
|
|
147
191
|
const finish = (ready) => {
|
|
148
192
|
if (settled)
|
|
149
193
|
return;
|
|
150
194
|
settled = true;
|
|
151
195
|
clearTimeout(timer);
|
|
152
|
-
this.admissionWaiters.
|
|
196
|
+
const index = this.admissionWaiters.indexOf(waiter);
|
|
197
|
+
if (index !== -1)
|
|
198
|
+
this.admissionWaiters.splice(index, 1);
|
|
153
199
|
resolve(ready);
|
|
154
200
|
};
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
this.admissionWaiters.add(onSignal);
|
|
201
|
+
timer = setTimeout(() => finish(false), remainingMs);
|
|
202
|
+
this.admissionWaiters.push(waiter);
|
|
158
203
|
});
|
|
159
204
|
if (!gotSignal)
|
|
160
|
-
throw new ResourceAdmissionQueueTimedOut(partitionKey,
|
|
205
|
+
throw new ResourceAdmissionQueueTimedOut(partitionKey, timeoutMs, workKind);
|
|
161
206
|
}
|
|
162
207
|
finally {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
this.
|
|
167
|
-
|
|
168
|
-
|
|
208
|
+
if (workKind === "foreground")
|
|
209
|
+
this.queuedForegroundAdmissions--;
|
|
210
|
+
else {
|
|
211
|
+
this.queuedBackgroundAdmissions--;
|
|
212
|
+
const remaining = (this.waitingCounts.get(ownerKey) ?? 1) - 1;
|
|
213
|
+
if (remaining <= 0)
|
|
214
|
+
this.waitingCounts.delete(ownerKey);
|
|
215
|
+
else
|
|
216
|
+
this.waitingCounts.set(ownerKey, remaining);
|
|
217
|
+
}
|
|
169
218
|
}
|
|
170
219
|
}
|
|
171
|
-
async admit(ownerKey, partitionKey, create, workKind) {
|
|
220
|
+
async admit(ownerKey, partitionKey, create, workKind, admissionGranted) {
|
|
172
221
|
const partitionLimit = this.partitionLimit(partitionKey);
|
|
222
|
+
if (this.queuedForegroundAdmissions > 0 && !(workKind === "foreground" && admissionGranted)) {
|
|
223
|
+
throw new NeedsAdmissionWait(workKind, this.admissionGeneration);
|
|
224
|
+
}
|
|
173
225
|
while (this.countPartition(partitionKey) >= partitionLimit) {
|
|
174
226
|
const victim = this.leastRecentlyUsedIdle(partitionKey);
|
|
175
227
|
if (!victim)
|
|
176
|
-
|
|
228
|
+
this.capacityUnavailable(workKind, partitionKey, this.maxActive, partitionLimit);
|
|
177
229
|
await this.evict(victim);
|
|
178
230
|
}
|
|
179
231
|
const { ceiling: baseCeiling, source: ceilingSource } = this.baseActiveCeiling();
|
|
180
232
|
this.lastEffectiveMaxActive = baseCeiling;
|
|
181
233
|
this.lastActiveCeilingSource = ceilingSource;
|
|
182
|
-
// "Borrowable": background's own effective ceiling is reduced,
|
|
183
|
-
//
|
|
184
|
-
// set-aside nothing else can reach. Foreground keeps using the full (possibly resource-
|
|
185
|
-
// budget-raised) baseCeiling unchanged.
|
|
234
|
+
// "Borrowable": background's own effective ceiling is reduced, while foreground keeps the
|
|
235
|
+
// full (possibly resource-budget-raised) base ceiling unchanged.
|
|
186
236
|
const effectiveMaxActive = workKind === "background" ? Math.max(baseCeiling - this.reservedForegroundSlots, 0) : baseCeiling;
|
|
187
237
|
while (this.entries.size >= effectiveMaxActive) {
|
|
188
238
|
const victim = this.leastRecentlyUsedIdle();
|
|
@@ -190,14 +240,12 @@ export class BoundedResourcePool {
|
|
|
190
240
|
await this.evict(victim);
|
|
191
241
|
continue;
|
|
192
242
|
}
|
|
193
|
-
|
|
194
|
-
throw new NeedsBackgroundAdmissionWait();
|
|
195
|
-
throw new ResourceCapacityExceeded(partitionKey, baseCeiling, partitionLimit);
|
|
243
|
+
this.capacityUnavailable(workKind, partitionKey, baseCeiling, partitionLimit);
|
|
196
244
|
}
|
|
197
245
|
while (this.options.resourcePolicy && !this.options.resourcePolicy.canAdmit(this.activePartitions(), partitionKey)) {
|
|
198
246
|
const victim = this.leastRecentlyUsedIdle();
|
|
199
247
|
if (!victim)
|
|
200
|
-
|
|
248
|
+
this.capacityUnavailable(workKind, partitionKey, baseCeiling, partitionLimit);
|
|
201
249
|
await this.evict(victim, "resource-pressure");
|
|
202
250
|
}
|
|
203
251
|
return {
|
|
@@ -225,6 +273,8 @@ export class BoundedResourcePool {
|
|
|
225
273
|
}
|
|
226
274
|
/** Acquires a lease for (ownerKey, partitionKey), reusing an already-admitted resource if one is warm, or admitting a fresh one via `create()` -- called only on an actual cache miss, never speculatively. workKind defaults to "foreground". */
|
|
227
275
|
async acquire(ownerKey, partitionKey, create, workKind = "foreground") {
|
|
276
|
+
const deadline = Date.now() + this.waitTimeoutMs(workKind);
|
|
277
|
+
let admissionGranted = false;
|
|
228
278
|
for (;;) {
|
|
229
279
|
try {
|
|
230
280
|
const entry = await this.serialized(async () => {
|
|
@@ -238,7 +288,7 @@ export class BoundedResourcePool {
|
|
|
238
288
|
entry = undefined;
|
|
239
289
|
}
|
|
240
290
|
if (!entry) {
|
|
241
|
-
entry = await this.admit(ownerKey, partitionKey, create, workKind);
|
|
291
|
+
entry = await this.admit(ownerKey, partitionKey, create, workKind, admissionGranted);
|
|
242
292
|
this.entries.set(key, entry);
|
|
243
293
|
}
|
|
244
294
|
entry.activeLeases++;
|
|
@@ -247,11 +297,10 @@ export class BoundedResourcePool {
|
|
|
247
297
|
return this.lease(entry.resource, [entry]);
|
|
248
298
|
}
|
|
249
299
|
catch (error) {
|
|
250
|
-
if (!(error instanceof
|
|
300
|
+
if (!(error instanceof NeedsAdmissionWait))
|
|
251
301
|
throw error;
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
await this.waitForAdmissionRoom(partitionKey, ownerKey);
|
|
302
|
+
await this.waitForAdmissionRoom(partitionKey, ownerKey, error.workKind, deadline, error.observedGeneration);
|
|
303
|
+
admissionGranted = true;
|
|
255
304
|
}
|
|
256
305
|
}
|
|
257
306
|
}
|
|
@@ -270,8 +319,7 @@ export class BoundedResourcePool {
|
|
|
270
319
|
entry.recencySequence = this.nextSequence++;
|
|
271
320
|
}
|
|
272
321
|
// A lease completing makes its entry newly idle -- exactly the condition a queued
|
|
273
|
-
//
|
|
274
|
-
// itself ends up evicting anything below.
|
|
322
|
+
// admission retries against.
|
|
275
323
|
this.notifyAdmissionWaiters();
|
|
276
324
|
await this.reconcileResources();
|
|
277
325
|
},
|
|
@@ -324,6 +372,7 @@ export class BoundedResourcePool {
|
|
|
324
372
|
activeCeilingSource: this.lastActiveCeilingSource,
|
|
325
373
|
absoluteMaxActive: this.absoluteMaxActive,
|
|
326
374
|
byPartition,
|
|
375
|
+
waitingForegroundAdmissions: this.queuedForegroundAdmissions,
|
|
327
376
|
waitingBackgroundAdmissions: this.queuedBackgroundAdmissions,
|
|
328
377
|
...(resources !== undefined ? { resources } : {}),
|
|
329
378
|
};
|
|
@@ -344,6 +393,8 @@ export class BoundedResourcePool {
|
|
|
344
393
|
const stale = Array.from(this.entries.entries()).filter(([, entry]) => entry.ownerKey === ownerKey);
|
|
345
394
|
for (const [key] of stale)
|
|
346
395
|
this.entries.delete(key);
|
|
396
|
+
for (let i = 0; i < stale.length; i++)
|
|
397
|
+
this.notifyAdmissionWaiters();
|
|
347
398
|
await Promise.all(stale.map(([, entry]) => entry.resource.close()));
|
|
348
399
|
}
|
|
349
400
|
/** Unconditional force-close of one (ownerKey, partitionKey) resource, if any -- the single-partition sibling of closeOwner, for a caller that has already identified exactly which partition needs invalidating. */
|
|
@@ -353,6 +404,7 @@ export class BoundedResourcePool {
|
|
|
353
404
|
if (!entry)
|
|
354
405
|
return;
|
|
355
406
|
this.entries.delete(key);
|
|
407
|
+
this.notifyAdmissionWaiters();
|
|
356
408
|
await entry.resource.close();
|
|
357
409
|
}
|
|
358
410
|
/**
|
|
@@ -373,6 +425,8 @@ export class BoundedResourcePool {
|
|
|
373
425
|
async closeAll() {
|
|
374
426
|
const entries = Array.from(this.entries.values());
|
|
375
427
|
this.entries.clear();
|
|
428
|
+
for (let i = 0; i < entries.length; i++)
|
|
429
|
+
this.notifyAdmissionWaiters();
|
|
376
430
|
await Promise.all(entries.map((entry) => entry.resource.close()));
|
|
377
431
|
}
|
|
378
432
|
async reconcileResourcesUnsafe() {
|
|
@@ -10,15 +10,17 @@ export declare class ResourceInUse extends Error {
|
|
|
10
10
|
readonly ownerKey: string;
|
|
11
11
|
constructor(ownerKey: string);
|
|
12
12
|
}
|
|
13
|
-
/** Raised when
|
|
13
|
+
/** Raised when one admission class is already at its configured queue bound. */
|
|
14
14
|
export declare class ResourceAdmissionQueueFull extends Error {
|
|
15
15
|
readonly partitionKey: string;
|
|
16
16
|
readonly maxQueued: number;
|
|
17
|
-
|
|
17
|
+
readonly workKind: "foreground" | "background";
|
|
18
|
+
constructor(partitionKey: string, maxQueued: number, workKind?: "foreground" | "background");
|
|
18
19
|
}
|
|
19
|
-
/** Raised when a queued
|
|
20
|
+
/** Raised when a queued admission waits past its configured timeout without room becoming available. */
|
|
20
21
|
export declare class ResourceAdmissionQueueTimedOut extends Error {
|
|
21
22
|
readonly partitionKey: string;
|
|
22
23
|
readonly timeoutMs: number;
|
|
23
|
-
|
|
24
|
+
readonly workKind: "foreground" | "background";
|
|
25
|
+
constructor(partitionKey: string, timeoutMs: number, workKind?: "foreground" | "background");
|
|
24
26
|
}
|
|
@@ -20,25 +20,29 @@ export class ResourceInUse extends Error {
|
|
|
20
20
|
this.name = "ResourceInUse";
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
/** Raised when
|
|
23
|
+
/** Raised when one admission class is already at its configured queue bound. */
|
|
24
24
|
export class ResourceAdmissionQueueFull extends Error {
|
|
25
25
|
partitionKey;
|
|
26
26
|
maxQueued;
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
workKind;
|
|
28
|
+
constructor(partitionKey, maxQueued, workKind = "background") {
|
|
29
|
+
super(`${workKind} admission for partition "${partitionKey}" is already waiting at capacity (${maxQueued} queued); retry later`);
|
|
29
30
|
this.partitionKey = partitionKey;
|
|
30
31
|
this.maxQueued = maxQueued;
|
|
32
|
+
this.workKind = workKind;
|
|
31
33
|
this.name = "ResourceAdmissionQueueFull";
|
|
32
34
|
}
|
|
33
35
|
}
|
|
34
|
-
/** Raised when a queued
|
|
36
|
+
/** Raised when a queued admission waits past its configured timeout without room becoming available. */
|
|
35
37
|
export class ResourceAdmissionQueueTimedOut extends Error {
|
|
36
38
|
partitionKey;
|
|
37
39
|
timeoutMs;
|
|
38
|
-
|
|
39
|
-
|
|
40
|
+
workKind;
|
|
41
|
+
constructor(partitionKey, timeoutMs, workKind = "background") {
|
|
42
|
+
super(`${workKind} admission for partition "${partitionKey}" waited ${timeoutMs}ms for a resource-pool slot and gave up`);
|
|
40
43
|
this.partitionKey = partitionKey;
|
|
41
44
|
this.timeoutMs = timeoutMs;
|
|
45
|
+
this.workKind = workKind;
|
|
42
46
|
this.name = "ResourceAdmissionQueueTimedOut";
|
|
43
47
|
}
|
|
44
48
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danypops/vehicle-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Vehicle's runtime-neutral wire contract: operation descriptors, schema codecs, failure shapes. Zero runtime dependencies, zero Bun-specific code -- the one thing every Vehicle client and server package depends on.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -42,6 +42,21 @@ export const DEFAULT_APPROVAL_EFFECTS: readonly VehicleEffect[] = ["destructive"
|
|
|
42
42
|
*/
|
|
43
43
|
export const VEHICLE_APPROVAL_RESOLVE_OPERATION_NAME = "vehicle.approval.resolve";
|
|
44
44
|
|
|
45
|
+
/**
|
|
46
|
+
* The name VehicleRegistry.configureApprovals() registers its own read-only status-lookup
|
|
47
|
+
* operation under. Unlike vehicle.approval.resolve, this one is deliberately left projectable
|
|
48
|
+
* as an ordinary Pi/model-callable tool: reading a requestId's own already-made decision (who
|
|
49
|
+
* decided, when, and any comment) can never let a caller grant its own pending request -- it
|
|
50
|
+
* only ever surfaces a decision a human (or other authority) already recorded. Exists because a
|
|
51
|
+
* gated invoke()'s own local HITL prompt and the eventual human decision can genuinely happen on
|
|
52
|
+
* two different calls (a caller-side timeout, a UI presented asynchronously outside the
|
|
53
|
+
* originating call's own lifetime, a session restart in between) -- without this, a decision
|
|
54
|
+
* (and any comment explaining it) made after the originating call already gave up is
|
|
55
|
+
* unrecoverable, since vehicle.approval.resolve's own vehicleApprovalResolvedEvent is transient
|
|
56
|
+
* pub/sub with no replay for a caller that wasn't subscribed at that exact moment.
|
|
57
|
+
*/
|
|
58
|
+
export const VEHICLE_APPROVAL_STATUS_OPERATION_NAME = "vehicle.approval.status";
|
|
59
|
+
|
|
45
60
|
/** How long a request stays resolvable before it lapses and must be re-requested. */
|
|
46
61
|
export const DEFAULT_APPROVAL_TIMEOUT_MS = 5 * 60_000;
|
|
47
62
|
|
|
@@ -15,23 +15,35 @@ const DEFAULT_MAX_ACTIVE = 3;
|
|
|
15
15
|
const DEFAULT_ABSOLUTE_MAX_ACTIVE = 32;
|
|
16
16
|
const DEFAULT_BACKGROUND_ADMISSION_QUEUE_TIMEOUT_MS = 10_000;
|
|
17
17
|
const DEFAULT_MAX_QUEUED_BACKGROUND_ADMISSIONS = 8;
|
|
18
|
+
const DEFAULT_FOREGROUND_ADMISSION_QUEUE_TIMEOUT_MS = 0;
|
|
19
|
+
const DEFAULT_MAX_QUEUED_FOREGROUND_ADMISSIONS = 8;
|
|
18
20
|
|
|
19
21
|
/**
|
|
20
22
|
* Distinguishes an interactive human/agent-facing request from a self-scheduled background one.
|
|
21
|
-
* Foreground
|
|
22
|
-
*
|
|
23
|
-
* that never opts in gets today's exact unreserved behavior.
|
|
23
|
+
* Foreground keeps the full ceiling and remains fail-fast unless its explicit queue timeout is
|
|
24
|
+
* enabled; background can be held below the reserved-slot ceiling. Defaults to "foreground".
|
|
24
25
|
*/
|
|
25
26
|
export type ResourceWorkKind = "foreground" | "background";
|
|
26
27
|
|
|
27
28
|
/**
|
|
28
29
|
* Internal signal only: admit() throws this to tell acquire() "release the serialized lock and
|
|
29
|
-
* wait outside it" -- never surfaced to a caller.
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* the starvation this exists to prevent.
|
|
30
|
+
* wait outside it" -- never surfaced to a caller. A bounded admission wait can take seconds;
|
|
31
|
+
* holding admissionTail (the single global admission mutex) for that span would block every other
|
|
32
|
+
* request and create the starvation this mechanism exists to prevent.
|
|
33
33
|
*/
|
|
34
|
-
class
|
|
34
|
+
class NeedsAdmissionWait extends Error {
|
|
35
|
+
constructor(
|
|
36
|
+
readonly workKind: ResourceWorkKind,
|
|
37
|
+
readonly observedGeneration: number,
|
|
38
|
+
) {
|
|
39
|
+
super();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
interface AdmissionWaiter {
|
|
44
|
+
readonly workKind: ResourceWorkKind;
|
|
45
|
+
readonly wake: () => void;
|
|
46
|
+
}
|
|
35
47
|
|
|
36
48
|
export type ResourcePoolEvent =
|
|
37
49
|
| { readonly kind: "admission-evicted" | "dead-replaced" | "resource-pressure-evicted"; readonly partitionKey: string }
|
|
@@ -52,7 +64,9 @@ export interface ResourcePoolStatus<Status = unknown> {
|
|
|
52
64
|
readonly absoluteMaxActive: number;
|
|
53
65
|
readonly byPartition: Readonly<Record<string, number>>;
|
|
54
66
|
readonly resources?: Status;
|
|
55
|
-
/** How many
|
|
67
|
+
/** How many opt-in foreground admissions are waiting for a lease to free. */
|
|
68
|
+
readonly waitingForegroundAdmissions: number;
|
|
69
|
+
/** How many background admissions are waiting for room below their effective ceiling. */
|
|
56
70
|
readonly waitingBackgroundAdmissions: number;
|
|
57
71
|
}
|
|
58
72
|
|
|
@@ -73,6 +87,10 @@ export interface ResourcePoolOptions<Status = unknown> {
|
|
|
73
87
|
readonly backgroundAdmissionQueueTimeoutMs?: number;
|
|
74
88
|
/** How many background admissions may be simultaneously waiting before a new one fails fast with ResourceAdmissionQueueFull instead of growing the wait queue further. Default 8. */
|
|
75
89
|
readonly maxQueuedBackgroundAdmissions?: number;
|
|
90
|
+
/** Opt-in foreground wait bound. Zero (default) preserves fail-fast ResourceCapacityExceeded behavior. */
|
|
91
|
+
readonly foregroundAdmissionQueueTimeoutMs?: number;
|
|
92
|
+
/** How many opted-in foreground admissions may wait simultaneously. Default 8. */
|
|
93
|
+
readonly maxQueuedForegroundAdmissions?: number;
|
|
76
94
|
/** Fed one (partitionKey, costHandle) pair per active entry with a costHandle on calibrateCosts(). */
|
|
77
95
|
readonly costRecorder?: { recordSample(partitionKey: string, costHandle: unknown): void };
|
|
78
96
|
readonly now?: () => number;
|
|
@@ -108,7 +126,11 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
108
126
|
private readonly reservedForegroundSlots: number;
|
|
109
127
|
private readonly backgroundAdmissionQueueTimeoutMs: number;
|
|
110
128
|
private readonly maxQueuedBackgroundAdmissions: number;
|
|
111
|
-
private readonly
|
|
129
|
+
private readonly foregroundAdmissionQueueTimeoutMs: number;
|
|
130
|
+
private readonly maxQueuedForegroundAdmissions: number;
|
|
131
|
+
private readonly admissionWaiters: AdmissionWaiter[] = [];
|
|
132
|
+
private admissionGeneration = 0;
|
|
133
|
+
private queuedForegroundAdmissions = 0;
|
|
112
134
|
private queuedBackgroundAdmissions = 0;
|
|
113
135
|
private readonly waitingCounts = new Map<string, number>();
|
|
114
136
|
|
|
@@ -138,6 +160,14 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
138
160
|
if (!Number.isSafeInteger(this.maxQueuedBackgroundAdmissions) || this.maxQueuedBackgroundAdmissions < 1) {
|
|
139
161
|
throw new TypeError("maxQueuedBackgroundAdmissions must be a positive safe integer");
|
|
140
162
|
}
|
|
163
|
+
this.foregroundAdmissionQueueTimeoutMs = options.foregroundAdmissionQueueTimeoutMs ?? DEFAULT_FOREGROUND_ADMISSION_QUEUE_TIMEOUT_MS;
|
|
164
|
+
if (!Number.isSafeInteger(this.foregroundAdmissionQueueTimeoutMs) || this.foregroundAdmissionQueueTimeoutMs < 0) {
|
|
165
|
+
throw new TypeError("foregroundAdmissionQueueTimeoutMs must be a non-negative safe integer");
|
|
166
|
+
}
|
|
167
|
+
this.maxQueuedForegroundAdmissions = options.maxQueuedForegroundAdmissions ?? DEFAULT_MAX_QUEUED_FOREGROUND_ADMISSIONS;
|
|
168
|
+
if (!Number.isSafeInteger(this.maxQueuedForegroundAdmissions) || this.maxQueuedForegroundAdmissions < 1) {
|
|
169
|
+
throw new TypeError("maxQueuedForegroundAdmissions must be a positive safe integer");
|
|
170
|
+
}
|
|
141
171
|
}
|
|
142
172
|
|
|
143
173
|
private key(ownerKey: OwnerKey, partitionKey: string): string {
|
|
@@ -196,51 +226,84 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
196
226
|
this.notifyAdmissionWaiters();
|
|
197
227
|
}
|
|
198
228
|
|
|
199
|
-
/** Wakes
|
|
229
|
+
/** Wakes one waiter per newly available opportunity, preferring foreground and preserving FIFO within each class. */
|
|
200
230
|
private notifyAdmissionWaiters(): void {
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
this.admissionWaiters.
|
|
204
|
-
|
|
231
|
+
this.admissionGeneration++;
|
|
232
|
+
if (this.admissionWaiters.length === 0) return;
|
|
233
|
+
const foregroundIndex = this.admissionWaiters.findIndex((waiter) => waiter.workKind === "foreground");
|
|
234
|
+
const [waiter] = this.admissionWaiters.splice(foregroundIndex === -1 ? 0 : foregroundIndex, 1);
|
|
235
|
+
waiter?.wake();
|
|
205
236
|
}
|
|
206
237
|
|
|
207
|
-
/** True while at least one background admission for this owner is
|
|
238
|
+
/** True while at least one background admission for this owner is waiting. */
|
|
208
239
|
waitingForAdmission(ownerKey: OwnerKey): boolean {
|
|
209
240
|
return (this.waitingCounts.get(ownerKey) ?? 0) > 0;
|
|
210
241
|
}
|
|
211
242
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
243
|
+
private waitTimeoutMs(workKind: ResourceWorkKind): number {
|
|
244
|
+
return workKind === "foreground" ? this.foregroundAdmissionQueueTimeoutMs : this.backgroundAdmissionQueueTimeoutMs;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
private canWait(workKind: ResourceWorkKind): boolean {
|
|
248
|
+
return workKind === "background" || this.foregroundAdmissionQueueTimeoutMs > 0;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
private capacityUnavailable(workKind: ResourceWorkKind, partitionKey: string, maxActive: number, partitionLimit: number): never {
|
|
252
|
+
if (this.canWait(workKind)) throw new NeedsAdmissionWait(workKind, this.admissionGeneration);
|
|
253
|
+
throw new ResourceCapacityExceeded(partitionKey, maxActive, partitionLimit);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Waits outside the serialized admission lock, with one total deadline across false wakes. */
|
|
257
|
+
private async waitForAdmissionRoom(
|
|
258
|
+
partitionKey: string,
|
|
259
|
+
ownerKey: OwnerKey,
|
|
260
|
+
workKind: ResourceWorkKind,
|
|
261
|
+
deadline: number,
|
|
262
|
+
observedGeneration: number,
|
|
263
|
+
): Promise<void> {
|
|
264
|
+
// A lease may complete after admit() releases the mutex but before this waiter is registered.
|
|
265
|
+
// Retry immediately when that happened rather than sleeping through a lost wake-up.
|
|
266
|
+
if (this.admissionGeneration !== observedGeneration) return;
|
|
267
|
+
const maxQueued = workKind === "foreground" ? this.maxQueuedForegroundAdmissions : this.maxQueuedBackgroundAdmissions;
|
|
268
|
+
const queued = workKind === "foreground" ? this.queuedForegroundAdmissions : this.queuedBackgroundAdmissions;
|
|
269
|
+
if (queued >= maxQueued) throw new ResourceAdmissionQueueFull(partitionKey, maxQueued, workKind);
|
|
270
|
+
const timeoutMs = this.waitTimeoutMs(workKind);
|
|
271
|
+
const remainingMs = deadline - Date.now();
|
|
272
|
+
if (remainingMs <= 0) throw new ResourceAdmissionQueueTimedOut(partitionKey, timeoutMs, workKind);
|
|
273
|
+
|
|
274
|
+
if (workKind === "foreground") this.queuedForegroundAdmissions++;
|
|
275
|
+
else {
|
|
276
|
+
this.queuedBackgroundAdmissions++;
|
|
277
|
+
this.waitingCounts.set(ownerKey, (this.waitingCounts.get(ownerKey) ?? 0) + 1);
|
|
221
278
|
}
|
|
222
|
-
this.queuedBackgroundAdmissions++;
|
|
223
|
-
this.waitingCounts.set(ownerKey, (this.waitingCounts.get(ownerKey) ?? 0) + 1);
|
|
224
279
|
try {
|
|
225
280
|
const gotSignal = await new Promise<boolean>((resolve) => {
|
|
226
281
|
let settled = false;
|
|
282
|
+
let timer: ReturnType<typeof setTimeout>;
|
|
283
|
+
const waiter: AdmissionWaiter = {
|
|
284
|
+
workKind,
|
|
285
|
+
wake: () => finish(true),
|
|
286
|
+
};
|
|
227
287
|
const finish = (ready: boolean): void => {
|
|
228
288
|
if (settled) return;
|
|
229
289
|
settled = true;
|
|
230
290
|
clearTimeout(timer);
|
|
231
|
-
this.admissionWaiters.
|
|
291
|
+
const index = this.admissionWaiters.indexOf(waiter);
|
|
292
|
+
if (index !== -1) this.admissionWaiters.splice(index, 1);
|
|
232
293
|
resolve(ready);
|
|
233
294
|
};
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
this.admissionWaiters.add(onSignal);
|
|
295
|
+
timer = setTimeout(() => finish(false), remainingMs);
|
|
296
|
+
this.admissionWaiters.push(waiter);
|
|
237
297
|
});
|
|
238
|
-
if (!gotSignal) throw new ResourceAdmissionQueueTimedOut(partitionKey,
|
|
298
|
+
if (!gotSignal) throw new ResourceAdmissionQueueTimedOut(partitionKey, timeoutMs, workKind);
|
|
239
299
|
} finally {
|
|
240
|
-
this.
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
300
|
+
if (workKind === "foreground") this.queuedForegroundAdmissions--;
|
|
301
|
+
else {
|
|
302
|
+
this.queuedBackgroundAdmissions--;
|
|
303
|
+
const remaining = (this.waitingCounts.get(ownerKey) ?? 1) - 1;
|
|
304
|
+
if (remaining <= 0) this.waitingCounts.delete(ownerKey);
|
|
305
|
+
else this.waitingCounts.set(ownerKey, remaining);
|
|
306
|
+
}
|
|
244
307
|
}
|
|
245
308
|
}
|
|
246
309
|
|
|
@@ -249,20 +312,22 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
249
312
|
partitionKey: string,
|
|
250
313
|
create: () => Resource,
|
|
251
314
|
workKind: ResourceWorkKind,
|
|
315
|
+
admissionGranted: boolean,
|
|
252
316
|
): Promise<PoolEntry<OwnerKey, Resource>> {
|
|
253
317
|
const partitionLimit = this.partitionLimit(partitionKey);
|
|
318
|
+
if (this.queuedForegroundAdmissions > 0 && !(workKind === "foreground" && admissionGranted)) {
|
|
319
|
+
throw new NeedsAdmissionWait(workKind, this.admissionGeneration);
|
|
320
|
+
}
|
|
254
321
|
while (this.countPartition(partitionKey) >= partitionLimit) {
|
|
255
322
|
const victim = this.leastRecentlyUsedIdle(partitionKey);
|
|
256
|
-
if (!victim)
|
|
323
|
+
if (!victim) this.capacityUnavailable(workKind, partitionKey, this.maxActive, partitionLimit);
|
|
257
324
|
await this.evict(victim);
|
|
258
325
|
}
|
|
259
326
|
const { ceiling: baseCeiling, source: ceilingSource } = this.baseActiveCeiling();
|
|
260
327
|
this.lastEffectiveMaxActive = baseCeiling;
|
|
261
328
|
this.lastActiveCeilingSource = ceilingSource;
|
|
262
|
-
// "Borrowable": background's own effective ceiling is reduced,
|
|
263
|
-
//
|
|
264
|
-
// set-aside nothing else can reach. Foreground keeps using the full (possibly resource-
|
|
265
|
-
// budget-raised) baseCeiling unchanged.
|
|
329
|
+
// "Borrowable": background's own effective ceiling is reduced, while foreground keeps the
|
|
330
|
+
// full (possibly resource-budget-raised) base ceiling unchanged.
|
|
266
331
|
const effectiveMaxActive = workKind === "background" ? Math.max(baseCeiling - this.reservedForegroundSlots, 0) : baseCeiling;
|
|
267
332
|
while (this.entries.size >= effectiveMaxActive) {
|
|
268
333
|
const victim = this.leastRecentlyUsedIdle();
|
|
@@ -270,12 +335,11 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
270
335
|
await this.evict(victim);
|
|
271
336
|
continue;
|
|
272
337
|
}
|
|
273
|
-
|
|
274
|
-
throw new ResourceCapacityExceeded(partitionKey, baseCeiling, partitionLimit);
|
|
338
|
+
this.capacityUnavailable(workKind, partitionKey, baseCeiling, partitionLimit);
|
|
275
339
|
}
|
|
276
340
|
while (this.options.resourcePolicy && !this.options.resourcePolicy.canAdmit(this.activePartitions(), partitionKey)) {
|
|
277
341
|
const victim = this.leastRecentlyUsedIdle();
|
|
278
|
-
if (!victim)
|
|
342
|
+
if (!victim) this.capacityUnavailable(workKind, partitionKey, baseCeiling, partitionLimit);
|
|
279
343
|
await this.evict(victim, "resource-pressure");
|
|
280
344
|
}
|
|
281
345
|
return {
|
|
@@ -309,6 +373,8 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
309
373
|
create: () => Resource,
|
|
310
374
|
workKind: ResourceWorkKind = "foreground",
|
|
311
375
|
): Promise<PoolLease<Resource>> {
|
|
376
|
+
const deadline = Date.now() + this.waitTimeoutMs(workKind);
|
|
377
|
+
let admissionGranted = false;
|
|
312
378
|
for (;;) {
|
|
313
379
|
try {
|
|
314
380
|
const entry = await this.serialized(async () => {
|
|
@@ -322,7 +388,7 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
322
388
|
entry = undefined;
|
|
323
389
|
}
|
|
324
390
|
if (!entry) {
|
|
325
|
-
entry = await this.admit(ownerKey, partitionKey, create, workKind);
|
|
391
|
+
entry = await this.admit(ownerKey, partitionKey, create, workKind, admissionGranted);
|
|
326
392
|
this.entries.set(key, entry);
|
|
327
393
|
}
|
|
328
394
|
entry.activeLeases++;
|
|
@@ -330,10 +396,9 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
330
396
|
});
|
|
331
397
|
return this.lease(entry.resource, [entry]);
|
|
332
398
|
} catch (error) {
|
|
333
|
-
if (!(error instanceof
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
await this.waitForAdmissionRoom(partitionKey, ownerKey);
|
|
399
|
+
if (!(error instanceof NeedsAdmissionWait)) throw error;
|
|
400
|
+
await this.waitForAdmissionRoom(partitionKey, ownerKey, error.workKind, deadline, error.observedGeneration);
|
|
401
|
+
admissionGranted = true;
|
|
337
402
|
}
|
|
338
403
|
}
|
|
339
404
|
}
|
|
@@ -352,8 +417,7 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
352
417
|
entry.recencySequence = this.nextSequence++;
|
|
353
418
|
}
|
|
354
419
|
// A lease completing makes its entry newly idle -- exactly the condition a queued
|
|
355
|
-
//
|
|
356
|
-
// itself ends up evicting anything below.
|
|
420
|
+
// admission retries against.
|
|
357
421
|
this.notifyAdmissionWaiters();
|
|
358
422
|
await this.reconcileResources();
|
|
359
423
|
},
|
|
@@ -405,6 +469,7 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
405
469
|
activeCeilingSource: this.lastActiveCeilingSource,
|
|
406
470
|
absoluteMaxActive: this.absoluteMaxActive,
|
|
407
471
|
byPartition,
|
|
472
|
+
waitingForegroundAdmissions: this.queuedForegroundAdmissions,
|
|
408
473
|
waitingBackgroundAdmissions: this.queuedBackgroundAdmissions,
|
|
409
474
|
...(resources !== undefined ? { resources } : {}),
|
|
410
475
|
};
|
|
@@ -424,6 +489,7 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
424
489
|
async closeOwner(ownerKey: OwnerKey): Promise<void> {
|
|
425
490
|
const stale = Array.from(this.entries.entries()).filter(([, entry]) => entry.ownerKey === ownerKey);
|
|
426
491
|
for (const [key] of stale) this.entries.delete(key);
|
|
492
|
+
for (let i = 0; i < stale.length; i++) this.notifyAdmissionWaiters();
|
|
427
493
|
await Promise.all(stale.map(([, entry]) => entry.resource.close()));
|
|
428
494
|
}
|
|
429
495
|
|
|
@@ -433,6 +499,7 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
433
499
|
const entry = this.entries.get(key);
|
|
434
500
|
if (!entry) return;
|
|
435
501
|
this.entries.delete(key);
|
|
502
|
+
this.notifyAdmissionWaiters();
|
|
436
503
|
await entry.resource.close();
|
|
437
504
|
}
|
|
438
505
|
|
|
@@ -453,6 +520,7 @@ export class BoundedResourcePool<OwnerKey extends string, Resource extends Poole
|
|
|
453
520
|
async closeAll(): Promise<void> {
|
|
454
521
|
const entries = Array.from(this.entries.values());
|
|
455
522
|
this.entries.clear();
|
|
523
|
+
for (let i = 0; i < entries.length; i++) this.notifyAdmissionWaiters();
|
|
456
524
|
await Promise.all(entries.map((entry) => entry.resource.close()));
|
|
457
525
|
}
|
|
458
526
|
|
|
@@ -20,26 +20,26 @@ export class ResourceInUse extends Error {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
/** Raised when
|
|
23
|
+
/** Raised when one admission class is already at its configured queue bound. */
|
|
24
24
|
export class ResourceAdmissionQueueFull extends Error {
|
|
25
25
|
constructor(
|
|
26
26
|
readonly partitionKey: string,
|
|
27
27
|
readonly maxQueued: number,
|
|
28
|
+
readonly workKind: "foreground" | "background" = "background",
|
|
28
29
|
) {
|
|
29
|
-
super(
|
|
30
|
+
super(`${workKind} admission for partition "${partitionKey}" is already waiting at capacity (${maxQueued} queued); retry later`);
|
|
30
31
|
this.name = "ResourceAdmissionQueueFull";
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
34
|
-
/** Raised when a queued
|
|
35
|
+
/** Raised when a queued admission waits past its configured timeout without room becoming available. */
|
|
35
36
|
export class ResourceAdmissionQueueTimedOut extends Error {
|
|
36
37
|
constructor(
|
|
37
38
|
readonly partitionKey: string,
|
|
38
39
|
readonly timeoutMs: number,
|
|
40
|
+
readonly workKind: "foreground" | "background" = "background",
|
|
39
41
|
) {
|
|
40
|
-
super(
|
|
41
|
-
`background admission for partition "${partitionKey}" waited ${timeoutMs}ms for a resource-pool slot and gave up -- foreground demand is holding every admittable slot`,
|
|
42
|
-
);
|
|
42
|
+
super(`${workKind} admission for partition "${partitionKey}" waited ${timeoutMs}ms for a resource-pool slot and gave up`);
|
|
43
43
|
this.name = "ResourceAdmissionQueueTimedOut";
|
|
44
44
|
}
|
|
45
45
|
}
|