@danypops/vehicle-core 0.18.3 → 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +6 -4
- package/dist/index.js +6 -4
- package/dist/resource-pool/bounded-resource-pool.d.ts +139 -0
- package/dist/resource-pool/bounded-resource-pool.js +427 -0
- package/dist/resource-pool/errors.d.ts +24 -0
- package/dist/resource-pool/errors.js +44 -0
- package/dist/resource-pool/index.d.ts +3 -0
- package/dist/resource-pool/index.js +3 -0
- package/dist/resource-pool/resource-policy.d.ts +28 -0
- package/dist/resource-pool/resource-policy.js +1 -0
- package/package.json +1 -1
- package/src/index.ts +6 -4
- package/src/resource-pool/bounded-resource-pool.ts +506 -0
- package/src/resource-pool/errors.ts +45 -0
- package/src/resource-pool/index.ts +3 -0
- package/src/resource-pool/resource-policy.ts +29 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Raised when no idle resource can be evicted to admit a new one within the current ceiling. */
|
|
2
|
+
export class ResourceCapacityExceeded extends Error {
|
|
3
|
+
partitionKey;
|
|
4
|
+
maxActive;
|
|
5
|
+
partitionLimit;
|
|
6
|
+
constructor(partitionKey, maxActive, partitionLimit) {
|
|
7
|
+
super(`no idle resource can be evicted to admit partition "${partitionKey}" within global capacity ${maxActive} and partition capacity ${partitionLimit}`);
|
|
8
|
+
this.partitionKey = partitionKey;
|
|
9
|
+
this.maxActive = maxActive;
|
|
10
|
+
this.partitionLimit = partitionLimit;
|
|
11
|
+
this.name = "ResourceCapacityExceeded";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
/** Raised by releaseOwnerIfIdle when at least one of the owner's own resources still has an active lease. */
|
|
15
|
+
export class ResourceInUse extends Error {
|
|
16
|
+
ownerKey;
|
|
17
|
+
constructor(ownerKey) {
|
|
18
|
+
super(`cannot release owner "${ownerKey}": a pooled resource for it still has an active lease`);
|
|
19
|
+
this.ownerKey = ownerKey;
|
|
20
|
+
this.name = "ResourceInUse";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/** Raised when background admission is already waiting at maxQueuedBackgroundAdmissions -- fails fast rather than growing the wait queue without bound. */
|
|
24
|
+
export class ResourceAdmissionQueueFull extends Error {
|
|
25
|
+
partitionKey;
|
|
26
|
+
maxQueued;
|
|
27
|
+
constructor(partitionKey, maxQueued) {
|
|
28
|
+
super(`background admission for partition "${partitionKey}" is already waiting at capacity (${maxQueued} queued); retry later`);
|
|
29
|
+
this.partitionKey = partitionKey;
|
|
30
|
+
this.maxQueued = maxQueued;
|
|
31
|
+
this.name = "ResourceAdmissionQueueFull";
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** Raised when a queued background admission waits past backgroundAdmissionQueueTimeoutMs without a slot freeing. */
|
|
35
|
+
export class ResourceAdmissionQueueTimedOut extends Error {
|
|
36
|
+
partitionKey;
|
|
37
|
+
timeoutMs;
|
|
38
|
+
constructor(partitionKey, timeoutMs) {
|
|
39
|
+
super(`background admission for partition "${partitionKey}" waited ${timeoutMs}ms for a resource-pool slot and gave up -- foreground demand is holding every admittable slot`);
|
|
40
|
+
this.partitionKey = partitionKey;
|
|
41
|
+
this.timeoutMs = timeoutMs;
|
|
42
|
+
this.name = "ResourceAdmissionQueueTimedOut";
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** Which ceiling actually constrained the most recent admission decision. */
|
|
2
|
+
export type ResourcePoolActiveCeilingSource = "configured" | "resource-budget" | "absolute-cap";
|
|
3
|
+
/**
|
|
4
|
+
* Optional plug point letting a real resource budget (memory, /proc-sampled process cost, ...)
|
|
5
|
+
* drive admission and retention decisions beyond the pool's own configured counts. A pool without
|
|
6
|
+
* one falls back to its configured maxActive/absoluteMaxActive alone -- this interface exists so a
|
|
7
|
+
* caller CAN plug in something smarter, not because every caller needs to.
|
|
8
|
+
*
|
|
9
|
+
* `Status` is left to the caller (default `unknown`) rather than fixed to one shape: the pool
|
|
10
|
+
* itself never inspects or reshapes what `status()` returns, only forwards it verbatim in its own
|
|
11
|
+
* status() report -- a caller with an existing status shape (its own field names, units, wire
|
|
12
|
+
* contract) plugs it in unchanged, no adapter required.
|
|
13
|
+
*/
|
|
14
|
+
export interface ResourcePoolResourcePolicy<Status = unknown> {
|
|
15
|
+
canAdmit(activePartitions: readonly string[], requestedPartition: string): boolean;
|
|
16
|
+
isOverBudget(activePartitions: readonly string[]): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* A conservative, count-shaped ceiling derived from a real budget and worst-case known
|
|
19
|
+
* per-partition cost -- lets a larger real budget actually raise how many resources the pool
|
|
20
|
+
* will try to keep active, instead of a fixed configured count being the permanent bottleneck
|
|
21
|
+
* regardless of how much is genuinely available. Never authoritative on its own: canAdmit's own
|
|
22
|
+
* precise per-attempt check still gates the actual admission. Returns undefined on any metric
|
|
23
|
+
* loss -- fails closed, never treated as "unlimited room."
|
|
24
|
+
*/
|
|
25
|
+
softActiveCeiling(activePartitions: readonly string[]): number | undefined;
|
|
26
|
+
maxIdleMs(configuredMaxIdleMs: number, activePartitions: readonly string[]): number;
|
|
27
|
+
status(activePartitions: readonly string[]): Status;
|
|
28
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danypops/vehicle-core",
|
|
3
|
-
"version": "0.18.
|
|
3
|
+
"version": "0.18.4",
|
|
4
4
|
"description": "Vehicle's runtime-neutral wire contract: operation descriptors, schema codecs, failure shapes. Zero runtime dependencies, zero Bun-specific code -- the one thing every Vehicle client and server package depends on.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/index.ts
CHANGED
|
@@ -6,10 +6,11 @@
|
|
|
6
6
|
* classification, invocation context), events, manifest, client (the port a
|
|
7
7
|
* caller programs against), approvals (the Approval Gate's wire shapes), jobs
|
|
8
8
|
* (Vehicle Jobs' pure pieces), schedules, watches, persistence (atomic
|
|
9
|
-
* JSON), concurrency (timer-based scheduling primitives),
|
|
10
|
-
* (argv-injection guards for any operation shelling out to a CLI)
|
|
11
|
-
*
|
|
12
|
-
*
|
|
9
|
+
* JSON), concurrency (timer-based scheduling primitives), cli-safety
|
|
10
|
+
* (argv-injection guards for any operation shelling out to a CLI), and
|
|
11
|
+
* resource-pool (bounded admission/eviction/leasing for a pooled expensive
|
|
12
|
+
* stateful resource) -- the latter four are technical utilities, not
|
|
13
|
+
* Vehicle protocol capabilities, kept distinct for that reason. Every symbol below is re-exported unchanged
|
|
13
14
|
* from its historical flat-file home, so root-level `import { X } from
|
|
14
15
|
* "@danypops/vehicle-core"` usage is completely unaffected by this layout.
|
|
15
16
|
*/
|
|
@@ -25,6 +26,7 @@ export * from "./jobs/index.js";
|
|
|
25
26
|
export * from "./manifest/index.js";
|
|
26
27
|
export * from "./operations/index.js";
|
|
27
28
|
export * from "./persistence/index.js";
|
|
29
|
+
export * from "./resource-pool/index.js";
|
|
28
30
|
export * from "./schedules/index.js";
|
|
29
31
|
export * from "./schemas/index.js";
|
|
30
32
|
export * from "./watches/index.js";
|
|
@@ -0,0 +1,506 @@
|
|
|
1
|
+
import { ResourceAdmissionQueueFull, ResourceAdmissionQueueTimedOut, ResourceCapacityExceeded, ResourceInUse } from "./errors.js";
|
|
2
|
+
import type { ResourcePoolActiveCeilingSource, ResourcePoolResourcePolicy } from "./resource-policy.js";
|
|
3
|
+
|
|
4
|
+
export { ResourceAdmissionQueueFull, ResourceAdmissionQueueTimedOut, ResourceCapacityExceeded, ResourceInUse };
|
|
5
|
+
|
|
6
|
+
/** A resource the pool can shut down when it goes cold. costHandle, when present, is an opaque token a cost-sampling hook may key off (e.g. a subprocess pid) -- undefined for a resource with nothing external to sample. */
|
|
7
|
+
export interface PooledResource {
|
|
8
|
+
close(): Promise<void>;
|
|
9
|
+
isAlive?(): boolean;
|
|
10
|
+
readonly costHandle?: unknown;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const DEFAULT_MAX_ACTIVE = 3;
|
|
14
|
+
/** A genuine structural ceiling on resource count -- independent of memory, protecting against pathological OS-level exhaustion (file descriptors, threads, scheduler overhead) that no amount of available headroom makes safe to exceed. maxActive itself can never be raised past this by a resource policy's own soft ceiling. */
|
|
15
|
+
const DEFAULT_ABSOLUTE_MAX_ACTIVE = 32;
|
|
16
|
+
const DEFAULT_BACKGROUND_ADMISSION_QUEUE_TIMEOUT_MS = 10_000;
|
|
17
|
+
const DEFAULT_MAX_QUEUED_BACKGROUND_ADMISSIONS = 8;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Distinguishes an interactive human/agent-facing request from a self-scheduled background one.
|
|
21
|
+
* Foreground admission is never queued or reduced below reservedForegroundSlots' effective
|
|
22
|
+
* ceiling -- background is the only work kind that ever waits. Defaults to "foreground": a caller
|
|
23
|
+
* that never opts in gets today's exact unreserved behavior.
|
|
24
|
+
*/
|
|
25
|
+
export type ResourceWorkKind = "foreground" | "background";
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Internal signal only: admit() throws this to tell acquire() "release the serialized lock and
|
|
29
|
+
* wait outside it" -- never surfaced to a caller. Waiting for a background admission's turn can
|
|
30
|
+
* legitimately take seconds; holding admissionTail (the single global admission mutex) for that
|
|
31
|
+
* whole span would block every OTHER admission request, including foreground's, which is exactly
|
|
32
|
+
* the starvation this exists to prevent.
|
|
33
|
+
*/
|
|
34
|
+
class NeedsBackgroundAdmissionWait extends Error {}
|
|
35
|
+
|
|
36
|
+
export type ResourcePoolEvent =
|
|
37
|
+
| { readonly kind: "admission-evicted" | "dead-replaced" | "resource-pressure-evicted"; readonly partitionKey: string }
|
|
38
|
+
| {
|
|
39
|
+
readonly kind: "close-failed";
|
|
40
|
+
readonly reason: "admission" | "dead-replacement" | "idle-reap" | "resource-pressure";
|
|
41
|
+
readonly partitionKey: string;
|
|
42
|
+
readonly errorName: string;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export interface ResourcePoolStatus<Status = unknown> {
|
|
46
|
+
readonly active: number;
|
|
47
|
+
readonly leased: number;
|
|
48
|
+
readonly maxActive: number;
|
|
49
|
+
/** The count ceiling actually in effect for the most recent admission -- may exceed maxActive when a resource policy's own soft ceiling raised it, never exceeds absoluteMaxActive. */
|
|
50
|
+
readonly effectiveMaxActive: number;
|
|
51
|
+
readonly activeCeilingSource: ResourcePoolActiveCeilingSource;
|
|
52
|
+
readonly absoluteMaxActive: number;
|
|
53
|
+
readonly byPartition: Readonly<Record<string, number>>;
|
|
54
|
+
readonly resources?: Status;
|
|
55
|
+
/** How many background admissions are currently waiting for a slot reserved for foreground work -- count-only, zero whenever reservedForegroundSlots is unset or nothing is contending. */
|
|
56
|
+
readonly waitingBackgroundAdmissions: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface PoolLease<Value> extends AsyncDisposable {
|
|
60
|
+
readonly value: Value;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface ResourcePoolOptions<Status = unknown> {
|
|
64
|
+
readonly maxActive?: number;
|
|
65
|
+
readonly partitionLimits?: Readonly<Record<string, number>>;
|
|
66
|
+
readonly resourcePolicy?: ResourcePoolResourcePolicy<Status>;
|
|
67
|
+
/** The hard structural ceiling a resource policy's own soft ceiling can never raise maxActive past. Defaults to 32. Must be >= maxActive. */
|
|
68
|
+
readonly absoluteMaxActive?: number;
|
|
69
|
+
readonly observe?: (event: ResourcePoolEvent) => void;
|
|
70
|
+
/** Slots background admission alone can never grow into. Default 0 (no reservation). */
|
|
71
|
+
readonly reservedForegroundSlots?: number;
|
|
72
|
+
/** How long a queued background admission waits for a slot before giving up with ResourceAdmissionQueueTimedOut. Default 10s. */
|
|
73
|
+
readonly backgroundAdmissionQueueTimeoutMs?: number;
|
|
74
|
+
/** How many background admissions may be simultaneously waiting before a new one fails fast with ResourceAdmissionQueueFull instead of growing the wait queue further. Default 8. */
|
|
75
|
+
readonly maxQueuedBackgroundAdmissions?: number;
|
|
76
|
+
/** Fed one (partitionKey, costHandle) pair per active entry with a costHandle on calibrateCosts(). */
|
|
77
|
+
readonly costRecorder?: { recordSample(partitionKey: string, costHandle: unknown): void };
|
|
78
|
+
readonly now?: () => number;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
interface PoolEntry<OwnerKey extends string, Resource extends PooledResource> {
|
|
82
|
+
readonly resource: Resource;
|
|
83
|
+
readonly ownerKey: OwnerKey;
|
|
84
|
+
readonly partitionKey: string;
|
|
85
|
+
recencySequence: number;
|
|
86
|
+
activeLeases: number;
|
|
87
|
+
lastUsedAt: number;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Owns the bounded lifecycle of pooled, expensive, stateful resources partitioned by an
|
|
92
|
+
* (ownerKey, partitionKey) pair -- e.g. one warm language-server process per (workspace,
|
|
93
|
+
* language), one warm browser context per (tenant, profile). Never knows how to build a resource:
|
|
94
|
+
* `acquire()` takes a lazy factory invoked only on an actual cache miss, so every caller-specific
|
|
95
|
+
* concept (what a resource even is, how to construct one) lives entirely in the caller's own
|
|
96
|
+
* closure at each call site.
|
|
97
|
+
*/
|
|
98
|
+
export class BoundedResourcePool<OwnerKey extends string, Resource extends PooledResource, Status = unknown> {
|
|
99
|
+
private readonly entries = new Map<string, PoolEntry<OwnerKey, Resource>>();
|
|
100
|
+
private readonly now: () => number;
|
|
101
|
+
private readonly maxActive: number;
|
|
102
|
+
private readonly absoluteMaxActive: number;
|
|
103
|
+
private lastActiveCeilingSource: ResourcePoolActiveCeilingSource = "configured";
|
|
104
|
+
private lastEffectiveMaxActive: number;
|
|
105
|
+
private readonly partitionLimits: Readonly<Record<string, number>>;
|
|
106
|
+
private admissionTail: Promise<void> = Promise.resolve();
|
|
107
|
+
private nextSequence = 0;
|
|
108
|
+
private readonly reservedForegroundSlots: number;
|
|
109
|
+
private readonly backgroundAdmissionQueueTimeoutMs: number;
|
|
110
|
+
private readonly maxQueuedBackgroundAdmissions: number;
|
|
111
|
+
private readonly admissionWaiters = new Set<() => void>();
|
|
112
|
+
private queuedBackgroundAdmissions = 0;
|
|
113
|
+
private readonly waitingCounts = new Map<string, number>();
|
|
114
|
+
|
|
115
|
+
constructor(private readonly options: ResourcePoolOptions<Status> = {}) {
|
|
116
|
+
this.now = options.now ?? Date.now;
|
|
117
|
+
this.maxActive = options.maxActive ?? DEFAULT_MAX_ACTIVE;
|
|
118
|
+
this.partitionLimits = options.partitionLimits ?? {};
|
|
119
|
+
if (!Number.isSafeInteger(this.maxActive) || this.maxActive < 1) throw new TypeError("maxActive must be a positive safe integer");
|
|
120
|
+
this.absoluteMaxActive = options.absoluteMaxActive ?? Math.max(DEFAULT_ABSOLUTE_MAX_ACTIVE, this.maxActive);
|
|
121
|
+
if (!Number.isSafeInteger(this.absoluteMaxActive) || this.absoluteMaxActive < this.maxActive) {
|
|
122
|
+
throw new TypeError("absoluteMaxActive must be a safe integer no smaller than maxActive");
|
|
123
|
+
}
|
|
124
|
+
this.lastEffectiveMaxActive = this.maxActive;
|
|
125
|
+
for (const [partitionKey, limit] of Object.entries(this.partitionLimits)) {
|
|
126
|
+
if (!partitionKey || !Number.isSafeInteger(limit) || limit < 1)
|
|
127
|
+
throw new TypeError("partition limits must be positive safe integers keyed by partition key");
|
|
128
|
+
}
|
|
129
|
+
this.reservedForegroundSlots = options.reservedForegroundSlots ?? 0;
|
|
130
|
+
if (!Number.isSafeInteger(this.reservedForegroundSlots) || this.reservedForegroundSlots < 0) {
|
|
131
|
+
throw new TypeError("reservedForegroundSlots must be a non-negative safe integer");
|
|
132
|
+
}
|
|
133
|
+
this.backgroundAdmissionQueueTimeoutMs = options.backgroundAdmissionQueueTimeoutMs ?? DEFAULT_BACKGROUND_ADMISSION_QUEUE_TIMEOUT_MS;
|
|
134
|
+
if (!Number.isSafeInteger(this.backgroundAdmissionQueueTimeoutMs) || this.backgroundAdmissionQueueTimeoutMs < 0) {
|
|
135
|
+
throw new TypeError("backgroundAdmissionQueueTimeoutMs must be a non-negative safe integer");
|
|
136
|
+
}
|
|
137
|
+
this.maxQueuedBackgroundAdmissions = options.maxQueuedBackgroundAdmissions ?? DEFAULT_MAX_QUEUED_BACKGROUND_ADMISSIONS;
|
|
138
|
+
if (!Number.isSafeInteger(this.maxQueuedBackgroundAdmissions) || this.maxQueuedBackgroundAdmissions < 1) {
|
|
139
|
+
throw new TypeError("maxQueuedBackgroundAdmissions must be a positive safe integer");
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
private key(ownerKey: OwnerKey, partitionKey: string): string {
|
|
144
|
+
return `${ownerKey}:${partitionKey}`;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
private partitionLimit(partitionKey: string): number {
|
|
148
|
+
return this.partitionLimits[partitionKey] ?? this.maxActive;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
private countPartition(partitionKey: string): number {
|
|
152
|
+
let count = 0;
|
|
153
|
+
for (const entry of this.entries.values()) if (entry.partitionKey === partitionKey) count++;
|
|
154
|
+
return count;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
private activePartitions(): string[] {
|
|
158
|
+
return Array.from(this.entries.values(), (entry) => entry.partitionKey);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
private leastRecentlyUsedIdle(partitionKey?: string): [string, PoolEntry<OwnerKey, Resource>] | undefined {
|
|
162
|
+
let selected: [string, PoolEntry<OwnerKey, Resource>] | undefined;
|
|
163
|
+
for (const candidate of this.entries) {
|
|
164
|
+
const entry = candidate[1];
|
|
165
|
+
if (entry.activeLeases > 0 || (partitionKey !== undefined && entry.partitionKey !== partitionKey)) continue;
|
|
166
|
+
const current = selected?.[1];
|
|
167
|
+
if (
|
|
168
|
+
!current ||
|
|
169
|
+
entry.lastUsedAt < current.lastUsedAt ||
|
|
170
|
+
(entry.lastUsedAt === current.lastUsedAt && entry.recencySequence < current.recencySequence)
|
|
171
|
+
)
|
|
172
|
+
selected = candidate;
|
|
173
|
+
}
|
|
174
|
+
return selected;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
private async evict(
|
|
178
|
+
entry: [string, PoolEntry<OwnerKey, Resource>],
|
|
179
|
+
reason: "admission" | "dead-replacement" | "resource-pressure" = "admission",
|
|
180
|
+
): Promise<void> {
|
|
181
|
+
try {
|
|
182
|
+
await entry[1].resource.close();
|
|
183
|
+
} catch (error) {
|
|
184
|
+
this.options.observe?.({
|
|
185
|
+
kind: "close-failed",
|
|
186
|
+
reason,
|
|
187
|
+
partitionKey: entry[1].partitionKey,
|
|
188
|
+
errorName: error instanceof Error ? error.name : "UnknownError",
|
|
189
|
+
});
|
|
190
|
+
throw error;
|
|
191
|
+
}
|
|
192
|
+
this.entries.delete(entry[0]);
|
|
193
|
+
const kind =
|
|
194
|
+
reason === "admission" ? "admission-evicted" : reason === "dead-replacement" ? "dead-replaced" : "resource-pressure-evicted";
|
|
195
|
+
this.options.observe?.({ kind, partitionKey: entry[1].partitionKey });
|
|
196
|
+
this.notifyAdmissionWaiters();
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Wakes every queued background admission to re-check the real state -- called whenever an entry is removed OR a lease completes (an idle candidate an admit() retry might now be able to evict). A false wake just re-checks and re-waits; never a correctness issue, only a wasted retry. */
|
|
200
|
+
private notifyAdmissionWaiters(): void {
|
|
201
|
+
if (this.admissionWaiters.size === 0) return;
|
|
202
|
+
const waiters = Array.from(this.admissionWaiters);
|
|
203
|
+
this.admissionWaiters.clear();
|
|
204
|
+
for (const waiter of waiters) waiter();
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** True while at least one background admission for this owner is currently waiting for a reserved-slot conflict to clear. */
|
|
208
|
+
waitingForAdmission(ownerKey: OwnerKey): boolean {
|
|
209
|
+
return (this.waitingCounts.get(ownerKey) ?? 0) > 0;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Runs entirely outside the serialized admission lock -- admissionTail is the single global
|
|
214
|
+
* admission mutex, and this wait can legitimately take up to backgroundAdmissionQueueTimeoutMs.
|
|
215
|
+
* Holding that lock for the whole wait would block every other admission request, foreground
|
|
216
|
+
* included, which is the exact starvation this exists to prevent.
|
|
217
|
+
*/
|
|
218
|
+
private async waitForAdmissionRoom(partitionKey: string, ownerKey: OwnerKey): Promise<void> {
|
|
219
|
+
if (this.queuedBackgroundAdmissions >= this.maxQueuedBackgroundAdmissions) {
|
|
220
|
+
throw new ResourceAdmissionQueueFull(partitionKey, this.maxQueuedBackgroundAdmissions);
|
|
221
|
+
}
|
|
222
|
+
this.queuedBackgroundAdmissions++;
|
|
223
|
+
this.waitingCounts.set(ownerKey, (this.waitingCounts.get(ownerKey) ?? 0) + 1);
|
|
224
|
+
try {
|
|
225
|
+
const gotSignal = await new Promise<boolean>((resolve) => {
|
|
226
|
+
let settled = false;
|
|
227
|
+
const finish = (ready: boolean): void => {
|
|
228
|
+
if (settled) return;
|
|
229
|
+
settled = true;
|
|
230
|
+
clearTimeout(timer);
|
|
231
|
+
this.admissionWaiters.delete(onSignal);
|
|
232
|
+
resolve(ready);
|
|
233
|
+
};
|
|
234
|
+
const onSignal = (): void => finish(true);
|
|
235
|
+
const timer = setTimeout(() => finish(false), this.backgroundAdmissionQueueTimeoutMs);
|
|
236
|
+
this.admissionWaiters.add(onSignal);
|
|
237
|
+
});
|
|
238
|
+
if (!gotSignal) throw new ResourceAdmissionQueueTimedOut(partitionKey, this.backgroundAdmissionQueueTimeoutMs);
|
|
239
|
+
} finally {
|
|
240
|
+
this.queuedBackgroundAdmissions--;
|
|
241
|
+
const remaining = (this.waitingCounts.get(ownerKey) ?? 1) - 1;
|
|
242
|
+
if (remaining <= 0) this.waitingCounts.delete(ownerKey);
|
|
243
|
+
else this.waitingCounts.set(ownerKey, remaining);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
private async admit(
|
|
248
|
+
ownerKey: OwnerKey,
|
|
249
|
+
partitionKey: string,
|
|
250
|
+
create: () => Resource,
|
|
251
|
+
workKind: ResourceWorkKind,
|
|
252
|
+
): Promise<PoolEntry<OwnerKey, Resource>> {
|
|
253
|
+
const partitionLimit = this.partitionLimit(partitionKey);
|
|
254
|
+
while (this.countPartition(partitionKey) >= partitionLimit) {
|
|
255
|
+
const victim = this.leastRecentlyUsedIdle(partitionKey);
|
|
256
|
+
if (!victim) throw new ResourceCapacityExceeded(partitionKey, this.maxActive, partitionLimit);
|
|
257
|
+
await this.evict(victim);
|
|
258
|
+
}
|
|
259
|
+
const { ceiling: baseCeiling, source: ceilingSource } = this.baseActiveCeiling();
|
|
260
|
+
this.lastEffectiveMaxActive = baseCeiling;
|
|
261
|
+
this.lastActiveCeilingSource = ceilingSource;
|
|
262
|
+
// "Borrowable": background's own effective ceiling is reduced, but only background is ever
|
|
263
|
+
// held to it -- it constrains what background alone can grow the pool into, not a hard
|
|
264
|
+
// set-aside nothing else can reach. Foreground keeps using the full (possibly resource-
|
|
265
|
+
// budget-raised) baseCeiling unchanged.
|
|
266
|
+
const effectiveMaxActive = workKind === "background" ? Math.max(baseCeiling - this.reservedForegroundSlots, 0) : baseCeiling;
|
|
267
|
+
while (this.entries.size >= effectiveMaxActive) {
|
|
268
|
+
const victim = this.leastRecentlyUsedIdle();
|
|
269
|
+
if (victim) {
|
|
270
|
+
await this.evict(victim);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
if (workKind === "background") throw new NeedsBackgroundAdmissionWait();
|
|
274
|
+
throw new ResourceCapacityExceeded(partitionKey, baseCeiling, partitionLimit);
|
|
275
|
+
}
|
|
276
|
+
while (this.options.resourcePolicy && !this.options.resourcePolicy.canAdmit(this.activePartitions(), partitionKey)) {
|
|
277
|
+
const victim = this.leastRecentlyUsedIdle();
|
|
278
|
+
if (!victim) throw new ResourceCapacityExceeded(partitionKey, baseCeiling, partitionLimit);
|
|
279
|
+
await this.evict(victim, "resource-pressure");
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
resource: create(),
|
|
283
|
+
ownerKey,
|
|
284
|
+
partitionKey,
|
|
285
|
+
recencySequence: this.nextSequence++,
|
|
286
|
+
activeLeases: 0,
|
|
287
|
+
lastUsedAt: this.now(),
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
private async serialized<Value>(operation: () => Promise<Value>): Promise<Value> {
|
|
292
|
+
const previous = this.admissionTail;
|
|
293
|
+
let release = (): void => {};
|
|
294
|
+
this.admissionTail = new Promise<void>((resolve) => {
|
|
295
|
+
release = resolve;
|
|
296
|
+
});
|
|
297
|
+
await previous;
|
|
298
|
+
try {
|
|
299
|
+
return await operation();
|
|
300
|
+
} finally {
|
|
301
|
+
release();
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Acquires a lease for (ownerKey, partitionKey), reusing an already-admitted resource if one is warm, or admitting a fresh one via `create()` -- called only on an actual cache miss, never speculatively. workKind defaults to "foreground". */
|
|
306
|
+
async acquire(
|
|
307
|
+
ownerKey: OwnerKey,
|
|
308
|
+
partitionKey: string,
|
|
309
|
+
create: () => Resource,
|
|
310
|
+
workKind: ResourceWorkKind = "foreground",
|
|
311
|
+
): Promise<PoolLease<Resource>> {
|
|
312
|
+
for (;;) {
|
|
313
|
+
try {
|
|
314
|
+
const entry = await this.serialized(async () => {
|
|
315
|
+
const key = this.key(ownerKey, partitionKey);
|
|
316
|
+
let entry = this.entries.get(key);
|
|
317
|
+
if (entry?.resource.isAlive?.() === false) {
|
|
318
|
+
if (entry.activeLeases > 0) {
|
|
319
|
+
throw new ResourceCapacityExceeded(partitionKey, this.maxActive, this.partitionLimit(partitionKey));
|
|
320
|
+
}
|
|
321
|
+
await this.evict([key, entry], "dead-replacement");
|
|
322
|
+
entry = undefined;
|
|
323
|
+
}
|
|
324
|
+
if (!entry) {
|
|
325
|
+
entry = await this.admit(ownerKey, partitionKey, create, workKind);
|
|
326
|
+
this.entries.set(key, entry);
|
|
327
|
+
}
|
|
328
|
+
entry.activeLeases++;
|
|
329
|
+
return entry;
|
|
330
|
+
});
|
|
331
|
+
return this.lease(entry.resource, [entry]);
|
|
332
|
+
} catch (error) {
|
|
333
|
+
if (!(error instanceof NeedsBackgroundAdmissionWait)) throw error;
|
|
334
|
+
// Outside the lock deliberately -- see waitForAdmissionRoom's own comment. Throws
|
|
335
|
+
// ResourceAdmissionQueueFull/TimedOut instead of looping back if it can't wait.
|
|
336
|
+
await this.waitForAdmissionRoom(partitionKey, ownerKey);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
private lease<Value>(value: Value, entries: readonly PoolEntry<OwnerKey, Resource>[]): PoolLease<Value> {
|
|
342
|
+
let released = false;
|
|
343
|
+
return {
|
|
344
|
+
value,
|
|
345
|
+
[Symbol.asyncDispose]: async () => {
|
|
346
|
+
if (released) return;
|
|
347
|
+
released = true;
|
|
348
|
+
const completedAt = this.now();
|
|
349
|
+
for (const entry of entries) {
|
|
350
|
+
entry.activeLeases--;
|
|
351
|
+
entry.lastUsedAt = completedAt;
|
|
352
|
+
entry.recencySequence = this.nextSequence++;
|
|
353
|
+
}
|
|
354
|
+
// A lease completing makes its entry newly idle -- exactly the condition a queued
|
|
355
|
+
// background admission's retry is waiting to find, whether or not resource pressure
|
|
356
|
+
// itself ends up evicting anything below.
|
|
357
|
+
this.notifyAdmissionWaiters();
|
|
358
|
+
await this.reconcileResources();
|
|
359
|
+
},
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
has(ownerKey: OwnerKey, partitionKey: string): boolean {
|
|
364
|
+
return this.entries.has(this.key(ownerKey, partitionKey));
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
hasAny(ownerKey: OwnerKey): boolean {
|
|
368
|
+
for (const entry of this.entries.values()) if (entry.ownerKey === ownerKey) return true;
|
|
369
|
+
return false;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** Every currently active resource belonging to `ownerKey` -- lets a caller's own fan-out (file-watch notifications, etc) stay pool-backed instead of duplicating the entry map. */
|
|
373
|
+
activeResourcesForOwner(ownerKey: OwnerKey): readonly Resource[] {
|
|
374
|
+
const resources: Resource[] = [];
|
|
375
|
+
for (const entry of this.entries.values()) if (entry.ownerKey === ownerKey) resources.push(entry.resource);
|
|
376
|
+
return resources;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Derives the count ceiling actually in effect right now, independent of any particular
|
|
381
|
+
* admission attempt -- the resource policy's own soft ceiling raises maxActive when it reports
|
|
382
|
+
* more room, clamped to absoluteMaxActive, and falls back to maxActive alone (source
|
|
383
|
+
* "configured") on any metric loss -- fails closed, never treated as "unlimited room."
|
|
384
|
+
*/
|
|
385
|
+
private baseActiveCeiling(): { readonly ceiling: number; readonly source: ResourcePoolActiveCeilingSource } {
|
|
386
|
+
const soft = this.options.resourcePolicy?.softActiveCeiling(this.activePartitions());
|
|
387
|
+
if (soft === undefined || !Number.isFinite(soft) || soft <= this.maxActive) return { ceiling: this.maxActive, source: "configured" };
|
|
388
|
+
const clamped = Math.min(Math.floor(soft), this.absoluteMaxActive);
|
|
389
|
+
return { ceiling: clamped, source: clamped >= this.absoluteMaxActive ? "absolute-cap" : "resource-budget" };
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
status(): ResourcePoolStatus<Status> {
|
|
393
|
+
const byPartition: Record<string, number> = {};
|
|
394
|
+
let leased = 0;
|
|
395
|
+
for (const entry of this.entries.values()) {
|
|
396
|
+
byPartition[entry.partitionKey] = (byPartition[entry.partitionKey] ?? 0) + 1;
|
|
397
|
+
if (entry.activeLeases > 0) leased++;
|
|
398
|
+
}
|
|
399
|
+
const resources = this.options.resourcePolicy?.status(this.activePartitions());
|
|
400
|
+
return {
|
|
401
|
+
active: this.entries.size,
|
|
402
|
+
leased,
|
|
403
|
+
maxActive: this.maxActive,
|
|
404
|
+
effectiveMaxActive: this.lastEffectiveMaxActive,
|
|
405
|
+
activeCeilingSource: this.lastActiveCeilingSource,
|
|
406
|
+
absoluteMaxActive: this.absoluteMaxActive,
|
|
407
|
+
byPartition,
|
|
408
|
+
waitingBackgroundAdmissions: this.queuedBackgroundAdmissions,
|
|
409
|
+
...(resources !== undefined ? { resources } : {}),
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** Samples every currently active entry with a real costHandle and folds it into the configured recorder, if any -- a no-op without one. Read-only over the entry map, so it deliberately does not run inside serialized(). */
|
|
414
|
+
calibrateCosts(): void {
|
|
415
|
+
const recorder = this.options.costRecorder;
|
|
416
|
+
if (!recorder) return;
|
|
417
|
+
for (const entry of this.entries.values()) {
|
|
418
|
+
if (entry.resource.costHandle === undefined) continue;
|
|
419
|
+
recorder.recordSample(entry.partitionKey, entry.resource.costHandle);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** Unconditional force-close of every one of `ownerKey`'s resources, regardless of any active lease -- for the case of a remote resource swapped out from under an already-warm one, where correctness requires closing regardless of who still holds it. */
|
|
424
|
+
async closeOwner(ownerKey: OwnerKey): Promise<void> {
|
|
425
|
+
const stale = Array.from(this.entries.entries()).filter(([, entry]) => entry.ownerKey === ownerKey);
|
|
426
|
+
for (const [key] of stale) this.entries.delete(key);
|
|
427
|
+
await Promise.all(stale.map(([, entry]) => entry.resource.close()));
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** Unconditional force-close of one (ownerKey, partitionKey) resource, if any -- the single-partition sibling of closeOwner, for a caller that has already identified exactly which partition needs invalidating. */
|
|
431
|
+
async closePartition(ownerKey: OwnerKey, partitionKey: string): Promise<void> {
|
|
432
|
+
const key = this.key(ownerKey, partitionKey);
|
|
433
|
+
const entry = this.entries.get(key);
|
|
434
|
+
if (!entry) return;
|
|
435
|
+
this.entries.delete(key);
|
|
436
|
+
await entry.resource.close();
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* The safe sibling of closeOwner: refuses (does not evict anything) while any of this owner's
|
|
441
|
+
* resources has an active lease. Serialized against concurrent admission so a lease can't be
|
|
442
|
+
* granted between the check and the close.
|
|
443
|
+
*/
|
|
444
|
+
async releaseOwnerIfIdle(ownerKey: OwnerKey): Promise<{ readonly closed: number }> {
|
|
445
|
+
return this.serialized(async () => {
|
|
446
|
+
const matching = Array.from(this.entries.entries()).filter(([, entry]) => entry.ownerKey === ownerKey);
|
|
447
|
+
if (matching.some(([, entry]) => entry.activeLeases > 0)) throw new ResourceInUse(ownerKey);
|
|
448
|
+
for (const pair of matching) await this.evict(pair, "admission");
|
|
449
|
+
return { closed: matching.length };
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
async closeAll(): Promise<void> {
|
|
454
|
+
const entries = Array.from(this.entries.values());
|
|
455
|
+
this.entries.clear();
|
|
456
|
+
await Promise.all(entries.map((entry) => entry.resource.close()));
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
private async reconcileResourcesUnsafe(): Promise<number> {
|
|
460
|
+
const policy = this.options.resourcePolicy;
|
|
461
|
+
if (!policy) return 0;
|
|
462
|
+
let reaped = 0;
|
|
463
|
+
while (policy.isOverBudget(this.activePartitions())) {
|
|
464
|
+
const victim = this.leastRecentlyUsedIdle();
|
|
465
|
+
if (!victim) break;
|
|
466
|
+
try {
|
|
467
|
+
await this.evict(victim, "resource-pressure");
|
|
468
|
+
reaped++;
|
|
469
|
+
} catch {
|
|
470
|
+
break;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
return reaped;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
async reconcileResources(): Promise<number> {
|
|
477
|
+
return this.serialized(() => this.reconcileResourcesUnsafe());
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
async reapIdle(maxIdleMs: number): Promise<number> {
|
|
481
|
+
return this.serialized(async () => {
|
|
482
|
+
let reaped = await this.reconcileResourcesUnsafe();
|
|
483
|
+
const now = this.now();
|
|
484
|
+
const effectiveMaxIdleMs = this.options.resourcePolicy?.maxIdleMs(maxIdleMs, this.activePartitions()) ?? maxIdleMs;
|
|
485
|
+
const idle = Array.from(this.entries.entries()).filter(
|
|
486
|
+
([, entry]) => entry.activeLeases === 0 && now - entry.lastUsedAt > effectiveMaxIdleMs,
|
|
487
|
+
);
|
|
488
|
+
for (const [key, entry] of idle) {
|
|
489
|
+
try {
|
|
490
|
+
await entry.resource.close();
|
|
491
|
+
if (this.entries.get(key) === entry) this.entries.delete(key);
|
|
492
|
+
reaped++;
|
|
493
|
+
} catch (error) {
|
|
494
|
+
this.options.observe?.({
|
|
495
|
+
kind: "close-failed",
|
|
496
|
+
reason: "idle-reap",
|
|
497
|
+
partitionKey: entry.partitionKey,
|
|
498
|
+
errorName: error instanceof Error ? error.name : "UnknownError",
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
if (idle.length > 0) this.notifyAdmissionWaiters();
|
|
503
|
+
return reaped;
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** Raised when no idle resource can be evicted to admit a new one within the current ceiling. */
|
|
2
|
+
export class ResourceCapacityExceeded extends Error {
|
|
3
|
+
constructor(
|
|
4
|
+
readonly partitionKey: string,
|
|
5
|
+
readonly maxActive: number,
|
|
6
|
+
readonly partitionLimit: number,
|
|
7
|
+
) {
|
|
8
|
+
super(
|
|
9
|
+
`no idle resource can be evicted to admit partition "${partitionKey}" within global capacity ${maxActive} and partition capacity ${partitionLimit}`,
|
|
10
|
+
);
|
|
11
|
+
this.name = "ResourceCapacityExceeded";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Raised by releaseOwnerIfIdle when at least one of the owner's own resources still has an active lease. */
|
|
16
|
+
export class ResourceInUse extends Error {
|
|
17
|
+
constructor(readonly ownerKey: string) {
|
|
18
|
+
super(`cannot release owner "${ownerKey}": a pooled resource for it still has an active lease`);
|
|
19
|
+
this.name = "ResourceInUse";
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Raised when background admission is already waiting at maxQueuedBackgroundAdmissions -- fails fast rather than growing the wait queue without bound. */
|
|
24
|
+
export class ResourceAdmissionQueueFull extends Error {
|
|
25
|
+
constructor(
|
|
26
|
+
readonly partitionKey: string,
|
|
27
|
+
readonly maxQueued: number,
|
|
28
|
+
) {
|
|
29
|
+
super(`background admission for partition "${partitionKey}" is already waiting at capacity (${maxQueued} queued); retry later`);
|
|
30
|
+
this.name = "ResourceAdmissionQueueFull";
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Raised when a queued background admission waits past backgroundAdmissionQueueTimeoutMs without a slot freeing. */
|
|
35
|
+
export class ResourceAdmissionQueueTimedOut extends Error {
|
|
36
|
+
constructor(
|
|
37
|
+
readonly partitionKey: string,
|
|
38
|
+
readonly timeoutMs: number,
|
|
39
|
+
) {
|
|
40
|
+
super(
|
|
41
|
+
`background admission for partition "${partitionKey}" waited ${timeoutMs}ms for a resource-pool slot and gave up -- foreground demand is holding every admittable slot`,
|
|
42
|
+
);
|
|
43
|
+
this.name = "ResourceAdmissionQueueTimedOut";
|
|
44
|
+
}
|
|
45
|
+
}
|