@haverstack/server 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.d.ts +8 -0
- package/dist/app.d.ts.map +1 -0
- package/dist/app.js +70 -0
- package/dist/app.js.map +1 -0
- package/dist/config.d.ts +21 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +115 -0
- package/dist/config.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/changeStreams.d.ts +17 -0
- package/dist/lib/changeStreams.d.ts.map +1 -0
- package/{src/lib/changeStreams.ts → dist/lib/changeStreams.js} +12 -12
- package/dist/lib/changeStreams.js.map +1 -0
- package/dist/lib/frameGate.d.ts +26 -0
- package/dist/lib/frameGate.d.ts.map +1 -0
- package/dist/lib/frameGate.js +44 -0
- package/dist/lib/frameGate.js.map +1 -0
- package/dist/lib/json.d.ts +10 -0
- package/dist/lib/json.d.ts.map +1 -0
- package/dist/lib/json.js +27 -0
- package/dist/lib/json.js.map +1 -0
- package/dist/lib/nonceStore.d.ts +50 -0
- package/dist/lib/nonceStore.d.ts.map +1 -0
- package/dist/lib/nonceStore.js +96 -0
- package/dist/lib/nonceStore.js.map +1 -0
- package/dist/lib/queryLimit.d.ts +4 -0
- package/dist/lib/queryLimit.d.ts.map +1 -0
- package/{src/lib/queryLimit.ts → dist/lib/queryLimit.js} +5 -6
- package/dist/lib/queryLimit.js.map +1 -0
- package/dist/lib/queryWorker/pool.d.ts +60 -0
- package/dist/lib/queryWorker/pool.d.ts.map +1 -0
- package/dist/lib/queryWorker/pool.js +278 -0
- package/dist/lib/queryWorker/pool.js.map +1 -0
- package/{src/lib/queryWorker/protocol.ts → dist/lib/queryWorker/protocol.d.ts} +21 -18
- package/dist/lib/queryWorker/protocol.d.ts.map +1 -0
- package/dist/lib/queryWorker/protocol.js +2 -0
- package/dist/lib/queryWorker/protocol.js.map +1 -0
- package/dist/lib/queryWorker/worker.d.ts +2 -0
- package/dist/lib/queryWorker/worker.d.ts.map +1 -0
- package/{src/lib/queryWorker/worker.ts → dist/lib/queryWorker/worker.js} +19 -22
- package/dist/lib/queryWorker/worker.js.map +1 -0
- package/dist/lib/resumeBuffer.d.ts +143 -0
- package/dist/lib/resumeBuffer.d.ts.map +1 -0
- package/dist/lib/resumeBuffer.js +216 -0
- package/dist/lib/resumeBuffer.js.map +1 -0
- package/dist/lib/resumeCursor.d.ts +16 -0
- package/dist/lib/resumeCursor.d.ts.map +1 -0
- package/{src/lib/resumeCursor.ts → dist/lib/resumeCursor.js} +22 -24
- package/dist/lib/resumeCursor.js.map +1 -0
- package/dist/lib/tokenSweep.d.ts +17 -0
- package/dist/lib/tokenSweep.d.ts.map +1 -0
- package/dist/lib/tokenSweep.js +40 -0
- package/dist/lib/tokenSweep.js.map +1 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +46 -0
- package/dist/main.js.map +1 -0
- package/dist/middleware/auth.d.ts +25 -0
- package/dist/middleware/auth.d.ts.map +1 -0
- package/dist/middleware/auth.js +74 -0
- package/dist/middleware/auth.js.map +1 -0
- package/dist/middleware/errors.d.ts +17 -0
- package/dist/middleware/errors.d.ts.map +1 -0
- package/dist/middleware/errors.js +64 -0
- package/dist/middleware/errors.js.map +1 -0
- package/dist/routes/attachments.d.ts +5 -0
- package/dist/routes/attachments.d.ts.map +1 -0
- package/dist/routes/attachments.js +108 -0
- package/dist/routes/attachments.js.map +1 -0
- package/dist/routes/auth.d.ts +6 -0
- package/dist/routes/auth.d.ts.map +1 -0
- package/dist/routes/auth.js +110 -0
- package/dist/routes/auth.js.map +1 -0
- package/dist/routes/changes.d.ts +22 -0
- package/dist/routes/changes.d.ts.map +1 -0
- package/dist/routes/changes.js +233 -0
- package/dist/routes/changes.js.map +1 -0
- package/dist/routes/entity.d.ts +5 -0
- package/dist/routes/entity.d.ts.map +1 -0
- package/dist/routes/entity.js +30 -0
- package/dist/routes/entity.js.map +1 -0
- package/dist/routes/health.d.ts +4 -0
- package/dist/routes/health.d.ts.map +1 -0
- package/dist/routes/health.js +7 -0
- package/dist/routes/health.js.map +1 -0
- package/dist/routes/records.d.ts +5 -0
- package/dist/routes/records.d.ts.map +1 -0
- package/dist/routes/records.js +231 -0
- package/dist/routes/records.js.map +1 -0
- package/dist/routes/tokens.d.ts +5 -0
- package/dist/routes/tokens.d.ts.map +1 -0
- package/dist/routes/tokens.js +65 -0
- package/dist/routes/tokens.js.map +1 -0
- package/dist/routes/types.d.ts +5 -0
- package/dist/routes/types.d.ts.map +1 -0
- package/dist/routes/types.js +46 -0
- package/dist/routes/types.js.map +1 -0
- package/dist/routes/wellknown.d.ts +6 -0
- package/dist/routes/wellknown.d.ts.map +1 -0
- package/dist/routes/wellknown.js +45 -0
- package/dist/routes/wellknown.js.map +1 -0
- package/dist/shutdown.d.ts +23 -0
- package/dist/shutdown.d.ts.map +1 -0
- package/dist/shutdown.js +32 -0
- package/dist/shutdown.js.map +1 -0
- package/dist/stack.d.ts +20 -0
- package/dist/stack.d.ts.map +1 -0
- package/dist/stack.js +50 -0
- package/dist/stack.js.map +1 -0
- package/dist/testing.d.ts +49 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +130 -0
- package/dist/testing.js.map +1 -0
- package/{src/types.ts → dist/types.d.ts} +5 -5
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/{src/wireError.ts → dist/wireError.d.ts} +2 -9
- package/dist/wireError.d.ts.map +1 -0
- package/dist/wireError.js +13 -0
- package/dist/wireError.js.map +1 -0
- package/package.json +42 -17
- package/.claude/settings.local.json +0 -22
- package/.dockerignore +0 -10
- package/.env.example +0 -90
- package/.github/pull_request_template.md +0 -23
- package/.github/workflows/ci.yml +0 -55
- package/.github/workflows/release.yml +0 -99
- package/.prettierignore +0 -3
- package/.prettierrc +0 -7
- package/AGENTS.md +0 -91
- package/CONTRIBUTING.md +0 -234
- package/Dockerfile +0 -30
- package/SECURITY.md +0 -54
- package/docs/api.md +0 -277
- package/docs/deployment.md +0 -212
- package/eslint.config.mjs +0 -18
- package/pnpm-workspace.yaml +0 -13
- package/src/app.ts +0 -86
- package/src/config.ts +0 -176
- package/src/index.ts +0 -54
- package/src/lib/frameGate.ts +0 -42
- package/src/lib/json.ts +0 -27
- package/src/lib/nonceStore.ts +0 -103
- package/src/lib/queryWorker/pool.ts +0 -324
- package/src/lib/resumeBuffer.ts +0 -268
- package/src/lib/tokenSweep.ts +0 -43
- package/src/middleware/auth.ts +0 -77
- package/src/middleware/errors.ts +0 -73
- package/src/routes/attachments.ts +0 -126
- package/src/routes/auth.ts +0 -137
- package/src/routes/changes.ts +0 -279
- package/src/routes/entity.ts +0 -33
- package/src/routes/health.ts +0 -8
- package/src/routes/records.ts +0 -259
- package/src/routes/tokens.ts +0 -85
- package/src/routes/types.ts +0 -57
- package/src/routes/wellknown.ts +0 -55
- package/src/shutdown.ts +0 -56
- package/src/stack.ts +0 -96
- package/tests/app.test.ts +0 -86
- package/tests/changeFeedClient.ts +0 -156
- package/tests/config.test.ts +0 -112
- package/tests/conformance.test.ts +0 -2322
- package/tests/fixtures/blockingWorker.ts +0 -27
- package/tests/fixtures/brokenWorker.ts +0 -7
- package/tests/lib/changeFeedClient.test.ts +0 -126
- package/tests/lib/frameGate.test.ts +0 -93
- package/tests/lib/queryLimit.test.ts +0 -33
- package/tests/lib/queryWorker.test.ts +0 -60
- package/tests/lib/queryWorkerPool.test.ts +0 -129
- package/tests/lib/resumeBuffer.test.ts +0 -341
- package/tests/lib/resumeCursor.test.ts +0 -46
- package/tests/middleware/errors.test.ts +0 -176
- package/tests/routes/associations.test.ts +0 -170
- package/tests/routes/attachments.test.ts +0 -621
- package/tests/routes/auth.test.ts +0 -448
- package/tests/routes/changes.test.ts +0 -567
- package/tests/routes/contentQuery.test.ts +0 -68
- package/tests/routes/entity.test.ts +0 -170
- package/tests/routes/groups.test.ts +0 -69
- package/tests/routes/records.test.ts +0 -1965
- package/tests/routes/relatedToQuery.test.ts +0 -182
- package/tests/routes/tokens.test.ts +0 -157
- package/tests/routes/tombstones.test.ts +0 -93
- package/tests/routes/types.test.ts +0 -129
- package/tests/routes/versions.test.ts +0 -177
- package/tests/routes/wellknown.test.ts +0 -71
- package/tests/setup.ts +0 -154
- package/tests/shutdown.test.ts +0 -121
- package/tests/stack.test.ts +0 -198
- package/tsconfig.build.json +0 -9
- package/tsconfig.json +0 -19
- package/vitest.config.ts +0 -12
|
@@ -1,324 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Main-thread side of the query worker boundary. Owns a small, fixed-size
|
|
3
|
-
* pool of workers (src/lib/queryWorker/worker.ts), each with its own
|
|
4
|
-
* LocalAdapter connection to the same stack.db, and dispatches
|
|
5
|
-
* ScopedStack.query() calls to them. The operator-facing account of why
|
|
6
|
-
* this exists is docs/deployment.md § Bounding query cost.
|
|
7
|
-
*
|
|
8
|
-
* Two bounds, deliberately not one:
|
|
9
|
-
*
|
|
10
|
-
* - `deadlineMs` bounds **execution**, and its clock starts when a
|
|
11
|
-
* request reaches a worker, not when it was accepted here. Execution
|
|
12
|
-
* is the only part that needs bounding: node:sqlite exposes no
|
|
13
|
-
* `sqlite3_interrupt` equivalent, so a query already inside the engine
|
|
14
|
-
* cannot be cancelled in place. Queueing makes a request slow, never
|
|
15
|
-
* failed.
|
|
16
|
-
*
|
|
17
|
-
* - `queueLimit` bounds **the wait**, so "slow" cannot quietly become
|
|
18
|
-
* unbounded in latency or retained memory. Past it the pool sheds load
|
|
19
|
-
* with StackTimeoutError — a retryable 503, which is honest about
|
|
20
|
-
* overload in a way a silently growing queue is not.
|
|
21
|
-
*
|
|
22
|
-
* Past its deadline the only lever left is termination: the caller is
|
|
23
|
-
* answered at once and the worker is terminated and replaced, so a stuck
|
|
24
|
-
* query costs one slot rather than wedging the pool. That recycle needs no
|
|
25
|
-
* backoff, being rate-limited by construction at `poolSize` terminations
|
|
26
|
-
* per `deadlineMs`.
|
|
27
|
-
*
|
|
28
|
-
* An *unexpected* death is the opposite case and does back off: respawning
|
|
29
|
-
* a worker that can't boot — an unreadable DB_PATH, say — as fast as the
|
|
30
|
-
* loop allows would spin threads and flood logs. After
|
|
31
|
-
* MAX_CONSECUTIVE_SPAWN_FAILURES the pool reports itself unhealthy and
|
|
32
|
-
* fails queries with the underlying spawn error, so a fatal
|
|
33
|
-
* misconfiguration surfaces as itself rather than as a pile of timeouts.
|
|
34
|
-
* Respawns continue at the capped interval, so the pool heals once the
|
|
35
|
-
* cause clears.
|
|
36
|
-
*
|
|
37
|
-
* Terminating mid-write is safe: WAL journaling leaves an uncommitted
|
|
38
|
-
* transaction simply absent once the file is reopened. Only query() routes
|
|
39
|
-
* through this pool — src/routes/records.ts holds the reason writes and
|
|
40
|
-
* by-id reads stay on the main thread.
|
|
41
|
-
*/
|
|
42
|
-
import { Worker } from 'node:worker_threads';
|
|
43
|
-
import type { StackQuery, QueryResult, TokenSession } from '@haverstack/core';
|
|
44
|
-
import { StackTimeoutError } from '@haverstack/core';
|
|
45
|
-
import { deserializeError } from '@haverstack/wire-types';
|
|
46
|
-
import type { Logger } from 'pino';
|
|
47
|
-
import type { QueryRequest, QueryResponse, QueryWorkerInit } from './protocol.js';
|
|
48
|
-
|
|
49
|
-
// import.meta.url ends in .ts under tsx (dev) and vitest (test), and in
|
|
50
|
-
// .js once tsc has compiled this file into dist/ — so which worker file
|
|
51
|
-
// and loader to use falls out of how *this* module was loaded, with no
|
|
52
|
-
// separate NODE_ENV check to keep in sync.
|
|
53
|
-
const RUNNING_FROM_SOURCE = import.meta.url.endsWith('.ts');
|
|
54
|
-
const WORKER_URL = new URL(RUNNING_FROM_SOURCE ? './worker.ts' : './worker.js', import.meta.url);
|
|
55
|
-
const WORKER_EXEC_ARGV = RUNNING_FROM_SOURCE ? ['--import', 'tsx'] : [];
|
|
56
|
-
|
|
57
|
-
/** Backoff schedule for respawning a worker that died unexpectedly. */
|
|
58
|
-
const RESPAWN_BASE_DELAY_MS = 100;
|
|
59
|
-
const RESPAWN_MAX_DELAY_MS = 30_000;
|
|
60
|
-
/** Consecutive unexpected deaths after which the pool reports itself unhealthy. */
|
|
61
|
-
const MAX_CONSECUTIVE_SPAWN_FAILURES = 5;
|
|
62
|
-
|
|
63
|
-
type Pending = {
|
|
64
|
-
resolve: (result: QueryResult) => void;
|
|
65
|
-
reject: (err: Error) => void;
|
|
66
|
-
deadlineMs: number;
|
|
67
|
-
/** The execution clock. Null while the request is still queued. */
|
|
68
|
-
timer: NodeJS.Timeout | null;
|
|
69
|
-
};
|
|
70
|
-
|
|
71
|
-
type Slot = {
|
|
72
|
-
/** Null while the worker is dead and its respawn is pending. */
|
|
73
|
-
worker: Worker | null;
|
|
74
|
-
/** id of the request currently running on this worker, or null if idle. */
|
|
75
|
-
busyWith: number | null;
|
|
76
|
-
respawnTimer: NodeJS.Timeout | null;
|
|
77
|
-
};
|
|
78
|
-
|
|
79
|
-
export type QueryWorkerPoolOptions = {
|
|
80
|
-
init: QueryWorkerInit;
|
|
81
|
-
poolSize: number;
|
|
82
|
-
/** Maximum requests waiting for a slot before the pool sheds load. */
|
|
83
|
-
queueLimit: number;
|
|
84
|
-
logger: Logger;
|
|
85
|
-
/**
|
|
86
|
-
* Worker entry point. Defaults to this directory's worker; overridable
|
|
87
|
-
* so tests can drive the pool with a fixture worker whose timing they
|
|
88
|
-
* control, rather than inferring pool behavior from real query latency.
|
|
89
|
-
*/
|
|
90
|
-
workerUrl?: URL;
|
|
91
|
-
};
|
|
92
|
-
|
|
93
|
-
export class QueryWorkerPool {
|
|
94
|
-
private readonly slots: Slot[] = [];
|
|
95
|
-
private readonly pending = new Map<number, Pending>();
|
|
96
|
-
/** FIFO of requests waiting for a slot to free up. */
|
|
97
|
-
private readonly waiting: Array<{ id: number; req: QueryRequest }> = [];
|
|
98
|
-
private nextId = 1;
|
|
99
|
-
private closed = false;
|
|
100
|
-
private consecutiveFailures = 0;
|
|
101
|
-
private lastFailure: Error | null = null;
|
|
102
|
-
|
|
103
|
-
private readonly init: QueryWorkerInit;
|
|
104
|
-
private readonly queueLimit: number;
|
|
105
|
-
private readonly logger: Logger;
|
|
106
|
-
private readonly workerUrl: URL;
|
|
107
|
-
|
|
108
|
-
constructor(opts: QueryWorkerPoolOptions) {
|
|
109
|
-
this.init = opts.init;
|
|
110
|
-
this.queueLimit = opts.queueLimit;
|
|
111
|
-
this.logger = opts.logger;
|
|
112
|
-
this.workerUrl = opts.workerUrl ?? WORKER_URL;
|
|
113
|
-
for (let i = 0; i < opts.poolSize; i++) {
|
|
114
|
-
const slot: Slot = { worker: null, busyWith: null, respawnTimer: null };
|
|
115
|
-
this.slots.push(slot);
|
|
116
|
-
this.startWorker(slot);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* True once every recent spawn attempt has failed — a standing fault
|
|
122
|
-
* (bad DB_PATH, unreadable file) rather than one unlucky crash.
|
|
123
|
-
*/
|
|
124
|
-
private get unhealthy(): boolean {
|
|
125
|
-
return this.consecutiveFailures >= MAX_CONSECUTIVE_SPAWN_FAILURES;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
private startWorker(slot: Slot): void {
|
|
129
|
-
if (this.closed) return;
|
|
130
|
-
slot.respawnTimer = null;
|
|
131
|
-
const worker = new Worker(this.workerUrl, {
|
|
132
|
-
workerData: this.init,
|
|
133
|
-
execArgv: WORKER_EXEC_ARGV,
|
|
134
|
-
});
|
|
135
|
-
slot.worker = worker;
|
|
136
|
-
slot.busyWith = null;
|
|
137
|
-
worker.on('message', (msg: QueryResponse) => this.handleMessage(slot, msg));
|
|
138
|
-
worker.on('error', (err: Error) => this.handleWorkerDeath(slot, err));
|
|
139
|
-
worker.on('exit', (code) => {
|
|
140
|
-
if (!this.closed && code !== 0) {
|
|
141
|
-
this.handleWorkerDeath(slot, new Error(`query worker exited with code ${code}`));
|
|
142
|
-
}
|
|
143
|
-
});
|
|
144
|
-
// postMessage before a worker finishes booting is fine — worker_threads
|
|
145
|
-
// queues it until the thread installs its listener.
|
|
146
|
-
this.drainWaiting(slot);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
private handleMessage(slot: Slot, msg: QueryResponse): void {
|
|
150
|
-
// A worker that answers is a working worker: clear any standing
|
|
151
|
-
// failure state so an earlier rough patch can't leave the pool
|
|
152
|
-
// permanently marked unhealthy.
|
|
153
|
-
this.consecutiveFailures = 0;
|
|
154
|
-
this.lastFailure = null;
|
|
155
|
-
slot.busyWith = null;
|
|
156
|
-
const pending = this.pending.get(msg.id);
|
|
157
|
-
// Already timed out (and rejected) before the worker answered — the
|
|
158
|
-
// late reply just frees the slot, above.
|
|
159
|
-
if (pending) {
|
|
160
|
-
this.pending.delete(msg.id);
|
|
161
|
-
if (pending.timer) clearTimeout(pending.timer);
|
|
162
|
-
if (msg.ok) pending.resolve(msg.result);
|
|
163
|
-
else pending.reject(msg.wire ? deserializeError(msg.wire.body) : new Error(msg.message));
|
|
164
|
-
}
|
|
165
|
-
this.drainWaiting(slot);
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
/**
|
|
169
|
-
* A worker crashed or exited on its own — not a deadline recycle, which
|
|
170
|
-
* is deliberate and handled by recycleSlot() without backoff.
|
|
171
|
-
*/
|
|
172
|
-
private handleWorkerDeath(slot: Slot, err: Error): void {
|
|
173
|
-
const failedId = slot.busyWith;
|
|
174
|
-
if (failedId !== null) {
|
|
175
|
-
const pending = this.pending.get(failedId);
|
|
176
|
-
if (pending) {
|
|
177
|
-
this.pending.delete(failedId);
|
|
178
|
-
if (pending.timer) clearTimeout(pending.timer);
|
|
179
|
-
pending.reject(err);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
this.teardown(slot);
|
|
183
|
-
this.consecutiveFailures++;
|
|
184
|
-
this.lastFailure = err;
|
|
185
|
-
const delay = Math.min(
|
|
186
|
-
RESPAWN_BASE_DELAY_MS * 2 ** (this.consecutiveFailures - 1),
|
|
187
|
-
RESPAWN_MAX_DELAY_MS,
|
|
188
|
-
);
|
|
189
|
-
this.logger.error(
|
|
190
|
-
{ err, consecutiveFailures: this.consecutiveFailures, retryInMs: delay },
|
|
191
|
-
'Query worker died unexpectedly; respawning after backoff',
|
|
192
|
-
);
|
|
193
|
-
// Nothing is going to serve the queue any time soon — answer it with
|
|
194
|
-
// the real cause instead of letting it wait out a fault that isn't
|
|
195
|
-
// going to clear on its own.
|
|
196
|
-
if (this.unhealthy) this.failQueued(err);
|
|
197
|
-
this.scheduleRespawn(slot, delay);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/** Deliberate recycle after a deadline: capacity is restored at once, no backoff. */
|
|
201
|
-
private recycleSlot(slot: Slot): void {
|
|
202
|
-
if (this.closed) return;
|
|
203
|
-
this.teardown(slot);
|
|
204
|
-
this.startWorker(slot);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
private teardown(slot: Slot): void {
|
|
208
|
-
const worker = slot.worker;
|
|
209
|
-
slot.worker = null;
|
|
210
|
-
slot.busyWith = null;
|
|
211
|
-
if (!worker) return;
|
|
212
|
-
worker.removeAllListeners();
|
|
213
|
-
void worker.terminate().catch(() => {});
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
private scheduleRespawn(slot: Slot, delay: number): void {
|
|
217
|
-
if (this.closed) return;
|
|
218
|
-
if (slot.respawnTimer) clearTimeout(slot.respawnTimer);
|
|
219
|
-
slot.respawnTimer = setTimeout(() => this.startWorker(slot), delay);
|
|
220
|
-
// A pending respawn must not by itself keep the process alive.
|
|
221
|
-
slot.respawnTimer.unref();
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
private failQueued(err: Error): void {
|
|
225
|
-
for (const { id } of this.waiting.splice(0)) {
|
|
226
|
-
const pending = this.pending.get(id);
|
|
227
|
-
if (!pending) continue;
|
|
228
|
-
this.pending.delete(id);
|
|
229
|
-
if (pending.timer) clearTimeout(pending.timer);
|
|
230
|
-
pending.reject(err);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
private drainWaiting(slot: Slot): void {
|
|
235
|
-
if (this.closed || !slot.worker || slot.busyWith !== null) return;
|
|
236
|
-
// Looped, not recursive: a long run of already-abandoned queue entries
|
|
237
|
-
// shouldn't cost stack depth.
|
|
238
|
-
for (;;) {
|
|
239
|
-
const next = this.waiting.shift();
|
|
240
|
-
if (!next) return;
|
|
241
|
-
const pending = this.pending.get(next.id);
|
|
242
|
-
// Abandoned while queued (pool went unhealthy, or close()).
|
|
243
|
-
if (!pending) continue;
|
|
244
|
-
this.startExecution(slot, next.id, next.req, pending);
|
|
245
|
-
return;
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
/** Hand a request to a worker and start its execution clock — see the module doc. */
|
|
250
|
-
private startExecution(slot: Slot, id: number, req: QueryRequest, pending: Pending): void {
|
|
251
|
-
pending.timer = setTimeout(() => this.onDeadline(id), pending.deadlineMs);
|
|
252
|
-
slot.busyWith = id;
|
|
253
|
-
slot.worker!.postMessage(req);
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
private onDeadline(id: number): void {
|
|
257
|
-
const pending = this.pending.get(id);
|
|
258
|
-
if (!pending) return;
|
|
259
|
-
this.pending.delete(id);
|
|
260
|
-
pending.reject(
|
|
261
|
-
new StackTimeoutError(`Query exceeded the ${pending.deadlineMs}ms execution deadline`),
|
|
262
|
-
);
|
|
263
|
-
const slot = this.slots.find((s) => s.busyWith === id);
|
|
264
|
-
if (slot) this.recycleSlot(slot);
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
/**
|
|
268
|
-
* Run a query on the pool. `deadlineMs` bounds execution only; a request
|
|
269
|
-
* that waits for a slot waits without penalty, until the queue itself is
|
|
270
|
-
* full.
|
|
271
|
-
*/
|
|
272
|
-
query(
|
|
273
|
-
session: TokenSession | null,
|
|
274
|
-
stackQuery: StackQuery,
|
|
275
|
-
deadlineMs: number,
|
|
276
|
-
): Promise<QueryResult> {
|
|
277
|
-
if (this.closed) return Promise.reject(new Error('Query worker pool is closed'));
|
|
278
|
-
// A standing spawn fault is a server fault, not a slow query: report
|
|
279
|
-
// the cause rather than making every caller wait out a deadline.
|
|
280
|
-
if (this.unhealthy) {
|
|
281
|
-
return Promise.reject(this.lastFailure ?? new Error('Query worker pool is unavailable'));
|
|
282
|
-
}
|
|
283
|
-
if (this.waiting.length >= this.queueLimit) {
|
|
284
|
-
return Promise.reject(
|
|
285
|
-
new StackTimeoutError(
|
|
286
|
-
`Query queue is full (${this.queueLimit} waiting); the server is shedding load rather than queueing further`,
|
|
287
|
-
),
|
|
288
|
-
);
|
|
289
|
-
}
|
|
290
|
-
const id = this.nextId++;
|
|
291
|
-
const req: QueryRequest = { id, session, query: stackQuery };
|
|
292
|
-
return new Promise<QueryResult>((resolve, reject) => {
|
|
293
|
-
const pending: Pending = { resolve, reject, deadlineMs, timer: null };
|
|
294
|
-
this.pending.set(id, pending);
|
|
295
|
-
const idle = this.slots.find((s) => s.worker !== null && s.busyWith === null);
|
|
296
|
-
if (idle) this.startExecution(idle, id, req, pending);
|
|
297
|
-
else this.waiting.push({ id, req });
|
|
298
|
-
});
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
async close(): Promise<void> {
|
|
302
|
-
this.closed = true;
|
|
303
|
-
for (const pending of this.pending.values()) {
|
|
304
|
-
if (pending.timer) clearTimeout(pending.timer);
|
|
305
|
-
pending.reject(new Error('Query worker pool is closing'));
|
|
306
|
-
}
|
|
307
|
-
this.pending.clear();
|
|
308
|
-
this.waiting.length = 0;
|
|
309
|
-
await Promise.all(
|
|
310
|
-
this.slots.map(async (slot) => {
|
|
311
|
-
if (slot.respawnTimer) {
|
|
312
|
-
clearTimeout(slot.respawnTimer);
|
|
313
|
-
slot.respawnTimer = null;
|
|
314
|
-
}
|
|
315
|
-
const worker = slot.worker;
|
|
316
|
-
slot.worker = null;
|
|
317
|
-
slot.busyWith = null;
|
|
318
|
-
if (!worker) return;
|
|
319
|
-
worker.removeAllListeners();
|
|
320
|
-
await worker.terminate();
|
|
321
|
-
}),
|
|
322
|
-
);
|
|
323
|
-
}
|
|
324
|
-
}
|
package/src/lib/resumeBuffer.ts
DELETED
|
@@ -1,268 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Per-(session, filter) resume buffers for GET /changes. Sits downstream
|
|
3
|
-
* of each connection's `ScopedStack.subscribe()`, so every entry here is a
|
|
4
|
-
* frame that subscription already delivered — something this session was
|
|
5
|
-
* cleared to see at the moment it happened. That is what keeps replay a
|
|
6
|
-
* pure re-send needing nothing from core, where a single server-wide
|
|
7
|
-
* buffer would owe a permission check per frame per reader.
|
|
8
|
-
*
|
|
9
|
-
* A buffer mints its own cursors (its position is a private counter, not a
|
|
10
|
-
* slice of some global sequence) and is fed independently of any one HTTP
|
|
11
|
-
* connection: it keeps collecting for a bounded window *after* the last
|
|
12
|
-
* connection using it disconnects, which is what lets a reconnect learn
|
|
13
|
-
* about changes made while nobody was listening. Depth-bounded on top of
|
|
14
|
-
* that: a session that stays away long enough to overflow the ring gets
|
|
15
|
-
* `overflow` rather than a silently incomplete replay.
|
|
16
|
-
*/
|
|
17
|
-
import type { ChangeFilter, RecordChange, Unsubscribe } from '@haverstack/core';
|
|
18
|
-
import { serializeChange } from '@haverstack/wire-types';
|
|
19
|
-
import type { WireRecordChange } from '@haverstack/wire-types';
|
|
20
|
-
import { encodeCursor } from './resumeCursor.js';
|
|
21
|
-
|
|
22
|
-
export type ResumeEntry = {
|
|
23
|
-
n: number;
|
|
24
|
-
recordId: string;
|
|
25
|
-
isPurge: boolean;
|
|
26
|
-
/** Wire-ready, seq already attached. */
|
|
27
|
-
frame: WireRecordChange;
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
export type EntriesAfterResult =
|
|
31
|
-
| { status: 'ok'; entries: ResumeEntry[] }
|
|
32
|
-
/** The buffer is the right one, but depth eviction already dropped what would be needed. */
|
|
33
|
-
| { status: 'overflow' }
|
|
34
|
-
/** A position this buffer never held, or no longer holds any record of. */
|
|
35
|
-
| { status: 'cursor_expired' };
|
|
36
|
-
|
|
37
|
-
/** One session+filter's resume buffer. Data and notification only — lifecycle lives in the registry. */
|
|
38
|
-
export class ResumeBuffer {
|
|
39
|
-
readonly id: string = crypto.randomUUID();
|
|
40
|
-
private readonly entries: ResumeEntry[] = [];
|
|
41
|
-
private currentN = 0;
|
|
42
|
-
private evictedUpTo = 0;
|
|
43
|
-
private readonly listeners = new Set<(entry: ResumeEntry) => void>();
|
|
44
|
-
|
|
45
|
-
/** Connections currently holding this buffer open. Registry-managed. */
|
|
46
|
-
liveCount = 0;
|
|
47
|
-
/** Set by the registry when liveCount drops to 0; cleared on reacquire. */
|
|
48
|
-
disconnectedAt: number | null = null;
|
|
49
|
-
|
|
50
|
-
constructor(private readonly depth: number) {}
|
|
51
|
-
|
|
52
|
-
/** The cursor naming this buffer's current position — what `ready` reports. */
|
|
53
|
-
headCursor(): string {
|
|
54
|
-
return encodeCursor(this.id, this.currentN);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Record one already-permission-filtered change, minting its cursor.
|
|
59
|
-
* Notifies every currently attached live connection with the same frame
|
|
60
|
-
* a replay would later produce.
|
|
61
|
-
*/
|
|
62
|
-
append(change: RecordChange): ResumeEntry {
|
|
63
|
-
this.currentN += 1;
|
|
64
|
-
const seq = encodeCursor(this.id, this.currentN);
|
|
65
|
-
const frame = serializeChange({ ...change, seq });
|
|
66
|
-
const entry: ResumeEntry = {
|
|
67
|
-
n: this.currentN,
|
|
68
|
-
recordId: change.recordId,
|
|
69
|
-
isPurge: change.kind === 'purged',
|
|
70
|
-
frame,
|
|
71
|
-
};
|
|
72
|
-
this.entries.push(entry);
|
|
73
|
-
if (this.entries.length > this.depth) {
|
|
74
|
-
const dropped = this.entries.shift();
|
|
75
|
-
if (dropped) this.evictedUpTo = dropped.n;
|
|
76
|
-
}
|
|
77
|
-
for (const listener of this.listeners) listener(entry);
|
|
78
|
-
return entry;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/** Attach a live connection. Returns the detach function. */
|
|
82
|
-
subscribeLive(listener: (entry: ResumeEntry) => void): () => void {
|
|
83
|
-
this.listeners.add(listener);
|
|
84
|
-
return () => {
|
|
85
|
-
this.listeners.delete(listener);
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Everything after position `n`, or why that can't be answered. `n` is
|
|
91
|
-
* this server's own minted position, already decoded and matched to
|
|
92
|
-
* this buffer's id by the caller.
|
|
93
|
-
*/
|
|
94
|
-
entriesAfter(n: number): EntriesAfterResult {
|
|
95
|
-
if (n > this.currentN || n < 0) return { status: 'cursor_expired' };
|
|
96
|
-
if (n < this.evictedUpTo) return { status: 'overflow' };
|
|
97
|
-
return { status: 'ok', entries: this.entries.filter((e) => e.n > n) };
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
export type ResumeBufferKeyParts = {
|
|
102
|
-
/** Both null for an anonymous connection. */
|
|
103
|
-
principalId: string | null;
|
|
104
|
-
subjectId: string | null;
|
|
105
|
-
filter: ChangeFilter;
|
|
106
|
-
includeRecords: boolean;
|
|
107
|
-
includeUnlisted: boolean;
|
|
108
|
-
};
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* Reproducible across a reconnect that re-sends the same query params —
|
|
112
|
-
* order-independent on the parts of `filter` that don't carry order of
|
|
113
|
-
* their own (`typeId`/`kinds` are matched as sets, not sequences, so
|
|
114
|
-
* re-sending them in a different order must key the same buffer).
|
|
115
|
-
*
|
|
116
|
-
* Two filters that mean different things must never key the same buffer.
|
|
117
|
-
* A buffer opens exactly one `ScopedStack.subscribe()`, carrying the
|
|
118
|
-
* filter of whichever connection minted it, so a collision hands one of
|
|
119
|
-
* the two colliding connections the *other's* filter: either a silent gap
|
|
120
|
-
* (events it asked for and never hears about, with no `reset` to tell it
|
|
121
|
-
* so) or frames outside its filter, which "filtering is exact, not
|
|
122
|
-
* advisory" forbids. See docs/spec/wire-format.md § Change feed.
|
|
123
|
-
*
|
|
124
|
-
* Hence an object rather than a positional array: `JSON.stringify` drops
|
|
125
|
-
* an undefined *property* but renders an undefined *array element* as
|
|
126
|
-
* `null`, which would collapse "no parentId filter" onto `parentId: null`
|
|
127
|
-
* — "root records only", a filter the wire format defines. An absent part
|
|
128
|
-
* is an absent key, which no present value can imitate. Key order is fixed
|
|
129
|
-
* by construction below.
|
|
130
|
-
*/
|
|
131
|
-
export function resumeBufferKey(parts: ResumeBufferKeyParts): string {
|
|
132
|
-
const { filter } = parts;
|
|
133
|
-
const typeId = filter.typeId
|
|
134
|
-
? [...(Array.isArray(filter.typeId) ? filter.typeId : [filter.typeId])].sort()
|
|
135
|
-
: undefined;
|
|
136
|
-
const kinds = filter.kinds ? [...filter.kinds].sort() : undefined;
|
|
137
|
-
return JSON.stringify({
|
|
138
|
-
principalId: parts.principalId,
|
|
139
|
-
subjectId: parts.subjectId,
|
|
140
|
-
includeRecords: parts.includeRecords,
|
|
141
|
-
includeUnlisted: parts.includeUnlisted,
|
|
142
|
-
...(typeId !== undefined && { typeId }),
|
|
143
|
-
...(filter.parentId !== undefined && { parentId: filter.parentId }),
|
|
144
|
-
...(filter.entityId !== undefined && { entityId: filter.entityId }),
|
|
145
|
-
...(kinds !== undefined && { kinds }),
|
|
146
|
-
});
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
export type ResumeBufferRegistryOptions = {
|
|
150
|
-
/** Max entries retained per buffer before the oldest is dropped. */
|
|
151
|
-
depth: number;
|
|
152
|
-
/** How long a buffer keeps being fed after its last connection disconnects, before it's dropped entirely. */
|
|
153
|
-
retentionMs: number;
|
|
154
|
-
};
|
|
155
|
-
|
|
156
|
-
/**
|
|
157
|
-
* Owns every live resume buffer, keyed by (session, filter, includeRecords).
|
|
158
|
-
* A buffer's underlying `ScopedStack.subscribe()` is opened once, when the
|
|
159
|
-
* buffer is first minted, and kept open — including across a gap with zero
|
|
160
|
-
* live connections — until the buffer itself is evicted.
|
|
161
|
-
*/
|
|
162
|
-
export class ResumeBufferRegistry {
|
|
163
|
-
private readonly buffers = new Map<string, ResumeBuffer>();
|
|
164
|
-
private readonly unsubscribes = new Map<string, Unsubscribe>();
|
|
165
|
-
private readonly evictionTimers = new Map<string, NodeJS.Timeout>();
|
|
166
|
-
/**
|
|
167
|
-
* Buffers whose subscription is still opening. A key lands in `buffers`
|
|
168
|
-
* only once its `ScopedStack.subscribe()` has resolved, so this is what
|
|
169
|
-
* keeps two connections arriving in the same tick from opening two
|
|
170
|
-
* subscriptions for one key — they await the same attempt and share
|
|
171
|
-
* whichever way it settles.
|
|
172
|
-
*/
|
|
173
|
-
private readonly acquiring = new Map<string, Promise<ResumeBuffer>>();
|
|
174
|
-
|
|
175
|
-
constructor(private readonly opts: ResumeBufferRegistryOptions) {}
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* Get this key's buffer, creating one and opening its scoped subscription
|
|
179
|
-
* if none is currently retained. `subscribeToStack` is called at most
|
|
180
|
-
* once per buffer instance.
|
|
181
|
-
*
|
|
182
|
-
* A failed subscription registers nothing: the next connection on this
|
|
183
|
-
* key retries it. Registering a buffer that nothing is feeding would
|
|
184
|
-
* hand every later connection on that key a `ready` frame followed by
|
|
185
|
-
* permanent silence — a gap with no `reset` to announce it, which is
|
|
186
|
-
* the one failure the feed contract calls untrustworthy.
|
|
187
|
-
*/
|
|
188
|
-
async acquire(
|
|
189
|
-
key: string,
|
|
190
|
-
subscribeToStack: (onChange: (change: RecordChange) => void) => Promise<Unsubscribe>,
|
|
191
|
-
): Promise<ResumeBuffer> {
|
|
192
|
-
const existing = this.buffers.get(key);
|
|
193
|
-
if (existing) {
|
|
194
|
-
this.cancelEviction(key);
|
|
195
|
-
existing.liveCount += 1;
|
|
196
|
-
existing.disconnectedAt = null;
|
|
197
|
-
return existing;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
const inFlight = this.acquiring.get(key);
|
|
201
|
-
if (inFlight) {
|
|
202
|
-
const buffer = await inFlight;
|
|
203
|
-
buffer.liveCount += 1;
|
|
204
|
-
return buffer;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
const attempt = (async () => {
|
|
208
|
-
const buffer = new ResumeBuffer(this.opts.depth);
|
|
209
|
-
// Awaited before either map is touched — the handler closes over
|
|
210
|
-
// `buffer`, so anything the stack emits while this is in flight is
|
|
211
|
-
// already being recorded by the time the buffer becomes reachable.
|
|
212
|
-
const unsubscribe = await subscribeToStack((change) => {
|
|
213
|
-
buffer.append(change);
|
|
214
|
-
});
|
|
215
|
-
this.buffers.set(key, buffer);
|
|
216
|
-
this.unsubscribes.set(key, unsubscribe);
|
|
217
|
-
return buffer;
|
|
218
|
-
})();
|
|
219
|
-
this.acquiring.set(key, attempt);
|
|
220
|
-
|
|
221
|
-
try {
|
|
222
|
-
const buffer = await attempt;
|
|
223
|
-
buffer.liveCount += 1;
|
|
224
|
-
return buffer;
|
|
225
|
-
} finally {
|
|
226
|
-
this.acquiring.delete(key);
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/** Detach one connection from this key's buffer, starting its retention countdown once none remain. */
|
|
231
|
-
release(key: string): void {
|
|
232
|
-
const buffer = this.buffers.get(key);
|
|
233
|
-
if (!buffer) return;
|
|
234
|
-
buffer.liveCount = Math.max(0, buffer.liveCount - 1);
|
|
235
|
-
if (buffer.liveCount > 0) return;
|
|
236
|
-
buffer.disconnectedAt = Date.now();
|
|
237
|
-
const timer = setTimeout(() => this.evict(key, buffer), this.opts.retentionMs);
|
|
238
|
-
timer.unref?.();
|
|
239
|
-
this.evictionTimers.set(key, timer);
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
private cancelEviction(key: string): void {
|
|
243
|
-
const timer = this.evictionTimers.get(key);
|
|
244
|
-
if (timer) {
|
|
245
|
-
clearTimeout(timer);
|
|
246
|
-
this.evictionTimers.delete(key);
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
private evict(key: string, buffer: ResumeBuffer): void {
|
|
251
|
-
if (this.buffers.get(key) !== buffer) return; // already replaced
|
|
252
|
-
if (buffer.liveCount > 0) return; // reacquired since the timer was set
|
|
253
|
-
this.buffers.delete(key);
|
|
254
|
-
this.evictionTimers.delete(key);
|
|
255
|
-
this.unsubscribes.get(key)?.();
|
|
256
|
-
this.unsubscribes.delete(key);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
/** Tear down every retained buffer immediately — for shutdown. */
|
|
260
|
-
closeAll(): void {
|
|
261
|
-
for (const timer of this.evictionTimers.values()) clearTimeout(timer);
|
|
262
|
-
this.evictionTimers.clear();
|
|
263
|
-
this.acquiring.clear();
|
|
264
|
-
for (const unsubscribe of this.unsubscribes.values()) unsubscribe();
|
|
265
|
-
this.unsubscribes.clear();
|
|
266
|
-
this.buffers.clear();
|
|
267
|
-
}
|
|
268
|
-
}
|
package/src/lib/tokenSweep.ts
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import type { Logger } from 'pino';
|
|
2
|
-
import type { StackTokenStore } from '@haverstack/core/wire';
|
|
3
|
-
|
|
4
|
-
// Once an hour at most. The sweep is O(n) in tokens ever issued — cheap
|
|
5
|
-
// relative to that interval, and never on the hot path twice in a row.
|
|
6
|
-
const SWEEP_INTERVAL_MS = 60 * 60 * 1000;
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Delete tokens past their `expiresAt`.
|
|
10
|
-
*
|
|
11
|
-
* `lookupToken()` already refuses an expired token, so this changes no
|
|
12
|
-
* authorization outcome — it reclaims rows, and nothing else does: the
|
|
13
|
-
* store deletes only on an explicit `revokeToken()`. Since `POST
|
|
14
|
-
* /auth/token` is unauthenticated, strangers drive the row count, and
|
|
15
|
-
* unbounded growth costs disk and slows every `listTokens()` caller.
|
|
16
|
-
*
|
|
17
|
-
* Returns a throttled sweep — call it freely; it runs at most once per
|
|
18
|
-
* interval and never rejects, since failing to reclaim rows must not fail
|
|
19
|
-
* the request that happened to trigger it.
|
|
20
|
-
*/
|
|
21
|
-
export function createExpiredTokenSweeper(
|
|
22
|
-
tokens: StackTokenStore,
|
|
23
|
-
logger: Logger,
|
|
24
|
-
): () => Promise<void> {
|
|
25
|
-
let lastSweptAt = 0;
|
|
26
|
-
return async () => {
|
|
27
|
-
const now = Date.now();
|
|
28
|
-
if (now - lastSweptAt < SWEEP_INTERVAL_MS) return;
|
|
29
|
-
// Claimed before the await, so two concurrent requests can't both start
|
|
30
|
-
// a sweep and race each other's revokeToken() calls.
|
|
31
|
-
lastSweptAt = now;
|
|
32
|
-
try {
|
|
33
|
-
const all = await tokens.listTokens();
|
|
34
|
-
const expired = all.filter((t) => t.expiresAt !== undefined && t.expiresAt.getTime() <= now);
|
|
35
|
-
for (const t of expired) await tokens.revokeToken(t.id);
|
|
36
|
-
if (expired.length > 0) {
|
|
37
|
-
logger.info({ revoked: expired.length }, 'Swept expired tokens');
|
|
38
|
-
}
|
|
39
|
-
} catch (err) {
|
|
40
|
-
logger.warn({ err }, 'Expired-token sweep failed');
|
|
41
|
-
}
|
|
42
|
-
};
|
|
43
|
-
}
|