@supalive/core 1.14.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index-B4UxN1ZL.d.ts +2365 -0
- package/dist/index-B4UxN1ZL.d.ts.map +1 -0
- package/dist/index-B8JFAyw4.d.ts +2377 -0
- package/dist/index-B8JFAyw4.d.ts.map +1 -0
- package/dist/index-CfcrWLLm.d.ts +2377 -0
- package/dist/index-CfcrWLLm.d.ts.map +1 -0
- package/dist/index-Ddr4NiPf.d.ts +2295 -0
- package/dist/index-Ddr4NiPf.d.ts.map +1 -0
- package/dist/index-ggi1DrSG.d.ts +2336 -0
- package/dist/index-ggi1DrSG.d.ts.map +1 -0
- package/dist/mysql-07ROfbBZ.d.ts +114 -0
- package/dist/mysql-07ROfbBZ.d.ts.map +1 -0
- package/dist/mysql-BTigT6mP.d.ts +114 -0
- package/dist/mysql-BTigT6mP.d.ts.map +1 -0
- package/dist/mysql-CXymz1Eu.d.ts +114 -0
- package/dist/mysql-CXymz1Eu.d.ts.map +1 -0
- package/dist/mysql-ClUK7S2j.js +623 -0
- package/dist/mysql-ClUK7S2j.js.map +1 -0
- package/dist/mysql-CtpWqOhO.d.ts +114 -0
- package/dist/mysql-CtpWqOhO.d.ts.map +1 -0
- package/dist/mysql-DWj2q3qR.d.ts +114 -0
- package/dist/mysql-DWj2q3qR.d.ts.map +1 -0
- package/dist/mysql-fTKZ1zaF.js +623 -0
- package/dist/mysql-fTKZ1zaF.js.map +1 -0
- package/dist/overlap-checker-1TwhkAwy.js +259 -0
- package/dist/overlap-checker-1TwhkAwy.js.map +1 -0
- package/dist/overlap-checker-CGlxWcyb.js +259 -0
- package/dist/overlap-checker-CGlxWcyb.js.map +1 -0
- package/dist/postgres-B5-5vgLJ.d.ts +118 -0
- package/dist/postgres-B5-5vgLJ.d.ts.map +1 -0
- package/dist/postgres-CN03Espg.js +868 -0
- package/dist/postgres-CN03Espg.js.map +1 -0
- package/dist/postgres-CRPJdZx2.js +868 -0
- package/dist/postgres-CRPJdZx2.js.map +1 -0
- package/dist/postgres-D0M1ri-C.d.ts +118 -0
- package/dist/postgres-D0M1ri-C.d.ts.map +1 -0
- package/dist/postgres-DCXnKaJr.d.ts +118 -0
- package/dist/postgres-DCXnKaJr.d.ts.map +1 -0
- package/dist/postgres-DgCYPf0j.d.ts +118 -0
- package/dist/postgres-DgCYPf0j.d.ts.map +1 -0
- package/dist/postgres-Dnshux6k.d.ts +118 -0
- package/dist/postgres-Dnshux6k.d.ts.map +1 -0
- package/dist/query-LJranz0c.js +795 -0
- package/dist/query-LJranz0c.js.map +1 -0
- package/dist/query-q1ZvKrZO.js +762 -0
- package/dist/query-q1ZvKrZO.js.map +1 -0
- package/dist/realtime_db-B0Tin87E.js +312 -0
- package/dist/realtime_db-B0Tin87E.js.map +1 -0
- package/dist/realtime_db-DVhhasJj.js +312 -0
- package/dist/realtime_db-DVhhasJj.js.map +1 -0
- package/dist/router-DP2ThAwh.js +274 -0
- package/dist/router-DP2ThAwh.js.map +1 -0
- package/dist/router-Dfnwka7K.js +337 -0
- package/dist/router-Dfnwka7K.js.map +1 -0
- package/dist/src/client/index.d.ts +2 -2
- package/dist/src/client/index.js +79 -17
- package/dist/src/client/index.js.map +1 -1
- package/dist/src/exports/mysql.d.ts +1 -1
- package/dist/src/exports/mysql.js +1 -1
- package/dist/src/exports/postgres.d.ts +1 -1
- package/dist/src/exports/postgres.js +1 -1
- package/dist/src/exports/procedure.d.ts +12 -9
- package/dist/src/exports/procedure.d.ts.map +1 -1
- package/dist/src/exports/procedure.js +65 -6
- package/dist/src/exports/procedure.js.map +1 -1
- package/dist/src/exports/schema-sql.d.ts +1 -1
- package/dist/src/exports/server.d.ts +12 -6
- package/dist/src/exports/server.d.ts.map +1 -1
- package/dist/src/exports/server.js +793 -253
- package/dist/src/exports/server.js.map +1 -1
- package/dist/src/exports/sub-manager-worker-entry.js +1 -1
- package/dist/src/exports/types.d.ts +2 -2
- package/dist/src/exports/types.js +1 -1
- package/dist/sub-worker-dispatch-D7jfw6ys.js +826 -0
- package/dist/sub-worker-dispatch-D7jfw6ys.js.map +1 -0
- package/dist/sub-worker-dispatch-qiSs5Xkm.js +826 -0
- package/dist/sub-worker-dispatch-qiSs5Xkm.js.map +1 -0
- package/dist/types_client_rpc-CWZmlWmM.js +145 -0
- package/dist/types_client_rpc-CWZmlWmM.js.map +1 -0
- package/dist/types_client_rpc-D4td5W0Y.js +141 -0
- package/dist/types_client_rpc-D4td5W0Y.js.map +1 -0
- package/dist/types_server-BSiGxbqb.d.ts +523 -0
- package/dist/types_server-BSiGxbqb.d.ts.map +1 -0
- package/dist/types_server-BhC19R2a.js.map +1 -1
- package/dist/types_server-BmIgDd8d.d.ts +614 -0
- package/dist/types_server-BmIgDd8d.d.ts.map +1 -0
- package/dist/types_server-C6B9_b1R.d.ts +662 -0
- package/dist/types_server-C6B9_b1R.d.ts.map +1 -0
- package/dist/types_server-DRr5yKy1.d.ts +655 -0
- package/dist/types_server-DRr5yKy1.d.ts.map +1 -0
- package/dist/types_server-DmCbve3w.d.ts +684 -0
- package/dist/types_server-DmCbve3w.d.ts.map +1 -0
- package/dist/types_server-Gs_vSVaS.d.ts +655 -0
- package/dist/types_server-Gs_vSVaS.d.ts.map +1 -0
- package/dist/types_server-ILUmPFvk.d.ts +614 -0
- package/dist/types_server-ILUmPFvk.d.ts.map +1 -0
- package/package.json +2 -1
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { l as primaryOnlyConn, n as getContextRegistry } from "../../router-DP2ThAwh.js";
|
|
2
2
|
import { n as stableStringify, r as supaliveStringify, t as groupByToMap } from "../../helper-zdJT5FUc.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { s as traceCacheCall } from "../../query-LJranz0c.js";
|
|
4
|
+
import { r as evaluateCacheFreshness } from "../../overlap-checker-1TwhkAwy.js";
|
|
5
|
+
import { i as AuthenticationError, o as ClientMessageSchema } from "../../types_client_rpc-CWZmlWmM.js";
|
|
5
6
|
import { n as RemoteSubscriptionMessageSchema } from "../../types_server-BhC19R2a.js";
|
|
6
|
-
import { n as sleep } from "../../realtime_db-
|
|
7
|
+
import { n as sleep } from "../../realtime_db-B0Tin87E.js";
|
|
8
|
+
import { C as appInstanceId, D as serializeMapValue, E as logger, S as RetentionWatermark, T as logLevelOf, _ as UnregisterSubscriptionResultSchema, a as bootstrapDbTypes, b as UpdateSubscriptionReadSetResultSchema, c as initCore, d as InvalidateWritesetParamsSchema, f as InvalidateWritesetResultSchema, g as UnregisterSubscriptionParamsSchema, h as RegisterSubscriptionResultSchema, i as DB_QUERY_TIMEOUT_MS, l as initDatabase, m as RegisterSubscriptionParamsSchema, n as SubscriptionWorker, o as createDatabase, p as RegisterSubscriptionBatchResultSchema, r as CORE_VERSION, s as initCacheLayer, t as dispatchWorkerMessage, u as CacheLayer, v as UnregisterSubscriptionsResultSchema, w as isProduction, x as RETENTION_WATERMARK_CHANNEL, y as UpdateSubscriptionReadSetParamsSchema } from "../../sub-worker-dispatch-qiSs5Xkm.js";
|
|
7
9
|
import { n as resolveReadRouting, t as makeConnRouting } from "../../read-routing-BUcQAwnd.js";
|
|
8
|
-
import {
|
|
10
|
+
import { ROOT_CONTEXT, SpanKind, SpanStatusCode, context, isSpanContextValid, metrics, propagation, trace } from "@opentelemetry/api";
|
|
9
11
|
import { v4 } from "uuid";
|
|
10
12
|
import { createServer } from "http";
|
|
11
13
|
import { WebSocket, WebSocketServer } from "ws";
|
|
@@ -72,6 +74,251 @@ async function generateCacheKey(queryName, input, queryIdentity) {
|
|
|
72
74
|
return `ck_${await getHashOf(`${queryName}_${typeof input === "string" ? input : stableStringify(input)}_${queryIdentity}`)}`;
|
|
73
75
|
}
|
|
74
76
|
//#endregion
|
|
77
|
+
//#region src/observability.ts
|
|
78
|
+
let rootLogger = logger;
|
|
79
|
+
/** Replace the process-wide root logger. Call once at startup, before creating
|
|
80
|
+
* the server, to route Supalive's request/lifecycle logs through your own pino
|
|
81
|
+
* instance. */
|
|
82
|
+
function configureRootLogger(logger) {
|
|
83
|
+
rootLogger = logger;
|
|
84
|
+
}
|
|
85
|
+
/** The current root logger (defaults to the env-configured singleton). */
|
|
86
|
+
function getRootLogger() {
|
|
87
|
+
return rootLogger;
|
|
88
|
+
}
|
|
89
|
+
const CLIENT_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
90
|
+
"NOT_FOUND",
|
|
91
|
+
"BAD_REQUEST",
|
|
92
|
+
"INVALID_MESSAGE",
|
|
93
|
+
"PARSE_ERROR",
|
|
94
|
+
"UNAUTHENTICATED",
|
|
95
|
+
"RATE_LIMITED",
|
|
96
|
+
"METHOD_NOT_ALLOWED",
|
|
97
|
+
"TOO_MANY_CONCURRENT_MUTATIONS",
|
|
98
|
+
"TOO_MANY_CONCURRENT_ACTIONS",
|
|
99
|
+
"SUBSCRIBE_ERROR"
|
|
100
|
+
]);
|
|
101
|
+
/**
|
|
102
|
+
* The bound observability surface for one server instance: an injectable logger,
|
|
103
|
+
* OTel metric instruments, and the `record()` seam every operation funnels
|
|
104
|
+
* through. Built once in the server constructor and shared with the HTTP RPC
|
|
105
|
+
* handler.
|
|
106
|
+
*/
|
|
107
|
+
var Observability = class {
|
|
108
|
+
logger;
|
|
109
|
+
requestLog;
|
|
110
|
+
onEvent;
|
|
111
|
+
requestCounter;
|
|
112
|
+
durationHistogram;
|
|
113
|
+
activeConnections;
|
|
114
|
+
activeSubscriptions;
|
|
115
|
+
tracer;
|
|
116
|
+
constructor(opts = {}) {
|
|
117
|
+
const base = opts.logger ?? getRootLogger();
|
|
118
|
+
this.logger = opts.level ? base.child({}, { level: opts.level }) : base;
|
|
119
|
+
this.requestLog = opts.requestLog ?? "all";
|
|
120
|
+
this.onEvent = opts.onEvent;
|
|
121
|
+
const meterName = opts.meterName ?? "@supalive/core";
|
|
122
|
+
const meter = metrics.getMeter(meterName, opts.meterVersion);
|
|
123
|
+
this.tracer = trace.getTracer(meterName, opts.meterVersion);
|
|
124
|
+
this.requestCounter = meter.createCounter("supalive_requests_total", { description: "Total Supalive operations (WS calls, subscribes, HTTP RPC, jobs)." });
|
|
125
|
+
this.durationHistogram = meter.createHistogram("supalive_request_duration_ms", {
|
|
126
|
+
description: "Operation duration in milliseconds.",
|
|
127
|
+
unit: "ms"
|
|
128
|
+
});
|
|
129
|
+
this.activeConnections = meter.createUpDownCounter("supalive_active_connections", { description: "Currently open WebSocket connections." });
|
|
130
|
+
this.activeSubscriptions = meter.createUpDownCounter("supalive_active_subscriptions", { description: "Currently registered live-query subscriptions on this instance." });
|
|
131
|
+
}
|
|
132
|
+
/** Record one completed operation: metrics + optional onEvent + one log line,
|
|
133
|
+
* and finalize the op's span (status/attributes/exception) when present. */
|
|
134
|
+
record(e) {
|
|
135
|
+
const traceId = this.finalizeSpan(e);
|
|
136
|
+
if (traceId) (e.extra ??= {}).traceId = traceId;
|
|
137
|
+
const attrs = {
|
|
138
|
+
transport: e.transport,
|
|
139
|
+
procedure: e.procedure ?? "",
|
|
140
|
+
kind: e.kind ?? "",
|
|
141
|
+
outcome: e.outcome,
|
|
142
|
+
code: e.code ?? ""
|
|
143
|
+
};
|
|
144
|
+
this.requestCounter.add(1, attrs);
|
|
145
|
+
this.durationHistogram.record(e.durationMs, attrs);
|
|
146
|
+
if (this.onEvent) try {
|
|
147
|
+
this.onEvent(e);
|
|
148
|
+
} catch {}
|
|
149
|
+
if (this.requestLog === "off") return;
|
|
150
|
+
if (e.outcome === "ok") {
|
|
151
|
+
if (this.requestLog !== "all") return;
|
|
152
|
+
this.logger.info(logFields(e), summary(e));
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
const fields = logFields(e);
|
|
156
|
+
if (!e.error && e.code != null && CLIENT_ERROR_CODES.has(e.code)) this.logger.warn(fields, summary(e));
|
|
157
|
+
else this.logger.error({
|
|
158
|
+
...fields,
|
|
159
|
+
err: e.error
|
|
160
|
+
}, summary(e));
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Start a SERVER span for an operation, as a child of `parent` (or the active
|
|
164
|
+
* context). No-op — returns a non-recording span — until a TracerProvider is
|
|
165
|
+
* registered by the app, so this costs nothing when tracing is disabled.
|
|
166
|
+
* Returns the span plus a context that has it active, for `context.with(...)`
|
|
167
|
+
* wrapping so nested handler work / `ctx.span` children attach correctly.
|
|
168
|
+
*/
|
|
169
|
+
startSpan(name, opts = {}) {
|
|
170
|
+
const parent = opts.parent ?? context.active();
|
|
171
|
+
const span = this.tracer.startSpan(name, {
|
|
172
|
+
kind: SpanKind.SERVER,
|
|
173
|
+
attributes: {
|
|
174
|
+
...opts.transport ? { "supalive.transport": opts.transport } : {},
|
|
175
|
+
...opts.procedure ? { "supalive.procedure": opts.procedure } : {},
|
|
176
|
+
...opts.kind ? { "supalive.kind": opts.kind } : {}
|
|
177
|
+
}
|
|
178
|
+
}, parent);
|
|
179
|
+
return {
|
|
180
|
+
span,
|
|
181
|
+
context: trace.setSpan(parent, span)
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Continue a trace from an inbound carrier (HTTP headers, or a WS message's
|
|
186
|
+
* `trace` field). Returns a context to pass as `startSpan({ parent })`. When
|
|
187
|
+
* no propagator is registered / no trace context is present, returns the root
|
|
188
|
+
* context (a fresh trace).
|
|
189
|
+
*/
|
|
190
|
+
extractContext(carrier) {
|
|
191
|
+
return propagation.extract(ROOT_CONTEXT, carrier);
|
|
192
|
+
}
|
|
193
|
+
/** Set the span's final status/attributes from the event, then end it.
|
|
194
|
+
* Returns the (valid) trace id for log correlation, or undefined. */
|
|
195
|
+
finalizeSpan(e) {
|
|
196
|
+
const span = e.span;
|
|
197
|
+
if (!span) return void 0;
|
|
198
|
+
try {
|
|
199
|
+
span.setAttribute("supalive.outcome", e.outcome);
|
|
200
|
+
if (e.code) span.setAttribute("supalive.code", e.code);
|
|
201
|
+
if (e.procedure) span.setAttribute("supalive.procedure", e.procedure);
|
|
202
|
+
if (e.kind) span.setAttribute("supalive.kind", e.kind);
|
|
203
|
+
if (e.outcome === "error") {
|
|
204
|
+
if (e.error instanceof Error) span.recordException(e.error);
|
|
205
|
+
span.setStatus({
|
|
206
|
+
code: SpanStatusCode.ERROR,
|
|
207
|
+
message: e.code
|
|
208
|
+
});
|
|
209
|
+
} else span.setStatus({ code: SpanStatusCode.OK });
|
|
210
|
+
} finally {
|
|
211
|
+
span.end();
|
|
212
|
+
}
|
|
213
|
+
const sc = span.spanContext();
|
|
214
|
+
return isSpanContextValid(sc) ? sc.traceId : void 0;
|
|
215
|
+
}
|
|
216
|
+
/** A request-scoped child logger for `ctx.log`, bound with the given fields
|
|
217
|
+
* (reqId, procedure, and trace id when tracing is on). */
|
|
218
|
+
childLogger(fields) {
|
|
219
|
+
return this.logger.child(fields);
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* A LAZY request-scoped `ctx.log`. Most handlers never log, and pino's
|
|
223
|
+
* `.child()` (bindings merge + serialization) isn't free — so this defers the
|
|
224
|
+
* child creation until the first `debug/info/warn/error` call and reuses it
|
|
225
|
+
* after. When a handler logs nothing, no child logger is ever allocated; the
|
|
226
|
+
* only per-request cost is this small delegator object. Returns the narrow
|
|
227
|
+
* {@link HandlerLogger} surface the handler context exposes.
|
|
228
|
+
*/
|
|
229
|
+
lazyChildLogger(fields) {
|
|
230
|
+
const base = this.logger;
|
|
231
|
+
let child;
|
|
232
|
+
const get = () => child ??= base.child(fields);
|
|
233
|
+
return {
|
|
234
|
+
debug: (...args) => get().debug(...args),
|
|
235
|
+
info: (...args) => get().info(...args),
|
|
236
|
+
warn: (...args) => get().warn(...args),
|
|
237
|
+
error: (...args) => get().error(...args)
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
/** The span's trace id, or undefined when tracing is off / the span is
|
|
241
|
+
* non-recording. For binding onto `ctx.log`. */
|
|
242
|
+
traceIdOf(span) {
|
|
243
|
+
if (!span) return void 0;
|
|
244
|
+
const sc = span.spanContext();
|
|
245
|
+
return isSpanContextValid(sc) ? sc.traceId : void 0;
|
|
246
|
+
}
|
|
247
|
+
connectionOpened() {
|
|
248
|
+
this.activeConnections.add(1);
|
|
249
|
+
}
|
|
250
|
+
connectionClosed() {
|
|
251
|
+
this.activeConnections.add(-1);
|
|
252
|
+
}
|
|
253
|
+
subscriptionAdded(n = 1) {
|
|
254
|
+
this.activeSubscriptions.add(n);
|
|
255
|
+
}
|
|
256
|
+
subscriptionRemoved(n = 1) {
|
|
257
|
+
this.activeSubscriptions.add(-n);
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
/** Start marker for an operation; pass to {@link elapsedMs} to get its duration. */
|
|
261
|
+
function startTimer() {
|
|
262
|
+
return performance.now();
|
|
263
|
+
}
|
|
264
|
+
function elapsedMs(startedAt) {
|
|
265
|
+
return Math.round((performance.now() - startedAt) * 1e3) / 1e3;
|
|
266
|
+
}
|
|
267
|
+
const coreTracer = trace.getTracer("@supalive/core");
|
|
268
|
+
/**
|
|
269
|
+
* Run a query handler inside a `query <procedureName>` span, nested under
|
|
270
|
+
* `parentContext` (the operation span) or the active context. The span is handed
|
|
271
|
+
* to `run` so it becomes the query handler's `ctx.span` and the parent of any DB
|
|
272
|
+
* spans. Cache HITs never call this (no execution), so a span appearing means a
|
|
273
|
+
* real recompute — including the otherwise-invisible background reactive path,
|
|
274
|
+
* where `parentContext` is omitted and the span is a root.
|
|
275
|
+
*
|
|
276
|
+
* No-op-cheap when tracing is off: the started span is non-recording, so we skip
|
|
277
|
+
* the `context.with` and just run.
|
|
278
|
+
*/
|
|
279
|
+
function traceQueryExecution(procedureName, trigger, parentContext, run) {
|
|
280
|
+
const parent = parentContext ?? context.active();
|
|
281
|
+
const span = coreTracer.startSpan(`query ${procedureName}`, { attributes: {
|
|
282
|
+
"supalive.kind": "query",
|
|
283
|
+
"supalive.procedure": procedureName,
|
|
284
|
+
"supalive.trigger": trigger
|
|
285
|
+
} }, parent);
|
|
286
|
+
const exec = async () => {
|
|
287
|
+
try {
|
|
288
|
+
const r = await run(span);
|
|
289
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
290
|
+
return r;
|
|
291
|
+
} catch (err) {
|
|
292
|
+
if (err instanceof Error) span.recordException(err);
|
|
293
|
+
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
294
|
+
throw err;
|
|
295
|
+
} finally {
|
|
296
|
+
span.end();
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
return span.isRecording() ? context.with(trace.setSpan(parent, span), exec) : exec();
|
|
300
|
+
}
|
|
301
|
+
function logFields(e) {
|
|
302
|
+
const f = {
|
|
303
|
+
op: e.op,
|
|
304
|
+
transport: e.transport,
|
|
305
|
+
outcome: e.outcome,
|
|
306
|
+
durMs: e.durationMs
|
|
307
|
+
};
|
|
308
|
+
if (e.procedure) f.procedure = e.procedure;
|
|
309
|
+
if (e.kind) f.kind = e.kind;
|
|
310
|
+
if (e.code) f.code = e.code;
|
|
311
|
+
if (e.reqId) f.reqId = e.reqId;
|
|
312
|
+
if (e.sessionId) f.sessionId = e.sessionId;
|
|
313
|
+
if (e.userId != null) f.userId = e.userId;
|
|
314
|
+
if (e.extra) Object.assign(f, e.extra);
|
|
315
|
+
return f;
|
|
316
|
+
}
|
|
317
|
+
function summary(e) {
|
|
318
|
+
const label = e.procedure ? `${e.kind ?? e.op} ${e.procedure}` : e.op;
|
|
319
|
+
return `${e.transport} ${label} ${e.outcome}`;
|
|
320
|
+
}
|
|
321
|
+
//#endregion
|
|
75
322
|
//#region src/server/one-shot-query.ts
|
|
76
323
|
/**
|
|
77
324
|
* Execute a one-shot query (WS `call` or HTTP RPC) with the same caching model
|
|
@@ -108,14 +355,24 @@ async function generateCacheKey(queryName, input, queryIdentity) {
|
|
|
108
355
|
* `ts` the result reflects, which the caller echoes so the client can advance
|
|
109
356
|
* its high-water.
|
|
110
357
|
*/
|
|
111
|
-
async function runCachedQuery(deps, procedure, procedureName, input, serverCtx, lastSnapshotTs, clientMinTs) {
|
|
358
|
+
async function runCachedQuery(deps, procedure, procedureName, input, serverCtx, lastSnapshotTs, clientMinTs, handlerObs) {
|
|
112
359
|
const queryIdentity = resolveQueryIdentity(procedure, serverCtx, input, deps.getUserId);
|
|
113
360
|
const cacheKey = await generateCacheKey(procedureName, stableStringify(input), queryIdentity);
|
|
114
361
|
const routing = await resolveReadRouting(deps.db, lastSnapshotTs, clientMinTs ?? 0n);
|
|
115
362
|
const readTs = routing.readTs;
|
|
116
|
-
const metadata = await
|
|
117
|
-
|
|
118
|
-
|
|
363
|
+
const { metadata, freshness } = await traceCacheCall("freshness", async () => {
|
|
364
|
+
const metadata = await deps.cache.getQueryCacheMetaData(cacheKey);
|
|
365
|
+
return {
|
|
366
|
+
metadata,
|
|
367
|
+
freshness: await evaluateCacheFreshness(routing.readImpl, metadata, readTs, deps.getMinRetainedTs())
|
|
368
|
+
};
|
|
369
|
+
});
|
|
370
|
+
if (freshness.fresh) {
|
|
371
|
+
const cached = await traceCacheCall("get", async (setHit) => {
|
|
372
|
+
const c = await deps.cache.getQueryCacheFor(cacheKey);
|
|
373
|
+
setHit(!!(c && metadata && c.version === metadata.version));
|
|
374
|
+
return c;
|
|
375
|
+
});
|
|
119
376
|
if (cached && metadata && cached.version === metadata.version) {
|
|
120
377
|
if (readTs > metadata.lastSnapshotTs) deps.cache.advanceQueryCacheMetadata(cacheKey, {
|
|
121
378
|
lastSnapshotTs: readTs,
|
|
@@ -134,11 +391,16 @@ async function runCachedQuery(deps, procedure, procedureName, input, serverCtx,
|
|
|
134
391
|
beginTs: readTs
|
|
135
392
|
}, async (ctx) => {
|
|
136
393
|
const fn = procedure.fn;
|
|
137
|
-
return
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
394
|
+
return traceQueryExecution(procedureName, "request", void 0, (querySpan) => {
|
|
395
|
+
const qctx = {
|
|
396
|
+
db: ctx,
|
|
397
|
+
serverCtx,
|
|
398
|
+
...makeConnRouting(ctx, routing),
|
|
399
|
+
log: handlerObs?.log,
|
|
400
|
+
span: querySpan
|
|
401
|
+
};
|
|
402
|
+
return fn(qctx, input);
|
|
403
|
+
});
|
|
142
404
|
});
|
|
143
405
|
await deps.cache.setQueryCacheAndMetadataFor(cacheKey, result.data, {
|
|
144
406
|
lastSnapshotTs: result.ts,
|
|
@@ -152,7 +414,7 @@ async function runCachedQuery(deps, procedure, procedureName, input, serverCtx,
|
|
|
152
414
|
}
|
|
153
415
|
//#endregion
|
|
154
416
|
//#region src/server/sub-manager-client.ts
|
|
155
|
-
const logger$
|
|
417
|
+
const logger$3 = logger.child({ name: "SUBS_MGR_CLIENT" }, { level: logLevelOf("SUB_MANAGER_CLIENT_LOG_LEVEL") });
|
|
156
418
|
const subscriptionManagerDataPack = {
|
|
157
419
|
encode: (data) => supaliveStringify(data),
|
|
158
420
|
decode: (data) => JSON.parse(data)
|
|
@@ -209,7 +471,7 @@ var SubManagerClient = class {
|
|
|
209
471
|
this.client.dataPack = subscriptionManagerDataPack;
|
|
210
472
|
this.client.on("open", this.handleOpen.bind(this));
|
|
211
473
|
this.client.on("close", this.handleClose.bind(this));
|
|
212
|
-
this.client.on("error", (err) => logger$
|
|
474
|
+
this.client.on("error", (err) => logger$3.warn(err, "sub-manager client socket error"));
|
|
213
475
|
}
|
|
214
476
|
/**
|
|
215
477
|
* Wire up the app-server-driven recovery. Must be called before any
|
|
@@ -270,7 +532,7 @@ var SubManagerClient = class {
|
|
|
270
532
|
appendToBuffer(writeSet) {
|
|
271
533
|
if (this.bufferedWriteSets.length + writeSet.length > this.bufferLimit) {
|
|
272
534
|
if (!this.bufferDroppedDuringOutage) {
|
|
273
|
-
logger$
|
|
535
|
+
logger$3.warn(`Invalidate buffer would exceed ${this.bufferLimit}; dropping buffered entries. Subscriptions will be caught up via commit-log replay on re-register.`);
|
|
274
536
|
this.bufferDroppedDuringOutage = true;
|
|
275
537
|
}
|
|
276
538
|
this.bufferedWriteSets = [];
|
|
@@ -287,7 +549,7 @@ var SubManagerClient = class {
|
|
|
287
549
|
const result = await this.call("invalidateWriteset", { writeSet }, InvalidateWritesetResultSchema);
|
|
288
550
|
if (this.affectedHandler && result.affected.length > 0) await this.affectedHandler(result.affected);
|
|
289
551
|
} catch (err) {
|
|
290
|
-
logger$
|
|
552
|
+
logger$3.error(err, "Failed to flush invalidate buffer after recovery");
|
|
291
553
|
}
|
|
292
554
|
}
|
|
293
555
|
markBusy() {
|
|
@@ -303,28 +565,28 @@ var SubManagerClient = class {
|
|
|
303
565
|
this.resolveReady();
|
|
304
566
|
}
|
|
305
567
|
handleClose() {
|
|
306
|
-
logger$
|
|
568
|
+
logger$3.info("sub-manager client: socket closed");
|
|
307
569
|
this.markBusy();
|
|
308
570
|
}
|
|
309
571
|
handleOpen() {
|
|
310
572
|
if (!this.hasConnectedOnce) {
|
|
311
573
|
this.hasConnectedOnce = true;
|
|
312
|
-
logger$
|
|
574
|
+
logger$3.info("sub-manager client: initial connect");
|
|
313
575
|
this.markReady();
|
|
314
576
|
return;
|
|
315
577
|
}
|
|
316
|
-
logger$
|
|
578
|
+
logger$3.info("sub-manager client: reconnected — starting recovery");
|
|
317
579
|
this.runRecovery();
|
|
318
580
|
}
|
|
319
581
|
async runRecovery() {
|
|
320
582
|
try {
|
|
321
583
|
if (this.recoveryDriver) await this.recoveryDriver();
|
|
322
|
-
else logger$
|
|
584
|
+
else logger$3.warn("sub-manager client: no recoveryDriver configured — skipping recovery step");
|
|
323
585
|
await this.flushBuffer();
|
|
324
586
|
} catch (err) {
|
|
325
|
-
logger$
|
|
587
|
+
logger$3.error(err, "Recovery failed; state will be marked ready anyway");
|
|
326
588
|
} finally {
|
|
327
|
-
logger$
|
|
589
|
+
logger$3.info("sub-manager client: recovery complete");
|
|
328
590
|
this.markReady();
|
|
329
591
|
}
|
|
330
592
|
}
|
|
@@ -342,7 +604,7 @@ var SubManagerClient = class {
|
|
|
342
604
|
};
|
|
343
605
|
//#endregion
|
|
344
606
|
//#region src/jobs/scheduler.ts
|
|
345
|
-
const logger$
|
|
607
|
+
const logger$2 = logger.child({ name: "SCHEDULER" }, { level: logLevelOf("SCHEDULER_LOG_LEVEL") });
|
|
346
608
|
function delayMsFrom(opts) {
|
|
347
609
|
if (opts?.runAt) return Math.max(0, opts.runAt.getTime() - Date.now());
|
|
348
610
|
if (opts?.delaySeconds && opts.delaySeconds > 0) return opts.delaySeconds * 1e3;
|
|
@@ -372,18 +634,18 @@ var DevScheduler = class {
|
|
|
372
634
|
for (const job of jobs) {
|
|
373
635
|
const endpoint = endpointFor(job.name);
|
|
374
636
|
const timer = setInterval(() => {
|
|
375
|
-
this.post(endpoint, {}).catch((err) => logger$
|
|
637
|
+
this.post(endpoint, {}).catch((err) => logger$2.error(err, `dev recurring job "${job.name}" failed`));
|
|
376
638
|
}, this.intervalMs);
|
|
377
639
|
timer.unref?.();
|
|
378
640
|
this.intervals.set(job.name, timer);
|
|
379
|
-
logger$
|
|
641
|
+
logger$2.info(`dev schedule "${job.name}" (cron "${job.cron}") → firing every ${this.intervalMs}ms`);
|
|
380
642
|
}
|
|
381
643
|
}
|
|
382
644
|
async enqueue(name, body, endpoint, opts) {
|
|
383
645
|
const id = crypto.randomUUID();
|
|
384
646
|
const timer = setTimeout(() => {
|
|
385
647
|
this.timeouts.delete(id);
|
|
386
|
-
this.post(endpoint, body).catch((err) => logger$
|
|
648
|
+
this.post(endpoint, body).catch((err) => logger$2.error(err, `dev one-shot job "${name}" failed`));
|
|
387
649
|
}, delayMsFrom(opts));
|
|
388
650
|
timer.unref?.();
|
|
389
651
|
this.timeouts.set(id, timer);
|
|
@@ -441,7 +703,7 @@ var QStashScheduler = class {
|
|
|
441
703
|
url: req.url
|
|
442
704
|
});
|
|
443
705
|
} catch (err) {
|
|
444
|
-
logger$
|
|
706
|
+
logger$2.warn(err, `signature verification failed for job "${req.name}"`);
|
|
445
707
|
return false;
|
|
446
708
|
}
|
|
447
709
|
}
|
|
@@ -451,15 +713,15 @@ var QStashScheduler = class {
|
|
|
451
713
|
try {
|
|
452
714
|
existing = await this.client.schedules.list();
|
|
453
715
|
} catch (err) {
|
|
454
|
-
logger$
|
|
716
|
+
logger$2.error(err, "failed to list QStash schedules; skipping reconcile of stale schedules");
|
|
455
717
|
}
|
|
456
718
|
for (const schedule of existing) {
|
|
457
719
|
const id = schedule.scheduleId;
|
|
458
720
|
if (id?.startsWith(SCHEDULE_ID_PREFIX) && !desired.has(id)) try {
|
|
459
721
|
await this.client.schedules.delete(id);
|
|
460
|
-
logger$
|
|
722
|
+
logger$2.info(`removed stale QStash schedule ${id}`);
|
|
461
723
|
} catch (err) {
|
|
462
|
-
logger$
|
|
724
|
+
logger$2.error(err, `failed to remove stale schedule ${id}`);
|
|
463
725
|
}
|
|
464
726
|
}
|
|
465
727
|
for (const [scheduleId, job] of desired) try {
|
|
@@ -468,9 +730,9 @@ var QStashScheduler = class {
|
|
|
468
730
|
destination: endpointFor(job.name),
|
|
469
731
|
cron: job.cron
|
|
470
732
|
});
|
|
471
|
-
logger$
|
|
733
|
+
logger$2.info(`synced QStash schedule "${job.name}" (${job.cron})`);
|
|
472
734
|
} catch (err) {
|
|
473
|
-
logger$
|
|
735
|
+
logger$2.error(err, `failed to sync schedule for job "${job.name}"`);
|
|
474
736
|
throw err;
|
|
475
737
|
}
|
|
476
738
|
}
|
|
@@ -557,7 +819,6 @@ var FixedWindowRateLimiter = class {
|
|
|
557
819
|
};
|
|
558
820
|
//#endregion
|
|
559
821
|
//#region src/server/http-rpc.ts
|
|
560
|
-
const logger$3 = logger.child({ name: "HTTP_RPC" }, { level: logLevelOf("SERVER_LOG_LEVEL") });
|
|
561
822
|
/**
|
|
562
823
|
* Handles the one-shot HTTP RPC transport: a single `POST {path}` endpoint for
|
|
563
824
|
* `query`/`mutation`/`action`. The procedure name travels in the request body
|
|
@@ -573,44 +834,87 @@ const logger$3 = logger.child({ name: "HTTP_RPC" }, { level: logLevelOf("SERVER_
|
|
|
573
834
|
*/
|
|
574
835
|
var HttpRpcHandler = class {
|
|
575
836
|
deps;
|
|
837
|
+
/** The RPC endpoint path (always set; claimed even when disabled so a call is
|
|
838
|
+
* explicitly rejected rather than silently falling through). */
|
|
576
839
|
path;
|
|
577
840
|
rpc;
|
|
841
|
+
rpcEnabled;
|
|
578
842
|
rateLimiter;
|
|
843
|
+
/** Job webhook prefix (`{jobPath}/`) — a job POST is `{jobPath}/<jobName>`. */
|
|
844
|
+
jobPrefix;
|
|
579
845
|
constructor(deps) {
|
|
580
846
|
this.deps = deps;
|
|
581
847
|
this.rpc = deps.rpc;
|
|
848
|
+
this.rpcEnabled = deps.rpcEnabled;
|
|
582
849
|
this.path = normalizeRpcPath(deps.rpc.path ?? "/_rpc");
|
|
583
|
-
if (deps.rpc.rateLimit !== false) {
|
|
850
|
+
if (this.rpcEnabled && deps.rpc.rateLimit !== false) {
|
|
584
851
|
const rl = deps.rpc.rateLimit ?? {};
|
|
585
852
|
this.rateLimiter = new FixedWindowRateLimiter(rl.windowMs ?? 6e4, rl.max ?? 120);
|
|
586
853
|
}
|
|
854
|
+
this.jobPrefix = `${deps.jobPath}/`;
|
|
587
855
|
}
|
|
588
|
-
/** True when this handler owns the given request path
|
|
589
|
-
* carried in the body now, so a single exact path is the
|
|
856
|
+
/** True when this handler owns the given request path as an RPC call. The
|
|
857
|
+
* procedure name is carried in the body now, so a single exact path is the
|
|
858
|
+
* whole RPC surface. Claims the path even when disabled so {@link handle}
|
|
859
|
+
* rejects the call explicitly. */
|
|
590
860
|
handlesPath(path) {
|
|
591
861
|
return path === this.path;
|
|
592
862
|
}
|
|
863
|
+
/** True when the path is a job webhook (`{jobPath}/<name>`). */
|
|
864
|
+
handlesJobPath(path) {
|
|
865
|
+
return path.startsWith(this.jobPrefix);
|
|
866
|
+
}
|
|
593
867
|
stop() {
|
|
594
868
|
this.rateLimiter?.stop();
|
|
595
869
|
}
|
|
596
870
|
async handle(req, res, path) {
|
|
871
|
+
if (!this.rpcEnabled) return this.send(res, 404, {
|
|
872
|
+
success: false,
|
|
873
|
+
error: "not found",
|
|
874
|
+
code: "NOT_FOUND"
|
|
875
|
+
});
|
|
597
876
|
this.applyCors(req, res);
|
|
598
877
|
if (req.method === "OPTIONS") {
|
|
599
878
|
res.writeHead(204);
|
|
600
879
|
res.end();
|
|
601
880
|
return;
|
|
602
881
|
}
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
882
|
+
const startedAt = startTimer();
|
|
883
|
+
const reqId = crypto.randomUUID().slice(0, 8);
|
|
884
|
+
let procName;
|
|
885
|
+
let kind;
|
|
886
|
+
let userId;
|
|
887
|
+
let span;
|
|
888
|
+
const record = (outcome, opts) => {
|
|
889
|
+
this.deps.obs.record({
|
|
890
|
+
op: "rpc",
|
|
891
|
+
transport: "http",
|
|
892
|
+
procedure: procName,
|
|
893
|
+
kind,
|
|
894
|
+
outcome,
|
|
895
|
+
durationMs: elapsedMs(startedAt),
|
|
896
|
+
code: opts?.code,
|
|
897
|
+
error: opts?.error,
|
|
898
|
+
reqId,
|
|
899
|
+
userId: userId ?? void 0,
|
|
900
|
+
span
|
|
901
|
+
});
|
|
902
|
+
};
|
|
903
|
+
if (req.method !== "POST") {
|
|
904
|
+
record("error", { code: "METHOD_NOT_ALLOWED" });
|
|
905
|
+
return this.send(res, 405, {
|
|
906
|
+
success: false,
|
|
907
|
+
error: "method not allowed",
|
|
908
|
+
code: "METHOD_NOT_ALLOWED"
|
|
909
|
+
});
|
|
910
|
+
}
|
|
608
911
|
if (this.rateLimiter) {
|
|
609
912
|
const keyBy = this.rpc.rateLimit ? this.rpc.rateLimit.keyBy : void 0;
|
|
610
913
|
const key = keyBy ? keyBy(req) : clientIp(req);
|
|
611
914
|
const rl = this.rateLimiter.check(key);
|
|
612
915
|
if (!rl.allowed) {
|
|
613
916
|
res.setHeader("retry-after", String(Math.ceil(rl.retryAfterMs / 1e3)));
|
|
917
|
+
record("error", { code: "RATE_LIMITED" });
|
|
614
918
|
return this.send(res, 429, {
|
|
615
919
|
success: false,
|
|
616
920
|
error: "rate limit exceeded",
|
|
@@ -629,44 +933,88 @@ var HttpRpcHandler = class {
|
|
|
629
933
|
rawInput = envelope.input ?? {};
|
|
630
934
|
minTs = typeof envelope.minTs === "string" ? parseMinTs$1(envelope.minTs) : void 0;
|
|
631
935
|
} catch {
|
|
936
|
+
record("error", { code: "BAD_REQUEST" });
|
|
632
937
|
return this.send(res, 400, {
|
|
633
938
|
success: false,
|
|
634
939
|
error: "invalid request",
|
|
635
940
|
code: "BAD_REQUEST"
|
|
636
941
|
});
|
|
637
942
|
}
|
|
943
|
+
procName = name;
|
|
638
944
|
const procedure = this.deps.getRegistry()?.getProcedure(name);
|
|
639
|
-
if (!procedure || procedure.internal || procedure.type === "job")
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
945
|
+
if (!procedure || procedure.internal || procedure.type === "job") {
|
|
946
|
+
record("error", { code: "NOT_FOUND" });
|
|
947
|
+
return this.send(res, 404, {
|
|
948
|
+
success: false,
|
|
949
|
+
error: "not found",
|
|
950
|
+
code: "NOT_FOUND"
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
kind = procedure.type;
|
|
644
954
|
const auth = await this.resolveAuth(req);
|
|
645
|
-
if (!auth.ok)
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
955
|
+
if (!auth.ok) {
|
|
956
|
+
record("error", { code: "UNAUTHENTICATED" });
|
|
957
|
+
return this.send(res, 401, {
|
|
958
|
+
success: false,
|
|
959
|
+
error: auth.message,
|
|
960
|
+
code: "UNAUTHENTICATED"
|
|
961
|
+
});
|
|
962
|
+
}
|
|
963
|
+
userId = this.deps.getUserId?.(auth.ctx);
|
|
650
964
|
let input;
|
|
651
965
|
try {
|
|
652
|
-
|
|
966
|
+
const parseRes = procedure.inputSchema.safeParse(rawInput);
|
|
967
|
+
if (!parseRes.success) {
|
|
968
|
+
record("error", { code: "BAD_REQUEST" });
|
|
969
|
+
return this.send(res, 400, {
|
|
970
|
+
success: false,
|
|
971
|
+
error: `Invalid input: ${parseRes.error.issues[0]?.message ?? "Unknow error"}`,
|
|
972
|
+
code: "BAD_REQUEST"
|
|
973
|
+
});
|
|
974
|
+
}
|
|
975
|
+
input = parseRes.data;
|
|
653
976
|
} catch (err) {
|
|
654
|
-
|
|
977
|
+
record("error", {
|
|
978
|
+
code: "BAD_REQUEST",
|
|
979
|
+
error: err
|
|
980
|
+
});
|
|
655
981
|
return this.send(res, 400, {
|
|
656
982
|
success: false,
|
|
657
983
|
error: "invalid input",
|
|
658
984
|
code: "BAD_REQUEST"
|
|
659
985
|
});
|
|
660
986
|
}
|
|
987
|
+
const started = this.deps.obs.startSpan(`rpc.${kind} ${name}`, {
|
|
988
|
+
transport: "http",
|
|
989
|
+
procedure: name,
|
|
990
|
+
kind,
|
|
991
|
+
parent: this.deps.obs.extractContext(req.headers)
|
|
992
|
+
});
|
|
993
|
+
span = started.span;
|
|
994
|
+
const traceId = this.deps.obs.traceIdOf(span);
|
|
995
|
+
const handlerObs = {
|
|
996
|
+
log: this.deps.obs.lazyChildLogger({
|
|
997
|
+
reqId,
|
|
998
|
+
procedure: name,
|
|
999
|
+
kind,
|
|
1000
|
+
...userId != null ? { userId } : {},
|
|
1001
|
+
...traceId ? { traceId } : {}
|
|
1002
|
+
}),
|
|
1003
|
+
span
|
|
1004
|
+
};
|
|
661
1005
|
try {
|
|
662
|
-
const { data, ts } = await this.runProcedure(procedure, input, auth.ctx, minTs);
|
|
1006
|
+
const { data, ts } = await context.with(started.context, () => this.runProcedure(procedure, input, auth.ctx, minTs, handlerObs));
|
|
1007
|
+
record("ok");
|
|
663
1008
|
return this.send(res, 200, {
|
|
664
1009
|
success: true,
|
|
665
1010
|
data,
|
|
666
1011
|
ts: ts?.toString()
|
|
667
1012
|
});
|
|
668
1013
|
} catch (err) {
|
|
669
|
-
|
|
1014
|
+
record("error", {
|
|
1015
|
+
code: "EXECUTION_ERROR",
|
|
1016
|
+
error: err
|
|
1017
|
+
});
|
|
670
1018
|
return this.send(res, 500, {
|
|
671
1019
|
success: false,
|
|
672
1020
|
error: err instanceof Error ? err.message : "execution failed",
|
|
@@ -674,10 +1022,97 @@ var HttpRpcHandler = class {
|
|
|
674
1022
|
});
|
|
675
1023
|
}
|
|
676
1024
|
}
|
|
677
|
-
|
|
1025
|
+
/**
|
|
1026
|
+
* Handles a job webhook: `POST {jobPath}/<jobName>`. Kept alongside {@link handle}
|
|
1027
|
+
* so the whole HTTP surface (RPC + jobs) lives in one class. Every job POST is
|
|
1028
|
+
* authenticated by the configured scheduler BEFORE any lookup or work — a forged
|
|
1029
|
+
* POST must not even learn whether a job name exists. Returns 500 on handler
|
|
1030
|
+
* failure so the scheduler retries (jobs must be idempotent).
|
|
1031
|
+
*/
|
|
1032
|
+
async handleJob(req, res, path) {
|
|
1033
|
+
if (req.method !== "POST") return this.send(res, 405, { error: "method not allowed" });
|
|
1034
|
+
const name = decodeURIComponent(path.slice(this.jobPrefix.length));
|
|
1035
|
+
if (!name || name.includes("/")) return this.send(res, 404, { error: "unknown job" });
|
|
1036
|
+
if (!this.deps.jobScheduler) return this.send(res, 503, { error: "jobs not enabled" });
|
|
1037
|
+
const startedAt = startTimer();
|
|
1038
|
+
const rawBody = await readBody(req);
|
|
1039
|
+
let verified = false;
|
|
1040
|
+
try {
|
|
1041
|
+
verified = await this.deps.jobScheduler.verify({
|
|
1042
|
+
name,
|
|
1043
|
+
rawBody,
|
|
1044
|
+
headers: lowercaseHeaders(req.headers),
|
|
1045
|
+
url: this.deps.endpointFor(name)
|
|
1046
|
+
});
|
|
1047
|
+
} catch (err) {
|
|
1048
|
+
this.deps.obs.logger.warn(err, `job "${name}" verification threw`);
|
|
1049
|
+
}
|
|
1050
|
+
if (!verified) return this.send(res, 401, { error: "invalid signature" });
|
|
1051
|
+
const procedure = this.deps.getRegistry()?.getProcedure(name);
|
|
1052
|
+
if (!procedure || procedure.type !== "job") return this.send(res, 404, { error: "unknown job" });
|
|
1053
|
+
let input;
|
|
1054
|
+
try {
|
|
1055
|
+
const parsed = rawBody.trim().length > 0 ? JSON.parse(rawBody) : {};
|
|
1056
|
+
input = procedure.inputSchema.parse(parsed);
|
|
1057
|
+
} catch (err) {
|
|
1058
|
+
this.deps.obs.logger.warn(err, `job "${name}" received an invalid payload`);
|
|
1059
|
+
return this.send(res, 400, { error: "invalid payload" });
|
|
1060
|
+
}
|
|
1061
|
+
const started = this.deps.obs.startSpan(`job ${name}`, {
|
|
1062
|
+
transport: "http",
|
|
1063
|
+
procedure: name,
|
|
1064
|
+
kind: "job",
|
|
1065
|
+
parent: this.deps.obs.extractContext(req.headers)
|
|
1066
|
+
});
|
|
1067
|
+
const span = started.span;
|
|
1068
|
+
const traceId = this.deps.obs.traceIdOf(span);
|
|
1069
|
+
const jobLog = this.deps.obs.lazyChildLogger({
|
|
1070
|
+
reqId: span && traceId ? traceId : name,
|
|
1071
|
+
procedure: name,
|
|
1072
|
+
kind: "job",
|
|
1073
|
+
...traceId ? { traceId } : {}
|
|
1074
|
+
});
|
|
1075
|
+
try {
|
|
1076
|
+
const serverCtx = this.deps.jobContext ? await this.deps.jobContext() : void 0;
|
|
1077
|
+
const fn = procedure.fn;
|
|
1078
|
+
const result = await context.with(started.context, () => fn({
|
|
1079
|
+
db: this.deps.db,
|
|
1080
|
+
serverCtx,
|
|
1081
|
+
log: jobLog,
|
|
1082
|
+
span
|
|
1083
|
+
}, input));
|
|
1084
|
+
this.deps.obs.record({
|
|
1085
|
+
op: "job",
|
|
1086
|
+
transport: "http",
|
|
1087
|
+
procedure: name,
|
|
1088
|
+
kind: "job",
|
|
1089
|
+
outcome: "ok",
|
|
1090
|
+
durationMs: elapsedMs(startedAt),
|
|
1091
|
+
span
|
|
1092
|
+
});
|
|
1093
|
+
return this.send(res, 200, {
|
|
1094
|
+
ok: true,
|
|
1095
|
+
result
|
|
1096
|
+
});
|
|
1097
|
+
} catch (err) {
|
|
1098
|
+
this.deps.obs.record({
|
|
1099
|
+
op: "job",
|
|
1100
|
+
transport: "http",
|
|
1101
|
+
procedure: name,
|
|
1102
|
+
kind: "job",
|
|
1103
|
+
outcome: "error",
|
|
1104
|
+
durationMs: elapsedMs(startedAt),
|
|
1105
|
+
code: "JOB_FAILED",
|
|
1106
|
+
error: err,
|
|
1107
|
+
span
|
|
1108
|
+
});
|
|
1109
|
+
return this.send(res, 500, { error: "job failed" });
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
async runProcedure(procedure, input, serverCtx, minTs, handlerObs) {
|
|
678
1113
|
if (procedure.type === "query") {
|
|
679
1114
|
const lastSnapshotTs = await this.deps.db.impl.getLatestSnapshotTimestamp();
|
|
680
|
-
return runCachedQuery(this.deps.oneShotDeps, procedure, procedure.name, input, serverCtx, lastSnapshotTs, minTs);
|
|
1115
|
+
return runCachedQuery(this.deps.oneShotDeps, procedure, procedure.name, input, serverCtx, lastSnapshotTs, minTs, handlerObs);
|
|
681
1116
|
}
|
|
682
1117
|
if (procedure.type === "mutation") {
|
|
683
1118
|
const res = await this.deps.db.mutation(async (ctx) => {
|
|
@@ -685,7 +1120,9 @@ var HttpRpcHandler = class {
|
|
|
685
1120
|
return fn({
|
|
686
1121
|
db: ctx,
|
|
687
1122
|
serverCtx,
|
|
688
|
-
...primaryOnlyConn
|
|
1123
|
+
...primaryOnlyConn,
|
|
1124
|
+
log: handlerObs?.log,
|
|
1125
|
+
span: handlerObs?.span
|
|
689
1126
|
}, input);
|
|
690
1127
|
});
|
|
691
1128
|
return {
|
|
@@ -698,7 +1135,9 @@ var HttpRpcHandler = class {
|
|
|
698
1135
|
db: this.deps.db,
|
|
699
1136
|
serverCtx,
|
|
700
1137
|
storage: this.deps.storage,
|
|
701
|
-
scheduler: this.deps.scheduler
|
|
1138
|
+
scheduler: this.deps.scheduler,
|
|
1139
|
+
log: handlerObs?.log,
|
|
1140
|
+
span: handlerObs?.span
|
|
702
1141
|
}, input) };
|
|
703
1142
|
}
|
|
704
1143
|
async resolveAuth(req) {
|
|
@@ -769,6 +1208,11 @@ function defaultHttpAuth(req) {
|
|
|
769
1208
|
const match = /^Bearer\s+(.+)$/i.exec(value);
|
|
770
1209
|
return match ? { token: match[1] } : void 0;
|
|
771
1210
|
}
|
|
1211
|
+
function lowercaseHeaders(headers) {
|
|
1212
|
+
const out = {};
|
|
1213
|
+
for (const [k, v] of Object.entries(headers)) out[k.toLowerCase()] = Array.isArray(v) ? v.join(",") : v;
|
|
1214
|
+
return out;
|
|
1215
|
+
}
|
|
772
1216
|
function clientIp(req) {
|
|
773
1217
|
const xff = req.headers["x-forwarded-for"];
|
|
774
1218
|
if (xff) {
|
|
@@ -840,7 +1284,6 @@ const CLIENT_TIMEOUT_MS = 12e4;
|
|
|
840
1284
|
* mutation triggers a new race.
|
|
841
1285
|
*/
|
|
842
1286
|
const RECOMPUTE_LOCK_TTL_SECONDS = Number(process.env.SUPALIVE_RECOMPUTE_LOCK_TTL_SECONDS ?? 10);
|
|
843
|
-
const logger$2 = logger.child({ "name": "SUPALIVE" }, { level: logLevelOf("SERVER_LOG_LEVEL") });
|
|
844
1287
|
/**
|
|
845
1288
|
* Supalive WebSocket Server
|
|
846
1289
|
*/
|
|
@@ -861,6 +1304,8 @@ var SupaliveWebSocketServer = class {
|
|
|
861
1304
|
scheduler;
|
|
862
1305
|
jobPath;
|
|
863
1306
|
jobClient;
|
|
1307
|
+
/** Owns the HTTP surface: one-shot RPC (when `config.rpc` is set) AND the job
|
|
1308
|
+
* webhook. Always constructed — jobs work even without RPC enabled. */
|
|
864
1309
|
rpcHandler;
|
|
865
1310
|
/** Optional object storage exposed to every procedure as `ctx.storage`. */
|
|
866
1311
|
storage;
|
|
@@ -872,6 +1317,10 @@ var SupaliveWebSocketServer = class {
|
|
|
872
1317
|
totalConnections: 0,
|
|
873
1318
|
activeSubscriptions: 0
|
|
874
1319
|
};
|
|
1320
|
+
/** Structured logs + OTel metrics + the per-operation `onEvent` seam. */
|
|
1321
|
+
obs;
|
|
1322
|
+
/** The request/lifecycle logger (injectable via `config.observability.logger`). */
|
|
1323
|
+
log;
|
|
875
1324
|
constructor(config) {
|
|
876
1325
|
this.config = {
|
|
877
1326
|
port: 3e3,
|
|
@@ -881,6 +1330,15 @@ var SupaliveWebSocketServer = class {
|
|
|
881
1330
|
...config
|
|
882
1331
|
};
|
|
883
1332
|
this.instanceId = appInstanceId;
|
|
1333
|
+
this.obs = new Observability({
|
|
1334
|
+
logger: (config.observability?.logger ?? logger).child({ name: "SUPALIVE" }, { level: config.observability?.level ?? logLevelOf("SERVER_LOG_LEVEL") }),
|
|
1335
|
+
level: void 0,
|
|
1336
|
+
requestLog: config.observability?.requestLog,
|
|
1337
|
+
onEvent: config.observability?.onEvent,
|
|
1338
|
+
meterName: config.observability?.meterName,
|
|
1339
|
+
meterVersion: config.observability?.meterVersion
|
|
1340
|
+
});
|
|
1341
|
+
this.log = this.obs.logger;
|
|
884
1342
|
if (config.subManager) this.subManagerClient = config.subManager;
|
|
885
1343
|
else if (config.subManagerUrl) this.subManagerClient = new SubManagerClient(config.subManagerUrl);
|
|
886
1344
|
else throw new Error("either `subManager` (in-process link) or `subManagerUrl` (WebSocket URL) is required");
|
|
@@ -890,7 +1348,7 @@ var SupaliveWebSocketServer = class {
|
|
|
890
1348
|
this.db = config.database;
|
|
891
1349
|
this.storage = config.storage;
|
|
892
1350
|
this.db.setOnAfterCommit((writeSet, commitTs) => {
|
|
893
|
-
this.handleMutationCommit(writeSet, commitTs).catch((err) =>
|
|
1351
|
+
this.handleMutationCommit(writeSet, commitTs).catch((err) => this.log.error(err, "post-commit invalidation failed"));
|
|
894
1352
|
});
|
|
895
1353
|
this.scheduler = config.scheduler;
|
|
896
1354
|
this.jobPath = normalizeJobPath(config.jobPath ?? "/_jobs");
|
|
@@ -901,20 +1359,27 @@ var SupaliveWebSocketServer = class {
|
|
|
901
1359
|
getUserId: config.getUserId,
|
|
902
1360
|
getMinRetainedTs: () => this.retentionWatermark.get()
|
|
903
1361
|
};
|
|
904
|
-
|
|
1362
|
+
this.rpcHandler = new HttpRpcHandler({
|
|
905
1363
|
getRegistry: () => this.contextRegistry,
|
|
906
1364
|
db: this.db,
|
|
907
1365
|
oneShotDeps: this.oneShotDeps,
|
|
908
1366
|
verifyAuth: config.verifyAuth,
|
|
909
|
-
rpc: config.rpc,
|
|
1367
|
+
rpc: config.rpc ?? {},
|
|
1368
|
+
rpcEnabled: config.rpc !== void 0,
|
|
910
1369
|
storage: this.storage,
|
|
911
|
-
scheduler: this.jobClient
|
|
1370
|
+
scheduler: this.jobClient,
|
|
1371
|
+
obs: this.obs,
|
|
1372
|
+
getUserId: config.getUserId,
|
|
1373
|
+
jobScheduler: this.scheduler,
|
|
1374
|
+
jobContext: config.jobContext,
|
|
1375
|
+
jobPath: this.jobPath,
|
|
1376
|
+
endpointFor: (name) => this.endpointFor(name)
|
|
912
1377
|
});
|
|
913
1378
|
this.subManagerClient.setRecoveryDriver(this.runSubManagerRecovery.bind(this));
|
|
914
1379
|
this.subManagerClient.setAffectedHandler(this.runAffectedHandler.bind(this));
|
|
915
1380
|
this.httpServer = createServer((req, res) => {
|
|
916
1381
|
this.handleHttpRequest(req, res).catch((err) => {
|
|
917
|
-
|
|
1382
|
+
this.log.error(err, "HTTP request handler error");
|
|
918
1383
|
if (!res.headersSent) {
|
|
919
1384
|
res.writeHead(500, { "content-type": "application/json" });
|
|
920
1385
|
res.end(JSON.stringify({ error: "internal error" }));
|
|
@@ -944,7 +1409,7 @@ var SupaliveWebSocketServer = class {
|
|
|
944
1409
|
startSessionHeartbeat(session) {
|
|
945
1410
|
session.heartbeatTimer = setInterval(() => {
|
|
946
1411
|
if (Date.now() - session.lastReceivedAt > CLIENT_TIMEOUT_MS) {
|
|
947
|
-
|
|
1412
|
+
this.log.info(`ws inactivity → terminate session ${session.id}`);
|
|
948
1413
|
try {
|
|
949
1414
|
session.ws.terminate();
|
|
950
1415
|
} catch {}
|
|
@@ -970,11 +1435,11 @@ var SupaliveWebSocketServer = class {
|
|
|
970
1435
|
try {
|
|
971
1436
|
await this.retentionWatermark.loadFrom(this.db.impl);
|
|
972
1437
|
} catch (err) {
|
|
973
|
-
|
|
1438
|
+
this.log.error(err, "failed to load initial retention watermark");
|
|
974
1439
|
}
|
|
975
1440
|
await new Promise((resolve) => {
|
|
976
1441
|
this.httpServer.listen(this.config.port, this.config.host, () => {
|
|
977
|
-
|
|
1442
|
+
this.log.info(`Server: ${this.instanceId} listening on ${this.config.host}:${this.config.port}`);
|
|
978
1443
|
resolve();
|
|
979
1444
|
});
|
|
980
1445
|
});
|
|
@@ -995,9 +1460,9 @@ var SupaliveWebSocketServer = class {
|
|
|
995
1460
|
if (jobs.length === 0) return;
|
|
996
1461
|
try {
|
|
997
1462
|
await this.scheduler.syncSchedules(jobs, (name) => this.endpointFor(name));
|
|
998
|
-
|
|
1463
|
+
this.log.info(`Synced ${jobs.length} recurring job schedule(s)`);
|
|
999
1464
|
} catch (err) {
|
|
1000
|
-
|
|
1465
|
+
this.log.error(err, "Failed to sync job schedules");
|
|
1001
1466
|
}
|
|
1002
1467
|
}
|
|
1003
1468
|
/** Absolute URL the scheduler POSTs to for a given job name. */
|
|
@@ -1028,9 +1493,9 @@ var SupaliveWebSocketServer = class {
|
|
|
1028
1493
|
async stop() {
|
|
1029
1494
|
this.wss.close();
|
|
1030
1495
|
for (const [ws] of this.sessions) ws.close(1001, "Server shutting down");
|
|
1031
|
-
this.rpcHandler
|
|
1496
|
+
this.rpcHandler.stop();
|
|
1032
1497
|
this.httpServer.close();
|
|
1033
|
-
|
|
1498
|
+
this.log.info("Server stopped");
|
|
1034
1499
|
}
|
|
1035
1500
|
getStats() {
|
|
1036
1501
|
return {
|
|
@@ -1054,63 +1519,25 @@ var SupaliveWebSocketServer = class {
|
|
|
1054
1519
|
status: "ok",
|
|
1055
1520
|
instanceId: this.instanceId
|
|
1056
1521
|
});
|
|
1057
|
-
if (this.rpcHandler
|
|
1058
|
-
|
|
1059
|
-
if (
|
|
1060
|
-
|
|
1061
|
-
const name = decodeURIComponent(path.slice(jobPrefix.length));
|
|
1062
|
-
if (!name || name.includes("/")) return sendJson(res, 404, { error: "unknown job" });
|
|
1063
|
-
if (!this.scheduler) return sendJson(res, 503, { error: "jobs not enabled" });
|
|
1064
|
-
const rawBody = await readRequestBody(req);
|
|
1065
|
-
let verified = false;
|
|
1066
|
-
try {
|
|
1067
|
-
verified = await this.scheduler.verify({
|
|
1068
|
-
name,
|
|
1069
|
-
rawBody,
|
|
1070
|
-
headers: lowercaseHeaders(req.headers),
|
|
1071
|
-
url: this.endpointFor(name)
|
|
1072
|
-
});
|
|
1073
|
-
} catch (err) {
|
|
1074
|
-
logger$2.warn(err, `job "${name}" verification threw`);
|
|
1075
|
-
}
|
|
1076
|
-
if (!verified) return sendJson(res, 401, { error: "invalid signature" });
|
|
1077
|
-
const procedure = this.contextRegistry?.getProcedure(name);
|
|
1078
|
-
if (!procedure || procedure.type !== "job") return sendJson(res, 404, { error: "unknown job" });
|
|
1079
|
-
let input;
|
|
1080
|
-
try {
|
|
1081
|
-
const parsed = rawBody.trim().length > 0 ? JSON.parse(rawBody) : {};
|
|
1082
|
-
input = procedure.inputSchema.parse(parsed);
|
|
1083
|
-
} catch (err) {
|
|
1084
|
-
logger$2.warn(err, `job "${name}" received an invalid payload`);
|
|
1085
|
-
return sendJson(res, 400, { error: "invalid payload" });
|
|
1086
|
-
}
|
|
1087
|
-
try {
|
|
1088
|
-
const serverCtx = this.config.jobContext ? await this.config.jobContext() : void 0;
|
|
1089
|
-
const fn = procedure.fn;
|
|
1090
|
-
const result = await fn({
|
|
1091
|
-
db: this.db,
|
|
1092
|
-
serverCtx
|
|
1093
|
-
}, input);
|
|
1094
|
-
logger$2.info(`job "${name}" completed`);
|
|
1095
|
-
return sendJson(res, 200, {
|
|
1096
|
-
ok: true,
|
|
1097
|
-
result
|
|
1098
|
-
});
|
|
1099
|
-
} catch (err) {
|
|
1100
|
-
logger$2.error(err, `job "${name}" failed`);
|
|
1101
|
-
return sendJson(res, 500, { error: "job failed" });
|
|
1522
|
+
if (this.rpcHandler.handlesPath(path)) return this.rpcHandler.handle(req, res, path);
|
|
1523
|
+
if (this.rpcHandler.handlesJobPath(path)) return this.rpcHandler.handleJob(req, res, path);
|
|
1524
|
+
if (this.config.onHttpRequest) {
|
|
1525
|
+
if (await this.config.onHttpRequest(req, res)) return;
|
|
1102
1526
|
}
|
|
1527
|
+
return sendJson(res, 404, { error: "not found" });
|
|
1103
1528
|
}
|
|
1104
1529
|
setupWebSocketHandlers() {
|
|
1105
1530
|
this.wss.on("connection", (ws, req) => {
|
|
1106
1531
|
this.handleConnection(ws, req).catch((err) => {
|
|
1107
|
-
|
|
1532
|
+
this.log.error(err, "Connection handler error");
|
|
1108
1533
|
});
|
|
1109
1534
|
});
|
|
1110
1535
|
}
|
|
1111
1536
|
async handleConnection(ws, req) {
|
|
1112
1537
|
this.stats.totalConnections++;
|
|
1113
1538
|
this.stats.activeConnections++;
|
|
1539
|
+
this.obs.connectionOpened();
|
|
1540
|
+
const connectedAt = startTimer();
|
|
1114
1541
|
const session = {
|
|
1115
1542
|
id: v4(),
|
|
1116
1543
|
ws,
|
|
@@ -1123,7 +1550,10 @@ var SupaliveWebSocketServer = class {
|
|
|
1123
1550
|
lastSentAt: Date.now(),
|
|
1124
1551
|
heartbeatTimer: null
|
|
1125
1552
|
};
|
|
1126
|
-
|
|
1553
|
+
this.log.info({
|
|
1554
|
+
sessionId: session.id,
|
|
1555
|
+
activeConnections: this.stats.activeConnections
|
|
1556
|
+
}, `New session: ${session.id}`);
|
|
1127
1557
|
this.sessions.set(ws, session);
|
|
1128
1558
|
this.sendMessage(session, {
|
|
1129
1559
|
type: "connected",
|
|
@@ -1132,7 +1562,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1132
1562
|
ws.on("message", (data) => {
|
|
1133
1563
|
session.lastReceivedAt = Date.now();
|
|
1134
1564
|
this.handleMessage(session, data.toString()).catch((err) => {
|
|
1135
|
-
|
|
1565
|
+
this.log.error(err, "Message handler error");
|
|
1136
1566
|
});
|
|
1137
1567
|
});
|
|
1138
1568
|
ws.on("ping", () => {
|
|
@@ -1143,17 +1573,24 @@ var SupaliveWebSocketServer = class {
|
|
|
1143
1573
|
});
|
|
1144
1574
|
this.startSessionHeartbeat(session);
|
|
1145
1575
|
ws.on("close", (code) => {
|
|
1146
|
-
logger$2.info(`Connection closed: ${session.id}, code: ${code}`);
|
|
1147
1576
|
this.clearSessionTimers(session);
|
|
1148
1577
|
this.handleDisconnect(session);
|
|
1149
1578
|
this.sessions.delete(ws);
|
|
1150
1579
|
this.stats.activeConnections--;
|
|
1580
|
+
this.obs.connectionClosed();
|
|
1581
|
+
this.log.info({
|
|
1582
|
+
sessionId: session.id,
|
|
1583
|
+
code,
|
|
1584
|
+
durMs: elapsedMs(connectedAt),
|
|
1585
|
+
activeConnections: this.stats.activeConnections
|
|
1586
|
+
}, `Connection closed: ${session.id}, code: ${code}`);
|
|
1151
1587
|
});
|
|
1152
1588
|
ws.on("error", (err) => {
|
|
1153
|
-
|
|
1589
|
+
this.log.error(err, "Websocket error");
|
|
1154
1590
|
});
|
|
1155
1591
|
}
|
|
1156
1592
|
async handleMessage(session, message) {
|
|
1593
|
+
const startedAt = startTimer();
|
|
1157
1594
|
try {
|
|
1158
1595
|
const rawMsg = JSON.parse(message);
|
|
1159
1596
|
const parseResult = ClientMessageSchema.safeParse(rawMsg);
|
|
@@ -1161,22 +1598,22 @@ var SupaliveWebSocketServer = class {
|
|
|
1161
1598
|
this.sendError(session, "", "Invalid message format", "INVALID_MESSAGE");
|
|
1162
1599
|
return;
|
|
1163
1600
|
}
|
|
1164
|
-
await this.processMessage(session, parseResult.data);
|
|
1601
|
+
await this.processMessage(session, parseResult.data, startedAt);
|
|
1165
1602
|
} catch (err) {
|
|
1166
|
-
|
|
1603
|
+
this.log.error(err, "Message handling error");
|
|
1167
1604
|
this.sendError(session, "", "Parse error", "PARSE_ERROR");
|
|
1168
1605
|
}
|
|
1169
1606
|
}
|
|
1170
|
-
async processMessage(session, msg) {
|
|
1607
|
+
async processMessage(session, msg, startedAt) {
|
|
1171
1608
|
switch (msg.type) {
|
|
1172
1609
|
case "auth":
|
|
1173
1610
|
await this.handleAuth(session, msg.data);
|
|
1174
1611
|
break;
|
|
1175
1612
|
case "call":
|
|
1176
|
-
await this.handleCall(session, msg.id, msg.procedure, msg.input, parseMinTs(msg.minTs));
|
|
1613
|
+
await this.handleCall(session, msg.id, msg.procedure, msg.input, startedAt, parseMinTs(msg.minTs), msg.trace);
|
|
1177
1614
|
break;
|
|
1178
1615
|
case "subscribe":
|
|
1179
|
-
await this.handleSubscribe(session, msg.id, msg.procedure, msg.input);
|
|
1616
|
+
await this.handleSubscribe(session, msg.id, msg.procedure, msg.input, startedAt, msg.trace);
|
|
1180
1617
|
break;
|
|
1181
1618
|
case "unsubscribe":
|
|
1182
1619
|
await this.handleUnsubscribe(session, msg.subId);
|
|
@@ -1224,16 +1661,17 @@ var SupaliveWebSocketServer = class {
|
|
|
1224
1661
|
* session. Callers (handleCall) own the response/error for their own
|
|
1225
1662
|
* op via the returned promise.
|
|
1226
1663
|
*/
|
|
1227
|
-
enqueueMutation(session, procedure, validatedInput) {
|
|
1664
|
+
enqueueMutation(session, procedure, validatedInput, handlerObs) {
|
|
1228
1665
|
return new Promise((resolve, reject) => {
|
|
1229
1666
|
const pending = {
|
|
1230
1667
|
procedure,
|
|
1231
1668
|
validatedInput,
|
|
1232
1669
|
resolve,
|
|
1233
|
-
reject
|
|
1670
|
+
reject,
|
|
1671
|
+
handlerObs
|
|
1234
1672
|
};
|
|
1235
1673
|
session.mutationQueue.push(pending);
|
|
1236
|
-
this.drainMutationQueue(session).catch((err) =>
|
|
1674
|
+
this.drainMutationQueue(session).catch((err) => this.log.error(err, "drainMutationQueue crashed"));
|
|
1237
1675
|
});
|
|
1238
1676
|
}
|
|
1239
1677
|
/**
|
|
@@ -1249,13 +1687,17 @@ var SupaliveWebSocketServer = class {
|
|
|
1249
1687
|
const op = session.mutationQueue.shift();
|
|
1250
1688
|
try {
|
|
1251
1689
|
const fn = op.procedure.fn;
|
|
1252
|
-
const
|
|
1690
|
+
const runMutation = () => this.db.mutation(async (ctx) => {
|
|
1253
1691
|
return fn({
|
|
1254
1692
|
db: ctx,
|
|
1255
1693
|
serverCtx: session.serverCtx,
|
|
1256
|
-
...primaryOnlyConn
|
|
1694
|
+
...primaryOnlyConn,
|
|
1695
|
+
log: op.handlerObs?.log,
|
|
1696
|
+
span: op.handlerObs?.span
|
|
1257
1697
|
}, op.validatedInput);
|
|
1258
1698
|
});
|
|
1699
|
+
const parentSpan = op.handlerObs?.span;
|
|
1700
|
+
const result = parentSpan?.isRecording() ? await context.with(trace.setSpan(context.active(), parentSpan), runMutation) : await runMutation();
|
|
1259
1701
|
op.resolve(result);
|
|
1260
1702
|
} catch (err) {
|
|
1261
1703
|
op.reject(err);
|
|
@@ -1265,81 +1707,181 @@ var SupaliveWebSocketServer = class {
|
|
|
1265
1707
|
session.mutationInFlight = false;
|
|
1266
1708
|
}
|
|
1267
1709
|
}
|
|
1268
|
-
async handleCall(session, id, procedureName, input, minTs) {
|
|
1710
|
+
async handleCall(session, id, procedureName, input, startedAt, minTs, trace) {
|
|
1711
|
+
const userId = this.config.getUserId?.(session.serverCtx) ?? void 0;
|
|
1712
|
+
let span;
|
|
1713
|
+
const record = (outcome, opts) => {
|
|
1714
|
+
this.obs.record({
|
|
1715
|
+
op: "rpc",
|
|
1716
|
+
transport: "ws",
|
|
1717
|
+
procedure: procedureName,
|
|
1718
|
+
kind: opts?.kind,
|
|
1719
|
+
outcome,
|
|
1720
|
+
durationMs: elapsedMs(startedAt),
|
|
1721
|
+
code: opts?.code,
|
|
1722
|
+
error: opts?.error,
|
|
1723
|
+
sessionId: session.id,
|
|
1724
|
+
reqId: id,
|
|
1725
|
+
userId,
|
|
1726
|
+
span
|
|
1727
|
+
});
|
|
1728
|
+
};
|
|
1269
1729
|
if (!this.router) {
|
|
1270
1730
|
this.sendError(session, id, "Router not registered", "SERVER_ERROR");
|
|
1731
|
+
record("error", { code: "SERVER_ERROR" });
|
|
1271
1732
|
return;
|
|
1272
1733
|
}
|
|
1273
1734
|
const procedure = this.contextRegistry.getProcedure(procedureName);
|
|
1274
1735
|
if (!procedure || procedure.type === "job") {
|
|
1275
1736
|
this.sendError(session, id, `Procedure "${procedureName}" not found`, "NOT_FOUND");
|
|
1737
|
+
record("error", { code: "NOT_FOUND" });
|
|
1276
1738
|
return;
|
|
1277
1739
|
}
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
this.
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1740
|
+
const kind = procedure.type;
|
|
1741
|
+
const parent = trace ? this.obs.extractContext({ traceparent: trace }) : void 0;
|
|
1742
|
+
const started = this.obs.startSpan(`rpc.${kind} ${procedureName}`, {
|
|
1743
|
+
transport: "ws",
|
|
1744
|
+
procedure: procedureName,
|
|
1745
|
+
kind,
|
|
1746
|
+
parent
|
|
1747
|
+
});
|
|
1748
|
+
span = started.span;
|
|
1749
|
+
const traceId = this.obs.traceIdOf(span);
|
|
1750
|
+
const handlerObs = {
|
|
1751
|
+
log: this.obs.lazyChildLogger({
|
|
1752
|
+
reqId: id,
|
|
1753
|
+
sessionId: session.id,
|
|
1754
|
+
procedure: procedureName,
|
|
1755
|
+
kind,
|
|
1756
|
+
...userId != null ? { userId } : {},
|
|
1757
|
+
...traceId ? { traceId } : {}
|
|
1758
|
+
}),
|
|
1759
|
+
span
|
|
1760
|
+
};
|
|
1761
|
+
await context.with(started.context, async () => {
|
|
1762
|
+
try {
|
|
1763
|
+
const validatedInput = procedure.inputSchema.parse(input);
|
|
1764
|
+
if (procedure.type === "query") {
|
|
1765
|
+
const lastSnapshotTs = await this.db.impl.getLatestSnapshotTimestamp();
|
|
1766
|
+
const { data, ts } = await runCachedQuery(this.oneShotDeps, procedure, procedureName, validatedInput, session.serverCtx, lastSnapshotTs, minTs, handlerObs);
|
|
1767
|
+
this.sendMessage(session, {
|
|
1768
|
+
type: "response",
|
|
1769
|
+
id,
|
|
1770
|
+
success: true,
|
|
1771
|
+
data,
|
|
1772
|
+
ts: ts.toString()
|
|
1773
|
+
});
|
|
1774
|
+
record("ok", { kind });
|
|
1775
|
+
} else if (procedure.type === "mutation") {
|
|
1776
|
+
if (session.mutationQueue.length >= MAX_MUTATION_QUEUE_BUFFER) {
|
|
1777
|
+
this.sendError(session, id, `Too many concurrent mutations on this connection (max ${MAX_MUTATION_QUEUE_BUFFER})`, "TOO_MANY_CONCURRENT_MUTATIONS");
|
|
1778
|
+
record("error", {
|
|
1779
|
+
kind,
|
|
1780
|
+
code: "TOO_MANY_CONCURRENT_MUTATIONS"
|
|
1781
|
+
});
|
|
1782
|
+
return;
|
|
1783
|
+
}
|
|
1784
|
+
const mutationResult = await this.enqueueMutation(session, procedure, validatedInput, handlerObs);
|
|
1785
|
+
this.sendMessage(session, {
|
|
1786
|
+
type: "response",
|
|
1787
|
+
id,
|
|
1788
|
+
success: true,
|
|
1789
|
+
data: mutationResult.result,
|
|
1790
|
+
ts: mutationResult.commitTs.toString()
|
|
1791
|
+
});
|
|
1792
|
+
record("ok", { kind });
|
|
1793
|
+
} else if (procedure.type === "action") {
|
|
1794
|
+
if (session.actionInFlight >= MAX_ACTION_CONCURRENCY) {
|
|
1795
|
+
this.sendError(session, id, `Too many concurrent actions on this connection (max ${MAX_ACTION_CONCURRENCY})`, "TOO_MANY_CONCURRENT_ACTIONS");
|
|
1796
|
+
record("error", {
|
|
1797
|
+
kind,
|
|
1798
|
+
code: "TOO_MANY_CONCURRENT_ACTIONS"
|
|
1799
|
+
});
|
|
1800
|
+
return;
|
|
1801
|
+
}
|
|
1802
|
+
session.actionInFlight++;
|
|
1803
|
+
let result;
|
|
1804
|
+
try {
|
|
1805
|
+
const fn = procedure.fn;
|
|
1806
|
+
result = await fn({
|
|
1807
|
+
db: this.db,
|
|
1808
|
+
serverCtx: session.serverCtx,
|
|
1809
|
+
storage: this.storage,
|
|
1810
|
+
scheduler: this.jobClient,
|
|
1811
|
+
log: handlerObs.log,
|
|
1812
|
+
span: handlerObs.span
|
|
1813
|
+
}, validatedInput);
|
|
1814
|
+
} finally {
|
|
1815
|
+
session.actionInFlight--;
|
|
1816
|
+
}
|
|
1817
|
+
this.sendMessage(session, {
|
|
1818
|
+
type: "response",
|
|
1819
|
+
id,
|
|
1820
|
+
success: true,
|
|
1821
|
+
data: result
|
|
1822
|
+
});
|
|
1823
|
+
record("ok", { kind });
|
|
1320
1824
|
}
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1825
|
+
} catch (err) {
|
|
1826
|
+
this.sendError(session, id, err instanceof Error ? err.message : "Execution failed", "EXECUTION_ERROR");
|
|
1827
|
+
record("error", {
|
|
1828
|
+
kind,
|
|
1829
|
+
code: "EXECUTION_ERROR",
|
|
1830
|
+
error: err
|
|
1326
1831
|
});
|
|
1327
1832
|
}
|
|
1328
|
-
}
|
|
1329
|
-
logger$2.error(err, `Error executing "${procedureName}"`);
|
|
1330
|
-
this.sendError(session, id, err instanceof Error ? err.message : "Execution failed", "EXECUTION_ERROR");
|
|
1331
|
-
}
|
|
1833
|
+
});
|
|
1332
1834
|
}
|
|
1333
|
-
async handleSubscribe(session, id, procedureName, input) {
|
|
1835
|
+
async handleSubscribe(session, id, procedureName, input, startedAt, trace) {
|
|
1836
|
+
const userId = this.config.getUserId?.(session.serverCtx) ?? void 0;
|
|
1837
|
+
let span;
|
|
1838
|
+
const record = (outcome, opts) => {
|
|
1839
|
+
this.obs.record({
|
|
1840
|
+
op: "subscribe",
|
|
1841
|
+
transport: "ws",
|
|
1842
|
+
procedure: procedureName,
|
|
1843
|
+
kind: "subscribe",
|
|
1844
|
+
outcome,
|
|
1845
|
+
durationMs: elapsedMs(startedAt),
|
|
1846
|
+
code: opts?.code,
|
|
1847
|
+
error: opts?.error,
|
|
1848
|
+
sessionId: session.id,
|
|
1849
|
+
reqId: id,
|
|
1850
|
+
userId,
|
|
1851
|
+
span
|
|
1852
|
+
});
|
|
1853
|
+
};
|
|
1334
1854
|
if (!this.router) {
|
|
1335
1855
|
this.sendError(session, id, "Router not registered", "SERVER_ERROR");
|
|
1856
|
+
record("error", { code: "SERVER_ERROR" });
|
|
1336
1857
|
return;
|
|
1337
1858
|
}
|
|
1338
1859
|
const procedure = this.contextRegistry.getProcedure(procedureName);
|
|
1339
1860
|
if (!procedure || procedure.type !== "query") {
|
|
1340
1861
|
this.sendError(session, id, `Procedure "${procedureName}" not found or not a query`, "NOT_FOUND");
|
|
1862
|
+
record("error", { code: "NOT_FOUND" });
|
|
1341
1863
|
return;
|
|
1342
1864
|
}
|
|
1865
|
+
const parent = trace ? this.obs.extractContext({ traceparent: trace }) : void 0;
|
|
1866
|
+
const started = this.obs.startSpan(`subscribe ${procedureName}`, {
|
|
1867
|
+
transport: "ws",
|
|
1868
|
+
procedure: procedureName,
|
|
1869
|
+
kind: "subscribe",
|
|
1870
|
+
parent
|
|
1871
|
+
});
|
|
1872
|
+
span = started.span;
|
|
1873
|
+
const traceId = this.obs.traceIdOf(span);
|
|
1874
|
+
const handlerObs = {
|
|
1875
|
+
log: this.obs.lazyChildLogger({
|
|
1876
|
+
reqId: id,
|
|
1877
|
+
sessionId: session.id,
|
|
1878
|
+
procedure: procedureName,
|
|
1879
|
+
kind: "subscribe",
|
|
1880
|
+
...userId != null ? { userId } : {},
|
|
1881
|
+
...traceId ? { traceId } : {}
|
|
1882
|
+
}),
|
|
1883
|
+
span
|
|
1884
|
+
};
|
|
1343
1885
|
try {
|
|
1344
1886
|
const validatedInput = procedure.inputSchema.parse(input);
|
|
1345
1887
|
const queryIdentity = resolveQueryIdentity(procedure, session.serverCtx, validatedInput, this.config.getUserId);
|
|
@@ -1359,7 +1901,11 @@ var SupaliveWebSocketServer = class {
|
|
|
1359
1901
|
data = cached.data;
|
|
1360
1902
|
servedTs = cached.lastSnapshotTs;
|
|
1361
1903
|
} else {
|
|
1362
|
-
liveResult = await this.executeLiveQuery(procedureName, validatedInput, session.serverCtx
|
|
1904
|
+
liveResult = await this.executeLiveQuery(procedureName, validatedInput, session.serverCtx, {
|
|
1905
|
+
log: handlerObs.log,
|
|
1906
|
+
parentContext: started.context,
|
|
1907
|
+
trigger: "subscribe"
|
|
1908
|
+
});
|
|
1363
1909
|
data = liveResult.data;
|
|
1364
1910
|
servedTs = liveResult.ts;
|
|
1365
1911
|
}
|
|
@@ -1384,7 +1930,11 @@ var SupaliveWebSocketServer = class {
|
|
|
1384
1930
|
}
|
|
1385
1931
|
}
|
|
1386
1932
|
if (rebuild) {
|
|
1387
|
-
liveResult = await this.executeLiveQuery(procedureName, validatedInput, session.serverCtx
|
|
1933
|
+
liveResult = await this.executeLiveQuery(procedureName, validatedInput, session.serverCtx, {
|
|
1934
|
+
log: handlerObs.log,
|
|
1935
|
+
parentContext: started.context,
|
|
1936
|
+
trigger: "subscribe"
|
|
1937
|
+
});
|
|
1388
1938
|
data = liveResult.data;
|
|
1389
1939
|
servedTs = liveResult.ts;
|
|
1390
1940
|
}
|
|
@@ -1394,6 +1944,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1394
1944
|
subscribedTimes: 1
|
|
1395
1945
|
});
|
|
1396
1946
|
this.stats.activeSubscriptions++;
|
|
1947
|
+
this.obs.subscriptionAdded();
|
|
1397
1948
|
}
|
|
1398
1949
|
if (liveResult) {
|
|
1399
1950
|
const [updateResult] = await Promise.all([this.subManagerClient.updateSubscriptionReadSet({
|
|
@@ -1403,7 +1954,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1403
1954
|
readSet: liveResult.readSet,
|
|
1404
1955
|
instanceName: this.instanceId
|
|
1405
1956
|
}), this.cache.setQueryCacheFor(cacheKey, data, liveResult.ts)]);
|
|
1406
|
-
if (!updateResult.tracked)
|
|
1957
|
+
if (!updateResult.tracked) this.log.warn(`updateSubscriptionReadSet returned tracked:false immediately after register for ${subId} — possible sub-manager restart mid-subscribe`);
|
|
1407
1958
|
}
|
|
1408
1959
|
dataHash = await getHashOf(data);
|
|
1409
1960
|
this.sendMessage(session, {
|
|
@@ -1419,10 +1970,13 @@ var SupaliveWebSocketServer = class {
|
|
|
1419
1970
|
dataHash,
|
|
1420
1971
|
ts: servedTs.toString()
|
|
1421
1972
|
});
|
|
1422
|
-
|
|
1973
|
+
record("ok", { code: void 0 });
|
|
1423
1974
|
} catch (err) {
|
|
1424
|
-
logger$2.error(err, `Error subscribing to "${procedureName}"`);
|
|
1425
1975
|
this.sendError(session, id, err instanceof Error ? err.message : "Subscribe failed", "SUBSCRIBE_ERROR");
|
|
1976
|
+
record("error", {
|
|
1977
|
+
code: "SUBSCRIBE_ERROR",
|
|
1978
|
+
error: err
|
|
1979
|
+
});
|
|
1426
1980
|
}
|
|
1427
1981
|
}
|
|
1428
1982
|
async handleUnsubscribe(session, subId) {
|
|
@@ -1436,19 +1990,24 @@ var SupaliveWebSocketServer = class {
|
|
|
1436
1990
|
session.subIds.delete(subId);
|
|
1437
1991
|
this.serverSubscriptions.delete(subId);
|
|
1438
1992
|
this.stats.activeSubscriptions--;
|
|
1993
|
+
this.obs.subscriptionRemoved();
|
|
1439
1994
|
try {
|
|
1440
1995
|
await this.subManagerClient.unregisterSubscription({
|
|
1441
1996
|
subId,
|
|
1442
1997
|
instanceName: this.instanceId
|
|
1443
1998
|
});
|
|
1444
1999
|
} catch (err) {
|
|
1445
|
-
|
|
2000
|
+
this.log.error(err, "Failed to unregister from sub-manager:");
|
|
1446
2001
|
}
|
|
1447
2002
|
}
|
|
1448
|
-
|
|
2003
|
+
this.log.debug({
|
|
2004
|
+
sessionId: session.id,
|
|
2005
|
+
subId,
|
|
2006
|
+
remaining: serverSub.subscribedTimes
|
|
2007
|
+
}, `Unsubscribed: ${subId} for session ${session.id}`);
|
|
1449
2008
|
}
|
|
1450
2009
|
async handleDisconnect(session) {
|
|
1451
|
-
|
|
2010
|
+
this.log.info(`Session disconnected: ${session.id}`);
|
|
1452
2011
|
let unregisterSubs = void 0;
|
|
1453
2012
|
for (const subId of session.subIds) {
|
|
1454
2013
|
const sub = this.serverSubscriptions.get(subId);
|
|
@@ -1466,13 +2025,14 @@ var SupaliveWebSocketServer = class {
|
|
|
1466
2025
|
instanceName: this.instanceId
|
|
1467
2026
|
})));
|
|
1468
2027
|
} catch (err) {
|
|
1469
|
-
|
|
2028
|
+
this.log.error(err, "Failed to unregister from sub-manager:");
|
|
1470
2029
|
}
|
|
1471
2030
|
for (const sub of unregisterSubs) if (sub.subscribedTimes <= 0) {
|
|
1472
2031
|
this.serverSubscriptions.delete(sub.subId);
|
|
1473
2032
|
this.stats.activeSubscriptions--;
|
|
2033
|
+
this.obs.subscriptionRemoved();
|
|
1474
2034
|
}
|
|
1475
|
-
|
|
2035
|
+
this.log.info(`All Subscription sessions is disconnected: ${session.id}, subs: ${unregisterSubs.length}`);
|
|
1476
2036
|
}
|
|
1477
2037
|
/**
|
|
1478
2038
|
* Drives recovery on a sub-manager reconnect. Iterates this app server's
|
|
@@ -1491,10 +2051,10 @@ var SupaliveWebSocketServer = class {
|
|
|
1491
2051
|
async runSubManagerRecovery() {
|
|
1492
2052
|
const snapshot = Array.from(this.serverSubscriptions.values());
|
|
1493
2053
|
if (snapshot.length === 0) {
|
|
1494
|
-
|
|
2054
|
+
this.log.info("Sub-manager recovery: no subscriptions to restore");
|
|
1495
2055
|
return;
|
|
1496
2056
|
}
|
|
1497
|
-
|
|
2057
|
+
this.log.info(`Sub-manager recovery: restoring ${snapshot.length} subscription(s)`);
|
|
1498
2058
|
const lastSnapshotTs = await this.db.impl.getLatestSnapshotTimestamp();
|
|
1499
2059
|
const subIds = snapshot.map((e) => e.subId);
|
|
1500
2060
|
const recordsMap = await this.cache.fetchRegistrationRecordsBatch(subIds);
|
|
@@ -1536,11 +2096,11 @@ var SupaliveWebSocketServer = class {
|
|
|
1536
2096
|
});
|
|
1537
2097
|
}
|
|
1538
2098
|
} catch (err) {
|
|
1539
|
-
|
|
2099
|
+
this.log.error(err, `Sub-manager recovery: failed to batch register ${i}-${i + batch.length}`);
|
|
1540
2100
|
}
|
|
1541
2101
|
}
|
|
1542
|
-
await Promise.allSettled(recompute.map((a) => this.reexecuteAffectedQuery(a)));
|
|
1543
|
-
|
|
2102
|
+
await Promise.allSettled(recompute.map((a) => this.reexecuteAffectedQuery(a, "recovery")));
|
|
2103
|
+
this.log.info(`Sub-manager recovery: done (${recompute.length} reexec'd)`);
|
|
1544
2104
|
}
|
|
1545
2105
|
async runAffectedHandler(affected) {
|
|
1546
2106
|
await Promise.allSettled(affected.map((a) => this.reexecuteAffectedQuery(a)));
|
|
@@ -1557,6 +2117,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1557
2117
|
if (serverSub) {
|
|
1558
2118
|
this.serverSubscriptions.delete(subId);
|
|
1559
2119
|
this.stats.activeSubscriptions = Math.max(0, this.stats.activeSubscriptions - 1);
|
|
2120
|
+
this.obs.subscriptionRemoved();
|
|
1560
2121
|
for (const [, session] of this.sessions) if (session.subIds.has(subId)) {
|
|
1561
2122
|
notified++;
|
|
1562
2123
|
session.subIds.delete(subId);
|
|
@@ -1567,18 +2128,18 @@ var SupaliveWebSocketServer = class {
|
|
|
1567
2128
|
});
|
|
1568
2129
|
}
|
|
1569
2130
|
}
|
|
1570
|
-
|
|
2131
|
+
this.log.warn(`Dropped subscription ${subId} locally (${reason}); notified ${notified} session(s)`);
|
|
1571
2132
|
}
|
|
1572
2133
|
async handleMutationCommit(writeSet, commitTs) {
|
|
1573
2134
|
if (writeSet.length === 0) return;
|
|
1574
2135
|
const tables = [...new Set(writeSet.map((w) => w.table))];
|
|
1575
|
-
|
|
2136
|
+
this.log.debug(`Mutation committed for tables: [${tables.join(", ")}]`);
|
|
1576
2137
|
try {
|
|
1577
2138
|
const result = await this.subManagerClient.invalidateWriteset({ writeSet });
|
|
1578
|
-
|
|
2139
|
+
this.log.debug({ affected: result }, "Sub-managers invalidation result");
|
|
1579
2140
|
if (result?.affected?.length > 0) await Promise.allSettled(result.affected.map((affected) => this.dispatchAffected(affected, commitTs)));
|
|
1580
2141
|
} catch (err) {
|
|
1581
|
-
|
|
2142
|
+
this.log.error(err, "Failed to notify sub-managers");
|
|
1582
2143
|
}
|
|
1583
2144
|
}
|
|
1584
2145
|
/**
|
|
@@ -1600,7 +2161,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1600
2161
|
return;
|
|
1601
2162
|
}
|
|
1602
2163
|
if (affected.notifyInstances.length === 0) {
|
|
1603
|
-
|
|
2164
|
+
this.log.debug(`dispatchAffected: no live instances hold ${affected.subId} — dropping`);
|
|
1604
2165
|
return;
|
|
1605
2166
|
}
|
|
1606
2167
|
await this.publishRecomputeRace(affected, commitTs);
|
|
@@ -1626,23 +2187,23 @@ var SupaliveWebSocketServer = class {
|
|
|
1626
2187
|
try {
|
|
1627
2188
|
await redisRetryPolicy.execute(() => this.cache.redis.publish(channel, payload));
|
|
1628
2189
|
} catch (err) {
|
|
1629
|
-
|
|
2190
|
+
this.log.error(err, `Failed to publish recompute-race to ${channel} after many retries`);
|
|
1630
2191
|
}
|
|
1631
2192
|
});
|
|
1632
2193
|
await Promise.all(publishPromises);
|
|
1633
2194
|
}
|
|
1634
|
-
async reexecuteAffectedQuery(affected) {
|
|
2195
|
+
async reexecuteAffectedQuery(affected, trigger = "recompute") {
|
|
1635
2196
|
let foundSession = null;
|
|
1636
2197
|
for (const [, session] of this.sessions) if (session.subIds.has(affected.subId)) {
|
|
1637
2198
|
foundSession = session;
|
|
1638
2199
|
break;
|
|
1639
2200
|
}
|
|
1640
2201
|
if (!foundSession) {
|
|
1641
|
-
|
|
2202
|
+
this.log.debug(`Session not found for subId: ${affected.subId}`);
|
|
1642
2203
|
return;
|
|
1643
2204
|
}
|
|
1644
2205
|
try {
|
|
1645
|
-
const liveResult = await this.executeLiveQuery(affected.queryName, affected.args, foundSession.serverCtx);
|
|
2206
|
+
const liveResult = await this.executeLiveQuery(affected.queryName, affected.args, foundSession.serverCtx, { trigger });
|
|
1646
2207
|
const dataHash = await getHashOf(liveResult.data);
|
|
1647
2208
|
const [updateResult] = await Promise.all([this.subManagerClient.updateSubscriptionReadSet({
|
|
1648
2209
|
cacheKey: affected.cacheKey,
|
|
@@ -1664,21 +2225,23 @@ var SupaliveWebSocketServer = class {
|
|
|
1664
2225
|
ts
|
|
1665
2226
|
});
|
|
1666
2227
|
if (affected.notifyInstances.length > 0) await this.publishSubscriptionUpdate(affected.subId, liveResult.data, dataHash, affected.notifyInstances, ts);
|
|
1667
|
-
|
|
2228
|
+
this.log.debug(`Re-executed affected query for subId: ${affected.subId}`);
|
|
1668
2229
|
} catch (err) {
|
|
1669
|
-
|
|
2230
|
+
this.log.error(err, `Failed to re-execute query for subId ${affected.subId}`);
|
|
1670
2231
|
}
|
|
1671
2232
|
}
|
|
1672
|
-
async executeLiveQuery(procedureName, input, serverCtx) {
|
|
2233
|
+
async executeLiveQuery(procedureName, input, serverCtx, opts) {
|
|
1673
2234
|
const procedure = this.contextRegistry.getProcedure(procedureName);
|
|
1674
2235
|
if (!procedure || procedure.type !== "query") throw new Error(`Procedure not defined: ${procedureName}`);
|
|
1675
2236
|
return this.db.liveQuery(async (ctx) => {
|
|
1676
2237
|
const fn = procedure.fn;
|
|
1677
|
-
return fn({
|
|
2238
|
+
return traceQueryExecution(procedureName, opts?.trigger ?? "recompute", opts?.parentContext, (querySpan) => fn({
|
|
1678
2239
|
db: ctx,
|
|
1679
2240
|
serverCtx,
|
|
1680
|
-
...primaryOnlyConn
|
|
1681
|
-
|
|
2241
|
+
...primaryOnlyConn,
|
|
2242
|
+
log: opts?.log,
|
|
2243
|
+
span: querySpan
|
|
2244
|
+
}, input));
|
|
1682
2245
|
});
|
|
1683
2246
|
}
|
|
1684
2247
|
sendMessage(session, message) {
|
|
@@ -1705,7 +2268,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1705
2268
|
const appChannels = [...this.appPubSubChannels];
|
|
1706
2269
|
await this.redisSub.subscribe(instanceChannel, schemaChannel, watermarkChannel, ...appChannels);
|
|
1707
2270
|
this.redisSub.on("connect", () => {
|
|
1708
|
-
|
|
2271
|
+
this.log.info(`Subscribed to Redis channels: ${[
|
|
1709
2272
|
instanceChannel,
|
|
1710
2273
|
schemaChannel,
|
|
1711
2274
|
watermarkChannel,
|
|
@@ -1714,20 +2277,20 @@ var SupaliveWebSocketServer = class {
|
|
|
1714
2277
|
});
|
|
1715
2278
|
this.redisSub.on("message", (channel, message) => {
|
|
1716
2279
|
if (channel === instanceChannel) this.handleRemoteInstanceMessage(message).catch((err) => {
|
|
1717
|
-
|
|
2280
|
+
this.log.error(err, "Error handling remote instance message");
|
|
1718
2281
|
});
|
|
1719
2282
|
else if (channel === schemaChannel) this.handleSchemaReload().catch((err) => {
|
|
1720
|
-
|
|
2283
|
+
this.log.error(err, "Error handling schema reload");
|
|
1721
2284
|
});
|
|
1722
2285
|
else if (channel === watermarkChannel) try {
|
|
1723
2286
|
this.retentionWatermark.set(BigInt(message));
|
|
1724
2287
|
} catch (err) {
|
|
1725
|
-
|
|
2288
|
+
this.log.error(err, `Bad retention watermark message: ${message}`);
|
|
1726
2289
|
}
|
|
1727
|
-
else if (this.appPubSubChannels.has(channel)) Promise.resolve(this.config.appPubSub?.onMessage(channel, message)).catch((err) =>
|
|
2290
|
+
else if (this.appPubSubChannels.has(channel)) Promise.resolve(this.config.appPubSub?.onMessage(channel, message)).catch((err) => this.log.error(err, `Error in appPubSub handler for channel ${channel}`));
|
|
1728
2291
|
});
|
|
1729
2292
|
this.redisSub.on("error", (err) => {
|
|
1730
|
-
|
|
2293
|
+
this.log.error(err, `Subscriber error for channels: ${instanceChannel}, ${schemaChannel}`);
|
|
1731
2294
|
});
|
|
1732
2295
|
}
|
|
1733
2296
|
/**
|
|
@@ -1737,13 +2300,13 @@ var SupaliveWebSocketServer = class {
|
|
|
1737
2300
|
* messages published by this instance (we already have the new types).
|
|
1738
2301
|
*/
|
|
1739
2302
|
async handleSchemaReload() {
|
|
1740
|
-
|
|
2303
|
+
this.log.info(`Schema reload request received in instance ${this.instanceId} — reloading schema...`);
|
|
1741
2304
|
try {
|
|
1742
2305
|
await sleep(1e3 + Math.random() * 5e3);
|
|
1743
2306
|
await bootstrapDbTypes(this.db, this.cache, true);
|
|
1744
|
-
|
|
2307
|
+
this.log.info(`Schema reload applied in instance ${this.instanceId}`);
|
|
1745
2308
|
} catch (err) {
|
|
1746
|
-
|
|
2309
|
+
this.log.error(err, "Failed to reload schema");
|
|
1747
2310
|
return;
|
|
1748
2311
|
}
|
|
1749
2312
|
}
|
|
@@ -1777,7 +2340,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1777
2340
|
try {
|
|
1778
2341
|
parsed = RemoteSubscriptionMessageSchema.parse(JSON.parse(message));
|
|
1779
2342
|
} catch (err) {
|
|
1780
|
-
|
|
2343
|
+
this.log.error(err, "Failed to parse remote instance message");
|
|
1781
2344
|
return;
|
|
1782
2345
|
}
|
|
1783
2346
|
if (parsed.originInstance === this.instanceId) return;
|
|
@@ -1800,7 +2363,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1800
2363
|
async handleRemoteRecomputeRace(req) {
|
|
1801
2364
|
if (!this.hasLocalSessionFor(req.subId)) return;
|
|
1802
2365
|
if (!await this.cache.tryAcquireRecomputeLock(this.instanceId, req.subId, req.commitTs, RECOMPUTE_LOCK_TTL_SECONDS)) {
|
|
1803
|
-
|
|
2366
|
+
this.log.debug(`recompute-race: lost lock for ${req.subId}@${req.commitTs}, waiting for winner's update`);
|
|
1804
2367
|
return;
|
|
1805
2368
|
}
|
|
1806
2369
|
await this.reexecuteAffectedQuery({
|
|
@@ -1826,7 +2389,7 @@ var SupaliveWebSocketServer = class {
|
|
|
1826
2389
|
try {
|
|
1827
2390
|
await redisRetryPolicy.execute(() => this.cache.redis.publish(channel, payload));
|
|
1828
2391
|
} catch (err) {
|
|
1829
|
-
|
|
2392
|
+
this.log.error(err, `Failed to publish to ${channel} after many retries`);
|
|
1830
2393
|
}
|
|
1831
2394
|
});
|
|
1832
2395
|
await Promise.all(publishPromises);
|
|
@@ -1853,29 +2416,6 @@ function sendJson(res, status, body) {
|
|
|
1853
2416
|
res.writeHead(status, { "content-type": "application/json" });
|
|
1854
2417
|
res.end(payload);
|
|
1855
2418
|
}
|
|
1856
|
-
function lowercaseHeaders(headers) {
|
|
1857
|
-
const out = {};
|
|
1858
|
-
for (const [k, v] of Object.entries(headers)) out[k.toLowerCase()] = Array.isArray(v) ? v.join(",") : v;
|
|
1859
|
-
return out;
|
|
1860
|
-
}
|
|
1861
|
-
/** Buffer an incoming request body to a string, capped to guard against abuse. */
|
|
1862
|
-
function readRequestBody(req, maxBytes = 1024 * 1024) {
|
|
1863
|
-
return new Promise((resolve, reject) => {
|
|
1864
|
-
const chunks = [];
|
|
1865
|
-
let size = 0;
|
|
1866
|
-
req.on("data", (chunk) => {
|
|
1867
|
-
size += chunk.length;
|
|
1868
|
-
if (size > maxBytes) {
|
|
1869
|
-
reject(/* @__PURE__ */ new Error("request body too large"));
|
|
1870
|
-
req.destroy();
|
|
1871
|
-
return;
|
|
1872
|
-
}
|
|
1873
|
-
chunks.push(chunk);
|
|
1874
|
-
});
|
|
1875
|
-
req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
|
|
1876
|
-
req.on("error", reject);
|
|
1877
|
-
});
|
|
1878
|
-
}
|
|
1879
2419
|
//#endregion
|
|
1880
2420
|
//#region src/server/hash.ts
|
|
1881
2421
|
/**
|
|
@@ -2223,6 +2763,6 @@ function resolveMaxConnectionsPerWorker(config, workers) {
|
|
|
2223
2763
|
return Math.max(1, Math.floor(total / workers));
|
|
2224
2764
|
}
|
|
2225
2765
|
//#endregion
|
|
2226
|
-
export { ANONYMOUS_IDENTITY, CORE_VERSION, CacheLayer, DB_QUERY_TIMEOUT_MS, DevScheduler, InvalidateWritesetParamsSchema, JobClient, QStashScheduler, RegisterSubscriptionParamsSchema, SubManagerClient, SubscriptionManager, SupaliveWebSocketServer, UnregisterSubscriptionParamsSchema, UpdateSubscriptionReadSetParamsSchema, appInstanceId, bootstrapDbTypes, createDatabase, generateCacheKey, generateSubscriptionId, getHashOf, initCacheLayer, initCore, initDatabase, isProduction, logLevelOf, logger, serializeMapValue };
|
|
2766
|
+
export { ANONYMOUS_IDENTITY, CORE_VERSION, CacheLayer, DB_QUERY_TIMEOUT_MS, DevScheduler, InvalidateWritesetParamsSchema, JobClient, Observability, QStashScheduler, RegisterSubscriptionParamsSchema, SubManagerClient, SubscriptionManager, SupaliveWebSocketServer, UnregisterSubscriptionParamsSchema, UpdateSubscriptionReadSetParamsSchema, appInstanceId, bootstrapDbTypes, configureRootLogger, createDatabase, elapsedMs, generateCacheKey, generateSubscriptionId, getHashOf, getRootLogger, initCacheLayer, initCore, initDatabase, isProduction, logLevelOf, logger, serializeMapValue, startTimer, traceQueryExecution };
|
|
2227
2767
|
|
|
2228
2768
|
//# sourceMappingURL=server.js.map
|