@supalive/core 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/db-ClCicssQ.js +28 -0
- package/dist/db-ClCicssQ.js.map +1 -0
- package/dist/{index-DouKwYL6.d.ts → index-E1_1iRI1.d.ts} +213 -9
- package/dist/index-E1_1iRI1.d.ts.map +1 -0
- package/dist/{mysql-Bxurg_PH.d.ts → mysql-BmkLy5oY.d.ts} +8 -3
- package/dist/mysql-BmkLy5oY.d.ts.map +1 -0
- package/dist/{mysql-bvnjYWvZ.js → mysql-Ca1B_0k_.js} +58 -25
- package/dist/mysql-Ca1B_0k_.js.map +1 -0
- package/dist/{overlap-checker-Ct_HP_iB.js → overlap-checker-ClJC0nxW.js} +52 -4
- package/dist/overlap-checker-ClJC0nxW.js.map +1 -0
- package/dist/{postgres-B8u1YN8X.js → postgres-CHDY1HPf.js} +59 -22
- package/dist/postgres-CHDY1HPf.js.map +1 -0
- package/dist/{postgres-Ck-xt72U.d.ts → postgres-v2ctNE6o.d.ts} +8 -3
- package/dist/postgres-v2ctNE6o.d.ts.map +1 -0
- package/dist/{procedure-DX87tyef.js → procedure-Ze1GE-N9.js} +33 -2
- package/dist/procedure-Ze1GE-N9.js.map +1 -0
- package/dist/{query-Bh0RNxDz.js → query-CrbJ_W0W.js} +21 -1
- package/dist/query-CrbJ_W0W.js.map +1 -0
- package/dist/{realtime_db-DOCgqbwi.js → realtime_db-BtwBABDY.js} +76 -51
- package/dist/realtime_db-BtwBABDY.js.map +1 -0
- package/dist/{router-DlTYWpop.js → router-CiWYeWUR.js} +8 -2
- package/dist/router-CiWYeWUR.js.map +1 -0
- package/dist/src/client/index.d.ts +1 -1
- package/dist/src/client/index.js +44 -5
- package/dist/src/client/index.js.map +1 -1
- package/dist/src/exports/mysql.d.ts +1 -1
- package/dist/src/exports/mysql.js +1 -1
- package/dist/src/exports/postgres.d.ts +1 -1
- package/dist/src/exports/postgres.js +1 -1
- package/dist/src/exports/procedure.d.ts +2 -2
- package/dist/src/exports/procedure.js +4 -4
- package/dist/src/exports/schema-sql.d.ts +35 -2
- package/dist/src/exports/schema-sql.d.ts.map +1 -1
- package/dist/src/exports/schema-sql.js +69 -19
- package/dist/src/exports/schema-sql.js.map +1 -1
- package/dist/src/exports/server.d.ts +64 -17
- package/dist/src/exports/server.d.ts.map +1 -1
- package/dist/src/exports/server.js +726 -64
- package/dist/src/exports/server.js.map +1 -1
- package/dist/src/exports/subscription-manager-worker-entry.js +8 -5
- package/dist/src/exports/subscription-manager-worker-entry.js.map +1 -1
- package/dist/src/exports/types.d.ts +3 -3
- package/dist/src/exports/types.js +1 -1
- package/dist/{subscription-worker-CqoWp6zB.js → subscription-worker-CAf6MUta.js} +306 -69
- package/dist/subscription-worker-CAf6MUta.js.map +1 -0
- package/dist/{types_db-DXj_l9n-.js → types_db-CTRm_awX.js} +7 -2
- package/dist/types_db-CTRm_awX.js.map +1 -0
- package/dist/types_server-CzAufcee.d.ts +420 -0
- package/dist/types_server-CzAufcee.d.ts.map +1 -0
- package/package.json +5 -4
- package/dist/index-CaWMDqq0.d.ts +0 -1754
- package/dist/index-CaWMDqq0.d.ts.map +0 -1
- package/dist/index-DouKwYL6.d.ts.map +0 -1
- package/dist/mysql-Bxurg_PH.d.ts.map +0 -1
- package/dist/mysql-Cxd4Uq5p.d.ts +0 -104
- package/dist/mysql-Cxd4Uq5p.d.ts.map +0 -1
- package/dist/mysql-bvnjYWvZ.js.map +0 -1
- package/dist/overlap-checker-Ct_HP_iB.js.map +0 -1
- package/dist/postgres-B8u1YN8X.js.map +0 -1
- package/dist/postgres-CfT_aDx3.d.ts +0 -108
- package/dist/postgres-CfT_aDx3.d.ts.map +0 -1
- package/dist/postgres-Ck-xt72U.d.ts.map +0 -1
- package/dist/procedure-DX87tyef.js.map +0 -1
- package/dist/query-Bh0RNxDz.js.map +0 -1
- package/dist/realtime_db-DOCgqbwi.js.map +0 -1
- package/dist/router-DlTYWpop.js.map +0 -1
- package/dist/src/exports/codegen.d.ts +0 -38
- package/dist/src/exports/codegen.d.ts.map +0 -1
- package/dist/src/exports/codegen.js +0 -134
- package/dist/src/exports/codegen.js.map +0 -1
- package/dist/subscription-worker-CqoWp6zB.js.map +0 -1
- package/dist/subscription-worker-PtmJWEj1.js +0 -511
- package/dist/subscription-worker-PtmJWEj1.js.map +0 -1
- package/dist/subscription-worker-pahX0zxg.js +0 -511
- package/dist/subscription-worker-pahX0zxg.js.map +0 -1
- package/dist/types_db-DXj_l9n-.js.map +0 -1
- package/dist/types_server-BW1ys_SK.d.ts +0 -216
- package/dist/types_server-BW1ys_SK.d.ts.map +0 -1
- package/dist/types_server-Dpsi0wpG.d.ts +0 -216
- package/dist/types_server-Dpsi0wpG.d.ts.map +0 -1
- package/dist/types_server-DzrIccyO.d.ts +0 -216
- package/dist/types_server-DzrIccyO.d.ts.map +0 -1
- package/dist/types_server-TFo6SWDH.d.ts +0 -216
- package/dist/types_server-TFo6SWDH.d.ts.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { a as mapWriteSetToRaw } from "../../overlap-checker-ClJC0nxW.js";
|
|
2
|
+
import { a as createDatabase, b as UpdateSubscriptionReadSetParamsSchema, f as RegisterSubscriptionBatchParamsSchema, g as UnregisterSubscriptionParamsSchema, m as RegisterSubscriptionParamsSchema, o as initCacheLayer, t as SubscriptionWorker, u as InvalidateWritesetParamsSchema, v as UnregisterSubscriptionsParamsSchema } from "../../subscription-worker-CAf6MUta.js";
|
|
3
3
|
import { parentPort } from "node:worker_threads";
|
|
4
4
|
//#region src/server/subscription-manager-worker-entry.ts
|
|
5
5
|
if (!parentPort) throw new Error("subscription-manager-worker-entry must run inside a worker_threads Worker");
|
|
@@ -17,7 +17,6 @@ async function init(payload) {
|
|
|
17
17
|
commandMaxRetries: 5,
|
|
18
18
|
createSubClient: false
|
|
19
19
|
});
|
|
20
|
-
let db;
|
|
21
20
|
if (payload.database.type === "postgres") db = createDatabase({
|
|
22
21
|
type: "postgres",
|
|
23
22
|
connectionString: payload.database.connectionString,
|
|
@@ -73,11 +72,15 @@ async function dispatch(req) {
|
|
|
73
72
|
case 9: return db.getLatestSnapshotTimestamp();
|
|
74
73
|
case 10: {
|
|
75
74
|
const ts = req.payload;
|
|
76
|
-
return
|
|
75
|
+
return worker.prune(ts);
|
|
77
76
|
}
|
|
77
|
+
case 12: return worker.refreshRetentionWatermark();
|
|
78
|
+
case 13:
|
|
79
|
+
worker.setRetentionWatermark(req.payload);
|
|
80
|
+
return null;
|
|
78
81
|
case 11:
|
|
79
82
|
try {
|
|
80
|
-
await
|
|
83
|
+
await db?.close();
|
|
81
84
|
} catch (_) {}
|
|
82
85
|
try {
|
|
83
86
|
await void 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"subscription-manager-worker-entry.js","names":[],"sources":["../../../src/server/subscription-manager-worker-entry.ts"],"sourcesContent":["import { parentPort } from \"node:worker_threads\";\nimport Redis from \"ioredis\";\nimport type { Database } from \"../db/db\";\nimport { mapWriteSetToRaw } from \"./overlap-checker\";\nimport {\n InvalidateWritesetParamsSchema,\n RegisterSubscriptionBatchParamsSchema,\n RegisterSubscriptionParamsSchema,\n UnregisterSubscriptionParamsSchema,\n UnregisterSubscriptionsParamsSchema,\n UpdateSubscriptionReadSetParamsSchema,\n} from \"./subscription-manager-types\";\nimport { SubscriptionWorker } from \"./subscription-worker\";\nimport {\n MsgType,\n type WorkerInitPayload,\n type WorkerRequest,\n type WorkerResponse,\n} from \"./subscription-manager-worker-protocol\";\nimport { createDatabase, initCacheLayer } from \"../db/init_db\";\n\nif (!parentPort) {\n throw new Error(\"subscription-manager-worker-entry must run inside a worker_threads Worker\");\n}\n\nconst port = parentPort;\n\nlet worker: SubscriptionWorker | null = null;\nlet db: Database | null = null;\nlet redis: Redis | null = null;\n\nasync function init(payload: WorkerInitPayload): Promise<void> {\n const { cacheLayer } = initCacheLayer({\n upstash: {\n url: `https://${process.env.UPSTASH_REDIS_HOST}`,\n token: process.env.UPSTASH_REDIS_TOKEN ?? \"\",\n redisUrl: process.env.UPSTASH_REDIS_URL ?? \"\"\n },\n cacheTtlSeconds: payload.cacheTTLSeconds,\n commandMaxRetries: 5,\n createSubClient: false,\n });\n\n
|
|
1
|
+
{"version":3,"file":"subscription-manager-worker-entry.js","names":[],"sources":["../../../src/server/subscription-manager-worker-entry.ts"],"sourcesContent":["import { parentPort } from \"node:worker_threads\";\nimport Redis from \"ioredis\";\nimport type { Database } from \"../db/db\";\nimport { mapWriteSetToRaw } from \"./overlap-checker\";\nimport {\n InvalidateWritesetParamsSchema,\n RegisterSubscriptionBatchParamsSchema,\n RegisterSubscriptionParamsSchema,\n UnregisterSubscriptionParamsSchema,\n UnregisterSubscriptionsParamsSchema,\n UpdateSubscriptionReadSetParamsSchema,\n} from \"./subscription-manager-types\";\nimport { SubscriptionWorker } from \"./subscription-worker\";\nimport {\n MsgType,\n type WorkerInitPayload,\n type WorkerRequest,\n type WorkerResponse,\n} from \"./subscription-manager-worker-protocol\";\nimport { createDatabase, initCacheLayer } from \"../db/init_db\";\n\nif (!parentPort) {\n throw new Error(\"subscription-manager-worker-entry must run inside a worker_threads Worker\");\n}\n\nconst port = parentPort;\n\nlet worker: SubscriptionWorker | null = null;\nlet db: Database | null = null;\nlet redis: Redis | null = null;\n\nasync function init(payload: WorkerInitPayload): Promise<void> {\n const { cacheLayer } = initCacheLayer({\n upstash: {\n url: `https://${process.env.UPSTASH_REDIS_HOST}`,\n token: process.env.UPSTASH_REDIS_TOKEN ?? \"\",\n redisUrl: process.env.UPSTASH_REDIS_URL ?? \"\"\n },\n cacheTtlSeconds: payload.cacheTTLSeconds,\n commandMaxRetries: 5,\n createSubClient: false,\n });\n\n if (payload.database.type === \"postgres\") {\n db = createDatabase({\n type: \"postgres\",\n connectionString: payload.database.connectionString,\n maxConnections: payload.maxConnectionsPerWorker,\n minConnections: payload.database.minConnections ?? 1,\n idleTimeoutMillis: payload.database.idleTimeoutMillis ?? 30_000,\n connectionTimeoutMillis: payload.database.connectionTimeoutMillis ?? 7_000,\n }, payload.workerId);\n } else {\n db = createDatabase({\n type: \"mysql\",\n connectionString: payload.database.connectionString,\n maxConnections: payload.maxConnectionsPerWorker,\n minConnections: payload.database.minConnections ?? 1,\n idleTimeoutMillis: payload.database.idleTimeoutMillis ?? 30_000,\n connectionTimeoutMillis: payload.database.connectionTimeoutMillis ?? 7_000,\n queueLimit: payload.database.queueLimit ?? 0,\n }, payload.workerId);\n }\n\n worker = new SubscriptionWorker(payload.workerId, cacheLayer, db);\n}\n\nasync function dispatch(req: WorkerRequest): Promise<unknown> {\n if (req.type === MsgType.Init) {\n await init(req.payload as WorkerInitPayload);\n return null;\n }\n\n if (!worker) {\n throw new Error(\"Worker received message before Init\");\n }\n\n switch (req.type) {\n case MsgType.Register: {\n const params = RegisterSubscriptionParamsSchema.parse(req.payload);\n return worker.register(params);\n }\n case MsgType.RegisterBatch: {\n const params = RegisterSubscriptionBatchParamsSchema.parse(req.payload);\n return Promise.all(params.map((p) => worker!.register(p)));\n }\n case MsgType.UpdateReadSet: {\n const params = UpdateSubscriptionReadSetParamsSchema.parse(req.payload);\n return worker.updateReadSet(params);\n }\n case MsgType.Unregister: {\n const params = UnregisterSubscriptionParamsSchema.parse(req.payload);\n return worker.unregister(params);\n }\n case MsgType.Unregisters: {\n const params = UnregisterSubscriptionsParamsSchema.parse(req.payload);\n return worker.unregisters(params);\n }\n case MsgType.Invalidate: {\n const { writeSet } = InvalidateWritesetParamsSchema.parse(req.payload);\n const rawWriteSet = mapWriteSetToRaw(writeSet);\n return worker.invalidate(rawWriteSet);\n }\n case MsgType.MinSubscriptionTs: {\n const m = worker.minSubscriptionTs();\n // bigint isn't structured-cloneable via JSON; postMessage handles\n // bigints natively, so we return it as-is.\n return m;\n }\n case MsgType.LatestSnapshotTs: {\n return db!.getLatestSnapshotTimestamp();\n }\n case MsgType.Prune: {\n const ts = req.payload as bigint;\n // Route through the worker so it deletes AND republishes the new watermark.\n return worker.prune(ts);\n }\n case MsgType.RefreshRetentionWatermark: {\n // Seed/repair the watermark from the DB and return it so the manager can\n // fan it out to the other workers.\n return worker.refreshRetentionWatermark();\n }\n case MsgType.SetRetentionWatermark: {\n worker.setRetentionWatermark(req.payload as bigint);\n return null;\n }\n case MsgType.Shutdown: {\n try { await db?.close(); } catch (_) { /* ignore */ }\n try { await (redis as any)?.quit?.(); } catch (_) { /* ignore */ }\n return null;\n }\n default: {\n throw new Error(`Unknown MsgType: ${req.type satisfies never}`);\n }\n }\n}\n\nport.on(\"message\", async (req: WorkerRequest) => {\n try {\n const result = await dispatch(req);\n const res: WorkerResponse = { reqId: req.reqId, ok: true, result };\n port.postMessage(res);\n } catch (err) {\n const e = err as Error;\n const res: WorkerResponse = {\n reqId: req.reqId,\n ok: false,\n error: { message: e?.message ?? String(err), stack: e?.stack },\n };\n port.postMessage(res);\n }\n});\n"],"mappings":";;;;AAqBA,IAAI,CAAC,YACH,MAAM,IAAI,MAAM,2EAA2E;AAG7F,MAAM,OAAO;AAEb,IAAI,SAAoC;AACxC,IAAI,KAAsB;AAG1B,eAAe,KAAK,SAA2C;CAC7D,MAAM,EAAE,eAAe,eAAe;EACpC,SAAS;GACP,KAAK,WAAW,QAAQ,IAAI;GAC5B,OAAO,QAAQ,IAAI,uBAAuB;GAC1C,UAAU,QAAQ,IAAI,qBAAqB;EAC7C;EACA,iBAAiB,QAAQ;EACzB,mBAAmB;EACnB,iBAAiB;CACnB,CAAC;CAED,IAAI,QAAQ,SAAS,SAAS,YAC5B,KAAK,eAAe;EAClB,MAAM;EACN,kBAAkB,QAAQ,SAAS;EACnC,gBAAgB,QAAQ;EACxB,gBAAgB,QAAQ,SAAS,kBAAkB;EACnD,mBAAmB,QAAQ,SAAS,qBAAqB;EACzD,yBAAyB,QAAQ,SAAS,2BAA2B;CACvE,GAAG,QAAQ,QAAQ;MAEnB,KAAK,eAAe;EAClB,MAAM;EACN,kBAAkB,QAAQ,SAAS;EACnC,gBAAgB,QAAQ;EACxB,gBAAgB,QAAQ,SAAS,kBAAkB;EACnD,mBAAmB,QAAQ,SAAS,qBAAqB;EACzD,yBAAyB,QAAQ,SAAS,2BAA2B;EACrE,YAAY,QAAQ,SAAS,cAAc;CAC7C,GAAG,QAAQ,QAAQ;CAGrB,SAAS,IAAI,mBAAmB,QAAQ,UAAU,YAAY,EAAE;AAClE;AAEA,eAAe,SAAS,KAAsC;CAC5D,IAAI,IAAI,SAAA,GAAuB;EAC7B,MAAM,KAAK,IAAI,OAA4B;EAC3C,OAAO;CACT;CAEA,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,qCAAqC;CAGvD,QAAQ,IAAI,MAAZ;EACE,KAAA,GAAuB;GACrB,MAAM,SAAS,iCAAiC,MAAM,IAAI,OAAO;GACjE,OAAO,OAAO,SAAS,MAAM;EAC/B;EACA,KAAA,GAA4B;GAC1B,MAAM,SAAS,sCAAsC,MAAM,IAAI,OAAO;GACtE,OAAO,QAAQ,IAAI,OAAO,KAAK,MAAM,OAAQ,SAAS,CAAC,CAAC,CAAC;EAC3D;EACA,KAAA,GAA4B;GAC1B,MAAM,SAAS,sCAAsC,MAAM,IAAI,OAAO;GACtE,OAAO,OAAO,cAAc,MAAM;EACpC;EACA,KAAA,GAAyB;GACvB,MAAM,SAAS,mCAAmC,MAAM,IAAI,OAAO;GACnE,OAAO,OAAO,WAAW,MAAM;EACjC;EACA,KAAA,GAA0B;GACxB,MAAM,SAAS,oCAAoC,MAAM,IAAI,OAAO;GACpE,OAAO,OAAO,YAAY,MAAM;EAClC;EACA,KAAA,GAAyB;GACvB,MAAM,EAAE,aAAa,+BAA+B,MAAM,IAAI,OAAO;GACrE,MAAM,cAAc,iBAAiB,QAAQ;GAC7C,OAAO,OAAO,WAAW,WAAW;EACtC;EACA,KAAA,GAIE,OAHU,OAAO,kBAGV;EAET,KAAA,GACE,OAAO,GAAI,2BAA2B;EAExC,KAAA,IAAoB;GAClB,MAAM,KAAK,IAAI;GAEf,OAAO,OAAO,MAAM,EAAE;EACxB;EACA,KAAA,IAGE,OAAO,OAAO,0BAA0B;EAE1C,KAAA;GACE,OAAO,sBAAsB,IAAI,OAAiB;GAClD,OAAO;EAET,KAAA;GACE,IAAI;IAAE,MAAM,IAAI,MAAM;GAAG,SAAS,GAAG,CAAe;GACpD,IAAI;IAAE,MAAM,KAAA;GAA0B,SAAS,GAAG,CAAe;GACjE,OAAO;EAET,SACE,MAAM,IAAI,MAAM,oBAAoB,IAAI,MAAsB;CAElE;AACF;AAEA,KAAK,GAAG,WAAW,OAAO,QAAuB;CAC/C,IAAI;EACF,MAAM,SAAS,MAAM,SAAS,GAAG;EACjC,MAAM,MAAsB;GAAE,OAAO,IAAI;GAAO,IAAI;GAAM;EAAO;EACjE,KAAK,YAAY,GAAG;CACtB,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,MAAsB;GAC1B,OAAO,IAAI;GACX,IAAI;GACJ,OAAO;IAAE,SAAS,GAAG,WAAW,OAAO,GAAG;IAAG,OAAO,GAAG;GAAM;EAC/D;EACA,KAAK,YAAY,GAAG;CACtB;AACF,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $n as
|
|
1
|
+
import { $n as AndPredicate, A as ResponseMessageSchema, Ar as RawPointReadSchema, B as SubscriptionUpdateMessageSchema, Br as WriteOp, C as ClientMessageSchema, Cr as QueryCacheEntrySchema, D as ErrorMessage, Dr as RangeRead, E as Context, Er as QuerySpec, F as SubscribeMessage, Fr as ReadEntry, H as UnsubscribeMessageSchema, Hr as bytesFromJson, I as SubscribeMessageSchema, Ir as ReadEntrySchema, L as SubscriptionGoneMessage, Lr as RetryConfig, M as ServerMessageSchema, Mr as RawRangeReadSchema, N as ServerPingMessage, Nr as RawReadEntry, O as IsInternal, Or as RangeReadSchema, P as ServerPingMessageSchema, Pr as RawReadEntrySchema, R as SubscriptionGoneMessageSchema, Rr as WriteEntry, S as ClientMessage, Sr as QueryCacheEntry, T as ConnectedMessageSchema, Tr as QueryCacheMetadataSchema, Ur as normalizeIdToBytes, V as UnsubscribeMessage, Vr as WriteOpSchema, Wr as normalizeToBytes, _ as AuthSuccessMessage, _r as OrPredicateSchema, ar as CommitTs, b as CallMessage, br as Predicate, cr as DEFAULT_RETRY, dr as LiveResult, er as AndPredicateSchema, fr as MutationResult, g as AuthMessageSchema, gr as OrPredicate, h as AuthMessage, hr as OccConflictError, ir as CommitLogEntry, j as ServerMessage, jr as RawRangeRead, k as ResponseMessage, kr as RawPointRead, lr as LeafPredicate, m as AuthFailedMessageSchema, mr as OccAbortError, nr as CachedPgMetadata, or as CompareOperator, p as AuthFailedMessage, pr as NO_RETRY, rr as CachedPgMetadataSchema, sr as CompareOperatorSchema, tr as BigIntSchema, ur as LeafPredicateSchema, v as AuthSuccessMessageSchema, vr as PointRead, w as ConnectedMessage, wr as QueryCacheMetadata, x as CallMessageSchema, xr as PredicateSchema, y as AuthenticationError, yr as PointReadSchema, z as SubscriptionUpdateMessage, zr as WriteEntrySchema } from "../../index-E1_1iRI1.js";
|
|
2
2
|
import { n as stableStringify, r as supaliveStringify, t as groupByToMap } from "../../helper-CiacMqje.js";
|
|
3
|
-
import { _ as
|
|
4
|
-
export { AndPredicate, AndPredicateSchema, AuthFailedMessage, AuthFailedMessageSchema, AuthMessage, AuthMessageSchema, AuthSuccessMessage, AuthSuccessMessageSchema, AuthenticationError, BigIntSchema, CachedPgMetadata, CachedPgMetadataSchema, CallMessage, CallMessageSchema, ClientMessage, ClientMessageSchema, CommitLogEntry, CommitTs, CompareOperator, CompareOperatorSchema, ConnectedMessage, ConnectedMessageSchema, Context, DEFAULT_RETRY, DatabaseConfig, ErrorMessage, InstanceId, IsInternal, LeafPredicate, LeafPredicateSchema, LiveResult, MutationResult, MySQLConfig, NO_RETRY, OccAbortError, OccConflictError, OrPredicate, OrPredicateSchema, PendingMutation, PointRead, PointReadSchema, PostgresConfig, Predicate, PredicateSchema, QueryCacheEntry, QueryCacheEntrySchema, QueryCacheMetadata, QueryCacheMetadataSchema, QuerySpec, RangeRead, RangeReadSchema, RawPointRead, RawPointReadSchema, RawRangeRead, RawRangeReadSchema, RawReadEntry, RawReadEntrySchema, ReadEntry, ReadEntrySchema, RemoteRecomputeRace, RemoteRecomputeRaceSchema, RemoteSubscriptionMessage, RemoteSubscriptionMessageSchema, RemoteSubscriptionUpdate, RemoteSubscriptionUpdateSchema, ResponseMessage, ResponseMessageSchema, RetryConfig, ServerMessage, ServerMessageSchema, ServerPingMessage, ServerPingMessageSchema, Session, SubId, SubscribeMessage, SubscribeMessageSchema, SubscriptionEntry, SubscriptionGoneMessage, SubscriptionGoneMessageSchema, SubscriptionManagerConfig, SubscriptionUpdateMessage, SubscriptionUpdateMessageSchema, SupaliveServerConfig, UnsubscribeMessage, UnsubscribeMessageSchema, UpstashConfig, WriteEntry, WriteEntrySchema, WriteOp, WriteOpSchema, bytesFromJson, groupByToMap, normalizeIdToBytes, normalizeToBytes, stableStringify, supaliveStringify };
|
|
3
|
+
import { _ as RpcRateLimitConfig, a as RemoteSubscriptionMessage, b as UpstashConfig, c as RemoteSubscriptionUpdateSchema, d as SubscriptionEntry, f as DatabaseConfig, g as RpcCorsConfig, h as RpcConfig, i as RemoteRecomputeRaceSchema, l as Session, m as PostgresConfig, n as PendingMutation, o as RemoteSubscriptionMessageSchema, p as MySQLConfig, r as RemoteRecomputeRace, s as RemoteSubscriptionUpdate, t as InstanceId, u as SubId, v as SubscriptionManagerConfig, y as SupaliveServerConfig } from "../../types_server-CzAufcee.js";
|
|
4
|
+
export { AndPredicate, AndPredicateSchema, AuthFailedMessage, AuthFailedMessageSchema, AuthMessage, AuthMessageSchema, AuthSuccessMessage, AuthSuccessMessageSchema, AuthenticationError, BigIntSchema, CachedPgMetadata, CachedPgMetadataSchema, CallMessage, CallMessageSchema, ClientMessage, ClientMessageSchema, CommitLogEntry, CommitTs, CompareOperator, CompareOperatorSchema, ConnectedMessage, ConnectedMessageSchema, Context, DEFAULT_RETRY, DatabaseConfig, ErrorMessage, InstanceId, IsInternal, LeafPredicate, LeafPredicateSchema, LiveResult, MutationResult, MySQLConfig, NO_RETRY, OccAbortError, OccConflictError, OrPredicate, OrPredicateSchema, PendingMutation, PointRead, PointReadSchema, PostgresConfig, Predicate, PredicateSchema, QueryCacheEntry, QueryCacheEntrySchema, QueryCacheMetadata, QueryCacheMetadataSchema, QuerySpec, RangeRead, RangeReadSchema, RawPointRead, RawPointReadSchema, RawRangeRead, RawRangeReadSchema, RawReadEntry, RawReadEntrySchema, ReadEntry, ReadEntrySchema, RemoteRecomputeRace, RemoteRecomputeRaceSchema, RemoteSubscriptionMessage, RemoteSubscriptionMessageSchema, RemoteSubscriptionUpdate, RemoteSubscriptionUpdateSchema, ResponseMessage, ResponseMessageSchema, RetryConfig, RpcConfig, RpcCorsConfig, RpcRateLimitConfig, ServerMessage, ServerMessageSchema, ServerPingMessage, ServerPingMessageSchema, Session, SubId, SubscribeMessage, SubscribeMessageSchema, SubscriptionEntry, SubscriptionGoneMessage, SubscriptionGoneMessageSchema, SubscriptionManagerConfig, SubscriptionUpdateMessage, SubscriptionUpdateMessageSchema, SupaliveServerConfig, UnsubscribeMessage, UnsubscribeMessageSchema, UpstashConfig, WriteEntry, WriteEntrySchema, WriteOp, WriteOpSchema, bytesFromJson, groupByToMap, normalizeIdToBytes, normalizeToBytes, stableStringify, supaliveStringify };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as stableStringify, r as supaliveStringify, t as groupByToMap } from "../../helper-wGbzz6zo.js";
|
|
2
|
-
import { C as normalizeIdToBytes, S as bytesFromJson, _ as RawRangeReadSchema, a as DEFAULT_RETRY, b as WriteEntrySchema, c as OccAbortError, d as PointReadSchema, f as PredicateSchema, g as RawPointReadSchema, h as RangeReadSchema, i as CompareOperatorSchema, l as OccConflictError, m as QueryCacheMetadataSchema, n as BigIntSchema, o as LeafPredicateSchema, p as QueryCacheEntrySchema, r as CachedPgMetadataSchema, s as NO_RETRY, t as AndPredicateSchema, u as OrPredicateSchema, v as RawReadEntrySchema, w as normalizeToBytes, x as WriteOpSchema, y as ReadEntrySchema } from "../../types_db-
|
|
2
|
+
import { C as normalizeIdToBytes, S as bytesFromJson, _ as RawRangeReadSchema, a as DEFAULT_RETRY, b as WriteEntrySchema, c as OccAbortError, d as PointReadSchema, f as PredicateSchema, g as RawPointReadSchema, h as RangeReadSchema, i as CompareOperatorSchema, l as OccConflictError, m as QueryCacheMetadataSchema, n as BigIntSchema, o as LeafPredicateSchema, p as QueryCacheEntrySchema, r as CachedPgMetadataSchema, s as NO_RETRY, t as AndPredicateSchema, u as OrPredicateSchema, v as RawReadEntrySchema, w as normalizeToBytes, x as WriteOpSchema, y as ReadEntrySchema } from "../../types_db-CTRm_awX.js";
|
|
3
3
|
import { a as CallMessageSchema, c as ResponseMessageSchema, d as SubscribeMessageSchema, f as SubscriptionGoneMessageSchema, i as AuthenticationError, l as ServerMessageSchema, m as UnsubscribeMessageSchema, n as AuthMessageSchema, o as ClientMessageSchema, p as SubscriptionUpdateMessageSchema, r as AuthSuccessMessageSchema, s as ConnectedMessageSchema, t as AuthFailedMessageSchema, u as ServerPingMessageSchema } from "../../types_client_rpc-CIsjQQGG.js";
|
|
4
4
|
import { n as RemoteSubscriptionMessageSchema, r as RemoteSubscriptionUpdateSchema, t as RemoteRecomputeRaceSchema } from "../../types_server-Dk6o_B7N.js";
|
|
5
5
|
export { AndPredicateSchema, AuthFailedMessageSchema, AuthMessageSchema, AuthSuccessMessageSchema, AuthenticationError, BigIntSchema, CachedPgMetadataSchema, CallMessageSchema, ClientMessageSchema, CompareOperatorSchema, ConnectedMessageSchema, DEFAULT_RETRY, LeafPredicateSchema, NO_RETRY, OccAbortError, OccConflictError, OrPredicateSchema, PointReadSchema, PredicateSchema, QueryCacheEntrySchema, QueryCacheMetadataSchema, RangeReadSchema, RawPointReadSchema, RawRangeReadSchema, RawReadEntrySchema, ReadEntrySchema, RemoteRecomputeRaceSchema, RemoteSubscriptionMessageSchema, RemoteSubscriptionUpdateSchema, ResponseMessageSchema, ServerMessageSchema, ServerPingMessageSchema, SubscribeMessageSchema, SubscriptionGoneMessageSchema, SubscriptionUpdateMessageSchema, UnsubscribeMessageSchema, WriteEntrySchema, WriteOpSchema, bytesFromJson, groupByToMap, normalizeIdToBytes, normalizeToBytes, stableStringify, supaliveStringify };
|
|
@@ -1,15 +1,86 @@
|
|
|
1
1
|
import { r as supaliveStringify } from "./helper-wGbzz6zo.js";
|
|
2
|
-
import { b as WriteEntrySchema, m as QueryCacheMetadataSchema, n as BigIntSchema, p as QueryCacheEntrySchema, r as CachedPgMetadataSchema, y as ReadEntrySchema } from "./types_db-
|
|
3
|
-
import {
|
|
4
|
-
import { t as MySqlDatabase } from "./mysql-
|
|
5
|
-
import { t as PgDatabase } from "./postgres-
|
|
6
|
-
import { t as SupaliveDb } from "./realtime_db-
|
|
2
|
+
import { b as WriteEntrySchema, m as QueryCacheMetadataSchema, n as BigIntSchema, p as QueryCacheEntrySchema, r as CachedPgMetadataSchema, y as ReadEntrySchema } from "./types_db-CTRm_awX.js";
|
|
3
|
+
import { i as mapReadSetToRaw, n as checkWriteSetAffectsReadSet, r as evaluateCacheFreshness } from "./overlap-checker-ClJC0nxW.js";
|
|
4
|
+
import { t as MySqlDatabase } from "./mysql-Ca1B_0k_.js";
|
|
5
|
+
import { t as PgDatabase } from "./postgres-CHDY1HPf.js";
|
|
6
|
+
import { t as SupaliveDb } from "./realtime_db-BtwBABDY.js";
|
|
7
7
|
import z from "zod";
|
|
8
8
|
import { v4 } from "uuid";
|
|
9
9
|
import { ExponentialBackoff, handleAll, retry } from "cockatiel";
|
|
10
10
|
import pino from "pino";
|
|
11
11
|
import "pino-pretty";
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
12
13
|
import Redis$1 from "ioredis";
|
|
14
|
+
//#region src/logger.ts
|
|
15
|
+
const appInstanceId = `instance-${v4().slice(0, 8)}`;
|
|
16
|
+
const isProduction = process.env.ENV === "prod";
|
|
17
|
+
const logger$1 = pino({
|
|
18
|
+
level: process.env.LOG_LEVEL || "info",
|
|
19
|
+
timestamp: pino.stdTimeFunctions.isoTime,
|
|
20
|
+
redact: { paths: ["user.password"] },
|
|
21
|
+
transport: isProduction ? void 0 : {
|
|
22
|
+
target: "pino-pretty",
|
|
23
|
+
options: { colorize: true }
|
|
24
|
+
},
|
|
25
|
+
serializers: { "columnsTypes": (map) => {
|
|
26
|
+
if (map instanceof Map) return serializeMapValue(map);
|
|
27
|
+
return map;
|
|
28
|
+
} },
|
|
29
|
+
base: { "srv": appInstanceId }
|
|
30
|
+
});
|
|
31
|
+
function logLevelOf(name) {
|
|
32
|
+
const defaultLogLevel = process.env.DEFAULT_LOG_LEVEL || "info";
|
|
33
|
+
return process.env[name]?.toLowerCase() || defaultLogLevel;
|
|
34
|
+
}
|
|
35
|
+
function serializeMapValue(map) {
|
|
36
|
+
const obj = {};
|
|
37
|
+
for (const [k, v] of map.entries()) {
|
|
38
|
+
obj[k] = {};
|
|
39
|
+
if (v instanceof Map) for (const [innerK, innerV] of v.entries()) obj[k][innerK] = innerV;
|
|
40
|
+
else obj[k] = v;
|
|
41
|
+
}
|
|
42
|
+
return obj;
|
|
43
|
+
}
|
|
44
|
+
//#endregion
|
|
45
|
+
//#region src/server/retention-watermark.ts
|
|
46
|
+
/** Redis pub/sub channel the sub-manager publishes the retention watermark on
|
|
47
|
+
* after each prune (and at startup). App-server instances subscribe and update
|
|
48
|
+
* their in-memory {@link RetentionWatermark}. Payload is the watermark `ts` as a
|
|
49
|
+
* string. */
|
|
50
|
+
const RETENTION_WATERMARK_CHANNEL = "supalive:retention_watermark";
|
|
51
|
+
/**
|
|
52
|
+
* In-memory holder for the commit-log retention watermark: the oldest commit-log
|
|
53
|
+
* ts still retained after prune (see {@link Database.getMinSnapshotTimestamp}).
|
|
54
|
+
* Read on the hot path by the query-cache freshness check — a plain variable, no
|
|
55
|
+
* per-check Redis/DB round trip. Loaded once at startup from the `metadata`
|
|
56
|
+
* table and kept fresh out-of-band: the sub-manager broadcasts the new watermark
|
|
57
|
+
* (in-process to its workers, over Redis pub/sub to app servers) each time it
|
|
58
|
+
* prunes.
|
|
59
|
+
*
|
|
60
|
+
* `set` is monotonic — the watermark only ever moves forward (pruning never
|
|
61
|
+
* un-deletes rows), so an out-of-order or stale update can never lower it, which
|
|
62
|
+
* keeps the freshness guard on the safe side (an over-estimate only costs an
|
|
63
|
+
* extra recompute; an under-estimate could serve stale data).
|
|
64
|
+
*
|
|
65
|
+
* Initial value is the -1n genesis (not 0), matching `getMinSnapshotTimestamp`
|
|
66
|
+
* and the empty-DB snapshot ts — so before anything is pruned the guard never
|
|
67
|
+
* rejects a genesis-era entry (whose snapshot is -1).
|
|
68
|
+
*/
|
|
69
|
+
var RetentionWatermark = class {
|
|
70
|
+
ts = -1n;
|
|
71
|
+
get() {
|
|
72
|
+
return this.ts;
|
|
73
|
+
}
|
|
74
|
+
set(ts) {
|
|
75
|
+
if (ts > this.ts) this.ts = ts;
|
|
76
|
+
}
|
|
77
|
+
/** Seed from the persisted watermark (metadata table). Returns the current value. */
|
|
78
|
+
async loadFrom(db) {
|
|
79
|
+
this.set(await db.getMinSnapshotTimestamp());
|
|
80
|
+
return this.ts;
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
//#endregion
|
|
13
84
|
//#region src/server/subscription-manager-types.ts
|
|
14
85
|
const RegisterSubscriptionParamsSchema = z.object({
|
|
15
86
|
/**
|
|
@@ -35,7 +106,8 @@ const RegisterSubscriptionParamsSchema = z.object({
|
|
|
35
106
|
const RegisterSubscriptionResultSchema = z.object({
|
|
36
107
|
cacheKey: z.string(),
|
|
37
108
|
recompute: z.boolean(),
|
|
38
|
-
readSet: z.array(ReadEntrySchema)
|
|
109
|
+
readSet: z.array(ReadEntrySchema),
|
|
110
|
+
version: z.string().optional()
|
|
39
111
|
});
|
|
40
112
|
const RegisterSubscriptionBatchParamsSchema = z.array(RegisterSubscriptionParamsSchema);
|
|
41
113
|
const RegisterSubscriptionBatchResultSchema = z.array(RegisterSubscriptionResultSchema);
|
|
@@ -82,37 +154,6 @@ const RegistrationRecordSchema = z.object({
|
|
|
82
154
|
cacheKey: z.string()
|
|
83
155
|
});
|
|
84
156
|
//#endregion
|
|
85
|
-
//#region src/logger.ts
|
|
86
|
-
const appInstanceId = `instance-${v4().slice(0, 8)}`;
|
|
87
|
-
const isProduction = process.env.ENV === "prod";
|
|
88
|
-
const logger$1 = pino({
|
|
89
|
-
level: process.env.LOG_LEVEL || "info",
|
|
90
|
-
timestamp: pino.stdTimeFunctions.isoTime,
|
|
91
|
-
redact: { paths: ["user.password"] },
|
|
92
|
-
transport: isProduction ? void 0 : {
|
|
93
|
-
target: "pino-pretty",
|
|
94
|
-
options: { colorize: true }
|
|
95
|
-
},
|
|
96
|
-
serializers: { "columnsTypes": (map) => {
|
|
97
|
-
if (map instanceof Map) return serializeMapValue(map);
|
|
98
|
-
return map;
|
|
99
|
-
} },
|
|
100
|
-
base: { "srv": appInstanceId }
|
|
101
|
-
});
|
|
102
|
-
function logLevelOf(name) {
|
|
103
|
-
const defaultLogLevel = process.env.DEFAULT_LOG_LEVEL || "info";
|
|
104
|
-
return process.env[name]?.toLowerCase() || defaultLogLevel;
|
|
105
|
-
}
|
|
106
|
-
function serializeMapValue(map) {
|
|
107
|
-
const obj = {};
|
|
108
|
-
for (const [k, v] of map.entries()) {
|
|
109
|
-
obj[k] = {};
|
|
110
|
-
if (v instanceof Map) for (const [innerK, innerV] of v.entries()) obj[k][innerK] = innerV;
|
|
111
|
-
else obj[k] = v;
|
|
112
|
-
}
|
|
113
|
-
return obj;
|
|
114
|
-
}
|
|
115
|
-
//#endregion
|
|
116
157
|
//#region src/db/cache.ts
|
|
117
158
|
const logger = logger$1.child({ "name": "CACHE_LAYER" }, { level: logLevelOf("SERVER_LOG_LEVEL") });
|
|
118
159
|
const redisRetryPolicy = retry(handleAll, {
|
|
@@ -123,18 +164,123 @@ const redisRetryPolicy = retry(handleAll, {
|
|
|
123
164
|
exponent: 2
|
|
124
165
|
})
|
|
125
166
|
});
|
|
167
|
+
/** Atomic one-shot recompute write: set data + metadata iff incoming version
|
|
168
|
+
* (ARGV[3]) is not older than the stored one. KEYS: data, meta. */
|
|
169
|
+
const WRITE_DATA_AND_META = `
|
|
170
|
+
local cur = redis.call('GET', KEYS[2])
|
|
171
|
+
if cur then
|
|
172
|
+
local ok, m = pcall(cjson.decode, cur)
|
|
173
|
+
if ok and type(m) == 'table' and m.version and tonumber(ARGV[3]) < tonumber(m.version) then
|
|
174
|
+
return 0
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[4]))
|
|
178
|
+
redis.call('SET', KEYS[2], ARGV[2], 'EX', tonumber(ARGV[4]))
|
|
179
|
+
return 1`;
|
|
180
|
+
/** Data-only write (subscribe path): set data iff incoming version (ARGV[2]) is
|
|
181
|
+
* not older than the stored metadata's version. KEYS: data, meta. */
|
|
182
|
+
const WRITE_DATA_ONLY = `
|
|
183
|
+
local cur = redis.call('GET', KEYS[2])
|
|
184
|
+
if cur then
|
|
185
|
+
local ok, m = pcall(cjson.decode, cur)
|
|
186
|
+
if ok and type(m) == 'table' and m.version and tonumber(ARGV[2]) < tonumber(m.version) then
|
|
187
|
+
return 0
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[3]))
|
|
191
|
+
return 1`;
|
|
192
|
+
/** Metadata-only replace (subscription recompute): set meta iff incoming version
|
|
193
|
+
* (ARGV[2]) is not older than the stored one. KEYS: meta. */
|
|
194
|
+
const REPLACE_META = `
|
|
195
|
+
local cur = redis.call('GET', KEYS[1])
|
|
196
|
+
if cur then
|
|
197
|
+
local ok, m = pcall(cjson.decode, cur)
|
|
198
|
+
if ok and type(m) == 'table' and m.version and tonumber(ARGV[2]) < tonumber(m.version) then
|
|
199
|
+
return 0
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[3]))
|
|
203
|
+
return 1`;
|
|
204
|
+
/** Metadata-only horizon advance (still-fresh subscription): bump lastSnapshotTs
|
|
205
|
+
* ONLY, without recomputing. Applies ARGV[1] iff the new horizon (ARGV[2]) is
|
|
206
|
+
* strictly ahead of the stored one AND no newer computation (stored.version >
|
|
207
|
+
* ARGV[3], the version the caller based its advance on) has landed meanwhile —
|
|
208
|
+
* so an advance never reverts a fresh recompute. KEYS: meta. */
|
|
209
|
+
const ADVANCE_META = `
|
|
210
|
+
local cur = redis.call('GET', KEYS[1])
|
|
211
|
+
if not cur then return 0 end
|
|
212
|
+
local ok, m = pcall(cjson.decode, cur)
|
|
213
|
+
if not ok or type(m) ~= 'table' then return 0 end
|
|
214
|
+
if tonumber(ARGV[2]) <= tonumber(m.lastSnapshotTs) then return 0 end
|
|
215
|
+
if m.version and tonumber(m.version) > tonumber(ARGV[3]) then return 0 end
|
|
216
|
+
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[4]))
|
|
217
|
+
return 1`;
|
|
126
218
|
var CacheLayer = class {
|
|
127
219
|
redis;
|
|
128
220
|
ttlSeconds;
|
|
221
|
+
/** Cached EVALSHA digests keyed by Lua source (see evalScript). */
|
|
222
|
+
scriptShaCache = /* @__PURE__ */ new Map();
|
|
129
223
|
constructor(upstash, ttlSeconds = 3600) {
|
|
130
224
|
this.redis = upstash;
|
|
131
225
|
this.ttlSeconds = ttlSeconds;
|
|
132
226
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
227
|
+
/**
|
|
228
|
+
* Run a Lua script via EVALSHA using the script's SHA1 (computed locally — the
|
|
229
|
+
* same digest Redis keys scripts by, so no SCRIPT LOAD round-trip is needed on
|
|
230
|
+
* the hot path). On NOSCRIPT (the script isn't in Redis's cache yet, e.g. first
|
|
231
|
+
* use or after a Redis restart/failover) load it explicitly via SCRIPT LOAD and
|
|
232
|
+
* re-run via EVALSHA — so the script ends up cached server-side and subsequent
|
|
233
|
+
* EVALSHA calls hit. The runtime client is always ioredis (see initCacheLayer).
|
|
234
|
+
*/
|
|
235
|
+
async evalScript(src, keys, args) {
|
|
236
|
+
const redis = this.redis;
|
|
237
|
+
let sha = this.scriptShaCache.get(src);
|
|
238
|
+
if (!sha) {
|
|
239
|
+
sha = createHash("sha1").update(src).digest("hex");
|
|
240
|
+
this.scriptShaCache.set(src, sha);
|
|
241
|
+
}
|
|
242
|
+
return redisRetryPolicy.execute(async () => {
|
|
243
|
+
try {
|
|
244
|
+
return await redis.evalsha(sha, keys.length, ...keys, ...args);
|
|
245
|
+
} catch (err) {
|
|
246
|
+
if (err instanceof Error && err.message.includes("NOSCRIPT")) {
|
|
247
|
+
const loadedSha = await redis.call("SCRIPT", "LOAD", src);
|
|
248
|
+
this.scriptShaCache.set(src, loadedSha);
|
|
249
|
+
return await redis.evalsha(loadedSha, keys.length, ...keys, ...args);
|
|
250
|
+
}
|
|
251
|
+
throw err;
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
/** One-shot recompute: atomically write data + metadata (CAS on version). */
|
|
256
|
+
async setQueryCacheAndMetadataFor(cacheKey, data, metaData, ttlSeconds) {
|
|
257
|
+
const ttl = ttlSeconds ?? this.ttlSeconds;
|
|
258
|
+
const dataJson = supaliveStringify({
|
|
259
|
+
data,
|
|
260
|
+
version: metaData.version
|
|
261
|
+
});
|
|
262
|
+
const metaJson = supaliveStringify(metaData);
|
|
263
|
+
await this.evalScript(WRITE_DATA_AND_META, [this.getQueryCacheDataKey(cacheKey), this.getQeruyCacheMetaKey(cacheKey)], [
|
|
264
|
+
dataJson,
|
|
265
|
+
metaJson,
|
|
266
|
+
metaData.version,
|
|
267
|
+
String(ttl)
|
|
268
|
+
]);
|
|
269
|
+
}
|
|
270
|
+
/** Subscribe path: write the data blob only (CAS against the metadata's
|
|
271
|
+
* version). The metadata is written separately by the sub-worker; the
|
|
272
|
+
* `version` links the two so a divergent pair is detected on read. */
|
|
273
|
+
async setQueryCacheFor(cacheKey, data, version, ttlSeconds) {
|
|
274
|
+
const ttl = ttlSeconds ?? this.ttlSeconds;
|
|
275
|
+
const dataJson = supaliveStringify({
|
|
276
|
+
data,
|
|
277
|
+
version
|
|
137
278
|
});
|
|
279
|
+
await this.evalScript(WRITE_DATA_ONLY, [this.getQueryCacheDataKey(cacheKey), this.getQeruyCacheMetaKey(cacheKey)], [
|
|
280
|
+
dataJson,
|
|
281
|
+
version,
|
|
282
|
+
String(ttl)
|
|
283
|
+
]);
|
|
138
284
|
}
|
|
139
285
|
async getQueryCacheFor(cacheKey) {
|
|
140
286
|
const dataRaw = await redisRetryPolicy.execute(() => this.redis.get(this.getQueryCacheDataKey(cacheKey)));
|
|
@@ -146,6 +292,31 @@ var CacheLayer = class {
|
|
|
146
292
|
return null;
|
|
147
293
|
}
|
|
148
294
|
}
|
|
295
|
+
/** Subscription recompute: replace the whole metadata (CAS on version). */
|
|
296
|
+
async replaceQueryCacheMetadata(cacheKey, metaData, ttlSeconds) {
|
|
297
|
+
const ttl = ttlSeconds ?? this.ttlSeconds;
|
|
298
|
+
const metaJson = supaliveStringify(metaData);
|
|
299
|
+
await this.evalScript(REPLACE_META, [this.getQeruyCacheMetaKey(cacheKey)], [
|
|
300
|
+
metaJson,
|
|
301
|
+
metaData.version,
|
|
302
|
+
String(ttl)
|
|
303
|
+
]);
|
|
304
|
+
}
|
|
305
|
+
/** Still-fresh subscription: advance `lastSnapshotTs` only, preserving the
|
|
306
|
+
* data's `version` + `readSet` (no recompute). Monotonic and non-clobbering
|
|
307
|
+
* — see ADVANCE_META. `metaData` carries the new horizon plus the version +
|
|
308
|
+
* readSet the caller read, which the script keeps unless a newer computation
|
|
309
|
+
* has since landed. */
|
|
310
|
+
async advanceQueryCacheMetadata(cacheKey, metaData, ttlSeconds) {
|
|
311
|
+
const ttl = ttlSeconds ?? this.ttlSeconds;
|
|
312
|
+
const metaJson = supaliveStringify(metaData);
|
|
313
|
+
await this.evalScript(ADVANCE_META, [this.getQeruyCacheMetaKey(cacheKey)], [
|
|
314
|
+
metaJson,
|
|
315
|
+
String(metaData.lastSnapshotTs),
|
|
316
|
+
metaData.version,
|
|
317
|
+
String(ttl)
|
|
318
|
+
]);
|
|
319
|
+
}
|
|
149
320
|
async getQueryCacheMetaData(cacheKey) {
|
|
150
321
|
const dataRaw = await redisRetryPolicy.execute(() => this.redis.get(this.getQeruyCacheMetaKey(cacheKey)));
|
|
151
322
|
if (!dataRaw) return null;
|
|
@@ -156,10 +327,6 @@ var CacheLayer = class {
|
|
|
156
327
|
return null;
|
|
157
328
|
}
|
|
158
329
|
}
|
|
159
|
-
async setQueryCacheMetadata(cacheKey, metaData) {
|
|
160
|
-
const metadata = supaliveStringify(metaData);
|
|
161
|
-
await redisRetryPolicy.execute(() => this.redis.set(this.getQeruyCacheMetaKey(cacheKey), metadata));
|
|
162
|
-
}
|
|
163
330
|
async persistRegistrationRecord(subId, record) {
|
|
164
331
|
const data = supaliveStringify(record);
|
|
165
332
|
await redisRetryPolicy.execute(() => this.redis.set(this.getRegistrationKey(subId), data));
|
|
@@ -242,6 +409,40 @@ var CacheLayer = class {
|
|
|
242
409
|
//#endregion
|
|
243
410
|
//#region src/db/init_db.ts
|
|
244
411
|
/**
|
|
412
|
+
* Current core schema version. Bump this whenever core needs a new non-table
|
|
413
|
+
* database operation (a sequence, a seed row, a backfill). Each increment adds
|
|
414
|
+
* a case to `Database.coreMigrationStatements` for that version step.
|
|
415
|
+
*
|
|
416
|
+
* The core *tables* (commit_logs, metadata) are NOT versioned here — they come
|
|
417
|
+
* from the generated schema (`coreTablesSql` / `generateSchemaSql`) which must
|
|
418
|
+
* be applied to the database BEFORE `initCore` runs.
|
|
419
|
+
*/
|
|
420
|
+
const CORE_VERSION = 1n;
|
|
421
|
+
/**
|
|
422
|
+
* Bring the core, non-table database state up to {@link CORE_VERSION}. Idempotent
|
|
423
|
+
* and safe to call from every process at startup: a lock-free version read
|
|
424
|
+
* short-circuits the common "already current" case, and the actual migration
|
|
425
|
+
* runs under a cross-process lock with a re-check so only one process applies
|
|
426
|
+
* it.
|
|
427
|
+
*
|
|
428
|
+
* Assumes the core tables already exist (apply the generated schema first);
|
|
429
|
+
* the `metadata` table is where the version and seed rows live.
|
|
430
|
+
*/
|
|
431
|
+
async function initCore(db) {
|
|
432
|
+
if (await db.getCoreVersion() >= 1n) return;
|
|
433
|
+
await db.withCoreInitLock(async (ctx) => {
|
|
434
|
+
const from = await ctx.getVersion();
|
|
435
|
+
if (from >= 1n) return;
|
|
436
|
+
for (let v = from + 1n; v <= CORE_VERSION; v++) {
|
|
437
|
+
const statements = db.coreMigrationStatements(v);
|
|
438
|
+
logger$1.info(`Migrating core schema ${v - 1n} -> ${v} (${statements.length} statements)`);
|
|
439
|
+
for (const sql of statements) await ctx.exec(sql);
|
|
440
|
+
}
|
|
441
|
+
await ctx.setVersion(CORE_VERSION);
|
|
442
|
+
logger$1.info(`Core schema now at version ${CORE_VERSION}`);
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
245
446
|
* Default and env override for the per-statement DB timeout. PG enforces
|
|
246
447
|
* this via `statement_timeout` (server cancels), MySQL via the per-query
|
|
247
448
|
* `timeout` option (driver issues KILL QUERY). Set 0 to disable.
|
|
@@ -250,6 +451,7 @@ const DB_QUERY_TIMEOUT_MS = Number(process.env.SUPALIVE_DB_QUERY_TIMEOUT_MS ?? 3
|
|
|
250
451
|
async function initDatabase(options) {
|
|
251
452
|
const impl = createDatabase(options.DbConfig, options.workerId);
|
|
252
453
|
const supaliveDb = SupaliveDb.create({ db: impl });
|
|
454
|
+
await initCore(impl);
|
|
253
455
|
await bootstrapDbTypes(supaliveDb, options.CacheLayer, options.forceBootstrapReload ?? false);
|
|
254
456
|
return supaliveDb;
|
|
255
457
|
}
|
|
@@ -325,6 +527,7 @@ var SubscriptionWorker = class {
|
|
|
325
527
|
subscriptions = /* @__PURE__ */ new Map();
|
|
326
528
|
cache;
|
|
327
529
|
db;
|
|
530
|
+
retentionWatermark = new RetentionWatermark();
|
|
328
531
|
logger;
|
|
329
532
|
constructor(id, cache, db) {
|
|
330
533
|
this.id = id;
|
|
@@ -343,13 +546,55 @@ var SubscriptionWorker = class {
|
|
|
343
546
|
}
|
|
344
547
|
/**
|
|
345
548
|
* Lowest `lastSnapshotTs` across this worker's subscriptions, or `null`
|
|
346
|
-
* when there are none. Used by the dispatcher to derive a safe prune
|
|
549
|
+
* when there are none. Used by the dispatcher to derive a safe prune watermark.
|
|
347
550
|
*/
|
|
348
551
|
minSubscriptionTs() {
|
|
349
552
|
let min = null;
|
|
350
553
|
for (const sub of this.subscriptions.values()) if (min === null || sub.lastSnapshotTs < min) min = sub.lastSnapshotTs;
|
|
351
554
|
return min;
|
|
352
555
|
}
|
|
556
|
+
/**
|
|
557
|
+
* Seed/repair the persisted retained watermark from the actual oldest retained
|
|
558
|
+
* commit log (`min(ts)`), then load it into memory and broadcast it. Called
|
|
559
|
+
* once at sub-manager startup — the `min(ts)` scan is off the hot path, and
|
|
560
|
+
* it corrects `metadata.min_retained_ts` even for a DB that was pruned before
|
|
561
|
+
* this feature existed. Returns the resulting watermark so the manager can
|
|
562
|
+
* broadcast it to the other workers.
|
|
563
|
+
*/
|
|
564
|
+
async refreshRetentionWatermark() {
|
|
565
|
+
const real = await this.db.getMinCommitLogTs() ?? 0n;
|
|
566
|
+
await this.db.updateMinSnapshotTimestamp(real);
|
|
567
|
+
const persisted = await this.db.getMinSnapshotTimestamp();
|
|
568
|
+
this.retentionWatermark.set(persisted);
|
|
569
|
+
await this.publishRetentionWatermark(persisted);
|
|
570
|
+
return this.retentionWatermark.get();
|
|
571
|
+
}
|
|
572
|
+
/** Adopt a watermark broadcast by the manager (in-process from the pruning
|
|
573
|
+
* worker). Monotonic, so an out-of-order message can't lower it. */
|
|
574
|
+
setRetentionWatermark(ts) {
|
|
575
|
+
this.retentionWatermark.set(ts);
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* Prune commit logs older than `minRetainedTs`. Advertises the new watermark FIRST —
|
|
579
|
+
* persist it, bump our in-memory copy, and broadcast it over Redis — and only
|
|
580
|
+
* then deletes, so no reader ever trusts a cache entry whose freshness scan
|
|
581
|
+
* would include rows we're about to remove. Returns rows deleted.
|
|
582
|
+
*/
|
|
583
|
+
async prune(minRetainedTs) {
|
|
584
|
+
await this.db.updateMinSnapshotTimestamp(minRetainedTs);
|
|
585
|
+
this.retentionWatermark.set(minRetainedTs);
|
|
586
|
+
await this.publishRetentionWatermark(minRetainedTs);
|
|
587
|
+
return await this.db.pruneCommitLogsBefore(minRetainedTs);
|
|
588
|
+
}
|
|
589
|
+
/** Broadcast the watermark to app-server instances over Redis pub/sub. Workers in
|
|
590
|
+
* this process get it in-process via {@link setRetentionWatermark}. */
|
|
591
|
+
async publishRetentionWatermark(ts) {
|
|
592
|
+
try {
|
|
593
|
+
await this.cache.redis.publish(RETENTION_WATERMARK_CHANNEL, ts.toString());
|
|
594
|
+
} catch (err) {
|
|
595
|
+
this.logger.error(err, "failed to publish commit-log watermark");
|
|
596
|
+
}
|
|
597
|
+
}
|
|
353
598
|
async register(params) {
|
|
354
599
|
const { subId, cacheKey, queryName, args, queryIdentity, instanceName, lastSnapshotTs } = params;
|
|
355
600
|
this.logger.info(`registering subscription: ${subId} from ${instanceName}`);
|
|
@@ -368,25 +613,15 @@ var SubscriptionWorker = class {
|
|
|
368
613
|
existing.timeout = null;
|
|
369
614
|
}
|
|
370
615
|
const metadata = await this.cache.getQueryCacheMetaData(cacheKey);
|
|
371
|
-
|
|
372
|
-
let
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
if (lastSnapshotTs !== metadata.lastSnapshotTs) this.cache.setQueryCacheMetadata(cacheKey, {
|
|
381
|
-
lastSnapshotTs,
|
|
382
|
-
readSet
|
|
383
|
-
}).catch((err) => this.logger.error(err, `Failed to update cache metadata for ${cacheKey}`));
|
|
384
|
-
metadata.lastSnapshotTs = lastSnapshotTs;
|
|
385
|
-
}
|
|
386
|
-
} else {
|
|
387
|
-
readSet = [];
|
|
388
|
-
rawReadSet = [];
|
|
389
|
-
}
|
|
616
|
+
const freshness = await evaluateCacheFreshness(this.db, metadata, lastSnapshotTs, this.retentionWatermark.get());
|
|
617
|
+
let needsRecompute = !freshness.fresh;
|
|
618
|
+
const readSet = freshness.readSet;
|
|
619
|
+
const rawReadSet = freshness.rawReadSet;
|
|
620
|
+
if (!needsRecompute && metadata && lastSnapshotTs > metadata.lastSnapshotTs) this.cache.advanceQueryCacheMetadata(cacheKey, {
|
|
621
|
+
lastSnapshotTs,
|
|
622
|
+
version: metadata.version,
|
|
623
|
+
readSet
|
|
624
|
+
}).catch((err) => this.logger.error(err, `Failed to advance cache metadata for ${cacheKey}`));
|
|
390
625
|
let newSubscription = {
|
|
391
626
|
subId,
|
|
392
627
|
queryName,
|
|
@@ -423,7 +658,8 @@ var SubscriptionWorker = class {
|
|
|
423
658
|
return {
|
|
424
659
|
cacheKey,
|
|
425
660
|
recompute: needsRecompute,
|
|
426
|
-
readSet
|
|
661
|
+
readSet,
|
|
662
|
+
version: metadata?.version
|
|
427
663
|
};
|
|
428
664
|
}
|
|
429
665
|
async updateReadSet(params) {
|
|
@@ -439,8 +675,9 @@ var SubscriptionWorker = class {
|
|
|
439
675
|
rawReadSet,
|
|
440
676
|
lastSnapshotTs
|
|
441
677
|
});
|
|
442
|
-
await this.cache.
|
|
678
|
+
await this.cache.replaceQueryCacheMetadata(cacheKey, {
|
|
443
679
|
lastSnapshotTs,
|
|
680
|
+
version: String(lastSnapshotTs),
|
|
444
681
|
readSet
|
|
445
682
|
});
|
|
446
683
|
return { tracked: true };
|
|
@@ -506,6 +743,6 @@ var SubscriptionWorker = class {
|
|
|
506
743
|
}
|
|
507
744
|
};
|
|
508
745
|
//#endregion
|
|
509
|
-
export {
|
|
746
|
+
export { RetentionWatermark as C, logger$1 as D, logLevelOf as E, serializeMapValue as O, RETENTION_WATERMARK_CHANNEL as S, isProduction as T, UnregisterSubscriptionResultSchema as _, createDatabase as a, UpdateSubscriptionReadSetParamsSchema as b, initDatabase as c, InvalidateWritesetResultSchema as d, RegisterSubscriptionBatchParamsSchema as f, UnregisterSubscriptionParamsSchema as g, RegisterSubscriptionResultSchema as h, bootstrapDbTypes as i, CacheLayer as l, RegisterSubscriptionParamsSchema as m, CORE_VERSION as n, initCacheLayer as o, RegisterSubscriptionBatchResultSchema as p, DB_QUERY_TIMEOUT_MS as r, initCore as s, SubscriptionWorker as t, InvalidateWritesetParamsSchema as u, UnregisterSubscriptionsParamsSchema as v, appInstanceId as w, UpdateSubscriptionReadSetResultSchema as x, UnregisterSubscriptionsResultSchema as y };
|
|
510
747
|
|
|
511
|
-
//# sourceMappingURL=subscription-worker-
|
|
748
|
+
//# sourceMappingURL=subscription-worker-CAf6MUta.js.map
|