@prisma/streams-server 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CODE_OF_CONDUCT.md +1 -1
- package/CONTRIBUTING.md +5 -5
- package/README.md +3 -3
- package/SECURITY.md +2 -2
- package/package.json +2 -2
- package/src/app_core.ts +114 -391
- package/src/db/db.ts +0 -54
- package/src/db/schema.ts +12 -6
- package/src/runtime/host_runtime.ts +5 -0
- package/src/segment/segmenter_worker.ts +4 -1
- package/src/segment/segmenter_workers.ts +2 -0
- package/src/sqlite/adapter.ts +30 -6
- package/src/touch/interpreter_worker.ts +22 -34
- package/src/touch/live_metrics.ts +18 -49
- package/src/touch/manager.ts +26 -168
- package/src/touch/spec.ts +8 -78
- package/src/touch/worker_pool.ts +2 -1
- package/src/touch/worker_protocol.ts +0 -2
- package/src/touch/naming.ts +0 -13
- package/src/touch/routing_key_notifier.ts +0 -275
package/src/db/db.ts
CHANGED
|
@@ -964,60 +964,6 @@ export class SqliteDurableStore {
|
|
|
964
964
|
}
|
|
965
965
|
}
|
|
966
966
|
|
|
967
|
-
findFirstWalOffsetForRoutingKeys(
|
|
968
|
-
stream: string,
|
|
969
|
-
startOffsetExclusive: bigint,
|
|
970
|
-
endOffsetInclusive: bigint,
|
|
971
|
-
routingKeys: Uint8Array[]
|
|
972
|
-
): bigint | null {
|
|
973
|
-
if (routingKeys.length === 0) return null;
|
|
974
|
-
const start = this.bindInt(startOffsetExclusive);
|
|
975
|
-
|
|
976
|
-
// Avoid tripping SQLite variable limits (commonly 999) when callers watch a
|
|
977
|
-
// large number of keys. Chunking is cheap because the touch WAL ranges we
|
|
978
|
-
// scan are typically small and indexed by (stream, routing_key, offset).
|
|
979
|
-
const MAX_KEYS_PER_QUERY = 400;
|
|
980
|
-
let best: bigint | null = null;
|
|
981
|
-
let endLimit = endOffsetInclusive;
|
|
982
|
-
const touchNameLooksDefault = stream.endsWith(".__touch");
|
|
983
|
-
const touchLikeClause = touchNameLooksDefault ? " AND stream LIKE '%.__touch'" : "";
|
|
984
|
-
|
|
985
|
-
for (let i = 0; i < routingKeys.length; i += MAX_KEYS_PER_QUERY) {
|
|
986
|
-
const chunk = routingKeys.slice(i, i + MAX_KEYS_PER_QUERY);
|
|
987
|
-
if (chunk.length === 0) continue;
|
|
988
|
-
const end = this.bindInt(endLimit);
|
|
989
|
-
const placeholders = chunk.map(() => "?").join(", ");
|
|
990
|
-
const stmt = this.db.query(
|
|
991
|
-
`SELECT MIN(offset) as offset
|
|
992
|
-
FROM wal
|
|
993
|
-
WHERE stream = ?${touchLikeClause}
|
|
994
|
-
AND offset > ?
|
|
995
|
-
AND offset <= ?
|
|
996
|
-
AND routing_key IN (${placeholders});`
|
|
997
|
-
);
|
|
998
|
-
try {
|
|
999
|
-
const row = stmt.get(stream, start, end, ...chunk) as any;
|
|
1000
|
-
if (!row || row.offset == null) continue;
|
|
1001
|
-
const off = this.toBigInt(row.offset);
|
|
1002
|
-
if (best == null || off < best) {
|
|
1003
|
-
best = off;
|
|
1004
|
-
// No need to search beyond the best match so far.
|
|
1005
|
-
endLimit = off - 1n;
|
|
1006
|
-
if (off <= startOffsetExclusive + 1n) break;
|
|
1007
|
-
}
|
|
1008
|
-
} finally {
|
|
1009
|
-
try {
|
|
1010
|
-
stmt.finalize?.();
|
|
1011
|
-
} catch {
|
|
1012
|
-
// ignore
|
|
1013
|
-
}
|
|
1014
|
-
}
|
|
1015
|
-
if (endLimit < startOffsetExclusive + 1n) break;
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
return best;
|
|
1019
|
-
}
|
|
1020
|
-
|
|
1021
967
|
nextSegmentIndexForStream(stream: string): number {
|
|
1022
968
|
const row = this.stmts.nextSegmentIndex.get(stream) as any;
|
|
1023
969
|
return Number(row?.next_idx ?? 0);
|
package/src/db/schema.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { dsError } from "../util/ds_error.ts";
|
|
|
9
9
|
* - local metadata store (streams/segments/manifests/schemas)
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
export const SCHEMA_VERSION =
|
|
12
|
+
export const SCHEMA_VERSION = 11;
|
|
13
13
|
|
|
14
14
|
export const DEFAULT_PRAGMAS_SQL = `
|
|
15
15
|
PRAGMA journal_mode = WAL;
|
|
@@ -75,8 +75,6 @@ CREATE TABLE IF NOT EXISTS wal (
|
|
|
75
75
|
CREATE UNIQUE INDEX IF NOT EXISTS wal_stream_offset_uniq ON wal(stream, offset);
|
|
76
76
|
CREATE INDEX IF NOT EXISTS wal_stream_offset_idx ON wal(stream, offset);
|
|
77
77
|
CREATE INDEX IF NOT EXISTS wal_ts_idx ON wal(ts_ms);
|
|
78
|
-
-- Partial index for internal companion touch streams (WAL-only, routing-key heavy).
|
|
79
|
-
CREATE INDEX IF NOT EXISTS wal_touch_stream_rk_offset_idx ON wal(stream, routing_key, offset) WHERE stream LIKE '%.__touch';
|
|
80
78
|
|
|
81
79
|
CREATE TABLE IF NOT EXISTS segments (
|
|
82
80
|
segment_id TEXT PRIMARY KEY,
|
|
@@ -134,7 +132,7 @@ CREATE TABLE IF NOT EXISTS stream_interpreters (
|
|
|
134
132
|
updated_at_ms INTEGER NOT NULL
|
|
135
133
|
);
|
|
136
134
|
|
|
137
|
-
-- Live
|
|
135
|
+
-- Live dynamic template registry (per base stream).
|
|
138
136
|
CREATE TABLE IF NOT EXISTS live_templates (
|
|
139
137
|
stream TEXT NOT NULL,
|
|
140
138
|
template_id TEXT NOT NULL,
|
|
@@ -267,7 +265,6 @@ const CREATE_INDEXES_V4_SQL = `
|
|
|
267
265
|
CREATE UNIQUE INDEX IF NOT EXISTS wal_stream_offset_uniq ON wal(stream, offset);
|
|
268
266
|
CREATE INDEX IF NOT EXISTS wal_stream_offset_idx ON wal(stream, offset);
|
|
269
267
|
CREATE INDEX IF NOT EXISTS wal_ts_idx ON wal(ts_ms);
|
|
270
|
-
CREATE INDEX IF NOT EXISTS wal_touch_stream_rk_offset_idx ON wal(stream, routing_key, offset) WHERE stream LIKE '%.__touch';
|
|
271
268
|
|
|
272
269
|
CREATE INDEX IF NOT EXISTS streams_pending_bytes_idx ON streams(pending_bytes);
|
|
273
270
|
CREATE INDEX IF NOT EXISTS streams_last_cut_idx ON streams(last_segment_cut_ms);
|
|
@@ -326,6 +323,8 @@ export function initSchema(db: SqliteDatabase, opts: { skipMigrations?: boolean
|
|
|
326
323
|
migrateV8ToV9(db);
|
|
327
324
|
} else if (version === 9) {
|
|
328
325
|
migrateV9ToV10(db);
|
|
326
|
+
} else if (version === 10) {
|
|
327
|
+
migrateV10ToV11(db);
|
|
329
328
|
} else {
|
|
330
329
|
throw dsError(`unexpected schema version: ${version} (expected ${SCHEMA_VERSION})`);
|
|
331
330
|
}
|
|
@@ -555,7 +554,6 @@ function migrateV6ToV7(db: SqliteDatabase): void {
|
|
|
555
554
|
|
|
556
555
|
function migrateV7ToV8(db: SqliteDatabase): void {
|
|
557
556
|
const tx = db.transaction(() => {
|
|
558
|
-
db.exec(`CREATE INDEX IF NOT EXISTS wal_touch_stream_rk_offset_idx ON wal(stream, routing_key, offset) WHERE stream LIKE '%.__touch';`);
|
|
559
557
|
db.exec(`UPDATE schema_version SET version = 8;`);
|
|
560
558
|
});
|
|
561
559
|
tx();
|
|
@@ -613,6 +611,14 @@ function migrateV9ToV10(db: SqliteDatabase): void {
|
|
|
613
611
|
`);
|
|
614
612
|
db.exec(`DROP TABLE wal_stats;`);
|
|
615
613
|
|
|
614
|
+
db.exec(`UPDATE schema_version SET version = 10;`);
|
|
615
|
+
});
|
|
616
|
+
tx();
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
function migrateV10ToV11(db: SqliteDatabase): void {
|
|
620
|
+
const tx = db.transaction(() => {
|
|
621
|
+
db.exec(`DROP INDEX IF EXISTS wal_touch_stream_rk_offset_idx;`);
|
|
616
622
|
db.exec(`UPDATE schema_version SET version = ${SCHEMA_VERSION};`);
|
|
617
623
|
});
|
|
618
624
|
tx();
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import { parentPort, workerData } from "node:worker_threads";
|
|
2
2
|
import type { Config } from "../config.ts";
|
|
3
3
|
import { SqliteDurableStore } from "../db/db.ts";
|
|
4
|
+
import type { HostRuntime } from "../runtime/host_runtime.ts";
|
|
5
|
+
import { setSqliteRuntimeOverride } from "../sqlite/adapter.ts";
|
|
4
6
|
import { Segmenter, type SegmenterHooks, type SegmenterOptions } from "./segmenter.ts";
|
|
5
7
|
import { initConsoleLogging } from "../util/log.ts";
|
|
6
8
|
|
|
7
9
|
initConsoleLogging();
|
|
8
10
|
|
|
9
|
-
const data = workerData as { config: Config; opts?: SegmenterOptions };
|
|
11
|
+
const data = workerData as { config: Config; hostRuntime?: HostRuntime; opts?: SegmenterOptions };
|
|
10
12
|
const cfg = data.config;
|
|
13
|
+
setSqliteRuntimeOverride(data.hostRuntime ?? null);
|
|
11
14
|
// The main server process initializes/migrates schema; workers should avoid
|
|
12
15
|
// concurrent migrations on the same sqlite file.
|
|
13
16
|
const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.sqliteCacheBytes, skipMigrations: true });
|
|
@@ -3,6 +3,7 @@ import { resolve } from "node:path";
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { Worker } from "node:worker_threads";
|
|
5
5
|
import type { Config } from "../config";
|
|
6
|
+
import { detectHostRuntime } from "../runtime/host_runtime.ts";
|
|
6
7
|
import type { SegmenterHooks, SegmenterOptions } from "./segmenter";
|
|
7
8
|
|
|
8
9
|
export type SegmenterController = {
|
|
@@ -63,6 +64,7 @@ export class SegmenterWorkerPool implements SegmenterController {
|
|
|
63
64
|
const worker = new Worker(workerSpec, {
|
|
64
65
|
workerData: {
|
|
65
66
|
config: this.config,
|
|
67
|
+
hostRuntime: detectHostRuntime(),
|
|
66
68
|
opts: this.opts,
|
|
67
69
|
},
|
|
68
70
|
type: "module",
|
package/src/sqlite/adapter.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import type { HostRuntime } from "../runtime/host_runtime.ts";
|
|
3
|
+
import { detectHostRuntime } from "../runtime/host_runtime.ts";
|
|
1
4
|
import { dsError } from "../util/ds_error.ts";
|
|
2
5
|
export interface SqliteStatement {
|
|
3
6
|
get(...params: any[]): any;
|
|
@@ -149,16 +152,37 @@ class NodeDatabaseAdapter implements SqliteDatabase {
|
|
|
149
152
|
}
|
|
150
153
|
|
|
151
154
|
let openImpl: ((path: string) => SqliteDatabase) | null = null;
|
|
155
|
+
let openImplRuntime: HostRuntime | null = null;
|
|
156
|
+
let runtimeOverride: HostRuntime | null = null;
|
|
157
|
+
const require = createRequire(import.meta.url);
|
|
152
158
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
+
function selectedRuntime(): HostRuntime {
|
|
160
|
+
return runtimeOverride ?? detectHostRuntime();
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function buildOpenImpl(runtime: HostRuntime): (path: string) => SqliteDatabase {
|
|
164
|
+
if (runtime === "bun") {
|
|
165
|
+
const { Database } = require("bun:sqlite") as { Database: new (path: string) => any };
|
|
166
|
+
return (path: string) => new BunDatabaseAdapter(new Database(path));
|
|
167
|
+
}
|
|
168
|
+
const { DatabaseSync } = require("node:sqlite") as { DatabaseSync: new (path: string) => any };
|
|
169
|
+
return (path: string) => new NodeDatabaseAdapter(new DatabaseSync(path));
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function setSqliteRuntimeOverride(runtime: HostRuntime | null): void {
|
|
173
|
+
runtimeOverride = runtime;
|
|
174
|
+
if (runtimeOverride && openImplRuntime && runtimeOverride !== openImplRuntime) {
|
|
175
|
+
openImpl = null;
|
|
176
|
+
openImplRuntime = null;
|
|
177
|
+
}
|
|
159
178
|
}
|
|
160
179
|
|
|
161
180
|
export function openSqliteDatabase(path: string): SqliteDatabase {
|
|
181
|
+
const runtime = selectedRuntime();
|
|
182
|
+
if (!openImpl || openImplRuntime !== runtime) {
|
|
183
|
+
openImpl = buildOpenImpl(runtime);
|
|
184
|
+
openImplRuntime = runtime;
|
|
185
|
+
}
|
|
162
186
|
if (!openImpl) throw dsError("sqlite adapter not initialized");
|
|
163
187
|
return openImpl(path);
|
|
164
188
|
}
|
|
@@ -2,17 +2,21 @@ import { parentPort, workerData } from "node:worker_threads";
|
|
|
2
2
|
import { Result } from "better-result";
|
|
3
3
|
import type { Config } from "../config.ts";
|
|
4
4
|
import { SqliteDurableStore } from "../db/db.ts";
|
|
5
|
+
import type { HostRuntime } from "../runtime/host_runtime.ts";
|
|
6
|
+
import { setSqliteRuntimeOverride } from "../sqlite/adapter.ts";
|
|
5
7
|
import { initConsoleLogging } from "../util/log.ts";
|
|
6
8
|
import type { ProcessRequest } from "./worker_protocol.ts";
|
|
7
9
|
import { interpretRecordToChanges } from "./engine.ts";
|
|
8
|
-
import { encodeTemplateArg,
|
|
10
|
+
import { encodeTemplateArg, tableKeyIdFor, templateKeyIdFor, watchKeyIdFor, type TemplateEncoding } from "./live_keys.ts";
|
|
9
11
|
import { isTouchEnabled } from "./spec.ts";
|
|
10
|
-
import { resolveTouchStreamName } from "./naming.ts";
|
|
11
12
|
|
|
12
13
|
initConsoleLogging();
|
|
13
14
|
|
|
14
|
-
const data = workerData as { config: Config };
|
|
15
|
+
const data = workerData as { config: Config; hostRuntime?: HostRuntime };
|
|
15
16
|
const cfg = data.config;
|
|
17
|
+
// Bun worker_threads can miss the Bun globals that the main thread sees.
|
|
18
|
+
// Use the parent host runtime hint before the worker opens SQLite.
|
|
19
|
+
setSqliteRuntimeOverride(data.hostRuntime ?? null);
|
|
16
20
|
// The main server process initializes/migrates schema; workers should avoid
|
|
17
21
|
// concurrent migrations on the same sqlite file.
|
|
18
22
|
const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.sqliteCacheBytes, skipMigrations: true });
|
|
@@ -50,9 +54,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
50
54
|
return;
|
|
51
55
|
}
|
|
52
56
|
const touch = interpreter.touch;
|
|
53
|
-
const derivedStream = resolveTouchStreamName(stream, touch);
|
|
54
|
-
const touchStorage = touch.storage ?? "memory";
|
|
55
|
-
const emitRoutingKey = touchStorage === "sqlite";
|
|
56
57
|
|
|
57
58
|
const fineBudgetRaw = msg.fineTouchBudget ?? touch.fineTouchBudgetPerBatch;
|
|
58
59
|
const fineBudget = fineBudgetRaw == null ? null : Math.max(0, Math.floor(fineBudgetRaw));
|
|
@@ -125,7 +126,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
125
126
|
|
|
126
127
|
type PendingTouch = {
|
|
127
128
|
keyId: number;
|
|
128
|
-
key?: string;
|
|
129
129
|
windowStartMs: number;
|
|
130
130
|
watermark: string;
|
|
131
131
|
entity: string;
|
|
@@ -136,17 +136,16 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
136
136
|
|
|
137
137
|
const pending = new Map<string, PendingTouch>();
|
|
138
138
|
const templateOnlyEntityTouch = new Map<string, EntityTemplateOnlyTouch>();
|
|
139
|
-
const touches: Array<{ keyId: number;
|
|
139
|
+
const touches: Array<{ keyId: number; watermark: string; entity: string; kind: "table" | "template"; templateId?: string }> = [];
|
|
140
140
|
let fineTouchesDroppedDueToBudget = 0;
|
|
141
141
|
let fineTouchesSkippedColdTemplate = 0;
|
|
142
142
|
|
|
143
143
|
const flush = (_mapKey: string, p: PendingTouch) => {
|
|
144
|
-
touches.push({ keyId: p.keyId >>> 0,
|
|
144
|
+
touches.push({ keyId: p.keyId >>> 0, watermark: p.watermark, entity: p.entity, kind: p.kind, templateId: p.templateId });
|
|
145
145
|
};
|
|
146
146
|
|
|
147
147
|
const queueTouch = (args: {
|
|
148
148
|
keyId: number;
|
|
149
|
-
key?: string;
|
|
150
149
|
tsMs: number;
|
|
151
150
|
watermark: string;
|
|
152
151
|
entity: string;
|
|
@@ -154,7 +153,7 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
154
153
|
templateId?: string;
|
|
155
154
|
windowMs: number;
|
|
156
155
|
}) => {
|
|
157
|
-
const mapKey =
|
|
156
|
+
const mapKey = `i:${args.keyId >>> 0}`;
|
|
158
157
|
const prev = pending.get(mapKey);
|
|
159
158
|
|
|
160
159
|
// Guardrail: cap fine/template touches (key cardinality) per batch.
|
|
@@ -177,7 +176,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
177
176
|
if (!prev) {
|
|
178
177
|
pending.set(mapKey, {
|
|
179
178
|
keyId: args.keyId >>> 0,
|
|
180
|
-
key: args.key,
|
|
181
179
|
windowStartMs: args.tsMs,
|
|
182
180
|
watermark: args.watermark,
|
|
183
181
|
entity: args.entity,
|
|
@@ -194,7 +192,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
194
192
|
flush(mapKey, prev);
|
|
195
193
|
pending.set(mapKey, {
|
|
196
194
|
keyId: args.keyId >>> 0,
|
|
197
|
-
key: args.key,
|
|
198
195
|
windowStartMs: args.tsMs,
|
|
199
196
|
watermark: args.watermark,
|
|
200
197
|
entity: args.entity,
|
|
@@ -237,7 +234,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
237
234
|
const coarseKeyId = tableKeyIdFor(entity);
|
|
238
235
|
queueTouch({
|
|
239
236
|
keyId: coarseKeyId,
|
|
240
|
-
key: emitRoutingKey ? tableKeyFor(entity) : undefined,
|
|
241
237
|
tsMs,
|
|
242
238
|
watermark,
|
|
243
239
|
entity,
|
|
@@ -267,7 +263,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
267
263
|
if (fineGranularity === "template") {
|
|
268
264
|
queueTouch({
|
|
269
265
|
keyId: templateKeyIdFor(tpl.templateId) >>> 0,
|
|
270
|
-
key: emitRoutingKey ? templateKeyFor(tpl.templateId) : undefined,
|
|
271
266
|
tsMs,
|
|
272
267
|
watermark,
|
|
273
268
|
entity,
|
|
@@ -282,9 +277,9 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
282
277
|
const afterObj = ch.after;
|
|
283
278
|
const beforeObj = ch.before;
|
|
284
279
|
|
|
285
|
-
const
|
|
280
|
+
const watchKeyIds = new Set<number>();
|
|
286
281
|
|
|
287
|
-
const compute = (obj: unknown):
|
|
282
|
+
const compute = (obj: unknown): number | null => {
|
|
288
283
|
if (!obj || typeof obj !== "object" || Array.isArray(obj)) return null;
|
|
289
284
|
const args: string[] = [];
|
|
290
285
|
for (let i = 0; i < tpl.fields.length; i++) {
|
|
@@ -295,19 +290,15 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
295
290
|
if (encoded == null) return null;
|
|
296
291
|
args.push(encoded);
|
|
297
292
|
}
|
|
298
|
-
|
|
299
|
-
const key = watchKeyFor(tpl.templateId, args);
|
|
300
|
-
return { keyId: Number.parseInt(key.slice(8), 16) >>> 0, key };
|
|
301
|
-
}
|
|
302
|
-
return { keyId: watchKeyIdFor(tpl.templateId, args) >>> 0 };
|
|
293
|
+
return watchKeyIdFor(tpl.templateId, args) >>> 0;
|
|
303
294
|
};
|
|
304
295
|
|
|
305
296
|
if (ch.op === "insert") {
|
|
306
297
|
const k = compute(afterObj);
|
|
307
|
-
if (k)
|
|
298
|
+
if (k != null) watchKeyIds.add(k >>> 0);
|
|
308
299
|
} else if (ch.op === "delete") {
|
|
309
300
|
const k = compute(beforeObj);
|
|
310
|
-
if (k)
|
|
301
|
+
if (k != null) watchKeyIds.add(k >>> 0);
|
|
311
302
|
} else {
|
|
312
303
|
// update: compute touches from both before and after (when possible).
|
|
313
304
|
// Policy for missing/insufficient before image:
|
|
@@ -317,9 +308,9 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
317
308
|
const kAfter = compute(afterObj);
|
|
318
309
|
const kBefore = compute(beforeObj);
|
|
319
310
|
|
|
320
|
-
if (kBefore) {
|
|
321
|
-
|
|
322
|
-
if (kAfter)
|
|
311
|
+
if (kBefore != null) {
|
|
312
|
+
watchKeyIds.add(kBefore >>> 0);
|
|
313
|
+
if (kAfter != null) watchKeyIds.add(kAfter >>> 0);
|
|
323
314
|
} else {
|
|
324
315
|
if (beforeObj === undefined) {
|
|
325
316
|
if (onMissingBefore === "error") {
|
|
@@ -335,17 +326,16 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
335
326
|
}
|
|
336
327
|
|
|
337
328
|
if (onMissingBefore === "skipBefore") {
|
|
338
|
-
if (kAfter)
|
|
329
|
+
if (kAfter != null) watchKeyIds.add(kAfter >>> 0);
|
|
339
330
|
} else {
|
|
340
331
|
// coarse: no fine touches
|
|
341
332
|
}
|
|
342
333
|
}
|
|
343
334
|
}
|
|
344
335
|
|
|
345
|
-
for (const
|
|
336
|
+
for (const watchKeyId of watchKeyIds) {
|
|
346
337
|
queueTouch({
|
|
347
338
|
keyId: watchKeyId >>> 0,
|
|
348
|
-
key: watchKey,
|
|
349
339
|
tsMs,
|
|
350
340
|
watermark,
|
|
351
341
|
entity,
|
|
@@ -369,7 +359,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
369
359
|
if (agg.offset < tpl.activeFromSourceOffset) continue;
|
|
370
360
|
queueTouch({
|
|
371
361
|
keyId: templateKeyIdFor(tpl.templateId) >>> 0,
|
|
372
|
-
key: emitRoutingKey ? templateKeyFor(tpl.templateId) : undefined,
|
|
373
362
|
tsMs: agg.tsMs,
|
|
374
363
|
watermark: agg.watermark,
|
|
375
364
|
entity,
|
|
@@ -386,8 +375,8 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
386
375
|
}
|
|
387
376
|
|
|
388
377
|
touches.sort((a, b) => {
|
|
389
|
-
const ak = a.
|
|
390
|
-
const bk = b.
|
|
378
|
+
const ak = a.keyId >>> 0;
|
|
379
|
+
const bk = b.keyId >>> 0;
|
|
391
380
|
if (ak < bk) return -1;
|
|
392
381
|
if (ak > bk) return 1;
|
|
393
382
|
const aw = BigInt(a.watermark);
|
|
@@ -408,7 +397,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
|
|
|
408
397
|
type: "result",
|
|
409
398
|
id: msg.id,
|
|
410
399
|
stream,
|
|
411
|
-
derivedStream,
|
|
412
400
|
processedThrough,
|
|
413
401
|
touches,
|
|
414
402
|
stats: {
|
|
@@ -4,8 +4,6 @@ import { STREAM_FLAG_TOUCH } from "../db/db";
|
|
|
4
4
|
import { encodeOffset } from "../offset";
|
|
5
5
|
import type { TouchConfig } from "./spec";
|
|
6
6
|
import type { TemplateLifecycleEvent } from "./live_templates";
|
|
7
|
-
import { resolveTouchStreamName } from "./naming";
|
|
8
|
-
import type { RoutingKeyNotifier } from "./routing_key_notifier";
|
|
9
7
|
import type { TouchJournalIntervalStats, TouchJournalMeta } from "./touch_journal";
|
|
10
8
|
import { Result } from "better-result";
|
|
11
9
|
|
|
@@ -207,8 +205,7 @@ export class LiveMetricsV2 {
|
|
|
207
205
|
private readonly snapshotIntervalMs: number;
|
|
208
206
|
private readonly snapshotChunkSize: number;
|
|
209
207
|
private readonly retentionMs: number;
|
|
210
|
-
private readonly
|
|
211
|
-
private readonly getTouchJournal?: (derivedStream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
|
|
208
|
+
private readonly getTouchJournal?: (stream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
|
|
212
209
|
private timer: any | null = null;
|
|
213
210
|
private snapshotTimer: any | null = null;
|
|
214
211
|
private retentionTimer: any | null = null;
|
|
@@ -234,8 +231,7 @@ export class LiveMetricsV2 {
|
|
|
234
231
|
snapshotIntervalMs?: number;
|
|
235
232
|
snapshotChunkSize?: number;
|
|
236
233
|
retentionMs?: number;
|
|
237
|
-
|
|
238
|
-
getTouchJournal?: (derivedStream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
|
|
234
|
+
getTouchJournal?: (stream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
|
|
239
235
|
}
|
|
240
236
|
) {
|
|
241
237
|
this.db = db;
|
|
@@ -246,7 +242,6 @@ export class LiveMetricsV2 {
|
|
|
246
242
|
this.snapshotIntervalMs = opts?.snapshotIntervalMs ?? 60_000;
|
|
247
243
|
this.snapshotChunkSize = opts?.snapshotChunkSize ?? 200;
|
|
248
244
|
this.retentionMs = opts?.retentionMs ?? 7 * 24 * 60 * 60 * 1000;
|
|
249
|
-
this.routingKeyNotifier = opts?.routingKeyNotifier;
|
|
250
245
|
this.getTouchJournal = opts?.getTouchJournal;
|
|
251
246
|
}
|
|
252
247
|
|
|
@@ -500,8 +495,6 @@ export class LiveMetricsV2 {
|
|
|
500
495
|
this.lagSumMs = 0;
|
|
501
496
|
this.lagSamples = 0;
|
|
502
497
|
|
|
503
|
-
const rkInterval = this.routingKeyNotifier?.snapshotAndResetIntervalStats() ?? null;
|
|
504
|
-
|
|
505
498
|
for (const st of states) {
|
|
506
499
|
const stream = st.stream;
|
|
507
500
|
const regRow = this.db.getStream(stream);
|
|
@@ -522,26 +515,8 @@ export class LiveMetricsV2 {
|
|
|
522
515
|
if (!touchCfg) continue;
|
|
523
516
|
|
|
524
517
|
const c = this.get(stream, touchCfg);
|
|
525
|
-
const
|
|
526
|
-
const
|
|
527
|
-
const journal = storage === "memory" ? this.getTouchJournal?.(derived) ?? null : null;
|
|
528
|
-
const trow = (() => {
|
|
529
|
-
try {
|
|
530
|
-
return this.db.getStream(derived);
|
|
531
|
-
} catch {
|
|
532
|
-
return null;
|
|
533
|
-
}
|
|
534
|
-
})();
|
|
535
|
-
const touchTailSeq = trow ? (trow.next_offset > 0n ? trow.next_offset - 1n : -1n) : -1n;
|
|
536
|
-
let touchWalOldestOffset: string | null = null;
|
|
537
|
-
try {
|
|
538
|
-
const oldest = this.db.getWalOldestOffset(derived);
|
|
539
|
-
touchWalOldestOffset = oldest == null || !trow ? null : encodeOffset(trow.epoch, oldest);
|
|
540
|
-
} catch {
|
|
541
|
-
touchWalOldestOffset = null;
|
|
542
|
-
}
|
|
543
|
-
const waitActive =
|
|
544
|
-
storage === "memory" ? (journal?.meta.activeWaiters ?? 0) : this.routingKeyNotifier ? this.routingKeyNotifier.getActiveWaiters(derived) : 0;
|
|
518
|
+
const journal = this.getTouchJournal?.(stream) ?? null;
|
|
519
|
+
const waitActive = journal?.meta.activeWaiters ?? 0;
|
|
545
520
|
const tailSeq = regRow.next_offset > 0n ? regRow.next_offset - 1n : -1n;
|
|
546
521
|
const interpretedThrough = st.interpreted_through;
|
|
547
522
|
const gcThrough = interpretedThrough < regRow.uploaded_through ? interpretedThrough : regRow.uploaded_through;
|
|
@@ -571,7 +546,6 @@ export class LiveMetricsV2 {
|
|
|
571
546
|
instanceId: this.instanceId,
|
|
572
547
|
region: this.region,
|
|
573
548
|
touch: {
|
|
574
|
-
storage,
|
|
575
549
|
coarseIntervalMs: c.touch.coarseIntervalMs,
|
|
576
550
|
coalesceWindowMs: c.touch.coalesceWindowMs,
|
|
577
551
|
mode: c.touch.mode,
|
|
@@ -596,16 +570,11 @@ export class LiveMetricsV2 {
|
|
|
596
570
|
fineTouchesSuppressedBatchesDueToLag: c.touch.fineTouchesSuppressedBatchesDueToLag,
|
|
597
571
|
fineTouchesSuppressedSecondsDueToLag: c.touch.fineTouchesSuppressedMsDueToLag / 1000,
|
|
598
572
|
fineTouchesSuppressedBatchesDueToBudget: c.touch.fineTouchesSuppressedBatchesDueToBudget,
|
|
599
|
-
cursor:
|
|
600
|
-
epoch:
|
|
601
|
-
generation:
|
|
602
|
-
pendingKeys:
|
|
603
|
-
overflowBuckets:
|
|
604
|
-
walTailOffset: trow ? encodeOffset(trow.epoch, touchTailSeq) : null,
|
|
605
|
-
walNextOffset: trow ? encodeOffset(trow.epoch, trow.next_offset) : null,
|
|
606
|
-
walOldestOffset: touchWalOldestOffset,
|
|
607
|
-
walRetainedRows: trow ? clampBigInt(trow.wal_rows) : 0,
|
|
608
|
-
walRetainedBytes: trow ? clampBigInt(trow.wal_bytes) : 0,
|
|
573
|
+
cursor: journal?.meta.cursor ?? null,
|
|
574
|
+
epoch: journal?.meta.epoch ?? null,
|
|
575
|
+
generation: journal?.meta.generation ?? null,
|
|
576
|
+
pendingKeys: journal?.meta.pendingKeys ?? 0,
|
|
577
|
+
overflowBuckets: journal?.meta.overflowBuckets ?? 0,
|
|
609
578
|
},
|
|
610
579
|
templates: {
|
|
611
580
|
active: activeTemplates,
|
|
@@ -624,15 +593,15 @@ export class LiveMetricsV2 {
|
|
|
624
593
|
avgLatencyMs: c.wait.calls > 0 ? c.wait.latencySumMs / c.wait.calls : 0,
|
|
625
594
|
p95LatencyMs: c.wait.latencyHist.p95(),
|
|
626
595
|
activeWaiters: waitActive,
|
|
627
|
-
timeoutsFired:
|
|
628
|
-
timeoutSweeps:
|
|
629
|
-
timeoutSweepMsSum:
|
|
630
|
-
timeoutSweepMsMax:
|
|
631
|
-
notifyWakeups:
|
|
632
|
-
notifyFlushes:
|
|
633
|
-
notifyWakeMsSum:
|
|
634
|
-
notifyWakeMsMax:
|
|
635
|
-
timeoutHeapSize:
|
|
596
|
+
timeoutsFired: journal?.interval.timeoutsFired ?? 0,
|
|
597
|
+
timeoutSweeps: journal?.interval.timeoutSweeps ?? 0,
|
|
598
|
+
timeoutSweepMsSum: journal?.interval.timeoutSweepMsSum ?? 0,
|
|
599
|
+
timeoutSweepMsMax: journal?.interval.timeoutSweepMsMax ?? 0,
|
|
600
|
+
notifyWakeups: journal?.interval.notifyWakeups ?? 0,
|
|
601
|
+
notifyFlushes: journal?.interval.notifyFlushes ?? 0,
|
|
602
|
+
notifyWakeMsSum: journal?.interval.notifyWakeMsSum ?? 0,
|
|
603
|
+
notifyWakeMsMax: journal?.interval.notifyWakeMsMax ?? 0,
|
|
604
|
+
timeoutHeapSize: journal?.interval.heapSize ?? 0,
|
|
636
605
|
},
|
|
637
606
|
interpreter: {
|
|
638
607
|
eventsIn: c.interpreter.eventsIn,
|