@prisma/streams-server 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +8 -0
- package/package.json +2 -1
- package/src/app.ts +290 -17
- package/src/app_core.ts +1833 -698
- package/src/app_local.ts +144 -4
- package/src/auto_tune.ts +62 -0
- package/src/bootstrap.ts +159 -1
- package/src/concurrency_gate.ts +108 -0
- package/src/config.ts +116 -14
- package/src/db/db.ts +1201 -131
- package/src/db/schema.ts +308 -8
- package/src/foreground_activity.ts +55 -0
- package/src/index/indexer.ts +254 -124
- package/src/index/lexicon_file_cache.ts +261 -0
- package/src/index/lexicon_format.ts +93 -0
- package/src/index/lexicon_indexer.ts +789 -0
- package/src/index/secondary_indexer.ts +824 -0
- package/src/index/secondary_schema.ts +105 -0
- package/src/ingest.ts +10 -12
- package/src/manifest.ts +143 -8
- package/src/memory.ts +183 -8
- package/src/metrics.ts +15 -29
- package/src/metrics_emitter.ts +26 -3
- package/src/notifier.ts +121 -5
- package/src/objectstore/accounting.ts +92 -0
- package/src/objectstore/mock_r2.ts +1 -1
- package/src/objectstore/r2.ts +17 -1
- package/src/profiles/evlog/schema.ts +234 -0
- package/src/profiles/evlog.ts +299 -0
- package/src/profiles/generic.ts +47 -0
- package/src/profiles/index.ts +205 -0
- package/src/profiles/metrics/block_format.ts +109 -0
- package/src/profiles/metrics/normalize.ts +366 -0
- package/src/profiles/metrics/schema.ts +319 -0
- package/src/profiles/metrics.ts +85 -0
- package/src/profiles/profile.ts +225 -0
- package/src/{touch/engine.ts → profiles/stateProtocol/changes.ts} +3 -20
- package/src/profiles/stateProtocol/routes.ts +389 -0
- package/src/profiles/stateProtocol/types.ts +6 -0
- package/src/profiles/stateProtocol/validation.ts +51 -0
- package/src/profiles/stateProtocol.ts +100 -0
- package/src/read_filter.ts +468 -0
- package/src/reader.ts +2151 -164
- package/src/runtime/host_runtime.ts +5 -0
- package/src/runtime_memory.ts +200 -0
- package/src/runtime_memory_sampler.ts +235 -0
- package/src/schema/read_json.ts +43 -0
- package/src/schema/registry.ts +563 -59
- package/src/search/agg_format.ts +638 -0
- package/src/search/aggregate.ts +389 -0
- package/src/search/binary/codec.ts +162 -0
- package/src/search/binary/docset.ts +67 -0
- package/src/search/binary/restart_strings.ts +181 -0
- package/src/search/binary/varint.ts +34 -0
- package/src/search/bitset.ts +19 -0
- package/src/search/col_format.ts +382 -0
- package/src/search/col_runtime.ts +59 -0
- package/src/search/column_encoding.ts +43 -0
- package/src/search/companion_file_cache.ts +319 -0
- package/src/search/companion_format.ts +313 -0
- package/src/search/companion_manager.ts +1086 -0
- package/src/search/companion_plan.ts +218 -0
- package/src/search/fts_format.ts +423 -0
- package/src/search/fts_runtime.ts +333 -0
- package/src/search/query.ts +875 -0
- package/src/search/schema.ts +245 -0
- package/src/segment/cache.ts +93 -2
- package/src/segment/cached_segment.ts +89 -0
- package/src/segment/format.ts +108 -36
- package/src/segment/segmenter.ts +79 -5
- package/src/segment/segmenter_worker.ts +35 -6
- package/src/segment/segmenter_workers.ts +42 -12
- package/src/server.ts +150 -36
- package/src/sqlite/adapter.ts +185 -14
- package/src/sqlite/runtime_stats.ts +163 -0
- package/src/stats.ts +3 -3
- package/src/stream_size_reconciler.ts +100 -0
- package/src/touch/canonical_change.ts +7 -0
- package/src/touch/live_metrics.ts +94 -64
- package/src/touch/live_templates.ts +15 -1
- package/src/touch/manager.ts +166 -88
- package/src/touch/{interpreter_worker.ts → processor_worker.ts} +19 -14
- package/src/touch/spec.ts +95 -92
- package/src/touch/touch_journal.ts +4 -0
- package/src/touch/worker_pool.ts +8 -14
- package/src/touch/worker_protocol.ts +3 -3
- package/src/uploader.ts +77 -6
- package/src/util/bloom256.ts +2 -2
- package/src/util/byte_lru.ts +73 -0
- package/src/util/lru.ts +8 -0
- package/src/util/stream_paths.ts +19 -0
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
import { Result } from "better-result";
|
|
2
|
+
import { encodeOffset } from "../../offset";
|
|
3
|
+
import { parseTouchCursor } from "../../touch/touch_journal";
|
|
4
|
+
import { touchKeyIdFromRoutingKeyResult } from "../../touch/touch_key_id";
|
|
5
|
+
import { tableKeyIdFor, templateKeyIdFor } from "../../touch/live_keys";
|
|
6
|
+
import type { TemplateDecl } from "../../touch/live_templates";
|
|
7
|
+
import type { TouchConfig } from "../../touch/spec";
|
|
8
|
+
import type { StreamTouchRouteArgs } from "../profile";
|
|
9
|
+
import { getStateProtocolTouchConfig } from "./validation";
|
|
10
|
+
|
|
11
|
+
function countActiveTemplates(stream: string, db: StreamTouchRouteArgs["db"]): number {
|
|
12
|
+
try {
|
|
13
|
+
const row = db.db.query(`SELECT COUNT(*) as cnt FROM live_templates WHERE stream=? AND state='active';`).get(stream) as any;
|
|
14
|
+
return Number(row?.cnt ?? 0);
|
|
15
|
+
} catch {
|
|
16
|
+
return 0;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function parseInactivityTtlResult(
|
|
21
|
+
raw: unknown,
|
|
22
|
+
defaultValue: number,
|
|
23
|
+
fieldPath: string
|
|
24
|
+
): Result<number, { message: string }> {
|
|
25
|
+
if (raw === undefined) return Result.ok(defaultValue);
|
|
26
|
+
if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) {
|
|
27
|
+
return Result.ok(Math.floor(raw));
|
|
28
|
+
}
|
|
29
|
+
return Result.err({ message: `${fieldPath} must be a non-negative number (ms)` });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function parseTemplateDeclsResult(raw: unknown, fieldPath: string): Result<TemplateDecl[], { message: string }> {
|
|
33
|
+
if (!Array.isArray(raw) || raw.length === 0) {
|
|
34
|
+
return Result.err({ message: `${fieldPath} must be a non-empty array` });
|
|
35
|
+
}
|
|
36
|
+
if (raw.length > 256) return Result.err({ message: `${fieldPath} too large (max 256)` });
|
|
37
|
+
|
|
38
|
+
const templates: TemplateDecl[] = [];
|
|
39
|
+
for (const t of raw) {
|
|
40
|
+
const entity = typeof t?.entity === "string" ? t.entity.trim() : "";
|
|
41
|
+
const fieldsRaw = t?.fields;
|
|
42
|
+
if (entity === "" || !Array.isArray(fieldsRaw) || fieldsRaw.length === 0 || fieldsRaw.length > 3) {
|
|
43
|
+
return Result.err({ message: `${fieldPath} contains invalid template definitions` });
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const fields: TemplateDecl["fields"] = [];
|
|
47
|
+
for (const f of fieldsRaw) {
|
|
48
|
+
const name = typeof f?.name === "string" ? f.name.trim() : "";
|
|
49
|
+
const encoding = f?.encoding;
|
|
50
|
+
if (name === "") {
|
|
51
|
+
return Result.err({ message: `${fieldPath} contains invalid template definitions` });
|
|
52
|
+
}
|
|
53
|
+
fields.push({ name, encoding });
|
|
54
|
+
}
|
|
55
|
+
if (fields.length !== fieldsRaw.length) {
|
|
56
|
+
return Result.err({ message: `${fieldPath} contains invalid template definitions` });
|
|
57
|
+
}
|
|
58
|
+
templates.push({ entity, fields });
|
|
59
|
+
}
|
|
60
|
+
return Result.ok(templates);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function handleTemplatesActivateRoute(args: StreamTouchRouteArgs, touchCfg: TouchConfig): Promise<Response> {
|
|
64
|
+
const { req, stream, streamRow, touchManager, respond } = args;
|
|
65
|
+
if (req.method !== "POST") return respond.badRequest("unsupported method");
|
|
66
|
+
|
|
67
|
+
let body: any;
|
|
68
|
+
try {
|
|
69
|
+
body = await req.json();
|
|
70
|
+
} catch {
|
|
71
|
+
return respond.badRequest("activate body must be valid JSON");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const templatesRes = parseTemplateDeclsResult(body?.templates, "activate.templates");
|
|
75
|
+
if (Result.isError(templatesRes)) return respond.badRequest(templatesRes.error.message);
|
|
76
|
+
|
|
77
|
+
const inactivityTtlRes = parseInactivityTtlResult(
|
|
78
|
+
body?.inactivityTtlMs,
|
|
79
|
+
touchCfg.templates?.defaultInactivityTtlMs ?? 60 * 60 * 1000,
|
|
80
|
+
"activate.inactivityTtlMs"
|
|
81
|
+
);
|
|
82
|
+
if (Result.isError(inactivityTtlRes)) return respond.badRequest(inactivityTtlRes.error.message);
|
|
83
|
+
|
|
84
|
+
const limits = {
|
|
85
|
+
maxActiveTemplatesPerStream: touchCfg.templates?.maxActiveTemplatesPerStream ?? 2048,
|
|
86
|
+
maxActiveTemplatesPerEntity: touchCfg.templates?.maxActiveTemplatesPerEntity ?? 256,
|
|
87
|
+
};
|
|
88
|
+
const activeFromTouchOffset = touchManager.getOrCreateJournal(stream, touchCfg).getCursor();
|
|
89
|
+
const res = touchManager.activateTemplates({
|
|
90
|
+
stream,
|
|
91
|
+
touchCfg,
|
|
92
|
+
baseStreamNextOffset: streamRow.next_offset,
|
|
93
|
+
activeFromTouchOffset,
|
|
94
|
+
templates: templatesRes.value,
|
|
95
|
+
inactivityTtlMs: inactivityTtlRes.value,
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
return respond.json(200, { activated: res.activated, denied: res.denied, limits });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function handleMetaRoute(args: StreamTouchRouteArgs, touchCfg: TouchConfig): Response {
|
|
102
|
+
const { stream, streamRow, db, touchManager, respond } = args;
|
|
103
|
+
const meta = touchManager.getOrCreateJournal(stream, touchCfg).getMeta();
|
|
104
|
+
const runtime = touchManager.getTouchRuntimeSnapshot({ stream, touchCfg });
|
|
105
|
+
const touchState = db.getStreamTouchState(stream);
|
|
106
|
+
return respond.json(200, {
|
|
107
|
+
...meta,
|
|
108
|
+
coarseIntervalMs: touchCfg.coarseIntervalMs ?? 100,
|
|
109
|
+
touchCoalesceWindowMs: touchCfg.touchCoalesceWindowMs ?? 100,
|
|
110
|
+
activeTemplates: countActiveTemplates(stream, db),
|
|
111
|
+
lagSourceOffsets: runtime.lagSourceOffsets,
|
|
112
|
+
touchMode: runtime.touchMode,
|
|
113
|
+
walScannedThrough: touchState ? encodeOffset(streamRow.epoch, touchState.processed_through) : null,
|
|
114
|
+
bucketMaxSourceOffsetSeq: meta.bucketMaxSourceOffsetSeq,
|
|
115
|
+
hotFineKeys: runtime.hotFineKeys,
|
|
116
|
+
hotTemplates: runtime.hotTemplates,
|
|
117
|
+
hotFineKeysActive: runtime.hotFineKeysActive,
|
|
118
|
+
hotFineKeysGrace: runtime.hotFineKeysGrace,
|
|
119
|
+
hotTemplatesActive: runtime.hotTemplatesActive,
|
|
120
|
+
hotTemplatesGrace: runtime.hotTemplatesGrace,
|
|
121
|
+
fineWaitersActive: runtime.fineWaitersActive,
|
|
122
|
+
coarseWaitersActive: runtime.coarseWaitersActive,
|
|
123
|
+
broadFineWaitersActive: runtime.broadFineWaitersActive,
|
|
124
|
+
hotKeyFilteringEnabled: runtime.hotKeyFilteringEnabled,
|
|
125
|
+
hotTemplateFilteringEnabled: runtime.hotTemplateFilteringEnabled,
|
|
126
|
+
scanRowsTotal: runtime.scanRowsTotal,
|
|
127
|
+
scanBatchesTotal: runtime.scanBatchesTotal,
|
|
128
|
+
scannedButEmitted0BatchesTotal: runtime.scannedButEmitted0BatchesTotal,
|
|
129
|
+
processedThroughDeltaTotal: runtime.processedThroughDeltaTotal,
|
|
130
|
+
touchesEmittedTotal: runtime.touchesEmittedTotal,
|
|
131
|
+
touchesTableTotal: runtime.touchesTableTotal,
|
|
132
|
+
touchesTemplateTotal: runtime.touchesTemplateTotal,
|
|
133
|
+
fineTouchesDroppedDueToBudgetTotal: runtime.fineTouchesDroppedDueToBudgetTotal,
|
|
134
|
+
fineTouchesSkippedColdTemplateTotal: runtime.fineTouchesSkippedColdTemplateTotal,
|
|
135
|
+
fineTouchesSkippedColdKeyTotal: runtime.fineTouchesSkippedColdKeyTotal,
|
|
136
|
+
fineTouchesSkippedTemplateBucketTotal: runtime.fineTouchesSkippedTemplateBucketTotal,
|
|
137
|
+
waitTouchedTotal: runtime.waitTouchedTotal,
|
|
138
|
+
waitTimeoutTotal: runtime.waitTimeoutTotal,
|
|
139
|
+
waitStaleTotal: runtime.waitStaleTotal,
|
|
140
|
+
journalFlushesTotal: runtime.journalFlushesTotal,
|
|
141
|
+
journalNotifyWakeupsTotal: runtime.journalNotifyWakeupsTotal,
|
|
142
|
+
journalNotifyWakeMsTotal: runtime.journalNotifyWakeMsTotal,
|
|
143
|
+
journalNotifyWakeMsMax: runtime.journalNotifyWakeMsMax,
|
|
144
|
+
journalTimeoutsFiredTotal: runtime.journalTimeoutsFiredTotal,
|
|
145
|
+
journalTimeoutSweepMsTotal: runtime.journalTimeoutSweepMsTotal,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function handleWaitRoute(args: StreamTouchRouteArgs, touchCfg: TouchConfig): Promise<Response> {
|
|
150
|
+
const { req, stream, streamRow, touchManager, respond } = args;
|
|
151
|
+
if (req.method !== "POST") return respond.badRequest("unsupported method");
|
|
152
|
+
|
|
153
|
+
const waitStartMs = Date.now();
|
|
154
|
+
let body: any;
|
|
155
|
+
try {
|
|
156
|
+
body = await req.json();
|
|
157
|
+
} catch {
|
|
158
|
+
return respond.badRequest("wait body must be valid JSON");
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const keysRaw = body?.keys;
|
|
162
|
+
if (keysRaw !== undefined && (!Array.isArray(keysRaw) || !keysRaw.every((k: any) => typeof k === "string" && k.trim() !== ""))) {
|
|
163
|
+
return respond.badRequest("wait.keys must be a non-empty string array when provided");
|
|
164
|
+
}
|
|
165
|
+
const keys = Array.isArray(keysRaw) ? Array.from(new Set(keysRaw.map((k: string) => k.trim()))) : [];
|
|
166
|
+
if (keys.length > 1024) return respond.badRequest("wait.keys too large (max 1024)");
|
|
167
|
+
|
|
168
|
+
const keyIdsRaw = body?.keyIds;
|
|
169
|
+
const keyIds =
|
|
170
|
+
Array.isArray(keyIdsRaw) && keyIdsRaw.length > 0
|
|
171
|
+
? Array.from(
|
|
172
|
+
new Set(
|
|
173
|
+
keyIdsRaw
|
|
174
|
+
.map((x: any) => Number(x))
|
|
175
|
+
.filter((n: number) => Number.isFinite(n) && Number.isInteger(n) && n >= 0 && n <= 0xffffffff)
|
|
176
|
+
)
|
|
177
|
+
).map((n) => n >>> 0)
|
|
178
|
+
: [];
|
|
179
|
+
if (Array.isArray(keyIdsRaw) && keyIds.length !== keyIdsRaw.length) {
|
|
180
|
+
return respond.badRequest("wait.keyIds must be a non-empty uint32 array when provided");
|
|
181
|
+
}
|
|
182
|
+
if (keys.length === 0 && keyIds.length === 0) return respond.badRequest("wait requires keys or keyIds");
|
|
183
|
+
if (keyIds.length > 1024) return respond.badRequest("wait.keyIds too large (max 1024)");
|
|
184
|
+
|
|
185
|
+
const cursorRaw = body?.cursor;
|
|
186
|
+
if (typeof cursorRaw !== "string" || cursorRaw.trim() === "") return respond.badRequest("wait.cursor must be a non-empty string");
|
|
187
|
+
const cursor = cursorRaw.trim();
|
|
188
|
+
|
|
189
|
+
const timeoutMsRaw = body?.timeoutMs;
|
|
190
|
+
const timeoutMs =
|
|
191
|
+
timeoutMsRaw === undefined
|
|
192
|
+
? 30_000
|
|
193
|
+
: typeof timeoutMsRaw === "number" && Number.isFinite(timeoutMsRaw)
|
|
194
|
+
? Math.max(0, Math.min(120_000, timeoutMsRaw))
|
|
195
|
+
: null;
|
|
196
|
+
if (timeoutMs == null) return respond.badRequest("wait.timeoutMs must be a number (ms)");
|
|
197
|
+
|
|
198
|
+
const templateIdsUsedRaw = body?.templateIdsUsed;
|
|
199
|
+
if (Array.isArray(templateIdsUsedRaw) && !templateIdsUsedRaw.every((x: any) => typeof x === "string" && x.trim() !== "")) {
|
|
200
|
+
return respond.badRequest("wait.templateIdsUsed must be a string array");
|
|
201
|
+
}
|
|
202
|
+
const templateIdsUsed =
|
|
203
|
+
Array.isArray(templateIdsUsedRaw) && templateIdsUsedRaw.length > 0
|
|
204
|
+
? Array.from(new Set(templateIdsUsedRaw.map((s: any) => (typeof s === "string" ? s.trim() : "")).filter((s: string) => s !== "")))
|
|
205
|
+
: [];
|
|
206
|
+
|
|
207
|
+
const interestModeRaw = body?.interestMode;
|
|
208
|
+
if (interestModeRaw !== undefined && interestModeRaw !== "fine" && interestModeRaw !== "coarse") {
|
|
209
|
+
return respond.badRequest("wait.interestMode must be 'fine' or 'coarse'");
|
|
210
|
+
}
|
|
211
|
+
const interestMode: "fine" | "coarse" = interestModeRaw === "coarse" ? "coarse" : "fine";
|
|
212
|
+
|
|
213
|
+
if (interestMode === "fine" && templateIdsUsed.length > 0) {
|
|
214
|
+
touchManager.heartbeatTemplates({ stream, touchCfg, templateIdsUsed });
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const declareTemplatesRaw = body?.declareTemplates;
|
|
218
|
+
if (Array.isArray(declareTemplatesRaw) && declareTemplatesRaw.length > 0) {
|
|
219
|
+
const templatesRes = parseTemplateDeclsResult(declareTemplatesRaw, "wait.declareTemplates");
|
|
220
|
+
if (Result.isError(templatesRes)) return respond.badRequest(templatesRes.error.message);
|
|
221
|
+
|
|
222
|
+
const inactivityTtlRes = parseInactivityTtlResult(
|
|
223
|
+
body?.inactivityTtlMs,
|
|
224
|
+
touchCfg.templates?.defaultInactivityTtlMs ?? 60 * 60 * 1000,
|
|
225
|
+
"wait.inactivityTtlMs"
|
|
226
|
+
);
|
|
227
|
+
if (Result.isError(inactivityTtlRes)) return respond.badRequest(inactivityTtlRes.error.message);
|
|
228
|
+
|
|
229
|
+
const activeFromTouchOffset = touchManager.getOrCreateJournal(stream, touchCfg).getCursor();
|
|
230
|
+
touchManager.activateTemplates({
|
|
231
|
+
stream,
|
|
232
|
+
touchCfg,
|
|
233
|
+
baseStreamNextOffset: streamRow.next_offset,
|
|
234
|
+
activeFromTouchOffset,
|
|
235
|
+
templates: templatesRes.value,
|
|
236
|
+
inactivityTtlMs: inactivityTtlRes.value,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const journal = touchManager.getOrCreateJournal(stream, touchCfg);
|
|
241
|
+
const runtime = touchManager.getTouchRuntimeSnapshot({ stream, touchCfg });
|
|
242
|
+
|
|
243
|
+
let rawFineKeyIds = keyIds;
|
|
244
|
+
if (keyIds.length === 0) {
|
|
245
|
+
const parsedKeyIds: number[] = [];
|
|
246
|
+
for (const key of keys) {
|
|
247
|
+
const keyIdRes = touchKeyIdFromRoutingKeyResult(key);
|
|
248
|
+
if (Result.isError(keyIdRes)) return respond.internalError();
|
|
249
|
+
parsedKeyIds.push(keyIdRes.value);
|
|
250
|
+
}
|
|
251
|
+
rawFineKeyIds = parsedKeyIds;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const templateWaitKeyIds = templateIdsUsed.length > 0 ? Array.from(new Set(templateIdsUsed.map((templateId) => templateKeyIdFor(templateId) >>> 0))) : [];
|
|
255
|
+
let waitKeyIds = rawFineKeyIds;
|
|
256
|
+
let effectiveWaitKind: "fineKey" | "templateKey" | "tableKey" = "fineKey";
|
|
257
|
+
|
|
258
|
+
if (interestMode === "coarse") {
|
|
259
|
+
effectiveWaitKind = "tableKey";
|
|
260
|
+
} else if (runtime.touchMode === "restricted" && templateIdsUsed.length > 0) {
|
|
261
|
+
effectiveWaitKind = "templateKey";
|
|
262
|
+
} else if (runtime.touchMode === "coarseOnly" && templateIdsUsed.length > 0) {
|
|
263
|
+
effectiveWaitKind = "tableKey";
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
if (effectiveWaitKind === "templateKey") {
|
|
267
|
+
waitKeyIds = templateWaitKeyIds;
|
|
268
|
+
} else if (effectiveWaitKind === "tableKey" && templateIdsUsed.length > 0) {
|
|
269
|
+
const entities = touchManager.resolveTemplateEntitiesForWait({ stream, templateIdsUsed });
|
|
270
|
+
waitKeyIds = Array.from(new Set(entities.map((entity) => tableKeyIdFor(entity) >>> 0)));
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
if (interestMode === "fine" && effectiveWaitKind === "fineKey" && templateWaitKeyIds.length > 0) {
|
|
274
|
+
const merged = new Set<number>();
|
|
275
|
+
for (const keyId of waitKeyIds) merged.add(keyId >>> 0);
|
|
276
|
+
for (const keyId of templateWaitKeyIds) merged.add(keyId >>> 0);
|
|
277
|
+
waitKeyIds = Array.from(merged);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if (waitKeyIds.length === 0) {
|
|
281
|
+
waitKeyIds = rawFineKeyIds;
|
|
282
|
+
effectiveWaitKind = "fineKey";
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
const hotInterestKeyIds = interestMode === "fine" ? rawFineKeyIds : waitKeyIds;
|
|
286
|
+
const releaseHotInterest = touchManager.beginHotWaitInterest({
|
|
287
|
+
stream,
|
|
288
|
+
touchCfg,
|
|
289
|
+
keyIds: hotInterestKeyIds,
|
|
290
|
+
templateIdsUsed,
|
|
291
|
+
interestMode,
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
try {
|
|
295
|
+
let sinceGen: number;
|
|
296
|
+
if (cursor === "now") {
|
|
297
|
+
sinceGen = journal.getGeneration();
|
|
298
|
+
} else {
|
|
299
|
+
const parsed = parseTouchCursor(cursor);
|
|
300
|
+
if (!parsed) return respond.badRequest("wait.cursor must be in the form <epochHex>:<generation> or 'now'");
|
|
301
|
+
if (parsed.epoch !== journal.getEpoch()) {
|
|
302
|
+
const latencyMs = Date.now() - waitStartMs;
|
|
303
|
+
touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "stale", latencyMs });
|
|
304
|
+
return respond.json(200, {
|
|
305
|
+
stale: true,
|
|
306
|
+
cursor: journal.getCursor(),
|
|
307
|
+
epoch: journal.getEpoch(),
|
|
308
|
+
generation: journal.getGeneration(),
|
|
309
|
+
effectiveWaitKind,
|
|
310
|
+
bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
|
|
311
|
+
flushAtMs: journal.getLastFlushAtMs(),
|
|
312
|
+
bucketStartMs: journal.getLastBucketStartMs(),
|
|
313
|
+
error: { code: "stale", message: "cursor epoch mismatch; rerun/re-subscribe and start from cursor" },
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
sinceGen = parsed.generation;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const nowGen = journal.getGeneration();
|
|
320
|
+
if (sinceGen > nowGen) sinceGen = nowGen;
|
|
321
|
+
|
|
322
|
+
if (journal.maybeTouchedSinceAny(waitKeyIds, sinceGen)) {
|
|
323
|
+
const latencyMs = Date.now() - waitStartMs;
|
|
324
|
+
touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs });
|
|
325
|
+
return respond.json(200, {
|
|
326
|
+
touched: true,
|
|
327
|
+
cursor: journal.getCursor(),
|
|
328
|
+
effectiveWaitKind,
|
|
329
|
+
bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
|
|
330
|
+
flushAtMs: journal.getLastFlushAtMs(),
|
|
331
|
+
bucketStartMs: journal.getLastBucketStartMs(),
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
const deadline = Date.now() + timeoutMs;
|
|
336
|
+
const remaining = deadline - Date.now();
|
|
337
|
+
if (remaining <= 0) {
|
|
338
|
+
const latencyMs = Date.now() - waitStartMs;
|
|
339
|
+
touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "timeout", latencyMs });
|
|
340
|
+
return respond.json(200, {
|
|
341
|
+
touched: false,
|
|
342
|
+
cursor: journal.getCursor(),
|
|
343
|
+
effectiveWaitKind,
|
|
344
|
+
bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
|
|
345
|
+
flushAtMs: journal.getLastFlushAtMs(),
|
|
346
|
+
bucketStartMs: journal.getLastBucketStartMs(),
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
const afterGen = journal.getGeneration();
|
|
351
|
+
const hit = await journal.waitForAny({ keys: waitKeyIds, afterGeneration: afterGen, timeoutMs: remaining, signal: req.signal });
|
|
352
|
+
if (req.signal.aborted) return new Response(null, { status: 204 });
|
|
353
|
+
|
|
354
|
+
if (hit == null) {
|
|
355
|
+
const latencyMs = Date.now() - waitStartMs;
|
|
356
|
+
touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "timeout", latencyMs });
|
|
357
|
+
return respond.json(200, {
|
|
358
|
+
touched: false,
|
|
359
|
+
cursor: journal.getCursor(),
|
|
360
|
+
effectiveWaitKind,
|
|
361
|
+
bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
|
|
362
|
+
flushAtMs: journal.getLastFlushAtMs(),
|
|
363
|
+
bucketStartMs: journal.getLastBucketStartMs(),
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const latencyMs = Date.now() - waitStartMs;
|
|
368
|
+
touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs });
|
|
369
|
+
return respond.json(200, {
|
|
370
|
+
touched: true,
|
|
371
|
+
cursor: journal.getCursor(),
|
|
372
|
+
effectiveWaitKind,
|
|
373
|
+
bucketMaxSourceOffsetSeq: hit.bucketMaxSourceOffsetSeq.toString(),
|
|
374
|
+
flushAtMs: hit.flushAtMs,
|
|
375
|
+
bucketStartMs: hit.bucketStartMs,
|
|
376
|
+
});
|
|
377
|
+
} finally {
|
|
378
|
+
releaseHotInterest();
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
export async function handleStateProtocolTouchRoute(args: StreamTouchRouteArgs): Promise<Response> {
|
|
383
|
+
const { route, profile, respond } = args;
|
|
384
|
+
const touchCfg = getStateProtocolTouchConfig(profile);
|
|
385
|
+
if (!touchCfg) return respond.notFound("touch not enabled");
|
|
386
|
+
if (route.kind === "templates_activate") return handleTemplatesActivateRoute(args, touchCfg);
|
|
387
|
+
if (route.kind === "meta") return handleMetaRoute(args, touchCfg);
|
|
388
|
+
return handleWaitRoute(args, touchCfg);
|
|
389
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { Result } from "better-result";
|
|
2
|
+
import type { CachedStreamProfile, StreamProfileSpec } from "../profile";
|
|
3
|
+
import {
|
|
4
|
+
cloneStreamProfileSpec,
|
|
5
|
+
expectPlainObjectResult,
|
|
6
|
+
rejectUnknownKeysResult,
|
|
7
|
+
} from "../profile";
|
|
8
|
+
import { validateTouchConfigResult, type TouchConfig } from "../../touch/spec";
|
|
9
|
+
import type { StateProtocolStreamProfile } from "./types";
|
|
10
|
+
|
|
11
|
+
export function isStateProtocolProfile(
|
|
12
|
+
profile: StreamProfileSpec | null | undefined
|
|
13
|
+
): profile is StateProtocolStreamProfile {
|
|
14
|
+
return !!profile && profile.kind === "state-protocol";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function getStateProtocolTouchConfig(profile: StreamProfileSpec | null | undefined): TouchConfig | null {
|
|
18
|
+
return isStateProtocolProfile(profile) && profile.touch?.enabled ? profile.touch : null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function cloneStateProtocolProfile(profile: StateProtocolStreamProfile): StateProtocolStreamProfile {
|
|
22
|
+
return cloneStreamProfileSpec(profile) as StateProtocolStreamProfile;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function cloneStateProtocolCache(cache: CachedStreamProfile | null): CachedStreamProfile | null {
|
|
26
|
+
if (!cache || cache.profile.kind !== "state-protocol") return null;
|
|
27
|
+
return {
|
|
28
|
+
profile: cloneStateProtocolProfile(cache.profile as StateProtocolStreamProfile),
|
|
29
|
+
updatedAtMs: cache.updatedAtMs,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function validateStateProtocolProfileResult(
|
|
34
|
+
raw: unknown,
|
|
35
|
+
path: string
|
|
36
|
+
): Result<StateProtocolStreamProfile, { message: string }> {
|
|
37
|
+
const objRes = expectPlainObjectResult(raw, path);
|
|
38
|
+
if (Result.isError(objRes)) return objRes;
|
|
39
|
+
if (objRes.value.kind !== "state-protocol") {
|
|
40
|
+
return Result.err({ message: `${path}.kind must be state-protocol` });
|
|
41
|
+
}
|
|
42
|
+
const keyCheck = rejectUnknownKeysResult(objRes.value, ["kind", "touch"], path);
|
|
43
|
+
if (Result.isError(keyCheck)) return keyCheck;
|
|
44
|
+
let touch = undefined;
|
|
45
|
+
if (objRes.value.touch !== undefined) {
|
|
46
|
+
const touchRes = validateTouchConfigResult(objRes.value.touch, `${path}.touch`);
|
|
47
|
+
if (Result.isError(touchRes)) return Result.err({ message: touchRes.error.message });
|
|
48
|
+
touch = touchRes.value;
|
|
49
|
+
}
|
|
50
|
+
return Result.ok(touch ? { kind: "state-protocol", touch } : { kind: "state-protocol" });
|
|
51
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { Result } from "better-result";
|
|
2
|
+
import type {
|
|
3
|
+
StreamProfileDefinition,
|
|
4
|
+
StreamProfilePersistResult,
|
|
5
|
+
StreamProfileReadResult,
|
|
6
|
+
StreamTouchCapability,
|
|
7
|
+
} from "./profile";
|
|
8
|
+
import { normalizeProfileContentType, parseStoredProfileJsonResult } from "./profile";
|
|
9
|
+
import { deriveStateProtocolChanges } from "./stateProtocol/changes";
|
|
10
|
+
import { handleStateProtocolTouchRoute } from "./stateProtocol/routes";
|
|
11
|
+
import type { StateProtocolStreamProfile } from "./stateProtocol/types";
|
|
12
|
+
import {
|
|
13
|
+
cloneStateProtocolCache,
|
|
14
|
+
cloneStateProtocolProfile,
|
|
15
|
+
getStateProtocolTouchConfig,
|
|
16
|
+
isStateProtocolProfile,
|
|
17
|
+
validateStateProtocolProfileResult,
|
|
18
|
+
} from "./stateProtocol/validation";
|
|
19
|
+
|
|
20
|
+
const STATE_PROTOCOL_TOUCH_CAPABILITY: StreamTouchCapability = {
|
|
21
|
+
getTouchConfig(profile) {
|
|
22
|
+
return getStateProtocolTouchConfig(profile);
|
|
23
|
+
},
|
|
24
|
+
|
|
25
|
+
syncState({ db, stream, profile }) {
|
|
26
|
+
if (getStateProtocolTouchConfig(profile)) db.ensureStreamTouchState(stream);
|
|
27
|
+
else db.deleteStreamTouchState(stream);
|
|
28
|
+
},
|
|
29
|
+
|
|
30
|
+
deriveCanonicalChanges(record) {
|
|
31
|
+
return deriveStateProtocolChanges(record);
|
|
32
|
+
},
|
|
33
|
+
|
|
34
|
+
async handleRoute(args) {
|
|
35
|
+
return handleStateProtocolTouchRoute(args);
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export const STATE_PROTOCOL_STREAM_PROFILE_DEFINITION: StreamProfileDefinition = {
|
|
40
|
+
kind: "state-protocol",
|
|
41
|
+
usesStoredProfileRow: true,
|
|
42
|
+
touch: STATE_PROTOCOL_TOUCH_CAPABILITY,
|
|
43
|
+
|
|
44
|
+
defaultProfile(): StateProtocolStreamProfile {
|
|
45
|
+
return { kind: "state-protocol" };
|
|
46
|
+
},
|
|
47
|
+
|
|
48
|
+
validateResult(raw, path) {
|
|
49
|
+
return validateStateProtocolProfileResult(raw, path);
|
|
50
|
+
},
|
|
51
|
+
|
|
52
|
+
readProfileResult({ row, cached }): Result<StreamProfileReadResult, { message: string }> {
|
|
53
|
+
if (!row) {
|
|
54
|
+
return Result.ok({ profile: { kind: "state-protocol" }, cache: null });
|
|
55
|
+
}
|
|
56
|
+
const cachedCopy = cloneStateProtocolCache(cached);
|
|
57
|
+
if (cachedCopy && cachedCopy.updatedAtMs === row.updated_at_ms) {
|
|
58
|
+
return Result.ok({
|
|
59
|
+
profile: cloneStateProtocolProfile(cachedCopy.profile as StateProtocolStreamProfile),
|
|
60
|
+
cache: cachedCopy,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
const parsedRes = parseStoredProfileJsonResult(row.profile_json);
|
|
64
|
+
if (Result.isError(parsedRes)) return parsedRes;
|
|
65
|
+
const profileRes = validateStateProtocolProfileResult(parsedRes.value, "profile");
|
|
66
|
+
if (Result.isError(profileRes)) return profileRes;
|
|
67
|
+
const profile = cloneStateProtocolProfile(profileRes.value);
|
|
68
|
+
return Result.ok({
|
|
69
|
+
profile: cloneStateProtocolProfile(profile),
|
|
70
|
+
cache: { profile, updatedAtMs: row.updated_at_ms },
|
|
71
|
+
});
|
|
72
|
+
},
|
|
73
|
+
|
|
74
|
+
persistProfileResult({ db, stream, streamRow, profile }): Result<StreamProfilePersistResult, { kind: "bad_request"; message: string; code?: string }> {
|
|
75
|
+
if (!isStateProtocolProfile(profile)) {
|
|
76
|
+
return Result.err({ kind: "bad_request", message: "invalid state-protocol profile" });
|
|
77
|
+
}
|
|
78
|
+
const contentType = normalizeProfileContentType(streamRow.content_type);
|
|
79
|
+
if (contentType !== "application/json") {
|
|
80
|
+
return Result.err({
|
|
81
|
+
kind: "bad_request",
|
|
82
|
+
message: "state-protocol profile requires application/json stream content-type",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const persistedProfile = cloneStateProtocolProfile(profile);
|
|
87
|
+
db.updateStreamProfile(stream, persistedProfile.kind);
|
|
88
|
+
db.upsertStreamProfile(stream, JSON.stringify(persistedProfile));
|
|
89
|
+
STATE_PROTOCOL_TOUCH_CAPABILITY.syncState({ db, stream, profile: persistedProfile });
|
|
90
|
+
const row = db.getStreamProfile(stream);
|
|
91
|
+
return Result.ok({
|
|
92
|
+
profile: cloneStateProtocolProfile(persistedProfile),
|
|
93
|
+
cache: {
|
|
94
|
+
profile: persistedProfile,
|
|
95
|
+
updatedAtMs: row?.updated_at_ms ?? db.nowMs(),
|
|
96
|
+
},
|
|
97
|
+
schemaRegistry: null,
|
|
98
|
+
});
|
|
99
|
+
},
|
|
100
|
+
};
|