@tungthedev/streams-server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CODE_OF_CONDUCT.md +45 -0
  2. package/CONTRIBUTING.md +76 -0
  3. package/LICENSE +201 -0
  4. package/README.md +58 -0
  5. package/SECURITY.md +42 -0
  6. package/bin/prisma-streams-server +2 -0
  7. package/package.json +46 -0
  8. package/src/app.ts +583 -0
  9. package/src/app_core.ts +3144 -0
  10. package/src/app_local.ts +206 -0
  11. package/src/auth.ts +124 -0
  12. package/src/auto_tune.ts +69 -0
  13. package/src/backpressure.ts +66 -0
  14. package/src/bootstrap.ts +613 -0
  15. package/src/compute/demo_entry.ts +415 -0
  16. package/src/compute/demo_site.ts +1242 -0
  17. package/src/compute/entry.ts +19 -0
  18. package/src/compute/package_entry.ts +4 -0
  19. package/src/compute/virtual-modules.d.ts +15 -0
  20. package/src/compute/worker_module_url.ts +9 -0
  21. package/src/concurrency_gate.ts +108 -0
  22. package/src/config.ts +402 -0
  23. package/src/db/bootstrap_store.ts +9 -0
  24. package/src/db/db.ts +2424 -0
  25. package/src/db/schema.ts +925 -0
  26. package/src/db/sqlite_manifest_snapshot.ts +81 -0
  27. package/src/db/sqlite_touch_store.ts +491 -0
  28. package/src/db/sqlite_wal_store.ts +472 -0
  29. package/src/details/full_mode_details.ts +568 -0
  30. package/src/expiry_sweeper.ts +47 -0
  31. package/src/foreground_activity.ts +55 -0
  32. package/src/hist.ts +169 -0
  33. package/src/index/binary_fuse.ts +379 -0
  34. package/src/index/indexer.ts +947 -0
  35. package/src/index/lexicon_file_cache.ts +261 -0
  36. package/src/index/lexicon_format.ts +93 -0
  37. package/src/index/lexicon_indexer.ts +863 -0
  38. package/src/index/run_cache.ts +84 -0
  39. package/src/index/run_format.ts +213 -0
  40. package/src/index/schedule.ts +28 -0
  41. package/src/index/secondary_indexer.ts +901 -0
  42. package/src/index/secondary_schema.ts +105 -0
  43. package/src/ingest.ts +309 -0
  44. package/src/lens/lens.ts +501 -0
  45. package/src/manifest.ts +249 -0
  46. package/src/memory.ts +334 -0
  47. package/src/metrics.ts +147 -0
  48. package/src/metrics_emitter.ts +83 -0
  49. package/src/notifier.ts +180 -0
  50. package/src/objectstore/accounting.ts +151 -0
  51. package/src/objectstore/interface.ts +13 -0
  52. package/src/objectstore/mock_r2.ts +269 -0
  53. package/src/objectstore/null.ts +32 -0
  54. package/src/objectstore/r2.ts +318 -0
  55. package/src/observe/pairing.ts +61 -0
  56. package/src/observe/request.ts +772 -0
  57. package/src/offset.ts +70 -0
  58. package/src/postgres/bootstrap.ts +269 -0
  59. package/src/postgres/companions.ts +197 -0
  60. package/src/postgres/control_restore.ts +109 -0
  61. package/src/postgres/details.ts +189 -0
  62. package/src/postgres/lexicon_index.ts +260 -0
  63. package/src/postgres/routing_index.ts +189 -0
  64. package/src/postgres/rows.ts +132 -0
  65. package/src/postgres/schema.ts +355 -0
  66. package/src/postgres/secondary_index.ts +238 -0
  67. package/src/postgres/segments.ts +900 -0
  68. package/src/postgres/stats.ts +103 -0
  69. package/src/postgres/store.ts +947 -0
  70. package/src/postgres/touch.ts +591 -0
  71. package/src/postgres/types.ts +32 -0
  72. package/src/profiles/evlog/schema.ts +234 -0
  73. package/src/profiles/evlog.ts +473 -0
  74. package/src/profiles/generic.ts +51 -0
  75. package/src/profiles/index.ts +237 -0
  76. package/src/profiles/metrics/block_format.ts +109 -0
  77. package/src/profiles/metrics/normalize.ts +366 -0
  78. package/src/profiles/metrics/schema.ts +319 -0
  79. package/src/profiles/metrics.ts +83 -0
  80. package/src/profiles/otelTraces/normalize.ts +955 -0
  81. package/src/profiles/otelTraces/otlp.ts +1002 -0
  82. package/src/profiles/otelTraces/schema.ts +408 -0
  83. package/src/profiles/otelTraces.ts +390 -0
  84. package/src/profiles/profile.ts +284 -0
  85. package/src/profiles/stateProtocol/change_event_conformance.typecheck.ts +35 -0
  86. package/src/profiles/stateProtocol/changes.ts +24 -0
  87. package/src/profiles/stateProtocol/ingest.ts +115 -0
  88. package/src/profiles/stateProtocol/routes.ts +511 -0
  89. package/src/profiles/stateProtocol/types.ts +6 -0
  90. package/src/profiles/stateProtocol/validation.ts +51 -0
  91. package/src/profiles/stateProtocol.ts +107 -0
  92. package/src/read_filter.ts +468 -0
  93. package/src/reader.ts +2986 -0
  94. package/src/runtime/hash.ts +156 -0
  95. package/src/runtime/hash_vendor/LICENSE.hash-wasm +38 -0
  96. package/src/runtime/hash_vendor/NOTICE.md +8 -0
  97. package/src/runtime/hash_vendor/xxhash3.umd.min.cjs +7 -0
  98. package/src/runtime/hash_vendor/xxhash32.umd.min.cjs +7 -0
  99. package/src/runtime/hash_vendor/xxhash64.umd.min.cjs +7 -0
  100. package/src/runtime/host_runtime.ts +5 -0
  101. package/src/runtime_memory.ts +200 -0
  102. package/src/runtime_memory_sampler.ts +237 -0
  103. package/src/schema/lens_schema.ts +290 -0
  104. package/src/schema/proof.ts +547 -0
  105. package/src/schema/read_json.ts +51 -0
  106. package/src/schema/registry.ts +966 -0
  107. package/src/search/agg_format.ts +638 -0
  108. package/src/search/aggregate.ts +409 -0
  109. package/src/search/binary/codec.ts +162 -0
  110. package/src/search/binary/docset.ts +67 -0
  111. package/src/search/binary/restart_strings.ts +181 -0
  112. package/src/search/binary/varint.ts +34 -0
  113. package/src/search/bitset.ts +19 -0
  114. package/src/search/col_format.ts +382 -0
  115. package/src/search/col_runtime.ts +59 -0
  116. package/src/search/column_encoding.ts +43 -0
  117. package/src/search/companion_file_cache.ts +319 -0
  118. package/src/search/companion_format.ts +327 -0
  119. package/src/search/companion_manager.ts +1305 -0
  120. package/src/search/companion_plan.ts +229 -0
  121. package/src/search/exact_format.ts +281 -0
  122. package/src/search/exact_runtime.ts +55 -0
  123. package/src/search/fts_format.ts +423 -0
  124. package/src/search/fts_runtime.ts +333 -0
  125. package/src/search/query.ts +875 -0
  126. package/src/search/schema.ts +245 -0
  127. package/src/segment/cache.ts +270 -0
  128. package/src/segment/cached_segment.ts +89 -0
  129. package/src/segment/format.ts +403 -0
  130. package/src/segment/segmenter.ts +412 -0
  131. package/src/segment/segmenter_worker.ts +72 -0
  132. package/src/segment/segmenter_workers.ts +130 -0
  133. package/src/server.ts +264 -0
  134. package/src/server_auto_tune.ts +158 -0
  135. package/src/sqlite/adapter.ts +335 -0
  136. package/src/sqlite/runtime_stats.ts +163 -0
  137. package/src/stats.ts +205 -0
  138. package/src/store/append.ts +50 -0
  139. package/src/store/bootstrap_restore_store.ts +71 -0
  140. package/src/store/capabilities.ts +86 -0
  141. package/src/store/full_mode_details_store.ts +71 -0
  142. package/src/store/index_store.ts +104 -0
  143. package/src/store/profile_touch_store.ts +1 -0
  144. package/src/store/rows.ts +144 -0
  145. package/src/store/schema_profile_store.ts +73 -0
  146. package/src/store/schema_publication.ts +6 -0
  147. package/src/store/segment_manifest_store.ts +129 -0
  148. package/src/store/segment_read_store.ts +22 -0
  149. package/src/store/stats_accounting_store.ts +83 -0
  150. package/src/store/touch_store.ts +98 -0
  151. package/src/store/wal_store.ts +21 -0
  152. package/src/stream_size_reconciler.ts +100 -0
  153. package/src/touch/canonical_change.ts +7 -0
  154. package/src/touch/live_keys.ts +158 -0
  155. package/src/touch/live_metrics.ts +841 -0
  156. package/src/touch/live_templates.ts +449 -0
  157. package/src/touch/manager.ts +1292 -0
  158. package/src/touch/process_batch.ts +576 -0
  159. package/src/touch/processor_worker.ts +85 -0
  160. package/src/touch/spec.ts +459 -0
  161. package/src/touch/touch_journal.ts +771 -0
  162. package/src/touch/touch_key_id.ts +20 -0
  163. package/src/touch/worker_pool.ts +191 -0
  164. package/src/touch/worker_protocol.ts +57 -0
  165. package/src/types/proper-lockfile.d.ts +1 -0
  166. package/src/uploader.ts +358 -0
  167. package/src/util/base32_crockford.ts +81 -0
  168. package/src/util/bloom256.ts +67 -0
  169. package/src/util/byte_lru.ts +73 -0
  170. package/src/util/cleanup.ts +22 -0
  171. package/src/util/crc32c.ts +29 -0
  172. package/src/util/ds_error.ts +15 -0
  173. package/src/util/duration.ts +17 -0
  174. package/src/util/endian.ts +53 -0
  175. package/src/util/json_pointer.ts +148 -0
  176. package/src/util/log.ts +25 -0
  177. package/src/util/lru.ts +53 -0
  178. package/src/util/retry.ts +35 -0
  179. package/src/util/siphash.ts +71 -0
  180. package/src/util/stream_paths.ts +50 -0
  181. package/src/util/time.ts +14 -0
  182. package/src/util/yield.ts +3 -0
  183. package/src/util/zstd.ts +24 -0
@@ -0,0 +1,35 @@
1
+ import type { ChangeEvent, ControlEvent } from "@durable-streams/state";
2
+
3
+ type Assert<T extends true> = T;
4
+ type AssertFalse<T extends false> = T;
5
+ type IsAssignable<From, To> = [From] extends [To] ? true : false;
6
+
7
+ type PrismaWalChangeEvent = {
8
+ type: "public.posts";
9
+ key: "42";
10
+ value: { id: number; title: string };
11
+ old_value: { id: number; title: string };
12
+ headers: {
13
+ operation: "update";
14
+ txid: string;
15
+ timestamp: string;
16
+ };
17
+ };
18
+
19
+ type _PrismaWalChangeEventMatchesStateProtocol = Assert<
20
+ IsAssignable<PrismaWalChangeEvent, ChangeEvent<{ id: number; title: string }>>
21
+ >;
22
+
23
+ type _StateProtocolUsesSnakeCaseBeforeImage = Assert<"old_value" extends keyof ChangeEvent<unknown> ? true : false>;
24
+ type _StateProtocolDoesNotUseCamelCaseBeforeImage = AssertFalse<"oldValue" extends keyof ChangeEvent<unknown> ? true : false>;
25
+
26
+ type PrismaWalControlEvent = {
27
+ headers: {
28
+ control: "reset";
29
+ offset: string;
30
+ };
31
+ };
32
+
33
+ type _PrismaWalControlEventMatchesStateProtocol = Assert<
34
+ IsAssignable<PrismaWalControlEvent, ControlEvent>
35
+ >;
@@ -0,0 +1,24 @@
1
+ import type { CanonicalChange } from "../../touch/canonical_change";
2
+
3
+ export function deriveStateProtocolChanges(record: unknown): CanonicalChange[] {
4
+ if (!record || typeof record !== "object" || Array.isArray(record)) return [];
5
+ const headers = (record as any).headers;
6
+ if (!headers || typeof headers !== "object" || Array.isArray(headers)) return [];
7
+
8
+ if (typeof (headers as any).control === "string") return [];
9
+
10
+ const opRaw = (headers as any).operation;
11
+ if (typeof opRaw !== "string") return [];
12
+ const op = opRaw;
13
+ if (op !== "insert" && op !== "update" && op !== "delete") return [];
14
+
15
+ const type = (record as any).type;
16
+ const key = (record as any).key;
17
+ if (typeof type !== "string" || type.trim() === "") return [];
18
+ if (typeof key !== "string" || key.trim() === "") return [];
19
+
20
+ const before = Object.prototype.hasOwnProperty.call(record, "old_value") ? (record as any).old_value : undefined;
21
+ const after = Object.prototype.hasOwnProperty.call(record, "value") ? (record as any).value : undefined;
22
+
23
+ return [{ entity: type, key, op, before, after }];
24
+ }
@@ -0,0 +1,115 @@
1
+ import { Result } from "better-result";
2
+ import { parseOffsetResult } from "../../offset";
3
+ import type { PreparedJsonRecord } from "../profile";
4
+ import { expectPlainObjectResult, rejectUnknownKeysResult } from "../profile";
5
+
6
+ const CHANGE_KEYS = ["type", "key", "value", "old_value", "headers"] as const;
7
+ const CHANGE_HEADER_KEYS = ["operation", "txid", "timestamp"] as const;
8
+ const CONTROL_KEYS = ["headers"] as const;
9
+ const CONTROL_HEADER_KEYS = ["control", "offset"] as const;
10
+
11
+ function isDateTimeString(value: string): boolean {
12
+ if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/.test(value)) {
13
+ return false;
14
+ }
15
+ return !Number.isNaN(Date.parse(value));
16
+ }
17
+
18
+ function nonEmptyStringFieldResult(
19
+ value: unknown,
20
+ path: string
21
+ ): Result<string, { message: string }> {
22
+ if (typeof value !== "string" || value.trim() === "") {
23
+ return Result.err({ message: `${path} must be a non-empty string` });
24
+ }
25
+ return Result.ok(value);
26
+ }
27
+
28
+ function validateChangeRecordResult(
29
+ record: Record<string, unknown>,
30
+ headers: Record<string, unknown>
31
+ ): Result<PreparedJsonRecord, { message: string }> {
32
+ const keyCheck = rejectUnknownKeysResult(record, CHANGE_KEYS, "state-protocol record");
33
+ if (Result.isError(keyCheck)) return keyCheck;
34
+
35
+ const headerKeyCheck = rejectUnknownKeysResult(headers, CHANGE_HEADER_KEYS, "state-protocol record.headers");
36
+ if (Result.isError(headerKeyCheck)) return headerKeyCheck;
37
+
38
+ const typeRes = nonEmptyStringFieldResult(record.type, "state-protocol record.type");
39
+ if (Result.isError(typeRes)) return typeRes;
40
+
41
+ const keyRes = nonEmptyStringFieldResult(record.key, "state-protocol record.key");
42
+ if (Result.isError(keyRes)) return keyRes;
43
+
44
+ const operation = headers.operation;
45
+ if (operation !== "insert" && operation !== "update" && operation !== "delete") {
46
+ return Result.err({ message: "state-protocol record.headers.operation must be insert, update, or delete" });
47
+ }
48
+
49
+ if ((operation === "insert" || operation === "update") && !Object.prototype.hasOwnProperty.call(record, "value")) {
50
+ return Result.err({ message: `state-protocol ${operation} records must include value` });
51
+ }
52
+
53
+ if (Object.prototype.hasOwnProperty.call(headers, "txid")) {
54
+ const txidRes = nonEmptyStringFieldResult(headers.txid, "state-protocol record.headers.txid");
55
+ if (Result.isError(txidRes)) return txidRes;
56
+ }
57
+
58
+ if (Object.prototype.hasOwnProperty.call(headers, "timestamp")) {
59
+ if (typeof headers.timestamp !== "string" || !isDateTimeString(headers.timestamp)) {
60
+ return Result.err({ message: "state-protocol record.headers.timestamp must be a valid RFC 3339 timestamp" });
61
+ }
62
+ }
63
+
64
+ return Result.ok({ value: record, routingKey: null });
65
+ }
66
+
67
+ function validateControlRecordResult(
68
+ record: Record<string, unknown>,
69
+ headers: Record<string, unknown>
70
+ ): Result<PreparedJsonRecord, { message: string }> {
71
+ const keyCheck = rejectUnknownKeysResult(record, CONTROL_KEYS, "state-protocol record");
72
+ if (Result.isError(keyCheck)) return keyCheck;
73
+
74
+ const headerKeyCheck = rejectUnknownKeysResult(headers, CONTROL_HEADER_KEYS, "state-protocol record.headers");
75
+ if (Result.isError(headerKeyCheck)) return headerKeyCheck;
76
+
77
+ const control = headers.control;
78
+ if (control !== "snapshot-start" && control !== "snapshot-end" && control !== "reset") {
79
+ return Result.err({ message: "state-protocol record.headers.control must be snapshot-start, snapshot-end, or reset" });
80
+ }
81
+
82
+ if (Object.prototype.hasOwnProperty.call(headers, "offset")) {
83
+ if (typeof headers.offset !== "string") {
84
+ return Result.err({ message: "state-protocol record.headers.offset must be a valid stream offset string" });
85
+ }
86
+ const offsetRes = parseOffsetResult(headers.offset);
87
+ if (Result.isError(offsetRes)) {
88
+ return Result.err({ message: "state-protocol record.headers.offset must be a valid stream offset string" });
89
+ }
90
+ }
91
+
92
+ return Result.ok({ value: record, routingKey: null });
93
+ }
94
+
95
+ export function validateStateProtocolRecordResult(value: unknown): Result<PreparedJsonRecord, { message: string }> {
96
+ const recordRes = expectPlainObjectResult(value, "state-protocol record");
97
+ if (Result.isError(recordRes)) {
98
+ return Result.err({ message: "state-protocol records must be JSON objects" });
99
+ }
100
+
101
+ const headersRes = expectPlainObjectResult(recordRes.value.headers, "state-protocol record.headers");
102
+ if (Result.isError(headersRes)) {
103
+ return Result.err({ message: "state-protocol record.headers must be an object" });
104
+ }
105
+
106
+ const hasControl = Object.prototype.hasOwnProperty.call(headersRes.value, "control");
107
+ const hasOperation = Object.prototype.hasOwnProperty.call(headersRes.value, "operation");
108
+
109
+ if (hasControl && hasOperation) {
110
+ return Result.err({ message: "state-protocol record.headers cannot mix control and operation" });
111
+ }
112
+ if (hasControl) return validateControlRecordResult(recordRes.value, headersRes.value);
113
+ if (hasOperation) return validateChangeRecordResult(recordRes.value, headersRes.value);
114
+ return Result.err({ message: "state-protocol record.headers must contain operation or control" });
115
+ }
@@ -0,0 +1,511 @@
1
+ import { Result } from "better-result";
2
+ import { encodeOffset } from "../../offset";
3
+ import { parseTouchCursor } from "../../touch/touch_journal";
4
+ import { touchKeyIdFromRoutingKeyResult } from "../../touch/touch_key_id";
5
+ import { tableKeyIdFor, templateKeyIdFor } from "../../touch/live_keys";
6
+ import type { TemplateDecl } from "../../touch/live_templates";
7
+ import type { TouchConfig } from "../../touch/spec";
8
+ import type { StreamTouchRouteArgs } from "../profile";
9
+ import { getStateProtocolTouchConfig } from "./validation";
10
+
11
+ const EXACT_FINE_WAIT_MAX_KEYS = 16;
12
+
13
+ async function countActiveTemplates(stream: string, db: StreamTouchRouteArgs["db"]): Promise<number> {
14
+ try {
15
+ return await db.countActiveLiveTemplates(stream);
16
+ } catch {
17
+ return 0;
18
+ }
19
+ }
20
+
21
+ function parseInactivityTtlResult(
22
+ raw: unknown,
23
+ defaultValue: number,
24
+ fieldPath: string
25
+ ): Result<number, { message: string }> {
26
+ if (raw === undefined) return Result.ok(defaultValue);
27
+ if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) {
28
+ return Result.ok(Math.floor(raw));
29
+ }
30
+ return Result.err({ message: `${fieldPath} must be a non-negative number (ms)` });
31
+ }
32
+
33
+ function parseTemplateDeclsResult(raw: unknown, fieldPath: string): Result<TemplateDecl[], { message: string }> {
34
+ if (!Array.isArray(raw) || raw.length === 0) {
35
+ return Result.err({ message: `${fieldPath} must be a non-empty array` });
36
+ }
37
+ if (raw.length > 256) return Result.err({ message: `${fieldPath} too large (max 256)` });
38
+
39
+ const templates: TemplateDecl[] = [];
40
+ for (const t of raw) {
41
+ const entity = typeof t?.entity === "string" ? t.entity.trim() : "";
42
+ const fieldsRaw = t?.fields;
43
+ if (entity === "" || !Array.isArray(fieldsRaw) || fieldsRaw.length === 0 || fieldsRaw.length > 3) {
44
+ return Result.err({ message: `${fieldPath} contains invalid template definitions` });
45
+ }
46
+
47
+ const fields: TemplateDecl["fields"] = [];
48
+ for (const f of fieldsRaw) {
49
+ const name = typeof f?.name === "string" ? f.name.trim() : "";
50
+ const encoding = f?.encoding;
51
+ if (name === "") {
52
+ return Result.err({ message: `${fieldPath} contains invalid template definitions` });
53
+ }
54
+ fields.push({ name, encoding });
55
+ }
56
+ if (fields.length !== fieldsRaw.length) {
57
+ return Result.err({ message: `${fieldPath} contains invalid template definitions` });
58
+ }
59
+ templates.push({ entity, fields });
60
+ }
61
+ return Result.ok(templates);
62
+ }
63
+
64
+ function parseWaitTimeoutMsQueryResult(raw: string | null, defaultValue: number, fieldPath: string): Result<number, { message: string }> {
65
+ if (raw == null || raw.trim() === "") return Result.ok(defaultValue);
66
+ const n = Number(raw);
67
+ if (!Number.isFinite(n)) return Result.err({ message: `${fieldPath} must be a number (ms)` });
68
+ return Result.ok(Math.max(0, Math.min(120_000, Math.floor(n))));
69
+ }
70
+
71
+ function normalizeExactFineWaitKeys(keys: string[]): string[] {
72
+ if (keys.length === 0 || keys.length > EXACT_FINE_WAIT_MAX_KEYS) return [];
73
+ const normalized = Array.from(new Set(keys.map((key) => key.trim().toLowerCase())));
74
+ if (!normalized.every((key) => /^[0-9a-f]{16}$/.test(key))) return [];
75
+ return normalized;
76
+ }
77
+
78
+ async function handleTemplatesActivateRoute(args: StreamTouchRouteArgs, touchCfg: TouchConfig): Promise<Response> {
79
+ const { req, stream, streamRow, touchManager, respond } = args;
80
+ if (req.method !== "POST") return respond.badRequest("unsupported method");
81
+
82
+ let body: any;
83
+ try {
84
+ body = await req.json();
85
+ } catch {
86
+ return respond.badRequest("activate body must be valid JSON");
87
+ }
88
+
89
+ const templatesRes = parseTemplateDeclsResult(body?.templates, "activate.templates");
90
+ if (Result.isError(templatesRes)) return respond.badRequest(templatesRes.error.message);
91
+
92
+ const inactivityTtlRes = parseInactivityTtlResult(
93
+ body?.inactivityTtlMs,
94
+ touchCfg.templates?.defaultInactivityTtlMs ?? 60 * 60 * 1000,
95
+ "activate.inactivityTtlMs"
96
+ );
97
+ if (Result.isError(inactivityTtlRes)) return respond.badRequest(inactivityTtlRes.error.message);
98
+
99
+ const limits = {
100
+ maxActiveTemplatesPerStream: touchCfg.templates?.maxActiveTemplatesPerStream ?? 2048,
101
+ maxActiveTemplatesPerEntity: touchCfg.templates?.maxActiveTemplatesPerEntity ?? 256,
102
+ };
103
+ const activeFromTouchOffset = touchManager.getOrCreateJournal(stream, touchCfg).getCursor();
104
+ const res = await touchManager.activateTemplates({
105
+ stream,
106
+ touchCfg,
107
+ baseStreamNextOffset: streamRow.next_offset,
108
+ activeFromTouchOffset,
109
+ templates: templatesRes.value,
110
+ inactivityTtlMs: inactivityTtlRes.value,
111
+ });
112
+
113
+ return respond.json(200, { activated: res.activated, denied: res.denied, limits });
114
+ }
115
+
116
+ async function buildMetaRoutePayload(args: StreamTouchRouteArgs, touchCfg: TouchConfig) {
117
+ const { stream, streamRow, db, touchManager } = args;
118
+ const meta = touchManager.getOrCreateJournal(stream, touchCfg).getMeta();
119
+ const runtime = touchManager.getTouchRuntimeSnapshot({ stream, touchCfg });
120
+ const touchState = await db.getStreamTouchState(stream);
121
+ return {
122
+ ...meta,
123
+ settled: meta.pendingKeys === 0 && runtime.lagSourceOffsets === 0,
124
+ coarseIntervalMs: touchCfg.coarseIntervalMs ?? 100,
125
+ touchCoalesceWindowMs: touchCfg.touchCoalesceWindowMs ?? 100,
126
+ activeTemplates: await countActiveTemplates(stream, db),
127
+ lagSourceOffsets: runtime.lagSourceOffsets,
128
+ touchMode: runtime.touchMode,
129
+ walScannedThrough: touchState ? encodeOffset(streamRow.epoch, touchState.processed_through) : null,
130
+ bucketMaxSourceOffsetSeq: meta.bucketMaxSourceOffsetSeq,
131
+ hotFineKeys: runtime.hotFineKeys,
132
+ hotTemplates: runtime.hotTemplates,
133
+ hotFineKeysActive: runtime.hotFineKeysActive,
134
+ hotFineKeysGrace: runtime.hotFineKeysGrace,
135
+ hotTemplatesActive: runtime.hotTemplatesActive,
136
+ hotTemplatesGrace: runtime.hotTemplatesGrace,
137
+ fineWaitersActive: runtime.fineWaitersActive,
138
+ coarseWaitersActive: runtime.coarseWaitersActive,
139
+ broadFineWaitersActive: runtime.broadFineWaitersActive,
140
+ hotKeyFilteringEnabled: runtime.hotKeyFilteringEnabled,
141
+ hotTemplateFilteringEnabled: runtime.hotTemplateFilteringEnabled,
142
+ scanRowsTotal: runtime.scanRowsTotal,
143
+ scanBatchesTotal: runtime.scanBatchesTotal,
144
+ scannedButEmitted0BatchesTotal: runtime.scannedButEmitted0BatchesTotal,
145
+ processedThroughDeltaTotal: runtime.processedThroughDeltaTotal,
146
+ touchesEmittedTotal: runtime.touchesEmittedTotal,
147
+ touchesTableTotal: runtime.touchesTableTotal,
148
+ touchesTemplateTotal: runtime.touchesTemplateTotal,
149
+ fineTouchesDroppedDueToBudgetTotal: runtime.fineTouchesDroppedDueToBudgetTotal,
150
+ fineTouchesSkippedColdTemplateTotal: runtime.fineTouchesSkippedColdTemplateTotal,
151
+ fineTouchesSkippedColdKeyTotal: runtime.fineTouchesSkippedColdKeyTotal,
152
+ fineTouchesSkippedTemplateBucketTotal: runtime.fineTouchesSkippedTemplateBucketTotal,
153
+ waitTouchedTotal: runtime.waitTouchedTotal,
154
+ waitTimeoutTotal: runtime.waitTimeoutTotal,
155
+ waitStaleTotal: runtime.waitStaleTotal,
156
+ journalFlushesTotal: runtime.journalFlushesTotal,
157
+ journalNotifyWakeupsTotal: runtime.journalNotifyWakeupsTotal,
158
+ journalNotifyWakeMsTotal: runtime.journalNotifyWakeMsTotal,
159
+ journalNotifyWakeMsMax: runtime.journalNotifyWakeMsMax,
160
+ journalTimeoutsFiredTotal: runtime.journalTimeoutsFiredTotal,
161
+ journalTimeoutSweepMsTotal: runtime.journalTimeoutSweepMsTotal,
162
+ };
163
+ }
164
+
165
+ async function handleMetaRoute(args: StreamTouchRouteArgs, touchCfg: TouchConfig): Promise<Response> {
166
+ const { req, respond } = args;
167
+ if (req.method !== "GET") return respond.badRequest("unsupported method");
168
+
169
+ const url = new URL(req.url);
170
+ const settleRaw = url.searchParams.get("settle");
171
+ if (settleRaw !== null && settleRaw !== "flush") {
172
+ return respond.badRequest("meta.settle must be 'flush' when provided");
173
+ }
174
+
175
+ const timeoutMsRes = parseWaitTimeoutMsQueryResult(url.searchParams.get("timeoutMs"), 30_000, "meta.timeoutMs");
176
+ if (Result.isError(timeoutMsRes)) return respond.badRequest(timeoutMsRes.error.message);
177
+
178
+ if (settleRaw !== "flush") {
179
+ return respond.json(200, await buildMetaRoutePayload(args, touchCfg));
180
+ }
181
+
182
+ const deadlineMs = Date.now() + timeoutMsRes.value;
183
+ for (;;) {
184
+ const payload = await buildMetaRoutePayload(args, touchCfg);
185
+ if (payload.settled || Date.now() >= deadlineMs) {
186
+ return respond.json(200, payload);
187
+ }
188
+ if (req.signal.aborted) return new Response(null, { status: 204 });
189
+ const remainingMs = Math.max(1, deadlineMs - Date.now());
190
+ await new Promise<void>((resolve) => {
191
+ const waitMs = Math.min(25, remainingMs);
192
+ const timer = setTimeout(() => {
193
+ req.signal.removeEventListener("abort", onAbort);
194
+ resolve();
195
+ }, waitMs);
196
+ const onAbort = () => {
197
+ clearTimeout(timer);
198
+ req.signal.removeEventListener("abort", onAbort);
199
+ resolve();
200
+ };
201
+ req.signal.addEventListener("abort", onAbort, { once: true });
202
+ });
203
+ }
204
+ }
205
+
206
+ async function handleWaitRoute(args: StreamTouchRouteArgs, touchCfg: TouchConfig): Promise<Response> {
207
+ const { req, stream, streamRow, touchManager, respond } = args;
208
+ if (req.method !== "POST") return respond.badRequest("unsupported method");
209
+
210
+ const waitStartMs = Date.now();
211
+ let body: any;
212
+ try {
213
+ body = await req.json();
214
+ } catch {
215
+ return respond.badRequest("wait body must be valid JSON");
216
+ }
217
+
218
+ const keysRaw = body?.keys;
219
+ if (keysRaw !== undefined && (!Array.isArray(keysRaw) || !keysRaw.every((k: any) => typeof k === "string" && k.trim() !== ""))) {
220
+ return respond.badRequest("wait.keys must be a non-empty string array when provided");
221
+ }
222
+ const keys = Array.isArray(keysRaw) ? Array.from(new Set(keysRaw.map((k: string) => k.trim()))) : [];
223
+ if (keys.length > 1024) return respond.badRequest("wait.keys too large (max 1024)");
224
+
225
+ const keyIdsRaw = body?.keyIds;
226
+ const keyIds =
227
+ Array.isArray(keyIdsRaw) && keyIdsRaw.length > 0
228
+ ? Array.from(
229
+ new Set(
230
+ keyIdsRaw
231
+ .map((x: any) => Number(x))
232
+ .filter((n: number) => Number.isFinite(n) && Number.isInteger(n) && n >= 0 && n <= 0xffffffff)
233
+ )
234
+ ).map((n) => n >>> 0)
235
+ : [];
236
+ if (Array.isArray(keyIdsRaw) && keyIds.length !== keyIdsRaw.length) {
237
+ return respond.badRequest("wait.keyIds must be a non-empty uint32 array when provided");
238
+ }
239
+ if (keys.length === 0 && keyIds.length === 0) return respond.badRequest("wait requires keys or keyIds");
240
+ if (keyIds.length > 1024) return respond.badRequest("wait.keyIds too large (max 1024)");
241
+
242
+ const exactRaw = body?.exact;
243
+ if (exactRaw !== undefined && typeof exactRaw !== "boolean") return respond.badRequest("wait.exact must be a boolean when provided");
244
+ const exactRequested = exactRaw === true;
245
+
246
+ const cursorRaw = body?.cursor;
247
+ if (typeof cursorRaw !== "string" || cursorRaw.trim() === "") return respond.badRequest("wait.cursor must be a non-empty string");
248
+ const cursor = cursorRaw.trim();
249
+
250
+ const timeoutMsRaw = body?.timeoutMs;
251
+ const timeoutMs =
252
+ timeoutMsRaw === undefined
253
+ ? 30_000
254
+ : typeof timeoutMsRaw === "number" && Number.isFinite(timeoutMsRaw)
255
+ ? Math.max(0, Math.min(120_000, timeoutMsRaw))
256
+ : null;
257
+ if (timeoutMs == null) return respond.badRequest("wait.timeoutMs must be a number (ms)");
258
+
259
+ const templateIdsUsedRaw = body?.templateIdsUsed;
260
+ if (Array.isArray(templateIdsUsedRaw) && !templateIdsUsedRaw.every((x: any) => typeof x === "string" && x.trim() !== "")) {
261
+ return respond.badRequest("wait.templateIdsUsed must be a string array");
262
+ }
263
+ const templateIdsUsed =
264
+ Array.isArray(templateIdsUsedRaw) && templateIdsUsedRaw.length > 0
265
+ ? Array.from(new Set(templateIdsUsedRaw.map((s: any) => (typeof s === "string" ? s.trim() : "")).filter((s: string) => s !== "")))
266
+ : [];
267
+
268
+ const interestModeRaw = body?.interestMode;
269
+ if (interestModeRaw !== undefined && interestModeRaw !== "fine" && interestModeRaw !== "coarse") {
270
+ return respond.badRequest("wait.interestMode must be 'fine' or 'coarse'");
271
+ }
272
+ const interestMode: "fine" | "coarse" = interestModeRaw === "coarse" ? "coarse" : "fine";
273
+
274
+ if (interestMode === "fine" && templateIdsUsed.length > 0) {
275
+ await touchManager.heartbeatTemplates({ stream, touchCfg, templateIdsUsed });
276
+ }
277
+
278
+ const declareTemplatesRaw = body?.declareTemplates;
279
+ if (Array.isArray(declareTemplatesRaw) && declareTemplatesRaw.length > 0) {
280
+ const templatesRes = parseTemplateDeclsResult(declareTemplatesRaw, "wait.declareTemplates");
281
+ if (Result.isError(templatesRes)) return respond.badRequest(templatesRes.error.message);
282
+
283
+ const inactivityTtlRes = parseInactivityTtlResult(
284
+ body?.inactivityTtlMs,
285
+ touchCfg.templates?.defaultInactivityTtlMs ?? 60 * 60 * 1000,
286
+ "wait.inactivityTtlMs"
287
+ );
288
+ if (Result.isError(inactivityTtlRes)) return respond.badRequest(inactivityTtlRes.error.message);
289
+
290
+ const activeFromTouchOffset = touchManager.getOrCreateJournal(stream, touchCfg).getCursor();
291
+ await touchManager.activateTemplates({
292
+ stream,
293
+ touchCfg,
294
+ baseStreamNextOffset: streamRow.next_offset,
295
+ activeFromTouchOffset,
296
+ templates: templatesRes.value,
297
+ inactivityTtlMs: inactivityTtlRes.value,
298
+ });
299
+ }
300
+
301
+ const journal = touchManager.getOrCreateJournal(stream, touchCfg);
302
+ const runtime = touchManager.getTouchRuntimeSnapshot({ stream, touchCfg });
303
+
304
+ let rawFineKeyIds = keyIds;
305
+ if (keyIds.length === 0) {
306
+ const parsedKeyIds: number[] = [];
307
+ for (const key of keys) {
308
+ const keyIdRes = touchKeyIdFromRoutingKeyResult(key);
309
+ if (Result.isError(keyIdRes)) return respond.internalError();
310
+ parsedKeyIds.push(keyIdRes.value);
311
+ }
312
+ rawFineKeyIds = parsedKeyIds;
313
+ }
314
+
315
+ const templateWaitKeyIds = templateIdsUsed.length > 0 ? Array.from(new Set(templateIdsUsed.map((templateId) => templateKeyIdFor(templateId) >>> 0))) : [];
316
+ let waitKeyIds = rawFineKeyIds;
317
+ let effectiveWaitKind: "fineKey" | "templateKey" | "tableKey" = "fineKey";
318
+
319
+ if (interestMode === "coarse") {
320
+ effectiveWaitKind = "tableKey";
321
+ } else if (runtime.touchMode === "restricted" && templateIdsUsed.length > 0) {
322
+ effectiveWaitKind = "templateKey";
323
+ } else if (runtime.touchMode === "coarseOnly" && templateIdsUsed.length > 0) {
324
+ effectiveWaitKind = "tableKey";
325
+ }
326
+
327
+ if (effectiveWaitKind === "templateKey") {
328
+ waitKeyIds = templateWaitKeyIds;
329
+ } else if (effectiveWaitKind === "tableKey" && templateIdsUsed.length > 0) {
330
+ const entities = await touchManager.resolveTemplateEntitiesForWait({ stream, templateIdsUsed });
331
+ waitKeyIds = Array.from(new Set(entities.map((entity) => tableKeyIdFor(entity) >>> 0)));
332
+ }
333
+
334
+ if (exactRequested && (interestMode !== "fine" || effectiveWaitKind !== "fineKey")) {
335
+ return respond.badRequest("wait.exact requires fine interest while runtime is in fine-key mode");
336
+ }
337
+
338
+ const exactFineRoutingKeys =
339
+ exactRequested && interestMode === "fine" && effectiveWaitKind === "fineKey" ? normalizeExactFineWaitKeys(keys) : [];
340
+ if (exactRequested && exactFineRoutingKeys.length === 0) {
341
+ return respond.badRequest("wait.exact requires 1 to 16 literal 64-bit routing keys");
342
+ }
343
+ const useExactFineKeyMatch = exactFineRoutingKeys.length > 0;
344
+ const exactFallbackKeyIds =
345
+ interestMode === "fine" && effectiveWaitKind === "fineKey" && templateWaitKeyIds.length > 0 ? templateWaitKeyIds : [];
346
+
347
+ if (interestMode === "fine" && effectiveWaitKind === "fineKey" && templateWaitKeyIds.length > 0 && !useExactFineKeyMatch) {
348
+ const merged = new Set<number>();
349
+ for (const keyId of waitKeyIds) merged.add(keyId >>> 0);
350
+ for (const keyId of templateWaitKeyIds) merged.add(keyId >>> 0);
351
+ waitKeyIds = Array.from(merged);
352
+ }
353
+
354
+ if (waitKeyIds.length === 0) {
355
+ waitKeyIds = rawFineKeyIds;
356
+ effectiveWaitKind = "fineKey";
357
+ }
358
+
359
+ const hotInterestKeyIds = interestMode === "fine" ? rawFineKeyIds : waitKeyIds;
360
+ const releaseHotInterest = touchManager.beginHotWaitInterest({
361
+ stream,
362
+ touchCfg,
363
+ keyIds: hotInterestKeyIds,
364
+ templateIdsUsed,
365
+ interestMode,
366
+ });
367
+
368
+ try {
369
+ let sinceGen: number;
370
+ if (cursor === "now") {
371
+ sinceGen = journal.getGeneration();
372
+ } else {
373
+ const parsed = parseTouchCursor(cursor);
374
+ if (!parsed) return respond.badRequest("wait.cursor must be in the form <epochHex>:<generation> or 'now'");
375
+ if (parsed.epoch !== journal.getEpoch()) {
376
+ const latencyMs = Date.now() - waitStartMs;
377
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "stale", latencyMs });
378
+ return respond.json(200, {
379
+ stale: true,
380
+ cursor: journal.getCursor(),
381
+ epoch: journal.getEpoch(),
382
+ generation: journal.getGeneration(),
383
+ effectiveWaitKind,
384
+ bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
385
+ flushAtMs: journal.getLastFlushAtMs(),
386
+ bucketStartMs: journal.getLastBucketStartMs(),
387
+ error: { code: "stale", message: "cursor epoch mismatch; rerun/re-subscribe and start from cursor" },
388
+ });
389
+ }
390
+ sinceGen = parsed.generation;
391
+ }
392
+
393
+ const nowGen = journal.getGeneration();
394
+ if (sinceGen > nowGen) sinceGen = nowGen;
395
+
396
+ if (useExactFineKeyMatch) {
397
+ const exactTouched = journal.exactTouchedSinceAny(exactFineRoutingKeys, sinceGen);
398
+ if (exactTouched === true) {
399
+ const latencyMs = Date.now() - waitStartMs;
400
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs });
401
+ return respond.json(200, {
402
+ touched: true,
403
+ cursor: journal.getCursor(),
404
+ effectiveWaitKind,
405
+ bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
406
+ flushAtMs: journal.getLastFlushAtMs(),
407
+ bucketStartMs: journal.getLastBucketStartMs(),
408
+ });
409
+ }
410
+ const fallbackTouched =
411
+ exactFallbackKeyIds.length > 0 ? journal.maybeTouchedSinceAny(exactFallbackKeyIds, sinceGen) : false;
412
+ if (fallbackTouched) {
413
+ const latencyMs = Date.now() - waitStartMs;
414
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs });
415
+ return respond.json(200, {
416
+ touched: true,
417
+ cursor: journal.getCursor(),
418
+ effectiveWaitKind,
419
+ bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
420
+ flushAtMs: journal.getLastFlushAtMs(),
421
+ bucketStartMs: journal.getLastBucketStartMs(),
422
+ });
423
+ }
424
+ if (exactTouched === false) {
425
+ // Exact recent history covers this cursor range and saw none of the watched keys.
426
+ } else if (journal.maybeTouchedSinceAny(waitKeyIds, sinceGen)) {
427
+ const latencyMs = Date.now() - waitStartMs;
428
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs });
429
+ return respond.json(200, {
430
+ touched: true,
431
+ cursor: journal.getCursor(),
432
+ effectiveWaitKind,
433
+ bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
434
+ flushAtMs: journal.getLastFlushAtMs(),
435
+ bucketStartMs: journal.getLastBucketStartMs(),
436
+ });
437
+ }
438
+ } else if (journal.maybeTouchedSinceAny(waitKeyIds, sinceGen)) {
439
+ const latencyMs = Date.now() - waitStartMs;
440
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs });
441
+ return respond.json(200, {
442
+ touched: true,
443
+ cursor: journal.getCursor(),
444
+ effectiveWaitKind,
445
+ bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
446
+ flushAtMs: journal.getLastFlushAtMs(),
447
+ bucketStartMs: journal.getLastBucketStartMs(),
448
+ });
449
+ }
450
+
451
+ const deadline = Date.now() + timeoutMs;
452
+ const remaining = deadline - Date.now();
453
+ if (remaining <= 0) {
454
+ const latencyMs = Date.now() - waitStartMs;
455
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "timeout", latencyMs });
456
+ return respond.json(200, {
457
+ touched: false,
458
+ cursor: journal.getCursor(),
459
+ effectiveWaitKind,
460
+ bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
461
+ flushAtMs: journal.getLastFlushAtMs(),
462
+ bucketStartMs: journal.getLastBucketStartMs(),
463
+ });
464
+ }
465
+
466
+ const afterGen = journal.getGeneration();
467
+ const hit = await journal.waitForAny({
468
+ keys: useExactFineKeyMatch ? exactFallbackKeyIds : waitKeyIds,
469
+ exactKeys: useExactFineKeyMatch ? exactFineRoutingKeys : null,
470
+ afterGeneration: afterGen,
471
+ timeoutMs: remaining,
472
+ signal: req.signal,
473
+ });
474
+ if (req.signal.aborted) return new Response(null, { status: 204 });
475
+
476
+ if (hit == null) {
477
+ const latencyMs = Date.now() - waitStartMs;
478
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "timeout", latencyMs });
479
+ return respond.json(200, {
480
+ touched: false,
481
+ cursor: journal.getCursor(),
482
+ effectiveWaitKind,
483
+ bucketMaxSourceOffsetSeq: journal.getLastFlushedSourceOffsetSeq().toString(),
484
+ flushAtMs: journal.getLastFlushAtMs(),
485
+ bucketStartMs: journal.getLastBucketStartMs(),
486
+ });
487
+ }
488
+
489
+ const latencyMs = Date.now() - waitStartMs;
490
+ touchManager.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs });
491
+ return respond.json(200, {
492
+ touched: true,
493
+ cursor: journal.getCursor(),
494
+ effectiveWaitKind,
495
+ bucketMaxSourceOffsetSeq: hit.bucketMaxSourceOffsetSeq.toString(),
496
+ flushAtMs: hit.flushAtMs,
497
+ bucketStartMs: hit.bucketStartMs,
498
+ });
499
+ } finally {
500
+ releaseHotInterest();
501
+ }
502
+ }
503
+
504
+ export async function handleStateProtocolTouchRoute(args: StreamTouchRouteArgs): Promise<Response> {
505
+ const { route, profile, respond } = args;
506
+ const touchCfg = getStateProtocolTouchConfig(profile);
507
+ if (!touchCfg) return respond.notFound("touch not enabled");
508
+ if (route.kind === "templates_activate") return handleTemplatesActivateRoute(args, touchCfg);
509
+ if (route.kind === "meta") return handleMetaRoute(args, touchCfg);
510
+ return handleWaitRoute(args, touchCfg);
511
+ }