@tungthedev/streams-server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CODE_OF_CONDUCT.md +45 -0
  2. package/CONTRIBUTING.md +76 -0
  3. package/LICENSE +201 -0
  4. package/README.md +58 -0
  5. package/SECURITY.md +42 -0
  6. package/bin/prisma-streams-server +2 -0
  7. package/package.json +46 -0
  8. package/src/app.ts +583 -0
  9. package/src/app_core.ts +3144 -0
  10. package/src/app_local.ts +206 -0
  11. package/src/auth.ts +124 -0
  12. package/src/auto_tune.ts +69 -0
  13. package/src/backpressure.ts +66 -0
  14. package/src/bootstrap.ts +613 -0
  15. package/src/compute/demo_entry.ts +415 -0
  16. package/src/compute/demo_site.ts +1242 -0
  17. package/src/compute/entry.ts +19 -0
  18. package/src/compute/package_entry.ts +4 -0
  19. package/src/compute/virtual-modules.d.ts +15 -0
  20. package/src/compute/worker_module_url.ts +9 -0
  21. package/src/concurrency_gate.ts +108 -0
  22. package/src/config.ts +402 -0
  23. package/src/db/bootstrap_store.ts +9 -0
  24. package/src/db/db.ts +2424 -0
  25. package/src/db/schema.ts +925 -0
  26. package/src/db/sqlite_manifest_snapshot.ts +81 -0
  27. package/src/db/sqlite_touch_store.ts +491 -0
  28. package/src/db/sqlite_wal_store.ts +472 -0
  29. package/src/details/full_mode_details.ts +568 -0
  30. package/src/expiry_sweeper.ts +47 -0
  31. package/src/foreground_activity.ts +55 -0
  32. package/src/hist.ts +169 -0
  33. package/src/index/binary_fuse.ts +379 -0
  34. package/src/index/indexer.ts +947 -0
  35. package/src/index/lexicon_file_cache.ts +261 -0
  36. package/src/index/lexicon_format.ts +93 -0
  37. package/src/index/lexicon_indexer.ts +863 -0
  38. package/src/index/run_cache.ts +84 -0
  39. package/src/index/run_format.ts +213 -0
  40. package/src/index/schedule.ts +28 -0
  41. package/src/index/secondary_indexer.ts +901 -0
  42. package/src/index/secondary_schema.ts +105 -0
  43. package/src/ingest.ts +309 -0
  44. package/src/lens/lens.ts +501 -0
  45. package/src/manifest.ts +249 -0
  46. package/src/memory.ts +334 -0
  47. package/src/metrics.ts +147 -0
  48. package/src/metrics_emitter.ts +83 -0
  49. package/src/notifier.ts +180 -0
  50. package/src/objectstore/accounting.ts +151 -0
  51. package/src/objectstore/interface.ts +13 -0
  52. package/src/objectstore/mock_r2.ts +269 -0
  53. package/src/objectstore/null.ts +32 -0
  54. package/src/objectstore/r2.ts +318 -0
  55. package/src/observe/pairing.ts +61 -0
  56. package/src/observe/request.ts +772 -0
  57. package/src/offset.ts +70 -0
  58. package/src/postgres/bootstrap.ts +269 -0
  59. package/src/postgres/companions.ts +197 -0
  60. package/src/postgres/control_restore.ts +109 -0
  61. package/src/postgres/details.ts +189 -0
  62. package/src/postgres/lexicon_index.ts +260 -0
  63. package/src/postgres/routing_index.ts +189 -0
  64. package/src/postgres/rows.ts +132 -0
  65. package/src/postgres/schema.ts +355 -0
  66. package/src/postgres/secondary_index.ts +238 -0
  67. package/src/postgres/segments.ts +900 -0
  68. package/src/postgres/stats.ts +103 -0
  69. package/src/postgres/store.ts +947 -0
  70. package/src/postgres/touch.ts +591 -0
  71. package/src/postgres/types.ts +32 -0
  72. package/src/profiles/evlog/schema.ts +234 -0
  73. package/src/profiles/evlog.ts +473 -0
  74. package/src/profiles/generic.ts +51 -0
  75. package/src/profiles/index.ts +237 -0
  76. package/src/profiles/metrics/block_format.ts +109 -0
  77. package/src/profiles/metrics/normalize.ts +366 -0
  78. package/src/profiles/metrics/schema.ts +319 -0
  79. package/src/profiles/metrics.ts +83 -0
  80. package/src/profiles/otelTraces/normalize.ts +955 -0
  81. package/src/profiles/otelTraces/otlp.ts +1002 -0
  82. package/src/profiles/otelTraces/schema.ts +408 -0
  83. package/src/profiles/otelTraces.ts +390 -0
  84. package/src/profiles/profile.ts +284 -0
  85. package/src/profiles/stateProtocol/change_event_conformance.typecheck.ts +35 -0
  86. package/src/profiles/stateProtocol/changes.ts +24 -0
  87. package/src/profiles/stateProtocol/ingest.ts +115 -0
  88. package/src/profiles/stateProtocol/routes.ts +511 -0
  89. package/src/profiles/stateProtocol/types.ts +6 -0
  90. package/src/profiles/stateProtocol/validation.ts +51 -0
  91. package/src/profiles/stateProtocol.ts +107 -0
  92. package/src/read_filter.ts +468 -0
  93. package/src/reader.ts +2986 -0
  94. package/src/runtime/hash.ts +156 -0
  95. package/src/runtime/hash_vendor/LICENSE.hash-wasm +38 -0
  96. package/src/runtime/hash_vendor/NOTICE.md +8 -0
  97. package/src/runtime/hash_vendor/xxhash3.umd.min.cjs +7 -0
  98. package/src/runtime/hash_vendor/xxhash32.umd.min.cjs +7 -0
  99. package/src/runtime/hash_vendor/xxhash64.umd.min.cjs +7 -0
  100. package/src/runtime/host_runtime.ts +5 -0
  101. package/src/runtime_memory.ts +200 -0
  102. package/src/runtime_memory_sampler.ts +237 -0
  103. package/src/schema/lens_schema.ts +290 -0
  104. package/src/schema/proof.ts +547 -0
  105. package/src/schema/read_json.ts +51 -0
  106. package/src/schema/registry.ts +966 -0
  107. package/src/search/agg_format.ts +638 -0
  108. package/src/search/aggregate.ts +409 -0
  109. package/src/search/binary/codec.ts +162 -0
  110. package/src/search/binary/docset.ts +67 -0
  111. package/src/search/binary/restart_strings.ts +181 -0
  112. package/src/search/binary/varint.ts +34 -0
  113. package/src/search/bitset.ts +19 -0
  114. package/src/search/col_format.ts +382 -0
  115. package/src/search/col_runtime.ts +59 -0
  116. package/src/search/column_encoding.ts +43 -0
  117. package/src/search/companion_file_cache.ts +319 -0
  118. package/src/search/companion_format.ts +327 -0
  119. package/src/search/companion_manager.ts +1305 -0
  120. package/src/search/companion_plan.ts +229 -0
  121. package/src/search/exact_format.ts +281 -0
  122. package/src/search/exact_runtime.ts +55 -0
  123. package/src/search/fts_format.ts +423 -0
  124. package/src/search/fts_runtime.ts +333 -0
  125. package/src/search/query.ts +875 -0
  126. package/src/search/schema.ts +245 -0
  127. package/src/segment/cache.ts +270 -0
  128. package/src/segment/cached_segment.ts +89 -0
  129. package/src/segment/format.ts +403 -0
  130. package/src/segment/segmenter.ts +412 -0
  131. package/src/segment/segmenter_worker.ts +72 -0
  132. package/src/segment/segmenter_workers.ts +130 -0
  133. package/src/server.ts +264 -0
  134. package/src/server_auto_tune.ts +158 -0
  135. package/src/sqlite/adapter.ts +335 -0
  136. package/src/sqlite/runtime_stats.ts +163 -0
  137. package/src/stats.ts +205 -0
  138. package/src/store/append.ts +50 -0
  139. package/src/store/bootstrap_restore_store.ts +71 -0
  140. package/src/store/capabilities.ts +86 -0
  141. package/src/store/full_mode_details_store.ts +71 -0
  142. package/src/store/index_store.ts +104 -0
  143. package/src/store/profile_touch_store.ts +1 -0
  144. package/src/store/rows.ts +144 -0
  145. package/src/store/schema_profile_store.ts +73 -0
  146. package/src/store/schema_publication.ts +6 -0
  147. package/src/store/segment_manifest_store.ts +129 -0
  148. package/src/store/segment_read_store.ts +22 -0
  149. package/src/store/stats_accounting_store.ts +83 -0
  150. package/src/store/touch_store.ts +98 -0
  151. package/src/store/wal_store.ts +21 -0
  152. package/src/stream_size_reconciler.ts +100 -0
  153. package/src/touch/canonical_change.ts +7 -0
  154. package/src/touch/live_keys.ts +158 -0
  155. package/src/touch/live_metrics.ts +841 -0
  156. package/src/touch/live_templates.ts +449 -0
  157. package/src/touch/manager.ts +1292 -0
  158. package/src/touch/process_batch.ts +576 -0
  159. package/src/touch/processor_worker.ts +85 -0
  160. package/src/touch/spec.ts +459 -0
  161. package/src/touch/touch_journal.ts +771 -0
  162. package/src/touch/touch_key_id.ts +20 -0
  163. package/src/touch/worker_pool.ts +191 -0
  164. package/src/touch/worker_protocol.ts +57 -0
  165. package/src/types/proper-lockfile.d.ts +1 -0
  166. package/src/uploader.ts +358 -0
  167. package/src/util/base32_crockford.ts +81 -0
  168. package/src/util/bloom256.ts +67 -0
  169. package/src/util/byte_lru.ts +73 -0
  170. package/src/util/cleanup.ts +22 -0
  171. package/src/util/crc32c.ts +29 -0
  172. package/src/util/ds_error.ts +15 -0
  173. package/src/util/duration.ts +17 -0
  174. package/src/util/endian.ts +53 -0
  175. package/src/util/json_pointer.ts +148 -0
  176. package/src/util/log.ts +25 -0
  177. package/src/util/lru.ts +53 -0
  178. package/src/util/retry.ts +35 -0
  179. package/src/util/siphash.ts +71 -0
  180. package/src/util/stream_paths.ts +50 -0
  181. package/src/util/time.ts +14 -0
  182. package/src/util/yield.ts +3 -0
  183. package/src/util/zstd.ts +24 -0
@@ -0,0 +1,390 @@
1
+ import { Result } from "better-result";
2
+ import type {
3
+ CachedStreamProfile,
4
+ StreamProfileDefinition,
5
+ StreamProfilePersistResult,
6
+ StreamProfileReadResult,
7
+ StreamProfileSpec,
8
+ UnifiedTimelineItem,
9
+ } from "./profile";
10
+ import {
11
+ cloneStreamProfileSpec,
12
+ expectPlainObjectResult,
13
+ normalizeProfileContentType,
14
+ parseStoredProfileJsonResult,
15
+ rejectUnknownKeysResult,
16
+ isPlainObject,
17
+ } from "./profile";
18
+ import { buildOtelTracesDefaultRegistry } from "./otelTraces/schema";
19
+ import {
20
+ DEFAULT_ATTRIBUTE_LIMITS,
21
+ DEFAULT_OTLP_LIMITS,
22
+ DEFAULT_STORE_CONFIG,
23
+ normalizeOtelTraceRecordResult,
24
+ type DbStatementMode,
25
+ type OtelTraceAttributeLimits,
26
+ type OtelTraceOtlpLimits,
27
+ type OtelTraceStoreConfig,
28
+ type OtelTracesStreamProfile,
29
+ type UrlMode,
30
+ } from "./otelTraces/normalize";
31
+ import { decodeOtlpTraceExportRequestResult } from "./otelTraces/otlp";
32
+
33
+ export type { OtelTracesStreamProfile };
34
+
35
+ function cloneOtelTracesProfile(profile: OtelTracesStreamProfile): OtelTracesStreamProfile {
36
+ return cloneStreamProfileSpec(profile) as OtelTracesStreamProfile;
37
+ }
38
+
39
+ function cloneOtelTracesCache(cache: CachedStreamProfile | null): CachedStreamProfile | null {
40
+ if (!cache || cache.profile.kind !== "otel-traces") return null;
41
+ return {
42
+ profile: cloneOtelTracesProfile(cache.profile as OtelTracesStreamProfile),
43
+ updatedAtMs: cache.updatedAtMs,
44
+ };
45
+ }
46
+
47
+ function isOtelTracesProfile(profile: StreamProfileSpec | null | undefined): profile is OtelTracesStreamProfile {
48
+ return !!profile && profile.kind === "otel-traces";
49
+ }
50
+
51
+ function parseStringArrayResult(raw: unknown, path: string, maxItems: number): Result<string[] | undefined, { message: string }> {
52
+ if (raw === undefined) return Result.ok(undefined);
53
+ if (!Array.isArray(raw)) return Result.err({ message: `${path} must be an array of strings` });
54
+ if (raw.length > maxItems) return Result.err({ message: `${path} too large (max ${maxItems})` });
55
+ const out: string[] = [];
56
+ const seen = new Set<string>();
57
+ for (const item of raw) {
58
+ if (typeof item !== "string") return Result.err({ message: `${path} must be an array of strings` });
59
+ const value = item.trim();
60
+ if (value === "") return Result.err({ message: `${path} must not contain empty strings` });
61
+ const key = value.toLowerCase();
62
+ if (seen.has(key)) continue;
63
+ seen.add(key);
64
+ out.push(path.endsWith("redactKeys") ? key : value);
65
+ }
66
+ return Result.ok(out);
67
+ }
68
+
69
+ function parsePositiveIntResult(raw: unknown, path: string, fallback: number): Result<number, { message: string }> {
70
+ if (raw === undefined) return Result.ok(fallback);
71
+ if (typeof raw !== "number" || !Number.isFinite(raw) || !Number.isInteger(raw) || raw <= 0) {
72
+ return Result.err({ message: `${path} must be a positive integer` });
73
+ }
74
+ return Result.ok(raw);
75
+ }
76
+
77
+ function parseAttributeLimitsResult(raw: unknown, path: string): Result<Partial<OtelTraceAttributeLimits> | undefined, { message: string }> {
78
+ if (raw === undefined) return Result.ok(undefined);
79
+ const objRes = expectPlainObjectResult(raw, path);
80
+ if (Result.isError(objRes)) return objRes;
81
+ const keyCheck = rejectUnknownKeysResult(
82
+ objRes.value,
83
+ ["maxAttributeValueBytes", "maxAttributesPerSpan", "maxEventsPerSpan", "maxLinksPerSpan", "maxStatementBytes"],
84
+ path
85
+ );
86
+ if (Result.isError(keyCheck)) return keyCheck;
87
+ const out: Partial<OtelTraceAttributeLimits> = {};
88
+ for (const key of Object.keys(DEFAULT_ATTRIBUTE_LIMITS) as Array<keyof OtelTraceAttributeLimits>) {
89
+ const valueRes = parsePositiveIntResult(objRes.value[key], `${path}.${key}`, DEFAULT_ATTRIBUTE_LIMITS[key]);
90
+ if (Result.isError(valueRes)) return valueRes;
91
+ if (objRes.value[key] !== undefined) out[key] = valueRes.value;
92
+ }
93
+ return Result.ok(Object.keys(out).length > 0 ? out : undefined);
94
+ }
95
+
96
+ function parseOtlpLimitsResult(raw: unknown, path: string): Result<Partial<OtelTraceOtlpLimits> | undefined, { message: string }> {
97
+ if (raw === undefined) return Result.ok(undefined);
98
+ const objRes = expectPlainObjectResult(raw, path);
99
+ if (Result.isError(objRes)) return objRes;
100
+ const keyCheck = rejectUnknownKeysResult(
101
+ objRes.value,
102
+ [
103
+ "maxCompressedBytes",
104
+ "maxDecodedBytes",
105
+ "maxResourceSpansPerRequest",
106
+ "maxScopeSpansPerRequest",
107
+ "maxSpansPerRequest",
108
+ "maxAnyValueDepth",
109
+ "maxArrayValuesPerAnyValue",
110
+ "maxKvListValuesPerAnyValue",
111
+ ],
112
+ path
113
+ );
114
+ if (Result.isError(keyCheck)) return keyCheck;
115
+ const out: Partial<OtelTraceOtlpLimits> = {};
116
+ for (const key of Object.keys(DEFAULT_OTLP_LIMITS) as Array<keyof OtelTraceOtlpLimits>) {
117
+ const valueRes = parsePositiveIntResult(objRes.value[key], `${path}.${key}`, DEFAULT_OTLP_LIMITS[key]);
118
+ if (Result.isError(valueRes)) return valueRes;
119
+ if (objRes.value[key] !== undefined) out[key] = valueRes.value;
120
+ }
121
+ return Result.ok(Object.keys(out).length > 0 ? out : undefined);
122
+ }
123
+
124
+ function parseStoreResult(raw: unknown, path: string): Result<Partial<OtelTraceStoreConfig> | undefined, { message: string }> {
125
+ if (raw === undefined) return Result.ok(undefined);
126
+ const objRes = expectPlainObjectResult(raw, path);
127
+ if (Result.isError(objRes)) return objRes;
128
+ const keyCheck = rejectUnknownKeysResult(objRes.value, ["rawResourceAttributes", "rawSpanAttributes", "rawEvents", "rawLinks"], path);
129
+ if (Result.isError(keyCheck)) return keyCheck;
130
+ const out: Partial<OtelTraceStoreConfig> = {};
131
+ for (const key of Object.keys(DEFAULT_STORE_CONFIG) as Array<keyof OtelTraceStoreConfig>) {
132
+ const value = objRes.value[key];
133
+ if (value === undefined) continue;
134
+ if (typeof value !== "boolean") return Result.err({ message: `${path}.${key} must be boolean` });
135
+ out[key] = value;
136
+ }
137
+ return Result.ok(Object.keys(out).length > 0 ? out : undefined);
138
+ }
139
+
140
+ function parseDbStatementModeResult(raw: unknown, path: string): Result<DbStatementMode | undefined, { message: string }> {
141
+ if (raw === undefined) return Result.ok(undefined);
142
+ if (raw === "drop" || raw === "raw") return Result.ok(raw);
143
+ return Result.err({ message: `${path} must be drop or raw` });
144
+ }
145
+
146
+ function parseUrlModeResult(raw: unknown, path: string): Result<UrlMode | undefined, { message: string }> {
147
+ if (raw === undefined) return Result.ok(undefined);
148
+ if (raw === "drop_query" || raw === "raw") return Result.ok(raw);
149
+ return Result.err({ message: `${path} must be drop_query or raw` });
150
+ }
151
+
152
+ function parseStreamNameResult(raw: unknown, path: string): Result<string | undefined, { message: string }> {
153
+ if (raw === undefined) return Result.ok(undefined);
154
+ if (typeof raw !== "string") return Result.err({ message: `${path} must be a string` });
155
+ const value = raw.trim();
156
+ if (value === "") return Result.err({ message: `${path} must not be empty` });
157
+ return Result.ok(value);
158
+ }
159
+
160
+ function parseOtelTracesObservabilityResult(raw: unknown, path: string): Result<OtelTracesStreamProfile["observability"] | undefined, { message: string }> {
161
+ if (raw === undefined) return Result.ok(undefined);
162
+ const objRes = expectPlainObjectResult(raw, path);
163
+ if (Result.isError(objRes)) return objRes;
164
+ const keyCheck = rejectUnknownKeysResult(objRes.value, ["request"], path);
165
+ if (Result.isError(keyCheck)) return keyCheck;
166
+
167
+ if (objRes.value.request === undefined) return Result.ok(undefined);
168
+ const requestRes = expectPlainObjectResult(objRes.value.request, `${path}.request`);
169
+ if (Result.isError(requestRes)) return requestRes;
170
+ const requestKeyCheck = rejectUnknownKeysResult(requestRes.value, ["eventsStream"], `${path}.request`);
171
+ if (Result.isError(requestKeyCheck)) return requestKeyCheck;
172
+ const eventsStreamRes = parseStreamNameResult(requestRes.value.eventsStream, `${path}.request.eventsStream`);
173
+ if (Result.isError(eventsStreamRes)) return eventsStreamRes;
174
+ if (!eventsStreamRes.value) return Result.ok(undefined);
175
+
176
+ return Result.ok({
177
+ request: {
178
+ eventsStream: eventsStreamRes.value,
179
+ },
180
+ });
181
+ }
182
+
183
+ function validateOtelTracesProfileResult(raw: unknown, path: string): Result<OtelTracesStreamProfile, { message: string }> {
184
+ const objRes = expectPlainObjectResult(raw, path);
185
+ if (Result.isError(objRes)) return objRes;
186
+ if (objRes.value.kind !== "otel-traces") return Result.err({ message: `${path}.kind must be otel-traces` });
187
+ const keyCheck = rejectUnknownKeysResult(
188
+ objRes.value,
189
+ ["kind", "redactKeys", "requestIdAttributes", "attributeLimits", "store", "dbStatementMode", "urlMode", "otlpLimits", "observability"],
190
+ path
191
+ );
192
+ if (Result.isError(keyCheck)) return keyCheck;
193
+ const redactKeysRes = parseStringArrayResult(objRes.value.redactKeys, `${path}.redactKeys`, 64);
194
+ if (Result.isError(redactKeysRes)) return redactKeysRes;
195
+ const requestIdAttributesRes = parseStringArrayResult(objRes.value.requestIdAttributes, `${path}.requestIdAttributes`, 64);
196
+ if (Result.isError(requestIdAttributesRes)) return requestIdAttributesRes;
197
+ const limitsRes = parseAttributeLimitsResult(objRes.value.attributeLimits, `${path}.attributeLimits`);
198
+ if (Result.isError(limitsRes)) return limitsRes;
199
+ const storeRes = parseStoreResult(objRes.value.store, `${path}.store`);
200
+ if (Result.isError(storeRes)) return storeRes;
201
+ const dbStatementModeRes = parseDbStatementModeResult(objRes.value.dbStatementMode, `${path}.dbStatementMode`);
202
+ if (Result.isError(dbStatementModeRes)) return dbStatementModeRes;
203
+ const urlModeRes = parseUrlModeResult(objRes.value.urlMode, `${path}.urlMode`);
204
+ if (Result.isError(urlModeRes)) return urlModeRes;
205
+ const otlpLimitsRes = parseOtlpLimitsResult(objRes.value.otlpLimits, `${path}.otlpLimits`);
206
+ if (Result.isError(otlpLimitsRes)) return otlpLimitsRes;
207
+ const observabilityRes = parseOtelTracesObservabilityResult(objRes.value.observability, `${path}.observability`);
208
+ if (Result.isError(observabilityRes)) return observabilityRes;
209
+ const profile: OtelTracesStreamProfile = { kind: "otel-traces" };
210
+ if (redactKeysRes.value) profile.redactKeys = redactKeysRes.value;
211
+ if (requestIdAttributesRes.value) profile.requestIdAttributes = requestIdAttributesRes.value;
212
+ if (limitsRes.value) profile.attributeLimits = limitsRes.value;
213
+ if (storeRes.value) profile.store = storeRes.value;
214
+ if (dbStatementModeRes.value) profile.dbStatementMode = dbStatementModeRes.value;
215
+ if (urlModeRes.value) profile.urlMode = urlModeRes.value;
216
+ if (otlpLimitsRes.value) profile.otlpLimits = otlpLimitsRes.value;
217
+ if (observabilityRes.value) profile.observability = observabilityRes.value;
218
+ return Result.ok(profile);
219
+ }
220
+
221
+ function getString(record: Record<string, unknown>, key: string): string | null {
222
+ const value = record[key];
223
+ return typeof value === "string" && value.trim() !== "" ? value : null;
224
+ }
225
+
226
+ function getNumber(record: Record<string, unknown>, key: string): number | null {
227
+ const value = record[key];
228
+ return typeof value === "number" && Number.isFinite(value) ? value : null;
229
+ }
230
+
231
+ function severityForSpan(record: Record<string, unknown>): "debug" | "info" | "warn" | "error" {
232
+ const status = isPlainObject(record.status) ? getString(record.status, "code") : null;
233
+ const error = isPlainObject(record.error) && record.error.isError === true;
234
+ return status === "error" || error ? "error" : "info";
235
+ }
236
+
237
+ function spanEventIsException(event: Record<string, unknown>): boolean {
238
+ const eventName = getString(event, "name")?.toLowerCase() ?? "";
239
+ if (eventName === "exception") return true;
240
+ const attributes = isPlainObject(event.attributes) ? event.attributes : {};
241
+ return getString(attributes, "exception.type") != null || getString(attributes, "exception.message") != null;
242
+ }
243
+
244
+ function buildOtelTimelineItems(args: { stream: string; offset?: string; record: unknown }): UnifiedTimelineItem[] {
245
+ if (!isPlainObject(args.record)) return [];
246
+ const record = args.record;
247
+ const traceId = getString(record, "traceId");
248
+ const spanId = getString(record, "spanId");
249
+ const parentSpanId = getString(record, "parentSpanId");
250
+ const requestId = getString(record, "requestId");
251
+ const service = getString(record, "service");
252
+ const title = getString(record, "name") ?? spanId ?? "span";
253
+ const timestamp = getString(record, "timestamp");
254
+ const endTimestamp = getString(record, "endTimestamp");
255
+ const duration = getNumber(record, "duration");
256
+ const severity = severityForSpan(record);
257
+ const source = { stream: args.stream, offset: args.offset, profile: "otel-traces" };
258
+ const ids = { requestId, traceId, spanId, parentSpanId };
259
+ const out: UnifiedTimelineItem[] = [];
260
+ if (timestamp) {
261
+ out.push({
262
+ kind: "otel.span.start",
263
+ time: timestamp,
264
+ duration,
265
+ service,
266
+ title,
267
+ severity,
268
+ ids,
269
+ source,
270
+ data: record,
271
+ });
272
+ }
273
+ if (Array.isArray(record.events)) {
274
+ for (const event of record.events) {
275
+ if (!isPlainObject(event)) continue;
276
+ const eventTime = getString(event, "timestamp");
277
+ const eventName = getString(event, "name") ?? "span event";
278
+ if (!eventTime) continue;
279
+ const isException = spanEventIsException(event);
280
+ out.push({
281
+ kind: isException ? "otel.exception" : "otel.span.event",
282
+ time: eventTime,
283
+ service,
284
+ title: eventName,
285
+ severity: isException ? "error" : severity,
286
+ ids,
287
+ source,
288
+ data: event,
289
+ });
290
+ }
291
+ }
292
+ if (endTimestamp) {
293
+ out.push({
294
+ kind: "otel.span.end",
295
+ time: endTimestamp,
296
+ duration,
297
+ service,
298
+ title,
299
+ severity,
300
+ ids,
301
+ source,
302
+ data: record,
303
+ });
304
+ }
305
+ return out;
306
+ }
307
+
308
+ export const OTEL_TRACES_STREAM_PROFILE_DEFINITION: StreamProfileDefinition = {
309
+ kind: "otel-traces",
310
+ usesStoredProfileRow: true,
311
+
312
+ defaultProfile(): OtelTracesStreamProfile {
313
+ return { kind: "otel-traces" };
314
+ },
315
+
316
+ validateResult(raw, path) {
317
+ return validateOtelTracesProfileResult(raw, path);
318
+ },
319
+
320
+ readProfileResult({ row, cached }): Result<StreamProfileReadResult, { message: string }> {
321
+ if (!row) return Result.ok({ profile: { kind: "otel-traces" }, cache: null });
322
+ const cachedCopy = cloneOtelTracesCache(cached);
323
+ if (cachedCopy && cachedCopy.updatedAtMs === row.updated_at_ms) {
324
+ return Result.ok({
325
+ profile: cloneOtelTracesProfile(cachedCopy.profile as OtelTracesStreamProfile),
326
+ cache: cachedCopy,
327
+ });
328
+ }
329
+ const parsedRes = parseStoredProfileJsonResult(row.profile_json);
330
+ if (Result.isError(parsedRes)) return parsedRes;
331
+ const profileRes = validateOtelTracesProfileResult(parsedRes.value, "profile");
332
+ if (Result.isError(profileRes)) return profileRes;
333
+ const profile = cloneOtelTracesProfile(profileRes.value);
334
+ return Result.ok({
335
+ profile: cloneOtelTracesProfile(profile),
336
+ cache: { profile, updatedAtMs: row.updated_at_ms },
337
+ });
338
+ },
339
+
340
+ persistProfileResult({ stream, streamRow, profile }): Result<StreamProfilePersistResult, { kind: "bad_request"; message: string; code?: string }> {
341
+ if (!isOtelTracesProfile(profile)) return Result.err({ kind: "bad_request", message: "invalid otel-traces profile" });
342
+ const contentType = normalizeProfileContentType(streamRow.content_type);
343
+ if (contentType !== "application/json") {
344
+ return Result.err({
345
+ kind: "bad_request",
346
+ message: "otel-traces profile requires application/json stream content-type",
347
+ });
348
+ }
349
+ if (streamRow.profile !== "otel-traces" && streamRow.next_offset > 0n) {
350
+ return Result.err({
351
+ kind: "bad_request",
352
+ message: "otel-traces profile must be installed before appending data",
353
+ });
354
+ }
355
+
356
+ const persistedProfile = cloneOtelTracesProfile(profile);
357
+ const registry = buildOtelTracesDefaultRegistry(stream);
358
+ return Result.ok({
359
+ profile: cloneOtelTracesProfile(persistedProfile),
360
+ cache: {
361
+ profile: persistedProfile,
362
+ updatedAtMs: 0n,
363
+ },
364
+ schemaRegistry: registry,
365
+ streamProfile: persistedProfile.kind,
366
+ profileJson: JSON.stringify(persistedProfile),
367
+ touchState: streamRow.profile === "state-protocol" ? "delete" : "preserve",
368
+ });
369
+ },
370
+
371
+ jsonIngest: {
372
+ prepareRecordResult({ profile, value }) {
373
+ if (!isOtelTracesProfile(profile)) return Result.err({ message: "invalid otel-traces profile" });
374
+ return normalizeOtelTraceRecordResult(profile, value);
375
+ },
376
+ },
377
+
378
+ otlpTraces: {
379
+ decodeExportRequestResult({ profile, stream, contentType, contentEncoding, body, maxDecodedBytes }) {
380
+ if (!isOtelTracesProfile(profile)) return Result.err({ status: 400, message: "invalid otel-traces profile" });
381
+ return decodeOtlpTraceExportRequestResult({ stream, profile, contentType, contentEncoding, body, maxDecodedBytes });
382
+ },
383
+ },
384
+
385
+ correlation: {
386
+ toTimelineItems(args) {
387
+ return buildOtelTimelineItems(args);
388
+ },
389
+ },
390
+ };
@@ -0,0 +1,284 @@
1
+ import { Result } from "better-result";
2
+ import type { SchemaRegistry } from "../schema/registry";
3
+ import type { StreamReadRow as StreamRow } from "../store/segment_read_store";
4
+ import type { ProfileTouchStatePlan } from "../store/profile_touch_store";
5
+ import type { MaybePromise } from "../store/capabilities";
6
+ import type { ProfileTouchControlStore, TouchRouteStore } from "../store/touch_store";
7
+ import type { TouchProcessorManager } from "../touch/manager";
8
+ import type { CanonicalChange } from "../touch/canonical_change";
9
+ import type { TouchConfig } from "../touch/spec";
10
+ import type { AggSummaryState } from "../search/agg_format";
11
+
12
+ export const STREAM_PROFILE_API_VERSION = "durable.streams/profile/v1" as const;
13
+ export const DEFAULT_STREAM_PROFILE = "generic" as const;
14
+
15
+ export type StreamProfileKind = string;
16
+
17
+ export type StreamProfileSpec = {
18
+ kind: StreamProfileKind;
19
+ [key: string]: unknown;
20
+ };
21
+
22
+ export type StreamProfileResource = {
23
+ apiVersion: typeof STREAM_PROFILE_API_VERSION;
24
+ profile: StreamProfileSpec;
25
+ };
26
+
27
+ export type StreamProfileMutationError = {
28
+ kind: "bad_request";
29
+ message: string;
30
+ code?: string;
31
+ };
32
+
33
+ export type StreamProfileReadError = {
34
+ kind: "invalid_profile";
35
+ message: string;
36
+ code?: string;
37
+ };
38
+
39
+ export type StreamProfileValidationError = {
40
+ message: string;
41
+ };
42
+
43
+ export type StoredProfileRow = {
44
+ stream: string;
45
+ profile_json: string;
46
+ updated_at_ms: bigint;
47
+ };
48
+
49
+ export type CachedStreamProfile = {
50
+ profile: StreamProfileSpec;
51
+ updatedAtMs: bigint;
52
+ };
53
+
54
+ export type StreamProfileReadResult = {
55
+ profile: StreamProfileSpec;
56
+ cache: CachedStreamProfile | null;
57
+ };
58
+
59
+ export type StreamProfilePersistResult = {
60
+ profile: StreamProfileSpec;
61
+ cache: CachedStreamProfile | null;
62
+ schemaRegistry?: SchemaRegistry | null;
63
+ streamProfile: string | null;
64
+ profileJson: string | null;
65
+ touchState: ProfileTouchStatePlan;
66
+ };
67
+
68
+ export type PersistProfileArgs = {
69
+ stream: string;
70
+ streamRow: StreamRow;
71
+ profile: StreamProfileSpec;
72
+ };
73
+
74
+ export type PreparedJsonRecord = {
75
+ value: unknown;
76
+ routingKey: string | null;
77
+ };
78
+
79
+ export type OtlpTraceExportResponseEncoding = "protobuf" | "json";
80
+
81
+ export type OtlpTraceExportResult = {
82
+ records: PreparedJsonRecord[];
83
+ acceptedSpans: number;
84
+ rejectedSpans: number;
85
+ warnings: string[];
86
+ responseEncoding: OtlpTraceExportResponseEncoding;
87
+ };
88
+
89
+ export type OtlpTraceExportError = {
90
+ message: string;
91
+ status?: 400 | 413 | 415;
92
+ };
93
+
94
+ export type UnifiedTimelineItem = {
95
+ kind: "evlog.event" | "otel.span.start" | "otel.span.end" | "otel.span.event" | "otel.exception";
96
+ time: string;
97
+ duration?: number | null;
98
+ service?: string | null;
99
+ title: string;
100
+ severity: "debug" | "info" | "warn" | "error";
101
+ ids: {
102
+ requestId?: string | null;
103
+ traceId?: string | null;
104
+ spanId?: string | null;
105
+ parentSpanId?: string | null;
106
+ };
107
+ source: {
108
+ stream: string;
109
+ offset?: string;
110
+ profile: string;
111
+ };
112
+ data: unknown;
113
+ };
114
+
115
+ export type MetricsCompanionRecord = {
116
+ metric: string;
117
+ unit: string;
118
+ metricKind: string;
119
+ temporality: string;
120
+ windowStartMs: number;
121
+ windowEndMs: number;
122
+ intervalMs: number;
123
+ stream: string | null;
124
+ instance: string | null;
125
+ attributes: Record<string, string>;
126
+ dimensionPairs: string[];
127
+ dimensionKey: string | null;
128
+ seriesKey: string;
129
+ summary: AggSummaryState;
130
+ };
131
+
132
+ export type NormalizedMetricsRecord = PreparedJsonRecord & {
133
+ value: Record<string, unknown>;
134
+ companion: MetricsCompanionRecord;
135
+ };
136
+
137
+ export type StreamTouchRoute =
138
+ | { kind: "meta" }
139
+ | { kind: "wait" }
140
+ | { kind: "templates_activate" };
141
+
142
+ export type StreamProfileTouchResponder = {
143
+ json(status: number, body: any, headers?: HeadersInit): Response;
144
+ badRequest(message: string): Response;
145
+ internalError(message?: string): Response;
146
+ notFound(message?: string): Response;
147
+ };
148
+
149
+ export type StreamTouchRouteStore = TouchRouteStore;
150
+
151
+ export type StreamTouchRouteArgs = {
152
+ route: StreamTouchRoute;
153
+ req: Request;
154
+ stream: string;
155
+ streamRow: StreamRow;
156
+ profile: StreamProfileSpec;
157
+ db: StreamTouchRouteStore;
158
+ touchManager: TouchProcessorManager;
159
+ respond: StreamProfileTouchResponder;
160
+ };
161
+
162
+ export interface StreamTouchCapability {
163
+ getTouchConfig(profile: StreamProfileSpec): TouchConfig | null;
164
+ syncState(args: { db: ProfileTouchControlStore; stream: string; profile: StreamProfileSpec }): MaybePromise<void>;
165
+ deriveCanonicalChanges(record: unknown, profile: StreamProfileSpec): CanonicalChange[];
166
+ handleRoute?(args: StreamTouchRouteArgs): Promise<Response>;
167
+ }
168
+
169
+ export interface StreamProfileJsonIngestCapability {
170
+ prepareRecordResult(args: { stream: string; profile: StreamProfileSpec; value: unknown }): Result<PreparedJsonRecord, StreamProfileValidationError>;
171
+ }
172
+
173
+ export interface StreamProfileOtlpTracesCapability {
174
+ decodeExportRequestResult(args: {
175
+ stream: string;
176
+ profile: StreamProfileSpec;
177
+ contentType: string;
178
+ contentEncoding: string | null;
179
+ body: Uint8Array;
180
+ maxDecodedBytes: number;
181
+ }): Result<OtlpTraceExportResult, OtlpTraceExportError>;
182
+ }
183
+
184
+ export interface StreamProfileCorrelationCapability {
185
+ toTimelineItems(args: { stream: string; offset?: string; record: unknown }): UnifiedTimelineItem[];
186
+ }
187
+
188
+ export interface StreamProfileMetricsCapability {
189
+ normalizeRecordResult(args: {
190
+ stream: string;
191
+ profile: StreamProfileSpec;
192
+ value: unknown;
193
+ }): Result<NormalizedMetricsRecord, StreamProfileValidationError>;
194
+ }
195
+
196
+ export interface StreamProfileDefinition {
197
+ kind: StreamProfileKind;
198
+ usesStoredProfileRow: boolean;
199
+ defaultProfile(): StreamProfileSpec;
200
+ validateResult(raw: unknown, path: string): Result<StreamProfileSpec, StreamProfileValidationError>;
201
+ readProfileResult(args: { row: StoredProfileRow | null; cached: CachedStreamProfile | null }): Result<StreamProfileReadResult, StreamProfileValidationError>;
202
+ persistProfileResult(args: PersistProfileArgs): Result<StreamProfilePersistResult, StreamProfileMutationError>;
203
+ touch?: StreamTouchCapability;
204
+ jsonIngest?: StreamProfileJsonIngestCapability;
205
+ otlpTraces?: StreamProfileOtlpTracesCapability;
206
+ correlation?: StreamProfileCorrelationCapability;
207
+ metrics?: StreamProfileMetricsCapability;
208
+ }
209
+
210
+ export function isPlainObject(value: unknown): value is Record<string, unknown> {
211
+ return !!value && typeof value === "object" && !Array.isArray(value);
212
+ }
213
+
214
+ export function expectPlainObjectResult(
215
+ value: unknown,
216
+ path: string
217
+ ): Result<Record<string, unknown>, StreamProfileValidationError> {
218
+ if (!isPlainObject(value)) return Result.err({ message: `${path} must be an object` });
219
+ return Result.ok(value);
220
+ }
221
+
222
+ export function rejectUnknownKeysResult(
223
+ obj: Record<string, unknown>,
224
+ allowed: readonly string[],
225
+ path: string
226
+ ): Result<void, StreamProfileValidationError> {
227
+ const allowedSet = new Set(allowed);
228
+ for (const key of Object.keys(obj)) {
229
+ if (!allowedSet.has(key)) return Result.err({ message: `${path}.${key} is not supported` });
230
+ }
231
+ return Result.ok(undefined);
232
+ }
233
+
234
+ export function normalizeProfileContentType(value: string | null): string | null {
235
+ if (!value) return null;
236
+ const base = value.split(";")[0]?.trim().toLowerCase();
237
+ return base ? base : null;
238
+ }
239
+
240
+ export function parseStoredProfileJsonResult(raw: string): Result<unknown, StreamProfileValidationError> {
241
+ try {
242
+ return Result.ok(JSON.parse(raw));
243
+ } catch (e: any) {
244
+ return Result.err({ message: String(e?.message ?? e) });
245
+ }
246
+ }
247
+
248
+ export function readProfileKindResult(
249
+ raw: unknown,
250
+ path = "profile"
251
+ ): Result<StreamProfileKind, StreamProfileValidationError> {
252
+ const objRes = expectPlainObjectResult(raw, path);
253
+ if (Result.isError(objRes)) return objRes;
254
+ const kind = typeof objRes.value.kind === "string" ? objRes.value.kind.trim() : "";
255
+ if (kind !== "") return Result.ok(kind);
256
+ return Result.err({ message: `${path}.kind must be a non-empty string` });
257
+ }
258
+
259
+ export function parseProfileUpdateEnvelopeResult(body: unknown): Result<unknown, StreamProfileValidationError> {
260
+ const bodyRes = expectPlainObjectResult(body, "profile update");
261
+ if (Result.isError(bodyRes)) {
262
+ return Result.err({ message: "profile update must be a JSON object" });
263
+ }
264
+ const keyCheck = rejectUnknownKeysResult(bodyRes.value, ["apiVersion", "profile"], "profileUpdate");
265
+ if (Result.isError(keyCheck)) return keyCheck;
266
+ if (bodyRes.value.apiVersion !== undefined && bodyRes.value.apiVersion !== STREAM_PROFILE_API_VERSION) {
267
+ return Result.err({ message: "invalid profile apiVersion" });
268
+ }
269
+ if (!Object.prototype.hasOwnProperty.call(bodyRes.value, "profile")) {
270
+ return Result.err({ message: "missing profile" });
271
+ }
272
+ return Result.ok(bodyRes.value.profile);
273
+ }
274
+
275
+ export function cloneStreamProfileSpec(profile: StreamProfileSpec): StreamProfileSpec {
276
+ return structuredClone(profile);
277
+ }
278
+
279
+ export function buildStreamProfileResource(profile: StreamProfileSpec): StreamProfileResource {
280
+ return {
281
+ apiVersion: STREAM_PROFILE_API_VERSION,
282
+ profile: cloneStreamProfileSpec(profile),
283
+ };
284
+ }