@tungthedev/streams-server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CODE_OF_CONDUCT.md +45 -0
  2. package/CONTRIBUTING.md +76 -0
  3. package/LICENSE +201 -0
  4. package/README.md +58 -0
  5. package/SECURITY.md +42 -0
  6. package/bin/prisma-streams-server +2 -0
  7. package/package.json +46 -0
  8. package/src/app.ts +583 -0
  9. package/src/app_core.ts +3144 -0
  10. package/src/app_local.ts +206 -0
  11. package/src/auth.ts +124 -0
  12. package/src/auto_tune.ts +69 -0
  13. package/src/backpressure.ts +66 -0
  14. package/src/bootstrap.ts +613 -0
  15. package/src/compute/demo_entry.ts +415 -0
  16. package/src/compute/demo_site.ts +1242 -0
  17. package/src/compute/entry.ts +19 -0
  18. package/src/compute/package_entry.ts +4 -0
  19. package/src/compute/virtual-modules.d.ts +15 -0
  20. package/src/compute/worker_module_url.ts +9 -0
  21. package/src/concurrency_gate.ts +108 -0
  22. package/src/config.ts +402 -0
  23. package/src/db/bootstrap_store.ts +9 -0
  24. package/src/db/db.ts +2424 -0
  25. package/src/db/schema.ts +925 -0
  26. package/src/db/sqlite_manifest_snapshot.ts +81 -0
  27. package/src/db/sqlite_touch_store.ts +491 -0
  28. package/src/db/sqlite_wal_store.ts +472 -0
  29. package/src/details/full_mode_details.ts +568 -0
  30. package/src/expiry_sweeper.ts +47 -0
  31. package/src/foreground_activity.ts +55 -0
  32. package/src/hist.ts +169 -0
  33. package/src/index/binary_fuse.ts +379 -0
  34. package/src/index/indexer.ts +947 -0
  35. package/src/index/lexicon_file_cache.ts +261 -0
  36. package/src/index/lexicon_format.ts +93 -0
  37. package/src/index/lexicon_indexer.ts +863 -0
  38. package/src/index/run_cache.ts +84 -0
  39. package/src/index/run_format.ts +213 -0
  40. package/src/index/schedule.ts +28 -0
  41. package/src/index/secondary_indexer.ts +901 -0
  42. package/src/index/secondary_schema.ts +105 -0
  43. package/src/ingest.ts +309 -0
  44. package/src/lens/lens.ts +501 -0
  45. package/src/manifest.ts +249 -0
  46. package/src/memory.ts +334 -0
  47. package/src/metrics.ts +147 -0
  48. package/src/metrics_emitter.ts +83 -0
  49. package/src/notifier.ts +180 -0
  50. package/src/objectstore/accounting.ts +151 -0
  51. package/src/objectstore/interface.ts +13 -0
  52. package/src/objectstore/mock_r2.ts +269 -0
  53. package/src/objectstore/null.ts +32 -0
  54. package/src/objectstore/r2.ts +318 -0
  55. package/src/observe/pairing.ts +61 -0
  56. package/src/observe/request.ts +772 -0
  57. package/src/offset.ts +70 -0
  58. package/src/postgres/bootstrap.ts +269 -0
  59. package/src/postgres/companions.ts +197 -0
  60. package/src/postgres/control_restore.ts +109 -0
  61. package/src/postgres/details.ts +189 -0
  62. package/src/postgres/lexicon_index.ts +260 -0
  63. package/src/postgres/routing_index.ts +189 -0
  64. package/src/postgres/rows.ts +132 -0
  65. package/src/postgres/schema.ts +355 -0
  66. package/src/postgres/secondary_index.ts +238 -0
  67. package/src/postgres/segments.ts +900 -0
  68. package/src/postgres/stats.ts +103 -0
  69. package/src/postgres/store.ts +947 -0
  70. package/src/postgres/touch.ts +591 -0
  71. package/src/postgres/types.ts +32 -0
  72. package/src/profiles/evlog/schema.ts +234 -0
  73. package/src/profiles/evlog.ts +473 -0
  74. package/src/profiles/generic.ts +51 -0
  75. package/src/profiles/index.ts +237 -0
  76. package/src/profiles/metrics/block_format.ts +109 -0
  77. package/src/profiles/metrics/normalize.ts +366 -0
  78. package/src/profiles/metrics/schema.ts +319 -0
  79. package/src/profiles/metrics.ts +83 -0
  80. package/src/profiles/otelTraces/normalize.ts +955 -0
  81. package/src/profiles/otelTraces/otlp.ts +1002 -0
  82. package/src/profiles/otelTraces/schema.ts +408 -0
  83. package/src/profiles/otelTraces.ts +390 -0
  84. package/src/profiles/profile.ts +284 -0
  85. package/src/profiles/stateProtocol/change_event_conformance.typecheck.ts +35 -0
  86. package/src/profiles/stateProtocol/changes.ts +24 -0
  87. package/src/profiles/stateProtocol/ingest.ts +115 -0
  88. package/src/profiles/stateProtocol/routes.ts +511 -0
  89. package/src/profiles/stateProtocol/types.ts +6 -0
  90. package/src/profiles/stateProtocol/validation.ts +51 -0
  91. package/src/profiles/stateProtocol.ts +107 -0
  92. package/src/read_filter.ts +468 -0
  93. package/src/reader.ts +2986 -0
  94. package/src/runtime/hash.ts +156 -0
  95. package/src/runtime/hash_vendor/LICENSE.hash-wasm +38 -0
  96. package/src/runtime/hash_vendor/NOTICE.md +8 -0
  97. package/src/runtime/hash_vendor/xxhash3.umd.min.cjs +7 -0
  98. package/src/runtime/hash_vendor/xxhash32.umd.min.cjs +7 -0
  99. package/src/runtime/hash_vendor/xxhash64.umd.min.cjs +7 -0
  100. package/src/runtime/host_runtime.ts +5 -0
  101. package/src/runtime_memory.ts +200 -0
  102. package/src/runtime_memory_sampler.ts +237 -0
  103. package/src/schema/lens_schema.ts +290 -0
  104. package/src/schema/proof.ts +547 -0
  105. package/src/schema/read_json.ts +51 -0
  106. package/src/schema/registry.ts +966 -0
  107. package/src/search/agg_format.ts +638 -0
  108. package/src/search/aggregate.ts +409 -0
  109. package/src/search/binary/codec.ts +162 -0
  110. package/src/search/binary/docset.ts +67 -0
  111. package/src/search/binary/restart_strings.ts +181 -0
  112. package/src/search/binary/varint.ts +34 -0
  113. package/src/search/bitset.ts +19 -0
  114. package/src/search/col_format.ts +382 -0
  115. package/src/search/col_runtime.ts +59 -0
  116. package/src/search/column_encoding.ts +43 -0
  117. package/src/search/companion_file_cache.ts +319 -0
  118. package/src/search/companion_format.ts +327 -0
  119. package/src/search/companion_manager.ts +1305 -0
  120. package/src/search/companion_plan.ts +229 -0
  121. package/src/search/exact_format.ts +281 -0
  122. package/src/search/exact_runtime.ts +55 -0
  123. package/src/search/fts_format.ts +423 -0
  124. package/src/search/fts_runtime.ts +333 -0
  125. package/src/search/query.ts +875 -0
  126. package/src/search/schema.ts +245 -0
  127. package/src/segment/cache.ts +270 -0
  128. package/src/segment/cached_segment.ts +89 -0
  129. package/src/segment/format.ts +403 -0
  130. package/src/segment/segmenter.ts +412 -0
  131. package/src/segment/segmenter_worker.ts +72 -0
  132. package/src/segment/segmenter_workers.ts +130 -0
  133. package/src/server.ts +264 -0
  134. package/src/server_auto_tune.ts +158 -0
  135. package/src/sqlite/adapter.ts +335 -0
  136. package/src/sqlite/runtime_stats.ts +163 -0
  137. package/src/stats.ts +205 -0
  138. package/src/store/append.ts +50 -0
  139. package/src/store/bootstrap_restore_store.ts +71 -0
  140. package/src/store/capabilities.ts +86 -0
  141. package/src/store/full_mode_details_store.ts +71 -0
  142. package/src/store/index_store.ts +104 -0
  143. package/src/store/profile_touch_store.ts +1 -0
  144. package/src/store/rows.ts +144 -0
  145. package/src/store/schema_profile_store.ts +73 -0
  146. package/src/store/schema_publication.ts +6 -0
  147. package/src/store/segment_manifest_store.ts +129 -0
  148. package/src/store/segment_read_store.ts +22 -0
  149. package/src/store/stats_accounting_store.ts +83 -0
  150. package/src/store/touch_store.ts +98 -0
  151. package/src/store/wal_store.ts +21 -0
  152. package/src/stream_size_reconciler.ts +100 -0
  153. package/src/touch/canonical_change.ts +7 -0
  154. package/src/touch/live_keys.ts +158 -0
  155. package/src/touch/live_metrics.ts +841 -0
  156. package/src/touch/live_templates.ts +449 -0
  157. package/src/touch/manager.ts +1292 -0
  158. package/src/touch/process_batch.ts +576 -0
  159. package/src/touch/processor_worker.ts +85 -0
  160. package/src/touch/spec.ts +459 -0
  161. package/src/touch/touch_journal.ts +771 -0
  162. package/src/touch/touch_key_id.ts +20 -0
  163. package/src/touch/worker_pool.ts +191 -0
  164. package/src/touch/worker_protocol.ts +57 -0
  165. package/src/types/proper-lockfile.d.ts +1 -0
  166. package/src/uploader.ts +358 -0
  167. package/src/util/base32_crockford.ts +81 -0
  168. package/src/util/bloom256.ts +67 -0
  169. package/src/util/byte_lru.ts +73 -0
  170. package/src/util/cleanup.ts +22 -0
  171. package/src/util/crc32c.ts +29 -0
  172. package/src/util/ds_error.ts +15 -0
  173. package/src/util/duration.ts +17 -0
  174. package/src/util/endian.ts +53 -0
  175. package/src/util/json_pointer.ts +148 -0
  176. package/src/util/log.ts +25 -0
  177. package/src/util/lru.ts +53 -0
  178. package/src/util/retry.ts +35 -0
  179. package/src/util/siphash.ts +71 -0
  180. package/src/util/stream_paths.ts +50 -0
  181. package/src/util/time.ts +14 -0
  182. package/src/util/yield.ts +3 -0
  183. package/src/util/zstd.ts +24 -0
@@ -0,0 +1,234 @@
1
+ import {
2
+ SCHEMA_REGISTRY_API_VERSION,
3
+ type SchemaRegistry,
4
+ type SearchConfig,
5
+ } from "../../schema/registry";
6
+
7
+ export const EVLOG_CANONICAL_SCHEMA = {
8
+ type: "object",
9
+ additionalProperties: false,
10
+ properties: {
11
+ timestamp: { type: "string" },
12
+ level: { type: "string", enum: ["debug", "info", "warn", "error"] },
13
+ service: { type: ["string", "null"] },
14
+ environment: { type: ["string", "null"] },
15
+ version: { type: ["string", "null"] },
16
+ region: { type: ["string", "null"] },
17
+ requestId: { type: ["string", "null"] },
18
+ traceId: { type: ["string", "null"] },
19
+ spanId: { type: ["string", "null"] },
20
+ method: { type: ["string", "null"] },
21
+ path: { type: ["string", "null"] },
22
+ status: { type: ["integer", "null"] },
23
+ duration: { type: ["number", "null"] },
24
+ message: { type: ["string", "null"] },
25
+ why: { type: ["string", "null"] },
26
+ fix: { type: ["string", "null"] },
27
+ link: { type: ["string", "null"] },
28
+ sampling: {
29
+ type: ["object", "null"],
30
+ additionalProperties: true,
31
+ },
32
+ redaction: {
33
+ type: "object",
34
+ additionalProperties: false,
35
+ properties: {
36
+ keys: {
37
+ type: "array",
38
+ items: { type: "string" },
39
+ },
40
+ },
41
+ required: ["keys"],
42
+ },
43
+ context: {
44
+ type: "object",
45
+ additionalProperties: true,
46
+ },
47
+ },
48
+ required: [
49
+ "timestamp",
50
+ "level",
51
+ "service",
52
+ "environment",
53
+ "version",
54
+ "region",
55
+ "requestId",
56
+ "traceId",
57
+ "spanId",
58
+ "method",
59
+ "path",
60
+ "status",
61
+ "duration",
62
+ "message",
63
+ "why",
64
+ "fix",
65
+ "link",
66
+ "sampling",
67
+ "redaction",
68
+ "context",
69
+ ],
70
+ } as const;
71
+
72
+ export const EVLOG_DEFAULT_SEARCH_CONFIG: SearchConfig = {
73
+ profile: "evlog",
74
+ primaryTimestampField: "timestamp",
75
+ aliases: {
76
+ env: "environment",
77
+ msg: "message",
78
+ req: "requestId",
79
+ span: "spanId",
80
+ time: "timestamp",
81
+ trace: "traceId",
82
+ ts: "timestamp",
83
+ },
84
+ defaultFields: [
85
+ { field: "message", boost: 2 },
86
+ { field: "why", boost: 1.5 },
87
+ { field: "fix", boost: 1.25 },
88
+ { field: "error.message", boost: 2 },
89
+ ],
90
+ fields: {
91
+ timestamp: {
92
+ kind: "date",
93
+ bindings: [{ version: 1, jsonPointer: "/timestamp" }],
94
+ exact: true,
95
+ column: true,
96
+ exists: true,
97
+ sortable: true,
98
+ aggregatable: true,
99
+ },
100
+ level: {
101
+ kind: "keyword",
102
+ bindings: [{ version: 1, jsonPointer: "/level" }],
103
+ normalizer: "lowercase_v1",
104
+ exact: true,
105
+ prefix: true,
106
+ exists: true,
107
+ sortable: true,
108
+ aggregatable: true,
109
+ },
110
+ service: {
111
+ kind: "keyword",
112
+ bindings: [{ version: 1, jsonPointer: "/service" }],
113
+ normalizer: "lowercase_v1",
114
+ exact: true,
115
+ prefix: true,
116
+ exists: true,
117
+ sortable: true,
118
+ aggregatable: true,
119
+ },
120
+ environment: {
121
+ kind: "keyword",
122
+ bindings: [{ version: 1, jsonPointer: "/environment" }],
123
+ normalizer: "lowercase_v1",
124
+ exact: true,
125
+ prefix: true,
126
+ exists: true,
127
+ sortable: true,
128
+ aggregatable: true,
129
+ },
130
+ requestId: {
131
+ kind: "keyword",
132
+ bindings: [{ version: 1, jsonPointer: "/requestId" }],
133
+ exact: true,
134
+ prefix: true,
135
+ exists: true,
136
+ sortable: true,
137
+ },
138
+ traceId: {
139
+ kind: "keyword",
140
+ bindings: [{ version: 1, jsonPointer: "/traceId" }],
141
+ exact: true,
142
+ prefix: true,
143
+ exists: true,
144
+ sortable: true,
145
+ },
146
+ spanId: {
147
+ kind: "keyword",
148
+ bindings: [{ version: 1, jsonPointer: "/spanId" }],
149
+ exact: true,
150
+ prefix: true,
151
+ exists: true,
152
+ sortable: true,
153
+ },
154
+ path: {
155
+ kind: "keyword",
156
+ bindings: [{ version: 1, jsonPointer: "/path" }],
157
+ exact: true,
158
+ prefix: true,
159
+ exists: true,
160
+ sortable: true,
161
+ aggregatable: true,
162
+ },
163
+ method: {
164
+ kind: "keyword",
165
+ bindings: [{ version: 1, jsonPointer: "/method" }],
166
+ normalizer: "lowercase_v1",
167
+ exact: true,
168
+ prefix: true,
169
+ exists: true,
170
+ sortable: true,
171
+ aggregatable: true,
172
+ },
173
+ status: {
174
+ kind: "integer",
175
+ bindings: [{ version: 1, jsonPointer: "/status" }],
176
+ exact: true,
177
+ column: true,
178
+ exists: true,
179
+ sortable: true,
180
+ aggregatable: true,
181
+ },
182
+ duration: {
183
+ kind: "float",
184
+ bindings: [{ version: 1, jsonPointer: "/duration" }],
185
+ exact: true,
186
+ column: true,
187
+ exists: true,
188
+ sortable: true,
189
+ aggregatable: true,
190
+ },
191
+ message: {
192
+ kind: "text",
193
+ bindings: [{ version: 1, jsonPointer: "/message" }],
194
+ analyzer: "unicode_word_v1",
195
+ exists: true,
196
+ positions: true,
197
+ },
198
+ why: {
199
+ kind: "text",
200
+ bindings: [{ version: 1, jsonPointer: "/why" }],
201
+ analyzer: "unicode_word_v1",
202
+ exists: true,
203
+ positions: true,
204
+ },
205
+ fix: {
206
+ kind: "text",
207
+ bindings: [{ version: 1, jsonPointer: "/fix" }],
208
+ analyzer: "unicode_word_v1",
209
+ exists: true,
210
+ positions: true,
211
+ },
212
+ "error.message": {
213
+ kind: "text",
214
+ bindings: [{ version: 1, jsonPointer: "/context/error/message" }],
215
+ analyzer: "unicode_word_v1",
216
+ exists: true,
217
+ positions: true,
218
+ },
219
+ },
220
+ };
221
+
222
+ export function buildEvlogDefaultRegistry(stream: string): SchemaRegistry {
223
+ return {
224
+ apiVersion: SCHEMA_REGISTRY_API_VERSION,
225
+ schema: stream,
226
+ currentVersion: 1,
227
+ search: structuredClone(EVLOG_DEFAULT_SEARCH_CONFIG),
228
+ boundaries: [{ offset: 0, version: 1 }],
229
+ schemas: {
230
+ "1": structuredClone(EVLOG_CANONICAL_SCHEMA),
231
+ },
232
+ lenses: {},
233
+ };
234
+ }
@@ -0,0 +1,473 @@
1
+ import { Result } from "better-result";
2
+ import type {
3
+ CachedStreamProfile,
4
+ PreparedJsonRecord,
5
+ StreamProfileDefinition,
6
+ StreamProfilePersistResult,
7
+ StreamProfileReadResult,
8
+ StreamProfileSpec,
9
+ UnifiedTimelineItem,
10
+ } from "./profile";
11
+ import {
12
+ cloneStreamProfileSpec,
13
+ expectPlainObjectResult,
14
+ isPlainObject,
15
+ normalizeProfileContentType,
16
+ parseStoredProfileJsonResult,
17
+ rejectUnknownKeysResult,
18
+ } from "./profile";
19
+ import { buildEvlogDefaultRegistry } from "./evlog/schema";
20
+
21
+ export type EvlogStreamProfile = {
22
+ kind: "evlog";
23
+ redactKeys?: string[];
24
+ correlation?: {
25
+ requestIdFields?: string[];
26
+ traceContextFields?: string[];
27
+ parseTraceparent?: boolean;
28
+ };
29
+ observability?: {
30
+ request?: {
31
+ tracesStream: string;
32
+ };
33
+ };
34
+ };
35
+
36
+ const DEFAULT_REDACT_KEYS = ["password", "token", "secret", "authorization", "cookie", "apikey"] as const;
37
+ const REDACTED_VALUE = "[REDACTED]";
38
+ const EVLOG_RESERVED_FIELDS = new Set([
39
+ "timestamp",
40
+ "level",
41
+ "service",
42
+ "environment",
43
+ "version",
44
+ "region",
45
+ "requestId",
46
+ "traceId",
47
+ "spanId",
48
+ "method",
49
+ "path",
50
+ "status",
51
+ "duration",
52
+ "message",
53
+ "why",
54
+ "fix",
55
+ "link",
56
+ "sampling",
57
+ "redaction",
58
+ "context",
59
+ ]);
60
+
61
+ type RedactionResult = {
62
+ value: unknown;
63
+ paths: string[];
64
+ };
65
+
66
+ function cloneEvlogProfile(profile: EvlogStreamProfile): EvlogStreamProfile {
67
+ return cloneStreamProfileSpec(profile) as EvlogStreamProfile;
68
+ }
69
+
70
+ function cloneEvlogCache(cache: CachedStreamProfile | null): CachedStreamProfile | null {
71
+ if (!cache || cache.profile.kind !== "evlog") return null;
72
+ return {
73
+ profile: cloneEvlogProfile(cache.profile as EvlogStreamProfile),
74
+ updatedAtMs: cache.updatedAtMs,
75
+ };
76
+ }
77
+
78
+ function isEvlogProfile(profile: StreamProfileSpec | null | undefined): profile is EvlogStreamProfile {
79
+ return !!profile && profile.kind === "evlog";
80
+ }
81
+
82
+ function parseRedactKeysResult(raw: unknown, path: string): Result<string[] | undefined, { message: string }> {
83
+ if (raw === undefined) return Result.ok(undefined);
84
+ if (!Array.isArray(raw)) return Result.err({ message: `${path} must be an array of strings` });
85
+ if (raw.length > 64) return Result.err({ message: `${path} too large (max 64)` });
86
+
87
+ const normalized: string[] = [];
88
+ const seen = new Set<string>();
89
+ for (const item of raw) {
90
+ if (typeof item !== "string") return Result.err({ message: `${path} must be an array of strings` });
91
+ const value = item.trim().toLowerCase();
92
+ if (value === "") return Result.err({ message: `${path} must not contain empty strings` });
93
+ if (seen.has(value)) continue;
94
+ seen.add(value);
95
+ normalized.push(value);
96
+ }
97
+ return Result.ok(normalized);
98
+ }
99
+
100
+ function parseStringListResult(raw: unknown, path: string, maxItems: number): Result<string[] | undefined, { message: string }> {
101
+ if (raw === undefined) return Result.ok(undefined);
102
+ if (!Array.isArray(raw)) return Result.err({ message: `${path} must be an array of strings` });
103
+ if (raw.length > maxItems) return Result.err({ message: `${path} too large (max ${maxItems})` });
104
+ const out: string[] = [];
105
+ const seen = new Set<string>();
106
+ for (const item of raw) {
107
+ if (typeof item !== "string") return Result.err({ message: `${path} must be an array of strings` });
108
+ const value = item.trim();
109
+ if (value === "") return Result.err({ message: `${path} must not contain empty strings` });
110
+ if (seen.has(value)) continue;
111
+ seen.add(value);
112
+ out.push(value);
113
+ }
114
+ return Result.ok(out);
115
+ }
116
+
117
+ function parseEvlogCorrelationResult(raw: unknown, path: string): Result<EvlogStreamProfile["correlation"] | undefined, { message: string }> {
118
+ if (raw === undefined) return Result.ok(undefined);
119
+ const objRes = expectPlainObjectResult(raw, path);
120
+ if (Result.isError(objRes)) return objRes;
121
+ const keyCheck = rejectUnknownKeysResult(objRes.value, ["requestIdFields", "traceContextFields", "parseTraceparent"], path);
122
+ if (Result.isError(keyCheck)) return keyCheck;
123
+ const requestIdFieldsRes = parseStringListResult(objRes.value.requestIdFields, `${path}.requestIdFields`, 64);
124
+ if (Result.isError(requestIdFieldsRes)) return requestIdFieldsRes;
125
+ const traceContextFieldsRes = parseStringListResult(objRes.value.traceContextFields, `${path}.traceContextFields`, 64);
126
+ if (Result.isError(traceContextFieldsRes)) return traceContextFieldsRes;
127
+ if (objRes.value.parseTraceparent !== undefined && typeof objRes.value.parseTraceparent !== "boolean") {
128
+ return Result.err({ message: `${path}.parseTraceparent must be boolean` });
129
+ }
130
+ const correlation: NonNullable<EvlogStreamProfile["correlation"]> = {};
131
+ if (requestIdFieldsRes.value) correlation.requestIdFields = requestIdFieldsRes.value;
132
+ if (traceContextFieldsRes.value) correlation.traceContextFields = traceContextFieldsRes.value;
133
+ if (objRes.value.parseTraceparent !== undefined) correlation.parseTraceparent = objRes.value.parseTraceparent;
134
+ return Result.ok(Object.keys(correlation).length > 0 ? correlation : undefined);
135
+ }
136
+
137
+ function parseStreamNameResult(raw: unknown, path: string): Result<string | undefined, { message: string }> {
138
+ if (raw === undefined) return Result.ok(undefined);
139
+ if (typeof raw !== "string") return Result.err({ message: `${path} must be a string` });
140
+ const value = raw.trim();
141
+ if (value === "") return Result.err({ message: `${path} must not be empty` });
142
+ return Result.ok(value);
143
+ }
144
+
145
+ function parseEvlogObservabilityResult(raw: unknown, path: string): Result<EvlogStreamProfile["observability"] | undefined, { message: string }> {
146
+ if (raw === undefined) return Result.ok(undefined);
147
+ const objRes = expectPlainObjectResult(raw, path);
148
+ if (Result.isError(objRes)) return objRes;
149
+ const keyCheck = rejectUnknownKeysResult(objRes.value, ["request"], path);
150
+ if (Result.isError(keyCheck)) return keyCheck;
151
+
152
+ if (objRes.value.request === undefined) return Result.ok(undefined);
153
+ const requestRes = expectPlainObjectResult(objRes.value.request, `${path}.request`);
154
+ if (Result.isError(requestRes)) return requestRes;
155
+ const requestKeyCheck = rejectUnknownKeysResult(requestRes.value, ["tracesStream"], `${path}.request`);
156
+ if (Result.isError(requestKeyCheck)) return requestKeyCheck;
157
+ const tracesStreamRes = parseStreamNameResult(requestRes.value.tracesStream, `${path}.request.tracesStream`);
158
+ if (Result.isError(tracesStreamRes)) return tracesStreamRes;
159
+ if (!tracesStreamRes.value) return Result.ok(undefined);
160
+
161
+ return Result.ok({
162
+ request: {
163
+ tracesStream: tracesStreamRes.value,
164
+ },
165
+ });
166
+ }
167
+
168
+ function validateEvlogProfileResult(raw: unknown, path: string): Result<EvlogStreamProfile, { message: string }> {
169
+ const objRes = expectPlainObjectResult(raw, path);
170
+ if (Result.isError(objRes)) return objRes;
171
+ if (objRes.value.kind !== "evlog") {
172
+ return Result.err({ message: `${path}.kind must be evlog` });
173
+ }
174
+ const keyCheck = rejectUnknownKeysResult(objRes.value, ["kind", "redactKeys", "correlation", "observability"], path);
175
+ if (Result.isError(keyCheck)) return keyCheck;
176
+ const redactKeysRes = parseRedactKeysResult(objRes.value.redactKeys, `${path}.redactKeys`);
177
+ if (Result.isError(redactKeysRes)) return redactKeysRes;
178
+ const correlationRes = parseEvlogCorrelationResult(objRes.value.correlation, `${path}.correlation`);
179
+ if (Result.isError(correlationRes)) return correlationRes;
180
+ const observabilityRes = parseEvlogObservabilityResult(objRes.value.observability, `${path}.observability`);
181
+ if (Result.isError(observabilityRes)) return observabilityRes;
182
+ const profile: EvlogStreamProfile = { kind: "evlog" };
183
+ if (redactKeysRes.value) profile.redactKeys = redactKeysRes.value;
184
+ if (correlationRes.value) profile.correlation = correlationRes.value;
185
+ if (observabilityRes.value) profile.observability = observabilityRes.value;
186
+ return Result.ok(profile);
187
+ }
188
+
189
+ function normalizeString(value: unknown): string | null {
190
+ if (typeof value !== "string") return null;
191
+ const trimmed = value.trim();
192
+ return trimmed === "" ? null : trimmed;
193
+ }
194
+
195
+ function normalizeTraceField(input: Record<string, unknown>, field: "traceId" | "spanId"): string | null {
196
+ const direct = normalizeString(input[field]);
197
+ if (direct) return direct;
198
+ const traceContext = isPlainObject(input.traceContext) ? input.traceContext : null;
199
+ return traceContext ? normalizeString(traceContext[field]) : null;
200
+ }
201
+
202
+ function readDottedString(input: Record<string, unknown>, path: string): string | null {
203
+ let cur: unknown = input;
204
+ for (const part of path.split(".")) {
205
+ if (!isPlainObject(cur)) return null;
206
+ cur = cur[part];
207
+ }
208
+ return normalizeString(cur);
209
+ }
210
+
211
+ function normalizeRequestId(input: Record<string, unknown>, profile: EvlogStreamProfile): string | null {
212
+ const fields = profile.correlation?.requestIdFields ?? ["requestId", "context.requestId"];
213
+ for (const field of fields) {
214
+ const value = readDottedString(input, field);
215
+ if (value) return value;
216
+ }
217
+ return null;
218
+ }
219
+
220
+ function normalizeConfiguredTraceField(input: Record<string, unknown>, profile: EvlogStreamProfile, field: "traceId" | "spanId"): string | null {
221
+ const fields = profile.correlation?.traceContextFields;
222
+ if (!fields) return normalizeTraceField(input, field);
223
+ for (const path of fields) {
224
+ if (path !== field && !path.endsWith(`.${field}`)) continue;
225
+ const value = readDottedString(input, path);
226
+ if (value) return value;
227
+ }
228
+ return normalizeTraceField(input, field);
229
+ }
230
+
231
+ function parseTraceparent(input: Record<string, unknown>): { traceId: string; spanId: string } | null {
232
+ for (const path of ["traceparent", "traceContext.traceparent", "context.traceparent", "headers.traceparent"]) {
233
+ const value = readDottedString(input, path);
234
+ if (!value) continue;
235
+ const match = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})(?:-.+)?$/i.exec(value);
236
+ if (!match) continue;
237
+ const traceId = match[2].toLowerCase();
238
+ const spanId = match[3].toLowerCase();
239
+ if (/^0+$/.test(traceId) || /^0+$/.test(spanId)) continue;
240
+ return { traceId, spanId };
241
+ }
242
+ return null;
243
+ }
244
+
245
+ function normalizeOptionalNumber(value: unknown): number | null {
246
+ if (typeof value === "number" && Number.isFinite(value)) return value;
247
+ if (typeof value === "string" && value.trim() !== "") {
248
+ const n = Number(value);
249
+ if (Number.isFinite(n)) return n;
250
+ }
251
+ return null;
252
+ }
253
+
254
+ function normalizeOptionalInteger(value: unknown): number | null {
255
+ if (typeof value === "number" && Number.isFinite(value) && Number.isInteger(value)) return value;
256
+ if (typeof value === "string" && value.trim() !== "") {
257
+ const n = Number(value);
258
+ if (Number.isFinite(n) && Number.isInteger(n)) return n;
259
+ }
260
+ return null;
261
+ }
262
+
263
+ function deriveLevel(input: Record<string, unknown>, status: number | null): string {
264
+ const direct = normalizeString(input.level)?.toLowerCase();
265
+ if (direct === "debug" || direct === "info" || direct === "warn" || direct === "error") {
266
+ return direct;
267
+ }
268
+ if (normalizeString(input.why) || normalizeString(input.fix) || normalizeString(input.link)) return "error";
269
+ if (status != null && status >= 500) return "error";
270
+ if (status != null && status >= 400) return "warn";
271
+ return "info";
272
+ }
273
+
274
+ function redactValue(value: unknown, redactKeys: Set<string>, path = ""): RedactionResult {
275
+ if (Array.isArray(value)) {
276
+ const items = value.map((item, index) => redactValue(item, redactKeys, path === "" ? String(index) : `${path}.${index}`));
277
+ return {
278
+ value: items.map((item) => item.value),
279
+ paths: items.flatMap((item) => item.paths),
280
+ };
281
+ }
282
+ if (!isPlainObject(value)) return { value: structuredClone(value), paths: [] };
283
+
284
+ const out: Record<string, unknown> = {};
285
+ const paths: string[] = [];
286
+ for (const [key, raw] of Object.entries(value)) {
287
+ const keyPath = path === "" ? key : `${path}.${key}`;
288
+ if (redactKeys.has(key.toLowerCase())) {
289
+ out[key] = REDACTED_VALUE;
290
+ paths.push(keyPath);
291
+ continue;
292
+ }
293
+ const nested = redactValue(raw, redactKeys, keyPath);
294
+ out[key] = nested.value;
295
+ paths.push(...nested.paths);
296
+ }
297
+ return { value: out, paths };
298
+ }
299
+
300
+ function buildContext(input: Record<string, unknown>): Record<string, unknown> {
301
+ const context: Record<string, unknown> = isPlainObject(input.context) ? structuredClone(input.context) : {};
302
+ for (const [key, value] of Object.entries(input)) {
303
+ if (EVLOG_RESERVED_FIELDS.has(key)) continue;
304
+ context[key] = structuredClone(value);
305
+ }
306
+ if (!isPlainObject(input.context) && Object.prototype.hasOwnProperty.call(input, "context")) {
307
+ context.context = structuredClone(input.context);
308
+ }
309
+ return context;
310
+ }
311
+
312
+ function normalizeEvlogRecordResult(profile: EvlogStreamProfile, value: unknown): Result<PreparedJsonRecord, { message: string }> {
313
+ const objRes = expectPlainObjectResult(value, "evlog record");
314
+ if (Result.isError(objRes)) return objRes;
315
+ const input = objRes.value;
316
+
317
+ const status = normalizeOptionalInteger(input.status);
318
+ const duration = normalizeOptionalNumber(input.duration);
319
+ const timestamp = normalizeString(input.timestamp) ?? new Date().toISOString();
320
+ const requestId = normalizeRequestId(input, profile);
321
+ const traceparent = profile.correlation?.parseTraceparent === false ? null : parseTraceparent(input);
322
+ const traceId = normalizeConfiguredTraceField(input, profile, "traceId") ?? traceparent?.traceId ?? null;
323
+ const spanId = normalizeConfiguredTraceField(input, profile, "spanId") ?? traceparent?.spanId ?? null;
324
+ const contextRes = redactValue(buildContext(input), new Set([...DEFAULT_REDACT_KEYS, ...(profile.redactKeys ?? [])]));
325
+
326
+ const normalized = {
327
+ timestamp,
328
+ level: deriveLevel(input, status),
329
+ service: normalizeString(input.service),
330
+ environment: normalizeString(input.environment),
331
+ version: normalizeString(input.version),
332
+ region: normalizeString(input.region),
333
+ requestId,
334
+ traceId,
335
+ spanId,
336
+ method: normalizeString(input.method),
337
+ path: normalizeString(input.path),
338
+ status,
339
+ duration,
340
+ message: normalizeString(input.message),
341
+ why: normalizeString(input.why),
342
+ fix: normalizeString(input.fix),
343
+ link: normalizeString(input.link),
344
+ sampling: Object.prototype.hasOwnProperty.call(input, "sampling") ? structuredClone(input.sampling) : null,
345
+ redaction: { keys: contextRes.paths },
346
+ context: contextRes.value as Record<string, unknown>,
347
+ };
348
+
349
+ return Result.ok({
350
+ value: normalized,
351
+ routingKey: requestId ?? traceId ?? null,
352
+ });
353
+ }
354
+
355
+ function evlogSeverity(record: Record<string, unknown>): "debug" | "info" | "warn" | "error" {
356
+ const level = normalizeString(record.level)?.toLowerCase();
357
+ if (level === "debug" || level === "info" || level === "warn" || level === "error") return level;
358
+ const status = normalizeOptionalInteger(record.status);
359
+ if (status != null && status >= 500) return "error";
360
+ if (status != null && status >= 400) return "warn";
361
+ return "info";
362
+ }
363
+
364
+ function evlogTimelineItems(args: { stream: string; offset?: string; record: unknown }): UnifiedTimelineItem[] {
365
+ if (!isPlainObject(args.record)) return [];
366
+ const record = args.record;
367
+ const timestamp = normalizeString(record.timestamp);
368
+ if (!timestamp) return [];
369
+ const message = normalizeString(record.message);
370
+ const method = normalizeString(record.method);
371
+ const path = normalizeString(record.path);
372
+ const title = message ?? ([method, path].filter(Boolean).join(" ") || "evlog event");
373
+ return [
374
+ {
375
+ kind: "evlog.event",
376
+ time: timestamp,
377
+ duration: normalizeOptionalNumber(record.duration),
378
+ service: normalizeString(record.service),
379
+ title,
380
+ severity: evlogSeverity(record),
381
+ ids: {
382
+ requestId: normalizeString(record.requestId),
383
+ traceId: normalizeString(record.traceId),
384
+ spanId: normalizeString(record.spanId),
385
+ },
386
+ source: {
387
+ stream: args.stream,
388
+ offset: args.offset,
389
+ profile: "evlog",
390
+ },
391
+ data: record,
392
+ },
393
+ ];
394
+ }
395
+
396
+ export const EVLOG_STREAM_PROFILE_DEFINITION: StreamProfileDefinition = {
397
+ kind: "evlog",
398
+ usesStoredProfileRow: true,
399
+
400
+ defaultProfile(): EvlogStreamProfile {
401
+ return { kind: "evlog" };
402
+ },
403
+
404
+ validateResult(raw, path) {
405
+ return validateEvlogProfileResult(raw, path);
406
+ },
407
+
408
+ readProfileResult({ row, cached }): Result<StreamProfileReadResult, { message: string }> {
409
+ if (!row) return Result.ok({ profile: { kind: "evlog" }, cache: null });
410
+ const cachedCopy = cloneEvlogCache(cached);
411
+ if (cachedCopy && cachedCopy.updatedAtMs === row.updated_at_ms) {
412
+ return Result.ok({
413
+ profile: cloneEvlogProfile(cachedCopy.profile as EvlogStreamProfile),
414
+ cache: cachedCopy,
415
+ });
416
+ }
417
+ const parsedRes = parseStoredProfileJsonResult(row.profile_json);
418
+ if (Result.isError(parsedRes)) return parsedRes;
419
+ const profileRes = validateEvlogProfileResult(parsedRes.value, "profile");
420
+ if (Result.isError(profileRes)) return profileRes;
421
+ const profile = cloneEvlogProfile(profileRes.value);
422
+ return Result.ok({
423
+ profile: cloneEvlogProfile(profile),
424
+ cache: { profile, updatedAtMs: row.updated_at_ms },
425
+ });
426
+ },
427
+
428
+ persistProfileResult({ stream, streamRow, profile }): Result<StreamProfilePersistResult, { kind: "bad_request"; message: string; code?: string }> {
429
+ if (!isEvlogProfile(profile)) {
430
+ return Result.err({ kind: "bad_request", message: "invalid evlog profile" });
431
+ }
432
+ const contentType = normalizeProfileContentType(streamRow.content_type);
433
+ if (contentType !== "application/json") {
434
+ return Result.err({
435
+ kind: "bad_request",
436
+ message: "evlog profile requires application/json stream content-type",
437
+ });
438
+ }
439
+ if (streamRow.profile !== "evlog" && streamRow.next_offset > 0n) {
440
+ return Result.err({
441
+ kind: "bad_request",
442
+ message: "evlog profile must be installed before appending data",
443
+ });
444
+ }
445
+
446
+ const persistedProfile = cloneEvlogProfile(profile);
447
+ const registry = buildEvlogDefaultRegistry(stream);
448
+ return Result.ok({
449
+ profile: cloneEvlogProfile(persistedProfile),
450
+ cache: {
451
+ profile: persistedProfile,
452
+ updatedAtMs: 0n,
453
+ },
454
+ schemaRegistry: registry,
455
+ streamProfile: persistedProfile.kind,
456
+ profileJson: JSON.stringify(persistedProfile),
457
+ touchState: streamRow.profile === "state-protocol" ? "delete" : "preserve",
458
+ });
459
+ },
460
+
461
+ jsonIngest: {
462
+ prepareRecordResult({ profile, value }) {
463
+ if (!isEvlogProfile(profile)) return Result.err({ message: "invalid evlog profile" });
464
+ return normalizeEvlogRecordResult(profile, value);
465
+ },
466
+ },
467
+
468
+ correlation: {
469
+ toTimelineItems(args) {
470
+ return evlogTimelineItems(args);
471
+ },
472
+ },
473
+ };