@prisma/streams-server 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CONTRIBUTING.md +8 -0
  2. package/package.json +2 -1
  3. package/src/app.ts +290 -17
  4. package/src/app_core.ts +1833 -698
  5. package/src/app_local.ts +144 -4
  6. package/src/auto_tune.ts +62 -0
  7. package/src/bootstrap.ts +159 -1
  8. package/src/concurrency_gate.ts +108 -0
  9. package/src/config.ts +116 -14
  10. package/src/db/db.ts +1201 -131
  11. package/src/db/schema.ts +308 -8
  12. package/src/foreground_activity.ts +55 -0
  13. package/src/index/indexer.ts +254 -124
  14. package/src/index/lexicon_file_cache.ts +261 -0
  15. package/src/index/lexicon_format.ts +93 -0
  16. package/src/index/lexicon_indexer.ts +789 -0
  17. package/src/index/secondary_indexer.ts +824 -0
  18. package/src/index/secondary_schema.ts +105 -0
  19. package/src/ingest.ts +10 -12
  20. package/src/manifest.ts +143 -8
  21. package/src/memory.ts +183 -8
  22. package/src/metrics.ts +15 -29
  23. package/src/metrics_emitter.ts +26 -3
  24. package/src/notifier.ts +121 -5
  25. package/src/objectstore/accounting.ts +92 -0
  26. package/src/objectstore/mock_r2.ts +1 -1
  27. package/src/objectstore/r2.ts +17 -1
  28. package/src/profiles/evlog/schema.ts +234 -0
  29. package/src/profiles/evlog.ts +299 -0
  30. package/src/profiles/generic.ts +47 -0
  31. package/src/profiles/index.ts +205 -0
  32. package/src/profiles/metrics/block_format.ts +109 -0
  33. package/src/profiles/metrics/normalize.ts +366 -0
  34. package/src/profiles/metrics/schema.ts +319 -0
  35. package/src/profiles/metrics.ts +85 -0
  36. package/src/profiles/profile.ts +225 -0
  37. package/src/{touch/engine.ts → profiles/stateProtocol/changes.ts} +3 -20
  38. package/src/profiles/stateProtocol/routes.ts +389 -0
  39. package/src/profiles/stateProtocol/types.ts +6 -0
  40. package/src/profiles/stateProtocol/validation.ts +51 -0
  41. package/src/profiles/stateProtocol.ts +100 -0
  42. package/src/read_filter.ts +468 -0
  43. package/src/reader.ts +2151 -164
  44. package/src/runtime/host_runtime.ts +5 -0
  45. package/src/runtime_memory.ts +200 -0
  46. package/src/runtime_memory_sampler.ts +235 -0
  47. package/src/schema/read_json.ts +43 -0
  48. package/src/schema/registry.ts +563 -59
  49. package/src/search/agg_format.ts +638 -0
  50. package/src/search/aggregate.ts +389 -0
  51. package/src/search/binary/codec.ts +162 -0
  52. package/src/search/binary/docset.ts +67 -0
  53. package/src/search/binary/restart_strings.ts +181 -0
  54. package/src/search/binary/varint.ts +34 -0
  55. package/src/search/bitset.ts +19 -0
  56. package/src/search/col_format.ts +382 -0
  57. package/src/search/col_runtime.ts +59 -0
  58. package/src/search/column_encoding.ts +43 -0
  59. package/src/search/companion_file_cache.ts +319 -0
  60. package/src/search/companion_format.ts +313 -0
  61. package/src/search/companion_manager.ts +1086 -0
  62. package/src/search/companion_plan.ts +218 -0
  63. package/src/search/fts_format.ts +423 -0
  64. package/src/search/fts_runtime.ts +333 -0
  65. package/src/search/query.ts +875 -0
  66. package/src/search/schema.ts +245 -0
  67. package/src/segment/cache.ts +93 -2
  68. package/src/segment/cached_segment.ts +89 -0
  69. package/src/segment/format.ts +108 -36
  70. package/src/segment/segmenter.ts +79 -5
  71. package/src/segment/segmenter_worker.ts +35 -6
  72. package/src/segment/segmenter_workers.ts +42 -12
  73. package/src/server.ts +150 -36
  74. package/src/sqlite/adapter.ts +185 -14
  75. package/src/sqlite/runtime_stats.ts +163 -0
  76. package/src/stats.ts +3 -3
  77. package/src/stream_size_reconciler.ts +100 -0
  78. package/src/touch/canonical_change.ts +7 -0
  79. package/src/touch/live_metrics.ts +94 -64
  80. package/src/touch/live_templates.ts +15 -1
  81. package/src/touch/manager.ts +166 -88
  82. package/src/touch/{interpreter_worker.ts → processor_worker.ts} +19 -14
  83. package/src/touch/spec.ts +95 -92
  84. package/src/touch/touch_journal.ts +4 -0
  85. package/src/touch/worker_pool.ts +8 -14
  86. package/src/touch/worker_protocol.ts +3 -3
  87. package/src/uploader.ts +77 -6
  88. package/src/util/bloom256.ts +2 -2
  89. package/src/util/byte_lru.ts +73 -0
  90. package/src/util/lru.ts +8 -0
  91. package/src/util/stream_paths.ts +19 -0
@@ -7,24 +7,78 @@ import { DURABLE_LENS_V1_SCHEMA } from "./lens_schema";
7
7
  import { compileLensResult, lensFromJson, type CompiledLens, type Lens } from "../lens/lens";
8
8
  import { validateLensAgainstSchemasResult, fillLensDefaultsResult } from "./proof";
9
9
  import { parseJsonPointerResult } from "../util/json_pointer";
10
- import {
11
- isTouchEnabled,
12
- validateStreamInterpreterConfigResult,
13
- type StreamInterpreterConfig,
14
- } from "../touch/spec";
10
+ import { parseDurationMsResult } from "../util/duration";
15
11
  import { dsError } from "../util/ds_error.ts";
16
12
 
13
+ export const SCHEMA_REGISTRY_API_VERSION = "durable.streams/schema-registry/v1" as const;
14
+
17
15
  export type RoutingKeyConfig = {
18
16
  jsonPointer: string;
19
17
  required: boolean;
20
18
  };
21
19
 
20
+ export type SearchFieldKind = "keyword" | "text" | "integer" | "float" | "date" | "bool";
21
+
22
+ export type SearchFieldBinding = {
23
+ version: number;
24
+ jsonPointer: string;
25
+ };
26
+
27
+ export type SearchDefaultField = {
28
+ field: string;
29
+ boost?: number;
30
+ };
31
+
32
+ export type SearchFieldConfig = {
33
+ kind: SearchFieldKind;
34
+ bindings: SearchFieldBinding[];
35
+ normalizer?: "identity_v1" | "lowercase_v1";
36
+ analyzer?: "unicode_word_v1";
37
+ exact?: boolean;
38
+ prefix?: boolean;
39
+ column?: boolean;
40
+ exists?: boolean;
41
+ sortable?: boolean;
42
+ aggregatable?: boolean;
43
+ contains?: boolean;
44
+ positions?: boolean;
45
+ };
46
+
47
+ export type SearchRollupMeasureConfig =
48
+ | { kind: "count" }
49
+ | { kind: "summary"; field: string; histogram?: "log2_v1" }
50
+ | {
51
+ kind: "summary_parts";
52
+ countJsonPointer: string;
53
+ sumJsonPointer: string;
54
+ minJsonPointer: string;
55
+ maxJsonPointer: string;
56
+ histogramJsonPointer?: string;
57
+ };
58
+
59
+ export type SearchRollupConfig = {
60
+ timestampField?: string;
61
+ dimensions?: string[];
62
+ intervals: string[];
63
+ measures: Record<string, SearchRollupMeasureConfig>;
64
+ };
65
+
66
+ export type SearchConfig = {
67
+ profile?: string;
68
+ primaryTimestampField: string;
69
+ defaultFields?: SearchDefaultField[];
70
+ containsDefaultFields?: string[];
71
+ aliases?: Record<string, string>;
72
+ fields: Record<string, SearchFieldConfig>;
73
+ rollups?: Record<string, SearchRollupConfig>;
74
+ };
75
+
22
76
  export type SchemaRegistry = {
23
- apiVersion: "durable.streams/schema-registry/v1";
77
+ apiVersion: typeof SCHEMA_REGISTRY_API_VERSION;
24
78
  schema: string;
25
79
  currentVersion: number;
26
80
  routingKey?: RoutingKeyConfig;
27
- interpreter?: StreamInterpreterConfig;
81
+ search?: SearchConfig;
28
82
  boundaries: Array<{ offset: number; version: number }>;
29
83
  schemas: Record<string, any>;
30
84
  lenses: Record<string, any>;
@@ -53,6 +107,18 @@ const AJV = new Ajv({
53
107
  validateSchema: false,
54
108
  });
55
109
 
110
+ function isDateTimeString(value: string): boolean {
111
+ if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/.test(value)) {
112
+ return false;
113
+ }
114
+ return !Number.isNaN(Date.parse(value));
115
+ }
116
+
117
+ AJV.addFormat("date-time", {
118
+ type: "string",
119
+ validate: isDateTimeString,
120
+ });
121
+
56
122
  const LENS_VALIDATOR = AJV.compile(DURABLE_LENS_V1_SCHEMA);
57
123
 
58
124
  function sha256Hex(input: string): string {
@@ -61,7 +127,7 @@ function sha256Hex(input: string): string {
61
127
 
62
128
  function defaultRegistry(stream: string): SchemaRegistry {
63
129
  return {
64
- apiVersion: "durable.streams/schema-registry/v1",
130
+ apiVersion: SCHEMA_REGISTRY_API_VERSION,
65
131
  schema: stream,
66
132
  currentVersion: 0,
67
133
  boundaries: [],
@@ -85,6 +151,388 @@ function ensureNoRefResult(schema: any): Result<void, { message: string }> {
85
151
  return Result.ok(undefined);
86
152
  }
87
153
 
154
+ function isPlainObject(value: unknown): value is Record<string, unknown> {
155
+ return !!value && typeof value === "object" && !Array.isArray(value);
156
+ }
157
+
158
+ function rejectUnknownKeysResult(
159
+ obj: Record<string, unknown>,
160
+ allowed: readonly string[],
161
+ path: string
162
+ ): Result<void, { message: string }> {
163
+ const allowedSet = new Set(allowed);
164
+ for (const key of Object.keys(obj)) {
165
+ if (!allowedSet.has(key)) return Result.err({ message: `${path}.${key} is not supported` });
166
+ }
167
+ return Result.ok(undefined);
168
+ }
169
+
170
+ function parseRoutingKeyConfigResult(raw: unknown, path: string): Result<RoutingKeyConfig | null, { message: string }> {
171
+ if (raw == null) return Result.ok(null);
172
+ if (!isPlainObject(raw)) return Result.err({ message: `${path} must be an object or null` });
173
+ const keyCheck = rejectUnknownKeysResult(raw, ["jsonPointer", "required"], path);
174
+ if (Result.isError(keyCheck)) return keyCheck;
175
+ if (typeof raw.jsonPointer !== "string") return Result.err({ message: `${path}.jsonPointer must be a string` });
176
+ const pointerRes = parseJsonPointerResult(raw.jsonPointer);
177
+ if (Result.isError(pointerRes)) return Result.err({ message: pointerRes.error.message });
178
+ if (typeof raw.required !== "boolean") return Result.err({ message: `${path}.required must be boolean` });
179
+ return Result.ok({ jsonPointer: raw.jsonPointer, required: raw.required });
180
+ }
181
+
182
+ function validateSearchFieldNameResult(name: string, path: string): Result<string, { message: string }> {
183
+ const trimmed = name.trim();
184
+ if (trimmed === "") return Result.err({ message: `${path} must not be empty` });
185
+ if (trimmed.length > 64) return Result.err({ message: `${path} too long (max 64)` });
186
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(trimmed)) {
187
+ return Result.err({ message: `${path} must match ^[a-zA-Z0-9][a-zA-Z0-9._-]*$` });
188
+ }
189
+ return Result.ok(trimmed);
190
+ }
191
+
192
+ function parseSearchFieldBindingResult(raw: unknown, path: string): Result<SearchFieldBinding, { message: string }> {
193
+ if (!isPlainObject(raw)) return Result.err({ message: `${path} must be an object` });
194
+ const keyCheck = rejectUnknownKeysResult(raw, ["version", "jsonPointer"], path);
195
+ if (Result.isError(keyCheck)) return keyCheck;
196
+ if (typeof raw.version !== "number" || !Number.isFinite(raw.version) || raw.version <= 0 || !Number.isInteger(raw.version)) {
197
+ return Result.err({ message: `${path}.version must be a positive integer` });
198
+ }
199
+ if (typeof raw.jsonPointer !== "string") return Result.err({ message: `${path}.jsonPointer must be a string` });
200
+ const pointerRes = parseJsonPointerResult(raw.jsonPointer);
201
+ if (Result.isError(pointerRes)) return Result.err({ message: pointerRes.error.message });
202
+ return Result.ok({
203
+ version: raw.version,
204
+ jsonPointer: raw.jsonPointer,
205
+ });
206
+ }
207
+
208
+ function parseSearchDefaultFieldResult(raw: unknown, path: string): Result<SearchDefaultField, { message: string }> {
209
+ if (!isPlainObject(raw)) return Result.err({ message: `${path} must be an object` });
210
+ const keyCheck = rejectUnknownKeysResult(raw, ["field", "boost"], path);
211
+ if (Result.isError(keyCheck)) return keyCheck;
212
+ if (typeof raw.field !== "string") return Result.err({ message: `${path}.field must be a string` });
213
+ const fieldRes = validateSearchFieldNameResult(raw.field, `${path}.field`);
214
+ if (Result.isError(fieldRes)) return fieldRes;
215
+ if (raw.boost !== undefined && (typeof raw.boost !== "number" || !Number.isFinite(raw.boost) || raw.boost <= 0)) {
216
+ return Result.err({ message: `${path}.boost must be a positive number` });
217
+ }
218
+ return Result.ok({
219
+ field: fieldRes.value,
220
+ boost: typeof raw.boost === "number" ? raw.boost : undefined,
221
+ });
222
+ }
223
+
224
+ function parseSearchFieldConfigResult(raw: unknown, path: string): Result<SearchFieldConfig, { message: string }> {
225
+ if (!isPlainObject(raw)) return Result.err({ message: `${path} must be an object` });
226
+ const keyCheck = rejectUnknownKeysResult(
227
+ raw,
228
+ ["kind", "bindings", "normalizer", "analyzer", "exact", "prefix", "column", "exists", "sortable", "aggregatable", "contains", "positions"],
229
+ path
230
+ );
231
+ if (Result.isError(keyCheck)) return keyCheck;
232
+ if (
233
+ raw.kind !== "keyword" &&
234
+ raw.kind !== "text" &&
235
+ raw.kind !== "integer" &&
236
+ raw.kind !== "float" &&
237
+ raw.kind !== "date" &&
238
+ raw.kind !== "bool"
239
+ ) {
240
+ return Result.err({ message: `${path}.kind must be keyword, text, integer, float, date, or bool` });
241
+ }
242
+ if (!Array.isArray(raw.bindings) || raw.bindings.length === 0) {
243
+ return Result.err({ message: `${path}.bindings must be a non-empty array` });
244
+ }
245
+ const bindings: SearchFieldBinding[] = [];
246
+ const seenVersions = new Set<number>();
247
+ for (let i = 0; i < raw.bindings.length; i++) {
248
+ const bindingRes = parseSearchFieldBindingResult(raw.bindings[i], `${path}.bindings[${i}]`);
249
+ if (Result.isError(bindingRes)) return bindingRes;
250
+ if (seenVersions.has(bindingRes.value.version)) {
251
+ return Result.err({ message: `${path}.bindings[${i}].version duplicates ${bindingRes.value.version}` });
252
+ }
253
+ seenVersions.add(bindingRes.value.version);
254
+ bindings.push(bindingRes.value);
255
+ }
256
+ if (raw.normalizer !== undefined && raw.normalizer !== "identity_v1" && raw.normalizer !== "lowercase_v1") {
257
+ return Result.err({ message: `${path}.normalizer must be identity_v1 or lowercase_v1` });
258
+ }
259
+ if (raw.analyzer !== undefined && raw.analyzer !== "unicode_word_v1") {
260
+ return Result.err({ message: `${path}.analyzer must be unicode_word_v1` });
261
+ }
262
+ const out: SearchFieldConfig = {
263
+ kind: raw.kind,
264
+ bindings,
265
+ normalizer: raw.normalizer as SearchFieldConfig["normalizer"] | undefined,
266
+ analyzer: raw.analyzer as SearchFieldConfig["analyzer"] | undefined,
267
+ exact: raw.exact === true ? true : undefined,
268
+ prefix: raw.prefix === true ? true : undefined,
269
+ column: raw.column === true ? true : undefined,
270
+ exists: raw.exists === true ? true : undefined,
271
+ sortable: raw.sortable === true ? true : undefined,
272
+ aggregatable: raw.aggregatable === true ? true : undefined,
273
+ contains: raw.contains === true ? true : undefined,
274
+ positions: raw.positions === true ? true : undefined,
275
+ };
276
+ if (out.kind === "text") {
277
+ if (!out.analyzer) return Result.err({ message: `${path}.analyzer is required for text fields` });
278
+ if (out.column) return Result.err({ message: `${path}.column is not supported for text fields` });
279
+ if (out.sortable) return Result.err({ message: `${path}.sortable is not supported for text fields` });
280
+ if (out.aggregatable) return Result.err({ message: `${path}.aggregatable is not supported for text fields` });
281
+ } else {
282
+ if (out.positions) return Result.err({ message: `${path}.positions is only supported for text fields` });
283
+ }
284
+ if (out.kind === "keyword") {
285
+ if (out.analyzer) return Result.err({ message: `${path}.analyzer is not supported for keyword fields` });
286
+ }
287
+ if (out.kind === "integer" || out.kind === "float" || out.kind === "date" || out.kind === "bool") {
288
+ if (out.prefix) return Result.err({ message: `${path}.prefix is not supported for typed fields` });
289
+ if (out.contains) return Result.err({ message: `${path}.contains is not supported for typed fields` });
290
+ if (out.normalizer) return Result.err({ message: `${path}.normalizer is not supported for typed fields` });
291
+ }
292
+ return Result.ok(out);
293
+ }
294
+
295
+ function parseSearchRollupMeasureResult(
296
+ raw: unknown,
297
+ path: string,
298
+ fields: Record<string, SearchFieldConfig>
299
+ ): Result<SearchRollupMeasureConfig, { message: string }> {
300
+ if (!isPlainObject(raw)) return Result.err({ message: `${path} must be an object` });
301
+ if (raw.kind === "count") {
302
+ const keyCheck = rejectUnknownKeysResult(raw, ["kind"], path);
303
+ if (Result.isError(keyCheck)) return keyCheck;
304
+ return Result.ok({ kind: "count" });
305
+ }
306
+ if (raw.kind === "summary") {
307
+ const keyCheck = rejectUnknownKeysResult(raw, ["kind", "field", "histogram"], path);
308
+ if (Result.isError(keyCheck)) return keyCheck;
309
+ if (typeof raw.field !== "string") return Result.err({ message: `${path}.field must be a string` });
310
+ const fieldRes = validateSearchFieldNameResult(raw.field, `${path}.field`);
311
+ if (Result.isError(fieldRes)) return fieldRes;
312
+ const field = fields[fieldRes.value];
313
+ if (!field) return Result.err({ message: `${path}.field must reference a declared search field` });
314
+ if (!field.aggregatable) return Result.err({ message: `${path}.field must reference an aggregatable search field` });
315
+ if (field.kind !== "integer" && field.kind !== "float") {
316
+ return Result.err({ message: `${path}.field must reference an integer or float field` });
317
+ }
318
+ if (raw.histogram !== undefined && raw.histogram !== "log2_v1") {
319
+ return Result.err({ message: `${path}.histogram must be log2_v1` });
320
+ }
321
+ return Result.ok({
322
+ kind: "summary",
323
+ field: fieldRes.value,
324
+ histogram: raw.histogram === "log2_v1" ? "log2_v1" : undefined,
325
+ });
326
+ }
327
+ if (raw.kind === "summary_parts") {
328
+ const keyCheck = rejectUnknownKeysResult(
329
+ raw,
330
+ ["kind", "countJsonPointer", "sumJsonPointer", "minJsonPointer", "maxJsonPointer", "histogramJsonPointer"],
331
+ path
332
+ );
333
+ if (Result.isError(keyCheck)) return keyCheck;
334
+ for (const key of ["countJsonPointer", "sumJsonPointer", "minJsonPointer", "maxJsonPointer"] as const) {
335
+ if (typeof raw[key] !== "string") return Result.err({ message: `${path}.${key} must be a string` });
336
+ const pointerRes = parseJsonPointerResult(raw[key]);
337
+ if (Result.isError(pointerRes)) return Result.err({ message: pointerRes.error.message });
338
+ }
339
+ if (raw.histogramJsonPointer !== undefined) {
340
+ if (typeof raw.histogramJsonPointer !== "string") return Result.err({ message: `${path}.histogramJsonPointer must be a string` });
341
+ const pointerRes = parseJsonPointerResult(raw.histogramJsonPointer);
342
+ if (Result.isError(pointerRes)) return Result.err({ message: pointerRes.error.message });
343
+ }
344
+ return Result.ok({
345
+ kind: "summary_parts",
346
+ countJsonPointer: raw.countJsonPointer as string,
347
+ sumJsonPointer: raw.sumJsonPointer as string,
348
+ minJsonPointer: raw.minJsonPointer as string,
349
+ maxJsonPointer: raw.maxJsonPointer as string,
350
+ histogramJsonPointer: typeof raw.histogramJsonPointer === "string" ? raw.histogramJsonPointer : undefined,
351
+ });
352
+ }
353
+ return Result.err({ message: `${path}.kind must be count, summary, or summary_parts` });
354
+ }
355
+
356
+ function parseSearchRollupConfigResult(
357
+ raw: unknown,
358
+ path: string,
359
+ fields: Record<string, SearchFieldConfig>,
360
+ primaryTimestampField: string
361
+ ): Result<SearchRollupConfig, { message: string }> {
362
+ if (!isPlainObject(raw)) return Result.err({ message: `${path} must be an object` });
363
+ const keyCheck = rejectUnknownKeysResult(raw, ["timestampField", "dimensions", "intervals", "measures"], path);
364
+ if (Result.isError(keyCheck)) return keyCheck;
365
+
366
+ const timestampFieldRaw = raw.timestampField === undefined ? primaryTimestampField : raw.timestampField;
367
+ if (typeof timestampFieldRaw !== "string") return Result.err({ message: `${path}.timestampField must be a string` });
368
+ const timestampFieldRes = validateSearchFieldNameResult(timestampFieldRaw, `${path}.timestampField`);
369
+ if (Result.isError(timestampFieldRes)) return timestampFieldRes;
370
+ const timestampField = fields[timestampFieldRes.value];
371
+ if (!timestampField) return Result.err({ message: `${path}.timestampField must reference a declared field` });
372
+ if (timestampField.kind !== "date") return Result.err({ message: `${path}.timestampField must reference a date field` });
373
+
374
+ let dimensions: string[] | undefined;
375
+ if (raw.dimensions !== undefined) {
376
+ if (!Array.isArray(raw.dimensions)) return Result.err({ message: `${path}.dimensions must be an array` });
377
+ dimensions = [];
378
+ const seen = new Set<string>();
379
+ for (let i = 0; i < raw.dimensions.length; i++) {
380
+ if (typeof raw.dimensions[i] !== "string") return Result.err({ message: `${path}.dimensions[${i}] must be a string` });
381
+ const dimRes = validateSearchFieldNameResult(raw.dimensions[i], `${path}.dimensions[${i}]`);
382
+ if (Result.isError(dimRes)) return dimRes;
383
+ if (seen.has(dimRes.value)) return Result.err({ message: `${path}.dimensions[${i}] duplicates ${dimRes.value}` });
384
+ const field = fields[dimRes.value];
385
+ if (!field) return Result.err({ message: `${path}.dimensions[${i}] must reference a declared field` });
386
+ if (!field.exact) return Result.err({ message: `${path}.dimensions[${i}] must reference an exact-capable field` });
387
+ seen.add(dimRes.value);
388
+ dimensions.push(dimRes.value);
389
+ }
390
+ }
391
+
392
+ if (!Array.isArray(raw.intervals) || raw.intervals.length === 0) {
393
+ return Result.err({ message: `${path}.intervals must be a non-empty array` });
394
+ }
395
+ const intervals: string[] = [];
396
+ const seenIntervals = new Set<string>();
397
+ for (let i = 0; i < raw.intervals.length; i++) {
398
+ if (typeof raw.intervals[i] !== "string") return Result.err({ message: `${path}.intervals[${i}] must be a string` });
399
+ const parsedRes = parseDurationMsResult(raw.intervals[i]);
400
+ if (Result.isError(parsedRes) || parsedRes.value <= 0) {
401
+ return Result.err({ message: `${path}.intervals[${i}] must be a positive duration string` });
402
+ }
403
+ if (seenIntervals.has(raw.intervals[i])) {
404
+ return Result.err({ message: `${path}.intervals[${i}] duplicates ${raw.intervals[i]}` });
405
+ }
406
+ seenIntervals.add(raw.intervals[i]);
407
+ intervals.push(raw.intervals[i]);
408
+ }
409
+
410
+ if (!isPlainObject(raw.measures) || Object.keys(raw.measures).length === 0) {
411
+ return Result.err({ message: `${path}.measures must be a non-empty object` });
412
+ }
413
+ const measures: Record<string, SearchRollupMeasureConfig> = {};
414
+ for (const [measureName, measureRaw] of Object.entries(raw.measures)) {
415
+ const nameRes = validateSearchFieldNameResult(measureName, `${path}.measures`);
416
+ if (Result.isError(nameRes)) return nameRes;
417
+ const measureRes = parseSearchRollupMeasureResult(measureRaw, `${path}.measures.${measureName}`, fields);
418
+ if (Result.isError(measureRes)) return measureRes;
419
+ measures[nameRes.value] = measureRes.value;
420
+ }
421
+
422
+ return Result.ok({
423
+ timestampField: timestampFieldRes.value,
424
+ dimensions,
425
+ intervals,
426
+ measures,
427
+ });
428
+ }
429
+
430
+ function parseSearchConfigResult(raw: unknown, path: string): Result<SearchConfig | null, { message: string }> {
431
+ if (raw == null) return Result.ok(null);
432
+ if (!isPlainObject(raw)) return Result.err({ message: `${path} must be an object` });
433
+ const keyCheck = rejectUnknownKeysResult(
434
+ raw,
435
+ ["profile", "primaryTimestampField", "defaultFields", "containsDefaultFields", "aliases", "fields", "rollups"],
436
+ path
437
+ );
438
+ if (Result.isError(keyCheck)) return keyCheck;
439
+ if (typeof raw.primaryTimestampField !== "string") {
440
+ return Result.err({ message: `${path}.primaryTimestampField must be a string` });
441
+ }
442
+ const primaryFieldRes = validateSearchFieldNameResult(raw.primaryTimestampField, `${path}.primaryTimestampField`);
443
+ if (Result.isError(primaryFieldRes)) return primaryFieldRes;
444
+ if (!isPlainObject(raw.fields) || Object.keys(raw.fields).length === 0) {
445
+ return Result.err({ message: `${path}.fields must be a non-empty object` });
446
+ }
447
+ const fields: Record<string, SearchFieldConfig> = {};
448
+ for (const [fieldName, fieldRaw] of Object.entries(raw.fields)) {
449
+ const nameRes = validateSearchFieldNameResult(fieldName, `${path}.fields`);
450
+ if (Result.isError(nameRes)) return nameRes;
451
+ const fieldRes = parseSearchFieldConfigResult(fieldRaw, `${path}.fields.${fieldName}`);
452
+ if (Result.isError(fieldRes)) return fieldRes;
453
+ fields[nameRes.value] = fieldRes.value;
454
+ }
455
+ if (!fields[primaryFieldRes.value]) {
456
+ return Result.err({ message: `${path}.primaryTimestampField must reference a declared field` });
457
+ }
458
+ if (fields[primaryFieldRes.value].kind !== "date") {
459
+ return Result.err({ message: `${path}.primaryTimestampField must reference a date field` });
460
+ }
461
+ let defaultFields: SearchDefaultField[] | undefined;
462
+ if (raw.defaultFields !== undefined) {
463
+ if (!Array.isArray(raw.defaultFields)) return Result.err({ message: `${path}.defaultFields must be an array` });
464
+ defaultFields = [];
465
+ for (let i = 0; i < raw.defaultFields.length; i++) {
466
+ const fieldRes = parseSearchDefaultFieldResult(raw.defaultFields[i], `${path}.defaultFields[${i}]`);
467
+ if (Result.isError(fieldRes)) return fieldRes;
468
+ if (!fields[fieldRes.value.field]) {
469
+ return Result.err({ message: `${path}.defaultFields[${i}].field must reference a declared field` });
470
+ }
471
+ defaultFields.push(fieldRes.value);
472
+ }
473
+ }
474
+ let containsDefaultFields: string[] | undefined;
475
+ if (raw.containsDefaultFields !== undefined) {
476
+ if (!Array.isArray(raw.containsDefaultFields)) {
477
+ return Result.err({ message: `${path}.containsDefaultFields must be an array` });
478
+ }
479
+ containsDefaultFields = [];
480
+ for (let i = 0; i < raw.containsDefaultFields.length; i++) {
481
+ if (typeof raw.containsDefaultFields[i] !== "string") {
482
+ return Result.err({ message: `${path}.containsDefaultFields[${i}] must be a string` });
483
+ }
484
+ const nameRes = validateSearchFieldNameResult(raw.containsDefaultFields[i], `${path}.containsDefaultFields[${i}]`);
485
+ if (Result.isError(nameRes)) return nameRes;
486
+ if (!fields[nameRes.value]) {
487
+ return Result.err({ message: `${path}.containsDefaultFields[${i}] must reference a declared field` });
488
+ }
489
+ containsDefaultFields.push(nameRes.value);
490
+ }
491
+ }
492
+ let aliases: Record<string, string> | undefined;
493
+ if (raw.aliases !== undefined) {
494
+ if (!isPlainObject(raw.aliases)) return Result.err({ message: `${path}.aliases must be an object` });
495
+ aliases = {};
496
+ for (const [aliasRaw, targetRaw] of Object.entries(raw.aliases)) {
497
+ const aliasRes = validateSearchFieldNameResult(aliasRaw, `${path}.aliases`);
498
+ if (Result.isError(aliasRes)) return aliasRes;
499
+ if (typeof targetRaw !== "string") return Result.err({ message: `${path}.aliases.${aliasRaw} must be a string` });
500
+ const targetRes = validateSearchFieldNameResult(targetRaw, `${path}.aliases.${aliasRaw}`);
501
+ if (Result.isError(targetRes)) return targetRes;
502
+ if (!fields[targetRes.value]) {
503
+ return Result.err({ message: `${path}.aliases.${aliasRaw} must reference a declared field` });
504
+ }
505
+ aliases[aliasRes.value] = targetRes.value;
506
+ }
507
+ }
508
+ let rollups: Record<string, SearchRollupConfig> | undefined;
509
+ if (raw.rollups !== undefined) {
510
+ if (!isPlainObject(raw.rollups)) return Result.err({ message: `${path}.rollups must be an object` });
511
+ rollups = {};
512
+ for (const [rollupName, rollupRaw] of Object.entries(raw.rollups)) {
513
+ const nameRes = validateSearchFieldNameResult(rollupName, `${path}.rollups`);
514
+ if (Result.isError(nameRes)) return nameRes;
515
+ const rollupRes = parseSearchRollupConfigResult(
516
+ rollupRaw,
517
+ `${path}.rollups.${rollupName}`,
518
+ fields,
519
+ primaryFieldRes.value
520
+ );
521
+ if (Result.isError(rollupRes)) return rollupRes;
522
+ rollups[nameRes.value] = rollupRes.value;
523
+ }
524
+ }
525
+ return Result.ok({
526
+ profile: typeof raw.profile === "string" ? raw.profile : undefined,
527
+ primaryTimestampField: primaryFieldRes.value,
528
+ defaultFields,
529
+ containsDefaultFields,
530
+ aliases,
531
+ fields,
532
+ rollups,
533
+ });
534
+ }
535
+
88
536
  function validateJsonSchemaResult(schema: any): Result<void, { message: string }> {
89
537
  const noRefRes = ensureNoRefResult(schema);
90
538
  if (Result.isError(noRefRes)) return noRefRes;
@@ -98,22 +546,52 @@ function validateJsonSchemaResult(schema: any): Result<void, { message: string }
98
546
  }
99
547
 
100
548
  function parseRegistryResult(stream: string, json: string): Result<SchemaRegistry, { message: string }> {
101
- let raw: any;
549
+ let raw: unknown;
102
550
  try {
103
551
  raw = JSON.parse(json);
104
552
  } catch (e: any) {
105
553
  return Result.err({ message: String(e?.message ?? e) });
106
554
  }
107
- if (!raw || typeof raw !== "object") return Result.err({ message: "invalid schema registry" });
108
- const reg = raw as SchemaRegistry;
109
- if (reg.apiVersion !== "durable.streams/schema-registry/v1") return Result.err({ message: "invalid registry apiVersion" });
110
- if (!reg.schema) reg.schema = stream;
111
- if (!Array.isArray(reg.boundaries)) reg.boundaries = [];
112
- if (!reg.schemas || typeof reg.schemas !== "object") reg.schemas = {};
113
- if (!reg.lenses || typeof reg.lenses !== "object") reg.lenses = {};
114
- if (typeof reg.currentVersion !== "number") reg.currentVersion = 0;
115
- if ((reg as any).interpreter === null) delete (reg as any).interpreter;
116
- return Result.ok(reg);
555
+ if (!isPlainObject(raw)) return Result.err({ message: "invalid schema registry" });
556
+ const keyCheck = rejectUnknownKeysResult(
557
+ raw,
558
+ ["apiVersion", "schema", "currentVersion", "routingKey", "search", "boundaries", "schemas", "lenses"],
559
+ "registry"
560
+ );
561
+ if (Result.isError(keyCheck)) return keyCheck;
562
+ if (raw.apiVersion !== SCHEMA_REGISTRY_API_VERSION) return Result.err({ message: "invalid registry apiVersion" });
563
+
564
+ const routingKeyRes = parseRoutingKeyConfigResult(raw.routingKey, "routingKey");
565
+ if (Result.isError(routingKeyRes)) return routingKeyRes;
566
+ const searchRes = parseSearchConfigResult(raw.search, "search");
567
+ if (Result.isError(searchRes)) return searchRes;
568
+
569
+ const boundariesRaw = Array.isArray(raw.boundaries) ? raw.boundaries : [];
570
+ const boundaries: Array<{ offset: number; version: number }> = [];
571
+ for (const item of boundariesRaw) {
572
+ if (!isPlainObject(item)) return Result.err({ message: "invalid boundary entry" });
573
+ const offset = typeof item.offset === "number" && Number.isFinite(item.offset) ? item.offset : null;
574
+ const version = typeof item.version === "number" && Number.isFinite(item.version) ? item.version : null;
575
+ if (offset == null || version == null) return Result.err({ message: "invalid boundary entry" });
576
+ boundaries.push({ offset, version });
577
+ }
578
+
579
+ const schemas = isPlainObject(raw.schemas) ? raw.schemas : {};
580
+ const lenses = isPlainObject(raw.lenses) ? raw.lenses : {};
581
+ const currentVersion =
582
+ typeof raw.currentVersion === "number" && Number.isFinite(raw.currentVersion) ? raw.currentVersion : 0;
583
+ const schemaName = typeof raw.schema === "string" && raw.schema.trim() !== "" ? raw.schema : stream;
584
+
585
+ return Result.ok({
586
+ apiVersion: SCHEMA_REGISTRY_API_VERSION,
587
+ schema: schemaName,
588
+ currentVersion,
589
+ routingKey: routingKeyRes.value ?? undefined,
590
+ search: searchRes.value ?? undefined,
591
+ boundaries,
592
+ schemas,
593
+ lenses,
594
+ });
117
595
  }
118
596
 
119
597
  function serializeRegistry(reg: SchemaRegistry): string {
@@ -129,6 +607,43 @@ function validateLensResult(raw: any): Result<Lens, { message: string }> {
129
607
  return Result.ok(raw as Lens);
130
608
  }
131
609
 
610
+ export function parseSchemaUpdateResult(
611
+ body: unknown
612
+ ): Result<{ schema?: any; lens?: any; routingKey?: RoutingKeyConfig | null; search?: SearchConfig | null }, { message: string }> {
613
+ if (!isPlainObject(body)) return Result.err({ message: "schema update must be a JSON object" });
614
+ const keyCheck = rejectUnknownKeysResult(body, ["apiVersion", "schema", "lens", "routingKey", "search"], "schemaUpdate");
615
+ if (Result.isError(keyCheck)) return keyCheck;
616
+ if (body.apiVersion !== undefined && body.apiVersion !== SCHEMA_REGISTRY_API_VERSION) {
617
+ return Result.err({ message: "invalid schema apiVersion" });
618
+ }
619
+
620
+ const hasSchema = Object.prototype.hasOwnProperty.call(body, "schema");
621
+ const hasRoutingKey = Object.prototype.hasOwnProperty.call(body, "routingKey");
622
+ const hasSearch = Object.prototype.hasOwnProperty.call(body, "search");
623
+ if (!hasSchema && !hasRoutingKey && !hasSearch) {
624
+ return Result.err({ message: "schema update must include schema, routingKey, or search" });
625
+ }
626
+ if (!hasSchema && body.lens !== undefined) {
627
+ return Result.err({ message: "schema update lens requires schema" });
628
+ }
629
+
630
+ const routingKeyRes = hasRoutingKey ? parseRoutingKeyConfigResult(body.routingKey, "routingKey") : Result.ok(null);
631
+ if (Result.isError(routingKeyRes)) return routingKeyRes;
632
+ if (hasSchema && hasRoutingKey && routingKeyRes.value == null) {
633
+ return Result.err({ message: "schema update routingKey must be an object when schema is provided" });
634
+ }
635
+
636
+ const searchRes = hasSearch ? parseSearchConfigResult(body.search, "search") : Result.ok(null);
637
+ if (Result.isError(searchRes)) return searchRes;
638
+
639
+ const out: { schema?: any; lens?: any; routingKey?: RoutingKeyConfig | null; search?: SearchConfig | null } = {};
640
+ if (hasSchema) out.schema = body.schema;
641
+ if (body.lens !== undefined) out.lens = body.lens;
642
+ if (hasRoutingKey) out.routingKey = routingKeyRes.value;
643
+ if (hasSearch) out.search = searchRes.value;
644
+ return Result.ok(out);
645
+ }
646
+
132
647
  function bigintToNumberSafeResult(v: bigint): Result<number, { message: string }> {
133
648
  const max = BigInt(Number.MAX_SAFE_INTEGER);
134
649
  if (v > max) return Result.err({ message: "offset exceeds MAX_SAFE_INTEGER" });
@@ -177,7 +692,7 @@ export class SchemaRegistryStore {
177
692
  updateRegistry(
178
693
  stream: string,
179
694
  streamRow: StreamRow,
180
- update: { schema: any; lens?: any; routingKey?: RoutingKeyConfig; interpreter?: StreamInterpreterConfig | null }
695
+ update: { schema: any; lens?: any; routingKey?: RoutingKeyConfig; search?: SearchConfig | null }
181
696
  ): SchemaRegistry {
182
697
  const res = this.updateRegistryResult(stream, streamRow, update);
183
698
  if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
@@ -187,9 +702,8 @@ export class SchemaRegistryStore {
187
702
  updateRegistryResult(
188
703
  stream: string,
189
704
  streamRow: StreamRow,
190
- update: { schema: any; lens?: any; routingKey?: RoutingKeyConfig; interpreter?: StreamInterpreterConfig | null }
705
+ update: { schema: any; lens?: any; routingKey?: RoutingKeyConfig; search?: SearchConfig | null }
191
706
  ): Result<SchemaRegistry, SchemaRegistryMutationError> {
192
- let validatedInterpreter: StreamInterpreterConfig | null | undefined = undefined;
193
707
  if (update.routingKey) {
194
708
  const pointerRes = parseJsonPointerResult(update.routingKey.jsonPointer);
195
709
  if (Result.isError(pointerRes)) {
@@ -199,17 +713,6 @@ export class SchemaRegistryStore {
199
713
  return Result.err({ kind: "bad_request", message: "routingKey.required must be boolean" });
200
714
  }
201
715
  }
202
- if (update.interpreter !== undefined) {
203
- if (update.interpreter === null) {
204
- validatedInterpreter = null;
205
- } else {
206
- const interpreterRes = validateStreamInterpreterConfigResult(update.interpreter);
207
- if (Result.isError(interpreterRes)) {
208
- return Result.err({ kind: "bad_request", message: interpreterRes.error.message });
209
- }
210
- validatedInterpreter = interpreterRes.value;
211
- }
212
- }
213
716
  if (update.schema === undefined) return Result.err({ kind: "bad_request", message: "missing schema" });
214
717
  const schemaRes = validateJsonSchemaResult(update.schema);
215
718
  if (Result.isError(schemaRes)) return Result.err({ kind: "bad_request", message: schemaRes.error.message });
@@ -238,13 +741,12 @@ export class SchemaRegistryStore {
238
741
  schema: stream,
239
742
  currentVersion: 1,
240
743
  routingKey: update.routingKey,
241
- interpreter: update.interpreter === undefined ? reg.interpreter : validatedInterpreter ?? undefined,
744
+ search: update.search === undefined ? reg.search : update.search ?? undefined,
242
745
  boundaries: [{ offset: 0, version: 1 }],
243
746
  schemas: { ...reg.schemas, ["1"]: update.schema },
244
747
  lenses: { ...reg.lenses },
245
748
  };
246
749
  this.persist(stream, nextReg);
247
- this.syncInterpreterState(stream, nextReg);
248
750
  return Result.ok(nextReg);
249
751
  }
250
752
 
@@ -277,13 +779,12 @@ export class SchemaRegistryStore {
277
779
  schema: reg.schema ?? stream,
278
780
  currentVersion: nextVersion,
279
781
  routingKey: update.routingKey ?? reg.routingKey,
280
- interpreter: update.interpreter === undefined ? reg.interpreter : validatedInterpreter ?? undefined,
782
+ search: update.search === undefined ? reg.search : update.search ?? undefined,
281
783
  boundaries: [...reg.boundaries, { offset: boundaryRes.value, version: nextVersion }],
282
784
  schemas: { ...reg.schemas, [String(nextVersion)]: update.schema },
283
785
  lenses: { ...reg.lenses, [String(currentVersion)]: defaultsRes.value },
284
786
  };
285
787
  this.persist(stream, nextReg);
286
- this.syncInterpreterState(stream, nextReg);
287
788
  return Result.ok(nextReg);
288
789
  }
289
790
 
@@ -310,50 +811,53 @@ export class SchemaRegistryStore {
310
811
  routingKey: routingKey ?? undefined,
311
812
  };
312
813
  this.persist(stream, nextReg);
313
- this.syncInterpreterState(stream, nextReg);
314
814
  return Result.ok(nextReg);
315
815
  }
316
816
 
317
- updateInterpreter(stream: string, interpreter: StreamInterpreterConfig | null): SchemaRegistry {
318
- const res = this.updateInterpreterResult(stream, interpreter);
817
+ updateSearch(stream: string, search: SearchConfig | null): SchemaRegistry {
818
+ const res = this.updateSearchResult(stream, search);
319
819
  if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
320
820
  return res.value;
321
821
  }
322
822
 
323
- updateInterpreterResult(stream: string, interpreter: StreamInterpreterConfig | null): Result<SchemaRegistry, SchemaRegistryMutationError> {
324
- let validatedInterpreter: StreamInterpreterConfig | null = interpreter;
325
- if (interpreter) {
326
- const interpreterRes = validateStreamInterpreterConfigResult(interpreter);
327
- if (Result.isError(interpreterRes)) {
328
- return Result.err({ kind: "bad_request", message: interpreterRes.error.message });
329
- }
330
- validatedInterpreter = interpreterRes.value;
331
- }
823
+ updateSearchResult(stream: string, search: SearchConfig | null): Result<SchemaRegistry, SchemaRegistryMutationError> {
824
+ const searchRes = parseSearchConfigResult(search, "search");
825
+ if (Result.isError(searchRes)) return Result.err({ kind: "bad_request", message: searchRes.error.message });
332
826
  const regRes = this.getRegistryResult(stream);
333
827
  if (Result.isError(regRes)) return Result.err({ kind: "bad_request", message: regRes.error.message, code: regRes.error.code });
828
+ if (searchRes.value && (regRes.value.currentVersion <= 0 || regRes.value.boundaries.length === 0)) {
829
+ return Result.err({
830
+ kind: "bad_request",
831
+ message: "search config requires an installed schema version",
832
+ });
833
+ }
334
834
  const nextReg: SchemaRegistry = {
335
835
  ...regRes.value,
336
- interpreter: validatedInterpreter ?? undefined,
836
+ search: searchRes.value ?? undefined,
337
837
  };
338
838
  this.persist(stream, nextReg);
339
- this.syncInterpreterState(stream, nextReg);
340
839
  return Result.ok(nextReg);
341
840
  }
342
841
 
842
+ replaceRegistry(stream: string, registry: SchemaRegistry): SchemaRegistry {
843
+ const res = this.replaceRegistryResult(stream, registry);
844
+ if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
845
+ return res.value;
846
+ }
847
+
848
+ replaceRegistryResult(stream: string, registry: SchemaRegistry): Result<SchemaRegistry, SchemaRegistryMutationError> {
849
+ const parseRes = parseRegistryResult(stream, JSON.stringify(registry));
850
+ if (Result.isError(parseRes)) return Result.err({ kind: "bad_request", message: parseRes.error.message });
851
+ this.persist(stream, parseRes.value);
852
+ return Result.ok(parseRes.value);
853
+ }
854
+
343
855
  private persist(stream: string, reg: SchemaRegistry): void {
344
856
  const json = serializeRegistry(reg);
345
857
  this.db.upsertSchemaRegistry(stream, json);
346
858
  this.registryCache.set(stream, { reg, updatedAtMs: this.db.nowMs() });
347
859
  }
348
860
 
349
- private syncInterpreterState(stream: string, reg: SchemaRegistry): void {
350
- if (isTouchEnabled(reg.interpreter)) {
351
- this.db.ensureStreamInterpreter(stream);
352
- } else {
353
- this.db.deleteStreamInterpreter(stream);
354
- }
355
- }
356
-
357
861
  getValidatorForVersion(reg: SchemaRegistry, version: number): Validator | null {
358
862
  const schema = reg.schemas[String(version)];
359
863
  if (!schema) return null;