@tungthedev/streams-server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CODE_OF_CONDUCT.md +45 -0
  2. package/CONTRIBUTING.md +76 -0
  3. package/LICENSE +201 -0
  4. package/README.md +58 -0
  5. package/SECURITY.md +42 -0
  6. package/bin/prisma-streams-server +2 -0
  7. package/package.json +46 -0
  8. package/src/app.ts +583 -0
  9. package/src/app_core.ts +3144 -0
  10. package/src/app_local.ts +206 -0
  11. package/src/auth.ts +124 -0
  12. package/src/auto_tune.ts +69 -0
  13. package/src/backpressure.ts +66 -0
  14. package/src/bootstrap.ts +613 -0
  15. package/src/compute/demo_entry.ts +415 -0
  16. package/src/compute/demo_site.ts +1242 -0
  17. package/src/compute/entry.ts +19 -0
  18. package/src/compute/package_entry.ts +4 -0
  19. package/src/compute/virtual-modules.d.ts +15 -0
  20. package/src/compute/worker_module_url.ts +9 -0
  21. package/src/concurrency_gate.ts +108 -0
  22. package/src/config.ts +402 -0
  23. package/src/db/bootstrap_store.ts +9 -0
  24. package/src/db/db.ts +2424 -0
  25. package/src/db/schema.ts +925 -0
  26. package/src/db/sqlite_manifest_snapshot.ts +81 -0
  27. package/src/db/sqlite_touch_store.ts +491 -0
  28. package/src/db/sqlite_wal_store.ts +472 -0
  29. package/src/details/full_mode_details.ts +568 -0
  30. package/src/expiry_sweeper.ts +47 -0
  31. package/src/foreground_activity.ts +55 -0
  32. package/src/hist.ts +169 -0
  33. package/src/index/binary_fuse.ts +379 -0
  34. package/src/index/indexer.ts +947 -0
  35. package/src/index/lexicon_file_cache.ts +261 -0
  36. package/src/index/lexicon_format.ts +93 -0
  37. package/src/index/lexicon_indexer.ts +863 -0
  38. package/src/index/run_cache.ts +84 -0
  39. package/src/index/run_format.ts +213 -0
  40. package/src/index/schedule.ts +28 -0
  41. package/src/index/secondary_indexer.ts +901 -0
  42. package/src/index/secondary_schema.ts +105 -0
  43. package/src/ingest.ts +309 -0
  44. package/src/lens/lens.ts +501 -0
  45. package/src/manifest.ts +249 -0
  46. package/src/memory.ts +334 -0
  47. package/src/metrics.ts +147 -0
  48. package/src/metrics_emitter.ts +83 -0
  49. package/src/notifier.ts +180 -0
  50. package/src/objectstore/accounting.ts +151 -0
  51. package/src/objectstore/interface.ts +13 -0
  52. package/src/objectstore/mock_r2.ts +269 -0
  53. package/src/objectstore/null.ts +32 -0
  54. package/src/objectstore/r2.ts +318 -0
  55. package/src/observe/pairing.ts +61 -0
  56. package/src/observe/request.ts +772 -0
  57. package/src/offset.ts +70 -0
  58. package/src/postgres/bootstrap.ts +269 -0
  59. package/src/postgres/companions.ts +197 -0
  60. package/src/postgres/control_restore.ts +109 -0
  61. package/src/postgres/details.ts +189 -0
  62. package/src/postgres/lexicon_index.ts +260 -0
  63. package/src/postgres/routing_index.ts +189 -0
  64. package/src/postgres/rows.ts +132 -0
  65. package/src/postgres/schema.ts +355 -0
  66. package/src/postgres/secondary_index.ts +238 -0
  67. package/src/postgres/segments.ts +900 -0
  68. package/src/postgres/stats.ts +103 -0
  69. package/src/postgres/store.ts +947 -0
  70. package/src/postgres/touch.ts +591 -0
  71. package/src/postgres/types.ts +32 -0
  72. package/src/profiles/evlog/schema.ts +234 -0
  73. package/src/profiles/evlog.ts +473 -0
  74. package/src/profiles/generic.ts +51 -0
  75. package/src/profiles/index.ts +237 -0
  76. package/src/profiles/metrics/block_format.ts +109 -0
  77. package/src/profiles/metrics/normalize.ts +366 -0
  78. package/src/profiles/metrics/schema.ts +319 -0
  79. package/src/profiles/metrics.ts +83 -0
  80. package/src/profiles/otelTraces/normalize.ts +955 -0
  81. package/src/profiles/otelTraces/otlp.ts +1002 -0
  82. package/src/profiles/otelTraces/schema.ts +408 -0
  83. package/src/profiles/otelTraces.ts +390 -0
  84. package/src/profiles/profile.ts +284 -0
  85. package/src/profiles/stateProtocol/change_event_conformance.typecheck.ts +35 -0
  86. package/src/profiles/stateProtocol/changes.ts +24 -0
  87. package/src/profiles/stateProtocol/ingest.ts +115 -0
  88. package/src/profiles/stateProtocol/routes.ts +511 -0
  89. package/src/profiles/stateProtocol/types.ts +6 -0
  90. package/src/profiles/stateProtocol/validation.ts +51 -0
  91. package/src/profiles/stateProtocol.ts +107 -0
  92. package/src/read_filter.ts +468 -0
  93. package/src/reader.ts +2986 -0
  94. package/src/runtime/hash.ts +156 -0
  95. package/src/runtime/hash_vendor/LICENSE.hash-wasm +38 -0
  96. package/src/runtime/hash_vendor/NOTICE.md +8 -0
  97. package/src/runtime/hash_vendor/xxhash3.umd.min.cjs +7 -0
  98. package/src/runtime/hash_vendor/xxhash32.umd.min.cjs +7 -0
  99. package/src/runtime/hash_vendor/xxhash64.umd.min.cjs +7 -0
  100. package/src/runtime/host_runtime.ts +5 -0
  101. package/src/runtime_memory.ts +200 -0
  102. package/src/runtime_memory_sampler.ts +237 -0
  103. package/src/schema/lens_schema.ts +290 -0
  104. package/src/schema/proof.ts +547 -0
  105. package/src/schema/read_json.ts +51 -0
  106. package/src/schema/registry.ts +966 -0
  107. package/src/search/agg_format.ts +638 -0
  108. package/src/search/aggregate.ts +409 -0
  109. package/src/search/binary/codec.ts +162 -0
  110. package/src/search/binary/docset.ts +67 -0
  111. package/src/search/binary/restart_strings.ts +181 -0
  112. package/src/search/binary/varint.ts +34 -0
  113. package/src/search/bitset.ts +19 -0
  114. package/src/search/col_format.ts +382 -0
  115. package/src/search/col_runtime.ts +59 -0
  116. package/src/search/column_encoding.ts +43 -0
  117. package/src/search/companion_file_cache.ts +319 -0
  118. package/src/search/companion_format.ts +327 -0
  119. package/src/search/companion_manager.ts +1305 -0
  120. package/src/search/companion_plan.ts +229 -0
  121. package/src/search/exact_format.ts +281 -0
  122. package/src/search/exact_runtime.ts +55 -0
  123. package/src/search/fts_format.ts +423 -0
  124. package/src/search/fts_runtime.ts +333 -0
  125. package/src/search/query.ts +875 -0
  126. package/src/search/schema.ts +245 -0
  127. package/src/segment/cache.ts +270 -0
  128. package/src/segment/cached_segment.ts +89 -0
  129. package/src/segment/format.ts +403 -0
  130. package/src/segment/segmenter.ts +412 -0
  131. package/src/segment/segmenter_worker.ts +72 -0
  132. package/src/segment/segmenter_workers.ts +130 -0
  133. package/src/server.ts +264 -0
  134. package/src/server_auto_tune.ts +158 -0
  135. package/src/sqlite/adapter.ts +335 -0
  136. package/src/sqlite/runtime_stats.ts +163 -0
  137. package/src/stats.ts +205 -0
  138. package/src/store/append.ts +50 -0
  139. package/src/store/bootstrap_restore_store.ts +71 -0
  140. package/src/store/capabilities.ts +86 -0
  141. package/src/store/full_mode_details_store.ts +71 -0
  142. package/src/store/index_store.ts +104 -0
  143. package/src/store/profile_touch_store.ts +1 -0
  144. package/src/store/rows.ts +144 -0
  145. package/src/store/schema_profile_store.ts +73 -0
  146. package/src/store/schema_publication.ts +6 -0
  147. package/src/store/segment_manifest_store.ts +129 -0
  148. package/src/store/segment_read_store.ts +22 -0
  149. package/src/store/stats_accounting_store.ts +83 -0
  150. package/src/store/touch_store.ts +98 -0
  151. package/src/store/wal_store.ts +21 -0
  152. package/src/stream_size_reconciler.ts +100 -0
  153. package/src/touch/canonical_change.ts +7 -0
  154. package/src/touch/live_keys.ts +158 -0
  155. package/src/touch/live_metrics.ts +841 -0
  156. package/src/touch/live_templates.ts +449 -0
  157. package/src/touch/manager.ts +1292 -0
  158. package/src/touch/process_batch.ts +576 -0
  159. package/src/touch/processor_worker.ts +85 -0
  160. package/src/touch/spec.ts +459 -0
  161. package/src/touch/touch_journal.ts +771 -0
  162. package/src/touch/touch_key_id.ts +20 -0
  163. package/src/touch/worker_pool.ts +191 -0
  164. package/src/touch/worker_protocol.ts +57 -0
  165. package/src/types/proper-lockfile.d.ts +1 -0
  166. package/src/uploader.ts +358 -0
  167. package/src/util/base32_crockford.ts +81 -0
  168. package/src/util/bloom256.ts +67 -0
  169. package/src/util/byte_lru.ts +73 -0
  170. package/src/util/cleanup.ts +22 -0
  171. package/src/util/crc32c.ts +29 -0
  172. package/src/util/ds_error.ts +15 -0
  173. package/src/util/duration.ts +17 -0
  174. package/src/util/endian.ts +53 -0
  175. package/src/util/json_pointer.ts +148 -0
  176. package/src/util/log.ts +25 -0
  177. package/src/util/lru.ts +53 -0
  178. package/src/util/retry.ts +35 -0
  179. package/src/util/siphash.ts +71 -0
  180. package/src/util/stream_paths.ts +50 -0
  181. package/src/util/time.ts +14 -0
  182. package/src/util/yield.ts +3 -0
  183. package/src/util/zstd.ts +24 -0
@@ -0,0 +1,6 @@
1
+ import type { TouchConfig } from "../../touch/spec";
2
+
3
+ export type StateProtocolStreamProfile = {
4
+ kind: "state-protocol";
5
+ touch?: TouchConfig;
6
+ };
@@ -0,0 +1,51 @@
1
+ import { Result } from "better-result";
2
+ import type { CachedStreamProfile, StreamProfileSpec } from "../profile";
3
+ import {
4
+ cloneStreamProfileSpec,
5
+ expectPlainObjectResult,
6
+ rejectUnknownKeysResult,
7
+ } from "../profile";
8
+ import { validateTouchConfigResult, type TouchConfig } from "../../touch/spec";
9
+ import type { StateProtocolStreamProfile } from "./types";
10
+
11
+ export function isStateProtocolProfile(
12
+ profile: StreamProfileSpec | null | undefined
13
+ ): profile is StateProtocolStreamProfile {
14
+ return !!profile && profile.kind === "state-protocol";
15
+ }
16
+
17
+ export function getStateProtocolTouchConfig(profile: StreamProfileSpec | null | undefined): TouchConfig | null {
18
+ return isStateProtocolProfile(profile) && profile.touch?.enabled ? profile.touch : null;
19
+ }
20
+
21
+ export function cloneStateProtocolProfile(profile: StateProtocolStreamProfile): StateProtocolStreamProfile {
22
+ return cloneStreamProfileSpec(profile) as StateProtocolStreamProfile;
23
+ }
24
+
25
+ export function cloneStateProtocolCache(cache: CachedStreamProfile | null): CachedStreamProfile | null {
26
+ if (!cache || cache.profile.kind !== "state-protocol") return null;
27
+ return {
28
+ profile: cloneStateProtocolProfile(cache.profile as StateProtocolStreamProfile),
29
+ updatedAtMs: cache.updatedAtMs,
30
+ };
31
+ }
32
+
33
+ export function validateStateProtocolProfileResult(
34
+ raw: unknown,
35
+ path: string
36
+ ): Result<StateProtocolStreamProfile, { message: string }> {
37
+ const objRes = expectPlainObjectResult(raw, path);
38
+ if (Result.isError(objRes)) return objRes;
39
+ if (objRes.value.kind !== "state-protocol") {
40
+ return Result.err({ message: `${path}.kind must be state-protocol` });
41
+ }
42
+ const keyCheck = rejectUnknownKeysResult(objRes.value, ["kind", "touch"], path);
43
+ if (Result.isError(keyCheck)) return keyCheck;
44
+ let touch = undefined;
45
+ if (objRes.value.touch !== undefined) {
46
+ const touchRes = validateTouchConfigResult(objRes.value.touch, `${path}.touch`);
47
+ if (Result.isError(touchRes)) return Result.err({ message: touchRes.error.message });
48
+ touch = touchRes.value;
49
+ }
50
+ return Result.ok(touch ? { kind: "state-protocol", touch } : { kind: "state-protocol" });
51
+ }
@@ -0,0 +1,107 @@
1
+ import { Result } from "better-result";
2
+ import type {
3
+ StreamProfileDefinition,
4
+ StreamProfilePersistResult,
5
+ StreamProfileReadResult,
6
+ StreamTouchCapability,
7
+ } from "./profile";
8
+ import { normalizeProfileContentType, parseStoredProfileJsonResult } from "./profile";
9
+ import { deriveStateProtocolChanges } from "./stateProtocol/changes";
10
+ import { handleStateProtocolTouchRoute } from "./stateProtocol/routes";
11
+ import type { StateProtocolStreamProfile } from "./stateProtocol/types";
12
+ import {
13
+ cloneStateProtocolCache,
14
+ cloneStateProtocolProfile,
15
+ getStateProtocolTouchConfig,
16
+ isStateProtocolProfile,
17
+ validateStateProtocolProfileResult,
18
+ } from "./stateProtocol/validation";
19
+ import { validateStateProtocolRecordResult } from "./stateProtocol/ingest";
20
+
21
+ const STATE_PROTOCOL_TOUCH_CAPABILITY: StreamTouchCapability = {
22
+ getTouchConfig(profile) {
23
+ return getStateProtocolTouchConfig(profile);
24
+ },
25
+
26
+ syncState({ db, stream, profile }) {
27
+ if (getStateProtocolTouchConfig(profile)) return db.ensureStreamTouchState(stream);
28
+ return db.deleteStreamTouchState(stream);
29
+ },
30
+
31
+ deriveCanonicalChanges(record) {
32
+ return deriveStateProtocolChanges(record);
33
+ },
34
+
35
+ async handleRoute(args) {
36
+ return handleStateProtocolTouchRoute(args);
37
+ },
38
+ };
39
+
40
+ export const STATE_PROTOCOL_STREAM_PROFILE_DEFINITION: StreamProfileDefinition = {
41
+ kind: "state-protocol",
42
+ usesStoredProfileRow: true,
43
+ touch: STATE_PROTOCOL_TOUCH_CAPABILITY,
44
+
45
+ defaultProfile(): StateProtocolStreamProfile {
46
+ return { kind: "state-protocol" };
47
+ },
48
+
49
+ validateResult(raw, path) {
50
+ return validateStateProtocolProfileResult(raw, path);
51
+ },
52
+
53
+ readProfileResult({ row, cached }): Result<StreamProfileReadResult, { message: string }> {
54
+ if (!row) {
55
+ return Result.ok({ profile: { kind: "state-protocol" }, cache: null });
56
+ }
57
+ const cachedCopy = cloneStateProtocolCache(cached);
58
+ if (cachedCopy && cachedCopy.updatedAtMs === row.updated_at_ms) {
59
+ return Result.ok({
60
+ profile: cloneStateProtocolProfile(cachedCopy.profile as StateProtocolStreamProfile),
61
+ cache: cachedCopy,
62
+ });
63
+ }
64
+ const parsedRes = parseStoredProfileJsonResult(row.profile_json);
65
+ if (Result.isError(parsedRes)) return parsedRes;
66
+ const profileRes = validateStateProtocolProfileResult(parsedRes.value, "profile");
67
+ if (Result.isError(profileRes)) return profileRes;
68
+ const profile = cloneStateProtocolProfile(profileRes.value);
69
+ return Result.ok({
70
+ profile: cloneStateProtocolProfile(profile),
71
+ cache: { profile, updatedAtMs: row.updated_at_ms },
72
+ });
73
+ },
74
+
75
+ persistProfileResult({ streamRow, profile }): Result<StreamProfilePersistResult, { kind: "bad_request"; message: string; code?: string }> {
76
+ if (!isStateProtocolProfile(profile)) {
77
+ return Result.err({ kind: "bad_request", message: "invalid state-protocol profile" });
78
+ }
79
+ const contentType = normalizeProfileContentType(streamRow.content_type);
80
+ if (contentType !== "application/json") {
81
+ return Result.err({
82
+ kind: "bad_request",
83
+ message: "state-protocol profile requires application/json stream content-type",
84
+ });
85
+ }
86
+
87
+ const persistedProfile = cloneStateProtocolProfile(profile);
88
+ return Result.ok({
89
+ profile: cloneStateProtocolProfile(persistedProfile),
90
+ cache: {
91
+ profile: persistedProfile,
92
+ updatedAtMs: 0n,
93
+ },
94
+ schemaRegistry: null,
95
+ streamProfile: persistedProfile.kind,
96
+ profileJson: JSON.stringify(persistedProfile),
97
+ touchState: getStateProtocolTouchConfig(persistedProfile) ? "ensure" : "delete",
98
+ });
99
+ },
100
+
101
+ jsonIngest: {
102
+ prepareRecordResult({ profile, value }) {
103
+ if (!isStateProtocolProfile(profile)) return Result.err({ message: "invalid state-protocol profile" });
104
+ return validateStateProtocolRecordResult(value);
105
+ },
106
+ },
107
+ };
@@ -0,0 +1,468 @@
1
+ import { Result } from "better-result";
2
+ import type { SchemaRegistry, SearchFieldConfig } from "./schema/registry";
3
+ import { extractSearchExactValuesResult, resolveSearchAlias } from "./search/schema";
4
+
5
+ type Token =
6
+ | { kind: "word"; value: string }
7
+ | { kind: "string"; value: string }
8
+ | { kind: "lparen" }
9
+ | { kind: "rparen" }
10
+ | { kind: "colon" }
11
+ | { kind: "op"; value: "=" | ">" | ">=" | "<" | "<=" }
12
+ | { kind: "minus" };
13
+
14
+ export type ReadFilterComparisonOp = "eq" | "gt" | "gte" | "lt" | "lte";
15
+
16
+ type FilterExpr =
17
+ | { kind: "and"; left: FilterExpr; right: FilterExpr }
18
+ | { kind: "or"; left: FilterExpr; right: FilterExpr }
19
+ | { kind: "not"; expr: FilterExpr }
20
+ | { kind: "has"; field: string }
21
+ | { kind: "compare"; field: string; op: ReadFilterComparisonOp; rawValue: string };
22
+
23
+ export type CompiledReadFilterClause = {
24
+ kind: "has" | "compare";
25
+ field: string;
26
+ index: SearchFieldConfig;
27
+ op?: ReadFilterComparisonOp;
28
+ canonicalValue?: string;
29
+ compareValue?: bigint | number | boolean | string;
30
+ };
31
+
32
+ export type CompiledReadFilter =
33
+ | { kind: "and"; left: CompiledReadFilter; right: CompiledReadFilter }
34
+ | { kind: "or"; left: CompiledReadFilter; right: CompiledReadFilter }
35
+ | { kind: "not"; expr: CompiledReadFilter }
36
+ | ({ kind: "has" } & CompiledReadFilterClause)
37
+ | ({ kind: "compare" } & CompiledReadFilterClause);
38
+
39
+ export type ReadFilterExactClause = {
40
+ field: string;
41
+ canonicalValue: string;
42
+ };
43
+
44
+ export type ReadFilterColumnClause = {
45
+ field: string;
46
+ op: ReadFilterComparisonOp | "has";
47
+ compareValue?: bigint | number | boolean;
48
+ };
49
+
50
+ function tokenizeResult(input: string): Result<Token[], { message: string }> {
51
+ const tokens: Token[] = [];
52
+ let i = 0;
53
+ while (i < input.length) {
54
+ const ch = input[i];
55
+ if (/\s/.test(ch)) {
56
+ i += 1;
57
+ continue;
58
+ }
59
+ if (ch === "(") {
60
+ tokens.push({ kind: "lparen" });
61
+ i += 1;
62
+ continue;
63
+ }
64
+ if (ch === ")") {
65
+ tokens.push({ kind: "rparen" });
66
+ i += 1;
67
+ continue;
68
+ }
69
+ if (ch === ":") {
70
+ tokens.push({ kind: "colon" });
71
+ i += 1;
72
+ continue;
73
+ }
74
+ if (ch === "-") {
75
+ tokens.push({ kind: "minus" });
76
+ i += 1;
77
+ continue;
78
+ }
79
+ if (ch === ">" || ch === "<" || ch === "=") {
80
+ if ((ch === ">" || ch === "<") && input[i + 1] === "=") {
81
+ tokens.push({ kind: "op", value: `${ch}=` as ">=" | "<=" });
82
+ i += 2;
83
+ continue;
84
+ }
85
+ tokens.push({ kind: "op", value: ch as "=" | ">" | "<" });
86
+ i += 1;
87
+ continue;
88
+ }
89
+ if (ch === "\"") {
90
+ let out = "";
91
+ i += 1;
92
+ while (i < input.length) {
93
+ const cur = input[i];
94
+ if (cur === "\\") {
95
+ if (i + 1 >= input.length) return Result.err({ message: "unterminated escape in filter string" });
96
+ out += input[i + 1];
97
+ i += 2;
98
+ continue;
99
+ }
100
+ if (cur === "\"") break;
101
+ out += cur;
102
+ i += 1;
103
+ }
104
+ if (i >= input.length || input[i] !== "\"") return Result.err({ message: "unterminated quoted string in filter" });
105
+ i += 1;
106
+ tokens.push({ kind: "string", value: out });
107
+ continue;
108
+ }
109
+ let j = i;
110
+ while (j < input.length) {
111
+ const cur = input[j];
112
+ if (/\s/.test(cur) || cur === "(" || cur === ")" || cur === ":" || cur === ">" || cur === "<" || cur === "=") break;
113
+ j += 1;
114
+ }
115
+ const word = input.slice(i, j);
116
+ if (word === "") return Result.err({ message: "invalid filter syntax" });
117
+ tokens.push({ kind: "word", value: word });
118
+ i = j;
119
+ }
120
+ return Result.ok(tokens);
121
+ }
122
+
123
+ class Parser {
124
+ constructor(private readonly tokens: Token[], private pos = 0) {}
125
+
126
+ parseResult(): Result<FilterExpr, { message: string }> {
127
+ const exprRes = this.parseOrResult();
128
+ if (Result.isError(exprRes)) return exprRes;
129
+ if (!this.isAtEnd()) return Result.err({ message: "unexpected token in filter" });
130
+ return exprRes;
131
+ }
132
+
133
+ private parseOrResult(): Result<FilterExpr, { message: string }> {
134
+ let leftRes = this.parseAndResult();
135
+ if (Result.isError(leftRes)) return leftRes;
136
+ let left = leftRes.value;
137
+ while (this.peekWord("OR")) {
138
+ this.pos += 1;
139
+ const rightRes = this.parseAndResult();
140
+ if (Result.isError(rightRes)) return rightRes;
141
+ left = { kind: "or", left, right: rightRes.value };
142
+ }
143
+ return Result.ok(left);
144
+ }
145
+
146
+ private parseAndResult(): Result<FilterExpr, { message: string }> {
147
+ let leftRes = this.parseUnaryResult();
148
+ if (Result.isError(leftRes)) return leftRes;
149
+ let left = leftRes.value;
150
+ while (!this.isAtEnd() && !this.peekKind("rparen") && !this.peekWord("OR")) {
151
+ if (this.peekWord("AND")) this.pos += 1;
152
+ const rightRes = this.parseUnaryResult();
153
+ if (Result.isError(rightRes)) return rightRes;
154
+ left = { kind: "and", left, right: rightRes.value };
155
+ }
156
+ return Result.ok(left);
157
+ }
158
+
159
+ private parseUnaryResult(): Result<FilterExpr, { message: string }> {
160
+ if (this.peekWord("NOT")) {
161
+ this.pos += 1;
162
+ const innerRes = this.parseUnaryResult();
163
+ if (Result.isError(innerRes)) return innerRes;
164
+ return Result.ok({ kind: "not", expr: innerRes.value });
165
+ }
166
+ if (this.peekKind("minus")) {
167
+ this.pos += 1;
168
+ const innerRes = this.parseUnaryResult();
169
+ if (Result.isError(innerRes)) return innerRes;
170
+ return Result.ok({ kind: "not", expr: innerRes.value });
171
+ }
172
+ return this.parsePrimaryResult();
173
+ }
174
+
175
+ private parsePrimaryResult(): Result<FilterExpr, { message: string }> {
176
+ if (this.peekKind("lparen")) {
177
+ this.pos += 1;
178
+ const exprRes = this.parseOrResult();
179
+ if (Result.isError(exprRes)) return exprRes;
180
+ if (!this.peekKind("rparen")) return Result.err({ message: "missing ')' in filter" });
181
+ this.pos += 1;
182
+ return exprRes;
183
+ }
184
+
185
+ const token = this.consumeWordOrString();
186
+ if (!token) return Result.err({ message: "expected clause in filter" });
187
+ if (token.kind !== "word") return Result.err({ message: "expected field name in filter" });
188
+ const fieldOrKeyword = token.value;
189
+
190
+ if (!this.peekKind("colon")) return Result.err({ message: "expected ':' in filter clause" });
191
+ this.pos += 1;
192
+
193
+ if (fieldOrKeyword === "has") {
194
+ const fieldToken = this.consumeWordOrString();
195
+ if (!fieldToken || fieldToken.kind !== "word") return Result.err({ message: "has: requires a field name" });
196
+ return Result.ok({ kind: "has", field: fieldToken.value });
197
+ }
198
+
199
+ let op: ReadFilterComparisonOp = "eq";
200
+ if (this.peekKind("op")) {
201
+ const raw = (this.tokens[this.pos] as Extract<Token, { kind: "op" }>).value;
202
+ this.pos += 1;
203
+ op = raw === "=" ? "eq" : raw === ">" ? "gt" : raw === ">=" ? "gte" : raw === "<" ? "lt" : "lte";
204
+ }
205
+ const valueToken = this.consumeWordOrString();
206
+ if (!valueToken) return Result.err({ message: "expected value in filter clause" });
207
+ return Result.ok({
208
+ kind: "compare",
209
+ field: fieldOrKeyword,
210
+ op,
211
+ rawValue: valueToken.value,
212
+ });
213
+ }
214
+
215
+ private consumeWordOrString(): Extract<Token, { kind: "word" | "string" }> | null {
216
+ const token = this.tokens[this.pos];
217
+ if (!token || (token.kind !== "word" && token.kind !== "string")) return null;
218
+ this.pos += 1;
219
+ return token;
220
+ }
221
+
222
+ private peekKind(kind: Token["kind"]): boolean {
223
+ return this.tokens[this.pos]?.kind === kind;
224
+ }
225
+
226
+ private peekWord(value: string): boolean {
227
+ const token = this.tokens[this.pos];
228
+ return token?.kind === "word" && token.value.toUpperCase() === value;
229
+ }
230
+
231
+ private isAtEnd(): boolean {
232
+ return this.pos >= this.tokens.length;
233
+ }
234
+ }
235
+
236
+ function canonicalizeFilterValue(index: SearchFieldConfig, rawValue: string): string | null {
237
+ switch (index.kind) {
238
+ case "keyword":
239
+ return index.normalizer === "lowercase_v1" ? rawValue.toLowerCase() : rawValue;
240
+ case "integer":
241
+ return /^-?(0|[1-9][0-9]*)$/.test(rawValue.trim()) ? String(BigInt(rawValue.trim())) : null;
242
+ case "float": {
243
+ const parsed = Number(rawValue);
244
+ return Number.isFinite(parsed) ? String(parsed) : null;
245
+ }
246
+ case "date": {
247
+ if (rawValue.trim() === "") return null;
248
+ const parsed = Date.parse(rawValue);
249
+ if (Number.isFinite(parsed)) return String(Math.trunc(parsed));
250
+ return /^-?(0|[1-9][0-9]*)$/.test(rawValue.trim()) ? String(BigInt(rawValue.trim())) : null;
251
+ }
252
+ case "bool": {
253
+ const lowered = rawValue.trim().toLowerCase();
254
+ return lowered === "true" || lowered === "false" ? lowered : null;
255
+ }
256
+ default:
257
+ return null;
258
+ }
259
+ }
260
+
261
+ function compileCompareValueResult(index: SearchFieldConfig, rawValue: string, op: ReadFilterComparisonOp): Result<{
262
+ canonicalValue: string;
263
+ compareValue: bigint | number | boolean | string;
264
+ }, { message: string }> {
265
+ const canonical = canonicalizeFilterValue(index, rawValue);
266
+ if (canonical == null) return Result.err({ message: "invalid value for filter field" });
267
+ if (op === "eq") {
268
+ if (index.kind === "integer" || index.kind === "date") {
269
+ return Result.ok({ canonicalValue: canonical, compareValue: BigInt(canonical) });
270
+ }
271
+ if (index.kind === "float") {
272
+ const parsed = Number(canonical);
273
+ if (!Number.isFinite(parsed)) return Result.err({ message: "invalid numeric value for filter field" });
274
+ return Result.ok({ canonicalValue: canonical, compareValue: parsed });
275
+ }
276
+ if (index.kind === "bool") {
277
+ return Result.ok({ canonicalValue: canonical, compareValue: canonical === "true" });
278
+ }
279
+ return Result.ok({ canonicalValue: canonical, compareValue: canonical });
280
+ }
281
+ if (index.kind === "integer" || index.kind === "date") {
282
+ return Result.ok({ canonicalValue: canonical, compareValue: BigInt(canonical) });
283
+ }
284
+ if (index.kind === "float") {
285
+ const parsed = Number(canonical);
286
+ if (!Number.isFinite(parsed)) return Result.err({ message: "invalid numeric value for filter field" });
287
+ return Result.ok({ canonicalValue: canonical, compareValue: parsed });
288
+ }
289
+ return Result.err({ message: "comparison operator not supported for filter field" });
290
+ }
291
+
292
+ function compileExprResult(
293
+ expr: FilterExpr,
294
+ indexByName: Map<string, { field: string; config: SearchFieldConfig }>
295
+ ): Result<CompiledReadFilter, { message: string }> {
296
+ if (expr.kind === "and") {
297
+ const leftRes = compileExprResult(expr.left, indexByName);
298
+ if (Result.isError(leftRes)) return leftRes;
299
+ const rightRes = compileExprResult(expr.right, indexByName);
300
+ if (Result.isError(rightRes)) return rightRes;
301
+ return Result.ok({ kind: "and", left: leftRes.value, right: rightRes.value });
302
+ }
303
+ if (expr.kind === "or") {
304
+ const leftRes = compileExprResult(expr.left, indexByName);
305
+ if (Result.isError(leftRes)) return leftRes;
306
+ const rightRes = compileExprResult(expr.right, indexByName);
307
+ if (Result.isError(rightRes)) return rightRes;
308
+ return Result.ok({ kind: "or", left: leftRes.value, right: rightRes.value });
309
+ }
310
+ if (expr.kind === "not") {
311
+ const innerRes = compileExprResult(expr.expr, indexByName);
312
+ if (Result.isError(innerRes)) return innerRes;
313
+ return Result.ok({ kind: "not", expr: innerRes.value });
314
+ }
315
+ const resolved = indexByName.get(expr.field);
316
+ if (!resolved) return Result.err({ message: `filter field ${expr.field} is not indexed` });
317
+ const index = resolved.config;
318
+ if (expr.kind === "has" && !index.exists && !index.exact && !index.column) {
319
+ return Result.err({ message: `filter field ${expr.field} does not support has:` });
320
+ }
321
+ if (expr.kind === "has") {
322
+ return Result.ok({ kind: "has", field: resolved.field, index });
323
+ }
324
+ const compareRes = compileCompareValueResult(index, expr.rawValue, expr.op);
325
+ if (Result.isError(compareRes)) return compareRes;
326
+ if (expr.op !== "eq" && !index.column) {
327
+ return Result.err({ message: `filter field ${expr.field} does not support comparisons` });
328
+ }
329
+ if (expr.op === "eq" && !index.exact && !index.column) {
330
+ return Result.err({ message: `filter field ${expr.field} does not support equality filters` });
331
+ }
332
+ return Result.ok({
333
+ kind: "compare",
334
+ field: resolved.field,
335
+ index,
336
+ op: expr.op,
337
+ canonicalValue: compareRes.value.canonicalValue,
338
+ compareValue: compareRes.value.compareValue,
339
+ });
340
+ }
341
+
342
+ export function parseReadFilterResult(
343
+ registry: SchemaRegistry,
344
+ input: string
345
+ ): Result<CompiledReadFilter, { message: string }> {
346
+ const trimmed = input.trim();
347
+ if (trimmed === "") return Result.err({ message: "filter must not be empty" });
348
+ const tokensRes = tokenizeResult(trimmed);
349
+ if (Result.isError(tokensRes)) return tokensRes;
350
+ const parser = new Parser(tokensRes.value);
351
+ const exprRes = parser.parseResult();
352
+ if (Result.isError(exprRes)) return exprRes;
353
+ const fields = registry.search?.fields ?? {};
354
+ const indexByName = new Map<string, { field: string; config: SearchFieldConfig }>();
355
+ for (const [fieldName, config] of Object.entries(fields)) indexByName.set(fieldName, { field: fieldName, config });
356
+ for (const alias of Object.keys(registry.search?.aliases ?? {})) {
357
+ const resolved = resolveSearchAlias(registry.search, alias);
358
+ const config = fields[resolved];
359
+ if (config) indexByName.set(alias, { field: resolved, config });
360
+ }
361
+ return compileExprResult(exprRes.value, indexByName);
362
+ }
363
+
364
+ function compareCanonicals(left: string, right: bigint | number | boolean | string, kind: SearchFieldConfig["kind"]): number {
365
+ if (kind === "integer" || kind === "date") {
366
+ const l = BigInt(left);
367
+ const r = right as bigint;
368
+ return l < r ? -1 : l > r ? 1 : 0;
369
+ }
370
+ if (kind === "float") {
371
+ const l = Number(left);
372
+ const r = right as number;
373
+ return l < r ? -1 : l > r ? 1 : 0;
374
+ }
375
+ if (kind === "bool") {
376
+ const l = left === "true";
377
+ const r = right === true || right === "true";
378
+ return l === r ? 0 : l ? 1 : -1;
379
+ }
380
+ const r = String(right);
381
+ return left < r ? -1 : left > r ? 1 : 0;
382
+ }
383
+
384
+ function evaluateClause(filter: CompiledReadFilterClause, values: string[] | undefined): boolean {
385
+ if (filter.kind === "has") return !!values && values.length > 0;
386
+ if (!values || values.length === 0) return false;
387
+ if (filter.op === "eq") return values.includes(filter.canonicalValue!);
388
+ for (const value of values) {
389
+ const cmp = compareCanonicals(value, filter.compareValue!, filter.index.kind);
390
+ if (filter.op === "gt" && cmp > 0) return true;
391
+ if (filter.op === "gte" && cmp >= 0) return true;
392
+ if (filter.op === "lt" && cmp < 0) return true;
393
+ if (filter.op === "lte" && cmp <= 0) return true;
394
+ }
395
+ return false;
396
+ }
397
+
398
+ export function evaluateReadFilterResult(
399
+ registry: SchemaRegistry,
400
+ offset: bigint,
401
+ filter: CompiledReadFilter,
402
+ value: unknown
403
+ ): Result<boolean, { message: string }> {
404
+ const valuesRes = extractSearchExactValuesResult(registry, offset, value);
405
+ if (Result.isError(valuesRes)) return valuesRes;
406
+ const values = valuesRes.value;
407
+ const evalNode = (node: CompiledReadFilter): boolean => {
408
+ if (node.kind === "and") return evalNode(node.left) && evalNode(node.right);
409
+ if (node.kind === "or") return evalNode(node.left) || evalNode(node.right);
410
+ if (node.kind === "not") return !evalNode(node.expr);
411
+ return evaluateClause(node, values.get(node.field));
412
+ };
413
+ return Result.ok(evalNode(filter));
414
+ }
415
+
416
+ export function collectPositiveExactFilterClauses(filter: CompiledReadFilter): ReadFilterExactClause[] {
417
+ const out: ReadFilterExactClause[] = [];
418
+ const visit = (node: CompiledReadFilter, negated: boolean): void => {
419
+ if (node.kind === "and") {
420
+ visit(node.left, negated);
421
+ visit(node.right, negated);
422
+ return;
423
+ }
424
+ if (node.kind === "not") {
425
+ visit(node.expr, !negated);
426
+ return;
427
+ }
428
+ if (negated || node.kind !== "compare" || node.op !== "eq" || !node.canonicalValue) return;
429
+ out.push({ field: node.field, canonicalValue: node.canonicalValue });
430
+ };
431
+ visit(filter, false);
432
+ return out;
433
+ }
434
+
435
+ export function collectPositiveColumnFilterClauses(filter: CompiledReadFilter): ReadFilterColumnClause[] {
436
+ const out: ReadFilterColumnClause[] = [];
437
+ const supportsColumn = (node: CompiledReadFilterClause): boolean =>
438
+ node.index.column === true &&
439
+ (node.index.kind === "integer" || node.index.kind === "float" || node.index.kind === "date" || node.index.kind === "bool");
440
+ const visit = (node: CompiledReadFilter, negated: boolean): void => {
441
+ if (node.kind === "and") {
442
+ visit(node.left, negated);
443
+ visit(node.right, negated);
444
+ return;
445
+ }
446
+ if (node.kind === "not") {
447
+ visit(node.expr, !negated);
448
+ return;
449
+ }
450
+ if (negated) return;
451
+ if (node.kind === "has") {
452
+ if (supportsColumn(node)) out.push({ field: node.field, op: "has" });
453
+ return;
454
+ }
455
+ if (node.kind === "compare" && supportsColumn(node)) {
456
+ out.push({
457
+ field: node.field,
458
+ op: node.op!,
459
+ compareValue:
460
+ typeof node.compareValue === "bigint" || typeof node.compareValue === "number" || typeof node.compareValue === "boolean"
461
+ ? node.compareValue
462
+ : undefined,
463
+ });
464
+ }
465
+ };
466
+ visit(filter, false);
467
+ return out;
468
+ }