@tungthedev/streams-server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CODE_OF_CONDUCT.md +45 -0
  2. package/CONTRIBUTING.md +76 -0
  3. package/LICENSE +201 -0
  4. package/README.md +58 -0
  5. package/SECURITY.md +42 -0
  6. package/bin/prisma-streams-server +2 -0
  7. package/package.json +46 -0
  8. package/src/app.ts +583 -0
  9. package/src/app_core.ts +3144 -0
  10. package/src/app_local.ts +206 -0
  11. package/src/auth.ts +124 -0
  12. package/src/auto_tune.ts +69 -0
  13. package/src/backpressure.ts +66 -0
  14. package/src/bootstrap.ts +613 -0
  15. package/src/compute/demo_entry.ts +415 -0
  16. package/src/compute/demo_site.ts +1242 -0
  17. package/src/compute/entry.ts +19 -0
  18. package/src/compute/package_entry.ts +4 -0
  19. package/src/compute/virtual-modules.d.ts +15 -0
  20. package/src/compute/worker_module_url.ts +9 -0
  21. package/src/concurrency_gate.ts +108 -0
  22. package/src/config.ts +402 -0
  23. package/src/db/bootstrap_store.ts +9 -0
  24. package/src/db/db.ts +2424 -0
  25. package/src/db/schema.ts +925 -0
  26. package/src/db/sqlite_manifest_snapshot.ts +81 -0
  27. package/src/db/sqlite_touch_store.ts +491 -0
  28. package/src/db/sqlite_wal_store.ts +472 -0
  29. package/src/details/full_mode_details.ts +568 -0
  30. package/src/expiry_sweeper.ts +47 -0
  31. package/src/foreground_activity.ts +55 -0
  32. package/src/hist.ts +169 -0
  33. package/src/index/binary_fuse.ts +379 -0
  34. package/src/index/indexer.ts +947 -0
  35. package/src/index/lexicon_file_cache.ts +261 -0
  36. package/src/index/lexicon_format.ts +93 -0
  37. package/src/index/lexicon_indexer.ts +863 -0
  38. package/src/index/run_cache.ts +84 -0
  39. package/src/index/run_format.ts +213 -0
  40. package/src/index/schedule.ts +28 -0
  41. package/src/index/secondary_indexer.ts +901 -0
  42. package/src/index/secondary_schema.ts +105 -0
  43. package/src/ingest.ts +309 -0
  44. package/src/lens/lens.ts +501 -0
  45. package/src/manifest.ts +249 -0
  46. package/src/memory.ts +334 -0
  47. package/src/metrics.ts +147 -0
  48. package/src/metrics_emitter.ts +83 -0
  49. package/src/notifier.ts +180 -0
  50. package/src/objectstore/accounting.ts +151 -0
  51. package/src/objectstore/interface.ts +13 -0
  52. package/src/objectstore/mock_r2.ts +269 -0
  53. package/src/objectstore/null.ts +32 -0
  54. package/src/objectstore/r2.ts +318 -0
  55. package/src/observe/pairing.ts +61 -0
  56. package/src/observe/request.ts +772 -0
  57. package/src/offset.ts +70 -0
  58. package/src/postgres/bootstrap.ts +269 -0
  59. package/src/postgres/companions.ts +197 -0
  60. package/src/postgres/control_restore.ts +109 -0
  61. package/src/postgres/details.ts +189 -0
  62. package/src/postgres/lexicon_index.ts +260 -0
  63. package/src/postgres/routing_index.ts +189 -0
  64. package/src/postgres/rows.ts +132 -0
  65. package/src/postgres/schema.ts +355 -0
  66. package/src/postgres/secondary_index.ts +238 -0
  67. package/src/postgres/segments.ts +900 -0
  68. package/src/postgres/stats.ts +103 -0
  69. package/src/postgres/store.ts +947 -0
  70. package/src/postgres/touch.ts +591 -0
  71. package/src/postgres/types.ts +32 -0
  72. package/src/profiles/evlog/schema.ts +234 -0
  73. package/src/profiles/evlog.ts +473 -0
  74. package/src/profiles/generic.ts +51 -0
  75. package/src/profiles/index.ts +237 -0
  76. package/src/profiles/metrics/block_format.ts +109 -0
  77. package/src/profiles/metrics/normalize.ts +366 -0
  78. package/src/profiles/metrics/schema.ts +319 -0
  79. package/src/profiles/metrics.ts +83 -0
  80. package/src/profiles/otelTraces/normalize.ts +955 -0
  81. package/src/profiles/otelTraces/otlp.ts +1002 -0
  82. package/src/profiles/otelTraces/schema.ts +408 -0
  83. package/src/profiles/otelTraces.ts +390 -0
  84. package/src/profiles/profile.ts +284 -0
  85. package/src/profiles/stateProtocol/change_event_conformance.typecheck.ts +35 -0
  86. package/src/profiles/stateProtocol/changes.ts +24 -0
  87. package/src/profiles/stateProtocol/ingest.ts +115 -0
  88. package/src/profiles/stateProtocol/routes.ts +511 -0
  89. package/src/profiles/stateProtocol/types.ts +6 -0
  90. package/src/profiles/stateProtocol/validation.ts +51 -0
  91. package/src/profiles/stateProtocol.ts +107 -0
  92. package/src/read_filter.ts +468 -0
  93. package/src/reader.ts +2986 -0
  94. package/src/runtime/hash.ts +156 -0
  95. package/src/runtime/hash_vendor/LICENSE.hash-wasm +38 -0
  96. package/src/runtime/hash_vendor/NOTICE.md +8 -0
  97. package/src/runtime/hash_vendor/xxhash3.umd.min.cjs +7 -0
  98. package/src/runtime/hash_vendor/xxhash32.umd.min.cjs +7 -0
  99. package/src/runtime/hash_vendor/xxhash64.umd.min.cjs +7 -0
  100. package/src/runtime/host_runtime.ts +5 -0
  101. package/src/runtime_memory.ts +200 -0
  102. package/src/runtime_memory_sampler.ts +237 -0
  103. package/src/schema/lens_schema.ts +290 -0
  104. package/src/schema/proof.ts +547 -0
  105. package/src/schema/read_json.ts +51 -0
  106. package/src/schema/registry.ts +966 -0
  107. package/src/search/agg_format.ts +638 -0
  108. package/src/search/aggregate.ts +409 -0
  109. package/src/search/binary/codec.ts +162 -0
  110. package/src/search/binary/docset.ts +67 -0
  111. package/src/search/binary/restart_strings.ts +181 -0
  112. package/src/search/binary/varint.ts +34 -0
  113. package/src/search/bitset.ts +19 -0
  114. package/src/search/col_format.ts +382 -0
  115. package/src/search/col_runtime.ts +59 -0
  116. package/src/search/column_encoding.ts +43 -0
  117. package/src/search/companion_file_cache.ts +319 -0
  118. package/src/search/companion_format.ts +327 -0
  119. package/src/search/companion_manager.ts +1305 -0
  120. package/src/search/companion_plan.ts +229 -0
  121. package/src/search/exact_format.ts +281 -0
  122. package/src/search/exact_runtime.ts +55 -0
  123. package/src/search/fts_format.ts +423 -0
  124. package/src/search/fts_runtime.ts +333 -0
  125. package/src/search/query.ts +875 -0
  126. package/src/search/schema.ts +245 -0
  127. package/src/segment/cache.ts +270 -0
  128. package/src/segment/cached_segment.ts +89 -0
  129. package/src/segment/format.ts +403 -0
  130. package/src/segment/segmenter.ts +412 -0
  131. package/src/segment/segmenter_worker.ts +72 -0
  132. package/src/segment/segmenter_workers.ts +130 -0
  133. package/src/server.ts +264 -0
  134. package/src/server_auto_tune.ts +158 -0
  135. package/src/sqlite/adapter.ts +335 -0
  136. package/src/sqlite/runtime_stats.ts +163 -0
  137. package/src/stats.ts +205 -0
  138. package/src/store/append.ts +50 -0
  139. package/src/store/bootstrap_restore_store.ts +71 -0
  140. package/src/store/capabilities.ts +86 -0
  141. package/src/store/full_mode_details_store.ts +71 -0
  142. package/src/store/index_store.ts +104 -0
  143. package/src/store/profile_touch_store.ts +1 -0
  144. package/src/store/rows.ts +144 -0
  145. package/src/store/schema_profile_store.ts +73 -0
  146. package/src/store/schema_publication.ts +6 -0
  147. package/src/store/segment_manifest_store.ts +129 -0
  148. package/src/store/segment_read_store.ts +22 -0
  149. package/src/store/stats_accounting_store.ts +83 -0
  150. package/src/store/touch_store.ts +98 -0
  151. package/src/store/wal_store.ts +21 -0
  152. package/src/stream_size_reconciler.ts +100 -0
  153. package/src/touch/canonical_change.ts +7 -0
  154. package/src/touch/live_keys.ts +158 -0
  155. package/src/touch/live_metrics.ts +841 -0
  156. package/src/touch/live_templates.ts +449 -0
  157. package/src/touch/manager.ts +1292 -0
  158. package/src/touch/process_batch.ts +576 -0
  159. package/src/touch/processor_worker.ts +85 -0
  160. package/src/touch/spec.ts +459 -0
  161. package/src/touch/touch_journal.ts +771 -0
  162. package/src/touch/touch_key_id.ts +20 -0
  163. package/src/touch/worker_pool.ts +191 -0
  164. package/src/touch/worker_protocol.ts +57 -0
  165. package/src/types/proper-lockfile.d.ts +1 -0
  166. package/src/uploader.ts +358 -0
  167. package/src/util/base32_crockford.ts +81 -0
  168. package/src/util/bloom256.ts +67 -0
  169. package/src/util/byte_lru.ts +73 -0
  170. package/src/util/cleanup.ts +22 -0
  171. package/src/util/crc32c.ts +29 -0
  172. package/src/util/ds_error.ts +15 -0
  173. package/src/util/duration.ts +17 -0
  174. package/src/util/endian.ts +53 -0
  175. package/src/util/json_pointer.ts +148 -0
  176. package/src/util/log.ts +25 -0
  177. package/src/util/lru.ts +53 -0
  178. package/src/util/retry.ts +35 -0
  179. package/src/util/siphash.ts +71 -0
  180. package/src/util/stream_paths.ts +50 -0
  181. package/src/util/time.ts +14 -0
  182. package/src/util/yield.ts +3 -0
  183. package/src/util/zstd.ts +24 -0
@@ -0,0 +1,335 @@
1
+ import { createRequire } from "node:module";
2
+ import type { HostRuntime } from "../runtime/host_runtime.ts";
3
+ import { detectHostRuntime } from "../runtime/host_runtime.ts";
4
+ import { dsError } from "../util/ds_error.ts";
5
+ export interface SqliteStatement {
6
+ get(...params: any[]): any;
7
+ all(...params: any[]): any[];
8
+ run(...params: any[]): { changes: number | bigint; lastInsertRowid: number | bigint };
9
+ iterate(...params: any[]): Iterable<any>;
10
+ finalize?(): void;
11
+ }
12
+
13
+ export interface SqliteDatabase {
14
+ exec(sql: string): void;
15
+ query(sql: string): SqliteStatement;
16
+ prepare(sql: string): SqliteStatement;
17
+ transaction<T>(fn: () => T): () => T;
18
+ close(): void;
19
+ }
20
+
21
+ type SqliteAdapterRuntimeCounts = {
22
+ open_connections: number;
23
+ prepared_statements: number;
24
+ };
25
+
26
+ const sqliteAdapterRuntimeCounts: SqliteAdapterRuntimeCounts = {
27
+ open_connections: 0,
28
+ prepared_statements: 0,
29
+ };
30
+
31
+ function incrementSqliteConnection(): void {
32
+ sqliteAdapterRuntimeCounts.open_connections += 1;
33
+ }
34
+
35
+ function decrementSqliteConnection(): void {
36
+ sqliteAdapterRuntimeCounts.open_connections = Math.max(0, sqliteAdapterRuntimeCounts.open_connections - 1);
37
+ }
38
+
39
+ function incrementPreparedStatement(): void {
40
+ sqliteAdapterRuntimeCounts.prepared_statements += 1;
41
+ }
42
+
43
+ function decrementPreparedStatement(): void {
44
+ sqliteAdapterRuntimeCounts.prepared_statements = Math.max(0, sqliteAdapterRuntimeCounts.prepared_statements - 1);
45
+ }
46
+
47
+ export function getSqliteAdapterRuntimeCounts(): SqliteAdapterRuntimeCounts {
48
+ return { ...sqliteAdapterRuntimeCounts };
49
+ }
50
+
51
+ class BunStatementAdapter implements SqliteStatement {
52
+ private readonly stmt: any;
53
+ private readonly onFinalize?: () => void;
54
+ private finalized = false;
55
+
56
+ constructor(stmt: any, onFinalize?: () => void) {
57
+ this.stmt = stmt;
58
+ this.onFinalize = onFinalize;
59
+ }
60
+
61
+ get(...params: any[]): any {
62
+ return this.stmt.get(...params);
63
+ }
64
+
65
+ all(...params: any[]): any[] {
66
+ return this.stmt.all(...params);
67
+ }
68
+
69
+ run(...params: any[]): { changes: number | bigint; lastInsertRowid: number | bigint } {
70
+ return this.stmt.run(...params);
71
+ }
72
+
73
+ iterate(...params: any[]): Iterable<any> {
74
+ return this.stmt.iterate(...params);
75
+ }
76
+
77
+ finalize(): void {
78
+ if (this.finalized) return;
79
+ this.finalized = true;
80
+ try {
81
+ if (typeof this.stmt.finalize === "function") this.stmt.finalize();
82
+ } finally {
83
+ this.onFinalize?.();
84
+ }
85
+ }
86
+ }
87
+
88
+ class BunDatabaseAdapter implements SqliteDatabase {
89
+ private readonly db: any;
90
+ private preparedStatementCount = 0;
91
+ private closed = false;
92
+ private readonly statementCache = new Map<string, BunStatementAdapter>();
93
+
94
+ constructor(db: any) {
95
+ this.db = db;
96
+ incrementSqliteConnection();
97
+ }
98
+
99
+ private trackStatement(): () => void {
100
+ let released = false;
101
+ this.preparedStatementCount += 1;
102
+ incrementPreparedStatement();
103
+ return () => {
104
+ if (released) return;
105
+ released = true;
106
+ this.preparedStatementCount = Math.max(0, this.preparedStatementCount - 1);
107
+ decrementPreparedStatement();
108
+ };
109
+ }
110
+
111
+ exec(sql: string): void {
112
+ this.db.exec(sql);
113
+ }
114
+
115
+ prepare(sql: string): SqliteStatement {
116
+ return new BunStatementAdapter(this.db.query(sql), this.trackStatement());
117
+ }
118
+
119
+ query(sql: string): SqliteStatement {
120
+ const cached = this.statementCache.get(sql);
121
+ if (cached) return cached;
122
+ let adapter: BunStatementAdapter;
123
+ const release = this.trackStatement();
124
+ adapter = new BunStatementAdapter(this.db.query(sql), () => {
125
+ this.statementCache.delete(sql);
126
+ release();
127
+ });
128
+ this.statementCache.set(sql, adapter);
129
+ return adapter;
130
+ }
131
+
132
+ transaction<T>(fn: () => T): () => T {
133
+ return this.db.transaction(fn);
134
+ }
135
+
136
+ close(): void {
137
+ if (this.closed) return;
138
+ this.closed = true;
139
+ const cachedStatements = Array.from(this.statementCache.values());
140
+ this.statementCache.clear();
141
+ for (const stmt of cachedStatements) {
142
+ try {
143
+ stmt.finalize?.();
144
+ } catch {
145
+ // Ignore finalizer failures during shutdown.
146
+ }
147
+ }
148
+ while (this.preparedStatementCount > 0) {
149
+ this.preparedStatementCount -= 1;
150
+ decrementPreparedStatement();
151
+ }
152
+ try {
153
+ this.db.close();
154
+ } finally {
155
+ decrementSqliteConnection();
156
+ }
157
+ }
158
+ }
159
+
160
+ class NodeStatementAdapter implements SqliteStatement {
161
+ private readonly stmt: any;
162
+ private readonly onFinalize?: () => void;
163
+ private finalized = false;
164
+
165
+ constructor(stmt: any, onFinalize?: () => void) {
166
+ this.stmt = stmt;
167
+ this.onFinalize = onFinalize;
168
+ }
169
+
170
+ get(...params: any[]): any {
171
+ return this.stmt.get(...params);
172
+ }
173
+
174
+ all(...params: any[]): any[] {
175
+ return this.stmt.all(...params);
176
+ }
177
+
178
+ run(...params: any[]): { changes: number | bigint; lastInsertRowid: number | bigint } {
179
+ return this.stmt.run(...params);
180
+ }
181
+
182
+ iterate(...params: any[]): Iterable<any> {
183
+ return this.stmt.iterate(...params);
184
+ }
185
+
186
+ finalize(): void {
187
+ if (this.finalized) return;
188
+ this.finalized = true;
189
+ try {
190
+ if (typeof this.stmt.finalize === "function") this.stmt.finalize();
191
+ } finally {
192
+ this.onFinalize?.();
193
+ }
194
+ }
195
+ }
196
+
197
+ class NodeDatabaseAdapter implements SqliteDatabase {
198
+ private txDepth = 0;
199
+ private txCounter = 0;
200
+ private readonly db: any;
201
+ private preparedStatementCount = 0;
202
+ private closed = false;
203
+ private readonly statementCache = new Map<string, NodeStatementAdapter>();
204
+
205
+ constructor(db: any) {
206
+ this.db = db;
207
+ incrementSqliteConnection();
208
+ }
209
+
210
+ private trackStatement(): () => void {
211
+ let released = false;
212
+ this.preparedStatementCount += 1;
213
+ incrementPreparedStatement();
214
+ return () => {
215
+ if (released) return;
216
+ released = true;
217
+ this.preparedStatementCount = Math.max(0, this.preparedStatementCount - 1);
218
+ decrementPreparedStatement();
219
+ };
220
+ }
221
+
222
+ exec(sql: string): void {
223
+ this.db.exec(sql);
224
+ }
225
+
226
+ prepare(sql: string): SqliteStatement {
227
+ const stmt = this.db.prepare(sql);
228
+ if (typeof stmt?.setReadBigInts === "function") stmt.setReadBigInts(true);
229
+ return new NodeStatementAdapter(stmt, this.trackStatement());
230
+ }
231
+
232
+ query(sql: string): SqliteStatement {
233
+ const cached = this.statementCache.get(sql);
234
+ if (cached) return cached;
235
+ const stmt = this.db.prepare(sql);
236
+ if (typeof stmt?.setReadBigInts === "function") stmt.setReadBigInts(true);
237
+ let adapter: NodeStatementAdapter;
238
+ const release = this.trackStatement();
239
+ adapter = new NodeStatementAdapter(stmt, () => {
240
+ this.statementCache.delete(sql);
241
+ release();
242
+ });
243
+ this.statementCache.set(sql, adapter);
244
+ return adapter;
245
+ }
246
+
247
+ transaction<T>(fn: () => T): () => T {
248
+ return () => {
249
+ const nested = this.txDepth > 0;
250
+ const savepoint = `ds_tx_${++this.txCounter}`;
251
+ this.txDepth += 1;
252
+ try {
253
+ if (nested) this.db.exec(`SAVEPOINT ${savepoint};`);
254
+ else this.db.exec("BEGIN;");
255
+ const out = fn();
256
+ if (nested) this.db.exec(`RELEASE SAVEPOINT ${savepoint};`);
257
+ else this.db.exec("COMMIT;");
258
+ return out;
259
+ } catch (err) {
260
+ try {
261
+ if (nested) {
262
+ this.db.exec(`ROLLBACK TO SAVEPOINT ${savepoint};`);
263
+ this.db.exec(`RELEASE SAVEPOINT ${savepoint};`);
264
+ } else {
265
+ this.db.exec("ROLLBACK;");
266
+ }
267
+ } catch {
268
+ // Ignore secondary rollback failures.
269
+ }
270
+ throw err;
271
+ } finally {
272
+ this.txDepth = Math.max(0, this.txDepth - 1);
273
+ }
274
+ };
275
+ }
276
+
277
+ close(): void {
278
+ if (this.closed) return;
279
+ this.closed = true;
280
+ const cachedStatements = Array.from(this.statementCache.values());
281
+ this.statementCache.clear();
282
+ for (const stmt of cachedStatements) {
283
+ try {
284
+ stmt.finalize?.();
285
+ } catch {
286
+ // Ignore finalizer failures during shutdown.
287
+ }
288
+ }
289
+ while (this.preparedStatementCount > 0) {
290
+ this.preparedStatementCount -= 1;
291
+ decrementPreparedStatement();
292
+ }
293
+ try {
294
+ this.db.close();
295
+ } finally {
296
+ decrementSqliteConnection();
297
+ }
298
+ }
299
+ }
300
+
301
+ let openImpl: ((path: string) => SqliteDatabase) | null = null;
302
+ let openImplRuntime: HostRuntime | null = null;
303
+ let runtimeOverride: HostRuntime | null = null;
304
+ const require = createRequire(import.meta.url);
305
+
306
+ function selectedRuntime(): HostRuntime {
307
+ return runtimeOverride ?? detectHostRuntime();
308
+ }
309
+
310
+ function buildOpenImpl(runtime: HostRuntime): (path: string) => SqliteDatabase {
311
+ if (runtime === "bun") {
312
+ const { Database } = require("bun:sqlite") as { Database: new (path: string) => any };
313
+ return (path: string) => new BunDatabaseAdapter(new Database(path));
314
+ }
315
+ const { DatabaseSync } = require("node:sqlite") as { DatabaseSync: new (path: string) => any };
316
+ return (path: string) => new NodeDatabaseAdapter(new DatabaseSync(path));
317
+ }
318
+
319
+ export function setSqliteRuntimeOverride(runtime: HostRuntime | null): void {
320
+ runtimeOverride = runtime;
321
+ if (runtimeOverride && openImplRuntime && runtimeOverride !== openImplRuntime) {
322
+ openImpl = null;
323
+ openImplRuntime = null;
324
+ }
325
+ }
326
+
327
+ export function openSqliteDatabase(path: string): SqliteDatabase {
328
+ const runtime = selectedRuntime();
329
+ if (!openImpl || openImplRuntime !== runtime) {
330
+ openImpl = buildOpenImpl(runtime);
331
+ openImplRuntime = runtime;
332
+ }
333
+ if (!openImpl) throw dsError("sqlite adapter not initialized");
334
+ return openImpl(path);
335
+ }
@@ -0,0 +1,163 @@
1
+ import { createRequire } from "node:module";
2
+ import { detectHostRuntime } from "../runtime/host_runtime.ts";
3
+ import { getSqliteAdapterRuntimeCounts } from "./adapter.ts";
4
+ import type { SqliteRuntimeMemoryStats } from "../runtime_memory.ts";
5
+
6
+ const SQLITE_STATUS_MEMORY_USED = 0;
7
+ const SQLITE_STATUS_PAGECACHE_USED = 1;
8
+ const SQLITE_STATUS_PAGECACHE_OVERFLOW = 2;
9
+ const SQLITE_STATUS_MALLOC_COUNT = 9;
10
+
11
+ type SqliteStatus64Fn = (op: number, currentPtr: number, highwaterPtr: number, resetFlag: number) => number;
12
+ type BunFfiModule = {
13
+ dlopen: (
14
+ path: string,
15
+ symbols: {
16
+ sqlite3_status64: {
17
+ args: ["i32", "ptr", "ptr", "i32"];
18
+ returns: "i32";
19
+ };
20
+ }
21
+ ) => { symbols: { sqlite3_status64: SqliteStatus64Fn }; close: () => void };
22
+ ptr: (value: TypedArray) => number;
23
+ };
24
+
25
+ type TypedArray = BigInt64Array | BigUint64Array | Int32Array | Uint32Array;
26
+
27
+ type SqliteLibraryHandle = {
28
+ sqlite3_status64: SqliteStatus64Fn;
29
+ close: () => void;
30
+ };
31
+
32
+ type CachedSqliteRuntimeStats = {
33
+ at_ms: number;
34
+ value: SqliteRuntimeMemoryStats;
35
+ };
36
+
37
+ let ffiModule: BunFfiModule | null | undefined;
38
+ let sqliteLibrary: SqliteLibraryHandle | null | undefined;
39
+ let cachedRuntimeStats: CachedSqliteRuntimeStats | null = null;
40
+ const require = createRequire(import.meta.url);
41
+
42
+ function sqliteLibraryNames(): string[] {
43
+ if (process.platform === "darwin") {
44
+ return ["libsqlite3.dylib", "/usr/lib/libsqlite3.dylib"];
45
+ }
46
+ if (process.platform === "linux") {
47
+ return ["libsqlite3.so.0", "libsqlite3.so", "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "/lib/x86_64-linux-gnu/libsqlite3.so.0"];
48
+ }
49
+ return ["libsqlite3.so", "libsqlite3.dylib"];
50
+ }
51
+
52
+ function loadBunFfi(): BunFfiModule | null {
53
+ if (ffiModule !== undefined) return ffiModule;
54
+ if (detectHostRuntime() !== "bun") {
55
+ ffiModule = null;
56
+ return ffiModule;
57
+ }
58
+ try {
59
+ ffiModule = require("bun:ffi") as BunFfiModule;
60
+ } catch {
61
+ ffiModule = null;
62
+ }
63
+ return ffiModule;
64
+ }
65
+
66
+ function loadSqliteLibrary(): SqliteLibraryHandle | null {
67
+ if (sqliteLibrary !== undefined) return sqliteLibrary;
68
+ const ffi = loadBunFfi();
69
+ if (!ffi) {
70
+ sqliteLibrary = null;
71
+ return sqliteLibrary;
72
+ }
73
+ for (const name of sqliteLibraryNames()) {
74
+ try {
75
+ const lib = ffi.dlopen(name, {
76
+ sqlite3_status64: {
77
+ args: ["i32", "ptr", "ptr", "i32"],
78
+ returns: "i32",
79
+ },
80
+ });
81
+ sqliteLibrary = {
82
+ sqlite3_status64: lib.symbols.sqlite3_status64,
83
+ close: () => lib.close(),
84
+ };
85
+ return sqliteLibrary;
86
+ } catch {
87
+ // Try the next library name.
88
+ }
89
+ }
90
+ sqliteLibrary = null;
91
+ return sqliteLibrary;
92
+ }
93
+
94
+ function readStatus64(
95
+ lib: SqliteLibraryHandle,
96
+ ffi: BunFfiModule,
97
+ op: number
98
+ ): { current: number; highwater: number } | null {
99
+ const current = new BigInt64Array(1);
100
+ const highwater = new BigInt64Array(1);
101
+ try {
102
+ const rc = lib.sqlite3_status64(op, ffi.ptr(current), ffi.ptr(highwater), 0);
103
+ if (rc !== 0) return null;
104
+ return {
105
+ current: Number(current[0]),
106
+ highwater: Number(highwater[0]),
107
+ };
108
+ } catch {
109
+ return null;
110
+ }
111
+ }
112
+
113
+ export function readSqliteRuntimeMemoryStats(ttlMs = 1_000): SqliteRuntimeMemoryStats {
114
+ const now = Date.now();
115
+ if (cachedRuntimeStats && now - cachedRuntimeStats.at_ms < Math.max(0, ttlMs)) {
116
+ return cachedRuntimeStats.value;
117
+ }
118
+
119
+ const adapterCounts = getSqliteAdapterRuntimeCounts();
120
+ const unavailable: SqliteRuntimeMemoryStats = {
121
+ available: false,
122
+ source: "unavailable",
123
+ memory_used_bytes: 0,
124
+ memory_highwater_bytes: 0,
125
+ pagecache_used_slots: 0,
126
+ pagecache_used_slots_highwater: 0,
127
+ pagecache_overflow_bytes: 0,
128
+ pagecache_overflow_highwater_bytes: 0,
129
+ malloc_count: 0,
130
+ malloc_count_highwater: 0,
131
+ open_connections: adapterCounts.open_connections,
132
+ prepared_statements: adapterCounts.prepared_statements,
133
+ };
134
+
135
+ const ffi = loadBunFfi();
136
+ const lib = loadSqliteLibrary();
137
+ if (!ffi || !lib) {
138
+ cachedRuntimeStats = { at_ms: now, value: unavailable };
139
+ return unavailable;
140
+ }
141
+
142
+ const memoryUsed = readStatus64(lib, ffi, SQLITE_STATUS_MEMORY_USED);
143
+ const pagecacheUsed = readStatus64(lib, ffi, SQLITE_STATUS_PAGECACHE_USED);
144
+ const pagecacheOverflow = readStatus64(lib, ffi, SQLITE_STATUS_PAGECACHE_OVERFLOW);
145
+ const mallocCount = readStatus64(lib, ffi, SQLITE_STATUS_MALLOC_COUNT);
146
+
147
+ const stats: SqliteRuntimeMemoryStats = {
148
+ available: memoryUsed != null,
149
+ source: memoryUsed != null ? "sqlite3_status64" : "unavailable",
150
+ memory_used_bytes: Math.max(0, Math.floor(memoryUsed?.current ?? 0)),
151
+ memory_highwater_bytes: Math.max(0, Math.floor(memoryUsed?.highwater ?? 0)),
152
+ pagecache_used_slots: Math.max(0, Math.floor(pagecacheUsed?.current ?? 0)),
153
+ pagecache_used_slots_highwater: Math.max(0, Math.floor(pagecacheUsed?.highwater ?? 0)),
154
+ pagecache_overflow_bytes: Math.max(0, Math.floor(pagecacheOverflow?.current ?? 0)),
155
+ pagecache_overflow_highwater_bytes: Math.max(0, Math.floor(pagecacheOverflow?.highwater ?? 0)),
156
+ malloc_count: Math.max(0, Math.floor(mallocCount?.current ?? 0)),
157
+ malloc_count_highwater: Math.max(0, Math.floor(mallocCount?.highwater ?? 0)),
158
+ open_connections: adapterCounts.open_connections,
159
+ prepared_statements: adapterCounts.prepared_statements,
160
+ };
161
+ cachedRuntimeStats = { at_ms: now, value: stats };
162
+ return stats;
163
+ }
package/src/stats.ts ADDED
@@ -0,0 +1,205 @@
1
+ import type { UploaderController } from "./uploader";
2
+ import type { MemoryPressureMonitor } from "./memory";
3
+ import type { BackpressureGate } from "./backpressure";
4
+ import type { IngestQueue } from "./ingest";
5
+ import type { StorageStatsStore } from "./store/stats_accounting_store";
6
+
7
+ export type StatsSnapshot = {
8
+ ingestedBytes: number;
9
+ walBytes: number;
10
+ sealedPayloadBytes: number;
11
+ sealedBytes: number;
12
+ uploadedBytes: number;
13
+ segmentsSealed: number;
14
+ backpressureOverMs: number;
15
+ activeStreams: number;
16
+ };
17
+
18
+ export class StatsCollector {
19
+ private readonly backpressureBudgetMs: number;
20
+ private ingestedBytes = 0;
21
+ private walBytes = 0;
22
+ private sealedPayloadBytes = 0;
23
+ private sealedBytes = 0;
24
+ private uploadedBytes = 0;
25
+ private segmentsSealed = 0;
26
+ private backpressureOverMs = 0;
27
+ private readonly activeStreams = new Set<string>();
28
+
29
+ constructor(opts?: { backpressureBudgetMs?: number }) {
30
+ const raw = opts?.backpressureBudgetMs ?? 1;
31
+ this.backpressureBudgetMs = Number.isFinite(raw) ? Math.max(0, raw) : 1;
32
+ }
33
+
34
+ recordIngested(bytes: number): void {
35
+ this.ingestedBytes += bytes;
36
+ }
37
+
38
+ recordWalCommitBytes(bytes: number): void {
39
+ this.walBytes += bytes;
40
+ }
41
+
42
+ recordSegmentSealed(payloadBytes: number, segmentBytes: number): void {
43
+ this.sealedPayloadBytes += payloadBytes;
44
+ this.sealedBytes += segmentBytes;
45
+ this.segmentsSealed += 1;
46
+ }
47
+
48
+ recordUploadedBytes(bytes: number): void {
49
+ this.uploadedBytes += bytes;
50
+ }
51
+
52
+ getBackpressureBudgetMs(): number {
53
+ return this.backpressureBudgetMs;
54
+ }
55
+
56
+ recordBackpressureOverMs(overMs: number): void {
57
+ if (overMs <= 0) return;
58
+ this.backpressureOverMs += Math.max(0, overMs);
59
+ }
60
+
61
+ recordStreamTouched(stream: string): void {
62
+ this.activeStreams.add(stream);
63
+ }
64
+
65
+ snapshotAndReset(): StatsSnapshot {
66
+ const snapshot: StatsSnapshot = {
67
+ ingestedBytes: this.ingestedBytes,
68
+ walBytes: this.walBytes,
69
+ sealedPayloadBytes: this.sealedPayloadBytes,
70
+ sealedBytes: this.sealedBytes,
71
+ uploadedBytes: this.uploadedBytes,
72
+ segmentsSealed: this.segmentsSealed,
73
+ backpressureOverMs: this.backpressureOverMs,
74
+ activeStreams: this.activeStreams.size,
75
+ };
76
+ this.ingestedBytes = 0;
77
+ this.walBytes = 0;
78
+ this.sealedPayloadBytes = 0;
79
+ this.sealedBytes = 0;
80
+ this.uploadedBytes = 0;
81
+ this.segmentsSealed = 0;
82
+ this.backpressureOverMs = 0;
83
+ this.activeStreams.clear();
84
+ return snapshot;
85
+ }
86
+ }
87
+
88
+ function formatBytes(bytes: number): string {
89
+ const units = ["b", "kb", "mb", "gb"];
90
+ let value = bytes;
91
+ let idx = 0;
92
+ while (value >= 1024 && idx < units.length - 1) {
93
+ value /= 1024;
94
+ idx += 1;
95
+ }
96
+ const digits = idx === 0 ? 0 : 1;
97
+ return `${value.toFixed(digits)}${units[idx]}`;
98
+ }
99
+
100
+ export class StatsReporter {
101
+ private timer: any | null = null;
102
+ private sampleTimer: any | null = null;
103
+ private running = false;
104
+ private lastTickMs: number | null = null;
105
+ private lastSampleMs: number | null = null;
106
+ private rejectActiveMs = 0;
107
+ private readonly intervalMs: number;
108
+ private readonly stats: StatsCollector;
109
+ private readonly storageStats: StorageStatsStore;
110
+ private readonly uploader: UploaderController;
111
+ private readonly ingest?: IngestQueue;
112
+ private readonly backpressure?: BackpressureGate;
113
+ private readonly memory?: MemoryPressureMonitor;
114
+
115
+ constructor(
116
+ stats: StatsCollector,
117
+ storageStats: StorageStatsStore,
118
+ uploader: UploaderController,
119
+ ingest?: IngestQueue,
120
+ backpressure?: BackpressureGate,
121
+ memory?: MemoryPressureMonitor,
122
+ intervalMs = 60_000
123
+ ) {
124
+ this.stats = stats;
125
+ this.storageStats = storageStats;
126
+ this.uploader = uploader;
127
+ this.ingest = ingest;
128
+ this.backpressure = backpressure;
129
+ this.memory = memory;
130
+ this.intervalMs = intervalMs;
131
+ }
132
+
133
+ start(): void {
134
+ if (this.timer) return;
135
+ if (!this.sampleTimer) {
136
+ this.sampleTimer = setInterval(() => this.sample(), 250);
137
+ this.sample();
138
+ }
139
+ this.timer = setInterval(() => {
140
+ void this.tick();
141
+ }, this.intervalMs);
142
+ }
143
+
144
+ stop(): void {
145
+ if (this.timer) clearInterval(this.timer);
146
+ this.timer = null;
147
+ if (this.sampleTimer) clearInterval(this.sampleTimer);
148
+ this.sampleTimer = null;
149
+ }
150
+
151
+ private sample(): void {
152
+ const now = Date.now();
153
+ const last = this.lastSampleMs;
154
+ this.lastSampleMs = now;
155
+ if (!last) return;
156
+ const dt = Math.max(0, now - last);
157
+
158
+ const rejectActive =
159
+ (this.memory?.isOverLimit() ?? false) ||
160
+ (this.ingest?.isQueueFull() ?? false) ||
161
+ (this.backpressure?.isOverLimit() ?? false);
162
+ if (rejectActive) this.rejectActiveMs += dt;
163
+ }
164
+
165
+ private async tick(): Promise<void> {
166
+ if (this.running) return;
167
+ this.running = true;
168
+ try {
169
+ const nowMs = Date.now();
170
+ const windowMs = this.lastTickMs ? Math.max(1, nowMs - this.lastTickMs) : this.intervalMs;
171
+ this.lastTickMs = nowMs;
172
+ const snap = this.stats.snapshotAndReset();
173
+ const storedBytes = snap.walBytes + snap.sealedBytes;
174
+ const compression =
175
+ snap.sealedBytes > 0 ? `${(snap.sealedPayloadBytes / snap.sealedBytes).toFixed(2)}x` : "n/a";
176
+ const queueWaitPct = snap.backpressureOverMs > 0 ? (snap.backpressureOverMs / windowMs) * 100 : 0;
177
+ const rejectPct = this.rejectActiveMs > 0 ? (this.rejectActiveMs / windowMs) * 100 : 0;
178
+ const backpressurePct = Math.min(100, Math.max(queueWaitPct, rejectPct));
179
+ this.rejectActiveMs = 0;
180
+ const avgSegmentSize = snap.segmentsSealed > 0 ? formatBytes(snap.sealedBytes / snap.segmentsSealed) : "n/a";
181
+ const totalStreams = await this.storageStats.countStreams();
182
+ const segmentsWaiting = this.uploader.countSegmentsWaiting();
183
+ const walDbBytes = await this.storageStats.getWalDbSizeBytes();
184
+ const metaDbBytes = await this.storageStats.getMetaDbSizeBytes();
185
+ const maxRss = this.memory ? formatBytes(this.memory.snapshotMaxRssBytes(true)) : null;
186
+ const line =
187
+ `ingested=${formatBytes(snap.ingestedBytes)} ` +
188
+ `stored=${formatBytes(storedBytes)} ` +
189
+ `compression=${compression} ` +
190
+ `uploaded=${formatBytes(snap.uploadedBytes)} ` +
191
+ `streams-touched=${snap.activeStreams}/${totalStreams} ` +
192
+ `segments-sealed=${snap.segmentsSealed} ` +
193
+ `segments-waiting=${segmentsWaiting} ` +
194
+ `avg-segment-size=${avgSegmentSize} ` +
195
+ `wal-size=${formatBytes(walDbBytes)} ` +
196
+ `meta-size=${formatBytes(metaDbBytes)} ` +
197
+ `backpressure=${backpressurePct.toFixed(1)}%` +
198
+ (maxRss ? ` max-rss=${maxRss}` : "");
199
+ // eslint-disable-next-line no-console
200
+ console.log(line);
201
+ } finally {
202
+ this.running = false;
203
+ }
204
+ }
205
+ }