@mblaney/holster 1.0.34 → 1.0.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/build/holster.js +3 -3
  2. package/build/holster.js.map +3 -3
  3. package/dist/holster.js +4 -0
  4. package/dist/holster.js.map +7 -0
  5. package/package.json +23 -5
  6. package/src/get.js +11 -19
  7. package/src/ham.js +17 -47
  8. package/src/holster.js +106 -28
  9. package/src/sea.js +0 -52
  10. package/src/store.js +7 -7
  11. package/src/wire.js +8 -2
  12. package/ts/src/array.ts +78 -0
  13. package/ts/src/buffer.ts +441 -0
  14. package/ts/src/dup.ts +49 -0
  15. package/ts/src/get.ts +83 -0
  16. package/ts/src/ham.ts +254 -0
  17. package/ts/src/holster.ts +931 -0
  18. package/ts/src/index.ts +35 -0
  19. package/ts/src/radisk.ts +585 -0
  20. package/ts/src/radix.ts +157 -0
  21. package/ts/src/schemas.ts +519 -0
  22. package/ts/src/sea-utils.ts +103 -0
  23. package/ts/src/sea.ts +388 -0
  24. package/ts/src/store.ts +321 -0
  25. package/ts/src/user.ts +381 -0
  26. package/ts/src/utils.ts +232 -0
  27. package/ts/src/wire.ts +829 -0
  28. package/.dockerignore +0 -9
  29. package/.github/workflows/node.js.yml +0 -31
  30. package/.github/workflows/npm-publish.yml +0 -33
  31. package/Dockerfile +0 -20
  32. package/examples/index.html +0 -33
  33. package/examples/index.js +0 -49
  34. package/examples/user.html +0 -31
  35. package/examples/user.js +0 -93
  36. package/examples/wire.html +0 -27
  37. package/examples/wire.js +0 -55
  38. package/test/holster.get.test.js +0 -94
  39. package/test/holster.lex.test.js +0 -196
  40. package/test/holster.off.test.js +0 -53
  41. package/test/holster.on.test.js +0 -264
  42. package/test/holster.put.test.js +0 -358
  43. package/test/holster.secure.test.js +0 -202
  44. package/test/holster.user.get.test.js +0 -114
  45. package/test/holster.user.lex.test.js +0 -149
  46. package/test/holster.user.off.test.js +0 -73
  47. package/test/holster.user.on.test.js +0 -300
  48. package/test/holster.user.put.test.js +0 -463
  49. package/test/kitty/! +0 -9
  50. package/test/names.js +0 -225
  51. package/test/radisk.cache.test.js +0 -213
  52. package/test/radisk.test.js +0 -331
  53. package/test/radix.test.js +0 -117
  54. package/test/sea.test.js +0 -284
  55. package/test/split.test.js +0 -317
  56. package/test/store.test.js +0 -316
  57. package/test/system/basic.test.js +0 -29
  58. package/test/system/chained-on.test.js +0 -41
  59. package/test/system/concurrent-listeners.test.js +0 -125
  60. package/test/system/immediate-read.test.js +0 -30
  61. package/test/system/listener-before-data.test.js +0 -51
  62. package/test/system/promise-wrapper.test.js +0 -47
  63. package/test/system/user-chained-on.test.js +0 -60
  64. package/test/system/user-concurrent-listeners.test.js +0 -142
  65. package/test/system/user-listener-before-data.test.js +0 -66
  66. package/test/user.test.js +0 -173
  67. package/test/wire.test.js +0 -158
package/ts/src/wire.ts ADDED
@@ -0,0 +1,829 @@
1
+ /**
2
+ * Wire - WebSocket-based network protocol
3
+ * Handles message routing, rate limiting, and graph synchronization
4
+ */
5
+
6
+ import Dup from "./dup.ts"
7
+ import Get from "./get.ts"
8
+ import Ham from "./ham.ts"
9
+ import Store from "./store.ts"
10
+ import * as utils from "./utils.ts"
11
+ import type {
12
+ Graph,
13
+ WireMessage,
14
+ WireOptions,
15
+ Lex,
16
+ ListenMap,
17
+ HolsterOptions,
18
+ GraphValue,
19
+ } from "./schemas.ts"
20
+
21
+ // Unified WebSocket type that works for both Node.js ws and browser WebSocket
22
+ type UnifiedWebSocket = WebSocket & {
23
+ _retryHandler?: ReturnType<typeof createRetryHandler>
24
+ on?: (event: string, cb: (...args: unknown[]) => void) => void
25
+ send: (data: string | Buffer | ArrayBuffer | Uint8Array, options?: unknown, cb?: (err?: Error) => void) => void
26
+ }
27
+
28
+ const isNode = typeof document === "undefined"
29
+
30
+ // Dynamic import for Node.js ws module - won't execute in browser/service worker
31
+ const wsModule = isNode ? await import("ws") : undefined
32
+
33
+ if (typeof globalThis.WebSocket === "undefined" && wsModule) {
34
+ globalThis.WebSocket = wsModule.WebSocket as unknown as typeof WebSocket
35
+ }
36
+
37
+ // ============================================================================
38
+ // Rate Limiting
39
+ // ============================================================================
40
+
41
+ interface RateLimiterClient {
42
+ requests: number[]
43
+ lastCleanup: number
44
+ throttleCount: number
45
+ }
46
+
47
+ const createRateLimiter = (isTestEnv: boolean) => {
48
+ const clients = new Map<string, RateLimiterClient>()
49
+ const maxRequests = 1500
50
+ const windowMs = 60000
51
+ const disconnectThreshold = 10
52
+ let cleanupInterval: NodeJS.Timeout | null = null
53
+
54
+ const cleanup = (): void => {
55
+ const now = Date.now()
56
+ for (const [_clientId, data] of Array.from(clients.entries())) {
57
+ if (now - data.lastCleanup > windowMs) {
58
+ data.requests = []
59
+ data.lastCleanup = now
60
+ }
61
+ data.requests = data.requests.filter(time => now - time < windowMs)
62
+ if (now - data.lastCleanup > windowMs * 10) {
63
+ data.throttleCount = 0
64
+ }
65
+ }
66
+ }
67
+
68
+ if (!isTestEnv) {
69
+ cleanupInterval = setInterval(cleanup, windowMs / 4)
70
+ }
71
+
72
+ return {
73
+ getDelay: (clientId: string): number => {
74
+ const now = Date.now()
75
+ const client = clients.get(clientId) || {
76
+ requests: [],
77
+ lastCleanup: now,
78
+ throttleCount: 0,
79
+ }
80
+
81
+ client.requests = client.requests.filter(time => now - time < windowMs)
82
+
83
+ if (client.requests.length >= maxRequests) {
84
+ const oldestRequest = Math.min(...client.requests)
85
+ const delay = windowMs - (now - oldestRequest)
86
+ client.throttleCount = (client.throttleCount || 0) + 1
87
+ clients.set(clientId, client)
88
+ return Math.max(0, delay)
89
+ }
90
+
91
+ client.requests.push(now)
92
+ clients.set(clientId, client)
93
+ return 0
94
+ },
95
+
96
+ getRemainingRequests: (clientId: string): number => {
97
+ const client = clients.get(clientId)
98
+ if (!client) return maxRequests
99
+
100
+ const now = Date.now()
101
+ const validRequests = client.requests.filter(time => now - time < windowMs)
102
+ return Math.max(0, maxRequests - validRequests.length)
103
+ },
104
+
105
+ getThrottleCount: (clientId: string): number => {
106
+ const client = clients.get(clientId)
107
+ return client ? client.throttleCount || 0 : 0
108
+ },
109
+
110
+ shouldDisconnect: (clientId: string): boolean => {
111
+ const client = clients.get(clientId)
112
+ if (!client) return false
113
+ return client.throttleCount >= disconnectThreshold
114
+ },
115
+
116
+ destroy: (): void => {
117
+ if (cleanupInterval) {
118
+ clearInterval(cleanupInterval)
119
+ cleanupInterval = null
120
+ }
121
+ clients.clear()
122
+ },
123
+ }
124
+ }
125
+
126
+ // ============================================================================
127
+ // Message Validation
128
+ // ============================================================================
129
+
130
+ const safeJSONParse = (
131
+ data: string | Buffer,
132
+ maxSize = 1024 * 1024
133
+ ): { success: boolean; data?: WireMessage; error?: string } => {
134
+ try {
135
+ if (typeof data === "string" && data.length > maxSize) {
136
+ throw new Error("Message too large")
137
+ }
138
+ return { success: true, data: JSON.parse(data.toString()) }
139
+ } catch (error) {
140
+ return { success: false, error: (error as Error).message }
141
+ }
142
+ }
143
+
144
+ const validateMessage = (
145
+ data: string | Buffer,
146
+ maxSize = 1024 * 1024
147
+ ): { valid: boolean; error?: string } => {
148
+ if (typeof data === "string" && data.length > maxSize) {
149
+ return { valid: false, error: "Message too large" }
150
+ }
151
+ if (Buffer.isBuffer && Buffer.isBuffer(data) && data.length > maxSize) {
152
+ return { valid: false, error: "Message too large" }
153
+ }
154
+ return { valid: true }
155
+ }
156
+
157
+ // ============================================================================
158
+ // Connection Management
159
+ // ============================================================================
160
+
161
+ const createConnectionManager = (maxConnections = 1000) => {
162
+ const connections = new Set<WebSocket>()
163
+
164
+ return {
165
+ add: (ws: WebSocket): boolean => {
166
+ if (connections.size >= maxConnections) {
167
+ return false
168
+ }
169
+ connections.add(ws)
170
+
171
+ const originalClose = ws.close.bind(ws)
172
+ ws.close = ((...args: unknown[]) => {
173
+ connections.delete(ws)
174
+ return originalClose(...(args as [number?, string?]))
175
+ }) as typeof ws.close
176
+
177
+ return true
178
+ },
179
+
180
+ remove: (ws: WebSocket): void => {
181
+ connections.delete(ws)
182
+ },
183
+
184
+ count: (): number => connections.size,
185
+
186
+ isFull: (): boolean => connections.size >= maxConnections,
187
+ }
188
+ }
189
+
190
+ // ============================================================================
191
+ // Retry Handler
192
+ // ============================================================================
193
+
194
+ const createRetryHandler = (maxRetries = 5) => {
195
+ let retryCount = 0
196
+
197
+ return {
198
+ shouldRetry: (): boolean => retryCount < maxRetries,
199
+ getDelay: (): number => Math.min(1000 * Math.pow(2, retryCount), 30000),
200
+ increment: (): void => {
201
+ retryCount++
202
+ },
203
+ reset: (): void => {
204
+ retryCount = 0
205
+ },
206
+ }
207
+ }
208
+
209
+ // ============================================================================
210
+ // Wire Protocol
211
+ // ============================================================================
212
+
213
+ export interface WireAPI {
214
+ get: (lex: Lex, cb: (msg: WireMessage) => void, _opt?: WireOptions) => void
215
+ put: (data: Graph, cb?: (err?: string | null) => void) => Promise<void> | void
216
+ on: (lex: Lex, cb: () => void, _get?: boolean, _opt?: WireOptions) => void
217
+ off: (lex: Lex, cb: () => void) => void
218
+ }
219
+
220
+ const Wire = (opt: HolsterOptions): WireAPI => {
221
+ const options = opt || {}
222
+ const dup = Dup(options.maxAge)
223
+ const store = Store(options as never)
224
+ const graph: Graph = {}
225
+ const queue: Record<string, (msg: WireMessage) => void> = {}
226
+ const listen: ListenMap = {}
227
+
228
+ const pendingReferences = new Set<string>()
229
+ const pendingTimeouts = new Map<
230
+ string,
231
+ { lex: Lex; wait: number; timeoutId?: NodeJS.Timeout }
232
+ >()
233
+
234
+ const hasSoul = async (soul: string): Promise<boolean> => {
235
+ if (graph[soul]) return true
236
+ return new Promise(resolve => {
237
+ store.get({ "#": soul }, (err, data) => {
238
+ resolve(!err && !!data && !!data[soul])
239
+ })
240
+ })
241
+ }
242
+
243
+ const isTestEnv = options.wss && (options.wss as { constructor?: { name?: string } }).constructor?.name === "Server"
244
+ const rateLimiter = createRateLimiter(isTestEnv)
245
+ const connectionManager = createConnectionManager(options.maxConnections || 1000)
246
+
247
+ const check = async (
248
+ data: Graph,
249
+ send: (msg: string) => { err?: string } | void,
250
+ cb?: (err?: string | null) => void
251
+ ): Promise<boolean> => {
252
+ const key = utils.userPublicKey
253
+
254
+ for (const soul of Object.keys(data)) {
255
+ const msg = await new Promise<WireMessage>(res => {
256
+ getWithCallback({ "#": soul, ".": key }, res, send)
257
+ })
258
+ if (msg.err) {
259
+ if (cb) cb(msg.err)
260
+ return false
261
+ }
262
+
263
+ const node = data[soul]!
264
+ if (
265
+ !msg.put ||
266
+ !msg.put[soul] ||
267
+ node[key] === undefined ||
268
+ msg.put[soul]![key] === node[key]
269
+ ) {
270
+ continue
271
+ }
272
+
273
+ if (cb) {
274
+ cb(`error in wire check public key does not match for soul: ${soul}`)
275
+ }
276
+ return false
277
+ }
278
+
279
+ return true
280
+ }
281
+
282
+ const get = (msg: { get: Lex; "#": string }, send: (msg: string) => void): void => {
283
+ const ack = Get(msg.get, graph)
284
+ if (ack) {
285
+ send(
286
+ JSON.stringify({
287
+ "#": dup.track(utils.text.random(9)),
288
+ "@": msg["#"],
289
+ put: ack,
290
+ })
291
+ )
292
+ } else {
293
+ store.get(
294
+ msg.get,
295
+ (err, ack) => {
296
+ send(
297
+ JSON.stringify({
298
+ "#": dup.track(utils.text.random(9)),
299
+ "@": msg["#"],
300
+ put: ack,
301
+ err: err,
302
+ })
303
+ )
304
+ },
305
+ { secure: true }
306
+ )
307
+ }
308
+ }
309
+
310
+ const put = async (
311
+ msg: { put: Graph; "#": string },
312
+ send: (msg: string) => void
313
+ ): Promise<void> => {
314
+ const update = await Ham.mix(msg.put, graph, options.secure || false, listen)
315
+ if (Object.keys(update.now).length === 0) {
316
+ if (Object.keys(update.defer).length !== 0) {
317
+ setTimeout(() => put({ put: update.defer, "#": msg["#"] }, send), update.wait)
318
+ }
319
+ return
320
+ }
321
+
322
+ if (!(await check(update.now, send as never))) return
323
+
324
+ store.put(update.now, err => {
325
+ send(
326
+ JSON.stringify({
327
+ "#": dup.track(utils.text.random(9)),
328
+ "@": msg["#"],
329
+ err: err,
330
+ })
331
+ )
332
+ update.listeners.forEach(cb => cb())
333
+ })
334
+
335
+ if (Object.keys(update.defer).length !== 0) {
336
+ setTimeout(() => put({ put: update.defer, "#": msg["#"] }, send), update.wait)
337
+ }
338
+ }
339
+
340
+ const getWithCallback = (
341
+ lex: Lex,
342
+ cb: (msg: WireMessage) => void,
343
+ send: (msg: string) => { err?: string } | void,
344
+ _opt?: WireOptions
345
+ ): void => {
346
+ if (!cb) return
347
+
348
+ const opts = _opt || {}
349
+ const ack = Get(lex, graph, opts.fast)
350
+ const track = utils.text.random(9)
351
+ const request = JSON.stringify({
352
+ "#": dup.track(track),
353
+ get: lex,
354
+ })
355
+
356
+ if (ack) {
357
+ const sendResult = send(request)
358
+ if (sendResult && sendResult.err) {
359
+ cb({ err: sendResult.err })
360
+ return
361
+ }
362
+ cb({ put: ack })
363
+ return
364
+ }
365
+
366
+ store.get(
367
+ lex,
368
+ (err, ack) => {
369
+ if (ack) {
370
+ // Populate in-memory graph from store data before sending the server
371
+ // request, so stale server responses with older state are correctly
372
+ // rejected by ham.mix.
373
+ for (const [soul, node] of Object.entries(ack)) {
374
+ if (!graph[soul] && node) graph[soul] = {...node} as unknown as Graph[string]
375
+ }
376
+ const sendResult = send(request)
377
+ if (sendResult && sendResult.err) {
378
+ cb({ err: sendResult.err })
379
+ return
380
+ }
381
+ cb(err ? { put: ack, err: err } : { put: ack })
382
+ return
383
+ }
384
+
385
+ if (err) console.log(err)
386
+
387
+ queue[track] = cb
388
+
389
+ pendingTimeouts.set(track, {
390
+ lex: lex,
391
+ wait: opts.wait || 100,
392
+ })
393
+
394
+ const sendResult = send(request)
395
+ if (sendResult && sendResult.err) {
396
+ cb({ err: sendResult.err })
397
+ delete queue[track]
398
+ pendingTimeouts.delete(track)
399
+ return
400
+ }
401
+ },
402
+ opts
403
+ )
404
+ }
405
+
406
+ const api = (send: (msg: string) => { err?: string } | void): WireAPI => {
407
+ return {
408
+ get: (lex, cb, _opt) => {
409
+ if (lex && typeof lex["."] === "number") {
410
+ lex = { ...lex, ".": String(lex["."]) }
411
+ }
412
+ if (lex && lex["#"]) {
413
+ pendingReferences.add(lex["#"])
414
+ }
415
+ getWithCallback(lex, cb, send, _opt)
416
+ },
417
+ put: async (data, cb) => {
418
+ const update = await Ham.mix(data, graph, options.secure || false, listen)
419
+ if (Object.keys(update.now).length === 0) {
420
+ if (cb) cb(null)
421
+ return
422
+ }
423
+
424
+ if (!(await check(update.now, send as never, cb))) {
425
+ return
426
+ }
427
+
428
+ for (const [_soul, node] of Object.entries(update.now)) {
429
+ if (node && typeof node === "object") {
430
+ for (const [_key, value] of Object.entries(node)) {
431
+ const soulId = utils.rel.is(value as GraphValue)
432
+ if (soulId) {
433
+ pendingReferences.add(soulId)
434
+ }
435
+ }
436
+ }
437
+ }
438
+
439
+ store.put(update.now, err => {
440
+ if (cb) cb(err)
441
+ update.listeners.forEach(l => l())
442
+ })
443
+
444
+ const sendResult = send(
445
+ JSON.stringify({
446
+ "#": dup.track(utils.text.random(9)),
447
+ put: data,
448
+ })
449
+ )
450
+ if (sendResult && sendResult.err) {
451
+ if (cb) cb(sendResult.err)
452
+ return
453
+ }
454
+ },
455
+ on: (lex, cb, _get, _opt) => {
456
+ if (lex && typeof lex["."] === "number") {
457
+ lex = { ...lex, ".": String(lex["."]) }
458
+ }
459
+ const soul = lex && lex["#"]
460
+ if (!soul || !cb) return
461
+
462
+ if (listen[soul]) {
463
+ listen[soul]!.push({ ".": lex["."], cb: cb as never })
464
+ } else {
465
+ listen[soul] = [{ ".": lex["."], cb: cb as never }]
466
+ }
467
+ if (_get) getWithCallback(lex, cb as never, send, _opt)
468
+ },
469
+ off: (lex, cb) => {
470
+ const soul = lex && lex["#"]
471
+ if (!soul || !listen[soul]) return
472
+
473
+ if (cb) {
474
+ const found = listen[soul]!.find(l => l.cb === (cb as never))
475
+ if (found) {
476
+ listen[soul]!.splice(listen[soul]!.indexOf(found), 1)
477
+ }
478
+ } else {
479
+ delete listen[soul]
480
+ }
481
+ },
482
+ }
483
+ }
484
+
485
+ if (isNode) {
486
+ let wss = options.wss
487
+ let clients = (): WebSocket[] =>
488
+ (wss as { clients?: () => WebSocket[] }).clients?.() || []
489
+ if (!wss) {
490
+ const config = options.server
491
+ ? { server: options.server }
492
+ : { port: options.port || 8765 }
493
+ wss = new wsModule!.WebSocketServer(config)
494
+ clients = () => Array.from((wss as { clients: Set<WebSocket> }).clients || [])
495
+ }
496
+
497
+ const send = (data: string, isBinary?: boolean): void => {
498
+ const msg = JSON.parse(data) as WireMessage
499
+ const trackId = msg["#"]
500
+ if (trackId && pendingTimeouts.has(trackId)) {
501
+ const timeoutConfig = pendingTimeouts.get(trackId)!
502
+ pendingTimeouts.delete(trackId)
503
+
504
+ timeoutConfig.timeoutId = setTimeout(() => {
505
+ const cb = queue[trackId]
506
+ if (cb) {
507
+ const id = timeoutConfig.lex["#"]
508
+ const ack: Graph = { [id]: null as never }
509
+ if (typeof timeoutConfig.lex["."] === "string") {
510
+ ack[id] = { [timeoutConfig.lex["."]]: null } as never
511
+ }
512
+ cb({ "#": trackId, put: ack })
513
+ delete queue[trackId]
514
+ }
515
+ }, timeoutConfig.wait)
516
+ }
517
+
518
+ clients().forEach(client => {
519
+ const ws = client as UnifiedWebSocket
520
+ if (ws && ws.readyState === WebSocket.OPEN) {
521
+ ws.send(data, { binary: isBinary })
522
+ } else if (ws) {
523
+ const retryHandler = ws._retryHandler || createRetryHandler()
524
+ ws._retryHandler = retryHandler
525
+
526
+ if (retryHandler.shouldRetry()) {
527
+ const delay = retryHandler.getDelay()
528
+ retryHandler.increment()
529
+
530
+ setTimeout(() => {
531
+ if (ws && ws.readyState === WebSocket.OPEN) {
532
+ retryHandler.reset()
533
+ ws.send(data, { binary: isBinary })
534
+ }
535
+ }, delay)
536
+ }
537
+ }
538
+ })
539
+ }
540
+
541
+ ;(wss as unknown as { on: (event: string, cb: (ws: UnifiedWebSocket) => void) => void }).on(
542
+ "connection",
543
+ (ws: UnifiedWebSocket) => {
544
+ if (!connectionManager.add(ws)) {
545
+ console.log("Connection limit reached, rejecting connection")
546
+ ws.close(1013, "Connection limit reached - try again later")
547
+ return
548
+ }
549
+
550
+ const clientId = utils.text.random(9)
551
+ console.log(`New WebSocket client connected: ${clientId}`)
552
+
553
+ ws.on?.("error", ((...args: unknown[]) => {
554
+ const error = args[0] as Error
555
+ console.log("WebSocket error:", error)
556
+ connectionManager.remove(ws)
557
+ }) as (...args: unknown[]) => void)
558
+
559
+ ws.on?.("close", (() => {
560
+ console.log(`WebSocket client disconnected: ${clientId}`)
561
+ connectionManager.remove(ws)
562
+ }) as (...args: unknown[]) => void)
563
+
564
+ ws.on?.("message", ((...args: unknown[]) => {
565
+ const data = args[0] as Buffer
566
+ const isBinary = args[1] as boolean
567
+ const validation = validateMessage(data, options.maxMessageSize)
568
+ if (!validation.valid) {
569
+ console.warn(`Invalid message: ${validation.error}`)
570
+ ws.send(JSON.stringify({ error: validation.error }))
571
+ return
572
+ }
573
+
574
+ const parseResult = safeJSONParse(data, options.maxMessageSize)
575
+ if (!parseResult.success) {
576
+ console.warn(`JSON parse error: ${parseResult.error}`)
577
+ ws.send(JSON.stringify({ error: "Invalid JSON" }))
578
+ return
579
+ }
580
+
581
+ const msg = parseResult.data!
582
+ if (!msg["#"]) return // Incoming messages must have '#'
583
+ if (dup.check(msg["#"])) return
584
+
585
+ const delay = rateLimiter.getDelay(clientId)
586
+ if (delay > 0) {
587
+ const throttleCount = rateLimiter.getThrottleCount(clientId)
588
+ console.log(
589
+ `Client ${clientId}: rate limit exceeded, delay would be ${delay}ms, dropping message (throttle count: ${throttleCount})`
590
+ )
591
+ if (rateLimiter.shouldDisconnect(clientId)) {
592
+ console.log(
593
+ `Client ${clientId}: Disconnecting after ${throttleCount} throttle violations`
594
+ )
595
+ ws.close(1008, "Rate limit violations")
596
+ return
597
+ }
598
+
599
+ ws.send(
600
+ JSON.stringify({
601
+ "#": dup.track(utils.text.random(9)),
602
+ "@": msg["#"],
603
+ err: `Rate limit exceeded. Slow down requests. Wait ${Math.ceil(delay / 1000)}s`,
604
+ throttle: delay,
605
+ })
606
+ )
607
+ return
608
+ }
609
+
610
+ const processMessage = async (): Promise<void> => {
611
+ dup.track(msg["#"]!)
612
+
613
+ if (msg.get) get(msg as never, send)
614
+ if (msg.put) await put(msg as never, send)
615
+ send(data.toString(), isBinary)
616
+
617
+ const id = msg["@"]
618
+ const cb = queue[id!]
619
+ if (cb) {
620
+ delete (msg as { "#"?: string })["#"]
621
+ delete (msg as { "@"?: string })["@"]
622
+ cb(msg)
623
+ delete queue[id!]
624
+ }
625
+ }
626
+
627
+ if (delay > 0) {
628
+ setTimeout(processMessage, delay)
629
+ } else {
630
+ processMessage()
631
+ }
632
+ }) as (...args: unknown[]) => void)
633
+ }
634
+ )
635
+ return api(send)
636
+ }
637
+
638
+ // Browser logic
639
+ const peers: WebSocket[] = []
640
+ let clientThrottled = false
641
+ let throttleUntil = 0
642
+ let messageQueue: string[] = []
643
+ let queueProcessor: NodeJS.Timeout | null = null
644
+ const maxQueueLength = options.maxQueueLength || 10000
645
+
646
+ const processQueue = (): void => {
647
+ if (messageQueue.length === 0) {
648
+ queueProcessor = null
649
+ return
650
+ }
651
+
652
+ const now = Date.now()
653
+ if (clientThrottled && now < throttleUntil) {
654
+ queueProcessor = setTimeout(processQueue, Math.min(1000, throttleUntil - now))
655
+ return
656
+ }
657
+
658
+ if (clientThrottled && now >= throttleUntil) {
659
+ console.log(`Throttle period ended, resuming queue processing`)
660
+ clientThrottled = false
661
+ throttleUntil = 0
662
+ }
663
+
664
+ const data = messageQueue[0]!
665
+ const sent = sendToPeers(data)
666
+
667
+ if (sent) {
668
+ messageQueue.shift()
669
+
670
+ const msg = JSON.parse(data) as WireMessage
671
+ const trackId = msg["#"]
672
+ if (trackId && pendingTimeouts.has(trackId)) {
673
+ const timeoutConfig = pendingTimeouts.get(trackId)!
674
+ pendingTimeouts.delete(trackId)
675
+
676
+ timeoutConfig.timeoutId = setTimeout(() => {
677
+ const cb = queue[trackId]
678
+ if (cb) {
679
+ const id = timeoutConfig.lex["#"]
680
+ const ack: Graph = { [id]: null as never }
681
+ if (typeof timeoutConfig.lex["."] === "string") {
682
+ ack[id] = { [timeoutConfig.lex["."]]: null } as never
683
+ }
684
+ cb({ "#": trackId, put: ack })
685
+ delete queue[trackId]
686
+ }
687
+ }, timeoutConfig.wait)
688
+ }
689
+ }
690
+
691
+ if (messageQueue.length > 0) {
692
+ queueProcessor = setTimeout(processQueue, 50)
693
+ } else {
694
+ queueProcessor = null
695
+ }
696
+ }
697
+
698
+ const sendToPeers = (data: string): boolean => {
699
+ let sentToAtLeastOne = false
700
+ peers.forEach(peer => {
701
+ const ws = peer as UnifiedWebSocket
702
+ if (ws && ws.readyState === WebSocket.OPEN) {
703
+ ws.send(data)
704
+ sentToAtLeastOne = true
705
+ } else if (ws) {
706
+ const retryHandler = ws._retryHandler || createRetryHandler()
707
+ ws._retryHandler = retryHandler
708
+
709
+ if (retryHandler.shouldRetry()) {
710
+ const delay = retryHandler.getDelay()
711
+ retryHandler.increment()
712
+
713
+ setTimeout(() => {
714
+ if (peer && peer.readyState === WebSocket.OPEN) {
715
+ retryHandler.reset()
716
+ peer.send(data)
717
+ }
718
+ }, delay)
719
+ }
720
+ }
721
+ })
722
+ return sentToAtLeastOne
723
+ }
724
+
725
+ const send = (data: string): { err?: string } | void => {
726
+ if (messageQueue.length >= maxQueueLength) {
727
+ return {
728
+ err: `Message queue exceeded maximum length (${maxQueueLength}). Update query logic to request less data.`,
729
+ }
730
+ }
731
+
732
+ messageQueue.push(data)
733
+ if (!queueProcessor) {
734
+ queueProcessor = setTimeout(processQueue, 50)
735
+ }
736
+ }
737
+
738
+ if (!(options.peers instanceof Array) || options.peers.length === 0) {
739
+ options.peers = [`ws://localhost:${options.port || 8765}`]
740
+ }
741
+
742
+ options.peers.forEach((peer: string) => {
743
+ const start = (): void => {
744
+ let ws: WebSocket | null = new WebSocket(peer)
745
+ peers.push(ws)
746
+ const retryHandler = createRetryHandler()
747
+
748
+ ws.onclose = () => {
749
+ if (ws && peers.indexOf(ws) !== -1) {
750
+ peers.splice(peers.indexOf(ws), 1)
751
+ }
752
+ ws = null
753
+
754
+ if (retryHandler.shouldRetry()) {
755
+ const delay = retryHandler.getDelay()
756
+ retryHandler.increment()
757
+ setTimeout(start, delay)
758
+ }
759
+ }
760
+
761
+ ws.onopen = () => {
762
+ retryHandler.reset()
763
+ }
764
+
765
+ ws.onerror = (e: Event) => {
766
+ console.log(e)
767
+ }
768
+
769
+ ws.onmessage = async (m: MessageEvent) => {
770
+ const msg = JSON.parse(m.data as string) as WireMessage
771
+ if (!msg["#"]) return // Incoming messages must have '#'
772
+ if (dup.check(msg["#"])) return
773
+
774
+ if (msg.throttle && msg.err) {
775
+ console.log(`Server throttling: ${msg.err}`)
776
+ clientThrottled = true
777
+ throttleUntil = Date.now() + msg.throttle
778
+ return
779
+ }
780
+
781
+ dup.track(msg["#"]!)
782
+ if (msg.get) get(msg as never, send as never)
783
+ if (msg.put) {
784
+ const filteredPut: Graph = {}
785
+ for (const [soul, node] of Object.entries(msg.put)) {
786
+ let shouldStore = false
787
+ if (await hasSoul(soul)) {
788
+ shouldStore = true
789
+ if (node && typeof node === "object") {
790
+ for (const [_key, value] of Object.entries(node)) {
791
+ const soulId = utils.rel.is(value as GraphValue)
792
+ if (soulId) {
793
+ pendingReferences.add(soulId)
794
+ }
795
+ }
796
+ }
797
+ }
798
+ if (pendingReferences.has(soul)) {
799
+ shouldStore = true
800
+ pendingReferences.delete(soul)
801
+ }
802
+ if (shouldStore) {
803
+ filteredPut[soul] = node
804
+ }
805
+ }
806
+ if (Object.keys(filteredPut).length > 0) {
807
+ const filteredMsg = { put: filteredPut, "#": msg["#"]! }
808
+ await put(filteredMsg, send as never)
809
+ }
810
+ }
811
+
812
+ const id = msg["@"]
813
+ const cb = queue[id!]
814
+ if (cb) {
815
+ delete (msg as { "#"?: string })["#"]
816
+ delete (msg as { "@"?: string })["@"]
817
+ cb(msg)
818
+ delete queue[id!]
819
+ }
820
+ }
821
+ }
822
+ start()
823
+ })
824
+
825
+ return api(send)
826
+ }
827
+
828
+ export default Wire
829
+