@mblaney/holster 1.0.35 → 1.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/holster.js +3 -3
- package/build/holster.js.map +3 -3
- package/dist/holster.js +4 -0
- package/dist/holster.js.map +7 -0
- package/package.json +23 -5
- package/src/get.js +6 -18
- package/src/ham.js +17 -47
- package/src/holster.js +106 -28
- package/src/sea.js +0 -52
- package/src/store.js +7 -7
- package/src/wire.js +7 -1
- package/ts/src/array.ts +78 -0
- package/ts/src/buffer.ts +441 -0
- package/ts/src/dup.ts +49 -0
- package/ts/src/get.ts +83 -0
- package/ts/src/ham.ts +254 -0
- package/ts/src/holster.ts +931 -0
- package/ts/src/index.ts +35 -0
- package/ts/src/radisk.ts +585 -0
- package/ts/src/radix.ts +157 -0
- package/ts/src/schemas.ts +519 -0
- package/ts/src/sea-utils.ts +103 -0
- package/ts/src/sea.ts +388 -0
- package/ts/src/store.ts +321 -0
- package/ts/src/user.ts +381 -0
- package/ts/src/utils.ts +232 -0
- package/ts/src/wire.ts +829 -0
- package/.dockerignore +0 -9
- package/.github/workflows/node.js.yml +0 -31
- package/.github/workflows/npm-publish.yml +0 -33
- package/Dockerfile +0 -20
- package/examples/index.html +0 -33
- package/examples/index.js +0 -49
- package/examples/user.html +0 -31
- package/examples/user.js +0 -93
- package/examples/wire.html +0 -27
- package/examples/wire.js +0 -55
- package/test/holster.get.test.js +0 -94
- package/test/holster.lex.test.js +0 -196
- package/test/holster.off.test.js +0 -53
- package/test/holster.on.test.js +0 -264
- package/test/holster.put.test.js +0 -358
- package/test/holster.secure.test.js +0 -202
- package/test/holster.user.get.test.js +0 -114
- package/test/holster.user.lex.test.js +0 -149
- package/test/holster.user.off.test.js +0 -73
- package/test/holster.user.on.test.js +0 -300
- package/test/holster.user.put.test.js +0 -463
- package/test/kitty/! +0 -9
- package/test/names.js +0 -225
- package/test/radisk.cache.test.js +0 -213
- package/test/radisk.test.js +0 -331
- package/test/radix.test.js +0 -117
- package/test/sea.test.js +0 -284
- package/test/split.test.js +0 -317
- package/test/store.test.js +0 -316
- package/test/system/basic.test.js +0 -29
- package/test/system/chained-on.test.js +0 -41
- package/test/system/concurrent-listeners.test.js +0 -125
- package/test/system/immediate-read.test.js +0 -30
- package/test/system/listener-before-data.test.js +0 -51
- package/test/system/promise-wrapper.test.js +0 -47
- package/test/system/user-chained-on.test.js +0 -60
- package/test/system/user-concurrent-listeners.test.js +0 -142
- package/test/system/user-listener-before-data.test.js +0 -66
- package/test/user.test.js +0 -173
- package/test/wire.test.js +0 -158
package/ts/src/wire.ts
ADDED
|
@@ -0,0 +1,829 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wire - WebSocket-based network protocol
|
|
3
|
+
* Handles message routing, rate limiting, and graph synchronization
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import Dup from "./dup.ts"
|
|
7
|
+
import Get from "./get.ts"
|
|
8
|
+
import Ham from "./ham.ts"
|
|
9
|
+
import Store from "./store.ts"
|
|
10
|
+
import * as utils from "./utils.ts"
|
|
11
|
+
import type {
|
|
12
|
+
Graph,
|
|
13
|
+
WireMessage,
|
|
14
|
+
WireOptions,
|
|
15
|
+
Lex,
|
|
16
|
+
ListenMap,
|
|
17
|
+
HolsterOptions,
|
|
18
|
+
GraphValue,
|
|
19
|
+
} from "./schemas.ts"
|
|
20
|
+
|
|
21
|
+
// Unified WebSocket type that works for both Node.js ws and browser WebSocket
|
|
22
|
+
type UnifiedWebSocket = WebSocket & {
|
|
23
|
+
_retryHandler?: ReturnType<typeof createRetryHandler>
|
|
24
|
+
on?: (event: string, cb: (...args: unknown[]) => void) => void
|
|
25
|
+
send: (data: string | Buffer | ArrayBuffer | Uint8Array, options?: unknown, cb?: (err?: Error) => void) => void
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const isNode = typeof document === "undefined"
|
|
29
|
+
|
|
30
|
+
// Dynamic import for Node.js ws module - won't execute in browser/service worker
|
|
31
|
+
const wsModule = isNode ? await import("ws") : undefined
|
|
32
|
+
|
|
33
|
+
if (typeof globalThis.WebSocket === "undefined" && wsModule) {
|
|
34
|
+
globalThis.WebSocket = wsModule.WebSocket as unknown as typeof WebSocket
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ============================================================================
|
|
38
|
+
// Rate Limiting
|
|
39
|
+
// ============================================================================
|
|
40
|
+
|
|
41
|
+
interface RateLimiterClient {
|
|
42
|
+
requests: number[]
|
|
43
|
+
lastCleanup: number
|
|
44
|
+
throttleCount: number
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const createRateLimiter = (isTestEnv: boolean) => {
|
|
48
|
+
const clients = new Map<string, RateLimiterClient>()
|
|
49
|
+
const maxRequests = 1500
|
|
50
|
+
const windowMs = 60000
|
|
51
|
+
const disconnectThreshold = 10
|
|
52
|
+
let cleanupInterval: NodeJS.Timeout | null = null
|
|
53
|
+
|
|
54
|
+
const cleanup = (): void => {
|
|
55
|
+
const now = Date.now()
|
|
56
|
+
for (const [_clientId, data] of Array.from(clients.entries())) {
|
|
57
|
+
if (now - data.lastCleanup > windowMs) {
|
|
58
|
+
data.requests = []
|
|
59
|
+
data.lastCleanup = now
|
|
60
|
+
}
|
|
61
|
+
data.requests = data.requests.filter(time => now - time < windowMs)
|
|
62
|
+
if (now - data.lastCleanup > windowMs * 10) {
|
|
63
|
+
data.throttleCount = 0
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (!isTestEnv) {
|
|
69
|
+
cleanupInterval = setInterval(cleanup, windowMs / 4)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
getDelay: (clientId: string): number => {
|
|
74
|
+
const now = Date.now()
|
|
75
|
+
const client = clients.get(clientId) || {
|
|
76
|
+
requests: [],
|
|
77
|
+
lastCleanup: now,
|
|
78
|
+
throttleCount: 0,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
client.requests = client.requests.filter(time => now - time < windowMs)
|
|
82
|
+
|
|
83
|
+
if (client.requests.length >= maxRequests) {
|
|
84
|
+
const oldestRequest = Math.min(...client.requests)
|
|
85
|
+
const delay = windowMs - (now - oldestRequest)
|
|
86
|
+
client.throttleCount = (client.throttleCount || 0) + 1
|
|
87
|
+
clients.set(clientId, client)
|
|
88
|
+
return Math.max(0, delay)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
client.requests.push(now)
|
|
92
|
+
clients.set(clientId, client)
|
|
93
|
+
return 0
|
|
94
|
+
},
|
|
95
|
+
|
|
96
|
+
getRemainingRequests: (clientId: string): number => {
|
|
97
|
+
const client = clients.get(clientId)
|
|
98
|
+
if (!client) return maxRequests
|
|
99
|
+
|
|
100
|
+
const now = Date.now()
|
|
101
|
+
const validRequests = client.requests.filter(time => now - time < windowMs)
|
|
102
|
+
return Math.max(0, maxRequests - validRequests.length)
|
|
103
|
+
},
|
|
104
|
+
|
|
105
|
+
getThrottleCount: (clientId: string): number => {
|
|
106
|
+
const client = clients.get(clientId)
|
|
107
|
+
return client ? client.throttleCount || 0 : 0
|
|
108
|
+
},
|
|
109
|
+
|
|
110
|
+
shouldDisconnect: (clientId: string): boolean => {
|
|
111
|
+
const client = clients.get(clientId)
|
|
112
|
+
if (!client) return false
|
|
113
|
+
return client.throttleCount >= disconnectThreshold
|
|
114
|
+
},
|
|
115
|
+
|
|
116
|
+
destroy: (): void => {
|
|
117
|
+
if (cleanupInterval) {
|
|
118
|
+
clearInterval(cleanupInterval)
|
|
119
|
+
cleanupInterval = null
|
|
120
|
+
}
|
|
121
|
+
clients.clear()
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ============================================================================
|
|
127
|
+
// Message Validation
|
|
128
|
+
// ============================================================================
|
|
129
|
+
|
|
130
|
+
const safeJSONParse = (
|
|
131
|
+
data: string | Buffer,
|
|
132
|
+
maxSize = 1024 * 1024
|
|
133
|
+
): { success: boolean; data?: WireMessage; error?: string } => {
|
|
134
|
+
try {
|
|
135
|
+
if (typeof data === "string" && data.length > maxSize) {
|
|
136
|
+
throw new Error("Message too large")
|
|
137
|
+
}
|
|
138
|
+
return { success: true, data: JSON.parse(data.toString()) }
|
|
139
|
+
} catch (error) {
|
|
140
|
+
return { success: false, error: (error as Error).message }
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const validateMessage = (
|
|
145
|
+
data: string | Buffer,
|
|
146
|
+
maxSize = 1024 * 1024
|
|
147
|
+
): { valid: boolean; error?: string } => {
|
|
148
|
+
if (typeof data === "string" && data.length > maxSize) {
|
|
149
|
+
return { valid: false, error: "Message too large" }
|
|
150
|
+
}
|
|
151
|
+
if (Buffer.isBuffer && Buffer.isBuffer(data) && data.length > maxSize) {
|
|
152
|
+
return { valid: false, error: "Message too large" }
|
|
153
|
+
}
|
|
154
|
+
return { valid: true }
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// ============================================================================
|
|
158
|
+
// Connection Management
|
|
159
|
+
// ============================================================================
|
|
160
|
+
|
|
161
|
+
const createConnectionManager = (maxConnections = 1000) => {
|
|
162
|
+
const connections = new Set<WebSocket>()
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
add: (ws: WebSocket): boolean => {
|
|
166
|
+
if (connections.size >= maxConnections) {
|
|
167
|
+
return false
|
|
168
|
+
}
|
|
169
|
+
connections.add(ws)
|
|
170
|
+
|
|
171
|
+
const originalClose = ws.close.bind(ws)
|
|
172
|
+
ws.close = ((...args: unknown[]) => {
|
|
173
|
+
connections.delete(ws)
|
|
174
|
+
return originalClose(...(args as [number?, string?]))
|
|
175
|
+
}) as typeof ws.close
|
|
176
|
+
|
|
177
|
+
return true
|
|
178
|
+
},
|
|
179
|
+
|
|
180
|
+
remove: (ws: WebSocket): void => {
|
|
181
|
+
connections.delete(ws)
|
|
182
|
+
},
|
|
183
|
+
|
|
184
|
+
count: (): number => connections.size,
|
|
185
|
+
|
|
186
|
+
isFull: (): boolean => connections.size >= maxConnections,
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// ============================================================================
|
|
191
|
+
// Retry Handler
|
|
192
|
+
// ============================================================================
|
|
193
|
+
|
|
194
|
+
const createRetryHandler = (maxRetries = 5) => {
|
|
195
|
+
let retryCount = 0
|
|
196
|
+
|
|
197
|
+
return {
|
|
198
|
+
shouldRetry: (): boolean => retryCount < maxRetries,
|
|
199
|
+
getDelay: (): number => Math.min(1000 * Math.pow(2, retryCount), 30000),
|
|
200
|
+
increment: (): void => {
|
|
201
|
+
retryCount++
|
|
202
|
+
},
|
|
203
|
+
reset: (): void => {
|
|
204
|
+
retryCount = 0
|
|
205
|
+
},
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// ============================================================================
|
|
210
|
+
// Wire Protocol
|
|
211
|
+
// ============================================================================
|
|
212
|
+
|
|
213
|
+
export interface WireAPI {
|
|
214
|
+
get: (lex: Lex, cb: (msg: WireMessage) => void, _opt?: WireOptions) => void
|
|
215
|
+
put: (data: Graph, cb?: (err?: string | null) => void) => Promise<void> | void
|
|
216
|
+
on: (lex: Lex, cb: () => void, _get?: boolean, _opt?: WireOptions) => void
|
|
217
|
+
off: (lex: Lex, cb: () => void) => void
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const Wire = (opt: HolsterOptions): WireAPI => {
|
|
221
|
+
const options = opt || {}
|
|
222
|
+
const dup = Dup(options.maxAge)
|
|
223
|
+
const store = Store(options as never)
|
|
224
|
+
const graph: Graph = {}
|
|
225
|
+
const queue: Record<string, (msg: WireMessage) => void> = {}
|
|
226
|
+
const listen: ListenMap = {}
|
|
227
|
+
|
|
228
|
+
const pendingReferences = new Set<string>()
|
|
229
|
+
const pendingTimeouts = new Map<
|
|
230
|
+
string,
|
|
231
|
+
{ lex: Lex; wait: number; timeoutId?: NodeJS.Timeout }
|
|
232
|
+
>()
|
|
233
|
+
|
|
234
|
+
const hasSoul = async (soul: string): Promise<boolean> => {
|
|
235
|
+
if (graph[soul]) return true
|
|
236
|
+
return new Promise(resolve => {
|
|
237
|
+
store.get({ "#": soul }, (err, data) => {
|
|
238
|
+
resolve(!err && !!data && !!data[soul])
|
|
239
|
+
})
|
|
240
|
+
})
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const isTestEnv = options.wss && (options.wss as { constructor?: { name?: string } }).constructor?.name === "Server"
|
|
244
|
+
const rateLimiter = createRateLimiter(isTestEnv)
|
|
245
|
+
const connectionManager = createConnectionManager(options.maxConnections || 1000)
|
|
246
|
+
|
|
247
|
+
const check = async (
|
|
248
|
+
data: Graph,
|
|
249
|
+
send: (msg: string) => { err?: string } | void,
|
|
250
|
+
cb?: (err?: string | null) => void
|
|
251
|
+
): Promise<boolean> => {
|
|
252
|
+
const key = utils.userPublicKey
|
|
253
|
+
|
|
254
|
+
for (const soul of Object.keys(data)) {
|
|
255
|
+
const msg = await new Promise<WireMessage>(res => {
|
|
256
|
+
getWithCallback({ "#": soul, ".": key }, res, send)
|
|
257
|
+
})
|
|
258
|
+
if (msg.err) {
|
|
259
|
+
if (cb) cb(msg.err)
|
|
260
|
+
return false
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const node = data[soul]!
|
|
264
|
+
if (
|
|
265
|
+
!msg.put ||
|
|
266
|
+
!msg.put[soul] ||
|
|
267
|
+
node[key] === undefined ||
|
|
268
|
+
msg.put[soul]![key] === node[key]
|
|
269
|
+
) {
|
|
270
|
+
continue
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
if (cb) {
|
|
274
|
+
cb(`error in wire check public key does not match for soul: ${soul}`)
|
|
275
|
+
}
|
|
276
|
+
return false
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return true
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const get = (msg: { get: Lex; "#": string }, send: (msg: string) => void): void => {
|
|
283
|
+
const ack = Get(msg.get, graph)
|
|
284
|
+
if (ack) {
|
|
285
|
+
send(
|
|
286
|
+
JSON.stringify({
|
|
287
|
+
"#": dup.track(utils.text.random(9)),
|
|
288
|
+
"@": msg["#"],
|
|
289
|
+
put: ack,
|
|
290
|
+
})
|
|
291
|
+
)
|
|
292
|
+
} else {
|
|
293
|
+
store.get(
|
|
294
|
+
msg.get,
|
|
295
|
+
(err, ack) => {
|
|
296
|
+
send(
|
|
297
|
+
JSON.stringify({
|
|
298
|
+
"#": dup.track(utils.text.random(9)),
|
|
299
|
+
"@": msg["#"],
|
|
300
|
+
put: ack,
|
|
301
|
+
err: err,
|
|
302
|
+
})
|
|
303
|
+
)
|
|
304
|
+
},
|
|
305
|
+
{ secure: true }
|
|
306
|
+
)
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const put = async (
|
|
311
|
+
msg: { put: Graph; "#": string },
|
|
312
|
+
send: (msg: string) => void
|
|
313
|
+
): Promise<void> => {
|
|
314
|
+
const update = await Ham.mix(msg.put, graph, options.secure || false, listen)
|
|
315
|
+
if (Object.keys(update.now).length === 0) {
|
|
316
|
+
if (Object.keys(update.defer).length !== 0) {
|
|
317
|
+
setTimeout(() => put({ put: update.defer, "#": msg["#"] }, send), update.wait)
|
|
318
|
+
}
|
|
319
|
+
return
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if (!(await check(update.now, send as never))) return
|
|
323
|
+
|
|
324
|
+
store.put(update.now, err => {
|
|
325
|
+
send(
|
|
326
|
+
JSON.stringify({
|
|
327
|
+
"#": dup.track(utils.text.random(9)),
|
|
328
|
+
"@": msg["#"],
|
|
329
|
+
err: err,
|
|
330
|
+
})
|
|
331
|
+
)
|
|
332
|
+
update.listeners.forEach(cb => cb())
|
|
333
|
+
})
|
|
334
|
+
|
|
335
|
+
if (Object.keys(update.defer).length !== 0) {
|
|
336
|
+
setTimeout(() => put({ put: update.defer, "#": msg["#"] }, send), update.wait)
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const getWithCallback = (
|
|
341
|
+
lex: Lex,
|
|
342
|
+
cb: (msg: WireMessage) => void,
|
|
343
|
+
send: (msg: string) => { err?: string } | void,
|
|
344
|
+
_opt?: WireOptions
|
|
345
|
+
): void => {
|
|
346
|
+
if (!cb) return
|
|
347
|
+
|
|
348
|
+
const opts = _opt || {}
|
|
349
|
+
const ack = Get(lex, graph, opts.fast)
|
|
350
|
+
const track = utils.text.random(9)
|
|
351
|
+
const request = JSON.stringify({
|
|
352
|
+
"#": dup.track(track),
|
|
353
|
+
get: lex,
|
|
354
|
+
})
|
|
355
|
+
|
|
356
|
+
if (ack) {
|
|
357
|
+
const sendResult = send(request)
|
|
358
|
+
if (sendResult && sendResult.err) {
|
|
359
|
+
cb({ err: sendResult.err })
|
|
360
|
+
return
|
|
361
|
+
}
|
|
362
|
+
cb({ put: ack })
|
|
363
|
+
return
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
store.get(
|
|
367
|
+
lex,
|
|
368
|
+
(err, ack) => {
|
|
369
|
+
if (ack) {
|
|
370
|
+
// Populate in-memory graph from store data before sending the server
|
|
371
|
+
// request, so stale server responses with older state are correctly
|
|
372
|
+
// rejected by ham.mix.
|
|
373
|
+
for (const [soul, node] of Object.entries(ack)) {
|
|
374
|
+
if (!graph[soul] && node) graph[soul] = {...node} as unknown as Graph[string]
|
|
375
|
+
}
|
|
376
|
+
const sendResult = send(request)
|
|
377
|
+
if (sendResult && sendResult.err) {
|
|
378
|
+
cb({ err: sendResult.err })
|
|
379
|
+
return
|
|
380
|
+
}
|
|
381
|
+
cb(err ? { put: ack, err: err } : { put: ack })
|
|
382
|
+
return
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
if (err) console.log(err)
|
|
386
|
+
|
|
387
|
+
queue[track] = cb
|
|
388
|
+
|
|
389
|
+
pendingTimeouts.set(track, {
|
|
390
|
+
lex: lex,
|
|
391
|
+
wait: opts.wait || 100,
|
|
392
|
+
})
|
|
393
|
+
|
|
394
|
+
const sendResult = send(request)
|
|
395
|
+
if (sendResult && sendResult.err) {
|
|
396
|
+
cb({ err: sendResult.err })
|
|
397
|
+
delete queue[track]
|
|
398
|
+
pendingTimeouts.delete(track)
|
|
399
|
+
return
|
|
400
|
+
}
|
|
401
|
+
},
|
|
402
|
+
opts
|
|
403
|
+
)
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
const api = (send: (msg: string) => { err?: string } | void): WireAPI => {
|
|
407
|
+
return {
|
|
408
|
+
get: (lex, cb, _opt) => {
|
|
409
|
+
if (lex && typeof lex["."] === "number") {
|
|
410
|
+
lex = { ...lex, ".": String(lex["."]) }
|
|
411
|
+
}
|
|
412
|
+
if (lex && lex["#"]) {
|
|
413
|
+
pendingReferences.add(lex["#"])
|
|
414
|
+
}
|
|
415
|
+
getWithCallback(lex, cb, send, _opt)
|
|
416
|
+
},
|
|
417
|
+
put: async (data, cb) => {
|
|
418
|
+
const update = await Ham.mix(data, graph, options.secure || false, listen)
|
|
419
|
+
if (Object.keys(update.now).length === 0) {
|
|
420
|
+
if (cb) cb(null)
|
|
421
|
+
return
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
if (!(await check(update.now, send as never, cb))) {
|
|
425
|
+
return
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
for (const [_soul, node] of Object.entries(update.now)) {
|
|
429
|
+
if (node && typeof node === "object") {
|
|
430
|
+
for (const [_key, value] of Object.entries(node)) {
|
|
431
|
+
const soulId = utils.rel.is(value as GraphValue)
|
|
432
|
+
if (soulId) {
|
|
433
|
+
pendingReferences.add(soulId)
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
store.put(update.now, err => {
|
|
440
|
+
if (cb) cb(err)
|
|
441
|
+
update.listeners.forEach(l => l())
|
|
442
|
+
})
|
|
443
|
+
|
|
444
|
+
const sendResult = send(
|
|
445
|
+
JSON.stringify({
|
|
446
|
+
"#": dup.track(utils.text.random(9)),
|
|
447
|
+
put: data,
|
|
448
|
+
})
|
|
449
|
+
)
|
|
450
|
+
if (sendResult && sendResult.err) {
|
|
451
|
+
if (cb) cb(sendResult.err)
|
|
452
|
+
return
|
|
453
|
+
}
|
|
454
|
+
},
|
|
455
|
+
on: (lex, cb, _get, _opt) => {
|
|
456
|
+
if (lex && typeof lex["."] === "number") {
|
|
457
|
+
lex = { ...lex, ".": String(lex["."]) }
|
|
458
|
+
}
|
|
459
|
+
const soul = lex && lex["#"]
|
|
460
|
+
if (!soul || !cb) return
|
|
461
|
+
|
|
462
|
+
if (listen[soul]) {
|
|
463
|
+
listen[soul]!.push({ ".": lex["."], cb: cb as never })
|
|
464
|
+
} else {
|
|
465
|
+
listen[soul] = [{ ".": lex["."], cb: cb as never }]
|
|
466
|
+
}
|
|
467
|
+
if (_get) getWithCallback(lex, cb as never, send, _opt)
|
|
468
|
+
},
|
|
469
|
+
off: (lex, cb) => {
|
|
470
|
+
const soul = lex && lex["#"]
|
|
471
|
+
if (!soul || !listen[soul]) return
|
|
472
|
+
|
|
473
|
+
if (cb) {
|
|
474
|
+
const found = listen[soul]!.find(l => l.cb === (cb as never))
|
|
475
|
+
if (found) {
|
|
476
|
+
listen[soul]!.splice(listen[soul]!.indexOf(found), 1)
|
|
477
|
+
}
|
|
478
|
+
} else {
|
|
479
|
+
delete listen[soul]
|
|
480
|
+
}
|
|
481
|
+
},
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
if (isNode) {
|
|
486
|
+
let wss = options.wss
|
|
487
|
+
let clients = (): WebSocket[] =>
|
|
488
|
+
(wss as { clients?: () => WebSocket[] }).clients?.() || []
|
|
489
|
+
if (!wss) {
|
|
490
|
+
const config = options.server
|
|
491
|
+
? { server: options.server }
|
|
492
|
+
: { port: options.port || 8765 }
|
|
493
|
+
wss = new wsModule!.WebSocketServer(config)
|
|
494
|
+
clients = () => Array.from((wss as { clients: Set<WebSocket> }).clients || [])
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
const send = (data: string, isBinary?: boolean): void => {
|
|
498
|
+
const msg = JSON.parse(data) as WireMessage
|
|
499
|
+
const trackId = msg["#"]
|
|
500
|
+
if (trackId && pendingTimeouts.has(trackId)) {
|
|
501
|
+
const timeoutConfig = pendingTimeouts.get(trackId)!
|
|
502
|
+
pendingTimeouts.delete(trackId)
|
|
503
|
+
|
|
504
|
+
timeoutConfig.timeoutId = setTimeout(() => {
|
|
505
|
+
const cb = queue[trackId]
|
|
506
|
+
if (cb) {
|
|
507
|
+
const id = timeoutConfig.lex["#"]
|
|
508
|
+
const ack: Graph = { [id]: null as never }
|
|
509
|
+
if (typeof timeoutConfig.lex["."] === "string") {
|
|
510
|
+
ack[id] = { [timeoutConfig.lex["."]]: null } as never
|
|
511
|
+
}
|
|
512
|
+
cb({ "#": trackId, put: ack })
|
|
513
|
+
delete queue[trackId]
|
|
514
|
+
}
|
|
515
|
+
}, timeoutConfig.wait)
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
clients().forEach(client => {
|
|
519
|
+
const ws = client as UnifiedWebSocket
|
|
520
|
+
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
521
|
+
ws.send(data, { binary: isBinary })
|
|
522
|
+
} else if (ws) {
|
|
523
|
+
const retryHandler = ws._retryHandler || createRetryHandler()
|
|
524
|
+
ws._retryHandler = retryHandler
|
|
525
|
+
|
|
526
|
+
if (retryHandler.shouldRetry()) {
|
|
527
|
+
const delay = retryHandler.getDelay()
|
|
528
|
+
retryHandler.increment()
|
|
529
|
+
|
|
530
|
+
setTimeout(() => {
|
|
531
|
+
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
532
|
+
retryHandler.reset()
|
|
533
|
+
ws.send(data, { binary: isBinary })
|
|
534
|
+
}
|
|
535
|
+
}, delay)
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
})
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
;(wss as unknown as { on: (event: string, cb: (ws: UnifiedWebSocket) => void) => void }).on(
|
|
542
|
+
"connection",
|
|
543
|
+
(ws: UnifiedWebSocket) => {
|
|
544
|
+
if (!connectionManager.add(ws)) {
|
|
545
|
+
console.log("Connection limit reached, rejecting connection")
|
|
546
|
+
ws.close(1013, "Connection limit reached - try again later")
|
|
547
|
+
return
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const clientId = utils.text.random(9)
|
|
551
|
+
console.log(`New WebSocket client connected: ${clientId}`)
|
|
552
|
+
|
|
553
|
+
ws.on?.("error", ((...args: unknown[]) => {
|
|
554
|
+
const error = args[0] as Error
|
|
555
|
+
console.log("WebSocket error:", error)
|
|
556
|
+
connectionManager.remove(ws)
|
|
557
|
+
}) as (...args: unknown[]) => void)
|
|
558
|
+
|
|
559
|
+
ws.on?.("close", (() => {
|
|
560
|
+
console.log(`WebSocket client disconnected: ${clientId}`)
|
|
561
|
+
connectionManager.remove(ws)
|
|
562
|
+
}) as (...args: unknown[]) => void)
|
|
563
|
+
|
|
564
|
+
ws.on?.("message", ((...args: unknown[]) => {
|
|
565
|
+
const data = args[0] as Buffer
|
|
566
|
+
const isBinary = args[1] as boolean
|
|
567
|
+
const validation = validateMessage(data, options.maxMessageSize)
|
|
568
|
+
if (!validation.valid) {
|
|
569
|
+
console.warn(`Invalid message: ${validation.error}`)
|
|
570
|
+
ws.send(JSON.stringify({ error: validation.error }))
|
|
571
|
+
return
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
const parseResult = safeJSONParse(data, options.maxMessageSize)
|
|
575
|
+
if (!parseResult.success) {
|
|
576
|
+
console.warn(`JSON parse error: ${parseResult.error}`)
|
|
577
|
+
ws.send(JSON.stringify({ error: "Invalid JSON" }))
|
|
578
|
+
return
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
const msg = parseResult.data!
|
|
582
|
+
if (!msg["#"]) return // Incoming messages must have '#'
|
|
583
|
+
if (dup.check(msg["#"])) return
|
|
584
|
+
|
|
585
|
+
const delay = rateLimiter.getDelay(clientId)
|
|
586
|
+
if (delay > 0) {
|
|
587
|
+
const throttleCount = rateLimiter.getThrottleCount(clientId)
|
|
588
|
+
console.log(
|
|
589
|
+
`Client ${clientId}: rate limit exceeded, delay would be ${delay}ms, dropping message (throttle count: ${throttleCount})`
|
|
590
|
+
)
|
|
591
|
+
if (rateLimiter.shouldDisconnect(clientId)) {
|
|
592
|
+
console.log(
|
|
593
|
+
`Client ${clientId}: Disconnecting after ${throttleCount} throttle violations`
|
|
594
|
+
)
|
|
595
|
+
ws.close(1008, "Rate limit violations")
|
|
596
|
+
return
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
ws.send(
|
|
600
|
+
JSON.stringify({
|
|
601
|
+
"#": dup.track(utils.text.random(9)),
|
|
602
|
+
"@": msg["#"],
|
|
603
|
+
err: `Rate limit exceeded. Slow down requests. Wait ${Math.ceil(delay / 1000)}s`,
|
|
604
|
+
throttle: delay,
|
|
605
|
+
})
|
|
606
|
+
)
|
|
607
|
+
return
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
const processMessage = async (): Promise<void> => {
|
|
611
|
+
dup.track(msg["#"]!)
|
|
612
|
+
|
|
613
|
+
if (msg.get) get(msg as never, send)
|
|
614
|
+
if (msg.put) await put(msg as never, send)
|
|
615
|
+
send(data.toString(), isBinary)
|
|
616
|
+
|
|
617
|
+
const id = msg["@"]
|
|
618
|
+
const cb = queue[id!]
|
|
619
|
+
if (cb) {
|
|
620
|
+
delete (msg as { "#"?: string })["#"]
|
|
621
|
+
delete (msg as { "@"?: string })["@"]
|
|
622
|
+
cb(msg)
|
|
623
|
+
delete queue[id!]
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
if (delay > 0) {
|
|
628
|
+
setTimeout(processMessage, delay)
|
|
629
|
+
} else {
|
|
630
|
+
processMessage()
|
|
631
|
+
}
|
|
632
|
+
}) as (...args: unknown[]) => void)
|
|
633
|
+
}
|
|
634
|
+
)
|
|
635
|
+
return api(send)
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// Browser logic
|
|
639
|
+
const peers: WebSocket[] = []
|
|
640
|
+
let clientThrottled = false
|
|
641
|
+
let throttleUntil = 0
|
|
642
|
+
let messageQueue: string[] = []
|
|
643
|
+
let queueProcessor: NodeJS.Timeout | null = null
|
|
644
|
+
const maxQueueLength = options.maxQueueLength || 10000
|
|
645
|
+
|
|
646
|
+
const processQueue = (): void => {
|
|
647
|
+
if (messageQueue.length === 0) {
|
|
648
|
+
queueProcessor = null
|
|
649
|
+
return
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
const now = Date.now()
|
|
653
|
+
if (clientThrottled && now < throttleUntil) {
|
|
654
|
+
queueProcessor = setTimeout(processQueue, Math.min(1000, throttleUntil - now))
|
|
655
|
+
return
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
if (clientThrottled && now >= throttleUntil) {
|
|
659
|
+
console.log(`Throttle period ended, resuming queue processing`)
|
|
660
|
+
clientThrottled = false
|
|
661
|
+
throttleUntil = 0
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
const data = messageQueue[0]!
|
|
665
|
+
const sent = sendToPeers(data)
|
|
666
|
+
|
|
667
|
+
if (sent) {
|
|
668
|
+
messageQueue.shift()
|
|
669
|
+
|
|
670
|
+
const msg = JSON.parse(data) as WireMessage
|
|
671
|
+
const trackId = msg["#"]
|
|
672
|
+
if (trackId && pendingTimeouts.has(trackId)) {
|
|
673
|
+
const timeoutConfig = pendingTimeouts.get(trackId)!
|
|
674
|
+
pendingTimeouts.delete(trackId)
|
|
675
|
+
|
|
676
|
+
timeoutConfig.timeoutId = setTimeout(() => {
|
|
677
|
+
const cb = queue[trackId]
|
|
678
|
+
if (cb) {
|
|
679
|
+
const id = timeoutConfig.lex["#"]
|
|
680
|
+
const ack: Graph = { [id]: null as never }
|
|
681
|
+
if (typeof timeoutConfig.lex["."] === "string") {
|
|
682
|
+
ack[id] = { [timeoutConfig.lex["."]]: null } as never
|
|
683
|
+
}
|
|
684
|
+
cb({ "#": trackId, put: ack })
|
|
685
|
+
delete queue[trackId]
|
|
686
|
+
}
|
|
687
|
+
}, timeoutConfig.wait)
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
if (messageQueue.length > 0) {
|
|
692
|
+
queueProcessor = setTimeout(processQueue, 50)
|
|
693
|
+
} else {
|
|
694
|
+
queueProcessor = null
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
const sendToPeers = (data: string): boolean => {
|
|
699
|
+
let sentToAtLeastOne = false
|
|
700
|
+
peers.forEach(peer => {
|
|
701
|
+
const ws = peer as UnifiedWebSocket
|
|
702
|
+
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
703
|
+
ws.send(data)
|
|
704
|
+
sentToAtLeastOne = true
|
|
705
|
+
} else if (ws) {
|
|
706
|
+
const retryHandler = ws._retryHandler || createRetryHandler()
|
|
707
|
+
ws._retryHandler = retryHandler
|
|
708
|
+
|
|
709
|
+
if (retryHandler.shouldRetry()) {
|
|
710
|
+
const delay = retryHandler.getDelay()
|
|
711
|
+
retryHandler.increment()
|
|
712
|
+
|
|
713
|
+
setTimeout(() => {
|
|
714
|
+
if (peer && peer.readyState === WebSocket.OPEN) {
|
|
715
|
+
retryHandler.reset()
|
|
716
|
+
peer.send(data)
|
|
717
|
+
}
|
|
718
|
+
}, delay)
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
})
|
|
722
|
+
return sentToAtLeastOne
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
const send = (data: string): { err?: string } | void => {
|
|
726
|
+
if (messageQueue.length >= maxQueueLength) {
|
|
727
|
+
return {
|
|
728
|
+
err: `Message queue exceeded maximum length (${maxQueueLength}). Update query logic to request less data.`,
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
messageQueue.push(data)
|
|
733
|
+
if (!queueProcessor) {
|
|
734
|
+
queueProcessor = setTimeout(processQueue, 50)
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
if (!(options.peers instanceof Array) || options.peers.length === 0) {
|
|
739
|
+
options.peers = [`ws://localhost:${options.port || 8765}`]
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
options.peers.forEach((peer: string) => {
|
|
743
|
+
const start = (): void => {
|
|
744
|
+
let ws: WebSocket | null = new WebSocket(peer)
|
|
745
|
+
peers.push(ws)
|
|
746
|
+
const retryHandler = createRetryHandler()
|
|
747
|
+
|
|
748
|
+
ws.onclose = () => {
|
|
749
|
+
if (ws && peers.indexOf(ws) !== -1) {
|
|
750
|
+
peers.splice(peers.indexOf(ws), 1)
|
|
751
|
+
}
|
|
752
|
+
ws = null
|
|
753
|
+
|
|
754
|
+
if (retryHandler.shouldRetry()) {
|
|
755
|
+
const delay = retryHandler.getDelay()
|
|
756
|
+
retryHandler.increment()
|
|
757
|
+
setTimeout(start, delay)
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
ws.onopen = () => {
|
|
762
|
+
retryHandler.reset()
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
ws.onerror = (e: Event) => {
|
|
766
|
+
console.log(e)
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
ws.onmessage = async (m: MessageEvent) => {
|
|
770
|
+
const msg = JSON.parse(m.data as string) as WireMessage
|
|
771
|
+
if (!msg["#"]) return // Incoming messages must have '#'
|
|
772
|
+
if (dup.check(msg["#"])) return
|
|
773
|
+
|
|
774
|
+
if (msg.throttle && msg.err) {
|
|
775
|
+
console.log(`Server throttling: ${msg.err}`)
|
|
776
|
+
clientThrottled = true
|
|
777
|
+
throttleUntil = Date.now() + msg.throttle
|
|
778
|
+
return
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
dup.track(msg["#"]!)
|
|
782
|
+
if (msg.get) get(msg as never, send as never)
|
|
783
|
+
if (msg.put) {
|
|
784
|
+
const filteredPut: Graph = {}
|
|
785
|
+
for (const [soul, node] of Object.entries(msg.put)) {
|
|
786
|
+
let shouldStore = false
|
|
787
|
+
if (await hasSoul(soul)) {
|
|
788
|
+
shouldStore = true
|
|
789
|
+
if (node && typeof node === "object") {
|
|
790
|
+
for (const [_key, value] of Object.entries(node)) {
|
|
791
|
+
const soulId = utils.rel.is(value as GraphValue)
|
|
792
|
+
if (soulId) {
|
|
793
|
+
pendingReferences.add(soulId)
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
if (pendingReferences.has(soul)) {
|
|
799
|
+
shouldStore = true
|
|
800
|
+
pendingReferences.delete(soul)
|
|
801
|
+
}
|
|
802
|
+
if (shouldStore) {
|
|
803
|
+
filteredPut[soul] = node
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
if (Object.keys(filteredPut).length > 0) {
|
|
807
|
+
const filteredMsg = { put: filteredPut, "#": msg["#"]! }
|
|
808
|
+
await put(filteredMsg, send as never)
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
const id = msg["@"]
|
|
813
|
+
const cb = queue[id!]
|
|
814
|
+
if (cb) {
|
|
815
|
+
delete (msg as { "#"?: string })["#"]
|
|
816
|
+
delete (msg as { "@"?: string })["@"]
|
|
817
|
+
cb(msg)
|
|
818
|
+
delete queue[id!]
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
start()
|
|
823
|
+
})
|
|
824
|
+
|
|
825
|
+
return api(send)
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
export default Wire
|
|
829
|
+
|