@scp-dev/broker 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +38 -2
- package/dist/index.d.ts +422 -0
- package/dist/index.js +2514 -0
- package/dist/testing.d.ts +30 -0
- package/dist/testing.js +131 -0
- package/package.json +48 -4
- package/src/bridge.ts +642 -0
- package/src/client.ts +336 -0
- package/src/index.ts +6 -0
- package/src/judge-gate.ts +552 -0
- package/src/lock.ts +113 -0
- package/src/ndjson.ts +95 -0
- package/src/paths.ts +132 -0
- package/src/server.ts +1124 -0
- package/src/socket.ts +57 -0
- package/src/sqlite.ts +90 -0
- package/src/testing.ts +168 -0
package/src/bridge.ts
ADDED
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ActivityRow,
|
|
3
|
+
type EndpointView,
|
|
4
|
+
type Envelope,
|
|
5
|
+
findSecret,
|
|
6
|
+
matches,
|
|
7
|
+
newId,
|
|
8
|
+
PROTOCOL,
|
|
9
|
+
RelayError,
|
|
10
|
+
type Selector,
|
|
11
|
+
type SendParams,
|
|
12
|
+
type SendResult,
|
|
13
|
+
XLIMITS,
|
|
14
|
+
type XListed,
|
|
15
|
+
} from "@scp-dev/core"
|
|
16
|
+
import { type FederatedCore, HUB_LIMITS, type RemoteView } from "@scp-dev/hub"
|
|
17
|
+
|
|
18
|
+
/** what the broker gives the bridge: waking local waiters, telling local watchers, the local policy's state */
|
|
19
|
+
export interface BridgeHooks {
|
|
20
|
+
/** a local endpoint got a message */
|
|
21
|
+
wake(endpoint: string): void
|
|
22
|
+
/** a fleet event for local watchers only; never forwarded back */
|
|
23
|
+
fleet(event: Record<string, unknown>): void
|
|
24
|
+
/** loads the local policy; a reason when it is unusable */
|
|
25
|
+
policyError(): string | null
|
|
26
|
+
log?(line: string): void
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface BridgeConfig {
|
|
30
|
+
/** the hub's https origin, from the caller; the socket is <origin>/v1/relay/ws */
|
|
31
|
+
url: string
|
|
32
|
+
/** the account's bearer credential for the hub, opaque to the broker; null keeps the bridge down and says why in status() */
|
|
33
|
+
token: string | null
|
|
34
|
+
/** why there is no credential, for status() */
|
|
35
|
+
tokenError?: string | undefined
|
|
36
|
+
/** reads the credential again before each reconnect, so an expired or replaced one is picked up */
|
|
37
|
+
credentials?: (() => Promise<string | null>) | undefined
|
|
38
|
+
machine: string
|
|
39
|
+
/** which local endpoints are mirrored to the hub (default ["*"]) */
|
|
40
|
+
mirror?: Selector[] | undefined
|
|
41
|
+
/** drop cwd, status notes and activity summaries from what is mirrored */
|
|
42
|
+
redact?: boolean | undefined
|
|
43
|
+
/** let messages marked private leave this machine */
|
|
44
|
+
privateOk?: boolean | undefined
|
|
45
|
+
/** a WebSocket constructor (tests) */
|
|
46
|
+
webSocket?: WebSocketWithHeaders | undefined
|
|
47
|
+
/** reconnect backoff bounds (default 1 s to 60 s) */
|
|
48
|
+
backoff?: { minMs: number; maxMs: number } | undefined
|
|
49
|
+
/** how long a forward cut off by a dropped connection waits for the next one (default 30 s) */
|
|
50
|
+
forwardWaitMs?: number | undefined
|
|
51
|
+
/** the wait before a failed inject is tried again, doubling for each later try (default 1 s) */
|
|
52
|
+
injectBackoffMs?: number | undefined
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface BridgeOptions extends BridgeConfig {
|
|
56
|
+
core: FederatedCore
|
|
57
|
+
onRemote: BridgeHooks
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface BridgeStatus {
|
|
61
|
+
enabled: true
|
|
62
|
+
machine: string
|
|
63
|
+
url: string
|
|
64
|
+
connected: boolean
|
|
65
|
+
last_error: string | null
|
|
66
|
+
since: string
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface Bridge {
|
|
70
|
+
/** a send whose recipients are all on other machines, on behalf of the local endpoint `from` */
|
|
71
|
+
forward(from: string, params: SendParams, origin?: { tty?: boolean }): Promise<SendResult>
|
|
72
|
+
/** a local endpoint was claimed, let go or changed its status: mirrored as a delta */
|
|
73
|
+
changed(endpoint: string): void
|
|
74
|
+
/** rules, caps, groups or a title changed, which can touch several endpoints: every mirrored endpoint is sent again */
|
|
75
|
+
resync(): void
|
|
76
|
+
/** a local endpoint's activity, posted to the hub */
|
|
77
|
+
activity(endpoint: string, row: ActivityRow): void
|
|
78
|
+
/** the endpoints of other accounts this account's contacts let it reach */
|
|
79
|
+
contacts(): Promise<XListed[]>
|
|
80
|
+
status(): BridgeStatus
|
|
81
|
+
stop(): Promise<void>
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** a WebSocket constructor that takes request headers, as Bun's and Node's do */
|
|
85
|
+
export type WebSocketWithHeaders = new (url: string, options: { headers: Record<string, string> }) => WebSocket
|
|
86
|
+
|
|
87
|
+
type Pending = { resolve: (value: unknown) => void; reject: (error: Error) => void }
|
|
88
|
+
type Frame = {
|
|
89
|
+
id?: number
|
|
90
|
+
method?: string
|
|
91
|
+
params?: unknown
|
|
92
|
+
result?: unknown
|
|
93
|
+
error?: { message: string; data?: { code?: RelayError["code"]; retry_after?: number } }
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const CONNECT_TIMEOUT_MS = 15_000
|
|
97
|
+
/** room left in the hub's frame for the JSON-RPC envelope around a mirror's endpoints */
|
|
98
|
+
const MIRROR_FRAME_BYTES = HUB_LIMITS.frameBytes - 1024
|
|
99
|
+
/** an inject that throws this many times in a row is rejected to the hub as a dead letter */
|
|
100
|
+
const DEAD_LETTER_ATTEMPTS = 3
|
|
101
|
+
|
|
102
|
+
/** whether a send goes to other machines: all recipients remote, or none; a mixed list is refused */
|
|
103
|
+
export function remoteSend(core: FederatedCore, to: unknown): boolean {
|
|
104
|
+
const targets = Array.isArray(to) ? to : [to]
|
|
105
|
+
const remote = targets.filter((address) => core.isRemote(address)).length
|
|
106
|
+
if (remote && remote !== targets.length)
|
|
107
|
+
throw new RelayError("invalid_params", "a message goes to endpoints on this machine or on other machines, not both; send it twice")
|
|
108
|
+
return remote > 0
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** `<origin>/v1/relay/ws` with ws or wss */
|
|
112
|
+
export function relaySocketUrl(origin: string): string {
|
|
113
|
+
const url = new URL("/v1/relay/ws", origin)
|
|
114
|
+
url.protocol = url.protocol === "https:" ? "wss:" : "ws:"
|
|
115
|
+
return url.toString()
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** the broker's link to the hub: mirrors local endpoints, forwards remote sends, injects deliveries */
|
|
119
|
+
export function startBridge(options: BridgeOptions): Bridge {
|
|
120
|
+
const { core, onRemote: hooks, machine } = options
|
|
121
|
+
const selectors = options.mirror ?? ["*"]
|
|
122
|
+
const minMs = options.backoff?.minMs ?? 1000
|
|
123
|
+
const maxMs = options.backoff?.maxMs ?? 60_000
|
|
124
|
+
const forwardWaitMs = options.forwardWaitMs ?? 30_000
|
|
125
|
+
const injectBackoffMs = options.injectBackoffMs ?? 1000
|
|
126
|
+
const Socket = options.webSocket ?? (WebSocket as unknown as WebSocketWithHeaders)
|
|
127
|
+
const log = hooks.log ?? ((line: string) => console.error(line))
|
|
128
|
+
const encoder = new TextEncoder()
|
|
129
|
+
|
|
130
|
+
let token = options.token
|
|
131
|
+
let ws: WebSocket | null = null
|
|
132
|
+
let connected = false
|
|
133
|
+
/** hello went through on the current socket: deltas may go out */
|
|
134
|
+
let ready = false
|
|
135
|
+
let stopped = false
|
|
136
|
+
let lastError: string | null = options.token ? null : (options.tokenError ?? "no hub credential; the broker runs local-only")
|
|
137
|
+
let since = new Date().toISOString()
|
|
138
|
+
let attempt = 0
|
|
139
|
+
let account: string | null = null
|
|
140
|
+
let revokedReason: string | null = null
|
|
141
|
+
let retryTimer: ReturnType<typeof setTimeout> | undefined
|
|
142
|
+
let nextId = 1
|
|
143
|
+
const pending = new Map<number, Pending>()
|
|
144
|
+
let waitingForLink: (() => void)[] = []
|
|
145
|
+
let deliveries: Promise<void> = Promise.resolve()
|
|
146
|
+
let ackedMax = 0
|
|
147
|
+
const inflight = new Set<number>()
|
|
148
|
+
const injectFailures = new Map<number, number>()
|
|
149
|
+
const fromHub = new WeakSet<Error>()
|
|
150
|
+
|
|
151
|
+
const setState = (up: boolean, error: string | null) => {
|
|
152
|
+
if (up !== connected || error !== lastError) since = new Date().toISOString()
|
|
153
|
+
connected = up
|
|
154
|
+
lastError = error
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function request<T>(method: string, params: object = {}): Promise<T> {
|
|
158
|
+
const socket = ws
|
|
159
|
+
if (socket?.readyState !== 1) return Promise.reject(new RelayError("unavailable", "the hub is not connected"))
|
|
160
|
+
const id = nextId++
|
|
161
|
+
return new Promise<T>((resolve, reject) => {
|
|
162
|
+
pending.set(id, { resolve: resolve as (value: unknown) => void, reject })
|
|
163
|
+
try {
|
|
164
|
+
socket.send(JSON.stringify({ jsonrpc: "2.0", id, method, params }))
|
|
165
|
+
} catch (error) {
|
|
166
|
+
pending.delete(id)
|
|
167
|
+
reject(new RelayError("unavailable", `the hub connection failed (${error instanceof Error ? error.message : String(error)})`))
|
|
168
|
+
}
|
|
169
|
+
})
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function fireAndForget(method: string, params: object) {
|
|
173
|
+
request(method, params).catch((error: unknown) => {
|
|
174
|
+
if (!(error instanceof RelayError && error.code === "unavailable"))
|
|
175
|
+
log(`relay bridge: ${method} failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
176
|
+
})
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function connect() {
|
|
180
|
+
if (stopped || !token) return
|
|
181
|
+
let socket: WebSocket
|
|
182
|
+
try {
|
|
183
|
+
socket = new Socket(relaySocketUrl(options.url), { headers: { Authorization: `Bearer ${token}` } })
|
|
184
|
+
} catch (error) {
|
|
185
|
+
setState(false, `cannot open the hub socket (${error instanceof Error ? error.message : String(error)})`)
|
|
186
|
+
return retry()
|
|
187
|
+
}
|
|
188
|
+
ws = socket
|
|
189
|
+
revokedReason = null
|
|
190
|
+
let handshakeError: string | null = null
|
|
191
|
+
const timer = setTimeout(() => {
|
|
192
|
+
if (socket.readyState === 0) socket.close()
|
|
193
|
+
}, CONNECT_TIMEOUT_MS)
|
|
194
|
+
socket.addEventListener("open", () => {
|
|
195
|
+
clearTimeout(timer)
|
|
196
|
+
handshake(socket).catch((error: unknown) => {
|
|
197
|
+
if (ws !== socket) return
|
|
198
|
+
const relay = error instanceof RelayError ? error : null
|
|
199
|
+
if (relay && (relay.code === "unauthorized" || relay.code === "invalid_params")) {
|
|
200
|
+
stopped = true
|
|
201
|
+
handshakeError = `the hub refused this bridge: ${relay.message}`
|
|
202
|
+
} else if (relay?.code === "too_large") {
|
|
203
|
+
handshakeError = `the hub refused this bridge for now, retrying: ${relay.message}`
|
|
204
|
+
} else {
|
|
205
|
+
handshakeError = `the handshake failed: ${error instanceof Error ? error.message : String(error)}`
|
|
206
|
+
}
|
|
207
|
+
socket.close(1000, "handshake failed")
|
|
208
|
+
})
|
|
209
|
+
})
|
|
210
|
+
socket.addEventListener("message", (event) => onFrame(String((event as MessageEvent).data)))
|
|
211
|
+
socket.addEventListener("error", () => {})
|
|
212
|
+
socket.addEventListener("close", (event) => {
|
|
213
|
+
clearTimeout(timer)
|
|
214
|
+
if (ws !== socket) return
|
|
215
|
+
ws = null
|
|
216
|
+
ready = false
|
|
217
|
+
const close = event as CloseEvent
|
|
218
|
+
const wasUp = connected
|
|
219
|
+
for (const p of pending.values()) p.reject(new RelayError("unavailable", "the hub connection closed"))
|
|
220
|
+
pending.clear()
|
|
221
|
+
const why = `${close.code}${close.reason ? ` ${close.reason}` : ""}`
|
|
222
|
+
if (close.code === 1008) {
|
|
223
|
+
stopped = true
|
|
224
|
+
setState(false, `revoked or unauthorized: ${revokedReason ?? (close.reason || "the hub closed the bridge")}`)
|
|
225
|
+
} else if (close.code === 4000) {
|
|
226
|
+
stopped = true
|
|
227
|
+
setState(false, `replaced: ${close.reason || "another bridge connected for this machine"}`)
|
|
228
|
+
} else if (close.code === 4001) {
|
|
229
|
+
setState(false, `the credential expired, reconnecting (${why})`)
|
|
230
|
+
} else {
|
|
231
|
+
setState(false, handshakeError ?? (wasUp ? `disconnected (${why})` : `cannot connect to ${options.url} (${why})`))
|
|
232
|
+
}
|
|
233
|
+
retry()
|
|
234
|
+
})
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function retry() {
|
|
238
|
+
if (stopped) return
|
|
239
|
+
const base = Math.min(maxMs, minMs * 2 ** Math.min(attempt, 16))
|
|
240
|
+
attempt++
|
|
241
|
+
retryTimer = setTimeout(reconnect, base / 2 + Math.random() * (base / 2))
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async function reconnect() {
|
|
245
|
+
if (options.credentials && !stopped) {
|
|
246
|
+
try {
|
|
247
|
+
token = (await options.credentials()) ?? token
|
|
248
|
+
} catch (error) {
|
|
249
|
+
log(`relay bridge: the credential could not be read again: ${error instanceof Error ? error.message : String(error)}`)
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
connect()
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async function handshake(socket: WebSocket) {
|
|
256
|
+
const hello = await request<{ account: string; endpoints: RemoteView[] }>("hello", { protocol: PROTOCOL, machine, bridge: true })
|
|
257
|
+
account = hello.account
|
|
258
|
+
core.setAliases([account])
|
|
259
|
+
ready = true
|
|
260
|
+
applyRemote(hello.endpoints, true)
|
|
261
|
+
await Promise.all(fullMirror().map((params) => request("mirror", params)))
|
|
262
|
+
await request("watch")
|
|
263
|
+
await request("subscribe", { after: cursor() })
|
|
264
|
+
if (ws !== socket) return
|
|
265
|
+
attempt = 0
|
|
266
|
+
setState(true, null)
|
|
267
|
+
for (const fn of waitingForLink.splice(0)) fn()
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** the other machines' endpoints, grouped by machine; with `full`, a known machine missing from the list goes offline */
|
|
271
|
+
function applyRemote(views: RemoteView[], full: boolean) {
|
|
272
|
+
const byMachine = new Map<string, RemoteView[]>()
|
|
273
|
+
for (const view of views) {
|
|
274
|
+
if (view.machine === machine) continue
|
|
275
|
+
byMachine.set(view.machine, [...(byMachine.get(view.machine) ?? []), view])
|
|
276
|
+
}
|
|
277
|
+
if (full) for (const view of core.remoteViews()) if (view.machine !== account && !byMachine.has(view.machine)) byMachine.set(view.machine, [])
|
|
278
|
+
for (const [name, list] of byMachine) {
|
|
279
|
+
try {
|
|
280
|
+
core.mirror(name, list, { full })
|
|
281
|
+
} catch (error) {
|
|
282
|
+
log(`relay bridge: the endpoints of ${name} were not applied: ${error instanceof Error ? error.message : String(error)}`)
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function mirrored(view: EndpointView): boolean {
|
|
288
|
+
if (view.kind === "system" || core.machineOf(view.id) !== null) return false
|
|
289
|
+
const subject = {
|
|
290
|
+
handle: view.kind === "human" ? "" : view.address.slice(view.address.lastIndexOf("/") + 1),
|
|
291
|
+
harness: view.harness,
|
|
292
|
+
kind: view.kind,
|
|
293
|
+
labels: view.labels,
|
|
294
|
+
}
|
|
295
|
+
return selectors.some((selector) => matches(selector, subject))
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function outward(view: EndpointView): object {
|
|
299
|
+
const { title, ...scope } = core.mirrorScope(view.id)
|
|
300
|
+
return {
|
|
301
|
+
...scope,
|
|
302
|
+
...(title === undefined ? {} : { title: options.redact ? null : title }),
|
|
303
|
+
address: view.address,
|
|
304
|
+
kind: view.kind,
|
|
305
|
+
harness: view.harness,
|
|
306
|
+
model: view.model,
|
|
307
|
+
cwd: options.redact ? null : view.cwd,
|
|
308
|
+
labels: view.labels,
|
|
309
|
+
status: view.status ? { state: view.status.state, note: options.redact ? "" : view.status.note } : null,
|
|
310
|
+
presence: view.presence,
|
|
311
|
+
last_seen: view.last_seen,
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function localViews(): object[] {
|
|
316
|
+
return core.list().filter(mirrored).map(outward)
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* the full mirror in frames that fit the hub's, sent back to back. Several frames share a `sync`, and the hub marks offline
|
|
321
|
+
* only what the whole sync left out, once its last frame is in; the first keeps `full`.
|
|
322
|
+
*/
|
|
323
|
+
function fullMirror(): { endpoints: object[]; full: boolean; sync?: { id: string; part: number; parts: number } }[] {
|
|
324
|
+
const frames: object[][] = [[]]
|
|
325
|
+
let bytes = 0
|
|
326
|
+
for (const view of localViews()) {
|
|
327
|
+
const size = encoder.encode(JSON.stringify(view)).length + 1
|
|
328
|
+
if (size > MIRROR_FRAME_BYTES)
|
|
329
|
+
throw new RelayError("too_large", `${(view as { address: string }).address} mirrors ${size} bytes, over the hub's ${MIRROR_FRAME_BYTES} byte frame`)
|
|
330
|
+
if (bytes + size > MIRROR_FRAME_BYTES) {
|
|
331
|
+
frames.push([])
|
|
332
|
+
bytes = 0
|
|
333
|
+
}
|
|
334
|
+
frames.at(-1)!.push(view)
|
|
335
|
+
bytes += size
|
|
336
|
+
}
|
|
337
|
+
if (frames.length === 1) return [{ endpoints: frames[0]!, full: true }]
|
|
338
|
+
const id = newId("sync")
|
|
339
|
+
return frames.map((endpoints, i) => ({ endpoints, full: i === 0, sync: { id, part: i + 1, parts: frames.length } }))
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/** the handle the hub knows a local endpoint by ("" for the human inbox), or null when it is not mirrored */
|
|
343
|
+
function handleOf(endpoint: string): string | null {
|
|
344
|
+
const view = core.whoami(endpoint)
|
|
345
|
+
if (!mirrored(view)) return null
|
|
346
|
+
return view.kind === "human" ? "" : view.address.slice(view.address.lastIndexOf("/") + 1)
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** a hub address (`<account>/<machine>/<handle>` or `<account>/<handle>`) as the local core writes it */
|
|
350
|
+
function localAddress(address: unknown): unknown {
|
|
351
|
+
if (typeof address !== "string" || !account || !address.startsWith(`${account}/`)) return address
|
|
352
|
+
const rest = address.slice(account.length + 1)
|
|
353
|
+
return rest.includes("/") ? `${core.user}/${rest}` : `${core.user}/${account}/${rest}`
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** the lowest seq a fresh subscribe must replay from: every delivery at or below it was acked */
|
|
357
|
+
function cursor(): number {
|
|
358
|
+
return inflight.size ? Math.min(...inflight) - 1 : ackedMax
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function onFrame(text: string) {
|
|
362
|
+
let frame: Frame
|
|
363
|
+
try {
|
|
364
|
+
frame = JSON.parse(text) as Frame
|
|
365
|
+
} catch {
|
|
366
|
+
log("relay bridge: the hub sent a frame that is not JSON")
|
|
367
|
+
return
|
|
368
|
+
}
|
|
369
|
+
if (frame.method) return onNotification(frame.method, frame.params as Record<string, unknown>)
|
|
370
|
+
const waiter = frame.id !== undefined ? pending.get(frame.id) : undefined
|
|
371
|
+
if (!waiter) return
|
|
372
|
+
pending.delete(frame.id!)
|
|
373
|
+
if (!frame.error) return waiter.resolve(frame.result)
|
|
374
|
+
const error = new RelayError(frame.error.data?.code ?? "unavailable", frame.error.message, frame.error.data?.retry_after)
|
|
375
|
+
fromHub.add(error)
|
|
376
|
+
waiter.reject(error)
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
function onNotification(method: string, params: Record<string, unknown>) {
|
|
380
|
+
switch (method) {
|
|
381
|
+
case "deliver": {
|
|
382
|
+
const seq = params.seq
|
|
383
|
+
if (!Number.isInteger(seq)) return
|
|
384
|
+
inflight.add(seq as number)
|
|
385
|
+
deliveries = deliveries.then(() => deliver(seq as number, params.endpoint, params.message as Envelope))
|
|
386
|
+
return
|
|
387
|
+
}
|
|
388
|
+
case "endpoints": {
|
|
389
|
+
if (typeof params.machine !== "string" || params.machine === machine) return
|
|
390
|
+
try {
|
|
391
|
+
core.mirror(params.machine, params.endpoints, { full: params.full === true })
|
|
392
|
+
} catch (error) {
|
|
393
|
+
log(`relay bridge: the endpoints of ${params.machine} were not applied: ${error instanceof Error ? error.message : String(error)}`)
|
|
394
|
+
}
|
|
395
|
+
return
|
|
396
|
+
}
|
|
397
|
+
case "fleet":
|
|
398
|
+
return remoteFleet(params)
|
|
399
|
+
case "revoked":
|
|
400
|
+
revokedReason = typeof params.reason === "string" ? params.reason : null
|
|
401
|
+
return
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
async function deliver(seq: number, target: unknown, message: Envelope) {
|
|
406
|
+
try {
|
|
407
|
+
let refused: string | null = null
|
|
408
|
+
for (;;) {
|
|
409
|
+
try {
|
|
410
|
+
refused = inject(target, message)
|
|
411
|
+
break
|
|
412
|
+
} catch (error) {
|
|
413
|
+
if (error instanceof RelayError) {
|
|
414
|
+
refused = error.code
|
|
415
|
+
break
|
|
416
|
+
}
|
|
417
|
+
const attempts = (injectFailures.get(seq) ?? 0) + 1
|
|
418
|
+
injectFailures.set(seq, attempts)
|
|
419
|
+
const why = error instanceof Error ? error.message : String(error)
|
|
420
|
+
log(`relay bridge: delivery ${seq} failed (attempt ${attempts} of ${DEAD_LETTER_ATTEMPTS}): ${why}`)
|
|
421
|
+
if (attempts >= DEAD_LETTER_ATTEMPTS) {
|
|
422
|
+
await settled("reject", { seq, code: "dead_letter" })
|
|
423
|
+
settle(seq)
|
|
424
|
+
core.record("dead_letter", null, typeof message?.id === "string" ? message.id : null, { seq, target: String(target), attempts, error: why })
|
|
425
|
+
return
|
|
426
|
+
}
|
|
427
|
+
await new Promise((resolve) => setTimeout(resolve, injectBackoffMs * 2 ** (attempts - 1)))
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
if (refused !== null) {
|
|
431
|
+
await settled("reject", { seq, code: refused })
|
|
432
|
+
settle(seq)
|
|
433
|
+
return
|
|
434
|
+
}
|
|
435
|
+
await settled("ack", { seqs: [seq] })
|
|
436
|
+
settle(seq)
|
|
437
|
+
} catch (error) {
|
|
438
|
+
if (error instanceof RelayError && error.code === "unavailable") return
|
|
439
|
+
log(`relay bridge: delivery ${seq} failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/** admits a delivery to its local endpoint and tells the local waiters; the refusal's code, or null once admitted */
|
|
444
|
+
function inject(target: unknown, message: Envelope): string | null {
|
|
445
|
+
const endpoint = core.localTarget(target)
|
|
446
|
+
if ((hooks.policyError() ?? core.inboundRefusal(endpoint, message)) !== null) return "policy_refused"
|
|
447
|
+
if (core.admit(endpoint, message).duplicate) return null
|
|
448
|
+
hooks.wake(endpoint)
|
|
449
|
+
hooks.fleet({
|
|
450
|
+
type: "message",
|
|
451
|
+
machine: message.from.machine ?? null,
|
|
452
|
+
id: message.id,
|
|
453
|
+
thread: message.thread,
|
|
454
|
+
kind: message.kind,
|
|
455
|
+
from: localAddress(message.from.address),
|
|
456
|
+
to: [core.addressOf(endpoint)],
|
|
457
|
+
hop: message.hop,
|
|
458
|
+
})
|
|
459
|
+
return null
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/** an ack or reject; one the hub's ack limit refuses waits its retry_after and goes again, while the link holds */
|
|
463
|
+
async function settled(method: "ack" | "reject", params: object): Promise<void> {
|
|
464
|
+
for (;;) {
|
|
465
|
+
try {
|
|
466
|
+
await request(method, params)
|
|
467
|
+
return
|
|
468
|
+
} catch (error) {
|
|
469
|
+
if (!(error instanceof RelayError && error.code === "rate_limited")) throw error
|
|
470
|
+
await new Promise((resolve) => setTimeout(resolve, (error.retryAfterS ?? 1) * 1000))
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
function settle(seq: number) {
|
|
476
|
+
inflight.delete(seq)
|
|
477
|
+
injectFailures.delete(seq)
|
|
478
|
+
ackedMax = Math.max(ackedMax, seq)
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function remoteFleet(event: Record<string, unknown>) {
|
|
482
|
+
if (event.machine === machine) return
|
|
483
|
+
if (
|
|
484
|
+
event.type === "message" &&
|
|
485
|
+
Array.isArray(event.to) &&
|
|
486
|
+
account &&
|
|
487
|
+
event.to.some((to) => typeof to === "string" && to.startsWith(`${account}/${machine}/`))
|
|
488
|
+
)
|
|
489
|
+
return
|
|
490
|
+
const local: Record<string, unknown> = {
|
|
491
|
+
...event,
|
|
492
|
+
...(event.from !== undefined ? { from: localAddress(event.from) } : {}),
|
|
493
|
+
...(event.endpoint !== undefined ? { endpoint: localAddress(event.endpoint) } : {}),
|
|
494
|
+
}
|
|
495
|
+
if (Array.isArray(event.to)) local.to = event.to.map(localAddress)
|
|
496
|
+
if (event.type === "activity" && typeof event.machine === "string" && typeof event.endpoint === "string") {
|
|
497
|
+
try {
|
|
498
|
+
const proxy = core.remoteEndpoint(event.machine, event.endpoint.slice(event.endpoint.lastIndexOf("/") + 1))
|
|
499
|
+
const row = core.event(proxy, { kind: event.kind, tool: event.tool, summary: event.summary })
|
|
500
|
+
local.id = row.id
|
|
501
|
+
} catch {}
|
|
502
|
+
}
|
|
503
|
+
hooks.fleet(local)
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/** resolves when the link is up, false when `ms` passed or the bridge stopped first */
|
|
507
|
+
function linkUp(ms: number): Promise<boolean> {
|
|
508
|
+
if (connected) return Promise.resolve(true)
|
|
509
|
+
if (stopped) return Promise.resolve(false)
|
|
510
|
+
return new Promise((resolve) => {
|
|
511
|
+
const done = (up: boolean) => {
|
|
512
|
+
clearTimeout(timer)
|
|
513
|
+
waitingForLink = waitingForLink.filter((fn) => fn !== onUp)
|
|
514
|
+
resolve(up)
|
|
515
|
+
}
|
|
516
|
+
const onUp = () => done(true)
|
|
517
|
+
const timer = setTimeout(() => done(false), ms)
|
|
518
|
+
waitingForLink.push(onUp)
|
|
519
|
+
})
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function unavailable(): RelayError {
|
|
523
|
+
return new RelayError("unavailable", `the hub is not connected${lastError ? ` (${lastError})` : ""}; try again`, 5)
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** a message to another account: one recipient, no fan-out, a short body, nothing private or attached; never stripped to fit */
|
|
527
|
+
function crossAccount(params: SendParams, targets: unknown[]) {
|
|
528
|
+
if (targets.length !== 1) throw new RelayError("invalid_params", "a message to another account goes to exactly one endpoint: no lists, groups or tree:")
|
|
529
|
+
if (typeof params.body === "string") {
|
|
530
|
+
const bytes = new TextEncoder().encode(params.body).length
|
|
531
|
+
if (bytes > XLIMITS.bodyBytes)
|
|
532
|
+
throw new RelayError("too_large", `a message to another account is at most ${XLIMITS.bodyBytes} bytes; this one is ${bytes}`)
|
|
533
|
+
}
|
|
534
|
+
if (params.attachments?.length) throw new RelayError("data_class_refused", "attachments stay in this account; send a reference instead")
|
|
535
|
+
if (params.data_class === "private") throw new RelayError("data_class_refused", "a private message stays in this account")
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
async function forward(from: string, params: SendParams, origin: { tty?: boolean } = {}): Promise<SendResult> {
|
|
539
|
+
const sender = core.addressOf(from)
|
|
540
|
+
const targets = Array.isArray(params.to) ? params.to : [params.to]
|
|
541
|
+
if (targets.some((to) => core.foreignAccount(to) !== null)) crossAccount(params, targets)
|
|
542
|
+
for (const to of targets) {
|
|
543
|
+
core.scopeCheck(from, to, typeof params.reply_to === "string" ? params.reply_to : undefined)
|
|
544
|
+
const refusal = core.checkPolicy(sender, String(to))
|
|
545
|
+
if (refusal) throw new RelayError("policy_refused", `${refusal} (${String(to)})`)
|
|
546
|
+
}
|
|
547
|
+
if (typeof params.body !== "string") throw new RelayError("invalid_params", "body must be a string")
|
|
548
|
+
if (params.data_class === "private" && !options.privateOk) {
|
|
549
|
+
throw new RelayError(
|
|
550
|
+
"data_class_refused",
|
|
551
|
+
"a private message stays on this machine; allow private messages in the bridge configuration to send them through the hub",
|
|
552
|
+
)
|
|
553
|
+
}
|
|
554
|
+
const secret = findSecret(params.body) ?? (params.attachments ?? []).map((a) => findSecret(String(a?.path))).find(Boolean) ?? null
|
|
555
|
+
if (secret) throw new RelayError("secret_refused", `the message looks like it carries a credential (${secret}); send a reference instead`)
|
|
556
|
+
const earlier = core.forwarded(from, params)
|
|
557
|
+
if (earlier) return earlier
|
|
558
|
+
const handle = handleOf(from)
|
|
559
|
+
if (handle === null)
|
|
560
|
+
throw new RelayError("policy_refused", `${sender} is not mirrored to the hub (the bridge's mirror selectors), so it can't message other machines`)
|
|
561
|
+
if (!connected) throw unavailable()
|
|
562
|
+
const clientId = params.client_id ?? newId("fwd")
|
|
563
|
+
const sent: SendParams = { ...params, client_id: clientId, to: targets.map((to) => core.route(to)!) }
|
|
564
|
+
delete (sent as { wait_s?: unknown }).wait_s
|
|
565
|
+
delete sent.attachments
|
|
566
|
+
const deadline = Date.now() + forwardWaitMs
|
|
567
|
+
for (let tries = 0; ; ) {
|
|
568
|
+
try {
|
|
569
|
+
const result = await request<SendResult>("forward", {
|
|
570
|
+
from: handle,
|
|
571
|
+
params: sent,
|
|
572
|
+
hop_floor: core.causalHop(from),
|
|
573
|
+
...(origin.tty === undefined ? {} : { tty: origin.tty }),
|
|
574
|
+
})
|
|
575
|
+
core.recordForward(from, { ...params, client_id: clientId }, result, origin)
|
|
576
|
+
return result
|
|
577
|
+
} catch (error) {
|
|
578
|
+
if (!(error instanceof RelayError && error.code === "unavailable")) throw error
|
|
579
|
+
if (fromHub.has(error)) {
|
|
580
|
+
const base = Math.min(maxMs, minMs * 2 ** tries++)
|
|
581
|
+
const wait = Math.max(base / 2 + Math.random() * (base / 2), (error.retryAfterS ?? 0) * 1000)
|
|
582
|
+
if (Date.now() + wait >= deadline) throw error
|
|
583
|
+
await new Promise((resolve) => setTimeout(resolve, wait))
|
|
584
|
+
} else if (!(await linkUp(deadline - Date.now()))) throw unavailable()
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function changed(endpoint: string) {
|
|
590
|
+
if (!ready) return
|
|
591
|
+
let view: EndpointView
|
|
592
|
+
try {
|
|
593
|
+
view = core.whoami(endpoint)
|
|
594
|
+
} catch {
|
|
595
|
+
return
|
|
596
|
+
}
|
|
597
|
+
if (mirrored(view)) fireAndForget("mirror", { endpoints: [outward(view)], full: false })
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
function resync() {
|
|
601
|
+
if (!ready) return
|
|
602
|
+
let frames: ReturnType<typeof fullMirror>
|
|
603
|
+
try {
|
|
604
|
+
frames = fullMirror()
|
|
605
|
+
} catch (error) {
|
|
606
|
+
return log(`relay bridge: mirror failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
607
|
+
}
|
|
608
|
+
for (const params of frames) fireAndForget("mirror", params)
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
function activity(endpoint: string, row: ActivityRow) {
|
|
612
|
+
if (!ready) return
|
|
613
|
+
const handle = handleOf(endpoint)
|
|
614
|
+
if (handle === null) return
|
|
615
|
+
fireAndForget("event", { from: handle, kind: row.kind, tool: row.tool ?? undefined, summary: options.redact ? undefined : (row.summary ?? undefined) })
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
async function contacts(): Promise<XListed[]> {
|
|
619
|
+
if (!connected) throw unavailable()
|
|
620
|
+
return (await request<{ contacts: XListed[] }>("list", { contacts: true })).contacts
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function status(): BridgeStatus {
|
|
624
|
+
return { enabled: true, machine, url: options.url, connected, last_error: lastError, since }
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
async function stop() {
|
|
628
|
+
stopped = true
|
|
629
|
+
clearTimeout(retryTimer)
|
|
630
|
+
for (const fn of waitingForLink.splice(0)) fn()
|
|
631
|
+
const socket = ws
|
|
632
|
+
ws = null
|
|
633
|
+
connected = false
|
|
634
|
+
ready = false
|
|
635
|
+
for (const p of pending.values()) p.reject(new RelayError("unavailable", "the bridge stopped"))
|
|
636
|
+
pending.clear()
|
|
637
|
+
if (socket && socket.readyState <= 1) socket.close(1000, "the broker stopped")
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
connect()
|
|
641
|
+
return { forward, changed, resync, activity, contacts, status, stop }
|
|
642
|
+
}
|