@scp-dev/hub 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/hub.ts ADDED
@@ -0,0 +1,1602 @@
1
+ import {
2
+ allowlisted,
3
+ CAPABILITIES,
4
+ CLI_NAME,
5
+ type Claim,
6
+ type CrossAccountLink,
7
+ cleanText,
8
+ dayStart,
9
+ declaredLimits,
10
+ type EndpointKind,
11
+ type EndpointView,
12
+ type Envelope,
13
+ findSecret,
14
+ type GateVerdict,
15
+ gateRefusal,
16
+ HUMAN_HANDLE,
17
+ honestSender,
18
+ joinRedacted,
19
+ judgeDetailForAgents,
20
+ LIMITS,
21
+ MESSAGE_KINDS,
22
+ type MessageKind,
23
+ newId,
24
+ one,
25
+ PREAMBLE,
26
+ type Presence,
27
+ PROTOCOL,
28
+ parseXMessage,
29
+ RelayError,
30
+ randomHex,
31
+ replyGrant,
32
+ type SendParams,
33
+ type SendResult,
34
+ SERVER_FIELDS,
35
+ type Sql,
36
+ SYSTEM_HANDLE,
37
+ sameSecret,
38
+ untilTomorrow,
39
+ type XContact,
40
+ XCounters,
41
+ type XDelivered,
42
+ type XEnvelopeFrom,
43
+ type XListed,
44
+ XLIMITS,
45
+ type XMessage,
46
+ type XRelease,
47
+ xlimits,
48
+ } from "@scp-dev/core"
49
+ import { FederatedCore, MACHINE_NAME, type MirrorChange, type XEndpoint } from "./federation"
50
+
51
+ export type HubAuth = "account" | "account+bridge"
52
+
53
+ export type HubRole = "owner" | "admin" | "member"
54
+
55
+ /** what the host proved about the token, plus its privilege */
56
+ export interface HubGrant {
57
+ tokenId: string
58
+ userId: string
59
+ actorId: string | null
60
+ expiresAt: number
61
+ /** a scoped token, not a full login; a grant that doesn't say false counts as scoped */
62
+ scoped: boolean
63
+ /** the actor's org role; null on a personal account */
64
+ role: HubRole | null
65
+ /** labels set on every endpoint the token claims or mirrors, over the client's own for the same keys */
66
+ labels?: Record<string, string>
67
+ }
68
+
69
+ /** per-socket state; JSON-serialisable, so the host can persist it with the socket */
70
+ export interface HubConn {
71
+ id: string
72
+ grant: HubGrant
73
+ role: "new" | "client" | "bridge" | "observer"
74
+ /** the machine a bridge or agent client is bound to; a human's or observer's namespace for unqualified handles */
75
+ machine: string | null
76
+ /** the hub endpoint id a client connection acts as */
77
+ endpoint: string | null
78
+ watching: boolean
79
+ subscribed: boolean
80
+ /** whether a human client said a terminal is attached */
81
+ tty?: boolean
82
+ }
83
+
84
+ export interface HubSocket {
85
+ conn: HubConn
86
+ send(text: string): void
87
+ close(code: number, reason: string): void
88
+ }
89
+
90
+ export interface HubDeps {
91
+ sql: Sql
92
+ /** synchronous sha256 hex */
93
+ hash: (value: string) => string
94
+ /** runs fn in one synchronous storage transaction */
95
+ transaction?: <T>(fn: () => T) => T
96
+ now?: () => number
97
+ /** every live socket of this account (rebuilt from persisted state when the host restarts) */
98
+ sockets(): HubSocket[]
99
+ /** persist socket.conn after the hub changed it */
100
+ save(socket: HubSocket): void
101
+ /** structured log line, e.g. relay_action; never bodies or tokens */
102
+ log?: (event: string, fields: Record<string, unknown>) => void
103
+ /** overrides of {@link HUB_LIMITS}; `Infinity` disables a cap, count or byte limit, durations stay finite */
104
+ limits?: Partial<HubLimits>
105
+ /** the link to other accounts' relays; without it an address of another account is not found */
106
+ crossAccount?: CrossAccountLink
107
+ }
108
+
109
+ export const HUB_LIMITS = {
110
+ machines: 16,
111
+ endpoints: 512,
112
+ sockets: 64,
113
+ frameBytes: 262_144,
114
+ callsPerMinute: 120,
115
+ bridgeCallsPerMinute: 600,
116
+ accountCallsPerMinute: 1200,
117
+ acksPerMinute: 1200,
118
+ accountAcksPerMinute: 4800,
119
+ storedBytes: 256 * 1024 * 1024,
120
+ clientMachineIdleS: 14 * 86_400,
121
+ idlePurgeS: 30 * 86_400,
122
+ httpWaitS: 25,
123
+ revalidateMs: 300_000,
124
+ deniedS: 3600,
125
+ refusalsPerMinute: 10,
126
+ } as const
127
+
128
+ export type HubLimits = { [K in keyof typeof HUB_LIMITS]: number }
129
+
130
+ const DURATIONS = ["clientMachineIdleS", "idlePurgeS", "httpWaitS", "revalidateMs", "deniedS"] as const satisfies readonly (keyof HubLimits)[]
131
+
132
+ const SWEEP_MS = 10 * 60_000
133
+ const DAY_MS = 86_400_000
134
+ const MIN_ALARM_MS = 60_000
135
+ const REVALIDATE_RETRY_MS = 30_000
136
+ const REVALIDATE_ERROR = "revalidate_error:"
137
+ const DENIED = "denied:"
138
+ const TOUCH_MS = 10_000
139
+
140
+ const HUB_MIGRATION = [
141
+ `CREATE TABLE IF NOT EXISTS machines (name TEXT PRIMARY KEY, token_id TEXT NOT NULL, kind TEXT NOT NULL, created_at INTEGER NOT NULL, last_seen INTEGER)`,
142
+ `CREATE TABLE IF NOT EXISTS hub_state (key TEXT PRIMARY KEY, value INTEGER NOT NULL)`,
143
+ `CREATE TABLE IF NOT EXISTS machine_tombstones (name TEXT PRIMARY KEY, actor_id TEXT, user_id TEXT, at INTEGER NOT NULL)`,
144
+ ]
145
+
146
+ const OBSERVER_METHODS = new Set(["list", "snapshot", "watch", "log", "machines", "machines_rm"])
147
+ const CLIENT_METHODS = new Set(["whoami", "send", "ask", "ask_wait", "inbox", "ack", "status", "event", ...OBSERVER_METHODS])
148
+
149
+ type Request = { jsonrpc?: string; id?: string | number | null; method?: unknown; params?: unknown }
150
+ type Params = Record<string, unknown>
151
+
152
+ const iso = (ms: number | null): string | null => (ms === null ? null : new Date(ms).toISOString())
153
+
154
+ function rpcError(error: unknown): { code: number; message: string; data?: { code: string; retry_after?: number } } {
155
+ if (!(error instanceof RelayError)) return { code: -32603, message: "internal error", data: { code: "unavailable" } }
156
+ const code = error.code === "method_not_found" ? -32601 : error.code === "invalid_params" ? -32602 : -32000
157
+ return { code, message: error.message, data: { code: error.code, ...(error.retryAfterS ? { retry_after: error.retryAfterS } : {}) } }
158
+ }
159
+
160
+ /** a frame's place in a full mirror sent in several frames (`part` of `parts` under one `id`), or null for a mirror of one frame */
161
+ function mirrorSync(value: unknown): { id: string; part: number; parts: number } | null {
162
+ if (value === undefined || value === null) return null
163
+ const { id, part, parts } = value as { id?: unknown; part?: unknown; parts?: unknown }
164
+ if (
165
+ typeof id !== "string" ||
166
+ !id ||
167
+ id.length > 64 ||
168
+ !Number.isInteger(parts) ||
169
+ (parts as number) < 1 ||
170
+ (parts as number) > 4096 ||
171
+ !Number.isInteger(part) ||
172
+ (part as number) < 1 ||
173
+ (part as number) > (parts as number)
174
+ ) {
175
+ throw new RelayError("invalid_params", "sync is { id, part, parts } of a full mirror in several frames")
176
+ }
177
+ return { id, part: part as number, parts: parts as number }
178
+ }
179
+
180
+ function serverFields(params: Params): void {
181
+ for (const field of SERVER_FIELDS) if (field in params) throw new RelayError("invalid_params", `${field} is set by the relay, not the sender`)
182
+ }
183
+
184
+ function object(value: unknown, what: string): Params {
185
+ if (value === undefined) return {}
186
+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new RelayError("invalid_params", `${what} must be an object`)
187
+ return value as Params
188
+ }
189
+
190
+ /** a full login: a grant that doesn't say it is unscoped counts as scoped */
191
+ const fullLogin = (grant: HubGrant): boolean => grant.scoped === false
192
+
193
+ /** a full login of the member `actorId` (null: the personal account's owner) */
194
+ const fullOf = (grant: HubGrant, actorId: string | null): boolean => fullLogin(grant) && grant.actorId === actorId
195
+
196
+ /** a full login of an org owner or admin, or of a personal account (its owner) */
197
+ const accountAdmin = (grant: HubGrant): boolean =>
198
+ fullLogin(grant) && (grant.role === "owner" || grant.role === "admin" || ((grant.role ?? null) === null && grant.actorId === null))
199
+
200
+ /** the grant's labels over `labels`; a value that is not a label object is left for the core to refuse */
201
+ function grantLabels(grant: HubGrant, labels: unknown): unknown {
202
+ if (!grant.labels || !Object.keys(grant.labels).length) return labels
203
+ if (labels !== undefined && labels !== null && (typeof labels !== "object" || Array.isArray(labels))) return labels
204
+ return { ...(labels as Record<string, unknown> | null | undefined), ...grant.labels }
205
+ }
206
+
207
+ /** each mirrored view with the grant's labels over its own */
208
+ function grantViews(grant: HubGrant, views: unknown): unknown {
209
+ if (!grant.labels || !Array.isArray(views)) return views
210
+ return views.map((view: unknown) =>
211
+ view && typeof view === "object" && !Array.isArray(view) ? { ...view, labels: grantLabels(grant, (view as Params).labels) } : view,
212
+ )
213
+ }
214
+
215
+ function listFilters(params: Params): { all?: boolean; harness?: string | undefined; kind?: EndpointKind | undefined; presence?: Presence | undefined } {
216
+ return {
217
+ all: params.all === true,
218
+ harness: typeof params.harness === "string" ? params.harness : undefined,
219
+ kind: params.kind === "agent" || params.kind === "human" ? params.kind : undefined,
220
+ presence: params.presence === "online" || params.presence === "idle" || params.presence === "offline" ? params.presence : undefined,
221
+ }
222
+ }
223
+
224
+ /** the hosted relay of one account: machines, bridges, direct clients, push and replay over a FederatedCore; pure, sockets are injected */
225
+ export class RelayHub {
226
+ private core: FederatedCore | null = null
227
+ private account = ""
228
+ private readonly now: () => number
229
+ private readonly waiters = new Map<string, Set<() => void>>()
230
+ private readonly buckets = new Map<string, { rate: number; tokens: number; at: number; refused: number }>()
231
+ private readonly transient = new Set<HubSocket>()
232
+ private readonly gone = new Set<string>()
233
+ /** per machine, the full mirror its bridge is sending in several frames; lost on eviction, which only skips that sync's offline sweep */
234
+ private readonly syncs = new Map<string, { id: string; parts: number; next: number; seen: Set<string> }>()
235
+ private stored: number | null = null
236
+ private denied: Map<string, number> | null = null
237
+ private xcounters: XCounters | null = null
238
+ private xkey: string | null = null
239
+ private readonly limits: Readonly<HubLimits>
240
+
241
+ constructor(private readonly deps: HubDeps) {
242
+ this.now = deps.now ?? Date.now
243
+ this.limits = { ...HUB_LIMITS, ...deps.limits }
244
+ for (const key of DURATIONS) if (!Number.isFinite(this.limits[key])) throw new RangeError(`hub limit ${key} must be finite`)
245
+ }
246
+
247
+ /** idempotent: migrates and records the account handle (the address user part, e.g. "ada"); call before handling each request */
248
+ init(account: { user: string }): void {
249
+ if (this.core && this.account === account.user) return
250
+ for (const statement of HUB_MIGRATION) this.deps.sql.all(statement)
251
+ const columns = this.deps.sql.all<{ name: string }>(`PRAGMA table_info(machines)`).map((c) => c.name)
252
+ if (!columns.includes("user_id")) this.deps.sql.all(`ALTER TABLE machines ADD COLUMN user_id TEXT`)
253
+ if (!columns.includes("actor_id")) this.deps.sql.all(`ALTER TABLE machines ADD COLUMN actor_id TEXT`)
254
+ const core = new FederatedCore(this.deps.sql, {
255
+ user: account.user,
256
+ now: this.now,
257
+ hash: this.deps.hash,
258
+ ...(this.deps.transaction ? { transaction: this.deps.transaction } : {}),
259
+ auth: "account",
260
+ endpointCap: this.limits.endpoints,
261
+ refusalsPerMinute: this.limits.refusalsPerMinute,
262
+ capEverySender: true,
263
+ xinbox: this.deps.crossAccount !== undefined,
264
+ })
265
+ core.migrate()
266
+ this.core = core
267
+ this.account = account.user
268
+ if (this.deps.crossAccount) {
269
+ this.xkey = core.xstate("key", () => randomHex(32))
270
+ this.xcounters = new XCounters(this.deps.sql, this.now)
271
+ }
272
+ }
273
+
274
+ /** forgets the core and in-memory state after the object's storage was deleted; the next init migrates afresh */
275
+ reset(): void {
276
+ this.core = null
277
+ this.account = ""
278
+ this.stored = null
279
+ this.denied = null
280
+ this.xcounters = null
281
+ this.xkey = null
282
+ this.buckets.clear()
283
+ this.gone.clear()
284
+ }
285
+
286
+ /** a fresh connection state for an accepted socket */
287
+ static connection(grant: HubGrant): HubConn {
288
+ return { id: newId("conn"), grant, role: "new", machine: null, endpoint: null, watching: false, subscribed: false }
289
+ }
290
+
291
+ /** a socket would exceed the `sockets` limit: the host refuses it (429) instead of accepting */
292
+ full(): boolean {
293
+ return this.deps.sockets().length >= this.limits.sockets
294
+ }
295
+
296
+ /** one JSON-RPC text frame from a socket; replies and pushes go out through socket.send */
297
+ async message(socket: HubSocket, text: string): Promise<void> {
298
+ if (new TextEncoder().encode(text).length > this.limits.frameBytes) {
299
+ this.write(socket, { jsonrpc: "2.0", id: null, error: rpcError(new RelayError("too_large", `a frame is at most ${this.limits.frameBytes} bytes`)) })
300
+ socket.close(1009, "frame too large")
301
+ this.closed(socket)
302
+ return
303
+ }
304
+ let request: unknown
305
+ try {
306
+ request = JSON.parse(text)
307
+ } catch {
308
+ this.write(socket, { jsonrpc: "2.0", id: null, error: { code: -32700, message: "parse error" } })
309
+ return
310
+ }
311
+ if (!request || typeof request !== "object" || Array.isArray(request)) {
312
+ this.write(socket, { jsonrpc: "2.0", id: null, error: { code: -32600, message: "one JSON-RPC request per frame" } })
313
+ return
314
+ }
315
+ const response = await this.handle(socket, request as Request)
316
+ if (response) this.write(socket, response)
317
+ }
318
+
319
+ /** the socket closed (any reason) */
320
+ closed(socket: HubSocket): void {
321
+ this.release(socket, false)
322
+ }
323
+
324
+ /** POST /v1/relay/rpc: a JSON-RPC request or batch run in order on a short-lived connection; returns the response(s) */
325
+ async batch(grant: HubGrant, body: unknown): Promise<unknown> {
326
+ if (JSON.stringify(body ?? null).length > this.limits.frameBytes) {
327
+ return { jsonrpc: "2.0", id: null, error: rpcError(new RelayError("too_large", `a request is at most ${this.limits.frameBytes} bytes`)) }
328
+ }
329
+ const requests = Array.isArray(body) ? body : [body]
330
+ if (!requests.length) return { jsonrpc: "2.0", id: null, error: { code: -32600, message: "an empty batch" } }
331
+ const socket: HubSocket = { conn: RelayHub.connection(grant), send: () => {}, close: () => {} }
332
+ this.transient.add(socket)
333
+ const responses: unknown[] = []
334
+ try {
335
+ for (const request of requests) {
336
+ if (!request || typeof request !== "object" || Array.isArray(request)) {
337
+ responses.push({ jsonrpc: "2.0", id: null, error: { code: -32600, message: "a request is an object" } })
338
+ continue
339
+ }
340
+ const response = await this.handle(socket, request as Request)
341
+ if (response) responses.push(response)
342
+ }
343
+ } finally {
344
+ this.closed(socket)
345
+ this.transient.delete(socket)
346
+ }
347
+ return Array.isArray(body) ? responses : (responses[0] ?? null)
348
+ }
349
+
350
+ /**
351
+ * closes every socket (and in-flight /rpc call) whose grant matches with 1008, audits it, and unbinds the machines
352
+ * the matching grants bound behind a tombstone of their member, keeping their endpoint rows and queued messages;
353
+ * returns how many closed
354
+ */
355
+ revoke(match: { tokenId?: string; userId?: string; actorId?: string }): number {
356
+ const keys = (Object.entries(match) as [keyof HubGrant, string | undefined][]).filter(([, value]) => value !== undefined)
357
+ if (!keys.length) return 0
358
+ let closed = 0
359
+ const tokens = new Set(match.tokenId !== undefined ? [match.tokenId] : [])
360
+ for (const socket of [...this.deps.sockets(), ...this.transient]) {
361
+ if (this.gone.has(socket.conn.id) || !keys.every(([key, value]) => socket.conn.grant[key] === value)) continue
362
+ tokens.add(socket.conn.grant.tokenId)
363
+ this.kick(socket, "the token was revoked")
364
+ closed++
365
+ }
366
+ if (closed) this.relay.record("revoke", null, null, { sockets: closed })
367
+ const columns = match.tokenId !== undefined ? [["token_id", match.tokenId]] : keys.map(([key, value]) => [key === "userId" ? "user_id" : "actor_id", value])
368
+ const where = columns.map(([column]) => `${column} = ?`).join(" AND ")
369
+ const now = this.now()
370
+ for (const row of this.deps.sql.all<{ name: string; token_id: string; actor_id: string | null; user_id: string | null }>(
371
+ `DELETE FROM machines WHERE ${where} RETURNING name, token_id, actor_id, user_id`,
372
+ ...columns.map(([, value]) => value!),
373
+ )) {
374
+ tokens.add(row.token_id)
375
+ this.deps.sql.all(`INSERT OR REPLACE INTO machine_tombstones (name, actor_id, user_id, at) VALUES (?, ?, ?, ?)`, row.name, row.actor_id, row.user_id, now)
376
+ this.relay.record("machine_unbind", null, row.name, { machine: row.name })
377
+ }
378
+ const denied = this.deniedGrants()
379
+ for (const token of tokens) {
380
+ denied.set(DENIED + token, now)
381
+ this.setState(DENIED + token, now)
382
+ }
383
+ this.deps.log?.("relay_action", { account: this.account, method: "revoke", sockets: closed })
384
+ return closed
385
+ }
386
+
387
+ /**
388
+ * the host's alarm: revalidates each socket's grant and sweeps; returns the next alarm time in ms, or null. A failed
389
+ * check closes with 1008, an expired grant with 4001 (reconnect for a new one). A check that throws keeps the
390
+ * sockets and retries in 30 s; once a token's checks kept throwing for the `revalidateMs` limit its sockets close
391
+ * with the retryable 1013.
392
+ */
393
+ async alarm(validate: (grant: HubGrant) => Promise<boolean>): Promise<number | null> {
394
+ const now = this.now()
395
+ this.xcounters?.trueUp()
396
+ const checks = new Map<string, Promise<boolean | null>>()
397
+ const erroring = new Set<string>()
398
+ for (const socket of [...this.deps.sockets()]) {
399
+ const grant = socket.conn.grant
400
+ if (grant.expiresAt <= now) {
401
+ this.kick(socket, "the token grant expired, reconnect", 4001)
402
+ continue
403
+ }
404
+ const key = JSON.stringify([grant.tokenId, grant.userId, grant.actorId, grant.expiresAt, grant.scoped, grant.role])
405
+ let check = checks.get(key)
406
+ if (!check) {
407
+ check = validate(grant).catch((error: unknown) => {
408
+ this.deps.log?.("relay_revalidate_error", { account: this.account, error: error instanceof Error ? error.name : "error" })
409
+ return null
410
+ })
411
+ checks.set(key, check)
412
+ }
413
+ const valid = await check
414
+ if (valid === false) {
415
+ this.kick(socket, "the token is no longer valid")
416
+ } else if (valid === null) {
417
+ erroring.add(grant.tokenId)
418
+ const key = REVALIDATE_ERROR + grant.tokenId
419
+ let since = this.state(key)
420
+ if (since === null) {
421
+ since = now
422
+ this.setState(key, since)
423
+ }
424
+ if (now - since >= this.limits.revalidateMs) this.kick(socket, "the token could not be revalidated, try again", 1013)
425
+ }
426
+ }
427
+ for (const { key } of this.deps.sql.all<{ key: string }>(`SELECT key FROM hub_state WHERE key LIKE ?`, `${REVALIDATE_ERROR}%`)) {
428
+ const tokenId = key.slice(REVALIDATE_ERROR.length)
429
+ const lingering = this.deps.sockets().some((s) => s.conn.grant.tokenId === tokenId && !this.gone.has(s.conn.id))
430
+ if (!erroring.has(tokenId) || !lingering) this.deps.sql.all(`DELETE FROM hub_state WHERE key = ?`, key)
431
+ }
432
+ this.setState("last_validated", now)
433
+ if (now - (this.state("last_sweep") ?? 0) >= SWEEP_MS) {
434
+ this.relay.sweep()
435
+ this.expireMachines(now)
436
+ this.forgetTombstones(now)
437
+ const denied = this.deniedGrants()
438
+ for (const [key, at] of denied) {
439
+ if (at > now - this.limits.deniedS * 1000) continue
440
+ denied.delete(key)
441
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = ?`, key)
442
+ }
443
+ this.stored = null
444
+ this.gone.clear()
445
+ for (const [key, bucket] of this.buckets) if (bucket.tokens + ((now - bucket.at) * bucket.rate) / 60_000 >= bucket.rate) this.buckets.delete(key)
446
+ this.xcounters?.prune()
447
+ this.setState("last_sweep", now)
448
+ }
449
+ if (this.idle(now)) {
450
+ if (this.state("idle_since") === null) this.setState("idle_since", now)
451
+ } else {
452
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = 'idle_since'`)
453
+ }
454
+ return this.nextAlarm()
455
+ }
456
+
457
+ /** when the next alarm should fire (earliest grant expiry, revalidation or its retry after an error, sweep, retention), or null when the hub holds nothing */
458
+ nextAlarm(): number | null {
459
+ const now = this.now()
460
+ const trueUp = this.xcounters?.dueAt() ?? null
461
+ const held = this.retentionAlarm(now)
462
+ const retention = held === null || trueUp === null ? (held ?? trueUp) : Math.min(held, trueUp)
463
+ const sockets = this.deps.sockets()
464
+ if (!sockets.length) return retention
465
+ let validated = this.state("last_validated")
466
+ if (validated === null) {
467
+ validated = now
468
+ this.setState("last_validated", validated)
469
+ }
470
+ let swept = this.state("last_sweep")
471
+ if (swept === null) {
472
+ swept = now
473
+ this.setState("last_sweep", swept)
474
+ }
475
+ const retrying = one(this.deps.sql, `SELECT 1 FROM hub_state WHERE key LIKE ? LIMIT 1`, `${REVALIDATE_ERROR}%`) !== undefined
476
+ return Math.min(
477
+ validated + (retrying ? REVALIDATE_RETRY_MS : this.limits.revalidateMs),
478
+ swept + SWEEP_MS,
479
+ retention ?? Number.POSITIVE_INFINITY,
480
+ ...sockets.map((s) => s.conn.grant.expiresAt),
481
+ )
482
+ }
483
+
484
+ /** the account held no machines, tombstones, sockets or queued messages for the `idlePurgeS` limit: the object may delete its storage */
485
+ purgeDue(): boolean {
486
+ const since = this.state("idle_since")
487
+ return since !== null && this.now() - since >= this.limits.idlePurgeS * 1000 && this.idle(this.now())
488
+ }
489
+
490
+ /** while the hub holds rows: the earliest unacked delivery's expiry, at most a day and at least a minute away */
491
+ private retentionAlarm(now: number): number | null {
492
+ const sql = this.deps.sql
493
+ const held = one<{ held: number }>(
494
+ sql,
495
+ `SELECT EXISTS (SELECT 1 FROM machines) OR EXISTS (SELECT 1 FROM machine_tombstones) OR EXISTS (SELECT 1 FROM endpoints WHERE handle NOT IN (?, ?))
496
+ OR EXISTS (SELECT 1 FROM messages) OR EXISTS (SELECT 1 FROM deliveries) OR EXISTS (SELECT 1 FROM audit) AS held`,
497
+ HUMAN_HANDLE,
498
+ SYSTEM_HANDLE,
499
+ )!.held
500
+ if (!held) return null
501
+ const expiry = one<{ at: number | null }>(sql, `SELECT MIN(expires_at) AS at FROM deliveries WHERE acked_at IS NULL`)!.at
502
+ return Math.max(now + MIN_ALARM_MS, Math.min(expiry ?? Number.POSITIVE_INFINITY, now + DAY_MS))
503
+ }
504
+
505
+ /** no machine bound or tombstoned, no live socket and nothing queued */
506
+ private idle(now: number): boolean {
507
+ if (this.deps.sockets().some((s) => !this.gone.has(s.conn.id))) return false
508
+ return !one<{ busy: number }>(
509
+ this.deps.sql,
510
+ `SELECT EXISTS (SELECT 1 FROM machines) OR EXISTS (SELECT 1 FROM machine_tombstones) OR EXISTS (SELECT 1 FROM deliveries WHERE acked_at IS NULL AND expires_at > ?) AS busy`,
511
+ now,
512
+ )!.busy
513
+ }
514
+
515
+ // dispatch
516
+
517
+ private get relay(): FederatedCore {
518
+ if (!this.core) throw new Error("RelayHub.init was not called")
519
+ return this.core
520
+ }
521
+
522
+ private async handle(socket: HubSocket, request: Request): Promise<object | null> {
523
+ const id = request.id ?? null
524
+ const method = String(request.method)
525
+ let code = "ok"
526
+ try {
527
+ this.refuseDenied(socket)
528
+ this.spend(socket, method, request.params)
529
+ const result = await this.dispatch(socket, method, object(request.params, "params"))
530
+ return request.id === undefined ? null : { jsonrpc: "2.0", id, result }
531
+ } catch (error) {
532
+ code = error instanceof RelayError ? error.code : "internal"
533
+ if (code === "internal") this.deps.log?.("relay_internal_error", { account: this.account, method, error: error instanceof Error ? error.name : "error" })
534
+ return request.id === undefined ? null : { jsonrpc: "2.0", id, error: rpcError(error) }
535
+ } finally {
536
+ this.deps.log?.("relay_action", { account: this.account, machine: socket.conn.machine, method, code })
537
+ }
538
+ }
539
+
540
+ /**
541
+ * one call from the token's bucket for the connection's role (a bridge, or a bridge hello, has its own, larger one)
542
+ * and from the account's; acks (one token per seq) and rejects spend from separate ack buckets of the token and the
543
+ * account instead, so they never starve calls, and an activity event is refused once either call bucket is below a
544
+ * quarter, so sends keep the rest
545
+ */
546
+ private spend(socket: HubSocket, method: string, params: unknown): void {
547
+ const conn = socket.conn
548
+ const now = this.now()
549
+ if ((method === "ack" && (conn.role === "client" || conn.role === "bridge")) || (method === "reject" && conn.role === "bridge")) {
550
+ const seqs = (params as Params | null)?.seqs
551
+ const cost = method === "ack" && Array.isArray(seqs) ? Math.min(Math.max(seqs.length, 1), LIMITS.inboxPage) : 1
552
+ const token = this.bucket(`${conn.grant.tokenId}:ack`, this.limits.acksPerMinute, now)
553
+ const account = this.bucket("account:ack", this.limits.accountAcksPerMinute, now)
554
+ for (const [b, scope] of [
555
+ [token, "a token"],
556
+ [account, "an account"],
557
+ ] as const) {
558
+ if (b.tokens < cost)
559
+ throw new RelayError("rate_limited", `at most ${b.rate} acked seqs per minute on ${scope}`, Math.max(1, Math.ceil(((cost - b.tokens) * 60) / b.rate)))
560
+ }
561
+ token.tokens -= cost
562
+ account.tokens -= cost
563
+ return
564
+ }
565
+ const bridge = conn.role === "bridge" || (conn.role === "new" && method === "hello" && (params as Params | null)?.bridge === true)
566
+ const rate = bridge ? this.limits.bridgeCallsPerMinute : this.limits.callsPerMinute
567
+ const bucket = this.bucket(`${conn.grant.tokenId}:${bridge ? "bridge" : "client"}`, rate, now)
568
+ const account = this.bucket("account", this.limits.accountCallsPerMinute, now)
569
+ if (method === "event") {
570
+ for (const b of [bucket, account]) {
571
+ if (b.tokens < b.rate / 4)
572
+ throw new RelayError(
573
+ "rate_limited",
574
+ "activity events wait while the token or the account is near its call limit",
575
+ Math.max(1, Math.ceil(((b.rate / 4 - b.tokens) * 60) / b.rate)),
576
+ )
577
+ }
578
+ }
579
+ if (bucket.tokens < 1) {
580
+ if (++bucket.refused >= rate) {
581
+ socket.close(1013, "rate limited")
582
+ this.closed(socket)
583
+ }
584
+ throw new RelayError(
585
+ "rate_limited",
586
+ `at most ${rate} ${bridge ? "bridge " : ""}calls per minute on a token`,
587
+ Math.max(1, Math.ceil(((1 - bucket.tokens) * 60) / rate)),
588
+ )
589
+ }
590
+ if (account.tokens < 1) {
591
+ throw new RelayError(
592
+ "rate_limited",
593
+ `at most ${this.limits.accountCallsPerMinute} calls per minute on an account`,
594
+ Math.max(1, Math.ceil(((1 - account.tokens) * 60) / account.rate)),
595
+ )
596
+ }
597
+ bucket.tokens -= 1
598
+ bucket.refused = 0
599
+ account.tokens -= 1
600
+ }
601
+
602
+ private bucket(key: string, rate: number, now: number): { rate: number; tokens: number; at: number; refused: number } {
603
+ if (rate === Number.POSITIVE_INFINITY) return { rate, tokens: rate, at: now, refused: 0 }
604
+ const bucket = this.buckets.get(key) ?? { rate, tokens: rate, at: now, refused: 0 }
605
+ bucket.tokens = Math.min(rate, bucket.tokens + ((now - bucket.at) * rate) / 60_000)
606
+ bucket.at = now
607
+ this.buckets.set(key, bucket)
608
+ return bucket
609
+ }
610
+
611
+ private async dispatch(socket: HubSocket, method: string, params: Params): Promise<unknown> {
612
+ if (this.gone.has(socket.conn.id)) throw new RelayError("unauthorized", "this connection is closed")
613
+ if (method === "hello") return this.hello(socket, params)
614
+ if (socket.conn.role === "new") throw new RelayError("unauthorized", "call hello first")
615
+ if (socket.conn.role === "bridge") return this.bridgeMethod(socket, method, params)
616
+ if (socket.conn.role === "observer" && !OBSERVER_METHODS.has(method)) {
617
+ if (CLIENT_METHODS.has(method)) throw new RelayError("unauthorized", "this connection only observes: say hello as an agent or the human")
618
+ throw new RelayError("method_not_found", `no method ${method}`)
619
+ }
620
+ return this.clientMethod(socket, method, params)
621
+ }
622
+
623
+ private hello(socket: HubSocket, params: Params): unknown {
624
+ const core = this.relay
625
+ const conn = socket.conn
626
+ if (conn.role !== "new") throw new RelayError("invalid_params", "hello was already called on this connection")
627
+ if (params.protocol !== PROTOCOL) throw new RelayError("invalid_params", `this relay speaks ${PROTOCOL}`)
628
+ if (params.transient !== undefined)
629
+ throw new RelayError("invalid_params", "the hub takes no hook connections: hooks report to the local broker, which bridges them")
630
+ serverFields(params)
631
+ const observe = params.observe === true
632
+ const named = params.machine
633
+ if ((!observe || named !== undefined) && (typeof named !== "string" || !MACHINE_NAME.test(named) || named === "me" || named === this.account)) {
634
+ throw new RelayError("invalid_params", "machine is a name of lowercase letters, digits and -, other than me and the account handle")
635
+ }
636
+ const bridge = params.bridge === true
637
+ const human = params.human === true
638
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = 'idle_since'`)
639
+ const info = { account: this.account, capabilities: CAPABILITIES, server_time: new Date(this.now()).toISOString(), limits: { ...LIMITS, ...this.limits } }
640
+ if (observe) {
641
+ if (bridge || human || params.claim !== undefined || params.resume !== undefined)
642
+ throw new RelayError("invalid_params", "an observer hello has no bridge, human, claim or resume")
643
+ conn.role = "observer"
644
+ conn.machine = typeof named === "string" ? named : null
645
+ this.save(socket)
646
+ return { ...info, ...(conn.machine ? { machine: conn.machine } : {}) }
647
+ }
648
+ const machine = named as string
649
+ const base = { ...info, machine }
650
+ if (bridge && this.transient.has(socket)) throw new RelayError("invalid_params", "a bridge connects over the WebSocket")
651
+ if (human && typeof params.tty !== "boolean")
652
+ throw new RelayError("invalid_params", "a human hello says whether a terminal is attached (tty: true or false)")
653
+
654
+ if (human) {
655
+ if (!fullLogin(conn.grant)) throw new RelayError("unauthorized", "a scoped token acts as an agent: use --as")
656
+ const endpoint = core.claimHuman(params.tty as boolean)
657
+ conn.role = "client"
658
+ conn.machine = machine
659
+ conn.endpoint = endpoint.id
660
+ conn.tty = params.tty as boolean
661
+ this.save(socket)
662
+ this.fleet({ type: "presence", machine, endpoint: endpoint.address, presence: endpoint.presence, harness: endpoint.harness, labels: endpoint.labels })
663
+ return { ...base, endpoint, preamble: PREAMBLE }
664
+ }
665
+
666
+ const claimed = object(params.claim, "claim")
667
+ const claim = (conn.grant.labels ? { ...claimed, labels: grantLabels(conn.grant, claimed.labels) } : claimed) as Claim
668
+ const held = this.bindable(machine, conn.grant)
669
+
670
+ if (bridge) {
671
+ this.bind(machine, conn.grant, "bridge", held)
672
+ core.adoptClaimed(machine)
673
+ conn.role = "bridge"
674
+ conn.machine = machine
675
+ this.save(socket)
676
+ for (const other of [...this.deps.sockets()]) {
677
+ if (other.conn.id === conn.id || other.conn.role !== "bridge" || other.conn.machine !== machine) continue
678
+ other.close(4000, "replaced by a newer bridge for this machine")
679
+ this.release(other, true)
680
+ }
681
+ core.record("bridge", null, machine, { machine })
682
+ return { ...base, endpoints: core.remoteViews({ except: machine }) }
683
+ }
684
+
685
+ if (one<{ kind: string }>(this.deps.sql, `SELECT kind FROM machines WHERE name = ?`, machine)?.kind === "bridge") {
686
+ throw new RelayError(
687
+ "invalid_params",
688
+ `machine ${machine} is linked through its broker's bridge; run the command there without --cloud, or pick another --machine`,
689
+ )
690
+ }
691
+ let endpoint: EndpointView
692
+ let token: string | undefined
693
+ if (typeof params.resume === "string")
694
+ endpoint = core.resumeOn(machine, params.resume, conn.grant.labels ? (own) => grantLabels(conn.grant, own) : undefined)
695
+ else ({ endpoint, token } = core.claimOn(machine, claim, "account"))
696
+ this.bind(machine, conn.grant, "client", held)
697
+ conn.role = "client"
698
+ conn.machine = machine
699
+ conn.endpoint = endpoint.id
700
+ this.save(socket)
701
+ this.fleet({ type: "presence", machine, endpoint: endpoint.address, presence: endpoint.presence, harness: endpoint.harness, labels: endpoint.labels })
702
+ this.announceEndpoint(endpoint.id)
703
+ return { ...base, endpoint, ...(token ? { token } : {}), preamble: PREAMBLE }
704
+ }
705
+
706
+ /** refuses a machine bound to another token, a revoked token's machine to anyone but its member or an admin, or one past the machine cap; returns whether this token already holds it */
707
+ private bindable(machine: string, grant: HubGrant): boolean {
708
+ const bound = one<{ token_id: string }>(this.deps.sql, `SELECT token_id FROM machines WHERE name = ?`, machine)
709
+ if (bound && bound.token_id !== grant.tokenId) {
710
+ this.relay.record("refuse", null, machine, { code: "unauthorized", machine })
711
+ throw new RelayError("unauthorized", `machine ${machine} is bound to another token; release it with ${CLI_NAME} machines rm ${machine}`)
712
+ }
713
+ if (bound) return true
714
+ const tomb = one<{ actor_id: string | null }>(this.deps.sql, `SELECT actor_id FROM machine_tombstones WHERE name = ?`, machine)
715
+ if (tomb && !fullOf(grant, tomb.actor_id) && !accountAdmin(grant)) {
716
+ this.relay.record("refuse", null, machine, { code: "unauthorized", machine })
717
+ throw new RelayError(
718
+ "unauthorized",
719
+ `machine ${machine} was unbound when its token was revoked; a full login of the same member (or an org owner or admin) can take it back, or ${CLI_NAME} machines rm ${machine} drops its queued messages`,
720
+ )
721
+ }
722
+ const count = one<{ n: number }>(this.deps.sql, `SELECT COUNT(*) AS n FROM machines`)!.n
723
+ if (count >= this.limits.machines)
724
+ throw new RelayError("too_large", `at most ${this.limits.machines} machines per account; release one with ${CLI_NAME} machines rm`)
725
+ return false
726
+ }
727
+
728
+ /** binds the machine to the token once its hello succeeded; `held` is what bindable() returned */
729
+ private bind(machine: string, grant: HubGrant, kind: "bridge" | "client", held: boolean): void {
730
+ const core = this.relay
731
+ const now = this.now()
732
+ if (held) {
733
+ this.deps.sql.all(
734
+ `UPDATE machines SET last_seen = ?, user_id = ?, actor_id = ?, kind = CASE WHEN kind = 'bridge' THEN 'bridge' ELSE ? END WHERE name = ?`,
735
+ now,
736
+ grant.userId,
737
+ grant.actorId,
738
+ kind,
739
+ machine,
740
+ )
741
+ return
742
+ }
743
+ this.deps.sql.all(
744
+ `INSERT INTO machines (name, token_id, user_id, actor_id, kind, created_at, last_seen) VALUES (?, ?, ?, ?, ?, ?, ?)`,
745
+ machine,
746
+ grant.tokenId,
747
+ grant.userId,
748
+ grant.actorId,
749
+ kind,
750
+ now,
751
+ now,
752
+ )
753
+ this.deps.sql.all(`DELETE FROM machine_tombstones WHERE name = ?`, machine)
754
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = 'idle_since'`)
755
+ core.record("machine_bind", null, machine, { machine, kind })
756
+ this.deps.log?.("relay_action", { account: this.account, machine, method: "machine_bind", code: "ok" })
757
+ }
758
+
759
+ private async clientMethod(socket: HubSocket, method: string, params: Params): Promise<unknown> {
760
+ const core = this.relay
761
+ const conn = socket.conn
762
+ const me = conn.endpoint
763
+ if (me) core.touch(me, TOUCH_MS)
764
+ switch (method) {
765
+ case "list":
766
+ if (params.contacts === true) return { contacts: await this.contactList() }
767
+ return { endpoints: core.list(listFilters(params)) }
768
+ case "snapshot":
769
+ return core.snapshot({ all: params.all === true, activity: typeof params.activity === "number" ? params.activity : undefined })
770
+ case "watch":
771
+ conn.watching = true
772
+ this.save(socket)
773
+ return core.snapshot({ activity: 20 })
774
+ case "log":
775
+ // no hosted connection is a verified human at a terminal: judge rows lose their reason and kinds (the dashboard shows them)
776
+ return {
777
+ rows: core
778
+ .log({ since: params.since as number | undefined, limit: params.limit as number | undefined })
779
+ .map((row) => (row.event.startsWith("judge_") ? { ...row, detail: judgeDetailForAgents(row.detail) } : row)),
780
+ }
781
+ case "machines":
782
+ return { machines: this.machineList() }
783
+ case "machines_rm":
784
+ return this.releaseMachine(socket, params.name)
785
+ }
786
+ if (!me) throw new RelayError("unauthorized", "this connection has no endpoint")
787
+ switch (method) {
788
+ case "whoami":
789
+ return core.whoami(me)
790
+ case "send":
791
+ case "ask": {
792
+ serverFields(params)
793
+ const send = { ...(params as unknown as SendParams) }
794
+ delete (send as Params).wait_s
795
+ if (method === "ask") send.kind = "request"
796
+ const bytes = this.room(me, send.body)
797
+ const result = core.within(conn.machine, () => core.send(me, send, conn.tty === undefined ? {} : { tty: conn.tty }))
798
+ if (!result.duplicate) this.stash(bytes)
799
+ if (!result.duplicate) this.delivered(me, result, send.kind ?? "message")
800
+ if (method === "send") return result
801
+ return this.awaitReply(socket, me, result.id, this.waitMs(socket, params.wait_s), result)
802
+ }
803
+ case "ask_wait": {
804
+ if (typeof params.id !== "string") throw new RelayError("invalid_params", "id is the request's message id")
805
+ return this.awaitReply(socket, me, params.id, this.waitMs(socket, params.wait_s))
806
+ }
807
+ case "inbox": {
808
+ const query = { after: params.after as number | undefined, limit: params.limit as number | undefined }
809
+ const deadline = Date.now() + this.waitMs(socket, params.wait_s)
810
+ let messages = core.inbox(me, query)
811
+ while (!messages.length && !this.gone.has(conn.id) && Date.now() < deadline) {
812
+ await this.waitFor(me, deadline - Date.now(), socket)
813
+ messages = core.inbox(me, query)
814
+ }
815
+ return { messages, next: messages.length ? messages[messages.length - 1]!.seq : (query.after ?? 0) }
816
+ }
817
+ case "ack":
818
+ return { acked: core.ack(me, params.seqs) }
819
+ case "status": {
820
+ core.status(me, params.state, params.note)
821
+ const view = core.whoami(me)
822
+ this.fleet({ type: "status", machine: conn.machine, endpoint: view.address, status: view.status })
823
+ this.announceEndpoint(me)
824
+ return view
825
+ }
826
+ case "event": {
827
+ const row = core.event(me, params)
828
+ this.fleet({ type: "activity", machine: conn.machine, ...row })
829
+ return row
830
+ }
831
+ }
832
+ throw new RelayError("method_not_found", `no method ${method}`)
833
+ }
834
+
835
+ private async bridgeMethod(socket: HubSocket, method: string, params: Params): Promise<unknown> {
836
+ const core = this.relay
837
+ const conn = socket.conn
838
+ const machine = conn.machine!
839
+ switch (method) {
840
+ case "mirror": {
841
+ const sync = mirrorSync(params.sync)
842
+ if (!sync) {
843
+ const full = params.full === true
844
+ const changes = core.mirror(machine, grantViews(conn.grant, params.endpoints), { full, auth: "account+bridge" })
845
+ this.announce(machine, changes, full)
846
+ return { changed: changes.length }
847
+ }
848
+ let state = this.syncs.get(machine)
849
+ if (sync.part === 1) {
850
+ state = { id: sync.id, parts: sync.parts, next: 1, seen: new Set() }
851
+ this.syncs.set(machine, state)
852
+ }
853
+ const intact = state !== undefined && state.id === sync.id && state.parts === sync.parts && state.next === sync.part
854
+ if (!intact) this.syncs.delete(machine)
855
+ const closing = intact && sync.part === sync.parts
856
+ const changes = core.mirror(machine, grantViews(conn.grant, params.endpoints), {
857
+ full: closing,
858
+ auth: "account+bridge",
859
+ ...(intact ? { seen: state!.seen } : {}),
860
+ })
861
+ if (intact) state!.next++
862
+ if (closing) this.syncs.delete(machine)
863
+ this.announce(machine, changes, closing)
864
+ return { changed: changes.length }
865
+ }
866
+ case "forward": {
867
+ const sender = core.remoteEndpoint(machine, params.from)
868
+ const send = { ...object(params.params, "params") }
869
+ serverFields(send)
870
+ const asked = params.hop_floor ?? 0
871
+ if (!Number.isInteger(asked) || (asked as number) < 0 || (asked as number) > 1000) throw new RelayError("invalid_params", "hop_floor is a hop count")
872
+ if ([send.to].flat().some((to) => core.foreignAccount(to) !== null)) return this.xsend(sender, send, { tty: params.tty === true, hop: asked as number })
873
+ delete send.attachments
874
+ const floor = Math.min(asked as number, LIMITS.hopMax + 1)
875
+ if (floor > 0) core.markReceived(sender, floor - 1)
876
+ const bytes = this.room(sender, send.body)
877
+ const result = core.within(machine, () => core.send(sender, send as unknown as SendParams, { tty: params.tty === true }))
878
+ if (!result.duplicate) this.stash(bytes)
879
+ if (!result.duplicate) this.delivered(sender, result, (send.kind as string | undefined) ?? "message")
880
+ return result
881
+ }
882
+ case "subscribe": {
883
+ const after = params.after ?? 0
884
+ if (!Number.isInteger(after) || (after as number) < 0) throw new RelayError("invalid_params", "after is a seq")
885
+ if (this.deps.crossAccount) core.xsenders(await this.xinbound())
886
+ conn.subscribed = true
887
+ this.save(socket)
888
+ let cursor = 0
889
+ let replayed = 0
890
+ for (;;) {
891
+ const page = core.pending(machine, Math.max(cursor, after as number), 200, cursor)
892
+ for (const delivery of page)
893
+ this.write(socket, { jsonrpc: "2.0", method: "deliver", params: { seq: delivery.seq, endpoint: delivery.endpoint, message: delivery.message } })
894
+ replayed += page.length
895
+ if (page.length) cursor = page[page.length - 1]!.seq
896
+ if (page.length < 200) break
897
+ }
898
+ return { replayed, next: Math.max(cursor, after as number) }
899
+ }
900
+ case "ack":
901
+ return { acked: core.ackOn(machine, params.seqs) }
902
+ case "reject": {
903
+ const { notice } = core.rejectOn(machine, params.seq, params.code)
904
+ if (notice) this.route(notice.seq, notice.endpoint)
905
+ return { rejected: true }
906
+ }
907
+ case "event": {
908
+ const sender = core.remoteEndpoint(machine, params.from)
909
+ const row = core.event(sender, params)
910
+ this.fleet({ type: "activity", machine, ...row })
911
+ return row
912
+ }
913
+ case "list":
914
+ if (params.contacts === true) return { contacts: await this.contactList() }
915
+ return { endpoints: core.list(listFilters(params)) }
916
+ case "watch":
917
+ conn.watching = true
918
+ this.save(socket)
919
+ return core.snapshot({ activity: 20 })
920
+ }
921
+ throw new RelayError("method_not_found", `no method ${method} on a bridge`)
922
+ }
923
+
924
+ // other accounts
925
+
926
+ /**
927
+ * A bridge's forward to another account's endpoint, writing no row here: shape, the contact, the sender on this account's allowlist,
928
+ * its scope (or the reply grant of the message it answers), the caps, the scrubber, the egress gate, then the peer relay. Anything
929
+ * unresolvable is not_found, as for a missing endpoint.
930
+ */
931
+ private async xsend(
932
+ sender: string,
933
+ send: Params,
934
+ origin: { tty: boolean; hop: number; judged?: boolean; clientId?: string; id?: string },
935
+ ): Promise<SendResult> {
936
+ const core = this.relay
937
+ const link = this.deps.crossAccount
938
+ const targets = [send.to].flat()
939
+ const missing = () => new RelayError("not_found", `no endpoint ${targets.map(String).join(", ")}`)
940
+ if (targets.length !== 1) throw new RelayError("invalid_params", "a message to another account goes to exactly one endpoint: no lists, groups or tree:")
941
+ const target = core.foreignAccount(targets[0])
942
+ if (!target || !link || !this.xcounters || !this.xkey) throw missing()
943
+ if (typeof send.body !== "string") throw new RelayError("invalid_params", "body must be a string")
944
+ const bytes = new TextEncoder().encode(send.body).length
945
+ if (bytes > XLIMITS.bodyBytes) throw new RelayError("too_large", `a message to another account is at most ${XLIMITS.bodyBytes} bytes; this one is ${bytes}`)
946
+ if (send.attachments !== undefined && !(Array.isArray(send.attachments) && !send.attachments.length))
947
+ throw new RelayError("data_class_refused", "attachments stay in this account")
948
+ if (send.data_class === "private") throw new RelayError("data_class_refused", "a private message stays in this account")
949
+ if (send.data_class !== undefined && send.data_class !== "public") throw new RelayError("invalid_params", "data_class must be public or private")
950
+ const kind = (send.kind ?? "message") as MessageKind
951
+ if (!MESSAGE_KINDS.includes(kind)) throw new RelayError("invalid_params", `kind must be one of ${MESSAGE_KINDS.join(", ")}`)
952
+ if (send.reply_to !== undefined && typeof send.reply_to !== "string") throw new RelayError("invalid_params", "reply_to is a message id")
953
+ if (origin.hop > XLIMITS.hopMax)
954
+ throw new RelayError(
955
+ "loop_guard",
956
+ `this message would be hop ${origin.hop} of a chain across accounts (limit ${XLIMITS.hopMax}); stop and report to your user`,
957
+ )
958
+ const contact = await link.contactByHandle(target.account)
959
+ if (!contact || contact.dir === "in" || contact.peerHandle !== target.account) throw missing()
960
+ const me = core.xsender(sender)
961
+ if (!me || !allowlisted(contact.allow, me.machine, me.handle, me.depth)) throw missing()
962
+ const address = `${target.account}/${target.machine}/${target.handle || "me"}`
963
+ const replyTo = (send.reply_to as string | undefined) ?? null
964
+ const parent = replyTo ? core.xreceived(me.remote, replyTo) : null
965
+ const answered = parent && parent.from_account === contact.peerId ? (JSON.parse(parent.message) as Envelope) : null
966
+ const granted = answered && answered.from.address === address ? parent : null
967
+ if (!granted && !core.crossAdmission(sender, "out", target).ok) throw missing()
968
+ const now = this.now()
969
+ const thread = granted?.thread ?? cleanText(send.thread, "thread", 64) ?? newId("thr", now)
970
+ const hop = Math.max(origin.hop, granted ? answered!.hop + 1 : 0)
971
+ if (hop > XLIMITS.hopMax)
972
+ throw new RelayError("loop_guard", `this message would be hop ${hop} of a chain across accounts (limit ${XLIMITS.hopMax}); stop and report to your user`)
973
+ this.xcounters.check(contact, me.remote, address, thread)
974
+ const secret = findSecret(send.body)
975
+ if (secret) throw new RelayError("secret_refused", `the message looks like it carries a credential (${secret}); send a reference instead`)
976
+ const ttl = send.ttl_s ?? LIMITS.ttlDefaultS
977
+ if (!Number.isInteger(ttl) || (ttl as number) < 1 || (ttl as number) > LIMITS.ttlMaxS)
978
+ throw new RelayError("invalid_params", `ttl_s must be 1 to ${LIMITS.ttlMaxS}`)
979
+ const self = link.self
980
+ const own = `${self.handle}/${me.machine}/${me.handle || "me"}`
981
+ const human = me.kind === "human" && me.handle === "" && origin.tty
982
+ const from: XEnvelopeFrom = {
983
+ account: self.handle,
984
+ address: human ? `${self.handle}/me` : own,
985
+ human,
986
+ auth: "contact",
987
+ rel: `contact:${self.handle}`,
988
+ machine: me.machine,
989
+ harness: me.harness,
990
+ }
991
+ const id = origin.id ?? newId("msg", now)
992
+ const expiresAt = now + (ttl as number) * 1000
993
+ const declared = declaredLimits(contact.limits)
994
+ const msg: XMessage = {
995
+ id,
996
+ from,
997
+ to: `${target.machine}/${target.handle}`,
998
+ body: send.body,
999
+ kind,
1000
+ thread,
1001
+ reply_to: replyTo,
1002
+ hop,
1003
+ client_id: origin.clientId ?? this.deps.hash(`${own}|${address}|${cleanText(send.client_id, "client_id", 128) ?? newId("cid", now)}`),
1004
+ created_at: now,
1005
+ expires_at: expiresAt,
1006
+ ...(granted?.grant ? { reply_grant: granted.grant, reply_grant_expires_at: granted.expires_at } : {}),
1007
+ grant: replyGrant(this.deps.hash, this.xkey, id, own, contact.peerId, expiresAt),
1008
+ ...(declared ? { limits: declared } : {}),
1009
+ }
1010
+ const verdict: GateVerdict = origin.judged
1011
+ ? { verdict: "pass" }
1012
+ : await link.gate.egress(msg, { side: "egress", account: self.handle, peer: contact.peerHandle, peer_id: contact.peerId, sender })
1013
+ if (verdict.verdict === "refuse") throw gateRefusal(verdict, "egress")
1014
+ if (verdict.verdict === "hold") return { id: verdict.hold_id, thread, hop, deliveries: [], held: { hold_id: verdict.hold_id, side: "egress" } }
1015
+ if (verdict.verdict === "duplicate") return { id: verdict.hold_id, thread, hop, deliveries: [], duplicate: true }
1016
+ if (verdict.verdict === "redact") {
1017
+ const result = await this.xsend(sender, { ...send, body: verdict.body }, { ...origin, judged: true })
1018
+ const redacted = joinRedacted(verdict.redacted, result.redacted)
1019
+ return { ...result, ...(redacted ? { redacted } : {}) }
1020
+ }
1021
+ this.xcounters.check(contact, me.remote, address, thread)
1022
+ const refund = this.xcounters.sent(contact, me.remote, address, thread)
1023
+ let delivered: XDelivered
1024
+ try {
1025
+ delivered = await link.deliver(contact, msg)
1026
+ } catch (error) {
1027
+ if (!(error instanceof RelayError) || error.code === "unavailable") refund()
1028
+ throw error
1029
+ }
1030
+ this.fleet({ type: "message", machine: me.machine, id: delivered.id, thread, kind, from: from.address, to: [address], hop })
1031
+ return {
1032
+ id: delivered.id,
1033
+ thread,
1034
+ hop,
1035
+ deliveries: [{ endpoint: address, address, seq: 0, offline: delivered.state === "queued", last_seen: null }],
1036
+ ...(delivered.duplicate ? { duplicate: true } : {}),
1037
+ ...(delivered.held ? { held: { hold_id: delivered.held.hold_id, side: "ingress" as const } } : {}),
1038
+ ...(delivered.redacted ? { redacted: delivered.redacted } : {}),
1039
+ }
1040
+ }
1041
+
1042
+ /**
1043
+ * A message from the account `fromAccountId`'s relay (the host's internal route, never a client): the contact, the target on this
1044
+ * account's allowlist, its scope (or the reply grant), shape, a retry's earlier answer, the scrubber, the ingress gate, then one xinbox
1045
+ * row pushed to the target's bridge. Anything unresolvable is not_found, as for a missing endpoint.
1046
+ */
1047
+ async xreceive(fromAccountId: string, raw: unknown, judged = false): Promise<XDelivered> {
1048
+ const core = this.relay
1049
+ const link = this.deps.crossAccount
1050
+ const msg = parseXMessage(raw)
1051
+ const missing = () => new RelayError("not_found", `no endpoint ${link?.self.handle ?? this.account}/${msg.to}`)
1052
+ if (!link || !this.xcounters || !this.xkey) throw missing()
1053
+ const contact = await link.contactById(fromAccountId)
1054
+ if (!contact || contact.dir === "out" || contact.peerId !== fromAccountId || msg.from.account !== contact.peerHandle) throw missing()
1055
+ const target = core.xendpoint(msg.to)
1056
+ if (!target || !allowlisted(contact.allow, target.machine, target.handle, target.depth)) throw missing()
1057
+ const address = `${link.self.handle}/${target.machine}/${target.handle || "me"}`
1058
+ const grantExpiry = msg.reply_grant_expires_at
1059
+ const granted =
1060
+ msg.reply_to !== null &&
1061
+ msg.reply_grant !== undefined &&
1062
+ grantExpiry !== undefined &&
1063
+ grantExpiry > this.now() &&
1064
+ sameSecret(msg.reply_grant, replyGrant(this.deps.hash, this.xkey, msg.reply_to, address, fromAccountId, grantExpiry))
1065
+ const parts = msg.from.address.split("/")
1066
+ const peer = {
1067
+ account: contact.peerHandle,
1068
+ machine: parts.length === 3 ? parts[1]! : (msg.from.machine ?? "me"),
1069
+ handle: parts.length === 3 && parts[2] !== "me" ? parts[2]! : "",
1070
+ }
1071
+ if (!granted && !core.crossAdmission(target.id, "in", peer).ok) throw missing()
1072
+ if (!honestSender(msg.from)) throw new RelayError("invalid_params", "the sender is not one its relay may name")
1073
+ const bytes = new TextEncoder().encode(msg.body).length
1074
+ if (bytes > XLIMITS.bodyBytes) throw new RelayError("too_large", `a message from another account is at most ${XLIMITS.bodyBytes} bytes`)
1075
+ if (msg.hop > XLIMITS.hopMax) throw new RelayError("loop_guard", `hop ${msg.hop} is past the limit across accounts (${XLIMITS.hopMax})`)
1076
+ if (msg.expires_at <= this.now()) throw new RelayError("invalid_params", "the message expired")
1077
+ const earlier = core.xduplicate(fromAccountId, msg.client_id)
1078
+ if (earlier) return { id: earlier.id, state: earlier.acked ? "delivered" : "queued", duplicate: true }
1079
+ const envelope: Envelope = {
1080
+ id: msg.id,
1081
+ thread: msg.thread,
1082
+ reply_to: msg.reply_to,
1083
+ kind: msg.kind,
1084
+ from: {
1085
+ address: msg.from.address,
1086
+ endpoint: msg.from.address,
1087
+ user: contact.peerHandle,
1088
+ auth: "contact",
1089
+ ...(msg.from.machine ? { machine: msg.from.machine } : {}),
1090
+ claims: { harness: msg.from.harness ?? null, model: null },
1091
+ rel: `contact:${contact.peerHandle}`,
1092
+ account: contact.peerHandle,
1093
+ ...(msg.from.human ? { human: true as const } : {}),
1094
+ },
1095
+ to: address,
1096
+ body: { text: msg.body, mime: "text/markdown" },
1097
+ data_class: "public",
1098
+ attachments: [],
1099
+ hop: msg.hop,
1100
+ created_at: new Date(msg.created_at).toISOString(),
1101
+ expires_at: new Date(msg.expires_at).toISOString(),
1102
+ }
1103
+ const message = JSON.stringify(envelope)
1104
+ const stored = new TextEncoder().encode(message).length
1105
+ this.xadmit(fromAccountId, contact, msg, target, stored)
1106
+ const secret = findSecret(msg.body)
1107
+ if (secret) throw new RelayError("secret_refused", `the message looks like it carries a credential (${secret})`)
1108
+ const verdict: GateVerdict = judged
1109
+ ? { verdict: "pass" }
1110
+ : await link.gate.ingress(msg, { side: "ingress", account: link.self.handle, peer: contact.peerHandle, peer_id: fromAccountId })
1111
+ if (verdict.verdict === "refuse") throw gateRefusal(verdict, "ingress")
1112
+ if (verdict.verdict === "hold") return { id: msg.id, state: "queued", held: { hold_id: verdict.hold_id } }
1113
+ if (verdict.verdict === "duplicate") return { id: msg.id, state: "delivered", duplicate: true }
1114
+ if (verdict.verdict === "redact") return { ...(await this.xreceive(fromAccountId, { ...msg, body: verdict.body }, true)), redacted: verdict.redacted }
1115
+ const raced = () => {
1116
+ const row = core.xduplicate(fromAccountId, msg.client_id)
1117
+ return row ? { id: row.id, state: row.acked ? ("delivered" as const) : ("queued" as const), duplicate: true } : null
1118
+ }
1119
+ const again = raced()
1120
+ if (again) return again
1121
+ this.xadmit(fromAccountId, contact, msg, target, stored)
1122
+ const seq = core.xinsert({
1123
+ id: msg.id,
1124
+ from_account: fromAccountId,
1125
+ client_id: msg.client_id,
1126
+ endpoint: target.remote,
1127
+ sender: msg.from.address,
1128
+ message,
1129
+ thread: msg.thread,
1130
+ grant: msg.grant ?? null,
1131
+ expires_at: msg.expires_at,
1132
+ at: this.now(),
1133
+ })
1134
+ if (seq === null) return raced()!
1135
+ this.stash(stored)
1136
+ let pushed = false
1137
+ for (const socket of this.deps.sockets()) {
1138
+ if (socket.conn.role !== "bridge" || !socket.conn.subscribed || socket.conn.machine !== target.machine || this.gone.has(socket.conn.id)) continue
1139
+ this.write(socket, { jsonrpc: "2.0", method: "deliver", params: { seq, endpoint: target.remote, message: envelope } })
1140
+ pushed = true
1141
+ }
1142
+ this.wake(target.id)
1143
+ this.fleet({
1144
+ type: "message",
1145
+ machine: msg.from.machine ?? null,
1146
+ id: msg.id,
1147
+ thread: msg.thread,
1148
+ kind: msg.kind,
1149
+ from: msg.from.address,
1150
+ to: [address],
1151
+ hop: msg.hop,
1152
+ })
1153
+ return { id: msg.id, state: pushed ? "delivered" : "queued" }
1154
+ }
1155
+
1156
+ /** this account's endpoints the account `fromAccountId` may reach under its contact's allowlist: address, harness and presence only */
1157
+ async xlist(fromAccountId: string): Promise<XListed[]> {
1158
+ const link = this.deps.crossAccount
1159
+ if (!link) return []
1160
+ const contact = await link.contactById(fromAccountId)
1161
+ if (!contact || contact.dir === "out" || contact.peerId !== fromAccountId) return []
1162
+ return this.relay
1163
+ .xendpoints()
1164
+ .filter((e) => allowlisted(contact.allow, e.machine, e.handle, e.depth))
1165
+ .map((e) => ({ address: `${link.self.handle}/${e.machine}/${e.handle || "me"}`, harness: e.harness, presence: e.presence }))
1166
+ }
1167
+
1168
+ /**
1169
+ * throws when one more message from `fromAccountId` would pass a cap of this side: the pair a minute and an hour and the contact a
1170
+ * day (the lower of this account's limits for the contact and the sender's declared ones), the thread, the account's stored bytes or
1171
+ * the target's unread messages; counted from xinbox itself, so a wake keeps them
1172
+ */
1173
+ private xadmit(fromAccountId: string, contact: XContact, msg: XMessage, target: XEndpoint, bytes: number): void {
1174
+ const core = this.relay
1175
+ const now = this.now()
1176
+ const { perMinute, perHour, perDay } = xlimits(contact.limits, msg.limits)
1177
+ const counts = core.xcounts(fromAccountId, msg.from.address, target.remote, dayStart(now))
1178
+ const retry = (since: number | null, ms: number) => Math.max(1, Math.ceil(((since ?? now) + ms - now) / 1000))
1179
+ if (counts.minute >= perMinute)
1180
+ throw new RelayError("rate_limited", `${perMinute} messages from ${msg.from.address} to ${msg.to} a minute`, retry(counts.minuteAt, 60_000))
1181
+ if (counts.hour >= perHour)
1182
+ throw new RelayError("rate_limited", `${perHour} messages from ${msg.from.address} to ${msg.to} an hour`, retry(counts.hourAt, 3_600_000))
1183
+ if (counts.day >= perDay) throw new RelayError("rate_limited", `${perDay} messages from ${contact.peerHandle} today`, untilTomorrow(now))
1184
+ if (core.xthreadCount(target.remote, msg.thread) >= XLIMITS.threadMax)
1185
+ throw new RelayError("loop_guard", `thread ${msg.thread} reached ${XLIMITS.threadMax} messages across accounts`)
1186
+ const cap = this.limits.storedBytes
1187
+ if (this.storedNow() + bytes > cap) throw new RelayError("inbox_full", `the receiving account's relay holds its cap of ${cap} bytes of messages`)
1188
+ const unread = core.xunread(target)
1189
+ if (unread >= LIMITS.inboxMax) throw new RelayError("inbox_full", `${msg.to} has ${unread} unread messages`)
1190
+ }
1191
+
1192
+ /**
1193
+ * deletes the queued messages of every account that is no longer an active contact allowed to message this one, after the
1194
+ * contacts changed; nothing when the contacts cannot be loaded
1195
+ */
1196
+ async xprune(): Promise<void> {
1197
+ if (!this.deps.crossAccount || !this.core) return
1198
+ const senders = await this.xinbound()
1199
+ if (!senders || !this.core) return
1200
+ this.relay.xsenders(senders)
1201
+ if (this.relay.xprune(senders)) this.stored = null
1202
+ }
1203
+
1204
+ /** the accounts whose messages this account takes now: active contacts with dir in or both; null when they cannot be loaded */
1205
+ private async xinbound(): Promise<Set<string> | null> {
1206
+ const contacts = await this.deps.crossAccount?.contacts()
1207
+ return contacts ? new Set(contacts.filter((contact) => contact.dir !== "out").map((contact) => contact.peerId)) : null
1208
+ }
1209
+
1210
+ /** what every contact this account may message lets it reach */
1211
+ private async contactList(): Promise<XListed[]> {
1212
+ const link = this.deps.crossAccount
1213
+ if (!link) return []
1214
+ const contacts = ((await link.contacts()) ?? []).filter((contact) => contact.dir !== "in")
1215
+ const lists = await Promise.all(
1216
+ contacts.map((contact) =>
1217
+ link.list(contact).then(
1218
+ (list) => ({ contact, list }),
1219
+ () => ({ contact, list: [] as XListed[] }),
1220
+ ),
1221
+ ),
1222
+ )
1223
+ return lists.flatMap(({ contact, list }) =>
1224
+ list
1225
+ .filter((e) => typeof e?.address === "string" && e.address.startsWith(`${contact.peerHandle}/`))
1226
+ .map((e) => ({
1227
+ address: e.address,
1228
+ harness: typeof e.harness === "string" ? e.harness : null,
1229
+ presence: typeof e.presence === "string" ? e.presence : "offline",
1230
+ })),
1231
+ )
1232
+ }
1233
+
1234
+ // push and wait
1235
+
1236
+ private delivered(from: string, result: SendResult, kind: string): void {
1237
+ const core = this.relay
1238
+ for (const delivery of result.deliveries) this.route(delivery.seq, delivery.endpoint)
1239
+ this.fleet({
1240
+ type: "message",
1241
+ machine: core.machineOf(from),
1242
+ id: result.id,
1243
+ thread: result.thread,
1244
+ kind,
1245
+ from: core.addressOf(from),
1246
+ to: result.deliveries.map((d) => d.address),
1247
+ hop: result.hop,
1248
+ })
1249
+ }
1250
+
1251
+ /** the body's bytes when the account can hold them; refuses a send past the `storedBytes` limit */
1252
+ private room(sender: string, body: unknown): number {
1253
+ if (typeof body !== "string") return 0
1254
+ const bytes = new TextEncoder().encode(body).length
1255
+ const cap = this.limits.storedBytes
1256
+ if (this.storedNow() + bytes > cap) {
1257
+ this.relay.record("refuse", sender, null, { code: "too_large" })
1258
+ throw new RelayError(
1259
+ "too_large",
1260
+ `the account relay holds its cap of ${cap} bytes of message bodies; ack or let messages expire, or release a machine with ${CLI_NAME} machines rm`,
1261
+ )
1262
+ }
1263
+ return bytes
1264
+ }
1265
+
1266
+ /** bytes of message bodies held, and of the messages from other accounts */
1267
+ private storedNow(): number {
1268
+ const xinbox = this.deps.crossAccount ? ` + (SELECT COALESCE(SUM(LENGTH(CAST(message AS BLOB))), 0) FROM xinbox)` : ""
1269
+ this.stored ??= one<{ n: number }>(this.deps.sql, `SELECT (SELECT COALESCE(SUM(LENGTH(CAST(body AS BLOB))), 0) FROM messages)${xinbox} AS n`)!.n
1270
+ return this.stored
1271
+ }
1272
+
1273
+ private stash(bytes: number): void {
1274
+ if (this.stored !== null) this.stored += bytes
1275
+ }
1276
+
1277
+ /** wakes direct waiters and pushes a delivery for a mirrored row to the subscribed bridge of its machine */
1278
+ private route(seq: number, endpoint: string): void {
1279
+ this.wake(endpoint)
1280
+ const delivery = this.relay.delivery(seq)
1281
+ if (!delivery?.endpoint || !delivery.mirrored) return
1282
+ const machine = delivery.endpoint.slice(0, delivery.endpoint.indexOf("/"))
1283
+ for (const socket of this.deps.sockets()) {
1284
+ if (socket.conn.role === "bridge" && socket.conn.subscribed && socket.conn.machine === machine) {
1285
+ this.write(socket, { jsonrpc: "2.0", method: "deliver", params: { seq, endpoint: delivery.endpoint, message: delivery.message } })
1286
+ }
1287
+ }
1288
+ }
1289
+
1290
+ /** a machine's endpoint changes: `endpoints` to the other bridges, presence and status to watchers */
1291
+ private announce(machine: string, changes: MirrorChange[], full: boolean): void {
1292
+ if (full || changes.length) {
1293
+ const endpoints = full ? this.relay.remoteRowsOf(machine) : changes.map((c) => c.view)
1294
+ this.toBridges(machine, { machine, endpoints, full })
1295
+ }
1296
+ for (const change of changes) {
1297
+ const view = change.view
1298
+ if (change.added || change.presence)
1299
+ this.fleet({ type: "presence", machine, endpoint: view.address, presence: view.presence, harness: view.harness, labels: view.labels })
1300
+ if (change.status && !change.added) this.fleet({ type: "status", machine, endpoint: view.address, status: view.status })
1301
+ }
1302
+ }
1303
+
1304
+ /** a direct client's endpoint changed: the other bridges mirror it */
1305
+ private announceEndpoint(endpoint: string): void {
1306
+ const machine = this.relay.machineOf(endpoint)
1307
+ if (!machine) return
1308
+ const view = this.relay.remoteRowsOf(machine).find((v) => v.id === endpoint)
1309
+ if (view) this.toBridges(machine, { machine, endpoints: [view], full: false })
1310
+ }
1311
+
1312
+ private toBridges(machine: string, params: unknown): void {
1313
+ for (const socket of this.deps.sockets()) {
1314
+ if (socket.conn.role === "bridge" && socket.conn.machine !== machine) this.write(socket, { jsonrpc: "2.0", method: "endpoints", params })
1315
+ }
1316
+ }
1317
+
1318
+ private fleet(event: Record<string, unknown>): void {
1319
+ const at = new Date(this.now()).toISOString()
1320
+ for (const socket of this.deps.sockets()) if (socket.conn.watching) this.write(socket, { jsonrpc: "2.0", method: "fleet", params: { at, ...event } })
1321
+ }
1322
+
1323
+ private write(socket: HubSocket, value: unknown): void {
1324
+ if (this.gone.has(socket.conn.id)) return
1325
+ try {
1326
+ socket.send(JSON.stringify(value))
1327
+ } catch {
1328
+ this.deps.log?.("relay_send_failed", { account: this.account, machine: socket.conn.machine })
1329
+ }
1330
+ }
1331
+
1332
+ private save(socket: HubSocket): void {
1333
+ if (this.transient.has(socket)) return
1334
+ this.deps.save(socket)
1335
+ }
1336
+
1337
+ private waitMs(socket: HubSocket, value: unknown): number {
1338
+ if (value === undefined) return 0
1339
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new RelayError("invalid_params", "wait_s must be a number of seconds")
1340
+ return Math.min(value, this.transient.has(socket) ? this.limits.httpWaitS : LIMITS.waitMaxS) * 1000
1341
+ }
1342
+
1343
+ private wake(endpoint: string): void {
1344
+ const set = this.waiters.get(endpoint)
1345
+ if (!set) return
1346
+ this.waiters.delete(endpoint)
1347
+ for (const fn of set) fn()
1348
+ }
1349
+
1350
+ private waitFor(endpoint: string, ms: number, socket: HubSocket): Promise<void> {
1351
+ return new Promise((resolve) => {
1352
+ const done = () => {
1353
+ clearTimeout(timer)
1354
+ this.waiters.get(endpoint)?.delete(done)
1355
+ resolve()
1356
+ }
1357
+ const timer = setTimeout(done, ms)
1358
+ let set = this.waiters.get(endpoint)
1359
+ if (!set) {
1360
+ set = new Set()
1361
+ this.waiters.set(endpoint, set)
1362
+ }
1363
+ set.add(done)
1364
+ if (this.gone.has(socket.conn.id)) done()
1365
+ })
1366
+ }
1367
+
1368
+ private async awaitReply(socket: HubSocket, me: string, request: string, ms: number, sent?: object): Promise<unknown> {
1369
+ const core = this.relay
1370
+ const deadline = Date.now() + ms
1371
+ for (;;) {
1372
+ const reply = core.replyTo(me, request)
1373
+ if (reply) {
1374
+ core.ack(me, [reply.seq])
1375
+ return { ...sent, id: request, reply }
1376
+ }
1377
+ if (core.requestExpired(request)) return { ...sent, id: request, expired: true }
1378
+ if (this.gone.has(socket.conn.id) || Date.now() >= deadline) return { ...sent, id: request, pending: true }
1379
+ await this.waitFor(me, deadline - Date.now(), socket)
1380
+ }
1381
+ }
1382
+
1383
+ // connections and machines
1384
+
1385
+ private kick(socket: HubSocket, reason: string, code = 1008): void {
1386
+ this.write(socket, { jsonrpc: "2.0", method: "revoked", params: { reason } })
1387
+ socket.close(code, reason)
1388
+ this.release(socket, false)
1389
+ }
1390
+
1391
+ private release(socket: HubSocket, replaced: boolean): void {
1392
+ const conn = socket.conn
1393
+ if (this.gone.has(conn.id)) return
1394
+ this.gone.add(conn.id)
1395
+ if (conn.role === "new" || !this.core) return
1396
+ const core = this.core
1397
+ const bound = this.boundMachine(conn)
1398
+ if (conn.role === "client" && conn.endpoint) {
1399
+ const endpoint = conn.endpoint
1400
+ core.disconnect(endpoint)
1401
+ const machine = core.machineOf(endpoint)
1402
+ if (machine && one<{ kind: string }>(this.deps.sql, `SELECT kind FROM machines WHERE name = ?`, machine)?.kind === "bridge") {
1403
+ for (const delivery of core.adoptClaimed(machine, endpoint)) this.route(delivery.seq, delivery.endpoint)
1404
+ }
1405
+ this.wake(endpoint)
1406
+ const view = core.whoami(endpoint)
1407
+ this.fleet({ type: "presence", machine: conn.machine, endpoint: view.address, presence: view.presence })
1408
+ this.announceEndpoint(endpoint)
1409
+ } else if (conn.role === "bridge" && conn.machine) {
1410
+ const newer = this.deps
1411
+ .sockets()
1412
+ .some((s) => s.conn.id !== conn.id && s.conn.role === "bridge" && s.conn.machine === conn.machine && !this.gone.has(s.conn.id))
1413
+ if (!replaced && !newer) this.announce(conn.machine, core.markOffline(conn.machine), false)
1414
+ }
1415
+ if (bound) this.deps.sql.all(`UPDATE machines SET last_seen = ? WHERE name = ?`, this.now(), bound)
1416
+ conn.role = "new"
1417
+ conn.endpoint = null
1418
+ conn.subscribed = false
1419
+ conn.watching = false
1420
+ if (!this.transient.has(socket)) {
1421
+ try {
1422
+ this.deps.save(socket)
1423
+ } catch {}
1424
+ }
1425
+ }
1426
+
1427
+ /** the machine a bridge or agent client connection holds; a human's or observer's machine is only a namespace */
1428
+ private boundMachine(conn: HubConn): string | null {
1429
+ if (!conn.machine || !this.core) return null
1430
+ if (conn.role === "bridge") return conn.machine
1431
+ return conn.role === "client" && conn.endpoint !== null && this.core.machineOf(conn.endpoint) === conn.machine ? conn.machine : null
1432
+ }
1433
+
1434
+ /** machines a live connection holds */
1435
+ private liveMachines(): Set<string> {
1436
+ const live = new Set<string>()
1437
+ for (const socket of [...this.deps.sockets(), ...this.transient]) {
1438
+ const machine = this.gone.has(socket.conn.id) ? null : this.boundMachine(socket.conn)
1439
+ if (machine) live.add(machine)
1440
+ }
1441
+ return live
1442
+ }
1443
+
1444
+ /** bindings, then names a revoke left behind (kind `revoked`) */
1445
+ private machineList(): { name: string; kind: string; online: boolean; bridge: boolean; created_at: string | null; last_seen: string | null }[] {
1446
+ const sockets = this.deps.sockets().filter((s) => !this.gone.has(s.conn.id))
1447
+ const bound = this.deps.sql
1448
+ .all<{ name: string; kind: string; created_at: number; last_seen: number | null }>(`SELECT name, kind, created_at, last_seen FROM machines ORDER BY name`)
1449
+ .map((row) => ({
1450
+ name: row.name,
1451
+ kind: row.kind,
1452
+ online: sockets.some((s) => this.boundMachine(s.conn) === row.name),
1453
+ bridge: sockets.some((s) => s.conn.role === "bridge" && s.conn.machine === row.name),
1454
+ created_at: iso(row.created_at),
1455
+ last_seen: iso(row.last_seen),
1456
+ }))
1457
+ const revoked = this.deps.sql
1458
+ .all<{ name: string; at: number }>(`SELECT name, at FROM machine_tombstones WHERE name NOT IN (SELECT name FROM machines) ORDER BY name`)
1459
+ .map((row) => ({ name: row.name, kind: "revoked", online: false, bridge: false, created_at: null, last_seen: iso(row.at) }))
1460
+ return [...bound, ...revoked]
1461
+ }
1462
+
1463
+ /** only the token the machine is bound to, a full login of the member who bound it, or an admin may release it (or a revoked token's name) */
1464
+ private releaseMachine(caller: HubSocket, name: unknown): unknown {
1465
+ const core = this.relay
1466
+ if (typeof name !== "string" || !MACHINE_NAME.test(name)) throw new RelayError("invalid_params", "name is a machine name")
1467
+ const bound = one<{ token_id: string; actor_id: string | null }>(this.deps.sql, `SELECT token_id, actor_id FROM machines WHERE name = ?`, name)
1468
+ const tomb = bound ? undefined : one<{ actor_id: string | null }>(this.deps.sql, `SELECT actor_id FROM machine_tombstones WHERE name = ?`, name)
1469
+ if (!bound && !tomb) throw new RelayError("not_found", `no machine ${name}`)
1470
+ const grant = caller.conn.grant
1471
+ if (!(bound?.token_id === grant.tokenId || fullOf(grant, (bound ?? tomb)!.actor_id) || accountAdmin(grant))) {
1472
+ core.record("refuse", null, name, { code: "unauthorized", machine: name })
1473
+ throw new RelayError(
1474
+ "unauthorized",
1475
+ `only the token machine ${name} is bound to, a full login of the member who bound it, or an org owner or admin can release it`,
1476
+ )
1477
+ }
1478
+ for (const socket of [...this.deps.sockets()]) {
1479
+ if (socket.conn.id !== caller.conn.id && this.boundMachine(socket.conn) === name) this.kick(socket, `machine ${name} was released`)
1480
+ }
1481
+ const dropped = this.tx(() => {
1482
+ const count = core.dropMachine(name, { queue: true })
1483
+ this.deps.sql.all(`DELETE FROM machines WHERE name = ?`, name)
1484
+ this.deps.sql.all(`DELETE FROM machine_tombstones WHERE name = ?`, name)
1485
+ core.record("machine_release", null, name, { machine: name, endpoints: count })
1486
+ return count
1487
+ })
1488
+ this.stored = null
1489
+ this.toBridges(name, { machine: name, endpoints: [], full: true })
1490
+ this.deps.log?.("relay_action", { account: this.account, machine: name, method: "machine_release", code: "ok" })
1491
+ return { removed: name, endpoints: dropped }
1492
+ }
1493
+
1494
+ /** releases client bindings with no live connection, idle for the `clientMachineIdleS` limit, with nothing queued and no unanswered ask open; bridge bindings never expire */
1495
+ private expireMachines(now: number): void {
1496
+ const core = this.relay
1497
+ const live = this.liveMachines()
1498
+ const idle = this.deps.sql.all<{ name: string }>(
1499
+ `SELECT name FROM machines WHERE kind = 'client' AND COALESCE(last_seen, created_at) <= ?`,
1500
+ now - this.limits.clientMachineIdleS * 1000,
1501
+ )
1502
+ for (const { name } of idle) {
1503
+ if (live.has(name)) continue
1504
+ const queued = one(
1505
+ this.deps.sql,
1506
+ `SELECT 1 FROM deliveries d JOIN endpoints e ON e.id = d.endpoint WHERE substr(e.remote, 1, ?) = ? AND d.acked_at IS NULL AND d.expires_at > ? LIMIT 1`,
1507
+ name.length + 1,
1508
+ `${name}/`,
1509
+ now,
1510
+ )
1511
+ const asking = one(
1512
+ this.deps.sql,
1513
+ `SELECT 1 FROM messages m JOIN endpoints e ON e.id = m.from_ep WHERE substr(e.remote, 1, ?) = ? AND m.kind = 'request' AND m.expires_at > ?
1514
+ AND NOT EXISTS (SELECT 1 FROM messages r WHERE r.reply_to = m.id AND r.kind = 'reply') LIMIT 1`,
1515
+ name.length + 1,
1516
+ `${name}/`,
1517
+ now,
1518
+ )
1519
+ if (queued || asking) continue
1520
+ const dropped = this.tx(() => {
1521
+ const count = core.dropMachine(name)
1522
+ this.deps.sql.all(`DELETE FROM machines WHERE name = ?`, name)
1523
+ core.record("machine_expire", null, name, { machine: name, endpoints: count })
1524
+ return count
1525
+ })
1526
+ this.toBridges(name, { machine: name, endpoints: [], full: true })
1527
+ this.deps.log?.("relay_action", { account: this.account, machine: name, method: "machine_expire", code: "ok", endpoints: dropped })
1528
+ }
1529
+ }
1530
+
1531
+ /** drops the tombstones past their age with nothing queued for their machine, and retires the rows and xinbox of each name still unbound */
1532
+ private forgetTombstones(now: number): void {
1533
+ const forgotten = this.tx(() => {
1534
+ const names = this.deps.sql.all<{ name: string }>(
1535
+ `DELETE FROM machine_tombstones WHERE at <= ? AND NOT EXISTS (SELECT 1 FROM deliveries d JOIN endpoints e ON e.id = d.endpoint
1536
+ WHERE substr(e.remote, 1, length(machine_tombstones.name) + 1) = machine_tombstones.name || '/' AND d.acked_at IS NULL AND d.expires_at > ?)
1537
+ RETURNING name`,
1538
+ now - (LIMITS.ttlMaxS + LIMITS.purgeAfterS) * 1000,
1539
+ now,
1540
+ )
1541
+ const unbound = names.map((row) => row.name).filter((name) => !one(this.deps.sql, `SELECT 1 FROM machines WHERE name = ?`, name))
1542
+ for (const name of unbound) this.relay.record("machine_forget", null, name, { machine: name, endpoints: this.relay.dropMachine(name, { queue: true }) })
1543
+ return unbound
1544
+ })
1545
+ for (const name of forgotten) this.toBridges(name, { machine: name, endpoints: [], full: true })
1546
+ }
1547
+
1548
+ /** the hub_state keys (`denied:<tokenId>`) of tokens revoked lately, to when; loaded once per object lifetime */
1549
+ private deniedGrants(): Map<string, number> {
1550
+ this.denied ??= new Map(
1551
+ this.deps.sql.all<{ key: string; value: number }>(`SELECT key, value FROM hub_state WHERE key LIKE ?`, `${DENIED}%`).map((row) => [row.key, row.value]),
1552
+ )
1553
+ return this.denied
1554
+ }
1555
+
1556
+ /** a token revoked lately is refused even when the host accepted its grant before the revoke reached the object */
1557
+ private refuseDenied(socket: HubSocket): void {
1558
+ const at = this.deniedGrants().get(DENIED + socket.conn.grant.tokenId)
1559
+ if (at === undefined || at <= this.now() - this.limits.deniedS * 1000) return
1560
+ this.kick(socket, "the token was revoked")
1561
+ throw new RelayError("unauthorized", "the token was revoked")
1562
+ }
1563
+
1564
+ private tx<T>(fn: () => T): T {
1565
+ return this.deps.transaction ? this.deps.transaction(fn) : fn()
1566
+ }
1567
+
1568
+ private state(key: string): number | null {
1569
+ return one<{ value: number }>(this.deps.sql, `SELECT value FROM hub_state WHERE key = ?`, key)?.value ?? null
1570
+ }
1571
+
1572
+ private setState(key: string, value: number): void {
1573
+ this.deps.sql.all(`INSERT OR REPLACE INTO hub_state (key, value) VALUES (?, ?)`, key, value)
1574
+ }
1575
+
1576
+ /**
1577
+ * a held cross-account message its account's human released (the host claimed the hold first): the rest of that side's path, with
1578
+ * every check run again and only the gate skipped. Egress sends as the stored sender with the time left, under the hold id its sender
1579
+ * got, and with the message's own client id, so the peer answers both a retried release and the sender's retry of the original as the
1580
+ * duplicate. Ingress runs xreceive again
1581
+ */
1582
+ async xrelease(hold: XRelease): Promise<SendResult | XDelivered> {
1583
+ if (hold.side === "ingress") return this.xreceive(hold.from_account, hold.message, true)
1584
+ const msg = hold.message
1585
+ const left = Math.ceil((msg.expires_at - this.now()) / 1000)
1586
+ if (left < 1) throw new RelayError("invalid_params", "the held message expired")
1587
+ const send: Params = {
1588
+ to: `${hold.peer}/${msg.to.endsWith("/") ? `${msg.to}me` : msg.to}`,
1589
+ body: msg.body,
1590
+ kind: msg.kind,
1591
+ thread: msg.thread,
1592
+ ...(msg.reply_to ? { reply_to: msg.reply_to } : {}),
1593
+ ttl_s: Math.min(left, LIMITS.ttlMaxS),
1594
+ }
1595
+ return this.xsend(hold.sender, send, { tty: msg.from.human, hop: msg.hop, judged: true, clientId: msg.client_id, id: hold.hold_id })
1596
+ }
1597
+
1598
+ /** a status message from the relay to one of this account's endpoints (a held message's sender), pushed to its bridge */
1599
+ xnotice(endpoint: string, text: string): void {
1600
+ this.route(this.relay.judgeNoticeTo(endpoint, text), endpoint)
1601
+ }
1602
+ }