@scp-dev/hub 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/hub.ts ADDED
@@ -0,0 +1,1619 @@
1
+ import {
2
+ type ActivityRow,
3
+ allowlisted,
4
+ CAPABILITIES,
5
+ CLI_NAME,
6
+ type Claim,
7
+ type CrossAccountLink,
8
+ cleanText,
9
+ dayStart,
10
+ declaredLimits,
11
+ type EndpointKind,
12
+ type EndpointView,
13
+ type Envelope,
14
+ findSecret,
15
+ type GateVerdict,
16
+ gateRefusal,
17
+ HUMAN_HANDLE,
18
+ honestSender,
19
+ joinRedacted,
20
+ judgeDetailForAgents,
21
+ LIMITS,
22
+ MESSAGE_KINDS,
23
+ type MessageKind,
24
+ newId,
25
+ one,
26
+ PREAMBLE,
27
+ type Presence,
28
+ PROTOCOL,
29
+ parseXMessage,
30
+ RelayError,
31
+ randomHex,
32
+ replyGrant,
33
+ type SendParams,
34
+ type SendResult,
35
+ SERVER_FIELDS,
36
+ type Sql,
37
+ SYSTEM_HANDLE,
38
+ sameSecret,
39
+ untilTomorrow,
40
+ type XContact,
41
+ XCounters,
42
+ type XDelivered,
43
+ type XEnvelopeFrom,
44
+ type XListed,
45
+ XLIMITS,
46
+ type XMessage,
47
+ type XRelease,
48
+ xlimits,
49
+ } from "@scp-dev/core"
50
+ import { FederatedCore, MACHINE_NAME, type MirrorChange, type XEndpoint } from "./federation"
51
+
52
+ export type HubAuth = "account" | "account+bridge"
53
+
54
+ export type HubRole = "owner" | "admin" | "member"
55
+
56
+ /** what the host proved about the token, plus its privilege */
57
+ export interface HubGrant {
58
+ tokenId: string
59
+ userId: string
60
+ actorId: string | null
61
+ expiresAt: number
62
+ /** a scoped token, not a full login; a grant that doesn't say false counts as scoped */
63
+ scoped: boolean
64
+ /** the actor's org role; null on a personal account */
65
+ role: HubRole | null
66
+ /** labels set on every endpoint the token claims or mirrors, over the client's own for the same keys */
67
+ labels?: Record<string, string>
68
+ }
69
+
70
+ /** per-socket state; JSON-serialisable, so the host can persist it with the socket */
71
+ export interface HubConn {
72
+ id: string
73
+ grant: HubGrant
74
+ role: "new" | "client" | "bridge" | "observer"
75
+ /** the machine a bridge or agent client is bound to; a human's or observer's namespace for unqualified handles */
76
+ machine: string | null
77
+ /** the hub endpoint id a client connection acts as */
78
+ endpoint: string | null
79
+ watching: boolean
80
+ subscribed: boolean
81
+ /** whether a human client said a terminal is attached */
82
+ tty?: boolean
83
+ }
84
+
85
+ /** a bound machine, or a revoked token's name kept as a tombstone */
86
+ export interface MachineRow {
87
+ name: string
88
+ kind: string
89
+ online: boolean
90
+ bridge: boolean
91
+ created_at: string | null
92
+ last_seen: string | null
93
+ }
94
+
95
+ export interface HubSocket {
96
+ conn: HubConn
97
+ send(text: string): void
98
+ close(code: number, reason: string): void
99
+ }
100
+
101
+ export interface HubDeps {
102
+ sql: Sql
103
+ /** synchronous sha256 hex */
104
+ hash: (value: string) => string
105
+ /** runs fn in one synchronous storage transaction */
106
+ transaction?: <T>(fn: () => T) => T
107
+ now?: () => number
108
+ /** every live socket of this account (rebuilt from persisted state when the host restarts) */
109
+ sockets(): HubSocket[]
110
+ /** persist socket.conn after the hub changed it */
111
+ save(socket: HubSocket): void
112
+ /** structured log line, e.g. relay_action; never bodies or tokens */
113
+ log?: (event: string, fields: Record<string, unknown>) => void
114
+ /** overrides of {@link HUB_LIMITS}; `Infinity` disables a cap, count or byte limit, durations stay finite */
115
+ limits?: Partial<HubLimits>
116
+ /** the link to other accounts' relays; without it an address of another account is not found */
117
+ crossAccount?: CrossAccountLink
118
+ }
119
+
120
+ export const HUB_LIMITS = {
121
+ machines: 16,
122
+ endpoints: 512,
123
+ sockets: 64,
124
+ frameBytes: 262_144,
125
+ callsPerMinute: 120,
126
+ bridgeCallsPerMinute: 600,
127
+ accountCallsPerMinute: 1200,
128
+ acksPerMinute: 1200,
129
+ accountAcksPerMinute: 4800,
130
+ storedBytes: 256 * 1024 * 1024,
131
+ clientMachineIdleS: 14 * 86_400,
132
+ idlePurgeS: 30 * 86_400,
133
+ httpWaitS: 25,
134
+ revalidateMs: 300_000,
135
+ deniedS: 3600,
136
+ refusalsPerMinute: 10,
137
+ } as const
138
+
139
+ export type HubLimits = { [K in keyof typeof HUB_LIMITS]: number }
140
+
141
+ const DURATIONS = ["clientMachineIdleS", "idlePurgeS", "httpWaitS", "revalidateMs", "deniedS"] as const satisfies readonly (keyof HubLimits)[]
142
+
143
+ const SWEEP_MS = 10 * 60_000
144
+ const DAY_MS = 86_400_000
145
+ const MIN_ALARM_MS = 60_000
146
+ const REVALIDATE_RETRY_MS = 30_000
147
+ const REVALIDATE_ERROR = "revalidate_error:"
148
+ const DENIED = "denied:"
149
+ const TOUCH_MS = 10_000
150
+
151
+ const HUB_MIGRATION = [
152
+ `CREATE TABLE IF NOT EXISTS machines (name TEXT PRIMARY KEY, token_id TEXT NOT NULL, kind TEXT NOT NULL, created_at INTEGER NOT NULL, last_seen INTEGER)`,
153
+ `CREATE TABLE IF NOT EXISTS hub_state (key TEXT PRIMARY KEY, value INTEGER NOT NULL)`,
154
+ `CREATE TABLE IF NOT EXISTS machine_tombstones (name TEXT PRIMARY KEY, actor_id TEXT, user_id TEXT, at INTEGER NOT NULL)`,
155
+ ]
156
+
157
+ const OBSERVER_METHODS = new Set(["list", "snapshot", "watch", "log", "machines", "machines_rm"])
158
+ const CLIENT_METHODS = new Set(["whoami", "send", "ask", "ask_wait", "inbox", "ack", "status", "event", ...OBSERVER_METHODS])
159
+
160
+ type Request = { jsonrpc?: string; id?: string | number | null; method?: unknown; params?: unknown }
161
+ type Params = Record<string, unknown>
162
+
163
+ const iso = (ms: number | null): string | null => (ms === null ? null : new Date(ms).toISOString())
164
+
165
+ function rpcError(error: unknown): { code: number; message: string; data?: { code: string; retry_after?: number } } {
166
+ if (!(error instanceof RelayError)) return { code: -32603, message: "internal error", data: { code: "unavailable" } }
167
+ const code = error.code === "method_not_found" ? -32601 : error.code === "invalid_params" ? -32602 : -32000
168
+ return { code, message: error.message, data: { code: error.code, ...(error.retryAfterS ? { retry_after: error.retryAfterS } : {}) } }
169
+ }
170
+
171
+ /** a frame's place in a full mirror sent in several frames (`part` of `parts` under one `id`), or null for a mirror of one frame */
172
+ function mirrorSync(value: unknown): { id: string; part: number; parts: number } | null {
173
+ if (value === undefined || value === null) return null
174
+ const { id, part, parts } = value as { id?: unknown; part?: unknown; parts?: unknown }
175
+ if (
176
+ typeof id !== "string" ||
177
+ !id ||
178
+ id.length > 64 ||
179
+ !Number.isInteger(parts) ||
180
+ (parts as number) < 1 ||
181
+ (parts as number) > 4096 ||
182
+ !Number.isInteger(part) ||
183
+ (part as number) < 1 ||
184
+ (part as number) > (parts as number)
185
+ ) {
186
+ throw new RelayError("invalid_params", "sync is { id, part, parts } of a full mirror in several frames")
187
+ }
188
+ return { id, part: part as number, parts: parts as number }
189
+ }
190
+
191
+ function serverFields(params: Params): void {
192
+ for (const field of SERVER_FIELDS) if (field in params) throw new RelayError("invalid_params", `${field} is set by the relay, not the sender`)
193
+ }
194
+
195
+ function object(value: unknown, what: string): Params {
196
+ if (value === undefined) return {}
197
+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new RelayError("invalid_params", `${what} must be an object`)
198
+ return value as Params
199
+ }
200
+
201
+ /** a full login: a grant that doesn't say it is unscoped counts as scoped */
202
+ const fullLogin = (grant: HubGrant): boolean => grant.scoped === false
203
+
204
+ /** a full login of the member `actorId` (null: the personal account's owner) */
205
+ const fullOf = (grant: HubGrant, actorId: string | null): boolean => fullLogin(grant) && grant.actorId === actorId
206
+
207
+ /** a full login of an org owner or admin, or of a personal account (its owner) */
208
+ const accountAdmin = (grant: HubGrant): boolean =>
209
+ fullLogin(grant) && (grant.role === "owner" || grant.role === "admin" || ((grant.role ?? null) === null && grant.actorId === null))
210
+
211
+ /** the grant's labels over `labels`; a value that is not a label object is left for the core to refuse */
212
+ function grantLabels(grant: HubGrant, labels: unknown): unknown {
213
+ if (!grant.labels || !Object.keys(grant.labels).length) return labels
214
+ if (labels !== undefined && labels !== null && (typeof labels !== "object" || Array.isArray(labels))) return labels
215
+ return { ...(labels as Record<string, unknown> | null | undefined), ...grant.labels }
216
+ }
217
+
218
+ /** each mirrored view with the grant's labels over its own */
219
+ function grantViews(grant: HubGrant, views: unknown): unknown {
220
+ if (!grant.labels || !Array.isArray(views)) return views
221
+ return views.map((view: unknown) =>
222
+ view && typeof view === "object" && !Array.isArray(view) ? { ...view, labels: grantLabels(grant, (view as Params).labels) } : view,
223
+ )
224
+ }
225
+
226
+ function listFilters(params: Params): { all?: boolean; harness?: string | undefined; kind?: EndpointKind | undefined; presence?: Presence | undefined } {
227
+ return {
228
+ all: params.all === true,
229
+ harness: typeof params.harness === "string" ? params.harness : undefined,
230
+ kind: params.kind === "agent" || params.kind === "human" ? params.kind : undefined,
231
+ presence: params.presence === "online" || params.presence === "idle" || params.presence === "offline" ? params.presence : undefined,
232
+ }
233
+ }
234
+
235
+ /** the hosted relay of one account: machines, bridges, direct clients, push and replay over a FederatedCore; pure, sockets are injected */
236
+ export class RelayHub {
237
+ private core: FederatedCore | null = null
238
+ private account = ""
239
+ private readonly now: () => number
240
+ private readonly waiters = new Map<string, Set<() => void>>()
241
+ private readonly buckets = new Map<string, { rate: number; tokens: number; at: number; refused: number }>()
242
+ private readonly transient = new Set<HubSocket>()
243
+ private readonly gone = new Set<string>()
244
+ /** per machine, the full mirror its bridge is sending in several frames; lost on eviction, which only skips that sync's offline sweep */
245
+ private readonly syncs = new Map<string, { id: string; parts: number; next: number; seen: Set<string> }>()
246
+ private stored: number | null = null
247
+ private denied: Map<string, number> | null = null
248
+ private xcounters: XCounters | null = null
249
+ private xkey: string | null = null
250
+ private readonly limits: Readonly<HubLimits>
251
+
252
+ constructor(private readonly deps: HubDeps) {
253
+ this.now = deps.now ?? Date.now
254
+ this.limits = { ...HUB_LIMITS, ...deps.limits }
255
+ for (const key of DURATIONS) if (!Number.isFinite(this.limits[key])) throw new RangeError(`hub limit ${key} must be finite`)
256
+ }
257
+
258
+ /** idempotent: migrates and records the account handle (the address user part, e.g. "ada"); call before handling each request */
259
+ init(account: { user: string }): void {
260
+ if (this.core && this.account === account.user) return
261
+ for (const statement of HUB_MIGRATION) this.deps.sql.all(statement)
262
+ const columns = this.deps.sql.all<{ name: string }>(`PRAGMA table_info(machines)`).map((c) => c.name)
263
+ if (!columns.includes("user_id")) this.deps.sql.all(`ALTER TABLE machines ADD COLUMN user_id TEXT`)
264
+ if (!columns.includes("actor_id")) this.deps.sql.all(`ALTER TABLE machines ADD COLUMN actor_id TEXT`)
265
+ const core = new FederatedCore(this.deps.sql, {
266
+ user: account.user,
267
+ now: this.now,
268
+ hash: this.deps.hash,
269
+ ...(this.deps.transaction ? { transaction: this.deps.transaction } : {}),
270
+ auth: "account",
271
+ endpointCap: this.limits.endpoints,
272
+ refusalsPerMinute: this.limits.refusalsPerMinute,
273
+ capEverySender: true,
274
+ xinbox: this.deps.crossAccount !== undefined,
275
+ })
276
+ core.migrate()
277
+ this.core = core
278
+ this.account = account.user
279
+ if (this.deps.crossAccount) {
280
+ this.xkey = core.xstate("key", () => randomHex(32))
281
+ this.xcounters = new XCounters(this.deps.sql, this.now)
282
+ }
283
+ }
284
+
285
+ /** forgets the core and in-memory state after the object's storage was deleted; the next init migrates afresh */
286
+ reset(): void {
287
+ this.core = null
288
+ this.account = ""
289
+ this.stored = null
290
+ this.denied = null
291
+ this.xcounters = null
292
+ this.xkey = null
293
+ this.buckets.clear()
294
+ this.gone.clear()
295
+ }
296
+
297
+ /** a fresh connection state for an accepted socket */
298
+ static connection(grant: HubGrant): HubConn {
299
+ return { id: newId("conn"), grant, role: "new", machine: null, endpoint: null, watching: false, subscribed: false }
300
+ }
301
+
302
+ /** a socket would exceed the `sockets` limit: the host refuses it (429) instead of accepting */
303
+ full(): boolean {
304
+ return this.deps.sockets().length >= this.limits.sockets
305
+ }
306
+
307
+ /** one JSON-RPC text frame from a socket; replies and pushes go out through socket.send */
308
+ async message(socket: HubSocket, text: string): Promise<void> {
309
+ if (new TextEncoder().encode(text).length > this.limits.frameBytes) {
310
+ this.write(socket, { jsonrpc: "2.0", id: null, error: rpcError(new RelayError("too_large", `a frame is at most ${this.limits.frameBytes} bytes`)) })
311
+ socket.close(1009, "frame too large")
312
+ this.closed(socket)
313
+ return
314
+ }
315
+ let request: unknown
316
+ try {
317
+ request = JSON.parse(text)
318
+ } catch {
319
+ this.write(socket, { jsonrpc: "2.0", id: null, error: { code: -32700, message: "parse error" } })
320
+ return
321
+ }
322
+ if (!request || typeof request !== "object" || Array.isArray(request)) {
323
+ this.write(socket, { jsonrpc: "2.0", id: null, error: { code: -32600, message: "one JSON-RPC request per frame" } })
324
+ return
325
+ }
326
+ const response = await this.handle(socket, request as Request)
327
+ if (response) this.write(socket, response)
328
+ }
329
+
330
+ /** the socket closed (any reason) */
331
+ closed(socket: HubSocket): void {
332
+ this.release(socket, false)
333
+ }
334
+
335
+ /** POST /v1/relay/rpc: a JSON-RPC request or batch run in order on a short-lived connection; returns the response(s) */
336
+ async batch(grant: HubGrant, body: unknown): Promise<unknown> {
337
+ if (JSON.stringify(body ?? null).length > this.limits.frameBytes) {
338
+ return { jsonrpc: "2.0", id: null, error: rpcError(new RelayError("too_large", `a request is at most ${this.limits.frameBytes} bytes`)) }
339
+ }
340
+ const requests = Array.isArray(body) ? body : [body]
341
+ if (!requests.length) return { jsonrpc: "2.0", id: null, error: { code: -32600, message: "an empty batch" } }
342
+ const socket: HubSocket = { conn: RelayHub.connection(grant), send: () => {}, close: () => {} }
343
+ this.transient.add(socket)
344
+ const responses: unknown[] = []
345
+ try {
346
+ for (const request of requests) {
347
+ if (!request || typeof request !== "object" || Array.isArray(request)) {
348
+ responses.push({ jsonrpc: "2.0", id: null, error: { code: -32600, message: "a request is an object" } })
349
+ continue
350
+ }
351
+ const response = await this.handle(socket, request as Request)
352
+ if (response) responses.push(response)
353
+ }
354
+ } finally {
355
+ this.closed(socket)
356
+ this.transient.delete(socket)
357
+ }
358
+ return Array.isArray(body) ? responses : (responses[0] ?? null)
359
+ }
360
+
361
+ /**
362
+ * closes every socket (and in-flight /rpc call) whose grant matches with 1008, audits it, and unbinds the machines
363
+ * the matching grants bound behind a tombstone of their member, keeping their endpoint rows and queued messages;
364
+ * returns how many closed
365
+ */
366
+ revoke(match: { tokenId?: string; userId?: string; actorId?: string }): number {
367
+ const keys = (Object.entries(match) as [keyof HubGrant, string | undefined][]).filter(([, value]) => value !== undefined)
368
+ if (!keys.length) return 0
369
+ let closed = 0
370
+ const tokens = new Set(match.tokenId !== undefined ? [match.tokenId] : [])
371
+ for (const socket of [...this.deps.sockets(), ...this.transient]) {
372
+ if (this.gone.has(socket.conn.id) || !keys.every(([key, value]) => socket.conn.grant[key] === value)) continue
373
+ tokens.add(socket.conn.grant.tokenId)
374
+ this.kick(socket, "the token was revoked")
375
+ closed++
376
+ }
377
+ if (closed) this.relay.record("revoke", null, null, { sockets: closed })
378
+ const columns = match.tokenId !== undefined ? [["token_id", match.tokenId]] : keys.map(([key, value]) => [key === "userId" ? "user_id" : "actor_id", value])
379
+ const where = columns.map(([column]) => `${column} = ?`).join(" AND ")
380
+ const now = this.now()
381
+ for (const row of this.deps.sql.all<{ name: string; token_id: string; actor_id: string | null; user_id: string | null }>(
382
+ `DELETE FROM machines WHERE ${where} RETURNING name, token_id, actor_id, user_id`,
383
+ ...columns.map(([, value]) => value!),
384
+ )) {
385
+ tokens.add(row.token_id)
386
+ this.deps.sql.all(`INSERT OR REPLACE INTO machine_tombstones (name, actor_id, user_id, at) VALUES (?, ?, ?, ?)`, row.name, row.actor_id, row.user_id, now)
387
+ this.relay.record("machine_unbind", null, row.name, { machine: row.name })
388
+ }
389
+ const denied = this.deniedGrants()
390
+ for (const token of tokens) {
391
+ denied.set(DENIED + token, now)
392
+ this.setState(DENIED + token, now)
393
+ }
394
+ this.deps.log?.("relay_action", { account: this.account, method: "revoke", sockets: closed })
395
+ return closed
396
+ }
397
+
398
+ /**
399
+ * the host's alarm: revalidates each socket's grant and sweeps; returns the next alarm time in ms, or null. A failed
400
+ * check closes with 1008, an expired grant with 4001 (reconnect for a new one). A check that throws keeps the
401
+ * sockets and retries in 30 s; once a token's checks kept throwing for the `revalidateMs` limit its sockets close
402
+ * with the retryable 1013.
403
+ */
404
+ async alarm(validate: (grant: HubGrant) => Promise<boolean>): Promise<number | null> {
405
+ const now = this.now()
406
+ this.xcounters?.trueUp()
407
+ const checks = new Map<string, Promise<boolean | null>>()
408
+ const erroring = new Set<string>()
409
+ for (const socket of [...this.deps.sockets()]) {
410
+ const grant = socket.conn.grant
411
+ if (grant.expiresAt <= now) {
412
+ this.kick(socket, "the token grant expired, reconnect", 4001)
413
+ continue
414
+ }
415
+ const key = JSON.stringify([grant.tokenId, grant.userId, grant.actorId, grant.expiresAt, grant.scoped, grant.role])
416
+ let check = checks.get(key)
417
+ if (!check) {
418
+ check = validate(grant).catch((error: unknown) => {
419
+ this.deps.log?.("relay_revalidate_error", { account: this.account, error: error instanceof Error ? error.name : "error" })
420
+ return null
421
+ })
422
+ checks.set(key, check)
423
+ }
424
+ const valid = await check
425
+ if (valid === false) {
426
+ this.kick(socket, "the token is no longer valid")
427
+ } else if (valid === null) {
428
+ erroring.add(grant.tokenId)
429
+ const key = REVALIDATE_ERROR + grant.tokenId
430
+ let since = this.state(key)
431
+ if (since === null) {
432
+ since = now
433
+ this.setState(key, since)
434
+ }
435
+ if (now - since >= this.limits.revalidateMs) this.kick(socket, "the token could not be revalidated, try again", 1013)
436
+ }
437
+ }
438
+ for (const { key } of this.deps.sql.all<{ key: string }>(`SELECT key FROM hub_state WHERE key LIKE ?`, `${REVALIDATE_ERROR}%`)) {
439
+ const tokenId = key.slice(REVALIDATE_ERROR.length)
440
+ const lingering = this.deps.sockets().some((s) => s.conn.grant.tokenId === tokenId && !this.gone.has(s.conn.id))
441
+ if (!erroring.has(tokenId) || !lingering) this.deps.sql.all(`DELETE FROM hub_state WHERE key = ?`, key)
442
+ }
443
+ this.setState("last_validated", now)
444
+ if (now - (this.state("last_sweep") ?? 0) >= SWEEP_MS) {
445
+ this.relay.sweep()
446
+ this.expireMachines(now)
447
+ this.forgetTombstones(now)
448
+ const denied = this.deniedGrants()
449
+ for (const [key, at] of denied) {
450
+ if (at > now - this.limits.deniedS * 1000) continue
451
+ denied.delete(key)
452
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = ?`, key)
453
+ }
454
+ this.stored = null
455
+ this.gone.clear()
456
+ for (const [key, bucket] of this.buckets) if (bucket.tokens + ((now - bucket.at) * bucket.rate) / 60_000 >= bucket.rate) this.buckets.delete(key)
457
+ this.xcounters?.prune()
458
+ this.setState("last_sweep", now)
459
+ }
460
+ if (this.idle(now)) {
461
+ if (this.state("idle_since") === null) this.setState("idle_since", now)
462
+ } else {
463
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = 'idle_since'`)
464
+ }
465
+ return this.nextAlarm()
466
+ }
467
+
468
+ /** when the next alarm should fire (earliest grant expiry, revalidation or its retry after an error, sweep, retention), or null when the hub holds nothing */
469
+ nextAlarm(): number | null {
470
+ const now = this.now()
471
+ const trueUp = this.xcounters?.dueAt() ?? null
472
+ const held = this.retentionAlarm(now)
473
+ const retention = held === null || trueUp === null ? (held ?? trueUp) : Math.min(held, trueUp)
474
+ const sockets = this.deps.sockets()
475
+ if (!sockets.length) return retention
476
+ let validated = this.state("last_validated")
477
+ if (validated === null) {
478
+ validated = now
479
+ this.setState("last_validated", validated)
480
+ }
481
+ let swept = this.state("last_sweep")
482
+ if (swept === null) {
483
+ swept = now
484
+ this.setState("last_sweep", swept)
485
+ }
486
+ const retrying = one(this.deps.sql, `SELECT 1 FROM hub_state WHERE key LIKE ? LIMIT 1`, `${REVALIDATE_ERROR}%`) !== undefined
487
+ return Math.min(
488
+ validated + (retrying ? REVALIDATE_RETRY_MS : this.limits.revalidateMs),
489
+ swept + SWEEP_MS,
490
+ retention ?? Number.POSITIVE_INFINITY,
491
+ ...sockets.map((s) => s.conn.grant.expiresAt),
492
+ )
493
+ }
494
+
495
+ /** the account held no machines, tombstones, sockets or queued messages for the `idlePurgeS` limit: the object may delete its storage */
496
+ purgeDue(): boolean {
497
+ const since = this.state("idle_since")
498
+ return since !== null && this.now() - since >= this.limits.idlePurgeS * 1000 && this.idle(this.now())
499
+ }
500
+
501
+ /** while the hub holds rows: the earliest unacked delivery's expiry, at most a day and at least a minute away */
502
+ private retentionAlarm(now: number): number | null {
503
+ const sql = this.deps.sql
504
+ const held = one<{ held: number }>(
505
+ sql,
506
+ `SELECT EXISTS (SELECT 1 FROM machines) OR EXISTS (SELECT 1 FROM machine_tombstones) OR EXISTS (SELECT 1 FROM endpoints WHERE handle NOT IN (?, ?))
507
+ OR EXISTS (SELECT 1 FROM messages) OR EXISTS (SELECT 1 FROM deliveries) OR EXISTS (SELECT 1 FROM audit) AS held`,
508
+ HUMAN_HANDLE,
509
+ SYSTEM_HANDLE,
510
+ )!.held
511
+ if (!held) return null
512
+ const expiry = one<{ at: number | null }>(sql, `SELECT MIN(expires_at) AS at FROM deliveries WHERE acked_at IS NULL`)!.at
513
+ return Math.max(now + MIN_ALARM_MS, Math.min(expiry ?? Number.POSITIVE_INFINITY, now + DAY_MS))
514
+ }
515
+
516
+ /** no machine bound or tombstoned, no live socket and nothing queued */
517
+ private idle(now: number): boolean {
518
+ if (this.deps.sockets().some((s) => !this.gone.has(s.conn.id))) return false
519
+ return !one<{ busy: number }>(
520
+ this.deps.sql,
521
+ `SELECT EXISTS (SELECT 1 FROM machines) OR EXISTS (SELECT 1 FROM machine_tombstones) OR EXISTS (SELECT 1 FROM deliveries WHERE acked_at IS NULL AND expires_at > ?) AS busy`,
522
+ now,
523
+ )!.busy
524
+ }
525
+
526
+ // dispatch
527
+
528
+ private get relay(): FederatedCore {
529
+ if (!this.core) throw new Error("RelayHub.init was not called")
530
+ return this.core
531
+ }
532
+
533
+ private async handle(socket: HubSocket, request: Request): Promise<object | null> {
534
+ const id = request.id ?? null
535
+ const method = String(request.method)
536
+ let code = "ok"
537
+ try {
538
+ this.refuseDenied(socket)
539
+ this.spend(socket, method, request.params)
540
+ const result = await this.dispatch(socket, method, object(request.params, "params"))
541
+ return request.id === undefined ? null : { jsonrpc: "2.0", id, result }
542
+ } catch (error) {
543
+ code = error instanceof RelayError ? error.code : "internal"
544
+ if (code === "internal") this.deps.log?.("relay_internal_error", { account: this.account, method, error: error instanceof Error ? error.name : "error" })
545
+ return request.id === undefined ? null : { jsonrpc: "2.0", id, error: rpcError(error) }
546
+ } finally {
547
+ this.deps.log?.("relay_action", { account: this.account, machine: socket.conn.machine, method, code })
548
+ }
549
+ }
550
+
551
+ /**
552
+ * one call from the token's bucket for the connection's role (a bridge, or a bridge hello, has its own, larger one)
553
+ * and from the account's; acks (one token per seq) and rejects spend from separate ack buckets of the token and the
554
+ * account instead, so they never starve calls, and an activity event is refused once either call bucket is below a
555
+ * quarter, so sends keep the rest
556
+ */
557
+ private spend(socket: HubSocket, method: string, params: unknown): void {
558
+ const conn = socket.conn
559
+ const now = this.now()
560
+ if ((method === "ack" && (conn.role === "client" || conn.role === "bridge")) || (method === "reject" && conn.role === "bridge")) {
561
+ const seqs = (params as Params | null)?.seqs
562
+ const cost = method === "ack" && Array.isArray(seqs) ? Math.min(Math.max(seqs.length, 1), LIMITS.inboxPage) : 1
563
+ const token = this.bucket(`${conn.grant.tokenId}:ack`, this.limits.acksPerMinute, now)
564
+ const account = this.bucket("account:ack", this.limits.accountAcksPerMinute, now)
565
+ for (const [b, scope] of [
566
+ [token, "a token"],
567
+ [account, "an account"],
568
+ ] as const) {
569
+ if (b.tokens < cost)
570
+ throw new RelayError("rate_limited", `at most ${b.rate} acked seqs per minute on ${scope}`, Math.max(1, Math.ceil(((cost - b.tokens) * 60) / b.rate)))
571
+ }
572
+ token.tokens -= cost
573
+ account.tokens -= cost
574
+ return
575
+ }
576
+ const bridge = conn.role === "bridge" || (conn.role === "new" && method === "hello" && (params as Params | null)?.bridge === true)
577
+ const rate = bridge ? this.limits.bridgeCallsPerMinute : this.limits.callsPerMinute
578
+ const bucket = this.bucket(`${conn.grant.tokenId}:${bridge ? "bridge" : "client"}`, rate, now)
579
+ const account = this.bucket("account", this.limits.accountCallsPerMinute, now)
580
+ if (method === "event") {
581
+ for (const b of [bucket, account]) {
582
+ if (b.tokens < b.rate / 4)
583
+ throw new RelayError(
584
+ "rate_limited",
585
+ "activity events wait while the token or the account is near its call limit",
586
+ Math.max(1, Math.ceil(((b.rate / 4 - b.tokens) * 60) / b.rate)),
587
+ )
588
+ }
589
+ }
590
+ if (bucket.tokens < 1) {
591
+ if (++bucket.refused >= rate) {
592
+ socket.close(1013, "rate limited")
593
+ this.closed(socket)
594
+ }
595
+ throw new RelayError(
596
+ "rate_limited",
597
+ `at most ${rate} ${bridge ? "bridge " : ""}calls per minute on a token`,
598
+ Math.max(1, Math.ceil(((1 - bucket.tokens) * 60) / rate)),
599
+ )
600
+ }
601
+ if (account.tokens < 1) {
602
+ throw new RelayError(
603
+ "rate_limited",
604
+ `at most ${this.limits.accountCallsPerMinute} calls per minute on an account`,
605
+ Math.max(1, Math.ceil(((1 - account.tokens) * 60) / account.rate)),
606
+ )
607
+ }
608
+ bucket.tokens -= 1
609
+ bucket.refused = 0
610
+ account.tokens -= 1
611
+ }
612
+
613
+ private bucket(key: string, rate: number, now: number): { rate: number; tokens: number; at: number; refused: number } {
614
+ if (rate === Number.POSITIVE_INFINITY) return { rate, tokens: rate, at: now, refused: 0 }
615
+ const bucket = this.buckets.get(key) ?? { rate, tokens: rate, at: now, refused: 0 }
616
+ bucket.tokens = Math.min(rate, bucket.tokens + ((now - bucket.at) * rate) / 60_000)
617
+ bucket.at = now
618
+ this.buckets.set(key, bucket)
619
+ return bucket
620
+ }
621
+
622
+ private async dispatch(socket: HubSocket, method: string, params: Params): Promise<unknown> {
623
+ if (this.gone.has(socket.conn.id)) throw new RelayError("unauthorized", "this connection is closed")
624
+ if (method === "hello") return this.hello(socket, params)
625
+ if (socket.conn.role === "new") throw new RelayError("unauthorized", "call hello first")
626
+ if (socket.conn.role === "bridge") return this.bridgeMethod(socket, method, params)
627
+ if (socket.conn.role === "observer" && !OBSERVER_METHODS.has(method)) {
628
+ if (CLIENT_METHODS.has(method)) throw new RelayError("unauthorized", "this connection only observes: say hello as an agent or the human")
629
+ throw new RelayError("method_not_found", `no method ${method}`)
630
+ }
631
+ return this.clientMethod(socket, method, params)
632
+ }
633
+
634
+ private hello(socket: HubSocket, params: Params): unknown {
635
+ const core = this.relay
636
+ const conn = socket.conn
637
+ if (conn.role !== "new") throw new RelayError("invalid_params", "hello was already called on this connection")
638
+ if (params.protocol !== PROTOCOL) throw new RelayError("invalid_params", `this relay speaks ${PROTOCOL}`)
639
+ if (params.transient !== undefined)
640
+ throw new RelayError("invalid_params", "the hub takes no hook connections: hooks report to the local broker, which bridges them")
641
+ serverFields(params)
642
+ const observe = params.observe === true
643
+ const named = params.machine
644
+ if ((!observe || named !== undefined) && (typeof named !== "string" || !MACHINE_NAME.test(named) || named === "me" || named === this.account)) {
645
+ throw new RelayError("invalid_params", "machine is a name of lowercase letters, digits and -, other than me and the account handle")
646
+ }
647
+ const bridge = params.bridge === true
648
+ const human = params.human === true
649
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = 'idle_since'`)
650
+ const info = { account: this.account, capabilities: CAPABILITIES, server_time: new Date(this.now()).toISOString(), limits: { ...LIMITS, ...this.limits } }
651
+ if (observe) {
652
+ if (bridge || human || params.claim !== undefined || params.resume !== undefined)
653
+ throw new RelayError("invalid_params", "an observer hello has no bridge, human, claim or resume")
654
+ conn.role = "observer"
655
+ conn.machine = typeof named === "string" ? named : null
656
+ this.save(socket)
657
+ return { ...info, ...(conn.machine ? { machine: conn.machine } : {}) }
658
+ }
659
+ const machine = named as string
660
+ const base = { ...info, machine }
661
+ if (bridge && this.transient.has(socket)) throw new RelayError("invalid_params", "a bridge connects over the WebSocket")
662
+ if (human && typeof params.tty !== "boolean")
663
+ throw new RelayError("invalid_params", "a human hello says whether a terminal is attached (tty: true or false)")
664
+
665
+ if (human) {
666
+ if (!fullLogin(conn.grant)) throw new RelayError("unauthorized", "a scoped token acts as an agent: use --as")
667
+ const endpoint = core.claimHuman(params.tty as boolean)
668
+ conn.role = "client"
669
+ conn.machine = machine
670
+ conn.endpoint = endpoint.id
671
+ conn.tty = params.tty as boolean
672
+ this.save(socket)
673
+ this.fleet({ type: "presence", machine, endpoint: endpoint.address, presence: endpoint.presence, harness: endpoint.harness, labels: endpoint.labels })
674
+ return { ...base, endpoint, preamble: PREAMBLE }
675
+ }
676
+
677
+ const claimed = object(params.claim, "claim")
678
+ const claim = (conn.grant.labels ? { ...claimed, labels: grantLabels(conn.grant, claimed.labels) } : claimed) as Claim
679
+ const held = this.bindable(machine, conn.grant)
680
+
681
+ if (bridge) {
682
+ this.bind(machine, conn.grant, "bridge", held)
683
+ core.adoptClaimed(machine)
684
+ conn.role = "bridge"
685
+ conn.machine = machine
686
+ this.save(socket)
687
+ for (const other of [...this.deps.sockets()]) {
688
+ if (other.conn.id === conn.id || other.conn.role !== "bridge" || other.conn.machine !== machine) continue
689
+ other.close(4000, "replaced by a newer bridge for this machine")
690
+ this.release(other, true)
691
+ }
692
+ core.record("bridge", null, machine, { machine })
693
+ return { ...base, endpoints: core.remoteViews({ except: machine }) }
694
+ }
695
+
696
+ if (one<{ kind: string }>(this.deps.sql, `SELECT kind FROM machines WHERE name = ?`, machine)?.kind === "bridge") {
697
+ throw new RelayError(
698
+ "invalid_params",
699
+ `machine ${machine} is linked through its broker's bridge; run the command there without --cloud, or pick another --machine`,
700
+ )
701
+ }
702
+ let endpoint: EndpointView
703
+ let token: string | undefined
704
+ if (typeof params.resume === "string")
705
+ endpoint = core.resumeOn(machine, params.resume, conn.grant.labels ? (own) => grantLabels(conn.grant, own) : undefined)
706
+ else ({ endpoint, token } = core.claimOn(machine, claim, "account"))
707
+ this.bind(machine, conn.grant, "client", held)
708
+ conn.role = "client"
709
+ conn.machine = machine
710
+ conn.endpoint = endpoint.id
711
+ this.save(socket)
712
+ this.fleet({ type: "presence", machine, endpoint: endpoint.address, presence: endpoint.presence, harness: endpoint.harness, labels: endpoint.labels })
713
+ this.announceEndpoint(endpoint.id)
714
+ return { ...base, endpoint, ...(token ? { token } : {}), preamble: PREAMBLE }
715
+ }
716
+
717
+ /** refuses a machine bound to another token, a revoked token's machine to anyone but its member or an admin, or one past the machine cap; returns whether this token already holds it */
718
+ private bindable(machine: string, grant: HubGrant): boolean {
719
+ const bound = one<{ token_id: string }>(this.deps.sql, `SELECT token_id FROM machines WHERE name = ?`, machine)
720
+ if (bound && bound.token_id !== grant.tokenId) {
721
+ this.relay.record("refuse", null, machine, { code: "unauthorized", machine })
722
+ throw new RelayError("unauthorized", `machine ${machine} is bound to another token; release it with ${CLI_NAME} machines rm ${machine}`)
723
+ }
724
+ if (bound) return true
725
+ const tomb = one<{ actor_id: string | null }>(this.deps.sql, `SELECT actor_id FROM machine_tombstones WHERE name = ?`, machine)
726
+ if (tomb && !fullOf(grant, tomb.actor_id) && !accountAdmin(grant)) {
727
+ this.relay.record("refuse", null, machine, { code: "unauthorized", machine })
728
+ throw new RelayError(
729
+ "unauthorized",
730
+ `machine ${machine} was unbound when its token was revoked; a full login of the same member (or an org owner or admin) can take it back, or ${CLI_NAME} machines rm ${machine} drops its queued messages`,
731
+ )
732
+ }
733
+ const count = one<{ n: number }>(this.deps.sql, `SELECT COUNT(*) AS n FROM machines`)!.n
734
+ if (count >= this.limits.machines)
735
+ throw new RelayError("too_large", `at most ${this.limits.machines} machines per account; release one with ${CLI_NAME} machines rm`)
736
+ return false
737
+ }
738
+
739
+ /** binds the machine to the token once its hello succeeded; `held` is what bindable() returned */
740
+ private bind(machine: string, grant: HubGrant, kind: "bridge" | "client", held: boolean): void {
741
+ const core = this.relay
742
+ const now = this.now()
743
+ if (held) {
744
+ this.deps.sql.all(
745
+ `UPDATE machines SET last_seen = ?, user_id = ?, actor_id = ?, kind = CASE WHEN kind = 'bridge' THEN 'bridge' ELSE ? END WHERE name = ?`,
746
+ now,
747
+ grant.userId,
748
+ grant.actorId,
749
+ kind,
750
+ machine,
751
+ )
752
+ return
753
+ }
754
+ this.deps.sql.all(
755
+ `INSERT INTO machines (name, token_id, user_id, actor_id, kind, created_at, last_seen) VALUES (?, ?, ?, ?, ?, ?, ?)`,
756
+ machine,
757
+ grant.tokenId,
758
+ grant.userId,
759
+ grant.actorId,
760
+ kind,
761
+ now,
762
+ now,
763
+ )
764
+ this.deps.sql.all(`DELETE FROM machine_tombstones WHERE name = ?`, machine)
765
+ this.deps.sql.all(`DELETE FROM hub_state WHERE key = 'idle_since'`)
766
+ core.record("machine_bind", null, machine, { machine, kind })
767
+ this.deps.log?.("relay_action", { account: this.account, machine, method: "machine_bind", code: "ok" })
768
+ }
769
+
770
+ private async clientMethod(socket: HubSocket, method: string, params: Params): Promise<unknown> {
771
+ const core = this.relay
772
+ const conn = socket.conn
773
+ const me = conn.endpoint
774
+ if (me) core.touch(me, TOUCH_MS)
775
+ switch (method) {
776
+ case "list":
777
+ if (params.contacts === true) return { contacts: await this.contactList() }
778
+ return { endpoints: core.list(listFilters(params)) }
779
+ case "snapshot":
780
+ return core.snapshot({ all: params.all === true, activity: typeof params.activity === "number" ? params.activity : undefined })
781
+ case "watch":
782
+ conn.watching = true
783
+ this.save(socket)
784
+ return core.snapshot({ activity: 20 })
785
+ case "log":
786
+ // no hosted connection is a verified human at a terminal: judge rows lose their reason and kinds (the dashboard shows them)
787
+ return {
788
+ rows: core
789
+ .log({ since: params.since as number | undefined, limit: params.limit as number | undefined })
790
+ .map((row) => (row.event.startsWith("judge_") ? { ...row, detail: judgeDetailForAgents(row.detail) } : row)),
791
+ }
792
+ case "machines":
793
+ return { machines: this.machineList() }
794
+ case "machines_rm":
795
+ return this.releaseMachine(socket, params.name)
796
+ }
797
+ if (!me) throw new RelayError("unauthorized", "this connection has no endpoint")
798
+ switch (method) {
799
+ case "whoami":
800
+ return core.whoami(me)
801
+ case "send":
802
+ case "ask": {
803
+ serverFields(params)
804
+ const send = { ...(params as unknown as SendParams) }
805
+ delete (send as Params).wait_s
806
+ if (method === "ask") send.kind = "request"
807
+ const bytes = this.room(me, send.body)
808
+ const result = core.within(conn.machine, () => core.send(me, send, conn.tty === undefined ? {} : { tty: conn.tty }))
809
+ if (!result.duplicate) this.stash(bytes)
810
+ if (!result.duplicate) this.delivered(me, result, send.kind ?? "message")
811
+ if (method === "send") return result
812
+ return this.awaitReply(socket, me, result.id, this.waitMs(socket, params.wait_s), result)
813
+ }
814
+ case "ask_wait": {
815
+ if (typeof params.id !== "string") throw new RelayError("invalid_params", "id is the request's message id")
816
+ return this.awaitReply(socket, me, params.id, this.waitMs(socket, params.wait_s))
817
+ }
818
+ case "inbox": {
819
+ const query = { after: params.after as number | undefined, limit: params.limit as number | undefined }
820
+ const deadline = Date.now() + this.waitMs(socket, params.wait_s)
821
+ let messages = core.inbox(me, query)
822
+ while (!messages.length && !this.gone.has(conn.id) && Date.now() < deadline) {
823
+ await this.waitFor(me, deadline - Date.now(), socket)
824
+ messages = core.inbox(me, query)
825
+ }
826
+ return { messages, next: messages.length ? messages[messages.length - 1]!.seq : (query.after ?? 0) }
827
+ }
828
+ case "ack":
829
+ return { acked: core.ack(me, params.seqs) }
830
+ case "status": {
831
+ core.status(me, params.state, params.note)
832
+ const view = core.whoami(me)
833
+ this.fleet({ type: "status", machine: conn.machine, endpoint: view.address, status: view.status })
834
+ this.announceEndpoint(me)
835
+ return view
836
+ }
837
+ case "event": {
838
+ const row = core.event(me, params)
839
+ this.fleet({ type: "activity", machine: conn.machine, ...row })
840
+ return row
841
+ }
842
+ }
843
+ throw new RelayError("method_not_found", `no method ${method}`)
844
+ }
845
+
846
+ private async bridgeMethod(socket: HubSocket, method: string, params: Params): Promise<unknown> {
847
+ const core = this.relay
848
+ const conn = socket.conn
849
+ const machine = conn.machine!
850
+ switch (method) {
851
+ case "mirror": {
852
+ const sync = mirrorSync(params.sync)
853
+ if (!sync) {
854
+ const full = params.full === true
855
+ const changes = core.mirror(machine, grantViews(conn.grant, params.endpoints), { full, auth: "account+bridge" })
856
+ this.announce(machine, changes, full)
857
+ return { changed: changes.length }
858
+ }
859
+ let state = this.syncs.get(machine)
860
+ if (sync.part === 1) {
861
+ state = { id: sync.id, parts: sync.parts, next: 1, seen: new Set() }
862
+ this.syncs.set(machine, state)
863
+ }
864
+ const intact = state !== undefined && state.id === sync.id && state.parts === sync.parts && state.next === sync.part
865
+ if (!intact) this.syncs.delete(machine)
866
+ const closing = intact && sync.part === sync.parts
867
+ const changes = core.mirror(machine, grantViews(conn.grant, params.endpoints), {
868
+ full: closing,
869
+ auth: "account+bridge",
870
+ ...(intact ? { seen: state!.seen } : {}),
871
+ })
872
+ if (intact) state!.next++
873
+ if (closing) this.syncs.delete(machine)
874
+ this.announce(machine, changes, closing)
875
+ return { changed: changes.length }
876
+ }
877
+ case "forward": {
878
+ const sender = core.remoteEndpoint(machine, params.from)
879
+ const send = { ...object(params.params, "params") }
880
+ serverFields(send)
881
+ const asked = params.hop_floor ?? 0
882
+ if (!Number.isInteger(asked) || (asked as number) < 0 || (asked as number) > 1000) throw new RelayError("invalid_params", "hop_floor is a hop count")
883
+ if ([send.to].flat().some((to) => core.foreignAccount(to) !== null)) return this.xsend(sender, send, { tty: params.tty === true, hop: asked as number })
884
+ delete send.attachments
885
+ const floor = Math.min(asked as number, LIMITS.hopMax + 1)
886
+ if (floor > 0) core.markReceived(sender, floor - 1)
887
+ const bytes = this.room(sender, send.body)
888
+ const result = core.within(machine, () => core.send(sender, send as unknown as SendParams, { tty: params.tty === true }))
889
+ if (!result.duplicate) this.stash(bytes)
890
+ if (!result.duplicate) this.delivered(sender, result, (send.kind as string | undefined) ?? "message")
891
+ return result
892
+ }
893
+ case "subscribe": {
894
+ const after = params.after ?? 0
895
+ if (!Number.isInteger(after) || (after as number) < 0) throw new RelayError("invalid_params", "after is a seq")
896
+ if (this.deps.crossAccount) core.xsenders(await this.xinbound())
897
+ conn.subscribed = true
898
+ this.save(socket)
899
+ let cursor = 0
900
+ let replayed = 0
901
+ for (;;) {
902
+ const page = core.pending(machine, Math.max(cursor, after as number), 200, cursor)
903
+ for (const delivery of page)
904
+ this.write(socket, { jsonrpc: "2.0", method: "deliver", params: { seq: delivery.seq, endpoint: delivery.endpoint, message: delivery.message } })
905
+ replayed += page.length
906
+ if (page.length) cursor = page[page.length - 1]!.seq
907
+ if (page.length < 200) break
908
+ }
909
+ return { replayed, next: Math.max(cursor, after as number) }
910
+ }
911
+ case "ack":
912
+ return { acked: core.ackOn(machine, params.seqs) }
913
+ case "reject": {
914
+ const { notice } = core.rejectOn(machine, params.seq, params.code)
915
+ if (notice) this.route(notice.seq, notice.endpoint)
916
+ return { rejected: true }
917
+ }
918
+ case "event": {
919
+ const sender = core.remoteEndpoint(machine, params.from)
920
+ const row = core.event(sender, params)
921
+ this.fleet({ type: "activity", machine, ...row })
922
+ return row
923
+ }
924
+ case "list":
925
+ if (params.contacts === true) return { contacts: await this.contactList() }
926
+ return { endpoints: core.list(listFilters(params)) }
927
+ case "watch":
928
+ conn.watching = true
929
+ this.save(socket)
930
+ return core.snapshot({ activity: 20 })
931
+ }
932
+ throw new RelayError("method_not_found", `no method ${method} on a bridge`)
933
+ }
934
+
935
+ // other accounts
936
+
937
+ /**
938
+ * A bridge's forward to another account's endpoint, writing no row here: shape, the contact, the sender on this account's allowlist,
939
+ * its scope (or the reply grant of the message it answers), the caps, the scrubber, the egress gate, then the peer relay. Anything
940
+ * unresolvable is not_found, as for a missing endpoint.
941
+ */
942
+ private async xsend(
943
+ sender: string,
944
+ send: Params,
945
+ origin: { tty: boolean; hop: number; judged?: boolean; clientId?: string; id?: string },
946
+ ): Promise<SendResult> {
947
+ const core = this.relay
948
+ const link = this.deps.crossAccount
949
+ const targets = [send.to].flat()
950
+ const missing = () => new RelayError("not_found", `no endpoint ${targets.map(String).join(", ")}`)
951
+ if (targets.length !== 1) throw new RelayError("invalid_params", "a message to another account goes to exactly one endpoint: no lists, groups or tree:")
952
+ const target = core.foreignAccount(targets[0])
953
+ if (!target || !link || !this.xcounters || !this.xkey) throw missing()
954
+ if (typeof send.body !== "string") throw new RelayError("invalid_params", "body must be a string")
955
+ const bytes = new TextEncoder().encode(send.body).length
956
+ if (bytes > XLIMITS.bodyBytes) throw new RelayError("too_large", `a message to another account is at most ${XLIMITS.bodyBytes} bytes; this one is ${bytes}`)
957
+ if (send.attachments !== undefined && !(Array.isArray(send.attachments) && !send.attachments.length))
958
+ throw new RelayError("data_class_refused", "attachments stay in this account")
959
+ if (send.data_class === "private") throw new RelayError("data_class_refused", "a private message stays in this account")
960
+ if (send.data_class !== undefined && send.data_class !== "public") throw new RelayError("invalid_params", "data_class must be public or private")
961
+ const kind = (send.kind ?? "message") as MessageKind
962
+ if (!MESSAGE_KINDS.includes(kind)) throw new RelayError("invalid_params", `kind must be one of ${MESSAGE_KINDS.join(", ")}`)
963
+ if (send.reply_to !== undefined && typeof send.reply_to !== "string") throw new RelayError("invalid_params", "reply_to is a message id")
964
+ if (origin.hop > XLIMITS.hopMax)
965
+ throw new RelayError(
966
+ "loop_guard",
967
+ `this message would be hop ${origin.hop} of a chain across accounts (limit ${XLIMITS.hopMax}); stop and report to your user`,
968
+ )
969
+ const contact = await link.contactByHandle(target.account)
970
+ if (!contact || contact.dir === "in" || contact.peerHandle !== target.account) throw missing()
971
+ const me = core.xsender(sender)
972
+ if (!me || !allowlisted(contact.allow, me.machine, me.handle, me.depth)) throw missing()
973
+ const address = `${target.account}/${target.machine}/${target.handle || "me"}`
974
+ const replyTo = (send.reply_to as string | undefined) ?? null
975
+ const parent = replyTo ? core.xreceived(me.remote, replyTo) : null
976
+ const answered = parent && parent.from_account === contact.peerId ? (JSON.parse(parent.message) as Envelope) : null
977
+ const granted = answered && answered.from.address === address ? parent : null
978
+ if (!granted && !core.crossAdmission(sender, "out", target).ok) throw missing()
979
+ const now = this.now()
980
+ const thread = granted?.thread ?? cleanText(send.thread, "thread", 64) ?? newId("thr", now)
981
+ const hop = Math.max(origin.hop, granted ? answered!.hop + 1 : 0)
982
+ if (hop > XLIMITS.hopMax)
983
+ throw new RelayError("loop_guard", `this message would be hop ${hop} of a chain across accounts (limit ${XLIMITS.hopMax}); stop and report to your user`)
984
+ this.xcounters.check(contact, me.remote, address, thread)
985
+ const secret = findSecret(send.body)
986
+ if (secret) throw new RelayError("secret_refused", `the message looks like it carries a credential (${secret}); send a reference instead`)
987
+ const ttl = send.ttl_s ?? LIMITS.ttlDefaultS
988
+ if (!Number.isInteger(ttl) || (ttl as number) < 1 || (ttl as number) > LIMITS.ttlMaxS)
989
+ throw new RelayError("invalid_params", `ttl_s must be 1 to ${LIMITS.ttlMaxS}`)
990
+ const self = link.self
991
+ const own = `${self.handle}/${me.machine}/${me.handle || "me"}`
992
+ const human = me.kind === "human" && me.handle === "" && origin.tty
993
+ const from: XEnvelopeFrom = {
994
+ account: self.handle,
995
+ address: human ? `${self.handle}/me` : own,
996
+ human,
997
+ auth: "contact",
998
+ rel: `contact:${self.handle}`,
999
+ machine: me.machine,
1000
+ harness: me.harness,
1001
+ }
1002
+ const id = origin.id ?? newId("msg", now)
1003
+ const expiresAt = now + (ttl as number) * 1000
1004
+ const declared = declaredLimits(contact.limits)
1005
+ const msg: XMessage = {
1006
+ id,
1007
+ from,
1008
+ to: `${target.machine}/${target.handle}`,
1009
+ body: send.body,
1010
+ kind,
1011
+ thread,
1012
+ reply_to: replyTo,
1013
+ hop,
1014
+ client_id: origin.clientId ?? this.deps.hash(`${own}|${address}|${cleanText(send.client_id, "client_id", 128) ?? newId("cid", now)}`),
1015
+ created_at: now,
1016
+ expires_at: expiresAt,
1017
+ ...(granted?.grant ? { reply_grant: granted.grant, reply_grant_expires_at: granted.expires_at } : {}),
1018
+ grant: replyGrant(this.deps.hash, this.xkey, id, own, contact.peerId, expiresAt),
1019
+ ...(declared ? { limits: declared } : {}),
1020
+ }
1021
+ const verdict: GateVerdict = origin.judged
1022
+ ? { verdict: "pass" }
1023
+ : await link.gate.egress(msg, { side: "egress", account: self.handle, peer: contact.peerHandle, peer_id: contact.peerId, sender })
1024
+ if (verdict.verdict === "refuse") throw gateRefusal(verdict, "egress")
1025
+ if (verdict.verdict === "hold") return { id: verdict.hold_id, thread, hop, deliveries: [], held: { hold_id: verdict.hold_id, side: "egress" } }
1026
+ if (verdict.verdict === "duplicate") return { id: verdict.hold_id, thread, hop, deliveries: [], duplicate: true }
1027
+ if (verdict.verdict === "redact") {
1028
+ const result = await this.xsend(sender, { ...send, body: verdict.body }, { ...origin, judged: true })
1029
+ const redacted = joinRedacted(verdict.redacted, result.redacted)
1030
+ return { ...result, ...(redacted ? { redacted } : {}) }
1031
+ }
1032
+ this.xcounters.check(contact, me.remote, address, thread)
1033
+ const refund = this.xcounters.sent(contact, me.remote, address, thread)
1034
+ let delivered: XDelivered
1035
+ try {
1036
+ delivered = await link.deliver(contact, msg)
1037
+ } catch (error) {
1038
+ if (!(error instanceof RelayError) || error.code === "unavailable") refund()
1039
+ throw error
1040
+ }
1041
+ this.fleet({ type: "message", machine: me.machine, id: delivered.id, thread, kind, from: from.address, to: [address], hop })
1042
+ return {
1043
+ id: delivered.id,
1044
+ thread,
1045
+ hop,
1046
+ deliveries: [{ endpoint: address, address, seq: 0, offline: delivered.state === "queued", last_seen: null }],
1047
+ ...(delivered.duplicate ? { duplicate: true } : {}),
1048
+ ...(delivered.held ? { held: { hold_id: delivered.held.hold_id, side: "ingress" as const } } : {}),
1049
+ ...(delivered.redacted ? { redacted: delivered.redacted } : {}),
1050
+ }
1051
+ }
1052
+
1053
+ /**
1054
+ * A message from the account `fromAccountId`'s relay (the host's internal route, never a client): the contact, the target on this
1055
+ * account's allowlist, its scope (or the reply grant), shape, a retry's earlier answer, the scrubber, the ingress gate, then one xinbox
1056
+ * row pushed to the target's bridge. Anything unresolvable is not_found, as for a missing endpoint.
1057
+ */
1058
+ async xreceive(fromAccountId: string, raw: unknown, judged = false): Promise<XDelivered> {
1059
+ const core = this.relay
1060
+ const link = this.deps.crossAccount
1061
+ const msg = parseXMessage(raw)
1062
+ const missing = () => new RelayError("not_found", `no endpoint ${link?.self.handle ?? this.account}/${msg.to}`)
1063
+ if (!link || !this.xcounters || !this.xkey) throw missing()
1064
+ const contact = await link.contactById(fromAccountId)
1065
+ if (!contact || contact.dir === "out" || contact.peerId !== fromAccountId || msg.from.account !== contact.peerHandle) throw missing()
1066
+ const target = core.xendpoint(msg.to)
1067
+ if (!target || !allowlisted(contact.allow, target.machine, target.handle, target.depth)) throw missing()
1068
+ const address = `${link.self.handle}/${target.machine}/${target.handle || "me"}`
1069
+ const grantExpiry = msg.reply_grant_expires_at
1070
+ const granted =
1071
+ msg.reply_to !== null &&
1072
+ msg.reply_grant !== undefined &&
1073
+ grantExpiry !== undefined &&
1074
+ grantExpiry > this.now() &&
1075
+ sameSecret(msg.reply_grant, replyGrant(this.deps.hash, this.xkey, msg.reply_to, address, fromAccountId, grantExpiry))
1076
+ const parts = msg.from.address.split("/")
1077
+ const peer = {
1078
+ account: contact.peerHandle,
1079
+ machine: parts.length === 3 ? parts[1]! : (msg.from.machine ?? "me"),
1080
+ handle: parts.length === 3 && parts[2] !== "me" ? parts[2]! : "",
1081
+ }
1082
+ if (!granted && !core.crossAdmission(target.id, "in", peer).ok) throw missing()
1083
+ if (!honestSender(msg.from)) throw new RelayError("invalid_params", "the sender is not one its relay may name")
1084
+ const bytes = new TextEncoder().encode(msg.body).length
1085
+ if (bytes > XLIMITS.bodyBytes) throw new RelayError("too_large", `a message from another account is at most ${XLIMITS.bodyBytes} bytes`)
1086
+ if (msg.hop > XLIMITS.hopMax) throw new RelayError("loop_guard", `hop ${msg.hop} is past the limit across accounts (${XLIMITS.hopMax})`)
1087
+ if (msg.expires_at <= this.now()) throw new RelayError("invalid_params", "the message expired")
1088
+ const earlier = core.xduplicate(fromAccountId, msg.client_id)
1089
+ if (earlier) return { id: earlier.id, state: earlier.acked ? "delivered" : "queued", duplicate: true }
1090
+ const envelope: Envelope = {
1091
+ id: msg.id,
1092
+ thread: msg.thread,
1093
+ reply_to: msg.reply_to,
1094
+ kind: msg.kind,
1095
+ from: {
1096
+ address: msg.from.address,
1097
+ endpoint: msg.from.address,
1098
+ user: contact.peerHandle,
1099
+ auth: "contact",
1100
+ ...(msg.from.machine ? { machine: msg.from.machine } : {}),
1101
+ claims: { harness: msg.from.harness ?? null, model: null },
1102
+ rel: `contact:${contact.peerHandle}`,
1103
+ account: contact.peerHandle,
1104
+ ...(msg.from.human ? { human: true as const } : {}),
1105
+ },
1106
+ to: address,
1107
+ body: { text: msg.body, mime: "text/markdown" },
1108
+ data_class: "public",
1109
+ attachments: [],
1110
+ hop: msg.hop,
1111
+ created_at: new Date(msg.created_at).toISOString(),
1112
+ expires_at: new Date(msg.expires_at).toISOString(),
1113
+ }
1114
+ const message = JSON.stringify(envelope)
1115
+ const stored = new TextEncoder().encode(message).length
1116
+ this.xadmit(fromAccountId, contact, msg, target, stored)
1117
+ const secret = findSecret(msg.body)
1118
+ if (secret) throw new RelayError("secret_refused", `the message looks like it carries a credential (${secret})`)
1119
+ const verdict: GateVerdict = judged
1120
+ ? { verdict: "pass" }
1121
+ : await link.gate.ingress(msg, { side: "ingress", account: link.self.handle, peer: contact.peerHandle, peer_id: fromAccountId })
1122
+ if (verdict.verdict === "refuse") throw gateRefusal(verdict, "ingress")
1123
+ if (verdict.verdict === "hold") return { id: msg.id, state: "queued", held: { hold_id: verdict.hold_id } }
1124
+ if (verdict.verdict === "duplicate") return { id: msg.id, state: "delivered", duplicate: true }
1125
+ if (verdict.verdict === "redact") return { ...(await this.xreceive(fromAccountId, { ...msg, body: verdict.body }, true)), redacted: verdict.redacted }
1126
+ const raced = () => {
1127
+ const row = core.xduplicate(fromAccountId, msg.client_id)
1128
+ return row ? { id: row.id, state: row.acked ? ("delivered" as const) : ("queued" as const), duplicate: true } : null
1129
+ }
1130
+ const again = raced()
1131
+ if (again) return again
1132
+ this.xadmit(fromAccountId, contact, msg, target, stored)
1133
+ const seq = core.xinsert({
1134
+ id: msg.id,
1135
+ from_account: fromAccountId,
1136
+ client_id: msg.client_id,
1137
+ endpoint: target.remote,
1138
+ sender: msg.from.address,
1139
+ message,
1140
+ thread: msg.thread,
1141
+ grant: msg.grant ?? null,
1142
+ expires_at: msg.expires_at,
1143
+ at: this.now(),
1144
+ })
1145
+ if (seq === null) return raced()!
1146
+ this.stash(stored)
1147
+ let pushed = false
1148
+ for (const socket of this.deps.sockets()) {
1149
+ if (socket.conn.role !== "bridge" || !socket.conn.subscribed || socket.conn.machine !== target.machine || this.gone.has(socket.conn.id)) continue
1150
+ this.write(socket, { jsonrpc: "2.0", method: "deliver", params: { seq, endpoint: target.remote, message: envelope } })
1151
+ pushed = true
1152
+ }
1153
+ this.wake(target.id)
1154
+ this.fleet({
1155
+ type: "message",
1156
+ machine: msg.from.machine ?? null,
1157
+ id: msg.id,
1158
+ thread: msg.thread,
1159
+ kind: msg.kind,
1160
+ from: msg.from.address,
1161
+ to: [address],
1162
+ hop: msg.hop,
1163
+ })
1164
+ return { id: msg.id, state: pushed ? "delivered" : "queued" }
1165
+ }
1166
+
1167
+ /** this account's endpoints the account `fromAccountId` may reach under its contact's allowlist: address, harness and presence only */
1168
+ async xlist(fromAccountId: string): Promise<XListed[]> {
1169
+ const link = this.deps.crossAccount
1170
+ if (!link) return []
1171
+ const contact = await link.contactById(fromAccountId)
1172
+ if (!contact || contact.dir === "out" || contact.peerId !== fromAccountId) return []
1173
+ return this.relay
1174
+ .xendpoints()
1175
+ .filter((e) => allowlisted(contact.allow, e.machine, e.handle, e.depth))
1176
+ .map((e) => ({ address: `${link.self.handle}/${e.machine}/${e.handle || "me"}`, harness: e.harness, presence: e.presence }))
1177
+ }
1178
+
1179
+ /**
1180
+ * throws when one more message from `fromAccountId` would pass a cap of this side: the pair a minute and an hour and the contact a
1181
+ * day (the lower of this account's limits for the contact and the sender's declared ones), the thread, the account's stored bytes or
1182
+ * the target's unread messages; counted from xinbox itself, so a wake keeps them
1183
+ */
1184
+ private xadmit(fromAccountId: string, contact: XContact, msg: XMessage, target: XEndpoint, bytes: number): void {
1185
+ const core = this.relay
1186
+ const now = this.now()
1187
+ const { perMinute, perHour, perDay } = xlimits(contact.limits, msg.limits)
1188
+ const counts = core.xcounts(fromAccountId, msg.from.address, target.remote, dayStart(now))
1189
+ const retry = (since: number | null, ms: number) => Math.max(1, Math.ceil(((since ?? now) + ms - now) / 1000))
1190
+ if (counts.minute >= perMinute)
1191
+ throw new RelayError("rate_limited", `${perMinute} messages from ${msg.from.address} to ${msg.to} a minute`, retry(counts.minuteAt, 60_000))
1192
+ if (counts.hour >= perHour)
1193
+ throw new RelayError("rate_limited", `${perHour} messages from ${msg.from.address} to ${msg.to} an hour`, retry(counts.hourAt, 3_600_000))
1194
+ if (counts.day >= perDay) throw new RelayError("rate_limited", `${perDay} messages from ${contact.peerHandle} today`, untilTomorrow(now))
1195
+ if (core.xthreadCount(target.remote, msg.thread) >= XLIMITS.threadMax)
1196
+ throw new RelayError("loop_guard", `thread ${msg.thread} reached ${XLIMITS.threadMax} messages across accounts`)
1197
+ const cap = this.limits.storedBytes
1198
+ if (this.storedNow() + bytes > cap) throw new RelayError("inbox_full", `the receiving account's relay holds its cap of ${cap} bytes of messages`)
1199
+ const unread = core.xunread(target)
1200
+ if (unread >= LIMITS.inboxMax) throw new RelayError("inbox_full", `${msg.to} has ${unread} unread messages`)
1201
+ }
1202
+
1203
+ /**
1204
+ * deletes the queued messages of every account that is no longer an active contact allowed to message this one, after the
1205
+ * contacts changed; nothing when the contacts cannot be loaded
1206
+ */
1207
+ async xprune(): Promise<void> {
1208
+ if (!this.deps.crossAccount || !this.core) return
1209
+ const senders = await this.xinbound()
1210
+ if (!senders || !this.core) return
1211
+ this.relay.xsenders(senders)
1212
+ if (this.relay.xprune(senders)) this.stored = null
1213
+ }
1214
+
1215
+ /** the accounts whose messages this account takes now: active contacts with dir in or both; null when they cannot be loaded */
1216
+ private async xinbound(): Promise<Set<string> | null> {
1217
+ const contacts = await this.deps.crossAccount?.contacts()
1218
+ return contacts ? new Set(contacts.filter((contact) => contact.dir !== "out").map((contact) => contact.peerId)) : null
1219
+ }
1220
+
1221
+ /** what every contact this account may message lets it reach */
1222
+ private async contactList(): Promise<XListed[]> {
1223
+ const link = this.deps.crossAccount
1224
+ if (!link) return []
1225
+ const contacts = ((await link.contacts()) ?? []).filter((contact) => contact.dir !== "in")
1226
+ const lists = await Promise.all(
1227
+ contacts.map((contact) =>
1228
+ link.list(contact).then(
1229
+ (list) => ({ contact, list }),
1230
+ () => ({ contact, list: [] as XListed[] }),
1231
+ ),
1232
+ ),
1233
+ )
1234
+ return lists.flatMap(({ contact, list }) =>
1235
+ list
1236
+ .filter((e) => typeof e?.address === "string" && e.address.startsWith(`${contact.peerHandle}/`))
1237
+ .map((e) => ({
1238
+ address: e.address,
1239
+ harness: typeof e.harness === "string" ? e.harness : null,
1240
+ presence: typeof e.presence === "string" ? e.presence : "offline",
1241
+ })),
1242
+ )
1243
+ }
1244
+
1245
+ // push and wait
1246
+
1247
+ private delivered(from: string, result: SendResult, kind: string): void {
1248
+ const core = this.relay
1249
+ for (const delivery of result.deliveries) this.route(delivery.seq, delivery.endpoint)
1250
+ this.fleet({
1251
+ type: "message",
1252
+ machine: core.machineOf(from),
1253
+ id: result.id,
1254
+ thread: result.thread,
1255
+ kind,
1256
+ from: core.addressOf(from),
1257
+ to: result.deliveries.map((d) => d.address),
1258
+ hop: result.hop,
1259
+ })
1260
+ }
1261
+
1262
+ /** the body's bytes when the account can hold them; refuses a send past the `storedBytes` limit */
1263
+ private room(sender: string, body: unknown): number {
1264
+ if (typeof body !== "string") return 0
1265
+ const bytes = new TextEncoder().encode(body).length
1266
+ const cap = this.limits.storedBytes
1267
+ if (this.storedNow() + bytes > cap) {
1268
+ this.relay.record("refuse", sender, null, { code: "too_large" })
1269
+ throw new RelayError(
1270
+ "too_large",
1271
+ `the account relay holds its cap of ${cap} bytes of message bodies; ack or let messages expire, or release a machine with ${CLI_NAME} machines rm`,
1272
+ )
1273
+ }
1274
+ return bytes
1275
+ }
1276
+
1277
+ /** bytes of message bodies held, and of the messages from other accounts */
1278
+ private storedNow(): number {
1279
+ const xinbox = this.deps.crossAccount ? ` + (SELECT COALESCE(SUM(LENGTH(CAST(message AS BLOB))), 0) FROM xinbox)` : ""
1280
+ this.stored ??= one<{ n: number }>(this.deps.sql, `SELECT (SELECT COALESCE(SUM(LENGTH(CAST(body AS BLOB))), 0) FROM messages)${xinbox} AS n`)!.n
1281
+ return this.stored
1282
+ }
1283
+
1284
+ private stash(bytes: number): void {
1285
+ if (this.stored !== null) this.stored += bytes
1286
+ }
1287
+
1288
+ /** wakes direct waiters and pushes a delivery for a mirrored row to the subscribed bridge of its machine */
1289
+ private route(seq: number, endpoint: string): void {
1290
+ this.wake(endpoint)
1291
+ const delivery = this.relay.delivery(seq)
1292
+ if (!delivery?.endpoint || !delivery.mirrored) return
1293
+ const machine = delivery.endpoint.slice(0, delivery.endpoint.indexOf("/"))
1294
+ for (const socket of this.deps.sockets()) {
1295
+ if (socket.conn.role === "bridge" && socket.conn.subscribed && socket.conn.machine === machine) {
1296
+ this.write(socket, { jsonrpc: "2.0", method: "deliver", params: { seq, endpoint: delivery.endpoint, message: delivery.message } })
1297
+ }
1298
+ }
1299
+ }
1300
+
1301
+ /** a machine's endpoint changes: `endpoints` to the other bridges, presence and status to watchers */
1302
+ private announce(machine: string, changes: MirrorChange[], full: boolean): void {
1303
+ if (full || changes.length) {
1304
+ const endpoints = full ? this.relay.remoteRowsOf(machine) : changes.map((c) => c.view)
1305
+ this.toBridges(machine, { machine, endpoints, full })
1306
+ }
1307
+ for (const change of changes) {
1308
+ const view = change.view
1309
+ if (change.added || change.presence)
1310
+ this.fleet({ type: "presence", machine, endpoint: view.address, presence: view.presence, harness: view.harness, labels: view.labels })
1311
+ if (change.status && !change.added) this.fleet({ type: "status", machine, endpoint: view.address, status: view.status })
1312
+ }
1313
+ }
1314
+
1315
+ /** a direct client's endpoint changed: the other bridges mirror it */
1316
+ private announceEndpoint(endpoint: string): void {
1317
+ const machine = this.relay.machineOf(endpoint)
1318
+ if (!machine) return
1319
+ const view = this.relay.remoteRowsOf(machine).find((v) => v.id === endpoint)
1320
+ if (view) this.toBridges(machine, { machine, endpoints: [view], full: false })
1321
+ }
1322
+
1323
+ private toBridges(machine: string, params: unknown): void {
1324
+ for (const socket of this.deps.sockets()) {
1325
+ if (socket.conn.role === "bridge" && socket.conn.machine !== machine) this.write(socket, { jsonrpc: "2.0", method: "endpoints", params })
1326
+ }
1327
+ }
1328
+
1329
+ private fleet(event: Record<string, unknown>): void {
1330
+ const at = new Date(this.now()).toISOString()
1331
+ for (const socket of this.deps.sockets()) if (socket.conn.watching) this.write(socket, { jsonrpc: "2.0", method: "fleet", params: { at, ...event } })
1332
+ }
1333
+
1334
+ private write(socket: HubSocket, value: unknown): void {
1335
+ if (this.gone.has(socket.conn.id)) return
1336
+ try {
1337
+ socket.send(JSON.stringify(value))
1338
+ } catch {
1339
+ this.deps.log?.("relay_send_failed", { account: this.account, machine: socket.conn.machine })
1340
+ }
1341
+ }
1342
+
1343
+ private save(socket: HubSocket): void {
1344
+ if (this.transient.has(socket)) return
1345
+ this.deps.save(socket)
1346
+ }
1347
+
1348
+ private waitMs(socket: HubSocket, value: unknown): number {
1349
+ if (value === undefined) return 0
1350
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new RelayError("invalid_params", "wait_s must be a number of seconds")
1351
+ return Math.min(value, this.transient.has(socket) ? this.limits.httpWaitS : LIMITS.waitMaxS) * 1000
1352
+ }
1353
+
1354
+ private wake(endpoint: string): void {
1355
+ const set = this.waiters.get(endpoint)
1356
+ if (!set) return
1357
+ this.waiters.delete(endpoint)
1358
+ for (const fn of set) fn()
1359
+ }
1360
+
1361
+ private waitFor(endpoint: string, ms: number, socket: HubSocket): Promise<void> {
1362
+ return new Promise((resolve) => {
1363
+ const done = () => {
1364
+ clearTimeout(timer)
1365
+ this.waiters.get(endpoint)?.delete(done)
1366
+ resolve()
1367
+ }
1368
+ const timer = setTimeout(done, ms)
1369
+ let set = this.waiters.get(endpoint)
1370
+ if (!set) {
1371
+ set = new Set()
1372
+ this.waiters.set(endpoint, set)
1373
+ }
1374
+ set.add(done)
1375
+ if (this.gone.has(socket.conn.id)) done()
1376
+ })
1377
+ }
1378
+
1379
+ private async awaitReply(socket: HubSocket, me: string, request: string, ms: number, sent?: object): Promise<unknown> {
1380
+ const core = this.relay
1381
+ const deadline = Date.now() + ms
1382
+ for (;;) {
1383
+ const reply = core.replyTo(me, request)
1384
+ if (reply) {
1385
+ core.ack(me, [reply.seq])
1386
+ return { ...sent, id: request, reply }
1387
+ }
1388
+ if (core.requestExpired(request)) return { ...sent, id: request, expired: true }
1389
+ if (this.gone.has(socket.conn.id) || Date.now() >= deadline) return { ...sent, id: request, pending: true }
1390
+ await this.waitFor(me, deadline - Date.now(), socket)
1391
+ }
1392
+ }
1393
+
1394
+ // connections and machines
1395
+
1396
+ private kick(socket: HubSocket, reason: string, code = 1008): void {
1397
+ this.write(socket, { jsonrpc: "2.0", method: "revoked", params: { reason } })
1398
+ socket.close(code, reason)
1399
+ this.release(socket, false)
1400
+ }
1401
+
1402
+ private release(socket: HubSocket, replaced: boolean): void {
1403
+ const conn = socket.conn
1404
+ if (this.gone.has(conn.id)) return
1405
+ this.gone.add(conn.id)
1406
+ if (conn.role === "new" || !this.core) return
1407
+ const core = this.core
1408
+ const bound = this.boundMachine(conn)
1409
+ if (conn.role === "client" && conn.endpoint) {
1410
+ const endpoint = conn.endpoint
1411
+ core.disconnect(endpoint)
1412
+ const machine = core.machineOf(endpoint)
1413
+ if (machine && one<{ kind: string }>(this.deps.sql, `SELECT kind FROM machines WHERE name = ?`, machine)?.kind === "bridge") {
1414
+ for (const delivery of core.adoptClaimed(machine, endpoint)) this.route(delivery.seq, delivery.endpoint)
1415
+ }
1416
+ this.wake(endpoint)
1417
+ const view = core.whoami(endpoint)
1418
+ this.fleet({ type: "presence", machine: conn.machine, endpoint: view.address, presence: view.presence })
1419
+ this.announceEndpoint(endpoint)
1420
+ } else if (conn.role === "bridge" && conn.machine) {
1421
+ const newer = this.deps
1422
+ .sockets()
1423
+ .some((s) => s.conn.id !== conn.id && s.conn.role === "bridge" && s.conn.machine === conn.machine && !this.gone.has(s.conn.id))
1424
+ if (!replaced && !newer) this.announce(conn.machine, core.markOffline(conn.machine), false)
1425
+ }
1426
+ if (bound) this.deps.sql.all(`UPDATE machines SET last_seen = ? WHERE name = ?`, this.now(), bound)
1427
+ conn.role = "new"
1428
+ conn.endpoint = null
1429
+ conn.subscribed = false
1430
+ conn.watching = false
1431
+ if (!this.transient.has(socket)) {
1432
+ try {
1433
+ this.deps.save(socket)
1434
+ } catch {}
1435
+ }
1436
+ }
1437
+
1438
+ /** the machine a bridge or agent client connection holds; a human's or observer's machine is only a namespace */
1439
+ private boundMachine(conn: HubConn): string | null {
1440
+ if (!conn.machine || !this.core) return null
1441
+ if (conn.role === "bridge") return conn.machine
1442
+ return conn.role === "client" && conn.endpoint !== null && this.core.machineOf(conn.endpoint) === conn.machine ? conn.machine : null
1443
+ }
1444
+
1445
+ /** machines a live connection holds */
1446
+ private liveMachines(): Set<string> {
1447
+ const live = new Set<string>()
1448
+ for (const socket of [...this.deps.sockets(), ...this.transient]) {
1449
+ const machine = this.gone.has(socket.conn.id) ? null : this.boundMachine(socket.conn)
1450
+ if (machine) live.add(machine)
1451
+ }
1452
+ return live
1453
+ }
1454
+
1455
+ /** bindings, then names a revoke left behind (kind `revoked`) */
1456
+ /** a read-only view of the account for a dashboard: its machines, listed endpoints and latest activity */
1457
+ overview(activity = 20): { machines: MachineRow[]; endpoints: EndpointView[]; activity: ActivityRow[] } {
1458
+ const { endpoints, activity: rows } = this.relay.snapshot({ activity })
1459
+ return { machines: this.machineList(), endpoints, activity: rows }
1460
+ }
1461
+
1462
+ private machineList(): MachineRow[] {
1463
+ const sockets = this.deps.sockets().filter((s) => !this.gone.has(s.conn.id))
1464
+ const bound = this.deps.sql
1465
+ .all<{ name: string; kind: string; created_at: number; last_seen: number | null }>(`SELECT name, kind, created_at, last_seen FROM machines ORDER BY name`)
1466
+ .map((row) => ({
1467
+ name: row.name,
1468
+ kind: row.kind,
1469
+ online: sockets.some((s) => this.boundMachine(s.conn) === row.name),
1470
+ bridge: sockets.some((s) => s.conn.role === "bridge" && s.conn.machine === row.name),
1471
+ created_at: iso(row.created_at),
1472
+ last_seen: iso(row.last_seen),
1473
+ }))
1474
+ const revoked = this.deps.sql
1475
+ .all<{ name: string; at: number }>(`SELECT name, at FROM machine_tombstones WHERE name NOT IN (SELECT name FROM machines) ORDER BY name`)
1476
+ .map((row) => ({ name: row.name, kind: "revoked", online: false, bridge: false, created_at: null, last_seen: iso(row.at) }))
1477
+ return [...bound, ...revoked]
1478
+ }
1479
+
1480
+ /** only the token the machine is bound to, a full login of the member who bound it, or an admin may release it (or a revoked token's name) */
1481
+ private releaseMachine(caller: HubSocket, name: unknown): unknown {
1482
+ const core = this.relay
1483
+ if (typeof name !== "string" || !MACHINE_NAME.test(name)) throw new RelayError("invalid_params", "name is a machine name")
1484
+ const bound = one<{ token_id: string; actor_id: string | null }>(this.deps.sql, `SELECT token_id, actor_id FROM machines WHERE name = ?`, name)
1485
+ const tomb = bound ? undefined : one<{ actor_id: string | null }>(this.deps.sql, `SELECT actor_id FROM machine_tombstones WHERE name = ?`, name)
1486
+ if (!bound && !tomb) throw new RelayError("not_found", `no machine ${name}`)
1487
+ const grant = caller.conn.grant
1488
+ if (!(bound?.token_id === grant.tokenId || fullOf(grant, (bound ?? tomb)!.actor_id) || accountAdmin(grant))) {
1489
+ core.record("refuse", null, name, { code: "unauthorized", machine: name })
1490
+ throw new RelayError(
1491
+ "unauthorized",
1492
+ `only the token machine ${name} is bound to, a full login of the member who bound it, or an org owner or admin can release it`,
1493
+ )
1494
+ }
1495
+ for (const socket of [...this.deps.sockets()]) {
1496
+ if (socket.conn.id !== caller.conn.id && this.boundMachine(socket.conn) === name) this.kick(socket, `machine ${name} was released`)
1497
+ }
1498
+ const dropped = this.tx(() => {
1499
+ const count = core.dropMachine(name, { queue: true })
1500
+ this.deps.sql.all(`DELETE FROM machines WHERE name = ?`, name)
1501
+ this.deps.sql.all(`DELETE FROM machine_tombstones WHERE name = ?`, name)
1502
+ core.record("machine_release", null, name, { machine: name, endpoints: count })
1503
+ return count
1504
+ })
1505
+ this.stored = null
1506
+ this.toBridges(name, { machine: name, endpoints: [], full: true })
1507
+ this.deps.log?.("relay_action", { account: this.account, machine: name, method: "machine_release", code: "ok" })
1508
+ return { removed: name, endpoints: dropped }
1509
+ }
1510
+
1511
+ /** releases client bindings with no live connection, idle for the `clientMachineIdleS` limit, with nothing queued and no unanswered ask open; bridge bindings never expire */
1512
+ private expireMachines(now: number): void {
1513
+ const core = this.relay
1514
+ const live = this.liveMachines()
1515
+ const idle = this.deps.sql.all<{ name: string }>(
1516
+ `SELECT name FROM machines WHERE kind = 'client' AND COALESCE(last_seen, created_at) <= ?`,
1517
+ now - this.limits.clientMachineIdleS * 1000,
1518
+ )
1519
+ for (const { name } of idle) {
1520
+ if (live.has(name)) continue
1521
+ const queued = one(
1522
+ this.deps.sql,
1523
+ `SELECT 1 FROM deliveries d JOIN endpoints e ON e.id = d.endpoint WHERE substr(e.remote, 1, ?) = ? AND d.acked_at IS NULL AND d.expires_at > ? LIMIT 1`,
1524
+ name.length + 1,
1525
+ `${name}/`,
1526
+ now,
1527
+ )
1528
+ const asking = one(
1529
+ this.deps.sql,
1530
+ `SELECT 1 FROM messages m JOIN endpoints e ON e.id = m.from_ep WHERE substr(e.remote, 1, ?) = ? AND m.kind = 'request' AND m.expires_at > ?
1531
+ AND NOT EXISTS (SELECT 1 FROM messages r WHERE r.reply_to = m.id AND r.kind = 'reply') LIMIT 1`,
1532
+ name.length + 1,
1533
+ `${name}/`,
1534
+ now,
1535
+ )
1536
+ if (queued || asking) continue
1537
+ const dropped = this.tx(() => {
1538
+ const count = core.dropMachine(name)
1539
+ this.deps.sql.all(`DELETE FROM machines WHERE name = ?`, name)
1540
+ core.record("machine_expire", null, name, { machine: name, endpoints: count })
1541
+ return count
1542
+ })
1543
+ this.toBridges(name, { machine: name, endpoints: [], full: true })
1544
+ this.deps.log?.("relay_action", { account: this.account, machine: name, method: "machine_expire", code: "ok", endpoints: dropped })
1545
+ }
1546
+ }
1547
+
1548
+ /** drops the tombstones past their age with nothing queued for their machine, and retires the rows and xinbox of each name still unbound */
1549
+ private forgetTombstones(now: number): void {
1550
+ const forgotten = this.tx(() => {
1551
+ const names = this.deps.sql.all<{ name: string }>(
1552
+ `DELETE FROM machine_tombstones WHERE at <= ? AND NOT EXISTS (SELECT 1 FROM deliveries d JOIN endpoints e ON e.id = d.endpoint
1553
+ WHERE substr(e.remote, 1, length(machine_tombstones.name) + 1) = machine_tombstones.name || '/' AND d.acked_at IS NULL AND d.expires_at > ?)
1554
+ RETURNING name`,
1555
+ now - (LIMITS.ttlMaxS + LIMITS.purgeAfterS) * 1000,
1556
+ now,
1557
+ )
1558
+ const unbound = names.map((row) => row.name).filter((name) => !one(this.deps.sql, `SELECT 1 FROM machines WHERE name = ?`, name))
1559
+ for (const name of unbound) this.relay.record("machine_forget", null, name, { machine: name, endpoints: this.relay.dropMachine(name, { queue: true }) })
1560
+ return unbound
1561
+ })
1562
+ for (const name of forgotten) this.toBridges(name, { machine: name, endpoints: [], full: true })
1563
+ }
1564
+
1565
+ /** the hub_state keys (`denied:<tokenId>`) of tokens revoked lately, to when; loaded once per object lifetime */
1566
+ private deniedGrants(): Map<string, number> {
1567
+ this.denied ??= new Map(
1568
+ this.deps.sql.all<{ key: string; value: number }>(`SELECT key, value FROM hub_state WHERE key LIKE ?`, `${DENIED}%`).map((row) => [row.key, row.value]),
1569
+ )
1570
+ return this.denied
1571
+ }
1572
+
1573
+ /** a token revoked lately is refused even when the host accepted its grant before the revoke reached the object */
1574
+ private refuseDenied(socket: HubSocket): void {
1575
+ const at = this.deniedGrants().get(DENIED + socket.conn.grant.tokenId)
1576
+ if (at === undefined || at <= this.now() - this.limits.deniedS * 1000) return
1577
+ this.kick(socket, "the token was revoked")
1578
+ throw new RelayError("unauthorized", "the token was revoked")
1579
+ }
1580
+
1581
+ private tx<T>(fn: () => T): T {
1582
+ return this.deps.transaction ? this.deps.transaction(fn) : fn()
1583
+ }
1584
+
1585
+ private state(key: string): number | null {
1586
+ return one<{ value: number }>(this.deps.sql, `SELECT value FROM hub_state WHERE key = ?`, key)?.value ?? null
1587
+ }
1588
+
1589
+ private setState(key: string, value: number): void {
1590
+ this.deps.sql.all(`INSERT OR REPLACE INTO hub_state (key, value) VALUES (?, ?)`, key, value)
1591
+ }
1592
+
1593
+ /**
1594
+ * a held cross-account message its account's human released (the host claimed the hold first): the rest of that side's path, with
1595
+ * every check run again and only the gate skipped. Egress sends as the stored sender with the time left, under the hold id its sender
1596
+ * got, and with the message's own client id, so the peer answers both a retried release and the sender's retry of the original as the
1597
+ * duplicate. Ingress runs xreceive again
1598
+ */
1599
+ async xrelease(hold: XRelease): Promise<SendResult | XDelivered> {
1600
+ if (hold.side === "ingress") return this.xreceive(hold.from_account, hold.message, true)
1601
+ const msg = hold.message
1602
+ const left = Math.ceil((msg.expires_at - this.now()) / 1000)
1603
+ if (left < 1) throw new RelayError("invalid_params", "the held message expired")
1604
+ const send: Params = {
1605
+ to: `${hold.peer}/${msg.to.endsWith("/") ? `${msg.to}me` : msg.to}`,
1606
+ body: msg.body,
1607
+ kind: msg.kind,
1608
+ thread: msg.thread,
1609
+ ...(msg.reply_to ? { reply_to: msg.reply_to } : {}),
1610
+ ttl_s: Math.min(left, LIMITS.ttlMaxS),
1611
+ }
1612
+ return this.xsend(hold.sender, send, { tty: msg.from.human, hop: msg.hop, judged: true, clientId: msg.client_id, id: hold.hold_id })
1613
+ }
1614
+
1615
+ /** a status message from the relay to one of this account's endpoints (a held message's sender), pushed to its bridge */
1616
+ xnotice(endpoint: string, text: string): void {
1617
+ this.route(this.relay.judgeNoticeTo(endpoint, text), endpoint)
1618
+ }
1619
+ }