@threadedstack/wmd-web 0.0.0-stage → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2601 -2
- package/RELEASE.json +123 -0
- package/export-page/coi-sw.js +20 -0
- package/export-page/index.html +17 -0
- package/export-page/run.css +10 -0
- package/export-page/run.js +236 -0
- package/guest/arm64/Image +4 -0
- package/guest/arm64/Image-4k +4 -0
- package/guest/arm64/initramfs.cpio.gz +0 -0
- package/guest/arm64/vm.wasm +0 -0
- package/package.json +36 -4
- package/page/sw.js +381 -0
- package/relay/control.ts +139 -0
- package/relay/engine.ts +787 -0
- package/relay/hosted.ts +660 -0
- package/relay/keys.ts +53 -0
- package/relay/local-tls.ts +121 -0
- package/relay/lock.ts +81 -0
- package/relay/logfile.ts +138 -0
- package/relay/main.ts +642 -0
- package/relay/policy.ts +241 -0
- package/relay/relay.ts +266 -0
- package/relay/token.ts +116 -0
- package/relay/v1.ts +107 -0
- package/relay/wisp.ts +615 -0
- package/src/base.js +325 -0
- package/src/boot.js +2037 -0
- package/src/calib-worker.js +40 -0
- package/src/fs/9p/codec.js +303 -0
- package/src/fs/9p/server.js +521 -0
- package/src/fs/errno.js +43 -0
- package/src/fs/graph.js +380 -0
- package/src/fs/overlay.js +1018 -0
- package/src/fs/pack-write.js +165 -0
- package/src/fs/pack.js +427 -0
- package/src/fs/seed.js +22 -0
- package/src/fs/svc-snapshot.js +65 -0
- package/src/fs/upper-pack.js +112 -0
- package/src/host/base-cache.js +195 -0
- package/src/host/base-file.js +328 -0
- package/src/host/base-io.js +93 -0
- package/src/host/boot-events.js +209 -0
- package/src/host/chan-service.js +300 -0
- package/src/host/chan-services.js +689 -0
- package/src/host/chan.js +655 -0
- package/src/host/ckpt-file.js +395 -0
- package/src/host/ckpt-io.js +207 -0
- package/src/host/ckpt-point.js +160 -0
- package/src/host/ckpt-save.js +64 -0
- package/src/host/ckpt.js +434 -0
- package/src/host/clock.js +111 -0
- package/src/host/ctl.js +778 -0
- package/src/host/errno.js +18 -0
- package/src/host/fs.js +167 -0
- package/src/host/gcnudge.js +83 -0
- package/src/host/i64shim.js +81 -0
- package/src/host/jit-cache.js +341 -0
- package/src/host/jit-hub-core.js +208 -0
- package/src/host/jit-hub.js +93 -0
- package/src/host/jit-snap.js +226 -0
- package/src/host/jit.js +1903 -0
- package/src/host/layout.js +100 -0
- package/src/host/monitor.js +110 -0
- package/src/host/p9mount.js +27 -0
- package/src/host/ram-mem.js +125 -0
- package/src/host/stdio.js +143 -0
- package/src/host/threads.js +109 -0
- package/src/host/wasi.js +335 -0
- package/src/host/wmd-imports.js +71 -0
- package/src/host/yamux.js +387 -0
- package/src/image-files.js +154 -0
- package/src/net/dhcp-dns.js +126 -0
- package/src/net/net-worker.js +172 -0
- package/src/net/packet.js +227 -0
- package/src/net/peer.js +230 -0
- package/src/net/proto.js +189 -0
- package/src/net/relay-client.js +226 -0
- package/src/net/stack.js +867 -0
- package/src/net/switch.js +160 -0
- package/src/net/wisp-client.js +570 -0
- package/src/net/wisp-transport.js +196 -0
- package/src/net/wisp.js +511 -0
- package/src/registry/client.js +176 -0
- package/src/registry/converter.js +68 -0
- package/src/registry/hub.js +221 -0
- package/src/registry/oci.js +76 -0
- package/src/registry/pack-worker.js +70 -0
- package/src/registry/pull.js +258 -0
- package/src/registry/ref.js +110 -0
- package/src/registry/wasi-mem.js +355 -0
- package/src/worker.js +315 -0
- package/wmdpack.wasm +0 -0
package/relay/hosted.ts
ADDED
|
@@ -0,0 +1,660 @@
|
|
|
1
|
+
// W59 S7: the `--hosted` profile (plan §15): what a public relay we run adds to the engine.
|
|
2
|
+
//
|
|
3
|
+
// The governor holds every hosted-only limit and list, and nothing else: it does not touch sockets. The engine
|
|
4
|
+
// (engine.ts, its `guard` hook) and the Wisp adapter (wisp.ts) ask it, at the points where a session is admitted,
|
|
5
|
+
// a stream is opened or bytes move, and the relay (relay.ts) asks it for the state shown at /healthz and set by the
|
|
6
|
+
// control socket. Clocks are injectable, so every limit is tested without waiting.
|
|
7
|
+
//
|
|
8
|
+
// Keys: an "address group" is wisp.ts's `grp` (an IPv4 address, or an IPv6 /64: token.ts clientKey); a "coarse
|
|
9
|
+
// group" is the IPv4 /24 or IPv6 /48 around it. Page origins are a fairness control for cooperating browsers only: any
|
|
10
|
+
// script can send any Origin, so every origin budget is also bounded by the address's own share and caps.
|
|
11
|
+
|
|
12
|
+
import { readFileSync, renameSync, writeFileSync } from 'node:fs'
|
|
13
|
+
import { Bucket, type Session, type Shared, type Stream } from './engine.ts'
|
|
14
|
+
import { type Addr, type Cidr, addrString, cidrContains, hostedBlockedReason, parseCidr, parseIP } from './policy.ts'
|
|
15
|
+
|
|
16
|
+
const MIB = 1 << 20
|
|
17
|
+
const GIB = 2 ** 30
|
|
18
|
+
|
|
19
|
+
export type HostedOptions = {
|
|
20
|
+
abuseContact: string
|
|
21
|
+
version?: string
|
|
22
|
+
startStopped?: boolean // begin in stop-new: nothing is taken until `ctl resume` (a stop does not persist a restart otherwise)
|
|
23
|
+
now?: () => number // ms since the epoch (tests)
|
|
24
|
+
// per address group
|
|
25
|
+
groupStreams?: number // streams open at once, all its sessions (128)
|
|
26
|
+
groupConnectsPerMin?: number // (300)
|
|
27
|
+
groupBytesPerHour?: number // both ways (2 GiB)
|
|
28
|
+
groupDnsPerMin?: number // (600)
|
|
29
|
+
// per coarse group (IPv4 /24, IPv6 /48)
|
|
30
|
+
coarseSessions?: number // (64)
|
|
31
|
+
coarseInstancesPerHour?: number // new instances (240)
|
|
32
|
+
coarseRateDown?: number // bytes/s, all its addresses together (16 MiB/s)
|
|
33
|
+
coarseRateUp?: number // (4 MiB/s)
|
|
34
|
+
// per page origin
|
|
35
|
+
originTable?: number // origins remembered (1000, LRU)
|
|
36
|
+
originSessions?: number // sessions of one origin (400)
|
|
37
|
+
originShare?: number // the most of an origin one address group may use, 0-1 (0.25)
|
|
38
|
+
otherOriginSessions?: number // "other origins" and `Origin: null`: one shared budget (200)
|
|
39
|
+
noOriginSessions?: number // requests without an Origin (16)
|
|
40
|
+
originsPerAddrPerHour?: number // distinct origins one address group may use before it falls into "other" (8)
|
|
41
|
+
originRateDown?: number // bytes/s of one origin (32 MiB/s)
|
|
42
|
+
originRateUp?: number // (8 MiB/s)
|
|
43
|
+
// the whole relay
|
|
44
|
+
maxStreams?: number // streams open at once, all sessions: a file-descriptor budget below the host's limit (20,000)
|
|
45
|
+
maxRateDown?: number // bytes/s (64 MiB/s)
|
|
46
|
+
maxRateUp?: number // (16 MiB/s)
|
|
47
|
+
maxBytesPerDay?: number // both ways; 0 = no ceiling
|
|
48
|
+
maxBytesPerMonth?: number
|
|
49
|
+
usageFile?: string // the day and month counters survive a restart here
|
|
50
|
+
ceilingGraceMs?: number // after a ceiling is reached, streams already open may run this long, then they end (60 s)
|
|
51
|
+
onCeilingGrace?: (words: string) => void // called when the grace is over and the ceiling is still in force: end the open streams
|
|
52
|
+
warn?: (line: string) => void // one-line operator notices (stderr)
|
|
53
|
+
// against third parties
|
|
54
|
+
destConnectsPerMin?: number // per destination (host:port) per coarse group (600)
|
|
55
|
+
destStreams?: number // streams to one destination at once per coarse group (200)
|
|
56
|
+
distinctDests?: number // distinct destinations per address group per 10 minutes (100)
|
|
57
|
+
failedConnects?: number // refused, unreachable or timed-out dials per address group before it cools off (20)
|
|
58
|
+
failedWindowMs?: number // (5 min)
|
|
59
|
+
failedCoolMs?: number // (5 min)
|
|
60
|
+
denyCidrs?: string[] // dialled addresses refused besides the built-in ranges (the fleet's own neighbours)
|
|
61
|
+
tableMax?: number // address and coarse groups remembered per table (50000)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export type DenySpec = {
|
|
65
|
+
addresses?: string[] // client addresses or CIDRs ("203.0.113.0/24", "2001:db8:1:2::/64")
|
|
66
|
+
origins?: string[]
|
|
67
|
+
destinations?: string[] // "name", "name:port", ".suffix" (and its apex), or a literal address
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export type Refusal = { status: number; body: string }
|
|
71
|
+
|
|
72
|
+
/** A live session, as the deny lists see it. */
|
|
73
|
+
export type Handle = { ip: string; origin: string; end: (words: string) => void }
|
|
74
|
+
|
|
75
|
+
/** What a session holds from the governor until it ends. */
|
|
76
|
+
export type Admission = {
|
|
77
|
+
grp: string
|
|
78
|
+
coarse: string
|
|
79
|
+
budget: OriginBudget
|
|
80
|
+
addrBucket: Shared
|
|
81
|
+
down: Bucket[]
|
|
82
|
+
up: Bucket[]
|
|
83
|
+
handle: Handle
|
|
84
|
+
release: () => void
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
type OriginBudget = {
|
|
88
|
+
key: string
|
|
89
|
+
cap: number
|
|
90
|
+
sessions: number
|
|
91
|
+
perAddr: Map<string, { n: number; shared: Shared }>
|
|
92
|
+
shared: Shared
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** A sliding window of event times on an injectable clock. */
|
|
96
|
+
class Hits {
|
|
97
|
+
times: number[] = []
|
|
98
|
+
constructor(public max: number, public ms: number) {}
|
|
99
|
+
prune(t: number) {
|
|
100
|
+
let i = 0
|
|
101
|
+
while (i < this.times.length && t - this.times[i] > this.ms) i++
|
|
102
|
+
if (i) this.times.splice(0, i)
|
|
103
|
+
}
|
|
104
|
+
hit(t: number): boolean {
|
|
105
|
+
this.prune(t)
|
|
106
|
+
if (this.times.length >= this.max) return false
|
|
107
|
+
this.times.push(t)
|
|
108
|
+
return true
|
|
109
|
+
}
|
|
110
|
+
last(): number {
|
|
111
|
+
return this.times.length ? this.times[this.times.length - 1] : 0
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** A table that forgets its least recently used entry past `max`. */
|
|
116
|
+
class Lru<V> {
|
|
117
|
+
m = new Map<string, V>()
|
|
118
|
+
constructor(public max: number) {}
|
|
119
|
+
get(k: string): V | undefined {
|
|
120
|
+
const v = this.m.get(k)
|
|
121
|
+
if (v !== undefined) {
|
|
122
|
+
this.m.delete(k)
|
|
123
|
+
this.m.set(k, v)
|
|
124
|
+
}
|
|
125
|
+
return v
|
|
126
|
+
}
|
|
127
|
+
set(k: string, v: V) {
|
|
128
|
+
this.m.delete(k)
|
|
129
|
+
this.m.set(k, v)
|
|
130
|
+
while (this.m.size > this.max) this.m.delete(this.m.keys().next().value as string)
|
|
131
|
+
}
|
|
132
|
+
delete(k: string) {
|
|
133
|
+
this.m.delete(k)
|
|
134
|
+
}
|
|
135
|
+
get size() {
|
|
136
|
+
return this.m.size
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const ipv4Group = /^\d+\.\d+\.\d+\.\d+$/
|
|
141
|
+
|
|
142
|
+
/** The coarse group of an address group key (token.ts clientKey form): IPv4 /24, IPv6 /48. */
|
|
143
|
+
export function coarseKey(grp: string): string {
|
|
144
|
+
if (ipv4Group.test(grp)) return `c4:${grp.split('.').slice(0, 3).join('.')}`
|
|
145
|
+
if (grp.startsWith('v6:')) return `c6:${grp.slice(3).split(':').slice(0, 3).join(':')}`
|
|
146
|
+
return `c:${grp}`
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export type ClientAddr = { ip: string } | { error: string }
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The client address of a request behind trusted proxies (plan §15, H2). A peer that is not a trusted proxy is keyed on
|
|
153
|
+
* its own address and the header is ignored. A trusted proxy must send the header; its entries are walked from the right
|
|
154
|
+
* and the first one that is not itself a trusted proxy is the client.
|
|
155
|
+
*/
|
|
156
|
+
export function resolveClient(peer: string, header: string | null, trusted: Cidr[], headerName: string): ClientAddr {
|
|
157
|
+
const norm = (s: string): Addr | null => {
|
|
158
|
+
s = s.trim().replace(/^\[|\](:\d+)?$/g, '')
|
|
159
|
+
if (/^\d+\.\d+\.\d+\.\d+:\d+$/.test(s)) s = s.slice(0, s.lastIndexOf(':'))
|
|
160
|
+
return parseIP(s)
|
|
161
|
+
}
|
|
162
|
+
const isTrusted = (a: Addr) => trusted.some(c => cidrHas(c, a))
|
|
163
|
+
const p = norm(peer)
|
|
164
|
+
if (!p) return { ip: peer }
|
|
165
|
+
const peerIp = unmap(p)
|
|
166
|
+
if (!isTrusted(peerIp)) return { ip: addrText(peerIp) }
|
|
167
|
+
const entries = (header || '').split(',').map(x => x.trim()).filter(Boolean)
|
|
168
|
+
if (!entries.length) return { error: `this relay sits behind a proxy that must send ${headerName}; the request had none` }
|
|
169
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
170
|
+
const a = norm(entries[i])
|
|
171
|
+
if (!a) return { error: `${headerName} has an entry that is not an address` }
|
|
172
|
+
const u = unmap(a)
|
|
173
|
+
if (!isTrusted(u)) return { ip: addrText(u) }
|
|
174
|
+
}
|
|
175
|
+
return { error: `${headerName} names only proxies, no client` }
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const cidrHas = cidrContains
|
|
179
|
+
const unmap = (a: Addr): Addr => (a.v === 6 && a.b.slice(0, 10).every(x => x === 0) && a.b[10] === 255 && a.b[11] === 255 ? { v: 4, b: a.b.slice(12) } : a)
|
|
180
|
+
const addrText = (a: Addr) => (a.v === 4 ? addrString(a) : compact6(a))
|
|
181
|
+
/** IPv6 in the compressed form the clients' own text uses ("2001:db8::1"), so clientKey groups it. */
|
|
182
|
+
function compact6(a: Addr): string {
|
|
183
|
+
const w: number[] = []
|
|
184
|
+
for (let i = 0; i < 16; i += 2) w.push((a.b[i] << 8) | a.b[i + 1])
|
|
185
|
+
let best = -1
|
|
186
|
+
let bestLen = 0
|
|
187
|
+
for (let i = 0; i < 8; ) {
|
|
188
|
+
if (w[i]) { i++; continue }
|
|
189
|
+
let j = i
|
|
190
|
+
while (j < 8 && !w[j]) j++
|
|
191
|
+
if (j - i > bestLen && j - i > 1) [best, bestLen] = [i, j - i]
|
|
192
|
+
i = j
|
|
193
|
+
}
|
|
194
|
+
const h = (x: number[]) => x.map(n => n.toString(16)).join(':')
|
|
195
|
+
return best < 0 ? h(w) : `${h(w.slice(0, best))}::${h(w.slice(best + bestLen))}`
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
type Dest = { name: string; port: number; suffix: boolean }
|
|
199
|
+
|
|
200
|
+
export function createGovernor(o: HostedOptions) {
|
|
201
|
+
const now = o.now ?? Date.now
|
|
202
|
+
const tableMax = o.tableMax ?? 50_000
|
|
203
|
+
const c = {
|
|
204
|
+
groupStreams: o.groupStreams ?? 128,
|
|
205
|
+
groupConnectsPerMin: o.groupConnectsPerMin ?? 300,
|
|
206
|
+
groupBytesPerHour: o.groupBytesPerHour ?? 2 * GIB,
|
|
207
|
+
groupDnsPerMin: o.groupDnsPerMin ?? 600,
|
|
208
|
+
coarseSessions: o.coarseSessions ?? 64,
|
|
209
|
+
coarseInstancesPerHour: o.coarseInstancesPerHour ?? 240,
|
|
210
|
+
coarseRateDown: o.coarseRateDown ?? 16 * MIB,
|
|
211
|
+
coarseRateUp: o.coarseRateUp ?? 4 * MIB,
|
|
212
|
+
originTable: o.originTable ?? 1000,
|
|
213
|
+
originSessions: o.originSessions ?? 400,
|
|
214
|
+
originShare: o.originShare ?? 0.25,
|
|
215
|
+
otherOriginSessions: o.otherOriginSessions ?? 200,
|
|
216
|
+
noOriginSessions: o.noOriginSessions ?? 16,
|
|
217
|
+
originsPerAddrPerHour: o.originsPerAddrPerHour ?? 8,
|
|
218
|
+
originRateDown: o.originRateDown ?? 32 * MIB,
|
|
219
|
+
originRateUp: o.originRateUp ?? 8 * MIB,
|
|
220
|
+
maxStreams: o.maxStreams ?? 20_000,
|
|
221
|
+
maxRateDown: o.maxRateDown ?? 64 * MIB,
|
|
222
|
+
maxRateUp: o.maxRateUp ?? 16 * MIB,
|
|
223
|
+
maxBytesPerDay: o.maxBytesPerDay ?? 0,
|
|
224
|
+
maxBytesPerMonth: o.maxBytesPerMonth ?? 0,
|
|
225
|
+
destConnectsPerMin: o.destConnectsPerMin ?? 600,
|
|
226
|
+
destStreams: o.destStreams ?? 200,
|
|
227
|
+
distinctDests: o.distinctDests ?? 100,
|
|
228
|
+
failedConnects: o.failedConnects ?? 20,
|
|
229
|
+
failedWindowMs: o.failedWindowMs ?? 5 * 60_000,
|
|
230
|
+
failedCoolMs: o.failedCoolMs ?? 5 * 60_000,
|
|
231
|
+
ceilingGraceMs: o.ceilingGraceMs ?? 60_000,
|
|
232
|
+
}
|
|
233
|
+
const denyCidrs = (o.denyCidrs ?? []).map(s => {
|
|
234
|
+
const x = parseCidr(s)
|
|
235
|
+
if (!x) throw new Error(`deny-cidr '${s}' is not an address or CIDR`)
|
|
236
|
+
return x
|
|
237
|
+
})
|
|
238
|
+
const contact = o.abuseContact
|
|
239
|
+
const contactLine = `abuse contact: ${contact}\n`
|
|
240
|
+
|
|
241
|
+
// ---- state set by the operator (control socket) and by the ceilings
|
|
242
|
+
let stopped = !!o.startStopped
|
|
243
|
+
let draining = false
|
|
244
|
+
|
|
245
|
+
// ---- tables
|
|
246
|
+
const groupStreams = new Map<string, number>()
|
|
247
|
+
const groupConnects = new Lru<Hits>(tableMax)
|
|
248
|
+
const groupDns = new Lru<Hits>(tableMax)
|
|
249
|
+
const groupBytes = new Lru<{ hour: number; n: number }>(tableMax)
|
|
250
|
+
const groupDests = new Lru<Map<string, number>>(tableMax)
|
|
251
|
+
const groupFails = new Lru<{ hits: Hits; cool: number }>(tableMax)
|
|
252
|
+
const coarseSessions = new Map<string, number>()
|
|
253
|
+
const coarseInstances = new Lru<Hits>(tableMax)
|
|
254
|
+
const coarseShared = new Lru<Shared>(tableMax)
|
|
255
|
+
const destConnects = new Lru<Hits>(tableMax)
|
|
256
|
+
const destLive = new Map<string, number>()
|
|
257
|
+
const addrOrigins = new Lru<Map<string, number>>(tableMax) // per address group: origin -> first seen (this hour)
|
|
258
|
+
const origins = new Lru<OriginBudget>(c.originTable) // by origin; entries with live sessions are never evicted
|
|
259
|
+
const mk = (key: string, cap: number): OriginBudget => ({
|
|
260
|
+
key, cap, sessions: 0, perAddr: new Map(),
|
|
261
|
+
shared: { down: new Bucket(c.originRateDown), up: new Bucket(c.originRateUp) },
|
|
262
|
+
})
|
|
263
|
+
const other = mk('other', c.otherOriginSessions)
|
|
264
|
+
const none = mk('none', c.noOriginSessions)
|
|
265
|
+
const globalShared: Shared = { down: new Bucket(c.maxRateDown), up: new Bucket(c.maxRateUp) }
|
|
266
|
+
const live = new Set<Handle>()
|
|
267
|
+
let liveStreams = 0
|
|
268
|
+
|
|
269
|
+
// ---- deny lists
|
|
270
|
+
let denyAddr: Cidr[] = []
|
|
271
|
+
let denyOrigin = new Set<string>()
|
|
272
|
+
let denyDest: Dest[] = []
|
|
273
|
+
|
|
274
|
+
function reloadDeny(spec: DenySpec): { ended: number } {
|
|
275
|
+
const addrs: Cidr[] = []
|
|
276
|
+
for (const a of spec.addresses ?? []) {
|
|
277
|
+
const x = parseCidr(a)
|
|
278
|
+
if (!x) throw new Error(`deny list: '${a}' is not an address or CIDR`)
|
|
279
|
+
addrs.push(x)
|
|
280
|
+
}
|
|
281
|
+
const dests: Dest[] = []
|
|
282
|
+
for (const d of spec.destinations ?? []) {
|
|
283
|
+
const m = String(d).toLowerCase().trim().match(/^(\.?[^:]+)(?::(\d{1,5}))?$/)
|
|
284
|
+
if (!m) throw new Error(`deny list: '${d}' is not a destination (name, name:port or .suffix)`)
|
|
285
|
+
dests.push({ name: m[1].replace(/^\./, ''), port: m[2] ? Number(m[2]) : 0, suffix: m[1].startsWith('.') })
|
|
286
|
+
}
|
|
287
|
+
denyAddr = addrs
|
|
288
|
+
denyOrigin = new Set((spec.origins ?? []).map(String))
|
|
289
|
+
denyDest = dests
|
|
290
|
+
let ended = 0
|
|
291
|
+
for (const h of [...live]) {
|
|
292
|
+
const why = deniedClient(h.ip, h.origin)
|
|
293
|
+
if (why) {
|
|
294
|
+
ended++
|
|
295
|
+
h.end(why)
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
return { ended }
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function deniedClient(ip: string, origin: string): string | null {
|
|
302
|
+
const a = parseIP(ip)
|
|
303
|
+
if (a && denyAddr.some(x => cidrHas(x, unmap(a)))) return 'this address is blocked by the operator'
|
|
304
|
+
if (origin && denyOrigin.has(origin)) return 'this page origin is blocked by the operator'
|
|
305
|
+
return null
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
const deniedDestination = (host: string, port: number): boolean => {
|
|
309
|
+
const h = host.toLowerCase().replace(/\.$/, '')
|
|
310
|
+
return denyDest.some(d => (!d.port || d.port === port) && (h === d.name || (d.suffix && h.endsWith(`.${d.name}`))))
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// ---- ceilings
|
|
314
|
+
const day = (t: number) => new Date(t).toISOString().slice(0, 10)
|
|
315
|
+
const month = (t: number) => new Date(t).toISOString().slice(0, 7)
|
|
316
|
+
const warn = o.warn ?? ((m: string) => void process.stderr.write(`${m}\n`))
|
|
317
|
+
let usage = { day: day(now()), dayBytes: 0, month: month(now()), monthBytes: 0 }
|
|
318
|
+
if (o.usageFile) {
|
|
319
|
+
let raw: string | null = null
|
|
320
|
+
try { raw = readFileSync(o.usageFile, 'utf8') } catch {}
|
|
321
|
+
if (raw !== null) {
|
|
322
|
+
let u: any = null
|
|
323
|
+
try { u = JSON.parse(raw) } catch {}
|
|
324
|
+
const t = now()
|
|
325
|
+
const dayOk = (v: unknown) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(v) && Number.isFinite(Date.parse(`${v}T00:00:00Z`)) && day(Date.parse(`${v}T00:00:00Z`)) === v && v <= day(t + 86_400_000)
|
|
326
|
+
const next = new Date(t)
|
|
327
|
+
const monthMax = month(Date.UTC(next.getUTCFullYear(), next.getUTCMonth() + 1, 1))
|
|
328
|
+
const monthOk = (v: unknown) => typeof v === 'string' && /^\d{4}-\d{2}$/.test(v) && Number.isFinite(Date.parse(`${v}-01T00:00:00Z`)) && month(Date.parse(`${v}-01T00:00:00Z`)) === v && v <= monthMax
|
|
329
|
+
// a counter that is not a finite, safe, non-negative number, or is far past its own ceiling, is corrupt
|
|
330
|
+
const sane = (v: unknown, ceil?: number) => typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= Number.MAX_SAFE_INTEGER && (!ceil || v <= ceil * 2)
|
|
331
|
+
const problems: string[] = []
|
|
332
|
+
if (!u || typeof u !== 'object') problems.push('not a JSON object')
|
|
333
|
+
else {
|
|
334
|
+
usage.day = dayOk(u.day) ? u.day : (u.day !== undefined && problems.push('day'), usage.day)
|
|
335
|
+
usage.month = monthOk(u.month) ? u.month : (u.month !== undefined && problems.push('month'), usage.month)
|
|
336
|
+
if (sane(u.dayBytes, c.maxBytesPerDay)) usage.dayBytes = u.dayBytes
|
|
337
|
+
else problems.push('dayBytes')
|
|
338
|
+
if (sane(u.monthBytes, c.maxBytesPerMonth)) usage.monthBytes = u.monthBytes
|
|
339
|
+
else problems.push('monthBytes')
|
|
340
|
+
// a date that was reset starts its counter at 0 too
|
|
341
|
+
if (usage.day !== u.day) usage.dayBytes = 0
|
|
342
|
+
if (usage.month !== u.month) usage.monthBytes = 0
|
|
343
|
+
}
|
|
344
|
+
if (problems.length) warn(`wmd-relay: ${o.usageFile} is damaged (${problems.join(', ')}); the affected counters and dates start again from 0 and today`)
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
/** The counters restart only when the date moves forward: a clock set back must not zero them. */
|
|
348
|
+
function roll() {
|
|
349
|
+
const t = now()
|
|
350
|
+
if (day(t) > usage.day) {
|
|
351
|
+
usage.day = day(t)
|
|
352
|
+
usage.dayBytes = 0
|
|
353
|
+
}
|
|
354
|
+
if (month(t) > usage.month) {
|
|
355
|
+
usage.month = month(t)
|
|
356
|
+
usage.monthBytes = 0
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
/** The ceiling in force, or null. The rollover lifts it: the counters restart with the day and the month. */
|
|
360
|
+
function ceiling(): 'day' | 'month' | null {
|
|
361
|
+
roll()
|
|
362
|
+
const k = c.maxBytesPerMonth && usage.monthBytes >= c.maxBytesPerMonth ? 'month' : c.maxBytesPerDay && usage.dayBytes >= c.maxBytesPerDay ? 'day' : null
|
|
363
|
+
if (!k) {
|
|
364
|
+
ceilingSince = 0
|
|
365
|
+
if (graceTimer) clearTimeout(graceTimer)
|
|
366
|
+
graceTimer = null
|
|
367
|
+
} else if (!ceilingSince) {
|
|
368
|
+
ceilingSince = now()
|
|
369
|
+
graceTimer = setTimeout(() => {
|
|
370
|
+
graceTimer = null
|
|
371
|
+
const w = ceiling()
|
|
372
|
+
if (w) o.onCeilingGrace?.(ceilingWords(w))
|
|
373
|
+
}, c.ceilingGraceMs)
|
|
374
|
+
;(graceTimer as any).unref?.()
|
|
375
|
+
}
|
|
376
|
+
return k
|
|
377
|
+
}
|
|
378
|
+
let ceilingSince = 0
|
|
379
|
+
let graceTimer: ReturnType<typeof setTimeout> | null = null
|
|
380
|
+
/** True once a ceiling has been in force longer than the grace: even streams that are open end. */
|
|
381
|
+
const pastGrace = () => !!ceiling() && now() - ceilingSince >= c.ceilingGraceMs
|
|
382
|
+
const ceilingWords = (k: 'day' | 'month') => `the relay has reached its traffic limit for ${k === 'day' ? 'today' : 'this month'}`
|
|
383
|
+
function saveUsage() {
|
|
384
|
+
if (!o.usageFile) return
|
|
385
|
+
try {
|
|
386
|
+
const tmp = `${o.usageFile}.new`
|
|
387
|
+
writeFileSync(tmp, JSON.stringify(usage), { mode: 0o600 })
|
|
388
|
+
renameSync(tmp, o.usageFile)
|
|
389
|
+
} catch {}
|
|
390
|
+
}
|
|
391
|
+
const saver = o.usageFile ? setInterval(saveUsage, 15_000) : null
|
|
392
|
+
saver?.unref?.()
|
|
393
|
+
|
|
394
|
+
function accepting(): boolean {
|
|
395
|
+
return !stopped && !draining && !ceiling()
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** Why new sessions are not taken now, or null. */
|
|
399
|
+
function notAccepting(): string | null {
|
|
400
|
+
if (draining) return 'the relay is closed for maintenance'
|
|
401
|
+
if (stopped) return 'the relay is not taking new connections now'
|
|
402
|
+
const k = ceiling()
|
|
403
|
+
return k ? ceilingWords(k) : null
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// ---- sessions
|
|
407
|
+
function pickBudget(grp: string, origin: string): OriginBudget {
|
|
408
|
+
if (!origin) return none
|
|
409
|
+
if (origin === 'null') return other
|
|
410
|
+
const t = now()
|
|
411
|
+
let seen = addrOrigins.get(grp)
|
|
412
|
+
if (!seen) addrOrigins.set(grp, (seen = new Map()))
|
|
413
|
+
for (const [k, at] of seen) if (t - at > 3_600_000) seen.delete(k)
|
|
414
|
+
if (!seen.has(origin)) {
|
|
415
|
+
if (seen.size >= c.originsPerAddrPerHour) return other // a ninth distinct origin from one address
|
|
416
|
+
seen.set(origin, t)
|
|
417
|
+
}
|
|
418
|
+
let b = origins.get(origin)
|
|
419
|
+
if (b) return b
|
|
420
|
+
if (origins.size >= c.originTable) {
|
|
421
|
+
// evict the least recently used origin that has no live session; none free: this one is "other"
|
|
422
|
+
let victim: string | null = null
|
|
423
|
+
for (const [k, v] of origins.m) if (!v.sessions) { victim = k; break }
|
|
424
|
+
if (victim === null) return other
|
|
425
|
+
origins.delete(victim)
|
|
426
|
+
}
|
|
427
|
+
b = mk(origin, c.originSessions)
|
|
428
|
+
origins.set(origin, b)
|
|
429
|
+
return b
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/** A session of an address group from `ip` with this page origin is about to open. */
|
|
433
|
+
function admitSession(i: { ip: string; grp: string; origin: string; fresh: boolean; end: (words: string) => void }): { ok: true; adm: Admission } | { ok: false; why: string } {
|
|
434
|
+
const coarse = coarseKey(i.grp)
|
|
435
|
+
const t = now()
|
|
436
|
+
if ((coarseSessions.get(coarse) || 0) >= c.coarseSessions) return { ok: false, why: `more than ${c.coarseSessions} sessions from this network` }
|
|
437
|
+
if (i.fresh) {
|
|
438
|
+
let w = coarseInstances.get(coarse)
|
|
439
|
+
if (!w) coarseInstances.set(coarse, (w = new Hits(c.coarseInstancesPerHour, 3_600_000)))
|
|
440
|
+
if (!w.hit(t)) return { ok: false, why: `more than ${c.coarseInstancesPerHour} new instances an hour from this network` }
|
|
441
|
+
}
|
|
442
|
+
const budget = pickBudget(i.grp, i.origin)
|
|
443
|
+
const share = Math.max(1, Math.floor(budget.cap * c.originShare))
|
|
444
|
+
const mine = budget.perAddr.get(i.grp)
|
|
445
|
+
if (budget.sessions >= budget.cap) return { ok: false, why: `this page origin has no room for more sessions (${budget.cap})` }
|
|
446
|
+
if ((mine?.n ?? 0) >= share) return { ok: false, why: `more than ${share} sessions of this page origin from one address` }
|
|
447
|
+
let cs = coarseShared.get(coarse)
|
|
448
|
+
if (!cs) coarseShared.set(coarse, (cs = { down: new Bucket(c.coarseRateDown), up: new Bucket(c.coarseRateUp) }))
|
|
449
|
+
const entry = mine ?? { n: 0, shared: { down: new Bucket(Math.max(1, c.originRateDown * c.originShare)), up: new Bucket(Math.max(1, c.originRateUp * c.originShare)) } }
|
|
450
|
+
entry.n++
|
|
451
|
+
budget.perAddr.set(i.grp, entry)
|
|
452
|
+
budget.sessions++
|
|
453
|
+
coarseSessions.set(coarse, (coarseSessions.get(coarse) || 0) + 1)
|
|
454
|
+
const handle: Handle = { ip: i.ip, origin: i.origin, end: i.end }
|
|
455
|
+
live.add(handle)
|
|
456
|
+
let done = false
|
|
457
|
+
const adm: Admission = {
|
|
458
|
+
grp: i.grp, coarse, budget, addrBucket: entry.shared,
|
|
459
|
+
down: [cs.down, entry.shared.down, budget.shared.down, globalShared.down],
|
|
460
|
+
up: [cs.up, entry.shared.up, budget.shared.up, globalShared.up],
|
|
461
|
+
handle,
|
|
462
|
+
release() {
|
|
463
|
+
if (done) return
|
|
464
|
+
done = true
|
|
465
|
+
live.delete(handle)
|
|
466
|
+
budget.sessions--
|
|
467
|
+
const e = budget.perAddr.get(i.grp)
|
|
468
|
+
if (e && --e.n <= 0) budget.perAddr.delete(i.grp)
|
|
469
|
+
const n = (coarseSessions.get(coarse) || 1) - 1
|
|
470
|
+
if (n > 0) coarseSessions.set(coarse, n)
|
|
471
|
+
else coarseSessions.delete(coarse)
|
|
472
|
+
},
|
|
473
|
+
}
|
|
474
|
+
return { ok: true, adm }
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// ---- streams (engine guard)
|
|
478
|
+
const grpOf = (s: Session): string => s.hx?.grp ?? s.p.grp
|
|
479
|
+
const destKey = (host: string, port: number) => `${host.toLowerCase().replace(/\.$/, '')}:${port}`
|
|
480
|
+
|
|
481
|
+
/** Before a connect is resolved: the cooling-off, the group's rate, the scan shape and the destination's rate. Null, or the words. */
|
|
482
|
+
function admitConnect(s: Session, host: string, port: number): string | null {
|
|
483
|
+
const t = now()
|
|
484
|
+
const grp = grpOf(s)
|
|
485
|
+
const closed = notAccepting() // stop-new, drain and a ceiling end new streams too: open streams run to their end
|
|
486
|
+
if (closed) return closed
|
|
487
|
+
const f = groupFails.get(grp)
|
|
488
|
+
if (f && f.cool > t) return `too many failed connections from this address; new connections wait ${Math.ceil((f.cool - t) / 1000)} s`
|
|
489
|
+
const h = bytesHour(grp)
|
|
490
|
+
if (h.n >= c.groupBytesPerHour) return 'this address has used its bytes for this hour'
|
|
491
|
+
let w = groupConnects.get(grp)
|
|
492
|
+
if (!w) groupConnects.set(grp, (w = new Hits(c.groupConnectsPerMin, 60_000)))
|
|
493
|
+
if (!w.hit(t)) return `more than ${c.groupConnectsPerMin} connections a minute from this address`
|
|
494
|
+
const key = destKey(host, port)
|
|
495
|
+
let seen = groupDests.get(grp)
|
|
496
|
+
if (!seen) groupDests.set(grp, (seen = new Map()))
|
|
497
|
+
for (const [k, at] of seen) if (t - at > 600_000) seen.delete(k)
|
|
498
|
+
if (!seen.has(key) && seen.size >= c.distinctDests) return `more than ${c.distinctDests} different destinations in 10 minutes from this address`
|
|
499
|
+
seen.set(key, t)
|
|
500
|
+
const ck = `${coarseKey(grp)}|${key}`
|
|
501
|
+
let dw = destConnects.get(ck)
|
|
502
|
+
if (!dw) destConnects.set(ck, (dw = new Hits(c.destConnectsPerMin, 60_000)))
|
|
503
|
+
if (!dw.hit(t)) return `more than ${c.destConnectsPerMin} connections a minute to ${key} from this network`
|
|
504
|
+
return null
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** Just before the dial: the concurrent counts. Sets `st.rel`, which the engine calls once when the stream is gone. */
|
|
508
|
+
function openStream(s: Session, st: Stream, host: string, port: number): string | null {
|
|
509
|
+
const grp = grpOf(s)
|
|
510
|
+
const ck = `${coarseKey(grp)}|${destKey(host, port)}`
|
|
511
|
+
if (liveStreams >= c.maxStreams) return `the relay has ${c.maxStreams} streams open; try again shortly`
|
|
512
|
+
if ((groupStreams.get(grp) || 0) >= c.groupStreams) return `more than ${c.groupStreams} streams open from this address`
|
|
513
|
+
if ((destLive.get(ck) || 0) >= c.destStreams) return `more than ${c.destStreams} streams open to ${destKey(host, port)} from this network`
|
|
514
|
+
liveStreams++
|
|
515
|
+
groupStreams.set(grp, (groupStreams.get(grp) || 0) + 1)
|
|
516
|
+
destLive.set(ck, (destLive.get(ck) || 0) + 1)
|
|
517
|
+
st.rel = () => {
|
|
518
|
+
liveStreams--
|
|
519
|
+
for (const [m, k] of [[groupStreams, grp], [destLive, ck]] as const) {
|
|
520
|
+
const n = (m.get(k) || 1) - 1
|
|
521
|
+
if (n > 0) m.set(k, n)
|
|
522
|
+
else m.delete(k)
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
return null
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/** A dial the remote refused, that found no route or that timed out (or a name that did not resolve). */
|
|
529
|
+
function dialFailed(s: Session) {
|
|
530
|
+
const t = now()
|
|
531
|
+
const grp = grpOf(s)
|
|
532
|
+
let f = groupFails.get(grp)
|
|
533
|
+
if (!f) groupFails.set(grp, (f = { hits: new Hits(Infinity, c.failedWindowMs), cool: 0 }))
|
|
534
|
+
f.hits.prune(t)
|
|
535
|
+
f.hits.times.push(t)
|
|
536
|
+
if (f.hits.times.length >= c.failedConnects) {
|
|
537
|
+
f.cool = t + c.failedCoolMs
|
|
538
|
+
f.hits.times = []
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** DNS answers stop with new connections: stop-new, a drain and a ceiling (the same words). */
|
|
543
|
+
const dnsClosed = (): string | null => notAccepting()
|
|
544
|
+
|
|
545
|
+
function dnsAllowed(s: Session): boolean {
|
|
546
|
+
const grp = grpOf(s)
|
|
547
|
+
let w = groupDns.get(grp)
|
|
548
|
+
if (!w) groupDns.set(grp, (w = new Hits(c.groupDnsPerMin, 60_000)))
|
|
549
|
+
return w.hit(now())
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
function bytesHour(grp: string) {
|
|
553
|
+
const hour = Math.floor(now() / 3_600_000)
|
|
554
|
+
let b = groupBytes.get(grp)
|
|
555
|
+
if (!b || b.hour !== hour) groupBytes.set(grp, (b = { hour, n: 0 }))
|
|
556
|
+
return b
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
/** Bytes relayed for a session (either way). True while the group is within its hourly bytes. */
|
|
560
|
+
function bytes(s: Session, n: number): boolean {
|
|
561
|
+
roll()
|
|
562
|
+
usage.dayBytes += n
|
|
563
|
+
usage.monthBytes += n
|
|
564
|
+
const b = bytesHour(grpOf(s))
|
|
565
|
+
b.n += n
|
|
566
|
+
ceiling() // starts the grace clock the moment the counter crosses
|
|
567
|
+
return b.n <= c.groupBytesPerHour && !pastGrace()
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/** Why `bytes` just said no. */
|
|
571
|
+
function limitWhy(): string {
|
|
572
|
+
const k = ceiling()
|
|
573
|
+
return k && pastGrace() ? ceilingWords(k) : 'this address has used its bytes for this hour'
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/** Up to n bytes from the coarse, origin and relay-wide buckets of the session (what the group's own bucket let through). */
|
|
577
|
+
function take(s: Session, dir: 'down' | 'up', n: number): number {
|
|
578
|
+
const list: Bucket[] | undefined = s.hx?.[dir]
|
|
579
|
+
if (!list) return n
|
|
580
|
+
const got: number[] = []
|
|
581
|
+
let k = n
|
|
582
|
+
for (const b of list) {
|
|
583
|
+
k = b.take(k)
|
|
584
|
+
got.push(k)
|
|
585
|
+
if (!k) break
|
|
586
|
+
}
|
|
587
|
+
const last = got[got.length - 1]
|
|
588
|
+
for (let i = 0; i < got.length; i++) if (got[i] > last) list[i].tokens = Math.min(list[i].rate, list[i].tokens + got[i] - last)
|
|
589
|
+
return last
|
|
590
|
+
}
|
|
591
|
+
function waitMs(s: Session, dir: 'down' | 'up', n: number): number {
|
|
592
|
+
const list: Bucket[] | undefined = s.hx?.[dir]
|
|
593
|
+
return list ? Math.max(1, ...list.map(b => b.waitMs(n))) : 1
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function blockedAddr(a: Addr): string | null {
|
|
597
|
+
return hostedBlockedReason(a, denyCidrs)
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// ---- housekeeping
|
|
601
|
+
const sweep = setInterval(() => {
|
|
602
|
+
const t = now()
|
|
603
|
+
for (const m of [groupConnects, groupDns, coarseInstances]) for (const [k, w] of m.m) { w.prune(t); if (!w.times.length) m.delete(k) }
|
|
604
|
+
for (const [k, w] of destConnects.m) { w.prune(t); if (!w.times.length && !destLive.has(k)) destConnects.delete(k) }
|
|
605
|
+
for (const [k, f] of groupFails.m) { f.hits.prune(t); if (!f.hits.times.length && f.cool < t) groupFails.delete(k) }
|
|
606
|
+
for (const [k, b] of groupBytes.m) if (b.hour !== Math.floor(t / 3_600_000)) groupBytes.delete(k)
|
|
607
|
+
}, 60_000)
|
|
608
|
+
sweep.unref?.()
|
|
609
|
+
|
|
610
|
+
return {
|
|
611
|
+
options: c,
|
|
612
|
+
contact,
|
|
613
|
+
contactLine,
|
|
614
|
+
version: o.version,
|
|
615
|
+
// operator state
|
|
616
|
+
stopNew: () => void (stopped = true),
|
|
617
|
+
resume() {
|
|
618
|
+
stopped = false
|
|
619
|
+
draining = false
|
|
620
|
+
},
|
|
621
|
+
startDrain: () => void ((draining = true), (stopped = true)),
|
|
622
|
+
isStopped: () => stopped,
|
|
623
|
+
isDraining: () => draining,
|
|
624
|
+
accepting,
|
|
625
|
+
notAccepting,
|
|
626
|
+
ceiling,
|
|
627
|
+
ceilingWords,
|
|
628
|
+
usage: () => (roll(), { ...usage }),
|
|
629
|
+
saveUsage,
|
|
630
|
+
// deny lists
|
|
631
|
+
reloadDeny,
|
|
632
|
+
deniedClient,
|
|
633
|
+
isDeniedDest: deniedDestination,
|
|
634
|
+
// admission
|
|
635
|
+
admitSession,
|
|
636
|
+
admitConnect,
|
|
637
|
+
openStream,
|
|
638
|
+
dialFailed,
|
|
639
|
+
dnsAllowed,
|
|
640
|
+
dnsClosed,
|
|
641
|
+
bytes,
|
|
642
|
+
limitWhy,
|
|
643
|
+
take,
|
|
644
|
+
waitMs,
|
|
645
|
+
blockedAddr,
|
|
646
|
+
/** Sessions per origin class, for tests and status (counts only). */
|
|
647
|
+
liveStreams: () => liveStreams,
|
|
648
|
+
originStats: () => ({ table: origins.size, other: other.sessions, none: none.sessions }),
|
|
649
|
+
limitsLine: () =>
|
|
650
|
+
`${c.groupStreams} streams, ${c.groupConnectsPerMin} connects a minute, ${Math.round(c.groupBytesPerHour / MIB)} MiB an hour per address`,
|
|
651
|
+
stop() {
|
|
652
|
+
if (graceTimer) clearTimeout(graceTimer)
|
|
653
|
+
clearInterval(sweep)
|
|
654
|
+
if (saver) clearInterval(saver)
|
|
655
|
+
saveUsage()
|
|
656
|
+
},
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
export type Governor = ReturnType<typeof createGovernor>
|