@preventive/triage 1.0.0-alpha.4 → 1.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/client-sync.js +14 -11
- package/out/graph.js +1 -1
- package/out/view.css +1 -1
- package/out/view.js +6 -6
- package/package.json +4 -1
- package/server/auth.ts +5 -1
- package/server/config.ts +14 -1
- package/server/http.ts +77 -7
- package/server/index.ts +12 -9
- package/server/lifecycle.ts +8 -4
- package/server/objstore/fetch-mint-guard.ts +74 -0
- package/server/objstore/init.ts +26 -1
- package/server/objstore/rest-deny.ts +28 -0
- package/server/objstore/rest-mint.ts +224 -0
- package/server/objstore/rest.ts +26 -25
- package/server/objstore/sign.ts +105 -0
- package/server/sse-server.ts +42 -35
- package/server/sse-session.ts +28 -8
- package/server/sync-handlers.ts +145 -91
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
// REST endpoints for the objstore plane — the single-round-trip,
|
|
2
|
+
// SSE-session-independent alternative to the WS `objstore-fetch` /
|
|
3
|
+
// `objstore-put-begin` / `objstore-delete` handshakes (so these ops can't
|
|
4
|
+
// be interrupted by an SSE replica hop). One route, `POST
|
|
5
|
+
// /api/objstore/{tag}/{res}`, with a JSON body `{ op, ts, signature, ... }`:
|
|
6
|
+
// the workspace signs the request (the workspaceTag IS the Ed25519 pubkey,
|
|
7
|
+
// so verification is self-contained — no socket, no stored key), the server
|
|
8
|
+
// verifies it + freshness/replay-guards (the connection-nonce stand-in).
|
|
9
|
+
// fetch/put return the SAME token shape the WS path sends, for the UNCHANGED
|
|
10
|
+
// bearer-token GET / PUT byte transfers; delete mutates in place and returns
|
|
11
|
+
// `{ deletedVersion }` (no token — there are no bytes to move). See
|
|
12
|
+
// server/README.md "REST endpoints & tokens".
|
|
13
|
+
|
|
14
|
+
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
15
|
+
import { Buffer } from 'node:buffer'
|
|
16
|
+
|
|
17
|
+
import type { ObjstoreRestDeps, RouteMatch } from './rest.ts'
|
|
18
|
+
import { deny, denyConflict } from './rest-deny.ts'
|
|
19
|
+
import { MAX_CONTENT_LENGTH, beginPut, deleteObject, getLive, isValidContentHash, isValidIncarnation, isValidSignature, objectMetaWire } from './store.ts'
|
|
20
|
+
import { mintGetToken, mintPutToken } from './tokens.ts'
|
|
21
|
+
import { verifyObjstoreDeleteRestSig, verifyObjstoreFetchRestSig, verifyObjstorePutBeginRestSig } from './sign.ts'
|
|
22
|
+
import { createFetchMintGuard } from './fetch-mint-guard.ts'
|
|
23
|
+
|
|
24
|
+
// Per-process freshness + replay guard for the REST POSTs (fetch, put-begin,
|
|
25
|
+
// delete). They have no connection nonce to bind, so they bind a client
|
|
26
|
+
// timestamp — see ./fetch-mint-guard.ts. The three ops' signatures are
|
|
27
|
+
// globally unique (distinct canonical domains), so one guard dedups all three.
|
|
28
|
+
const mintGuard = createFetchMintGuard()
|
|
29
|
+
|
|
30
|
+
// Hard cap on a mint JSON body. The put-begin body (op, ts, signature,
|
|
31
|
+
// prevVersion, prevIncarnation, expectedLength, contentHash) is the larger
|
|
32
|
+
// of the two and still only a few hundred bytes; 4 KiB is comfortable
|
|
33
|
+
// headroom. Bounds the read so a hostile client can't stream an unbounded
|
|
34
|
+
// body into memory before the parse.
|
|
35
|
+
const MINT_BODY_MAX = 4096
|
|
36
|
+
|
|
37
|
+
// Read a small JSON request body up to `maxBytes`, returning the parsed
|
|
38
|
+
// value or null on overflow / parse failure / read error. Used only by
|
|
39
|
+
// the mint POST; the PUT body is the raw blob and streams to disk via a
|
|
40
|
+
// different path.
|
|
41
|
+
async function readJsonBody(req: IncomingMessage, maxBytes: number): Promise<unknown> {
|
|
42
|
+
const chunks: Buffer[] = []
|
|
43
|
+
let total = 0
|
|
44
|
+
try {
|
|
45
|
+
for await (const chunk of req) {
|
|
46
|
+
const buf = chunk as Buffer
|
|
47
|
+
total += buf.length
|
|
48
|
+
if (total > maxBytes) return null
|
|
49
|
+
chunks.push(buf)
|
|
50
|
+
}
|
|
51
|
+
} catch { return null }
|
|
52
|
+
try { return JSON.parse(Buffer.concat(chunks).toString('utf8')) }
|
|
53
|
+
catch { return null }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Extract + range-check the auth fields common to both mint ops. Gates the
|
|
57
|
+
// signature to the wire shape (`isValidSignature`, matching the WS plane) so
|
|
58
|
+
// an obviously-malformed sig is rejected up front rather than burning an
|
|
59
|
+
// Ed25519 verify — and, since this runs before `mintGuard.admit`, a bad sig
|
|
60
|
+
// never reaches the replay cache regardless.
|
|
61
|
+
function parseMintAuth(body: object): { ts: number; signature: string } | null {
|
|
62
|
+
const ts = (body as { ts?: unknown }).ts
|
|
63
|
+
const signature = (body as { signature?: unknown }).signature
|
|
64
|
+
if (!Number.isSafeInteger(ts) || (ts as number) < 0) return null
|
|
65
|
+
if (!isValidSignature(signature)) return null
|
|
66
|
+
return { ts: ts as number, signature }
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Dispatch the mint by body `op`. Auth-proxy friendly: the signature rides
|
|
70
|
+
// the JSON body (not a header or cookie, so it never collides with a
|
|
71
|
+
// cookie-based proxy), and the response is JSON — no redirect, no token in
|
|
72
|
+
// any URL. Client note: a replayed signature is rejected, so a retry MUST
|
|
73
|
+
// re-sign with a fresh `ts` rather than resend the same body.
|
|
74
|
+
export async function handleRestMint(
|
|
75
|
+
deps: ObjstoreRestDeps, req: IncomingMessage, res: ServerResponse, route: RouteMatch,
|
|
76
|
+
): Promise<void> {
|
|
77
|
+
const body = await readJsonBody(req, MINT_BODY_MAX)
|
|
78
|
+
if (!body || typeof body !== 'object') { deny(res, 400, 'bad-request'); return }
|
|
79
|
+
const op = (body as { op?: unknown }).op
|
|
80
|
+
if (op === 'fetch') { await handleRestFetchMint(deps, res, route, body); return }
|
|
81
|
+
if (op === 'put') { await handleRestPutBegin(deps, res, route, body); return }
|
|
82
|
+
if (op === 'delete') { await handleRestDelete(deps, res, route, body); return }
|
|
83
|
+
deny(res, 400, 'bad-request')
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// op:'fetch' — mirrors the WS `objstore-fetch` → `objstore-fetch-token`
|
|
87
|
+
// handshake; returns `{ ...meta, urlPath, token, expiresAt }` for the
|
|
88
|
+
// UNCHANGED bearer-token GET.
|
|
89
|
+
async function handleRestFetchMint(
|
|
90
|
+
deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
|
|
91
|
+
): Promise<void> {
|
|
92
|
+
const auth = parseMintAuth(body)
|
|
93
|
+
if (!auth) { deny(res, 400, 'bad-request'); return }
|
|
94
|
+
// Verify the signature BEFORE touching the replay guard so a bad-sig
|
|
95
|
+
// request can't consume cache space. The signature commits to THIS `ts`.
|
|
96
|
+
if (!await verifyObjstoreFetchRestSig(route.tag, route.resourceTag, auth.ts, auth.signature)) {
|
|
97
|
+
deny(res, 401, 'unauthorized'); return
|
|
98
|
+
}
|
|
99
|
+
// Freshness window + single-use dedup (the connection-nonce stand-in).
|
|
100
|
+
// 'stale'/'replay' are both opaque 401s — the client re-signs with a
|
|
101
|
+
// fresh `ts` and retries.
|
|
102
|
+
if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
|
|
103
|
+
const row = await getLive(deps.handle, route.tag, route.resourceTag)
|
|
104
|
+
if (!row) { deny(res, 404, 'not-found'); return }
|
|
105
|
+
const { token, exp } = mintGetToken(deps.secret, route.tag, route.resourceTag, row.version, row.incarnation)
|
|
106
|
+
res.writeHead(200, { 'content-type': 'application/json' })
|
|
107
|
+
res.end(JSON.stringify({
|
|
108
|
+
...objectMetaWire(row),
|
|
109
|
+
urlPath: `/api/objstore/${route.tag}/${route.resourceTag}`,
|
|
110
|
+
token,
|
|
111
|
+
expiresAt: exp,
|
|
112
|
+
}))
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// op:'put' — mirrors the WS `handlePutBegin`: verify the signed put fields,
|
|
116
|
+
// freshness/replay-guard, the new-workspace operator gate, then `beginPut`
|
|
117
|
+
// (advisory prev-check + staging row) and mint the put-token. Returns
|
|
118
|
+
// `{ stagingId, urlPath, token, expiresAt }` for the UNCHANGED bearer-token
|
|
119
|
+
// PUT; `409 conflict` (with current version/incarnation to rebase on) or
|
|
120
|
+
// `403 workspace-full` map `beginPut`'s refusals.
|
|
121
|
+
//
|
|
122
|
+
// New-workspace gate: REST can't read a socket's operator-auth state, so
|
|
123
|
+
// when a password is configured AND the workspace is never-before-seen this
|
|
124
|
+
// returns 401 — the client falls back to the in-band WS put-begin (which
|
|
125
|
+
// runs the operator auth flow). No-op for password-less deployments or
|
|
126
|
+
// existing workspaces (the common case).
|
|
127
|
+
async function handleRestPutBegin(
|
|
128
|
+
deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
|
|
129
|
+
): Promise<void> {
|
|
130
|
+
const auth = parseMintAuth(body)
|
|
131
|
+
if (!auth) { deny(res, 400, 'bad-request'); return }
|
|
132
|
+
// Field gates mirror `handlePutBegin`'s up-front rejects.
|
|
133
|
+
const expectedLength = (body as { expectedLength?: unknown }).expectedLength
|
|
134
|
+
if (!Number.isSafeInteger(expectedLength) || (expectedLength as number) < 0 || (expectedLength as number) > MAX_CONTENT_LENGTH) {
|
|
135
|
+
deny(res, 400, 'bad-request'); return
|
|
136
|
+
}
|
|
137
|
+
const prevVersionRaw = (body as { prevVersion?: unknown }).prevVersion
|
|
138
|
+
if (prevVersionRaw != null && (typeof prevVersionRaw !== 'number' || !Number.isSafeInteger(prevVersionRaw))) {
|
|
139
|
+
deny(res, 400, 'bad-request'); return
|
|
140
|
+
}
|
|
141
|
+
const prevVersion = typeof prevVersionRaw === 'number' ? prevVersionRaw : null
|
|
142
|
+
const prevIncarnationRaw = (body as { prevIncarnation?: unknown }).prevIncarnation
|
|
143
|
+
const prevIncarnation = typeof prevIncarnationRaw === 'string' ? prevIncarnationRaw : null
|
|
144
|
+
// prevVersion/prevIncarnation travel as a null-iff-null pair (matches the
|
|
145
|
+
// WS `validPrevPair`): a half-pair is malformed.
|
|
146
|
+
if ((prevVersion === null) !== (prevIncarnation === null)) { deny(res, 400, 'bad-request'); return }
|
|
147
|
+
// A non-null incarnation must be the wire shape (matches the WS path's
|
|
148
|
+
// `isValidIncarnation` gate). Signature-covered + CAS-checked regardless,
|
|
149
|
+
// but reject the obviously-malformed up front.
|
|
150
|
+
if (prevIncarnation !== null && !isValidIncarnation(prevIncarnation)) { deny(res, 400, 'bad-request'); return }
|
|
151
|
+
const contentHash = (body as { contentHash?: unknown }).contentHash
|
|
152
|
+
if (!isValidContentHash(contentHash)) { deny(res, 400, 'bad-request'); return }
|
|
153
|
+
|
|
154
|
+
const fields = {
|
|
155
|
+
workspaceTag: route.tag, resourceTag: route.resourceTag,
|
|
156
|
+
prevVersion, prevIncarnation, contentHash, expectedLength: expectedLength as number,
|
|
157
|
+
}
|
|
158
|
+
if (!await verifyObjstorePutBeginRestSig(fields, auth.ts, auth.signature)) { deny(res, 401, 'unauthorized'); return }
|
|
159
|
+
if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
|
|
160
|
+
// New-workspace operator gate — see the function header. Refusing here
|
|
161
|
+
// routes the client to its in-band WS put-begin fallback.
|
|
162
|
+
if (await deps.restPutGate(route.tag)) { deny(res, 401, 'unauthorized'); return }
|
|
163
|
+
const result = await beginPut(deps.handle, {
|
|
164
|
+
workspaceTag: route.tag, resourceTag: route.resourceTag,
|
|
165
|
+
prevVersion, prevIncarnation, expectedLength: expectedLength as number,
|
|
166
|
+
contentHash, signature: auth.signature,
|
|
167
|
+
})
|
|
168
|
+
if (!result.ok) {
|
|
169
|
+
if (result.reason === 'workspace-full') { deny(res, 403, 'workspace-full'); return }
|
|
170
|
+
denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null); return
|
|
171
|
+
}
|
|
172
|
+
const { token, exp } = mintPutToken(deps.secret, route.tag, route.resourceTag, result.stagingId, expectedLength as number)
|
|
173
|
+
res.writeHead(200, { 'content-type': 'application/json' })
|
|
174
|
+
res.end(JSON.stringify({
|
|
175
|
+
stagingId: result.stagingId,
|
|
176
|
+
urlPath: `/api/objstore/${route.tag}/${route.resourceTag}`,
|
|
177
|
+
token,
|
|
178
|
+
expiresAt: exp,
|
|
179
|
+
}))
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// op:'delete' — mirrors the WS `handleDelete`: verify the signed delete
|
|
183
|
+
// fields, freshness/replay-guard, then `deleteObject` (a precondition-checked
|
|
184
|
+
// version-CAS drop). Unlike put, there is NO operator gate — delete is
|
|
185
|
+
// signature-gated, idempotent, and creates nothing (the WS handleDelete has
|
|
186
|
+
// no authGate either). Returns 200 `{ deletedVersion }` (0 = idempotent
|
|
187
|
+
// no-op on a missing row with a null precondition); 409 conflict (stale
|
|
188
|
+
// prevVersion/incarnation); 404 not-found (a non-null precondition against a
|
|
189
|
+
// missing row). On a real drop it broadcasts `objstore-deleted` to
|
|
190
|
+
// subscribers (+ cross-instance), exactly like the WS path.
|
|
191
|
+
async function handleRestDelete(
|
|
192
|
+
deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
|
|
193
|
+
): Promise<void> {
|
|
194
|
+
const auth = parseMintAuth(body)
|
|
195
|
+
if (!auth) { deny(res, 400, 'bad-request'); return }
|
|
196
|
+
const prevVersionRaw = (body as { prevVersion?: unknown }).prevVersion
|
|
197
|
+
if (prevVersionRaw != null && (typeof prevVersionRaw !== 'number' || !Number.isSafeInteger(prevVersionRaw))) {
|
|
198
|
+
deny(res, 400, 'bad-request'); return
|
|
199
|
+
}
|
|
200
|
+
const prevVersion = typeof prevVersionRaw === 'number' ? prevVersionRaw : null
|
|
201
|
+
const prevIncarnationRaw = (body as { prevIncarnation?: unknown }).prevIncarnation
|
|
202
|
+
const prevIncarnation = typeof prevIncarnationRaw === 'string' ? prevIncarnationRaw : null
|
|
203
|
+
// null-iff-null pair (matches the WS `validPrevPair`); a non-null
|
|
204
|
+
// incarnation must be the wire shape.
|
|
205
|
+
if ((prevVersion === null) !== (prevIncarnation === null)) { deny(res, 400, 'bad-request'); return }
|
|
206
|
+
if (prevIncarnation !== null && !isValidIncarnation(prevIncarnation)) { deny(res, 400, 'bad-request'); return }
|
|
207
|
+
|
|
208
|
+
const fields = { workspaceTag: route.tag, resourceTag: route.resourceTag, prevVersion, prevIncarnation }
|
|
209
|
+
if (!await verifyObjstoreDeleteRestSig(fields, auth.ts, auth.signature)) { deny(res, 401, 'unauthorized'); return }
|
|
210
|
+
if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
|
|
211
|
+
const result = await deleteObject(deps.handle, route.tag, route.resourceTag, prevVersion, prevIncarnation)
|
|
212
|
+
if (!result.ok) {
|
|
213
|
+
if (result.reason === 'conflict') { denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null); return }
|
|
214
|
+
deny(res, 404, 'not-found'); return
|
|
215
|
+
}
|
|
216
|
+
res.writeHead(200, { 'content-type': 'application/json' })
|
|
217
|
+
res.end(JSON.stringify({ deletedVersion: result.deletedVersion }))
|
|
218
|
+
// deletedVersion 0 = nothing was live → nothing to broadcast. A real drop
|
|
219
|
+
// fans out to subscribers (including the originator, matching the WS path's
|
|
220
|
+
// `except: null`) so peers' `onDeleted` fire.
|
|
221
|
+
if (result.deletedVersion === 0) return
|
|
222
|
+
deps.broadcast(route.tag, { type: 'objstore-deleted', workspaceTag: route.tag, resourceTag: route.resourceTag, version: result.deletedVersion }, null)
|
|
223
|
+
deps.publishObjDeleted(route.tag, route.resourceTag, result.deletedVersion)
|
|
224
|
+
}
|
package/server/objstore/rest.ts
CHANGED
|
@@ -41,6 +41,8 @@ import {
|
|
|
41
41
|
} from './store.ts'
|
|
42
42
|
import type { LiveReader } from './blob.ts'
|
|
43
43
|
import { type TokenSecret, extractBearer, verifyToken } from './tokens.ts'
|
|
44
|
+
import { deny, denyConflict } from './rest-deny.ts'
|
|
45
|
+
import { handleRestMint } from './rest-mint.ts'
|
|
44
46
|
import { debugId, debugTag, errMsg, errStack } from '../util.ts'
|
|
45
47
|
|
|
46
48
|
// Server-side fault codes that should surface as 500 `io-error`
|
|
@@ -66,6 +68,18 @@ export type ObjstoreRestDeps = {
|
|
|
66
68
|
// workspace_object for the full metadata (version, hash, length,
|
|
67
69
|
// signature). SQLite mode passes a no-op.
|
|
68
70
|
publishObjPut: (tag: string, resourceTag: string) => void
|
|
71
|
+
// Cross-instance pub/sub for objstore-deleted — fired alongside the local
|
|
72
|
+
// `broadcast` after a successful REST delete mint, so peers on OTHER
|
|
73
|
+
// instances drop the resource in real time. Carries (tag, resourceTag,
|
|
74
|
+
// version) inline (the row is gone post-delete). SQLite mode passes a no-op.
|
|
75
|
+
publishObjDeleted: (tag: string, resourceTag: string, version: number) => void
|
|
76
|
+
// New-workspace operator gate for the REST put-begin mint — the
|
|
77
|
+
// connection-independent analog of the WS path's `authGate`. Returns
|
|
78
|
+
// `true` to DENY (a password is configured AND the workspace is
|
|
79
|
+
// never-before-seen), which routes the client to its in-band WS
|
|
80
|
+
// put-begin fallback (REST has no socket auth state to consult).
|
|
81
|
+
// No-config default is open (never deny), matching the WS authGate.
|
|
82
|
+
restPutGate: (workspaceTag: string) => Promise<boolean>
|
|
69
83
|
debug: boolean
|
|
70
84
|
}
|
|
71
85
|
|
|
@@ -116,34 +130,21 @@ export function matchRoute(url: string | undefined): RouteMatch | null {
|
|
|
116
130
|
return { tag: tag!, resourceTag: resourceTag! }
|
|
117
131
|
}
|
|
118
132
|
|
|
119
|
-
function deny(res: ServerResponse, status: number, body: string): void {
|
|
120
|
-
// Uniform `{ error: <reason> }` JSON envelope for every failure so
|
|
121
|
-
// clients parse one shape. Status + reason are NOT intentionally
|
|
122
|
-
// indistinguishable across causes — 401/404/405/410/411/500 each map
|
|
123
|
-
// to a documented reason in server/README.md and the client decides
|
|
124
|
-
// recovery from the code. Probe-distinguishing defense isn't a goal:
|
|
125
|
-
// every reason is reachable only after the route + bearer-token check
|
|
126
|
-
// passes (or as 401/404 from the public surface), so a probe gains no
|
|
127
|
-
// signal it couldn't otherwise enumerate.
|
|
128
|
-
res.writeHead(status, { 'content-type': 'application/json' })
|
|
129
|
-
res.end(JSON.stringify({ error: body }))
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
// Variant of `deny` that augments the JSON envelope with the live
|
|
133
|
-
// row's `currentVersion` + `currentIncarnation` so a REST PUT 409 lets
|
|
134
|
-
// the caller rebase onto the right precondition token. Without this the
|
|
135
|
-
// client only learns the slot is occupied — not at what (version,
|
|
136
|
-
// incarnation) — and retries blindly against a non-empty slot, looping
|
|
137
|
-
// indefinitely against a live row. Symmetric with the WS plane's
|
|
138
|
-
// `objstore-conflict` envelope.
|
|
139
|
-
function denyConflict(res: ServerResponse, currentVersion: number | null, currentIncarnation: string | null): void {
|
|
140
|
-
res.writeHead(409, { 'content-type': 'application/json' })
|
|
141
|
-
res.end(JSON.stringify({ error: 'conflict', currentVersion, currentIncarnation }))
|
|
142
|
-
}
|
|
143
|
-
|
|
144
133
|
export async function handleRest(deps: ObjstoreRestDeps, req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
145
134
|
const route = matchRoute(req.url)
|
|
146
135
|
if (!route) { deny(res, 404, 'not-found'); return }
|
|
136
|
+
// POST = REST mint (fetch or put-begin, by body `op`; signature-authed
|
|
137
|
+
// via the JSON body, no bearer token). Dispatched before the bearer-token
|
|
138
|
+
// gate below, which guards the token-authed GET/PUT byte transfers.
|
|
139
|
+
if (req.method === 'POST') {
|
|
140
|
+
try { await handleRestMint(deps, req, res, route) }
|
|
141
|
+
catch (err: unknown) {
|
|
142
|
+
if (deps.debug) console.warn('objstore POST error:', errStack(err))
|
|
143
|
+
if (res.headersSent) res.destroy()
|
|
144
|
+
else deny(res, 500, 'internal')
|
|
145
|
+
}
|
|
146
|
+
return
|
|
147
|
+
}
|
|
147
148
|
const token = extractBearer(req.headers['authorization'])
|
|
148
149
|
if (!token) { deny(res, 401, 'unauthorized'); return }
|
|
149
150
|
const payload = verifyToken(deps.secret, token)
|
package/server/objstore/sign.ts
CHANGED
|
@@ -10,6 +10,18 @@ import { isValidIncarnation } from './store.ts'
|
|
|
10
10
|
const OBJSTORE_PUT_DOMAIN = 'deepview-objstore.v1.put'
|
|
11
11
|
const OBJSTORE_DELETE_DOMAIN = 'deepview-objstore.v1.delete'
|
|
12
12
|
const OBJSTORE_FETCH_DOMAIN = 'deepview-objstore.v1.fetch'
|
|
13
|
+
// REST fetch-mint domain — MUST match the client's FETCH_REST_DOMAIN
|
|
14
|
+
// (client/sync/objstore-crypto.ts). Distinct from OBJSTORE_FETCH_DOMAIN so
|
|
15
|
+
// a WS-fetch signature can't be replayed against the REST mint endpoint.
|
|
16
|
+
const OBJSTORE_FETCH_REST_DOMAIN = 'deepview-objstore.v1.fetch-rest'
|
|
17
|
+
// REST put-begin domain — MUST match the client's PUT_REST_DOMAIN. Distinct
|
|
18
|
+
// from OBJSTORE_PUT_DOMAIN so a WS put-begin signature can't be replayed
|
|
19
|
+
// against the REST mint endpoint.
|
|
20
|
+
const OBJSTORE_PUT_REST_DOMAIN = 'deepview-objstore.v1.put-rest'
|
|
21
|
+
// REST delete domain — MUST match the client's DELETE_REST_DOMAIN. Distinct
|
|
22
|
+
// from OBJSTORE_DELETE_DOMAIN so a WS delete signature can't be replayed
|
|
23
|
+
// against the REST mint endpoint.
|
|
24
|
+
const OBJSTORE_DELETE_REST_DOMAIN = 'deepview-objstore.v1.delete-rest'
|
|
13
25
|
|
|
14
26
|
// Wire shapes the verifiers accept. Fields land here post-
|
|
15
27
|
// `JSON.parse`, so every value starts life as `unknown` — strict
|
|
@@ -95,6 +107,38 @@ function canonicalObjstoreFetch(msg: ObjstoreFetchMsg, connectionNonce: string):
|
|
|
95
107
|
].join('\n'))
|
|
96
108
|
}
|
|
97
109
|
|
|
110
|
+
// REST fetch-mint canonical. Binds a client epoch-ms timestamp (string-
|
|
111
|
+
// encoded to match the client) in place of the connection nonce; the REST
|
|
112
|
+
// handler enforces the freshness window + replay dedup.
|
|
113
|
+
function canonicalObjstoreFetchRest(workspaceTag: string, resourceTag: string, ts: number): Uint8Array<ArrayBuffer> {
|
|
114
|
+
return encodeUtf8([
|
|
115
|
+
OBJSTORE_FETCH_REST_DOMAIN,
|
|
116
|
+
workspaceTag,
|
|
117
|
+
resourceTag,
|
|
118
|
+
String(ts),
|
|
119
|
+
].join('\n'))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// REST put-begin canonical — the WS `canonicalObjstorePut` fields in the
|
|
123
|
+
// same order/coercion (intOrEmpty / strOrEmpty), under the put-rest domain
|
|
124
|
+
// and binding the client `ts` in place of the connection nonce.
|
|
125
|
+
function canonicalObjstorePutRest(
|
|
126
|
+
workspaceTag: string, resourceTag: string,
|
|
127
|
+
prevVersion: number | null, prevIncarnation: string | null,
|
|
128
|
+
contentHash: string, expectedLength: number, ts: number,
|
|
129
|
+
): Uint8Array<ArrayBuffer> {
|
|
130
|
+
return encodeUtf8([
|
|
131
|
+
OBJSTORE_PUT_REST_DOMAIN,
|
|
132
|
+
workspaceTag,
|
|
133
|
+
resourceTag,
|
|
134
|
+
intOrEmpty(prevVersion),
|
|
135
|
+
strOrEmpty(prevIncarnation),
|
|
136
|
+
contentHash,
|
|
137
|
+
String(expectedLength),
|
|
138
|
+
String(ts),
|
|
139
|
+
].join('\n'))
|
|
140
|
+
}
|
|
141
|
+
|
|
98
142
|
// `Number.isSafeInteger` rather than `Number.isInteger`: JSON numbers
|
|
99
143
|
// are IEEE-754 and integers above 2^53-1 aren't precisely
|
|
100
144
|
// representable. Accepting non-safe integers would let a signed
|
|
@@ -162,3 +206,64 @@ export function verifyObjstoreFetchSig(msg: ObjstoreFetchMsg, connectionNonce: u
|
|
|
162
206
|
if (typeof msg.resourceTag !== 'string') return Promise.resolve(false)
|
|
163
207
|
return verifyObjstoreSig(msg, connectionNonce, (nonce) => canonicalObjstoreFetch(msg, nonce))
|
|
164
208
|
}
|
|
209
|
+
|
|
210
|
+
// Verify a REST fetch-mint signature. Unlike the WS verifiers this takes
|
|
211
|
+
// the already-validated fields directly (the REST handler parsed +
|
|
212
|
+
// range-checked `ts` and `signature`) rather than a wire `msg` + socket
|
|
213
|
+
// nonce. The workspaceTag IS the Ed25519 public key, so verification is
|
|
214
|
+
// fully self-contained — no session or stored key needed. A throw from
|
|
215
|
+
// the canonical build (lone surrogate, etc.) is treated as a verify
|
|
216
|
+
// failure, never an escaping exception.
|
|
217
|
+
export function verifyObjstoreFetchRestSig(
|
|
218
|
+
workspaceTag: string, resourceTag: string, ts: number, signature: string,
|
|
219
|
+
): Promise<boolean> {
|
|
220
|
+
let payload: Uint8Array<ArrayBuffer>
|
|
221
|
+
try { payload = canonicalObjstoreFetchRest(workspaceTag, resourceTag, ts) }
|
|
222
|
+
catch { return Promise.resolve(false) }
|
|
223
|
+
return verifyEd25519(workspaceTag, payload, signature)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Verify a REST put-begin signature. Same self-contained shape as
|
|
227
|
+
// `verifyObjstoreFetchRestSig` — the caller (rest.ts) has already
|
|
228
|
+
// range-checked the put fields + `ts`. The workspaceTag IS the pubkey.
|
|
229
|
+
export function verifyObjstorePutBeginRestSig(
|
|
230
|
+
fields: { workspaceTag: string; resourceTag: string; prevVersion: number | null; prevIncarnation: string | null; contentHash: string; expectedLength: number },
|
|
231
|
+
ts: number, signature: string,
|
|
232
|
+
): Promise<boolean> {
|
|
233
|
+
let payload: Uint8Array<ArrayBuffer>
|
|
234
|
+
try {
|
|
235
|
+
payload = canonicalObjstorePutRest(
|
|
236
|
+
fields.workspaceTag, fields.resourceTag, fields.prevVersion, fields.prevIncarnation,
|
|
237
|
+
fields.contentHash, fields.expectedLength, ts,
|
|
238
|
+
)
|
|
239
|
+
} catch { return Promise.resolve(false) }
|
|
240
|
+
return verifyEd25519(fields.workspaceTag, payload, signature)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// REST delete-mint canonical. WS `canonicalObjstoreDelete` fields, under the
|
|
244
|
+
// delete-rest domain, binding the client `ts` in place of the nonce.
|
|
245
|
+
function canonicalObjstoreDeleteRest(
|
|
246
|
+
workspaceTag: string, resourceTag: string,
|
|
247
|
+
prevVersion: number | null, prevIncarnation: string | null, ts: number,
|
|
248
|
+
): Uint8Array<ArrayBuffer> {
|
|
249
|
+
return encodeUtf8([
|
|
250
|
+
OBJSTORE_DELETE_REST_DOMAIN,
|
|
251
|
+
workspaceTag,
|
|
252
|
+
resourceTag,
|
|
253
|
+
intOrEmpty(prevVersion),
|
|
254
|
+
strOrEmpty(prevIncarnation),
|
|
255
|
+
String(ts),
|
|
256
|
+
].join('\n'))
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// Verify a REST delete-mint signature. Self-contained (workspaceTag IS the
|
|
260
|
+
// pubkey); the caller (rest.ts) has range-checked `ts` + the prev pair.
|
|
261
|
+
export function verifyObjstoreDeleteRestSig(
|
|
262
|
+
fields: { workspaceTag: string; resourceTag: string; prevVersion: number | null; prevIncarnation: string | null },
|
|
263
|
+
ts: number, signature: string,
|
|
264
|
+
): Promise<boolean> {
|
|
265
|
+
let payload: Uint8Array<ArrayBuffer>
|
|
266
|
+
try { payload = canonicalObjstoreDeleteRest(fields.workspaceTag, fields.resourceTag, fields.prevVersion, fields.prevIncarnation, ts) }
|
|
267
|
+
catch { return Promise.resolve(false) }
|
|
268
|
+
return verifyEd25519(fields.workspaceTag, payload, signature)
|
|
269
|
+
}
|
package/server/sse-server.ts
CHANGED
|
@@ -52,6 +52,24 @@ import { errMsg, randomId } from './util.ts'
|
|
|
52
52
|
// path so the same `location` block routes both.
|
|
53
53
|
export const SSE_OPEN_PATH = '/api/sync/sse'
|
|
54
54
|
|
|
55
|
+
// Cadence of the server-driven keepalive sweep: every tick we write a `:`
|
|
56
|
+
// comment to each open session's downstream so intermediary proxies don't
|
|
57
|
+
// idle-close it (nginx et al. default to a ~60s read timeout). This is the
|
|
58
|
+
// server's own liveness upkeep — the client no longer POSTs a periodic ping
|
|
59
|
+
// (which forced a stream takeover every tick).
|
|
60
|
+
//
|
|
61
|
+
// Reaping model (replaces the old POST-driven idle timer): a session is
|
|
62
|
+
// dropped on its downstream response `close` — clean disconnect, or a
|
|
63
|
+
// half-open socket the per-session TCP keepalive forces closed (see
|
|
64
|
+
// SseSession.SOCKET_KEEPALIVE_MS) — NOT by this sweep. We intentionally
|
|
65
|
+
// trust connection-level liveness. The one topology this can't see is a
|
|
66
|
+
// buffering / TLS-terminating proxy that holds the upstream open after the
|
|
67
|
+
// real client vanished (keepalive then probes the proxy hop, not the
|
|
68
|
+
// client); such a session lingers until `maxSessions`, the hard backstop.
|
|
69
|
+
// This is the same exposure the WS heartbeat already has (its ping only
|
|
70
|
+
// proves the proxy↔server hop too), not a new class of leak.
|
|
71
|
+
const KEEPALIVE_SWEEP_MS = 30_000
|
|
72
|
+
|
|
55
73
|
export type SseServerDeps = {
|
|
56
74
|
// The WS dispatch is the cohesive unit; SSE just provides another
|
|
57
75
|
// transport into it. Closure over the same handler / hub / objstore
|
|
@@ -70,12 +88,6 @@ export type SseServerDeps = {
|
|
|
70
88
|
// the per-frame budget is the same as the WS plane after the
|
|
71
89
|
// dispatcher splits them.
|
|
72
90
|
maxBodyBytes: number
|
|
73
|
-
// Idle timeout for a session with no inbound POSTs. Detects the
|
|
74
|
-
// wandered-off browser tab the WS heartbeat sweep handles via
|
|
75
|
-
// ping/pong on real sockets. The client's JSON ping/pong heartbeat
|
|
76
|
-
// (every 15s) is the steady-state liveness signal; this is the
|
|
77
|
-
// hard ceiling.
|
|
78
|
-
sessionIdleMs: number
|
|
79
91
|
debug: boolean
|
|
80
92
|
}
|
|
81
93
|
|
|
@@ -86,6 +98,9 @@ export type SseServer = {
|
|
|
86
98
|
// Iterates active sessions. Lifecycle's graceful-shutdown loop
|
|
87
99
|
// reads this to close SSE sessions alongside WS clients.
|
|
88
100
|
sessions: () => Iterable<SseSession>
|
|
101
|
+
// The keepalive-sweep timer. Lifecycle clears it on shutdown (parity
|
|
102
|
+
// with the WS heartbeat timer) so a tick can't fire mid-teardown.
|
|
103
|
+
keepaliveTimer: ReturnType<typeof setInterval>
|
|
89
104
|
}
|
|
90
105
|
|
|
91
106
|
// Inbound POST body. Every field optional — an empty-body POST is a
|
|
@@ -98,7 +113,7 @@ type SseBody = {
|
|
|
98
113
|
}
|
|
99
114
|
|
|
100
115
|
export function installSseServer(deps: SseServerDeps): SseServer {
|
|
101
|
-
const { peerDeps, isShuttingDown, maxSessions, maxBodyBytes,
|
|
116
|
+
const { peerDeps, isShuttingDown, maxSessions, maxBodyBytes, debug } = deps
|
|
102
117
|
|
|
103
118
|
// Active SSE sessions, keyed by the random session id `createSession`
|
|
104
119
|
// mints on the first POST that lacks a `?id=` (or whose id this
|
|
@@ -108,26 +123,9 @@ export function installSseServer(deps: SseServerDeps): SseServer {
|
|
|
108
123
|
// mint a fresh session instead, so a multi-replica deployment doesn't
|
|
109
124
|
// require sticky LB routing to recover.
|
|
110
125
|
const sessions = new Map<string, SseSession>()
|
|
111
|
-
// Per-session idle timer handle. Reset on every inbound POST; fires
|
|
112
|
-
// after `sessionIdleMs` of silence to close a stranded session.
|
|
113
|
-
const idleTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
|
114
|
-
|
|
115
|
-
function armIdleTimer(sid: string, session: SseSession): void {
|
|
116
|
-
clearTimeout(idleTimers.get(sid))
|
|
117
|
-
if (sessionIdleMs <= 0) return
|
|
118
|
-
const t = setTimeout(() => {
|
|
119
|
-
if (debug) console.warn(`sse: session ${sid.slice(0, 8)}… idle ${sessionIdleMs}ms → close`)
|
|
120
|
-
try { session.terminate() } catch {}
|
|
121
|
-
}, sessionIdleMs)
|
|
122
|
-
t.unref?.()
|
|
123
|
-
idleTimers.set(sid, t)
|
|
124
|
-
}
|
|
125
126
|
|
|
126
127
|
function dropSession(sid: string): void {
|
|
127
128
|
sessions.delete(sid)
|
|
128
|
-
const t = idleTimers.get(sid)
|
|
129
|
-
if (t) clearTimeout(t)
|
|
130
|
-
idleTimers.delete(sid)
|
|
131
129
|
}
|
|
132
130
|
|
|
133
131
|
function writeSseHeaders(res: ServerResponse): void {
|
|
@@ -149,7 +147,7 @@ export function installSseServer(deps: SseServerDeps): SseServer {
|
|
|
149
147
|
res.write('retry: 1000\n\n')
|
|
150
148
|
}
|
|
151
149
|
|
|
152
|
-
function createSession(res: ServerResponse, req: HttpRequest):
|
|
150
|
+
function createSession(res: ServerResponse, req: HttpRequest): SseSession | null {
|
|
153
151
|
if (sessions.size >= maxSessions) {
|
|
154
152
|
if (debug) console.warn(`sse: refused open — sessions ${sessions.size} >= ${maxSessions}`)
|
|
155
153
|
return null
|
|
@@ -158,7 +156,6 @@ export function installSseServer(deps: SseServerDeps): SseServer {
|
|
|
158
156
|
const sid = randomId()
|
|
159
157
|
const session = new SseSession(res)
|
|
160
158
|
sessions.set(sid, session)
|
|
161
|
-
armIdleTimer(sid, session)
|
|
162
159
|
session.on('close', () => { dropSession(sid) })
|
|
163
160
|
// Announce the continuation token BEFORE the dispatcher emits its
|
|
164
161
|
// `challenge` frame so the client latches the id first and the
|
|
@@ -171,7 +168,7 @@ export function installSseServer(deps: SseServerDeps): SseServer {
|
|
|
171
168
|
// `setupPeerConnection`'s first action is the protocol `challenge`
|
|
172
169
|
// frame, on the default-named SSE channel.
|
|
173
170
|
setupPeerConnection(session as unknown as WebSocket, req, peerDeps)
|
|
174
|
-
return
|
|
171
|
+
return session
|
|
175
172
|
}
|
|
176
173
|
|
|
177
174
|
// Drives a POST body's `password` and `frames` through the shared
|
|
@@ -268,19 +265,16 @@ export function installSseServer(deps: SseServerDeps): SseServer {
|
|
|
268
265
|
// through to createSession with `res` still header-virgin, so
|
|
269
266
|
// the new session can writeSseHeaders without ERR_HTTP_HEADERS_SENT.
|
|
270
267
|
let session: SseSession | null = null
|
|
271
|
-
let sid: string | null = null
|
|
272
268
|
if (sidFromUrl) {
|
|
273
269
|
const existing = sessions.get(sidFromUrl)
|
|
274
270
|
if (existing && existing.readyState === existing.OPEN && existing.attachResponse(res)) {
|
|
275
271
|
writeSseHeaders(res)
|
|
276
272
|
session = existing
|
|
277
|
-
sid = sidFromUrl
|
|
278
|
-
armIdleTimer(sid, session)
|
|
279
273
|
}
|
|
280
274
|
}
|
|
281
275
|
if (!session) {
|
|
282
|
-
|
|
283
|
-
if (!
|
|
276
|
+
session = createSession(res, req)
|
|
277
|
+
if (!session) {
|
|
284
278
|
// Cap exceeded; createSession already logged. Response
|
|
285
279
|
// headers not yet written by writeSseHeaders, so send a
|
|
286
280
|
// 503 JSON instead.
|
|
@@ -288,8 +282,6 @@ export function installSseServer(deps: SseServerDeps): SseServer {
|
|
|
288
282
|
res.end(JSON.stringify({ error: 'too-many-sessions' }))
|
|
289
283
|
return
|
|
290
284
|
}
|
|
291
|
-
session = created.session
|
|
292
|
-
sid = created.sid
|
|
293
285
|
}
|
|
294
286
|
dispatchBody(session, body)
|
|
295
287
|
// Do NOT res.end() — the response stays open as the session's
|
|
@@ -323,7 +315,22 @@ export function installSseServer(deps: SseServerDeps): SseServer {
|
|
|
323
315
|
return true
|
|
324
316
|
}
|
|
325
317
|
|
|
326
|
-
|
|
318
|
+
// Server-driven keepalive sweep. Writes a `:` comment to every open
|
|
319
|
+
// session's downstream so proxies don't idle-close it. `unref` so it can't
|
|
320
|
+
// by itself hold the event loop open (parity with the WS heartbeat timer);
|
|
321
|
+
// skipped during shutdown so a tick can't write to a session the close
|
|
322
|
+
// loop is tearing down. Dead-session reaping is the response `close` event
|
|
323
|
+
// (see SseSession.wireResponse + the per-session TCP keepalive), NOT this
|
|
324
|
+
// sweep — so a half-open client is dropped without ever POSTing.
|
|
325
|
+
const keepaliveTimer = setInterval(() => {
|
|
326
|
+
if (isShuttingDown()) return
|
|
327
|
+
for (const session of sessions.values()) {
|
|
328
|
+
try { session.ping() } catch {}
|
|
329
|
+
}
|
|
330
|
+
}, KEEPALIVE_SWEEP_MS)
|
|
331
|
+
keepaliveTimer.unref?.()
|
|
332
|
+
|
|
333
|
+
return { handle, sessions: () => sessions.values(), keepaliveTimer }
|
|
327
334
|
}
|
|
328
335
|
|
|
329
336
|
// Bare-bones query parse for `id=<base64url>`. Avoids URLSearchParams
|
package/server/sse-session.ts
CHANGED
|
@@ -28,6 +28,20 @@ import { EventEmitter } from 'node:events'
|
|
|
28
28
|
import type { Buffer } from 'node:buffer'
|
|
29
29
|
import type { ServerResponse } from 'node:http'
|
|
30
30
|
|
|
31
|
+
// TCP keepalive idle delay for the downstream socket. With the client no
|
|
32
|
+
// longer POSTing a periodic ping (see client/sync/socket-transport.ts), a
|
|
33
|
+
// QUIET session whose client vanished without a FIN (crash, NAT/idle drop)
|
|
34
|
+
// has no application-level liveness signal — so we lean on the kernel:
|
|
35
|
+
// after this much idle the kernel starts probing, and a dead half-open
|
|
36
|
+
// socket surfaces as a `close`/`error` here. Node's `setKeepAlive` sets
|
|
37
|
+
// only TCP_KEEPIDLE (the delay to the FIRST probe), not the probe
|
|
38
|
+
// interval/count — so full teardown is this delay PLUS the OS's
|
|
39
|
+
// TCP_KEEPINTVL × TCP_KEEPCNT (~minutes on Linux defaults), still bounded
|
|
40
|
+
// and far below the kernel's hours-long default-off behaviour. `maxSessions`
|
|
41
|
+
// is the hard backstop. Behind a TLS-terminating / buffering proxy this
|
|
42
|
+
// probes the proxy hop, not the client — see the reaping note in sse-server.ts.
|
|
43
|
+
const SOCKET_KEEPALIVE_MS = 30_000
|
|
44
|
+
|
|
31
45
|
export class SseSession extends EventEmitter {
|
|
32
46
|
static readonly CONNECTING = 0
|
|
33
47
|
static readonly OPEN = 1
|
|
@@ -62,6 +76,10 @@ export class SseSession extends EventEmitter {
|
|
|
62
76
|
// spurious session 'error' that operators read as a real transport
|
|
63
77
|
// failure on a healthy session.
|
|
64
78
|
private wireResponse(res: ServerResponse): void {
|
|
79
|
+
// Probe half-open downstreams at the kernel level (see SOCKET_KEEPALIVE_MS).
|
|
80
|
+
// A failed probe trips the `close`/`error` handlers below, which is how a
|
|
81
|
+
// dead-but-quiet SSE client is reaped now that there's no client ping.
|
|
82
|
+
try { res.socket?.setKeepAlive(true, SOCKET_KEEPALIVE_MS) } catch {}
|
|
65
83
|
res.on('close', () => {
|
|
66
84
|
if (res !== this.currentRes) return // current-response guard (see above)
|
|
67
85
|
if (this.readyState === SseSession.CLOSED) return
|
|
@@ -142,9 +160,8 @@ export class SseSession extends EventEmitter {
|
|
|
142
160
|
// double-emit), which means without the explicit emit here neither
|
|
143
161
|
// sse-server's dropSession cleanup nor setupPeerConnection's
|
|
144
162
|
// unsubscribeAll/peers.delete would run on any server-initiated
|
|
145
|
-
// teardown — sessions / hub.subscribers
|
|
146
|
-
//
|
|
147
|
-
// is a no-op.
|
|
163
|
+
// teardown — the sessions map / hub.subscribers would leak per close.
|
|
164
|
+
// The wireResponse guard then ensures the later async fire is a no-op.
|
|
148
165
|
close(code?: number, reason?: string): void {
|
|
149
166
|
if (this.readyState === SseSession.CLOSED) return
|
|
150
167
|
const res = this.currentRes
|
|
@@ -173,11 +190,14 @@ export class SseSession extends EventEmitter {
|
|
|
173
190
|
this.emit('close')
|
|
174
191
|
}
|
|
175
192
|
|
|
176
|
-
//
|
|
177
|
-
//
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
//
|
|
193
|
+
// Server-driven keepalive, called on the SSE keepalive sweep in
|
|
194
|
+
// sse-server.ts. SSE has no `pong` equivalent, so we write a `:` comment
|
|
195
|
+
// line — ignored by the client parser — that keeps the downstream from
|
|
196
|
+
// being idle-closed by intermediary proxies (nginx et al. default to a
|
|
197
|
+
// ~60s read timeout). Dead-client detection is the response `close` event
|
|
198
|
+
// (clean disconnect, or a half-open socket the TCP keepalive in
|
|
199
|
+
// `wireResponse` forces closed), NOT this write: a `:`-comment write to a
|
|
200
|
+
// half-open socket just buffers, it doesn't synchronously throw.
|
|
181
201
|
ping(): void {
|
|
182
202
|
if (this.readyState !== SseSession.OPEN) return
|
|
183
203
|
const res = this.currentRes
|