@dokebi/websocket-hub 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +96 -49
- package/README.md +71 -43
- package/dist/delivery.d.ts +13 -47
- package/dist/delivery.js +19 -177
- package/dist/dependency.d.ts +4 -0
- package/dist/dependency.js +14 -0
- package/dist/event-queue.d.ts +6 -0
- package/dist/event-queue.js +21 -3
- package/dist/eviction.d.ts +9 -0
- package/dist/eviction.js +57 -0
- package/dist/hub-index.d.ts +29 -0
- package/dist/hub-index.js +146 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.js +25 -18
- package/dist/leader.d.ts +3 -1
- package/dist/leader.js +215 -76
- package/dist/local.d.ts +11 -7
- package/dist/local.js +94 -107
- package/dist/node.d.ts +2 -2
- package/dist/node.js +83 -21
- package/dist/options.d.ts +32 -34
- package/dist/options.js +115 -120
- package/dist/peer-client.d.ts +13 -4
- package/dist/peer-client.js +290 -172
- package/dist/protocol.d.ts +6 -0
- package/dist/protocol.js +7 -1
- package/dist/status-cli.js +7 -2
- package/dist/store.js +12 -7
- package/dist/types.d.ts +25 -8
- package/dist/types.js +22 -5
- package/dist/values.d.ts +4 -0
- package/dist/values.js +32 -0
- package/dist/wire.d.ts +8 -1
- package/dist/wire.js +14 -3
- package/options.schema.json +113 -36
- package/package.json +35 -11
package/AI.md
CHANGED
|
@@ -1,76 +1,123 @@
|
|
|
1
|
-
# AI
|
|
1
|
+
# AI guide — @dokebi/websocket-hub 0.4.0
|
|
2
2
|
|
|
3
|
-
WebSocket
|
|
3
|
+
The hub that manages WebSocket connection lists, rooms, broadcasts and kicks (it replaces the old `dokebi-sync`). A broadcast is **at most once**, with no receipt ACK or resend ([D-0057](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0057-websocket-broadcast-without-ack.md)). Usually **@dokebi/websocket uses it alone**, and app code uses websocket's `defineSocket`. Direct use is mostly operational status (`status()`, `dokebi-hub-status`) and replay defense for sealed frames (`nonces()`). Node >=24, core ^0.4.0, peers: @dokebi/redis, @dokebi/nats (cluster mode). Details are in the [README](README.md).
|
|
4
4
|
|
|
5
|
-
## 1.
|
|
5
|
+
## 1. Modes ([D-0029](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0029-hub-and-cluster-infrastructure.md))
|
|
6
6
|
|
|
7
|
-
| | `mode: local`(
|
|
7
|
+
| | `mode: local` (default) | `mode: cluster` |
|
|
8
8
|
|---|---|---|
|
|
9
|
-
|
|
|
10
|
-
|
|
|
11
|
-
|
|
|
12
|
-
| cluster
|
|
9
|
+
| Scope | one process (memory) | shared by processes and servers |
|
|
10
|
+
| Needs | nothing | Redis (required), NATS (optional: instant leader change notices; otherwise Redis polling) |
|
|
11
|
+
| Failure | state disappears with the process, no handover | every worker is a hub candidate, the leader is elected by a Redis lease, and when the leader dies real sockets stay and reconnect to the new hub |
|
|
12
|
+
| cluster workers | refused with `LOCAL_IN_CLUSTER` (allowed only with `allowInCluster: true`) | normal |
|
|
13
13
|
|
|
14
|
-
API
|
|
14
|
+
The API and broadcast semantics (at most once) are the same in both modes. `mode` keeps its name in 0.4.0: it names the deployment style, not a driver ([D-0071](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0071-option-vocabulary.md)).
|
|
15
15
|
|
|
16
|
-
## 2.
|
|
16
|
+
## 2. Configuration `options.dokebi-websocket-hub`
|
|
17
17
|
|
|
18
18
|
```yaml
|
|
19
19
|
dokebi-websocket-hub:
|
|
20
20
|
mode: cluster
|
|
21
|
-
namespace: shop #
|
|
22
|
-
serverId: web-a #
|
|
23
|
-
host: 0.0.0.0 #
|
|
21
|
+
namespace: shop # servers of the same namespace share a hub
|
|
22
|
+
serverId: web-a # different per server
|
|
23
|
+
host: 0.0.0.0 # hub TCP listen address
|
|
24
24
|
port: 0
|
|
25
|
-
advertiseHost: web-a.internal #
|
|
25
|
+
advertiseHost: web-a.internal # the address other servers use to reach this server's hub
|
|
26
26
|
redisConnection: primary
|
|
27
|
-
natsConnection: primary #
|
|
28
|
-
secret: (32
|
|
27
|
+
natsConnection: primary # optional
|
|
28
|
+
secret: (32+ random bytes, only in dokebi.yaml) # the same secret for the same namespace
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
|
|
|
32
|
-
|
|
33
|
-
| `
|
|
34
|
-
| `
|
|
35
|
-
| `
|
|
36
|
-
| `
|
|
37
|
-
| `
|
|
38
|
-
| `
|
|
39
|
-
| `
|
|
31
|
+
| Key | Type | Default | Description |
|
|
32
|
+
|---|---|---|---|
|
|
33
|
+
| `mode` | `local` \| `cluster` | `local` | §1 |
|
|
34
|
+
| `allowInCluster` | boolean | false | Allow local mode in a cluster worker |
|
|
35
|
+
| `namespace` / `serverId` | string ≤ 64 | `local` / `local` | Shared area / this server's id (`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) |
|
|
36
|
+
| `redisConnection` / `natsConnection` | string | — | dokebi-redis (required in cluster) / dokebi-nats (optional) connection names |
|
|
37
|
+
| `host` / `advertiseHost` / `port` | string / string / integer 0–65535 | `127.0.0.1` / `127.0.0.1` / 0 | Hub TCP listen address, address announced to peers, port (0 = any) |
|
|
38
|
+
| `secret` | string, secret | — (required in cluster) | 32–256 characters and 32–256 bytes |
|
|
39
|
+
| `leaseMs` / `pollIntervalMs` | integer ms | 3000 / 500 | Leader lease / polling without NATS |
|
|
40
|
+
| `ownerTtlMs` / `startupTimeoutMs` | integer ms | 15000 / 15000 | Server registration expiry / startup wait |
|
|
41
|
+
| `maxFrameBytes` / `maxPayloadBytes` | integer | 1048576 / 65536 | Hub TCP frame / message |
|
|
42
|
+
| `maxConnections` / `maxOwners` / `maxTotalConnections` | integer | 128 / 64 / 8192 | Per server / number of servers / all connections |
|
|
43
|
+
| `maxPending` / `maxPendingBytes` | integer | 128 / 4194304 | Pending limits (more broadcasts in progress than this give `BUSY`) |
|
|
44
|
+
|
|
45
|
+
Keys renamed in 0.4.0 (D-0071). The old name is accepted with the warning `OPTION_DEPRECATED`; old and new together fail with `INVALID_OPTIONS`; 0.5.0 rejects it.
|
|
46
|
+
|
|
47
|
+
| Old key | New key |
|
|
48
|
+
|---|---|
|
|
49
|
+
| `pollMs` | `pollIntervalMs` |
|
|
50
|
+
| `startupMs` | `startupTimeoutMs` |
|
|
51
|
+
| `queueBytes` | `maxPendingBytes` |
|
|
52
|
+
|
|
53
|
+
**Removed options (0.3.0, D-0057)**: `deliveryTimeoutMs`, `retryMs`, `maxDeliveries`, `deliveryBytes`. Start still works when they remain in dokebi.yaml: each key logs warn `OPTION_REMOVED` (`option`, `reason`; since 0.4.0 core logs it at setup) once and the value is ignored (the schema keeps them as deprecated until the next version drops them). Delete them.
|
|
54
|
+
|
|
55
|
+
**Checks (0.4.0, [D-0063](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0063-core-primitives.md))**: the options are one `defineOptions` declaration in `src/options.ts` that is the parser, and `options.schema.json` is generated from it at build time. Rules that only the hand parser used to keep are in the schema now: connection names (`namespace`, `serverId`, `redisConnection`, `natsConnection`) `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`, no spaces or control characters in `host` and `advertiseHost`, and the defaults the parser filled in (`namespace` and `serverId` = `local`, `host` and `advertiseHost` = `127.0.0.1`). Cross-field rules are unchanged: `pollIntervalMs × 3 ≤ leaseMs`, `ownerTtlMs ≥ leaseMs × 3`, base64(`maxPayloadBytes`) + 8192 ≤ `maxFrameBytes`, `maxPendingBytes ≥ maxFrameBytes`. `secret` must be 32–256 **characters** and 32–256 **bytes** (the old schema counted characters, the parser only bytes). On failure the core `INVALID_OPTIONS` is the cause of `INVALID_PLUGIN_OPTIONS`, with paths and issue codes in `details.issues` (no values): `REDIS_REQUIRED` in cluster mode without a Redis connection, `SECRET_REQUIRED` when the secret is missing or has the wrong byte count, otherwise `required`, `pattern`, `check` and so on.
|
|
56
|
+
|
|
57
|
+
While a server with different settings (secret, limits) is alive in the same namespace, a new one is refused at once with `CONFIG_MISMATCH`; once every server with the old settings is down, the new settings take over.
|
|
40
58
|
|
|
41
|
-
|
|
59
|
+
## 3. API (`import { dependency } from '@dokebi/websocket-hub/dependency'` → `getDependency`)
|
|
42
60
|
|
|
43
|
-
|
|
61
|
+
The provider descriptor lives in the subpath `@dokebi/websocket-hub/dependency` (D-0063 §6; a small module that loads neither the hub nor Redis nor NATS), and the package entry's `dependency` is the same object. Plugins and app code that always install the hub use the subpath. When the hub is optional (an optional peer), do not import it; keep `optionalDependency({ name: 'dokebi-websocket-hub', version: '^0.4.0' })` in your own file.
|
|
44
62
|
|
|
45
|
-
- `state`(starting
|
|
46
|
-
- `register(...)
|
|
47
|
-
- `
|
|
48
|
-
- `
|
|
63
|
+
- `state` (`starting`, `ready`, `recovering`, `closing`, `closed`), `role` (`active`, `standby`), `ownerEpoch`, `leader`, `whenReady(timeoutMs)` → boolean.
|
|
64
|
+
- `register(...)` → `HubConnection` (`identity`, `join`, `leave`, `updateMetadata`, `close`), `list(query)` → `HubSnapshot`, `broadcast(query, bytes)`, `kick(query)`, `events()`: the low-level API websocket uses. A query (`HubQuery`) is a required `tenant` plus optional `room`, `member`, `connection` (AND). **`events()` has a single consumer**; when websocket uses it, never read it separately. The `message` event (`{ id, connections, data }`) is the whole delivery: there is nothing to confirm (`acknowledge` was removed in 0.3.0).
|
|
65
|
+
- `broadcast` → receipt `HubReceipt` `{ id, recipients }`. Failures: `RECOVERING` (not connected to the leader, or the leader is changing; not sent), `OUTCOME_UNKNOWN` (the connection dropped after sending, before the answer; it may have been sent), `BUSY` (broadcasts in progress reached `maxPending`; in local mode, half of the event queue is still unread). The hub never resends. `kick` → `HubKickReceipt` `{ authorityEpoch, id }` (default code 4403).
|
|
66
|
+
- `status()` (D-0038) → `HubStatus` `{ mode, state, role, leaseState?, leader?, leaseRemainingMs?, servers?, connections?, serverId? }`. Unknown values are **left out**, not filled with 0 (`leaseState: 'unreachable'` when Redis does not answer).
|
|
67
|
+
- `nonces()` (D-0038) → `HubNonceStore` `{ shared, setIfNotExists(key, ttlSeconds) }`. cluster shares it between servers through Redis `SET NX PX` (`shared: true`); local keeps it in the process (`shared: false`; used in a cluster worker it warns `HUB_NONCES_NOT_SHARED` once). Pass it to the replay defense of sealed frames (@dokebi/seal `SealTerminator`). Only the shape matches; seal is never imported.
|
|
68
|
+
- Exports: `createPlugin`, `dependency`, `HubError`, types `HubApi`, `HubConnection`, `HubRegistration`, `HubRecord`, `HubQuery`, `HubLeader`, `HubSnapshot`, `HubReceipt`, `HubKickReceipt`, `HubEvent`, `HubLease`, `HubStatus`, `HubNonceStore`.
|
|
49
69
|
|
|
50
|
-
## 4.
|
|
70
|
+
## 4. Operations command `dokebi-hub-status`
|
|
51
71
|
|
|
52
72
|
```sh
|
|
53
|
-
npx dokebi-hub-status --config dokebi.yaml #
|
|
54
|
-
npx dokebi-hub-status --config dokebi.yaml --json #
|
|
73
|
+
npx dokebi-hub-status --config dokebi.yaml # human-readable
|
|
74
|
+
npx dokebi-hub-status --config dokebi.yaml --json # for machines
|
|
55
75
|
```
|
|
56
76
|
|
|
57
|
-
|
|
77
|
+
From outside the app process it reads only Redis and shows the leader, the remaining lease time, the number of live servers and the connection count (it never starts a hub or joins the election). When Redis does not answer it exits with code 1. In local mode it says the state lives only inside the process.
|
|
78
|
+
|
|
79
|
+
## 5. Delivery semantics (D-0057)
|
|
80
|
+
|
|
81
|
+
- **Success = sent out.** cluster: the leader picks the connections matching right now, writes one message frame to each server (owner) holding them, then gives the sending server a receipt. local: it is done once it is in this process's event queue (read by the websocket gateway). There is no receipt confirmation, resend, deduplication, completion record or deadline wait. It is **at most once**.
|
|
82
|
+
- **receipt** `{ id, recipients }`: `recipients` are the identifiers of the connections matching when it was sent (the SHA-256 of tenant, member and clientId, or of owner and connection without a clientId). **It does not mean they received it.** Connections that were reconnecting, that left before the frame was written, or that were closed with 1013 for being slow do not get it. Connections that arrive later do not get it either (nothing is kept).
|
|
83
|
+
- **Failures come at once and are never resent (no duplicates)**: `RECOVERING`, `OUTCOME_UNKNOWN`, `BUSY`. The caller decides whether to send again (resending after `OUTCOME_UNKNOWN` can deliver twice).
|
|
84
|
+
- **App guideline**: never leave work that must not be missed (order state, payment results) to a broadcast. Store it in the DB and let the page read the latest state from the server when it reconnects. Use broadcasts as "it changed, read again" notices. Work that must happen goes to @dokebi/job.
|
|
85
|
+
- **Protocol between hub servers (protocol 2)**: hello carries `protocol: 2`. The leader refuses other versions with `PROTOCOL_MISMATCH` (warn `Hub peer refused; protocol differs`), and the refused server logs error `Hub protocol differs from the leader; retrying` (`protocol`, `leaderProtocol`) and retries. A 0.3 server connected to a 0.2 leader (no version) is cut with the same error. During a rolling restart a server of another version cannot join the leader (while starting, `STARTUP_TIMEOUT` after `startupTimeoutMs`), so upgrade every server to the same version. The configuration fingerprint stays the same as 0.2 so that this version check, not `CONFIG_MISMATCH`, tells them apart.
|
|
86
|
+
- Broadcasts never go through Redis (only connection registrations and changes are stored in Redis, D-0042). No broadcast record stays in leader memory either.
|
|
87
|
+
- presence events come as the full list per owner (server). After a reconnect or leader change, owners that differ from the snapshot and owners that disappeared are announced again.
|
|
88
|
+
|
|
89
|
+
## 6. Index and performance ([D-0068](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0068-performance-stage4.md) §3)
|
|
90
|
+
|
|
91
|
+
- `HubIndex` (`src/hub-index.ts`) keeps tenant → member → connections, tenant → room → connections (insertion-ordered sets), owner → connection → record and recipientKey → record. local updates it on every register, change and close; the leader when the server list changes (`setOwner`/`removeOwner`, rebuilt from the snapshot at start).
|
|
92
|
+
- Selection walks only the smallest matching set (a named connection → one; member and room → the smaller; member; room; the whole tenant) and checks each with `matches`, so the answer equals a full scan (receipt semantics, presence and roster unchanged). The `DUPLICATE_CLIENT` check is O(1); `list` and `kick` use the same index.
|
|
93
|
+
- Evidence: a randomized 2000-step test (add, room change, remove, owner swap) compares every filter with the full scan; `benchmarks/scenarios/micro/hub-select.mjs` measures `select`; `websocket-load.bench.mjs 1024 300` passed with no missed, duplicated or misdirected messages and baseline receive p50/p95/p99 6.8/10.8/17.0 ms.
|
|
94
|
+
|
|
95
|
+
## 7. Errors
|
|
96
|
+
|
|
97
|
+
`HubError` (branch on `code`).
|
|
58
98
|
|
|
59
|
-
|
|
99
|
+
`HubError` is a `DokebiError` (0.4.0, D-0063): `plugin: 'dokebi-websocket-hub'`, and the message stays `WebSocket hub: <code>`. Codes an operator fixes carry a `fix`.
|
|
60
100
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
101
|
+
| Code | When |
|
|
102
|
+
|---|---|
|
|
103
|
+
| `LOCAL_IN_CLUSTER` | local mode started in a node:cluster worker (the role comes from core `context.runtime.role`). `fix`: `mode: cluster` or `allowInCluster: true`. Before 0.4.0 this error was HUB_LOCAL_IN_CLUSTER; the error log keeps that code |
|
|
104
|
+
| `LISTEN_FAILED` · `STARTUP_TIMEOUT` | Could not open the hub TCP · could not join the leader within `startupTimeoutMs` (with `fix`) |
|
|
105
|
+
| `CONFIG_MISMATCH` · `PROTOCOL_MISMATCH` · `UNAUTHORIZED` | Different settings, version or secret in the same namespace (with `fix`) |
|
|
106
|
+
| `RECOVERING` · `OUTCOME_UNKNOWN` · `BUSY` | Broadcast failures (§5) |
|
|
107
|
+
| `STORE_UNAVAILABLE` | Redis failure (the cause is in `cause`) |
|
|
108
|
+
| `INVALID_ARGUMENT` · `INVALID_OPTIONS` | Wrong API arguments or hub frame values (frame value codes are compared between servers, so they stay as in 0.3) |
|
|
109
|
+
| `CONNECTION_LIMIT` · `MESSAGE_TOO_LARGE` · `DUPLICATE_CLIENT` · `DUPLICATE_CONNECTION` · `CLOSED` · `EVENT_OVERFLOW` | Limits, duplicates, shutdown |
|
|
67
110
|
|
|
68
|
-
##
|
|
111
|
+
## 8. Cautions
|
|
69
112
|
|
|
70
|
-
-
|
|
71
|
-
- secret
|
|
72
|
-
-
|
|
73
|
-
-
|
|
74
|
-
-
|
|
113
|
+
- Never trust tenant, member or rooms sent by the client (the authentication function decides them).
|
|
114
|
+
- The secret lives only in dokebi.yaml, at least 32 random bytes. Every server of a namespace uses the same secret, options and Redis/NATS.
|
|
115
|
+
- Each server needs a different `serverId` and an `advertiseHost` and TCP port reachable from other servers (firewalls). The hub TCP has no built-in TLS; use a trusted private network or an encrypted tunnel.
|
|
116
|
+
- Never close the hub, Redis or NATS before the whole drain (core ordering ensures it).
|
|
117
|
+
- **Operations ([D-0049](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0049-redis-rejections-and-eviction.md))**: Redis for cluster mode uses `maxmemory-policy noeviction`. With an evicting policy the leader key gets evicted and the hub goes through `STALE_LEADER` and `RECOVERING` (OPS-002 G). Put caches on a separate Redis (`allkeys-lru`). At node start it reads `CONFIG GET maxmemory*` once; with `maxmemory > 0` and a policy other than `noeviction` it logs **error `REDIS_EVICTION_UNSAFE`** (`policy`, `maxmemory`, `connection`) and **starts anyway** (an operator may accept the risk). When safe, debug `REDIS_EVICTION_CHECKED`; a managed Redis or ACL that blocks CONFIG gives debug `REDIS_EVICTION_CHECK_SKIPPED` (`reason: rejected` + the full `error`); an older @dokebi/redis the same code (`reason: unsupported`); any other failure error `REDIS_EVICTION_CHECK_FAILED`. Redis error replies (OOM and so on) now stay as `COMMAND_REJECTED` with the server reply in the cause of `STORE_UNAVAILABLE`.
|
|
118
|
+
- Logs ([D-0047](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0047-logging-and-no-silent-errors.md)): exceptions are never dropped silently. Failures that cannot be turned into a defined error or response code are logged at `error`: a fixed English message + `code` + `error` (the full original error: name, whole message, stack and `Caused by: …` for each cause, address, port and errno text included; only URL credentials masked as `//***@`; cut only beyond 16 KB). Examples: lost leader connection `Hub connection dropped` (debug when the node itself closed it with `RECOVERING` or `CLOSED`, during `bye`, or after the leader announced `leaving` — D-0056), recovery start `Hub transport or store unavailable; recovering`, peer send failure `PEER_SEND_FAILED` (backpressure included), peer socket and frame `PEER_SOCKET_ERROR`, `PEER_FRAME_REJECTED` (a socket error before hello is debug `Hub peer connection failed before it was established`, and the startup retry or recovery log reports it with the cause), when the leader vanished first and `bye` could not be sent but Redis `leave` worked, debug `Hub goodbye not delivered; left through Redis` (a successful stop), a leader of another version `PROTOCOL_MISMATCH` (error; the refusing leader logs warn), removed options `OPTION_REMOVED` (warn, once per key at start), NATS `NATS_PUBLISH_FAILED`, `NATS_SUBSCRIPTION_FAILED`, local event queue overflow `EVENT_OVERFLOW`, a Redis failure of `status()` `STATUS_UNREACHABLE`. Redis failures stay as the `cause` of `STORE_UNAVAILABLE`. Places that run per frame or message write the same message and code once per interval (10 seconds) and group the rest as a `repeated` count (the remaining count is written at shutdown too). Bodies, secrets and tokens are never logged.
|
|
119
|
+
- debug (`DOKEBI_LOG_LEVEL=debug`): election, leader start and stop, lease loss, leader connection and reconnect (`Hub connected to leader`, ms taken, resync count), server registration, removal and expiry, connection list changes, broadcasts `Hub broadcast sent` (leader and local: `id`, `recipients`, `owners`, `ms`; no body), broadcasts of unknown outcome `OUTCOME_UNKNOWN` (not resent), refusals (`BUSY`, `STALE_*`), failures while still recovering (`Hub still recovering`), and the recovery time at info (`Hub recovered`).
|
|
120
|
+
- `dokebi-hub-status` writes the cause of a Redis failure to stderr (the stdout JSON stays as it is).
|
|
121
|
+
- Extension hooks for the hub are listed as candidates in [D-0064](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0064-extension-contract.md) §9; there is no extension class yet.
|
|
75
122
|
|
|
76
|
-
|
|
123
|
+
Evidence: `examples/18-websocket-hub` (real TCP peers, leader handover), `examples/23-websocket`, `examples/25-playground`, `tests/status.test.ts` (status, nonces, the command), D-0024, D-0029, D-0034, D-0038, D-0057 (replacing the ACK and resend of D-0025 and D-0042).
|
package/README.md
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
# @dokebi/websocket-hub
|
|
2
2
|
|
|
3
|
-
Node >=24, 0.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
- **`mode: local`(
|
|
8
|
-
- API
|
|
9
|
-
-
|
|
10
|
-
- `node:cluster`
|
|
11
|
-
- **`mode: cluster`:**
|
|
12
|
-
-
|
|
13
|
-
- `namespace
|
|
14
|
-
- NATS(`natsConnection`)
|
|
15
|
-
-
|
|
16
|
-
-
|
|
17
|
-
- **`whenReady(timeoutMs)`(D-0034):**
|
|
18
|
-
- **presence
|
|
3
|
+
Node >=24, version 0.4.0 (unpublished). An optional plugin of Dokebi `@dokebi/core ^0.4.0`: a TCP hub that replaces the former `@dokebi/sync`. AI agents start with [AI.md](AI.md).
|
|
4
|
+
|
|
5
|
+
There are two modes ([D-0029](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0029-hub-and-cluster-infrastructure.md)).
|
|
6
|
+
|
|
7
|
+
- **`mode: local` (default):** a hub for one process. It handles connection lists, rooms, broadcasts and kicks in memory, without Redis or NATS.
|
|
8
|
+
- The API and broadcast semantics are those of cluster mode (at most once, "Broadcast" below). A broadcast is done once it is in the event queue websocket reads. When half of that queue is still unread it is refused with `BUSY` (leaving room for presence and kicks).
|
|
9
|
+
- There is no failover.
|
|
10
|
+
- In `node:cluster` workers the workers cannot share connections, so start is refused with `LOCAL_IN_CLUSTER`. Allow it with `allowInCluster: true` only on purpose.
|
|
11
|
+
- **`mode: cluster`:** connections are shared by processes and servers.
|
|
12
|
+
- Every WebSocket worker is a hub candidate, and a Redis lease elects one. TCP carries user registrations, snapshots, broadcast and targeted messages and kicks, and Redis keeps the connection list.
|
|
13
|
+
- `namespace`, `serverId`, `redisConnection` and `secret` are required.
|
|
14
|
+
- NATS (`natsConnection`) is optional. With it, leader changes are announced at once; without it other servers notice through Redis polling (`pollIntervalMs`).
|
|
15
|
+
- When the hub dies, the real WebSockets of healthy servers stay and reconnect to the new hub.
|
|
16
|
+
- **Install check:** the install module's `check` looks at the mode and reports the required plugins: Redis in cluster mode, and NATS too when a NATS connection is set.
|
|
17
|
+
- **`whenReady(timeoutMs)` (D-0034):** true once ready, false when the time passes or it closes. websocket uses it to wait out short reconnects when accepting upgrades. It logs why the leader connection dropped (`Hub connection dropped`), failures while recovery continues (`Hub still recovering`) and the recovery time (`Hub recovered`).
|
|
18
|
+
- **presence events:** they come as the full list per owner (server) when connections come and go. When a peer reconnects or the leader changes, owners that differ from the snapshot and owners that disappeared are announced again as presence events, so consumers never miss connections that left meanwhile.
|
|
19
19
|
|
|
20
20
|
```yaml
|
|
21
21
|
schemaVersion: 1
|
|
@@ -39,9 +39,9 @@ options:
|
|
|
39
39
|
secret: replace-with-your-random-secret-at-least-32-bytes
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
-
serverId
|
|
42
|
+
`serverId`/`advertiseHost` are per server; `namespace`, `secret`, shared stores and distributed limits must match across the area. Each worker opens its own TCP port (`port` default 0), which must be reachable from other servers. The TCP has no built-in TLS, so use a trusted private network or an encrypted tunnel.
|
|
43
43
|
|
|
44
|
-
```ts
|
|
44
|
+
```ts snippet
|
|
45
45
|
import { Application, loadConfig } from '@dokebi/core';
|
|
46
46
|
import { createPlugin } from '@dokebi/websocket-hub';
|
|
47
47
|
const config = await loadConfig('./dokebi.yaml');
|
|
@@ -51,52 +51,80 @@ app.register(plugin);
|
|
|
51
51
|
await app.start();
|
|
52
52
|
const hub = app.get(plugin);
|
|
53
53
|
const connection = await hub.register({tenant:'demo',member:'alice',connection:'socket-1'});
|
|
54
|
-
// websocket
|
|
54
|
+
// With the websocket plugin, that plugin is the sole consumer instead.
|
|
55
55
|
const consumer = (async () => {
|
|
56
56
|
for await (const event of hub.events()) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
hub.acknowledge(event.id, event.connections);
|
|
60
|
-
}
|
|
57
|
+
// the message event is the whole delivery: write event.data to this server's event.connections (no ACK).
|
|
58
|
+
if (event.kind === 'message') writeToSockets(event.connections, event.data);
|
|
61
59
|
}
|
|
62
60
|
})();
|
|
63
61
|
const snapshot = await hub.list({tenant:'demo'});
|
|
64
62
|
const receipt = await hub.broadcast({tenant:'demo',member:'alice'}, new TextEncoder().encode('hello'));
|
|
63
|
+
// receipt: { id, recipients } — connections matching when sent (not a confirmation of receipt)
|
|
65
64
|
await connection.close();
|
|
66
65
|
await app.stop();
|
|
67
66
|
await consumer;
|
|
68
67
|
```
|
|
69
68
|
|
|
70
|
-
HubConnection
|
|
69
|
+
`HubConnection`: `identity` (ownerEpoch/connection), `join(room)`, `leave(room)`, `updateMetadata(record)`, `close()`. `HubApi`: `state`, `role`, `ownerEpoch`, `leader`, `register`, `list`, `broadcast`, `kick`, `events`, `whenReady`, `status`, `nonces`. A query is the AND of a required `tenant` and optional `room`/`member`/`connection`. `events` is the single async iterator of presence, message and kick events. kick (default code 4403) is a TCP request that returns `{ authorityEpoch, id }`.
|
|
71
70
|
|
|
72
|
-
|
|
71
|
+
| Key | Type | Default | Description |
|
|
72
|
+
|---|---|---|---|
|
|
73
|
+
| `leaseMs` | integer ms | 3000 | Leader lease |
|
|
74
|
+
| `pollIntervalMs` | integer ms | 500 | Polling without NATS |
|
|
75
|
+
| `ownerTtlMs` | integer ms | 15000 | Server registration expiry |
|
|
76
|
+
| `startupTimeoutMs` | integer ms | 15000 | Startup wait |
|
|
77
|
+
| `maxOwners` | integer | 64 | Servers |
|
|
78
|
+
| `maxConnections` | integer | 128 | Connections per worker |
|
|
79
|
+
| `maxTotalConnections` | integer | 8192 | All connections |
|
|
80
|
+
| `maxFrameBytes` | integer | 1048576 | Hub TCP frame |
|
|
81
|
+
| `maxPayloadBytes` | integer | 65536 | Message |
|
|
82
|
+
| `maxPending` | integer | 128 | Broadcasts in progress |
|
|
83
|
+
| `maxPendingBytes` | integer | 4194304 | Queue bytes |
|
|
73
84
|
|
|
74
|
-
|
|
85
|
+
A full snapshot also stays within the frame limit. Check the [option schema](options.schema.json). It requires `pollIntervalMs × 3 ≤ leaseMs` and `ownerTtlMs ≥ leaseMs × 3`. 0.4.0 renamed `pollMs` → `pollIntervalMs`, `startupMs` → `startupTimeoutMs` and `queueBytes` → `maxPendingBytes` ([D-0071](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0071-option-vocabulary.md)); the old names still work in 0.4.0 with an `OPTION_DEPRECATED` warning.
|
|
75
86
|
|
|
76
|
-
##
|
|
87
|
+
## Broadcast: at most once (0.3.0, [D-0057](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0057-websocket-broadcast-without-ack.md))
|
|
77
88
|
|
|
78
|
-
-
|
|
79
|
-
- **`
|
|
80
|
-
-
|
|
89
|
+
- **Success = sent out.** In cluster mode the leader picks the connections matching right now, writes one message frame to each server holding them, then gives the sending server the receipt `{ id, recipients }`. local is done once it is in the event queue. There is no client ACK, resend, deduplication, completion record or deadline wait. Broadcasts never go through Redis and never stay in leader memory.
|
|
90
|
+
- **`recipients` of the receipt** are the identifiers of the connections matching when it was sent (the SHA-256 of tenant, member and clientId, or of owner and connection without a clientId). It is not a confirmation of receipt. Connections that were reconnecting, that left before the frame was written or that were closed with 1013 for being slow do not get it, and connections arriving later do not get it.
|
|
91
|
+
- **Failures are reported at once and never resent.** `RECOVERING`: not connected to the leader (leader change included) — not sent. `OUTCOME_UNKNOWN`: the connection dropped after sending the request, before the answer — it may have been sent. `BUSY`: broadcasts in progress reached `maxPending`. The caller decides whether to send again.
|
|
92
|
+
- **App guideline:** never leave work that must not be missed (order state, payment results) to a broadcast. Store it in the DB and let the page read the latest state from the server when it reconnects. Use broadcasts as "it changed, read again" notices. Work that must happen goes to @dokebi/job.
|
|
93
|
+
- **Removed options:** `deliveryTimeoutMs`, `retryMs`, `maxDeliveries`, `deliveryBytes`. Start still works when they remain; each key logs warn `OPTION_REMOVED` once and is ignored. The schema keeps them as deprecated (removed in the next version).
|
|
94
|
+
- **Version between hub servers:** hello carries `protocol: 2`. The leader refuses servers of another version with `PROTOCOL_MISMATCH` (warn), and the refused server logs an error with both versions and retries. A 0.3 server connected to a 0.2 leader is cut too. When versions mix during a rolling restart, old servers cannot join the new leader, so upgrade every server of the namespace to the same version. The configuration fingerprint stays the same as 0.2 so the version check tells them apart (against a 0.1.x Redis outbox it is still `CONFIG_MISMATCH`).
|
|
95
|
+
- `clientId` is an identifier per authenticated device or tab. A concurrent registration of the same `(tenant, member, clientId)` is refused with `DUPLICATE_CLIENT`.
|
|
81
96
|
|
|
82
|
-
##
|
|
97
|
+
## Operational status and replay defense (D-0038)
|
|
83
98
|
|
|
84
|
-
|
|
99
|
+
- **`status()`:** returns this process (`mode`, `state`, `role`, `serverId`) together with the hub as Redis sees it (`leaseState`, `leader`, `leaseRemainingMs`, `servers`, `connections`). The calculation is one pure function, `hubStatus`. Unknown values are left out, not filled with 0 — with `leaseState: 'unreachable'` (Redis does not answer) there are no `servers` or `connections`. `none` means no leader (failover in progress, or no server is up).
|
|
100
|
+
- **`dokebi-hub-status --config dokebi.yaml [--json]`:** answers the same question from outside the app. It starts only the Redis plugin and reads the leader and owner records, without joining the election or registering. When Redis does not answer it exits with 1. In local mode it says the state lives only inside the process.
|
|
101
|
+
- **`nonces()`:** the one-time nonce store for sealed frames (passed as is to `nonces` of @dokebi/seal `SealTerminator`; only the shape matches and seal is never imported). cluster mode uses Redis `SET NX PX` (`shared: true`), so a frame replayed to another server is refused too. local mode keeps it in the process (`shared: false`) and warns `HUB_NONCES_NOT_SHARED` once when used in a cluster worker.
|
|
85
102
|
|
|
86
|
-
|
|
87
|
-
- `@dokebi/redis`·`@dokebi/nats`는 `peerDependencies`다. 앱이 두 패키지를 함께 설치해야 한다.
|
|
88
|
-
- 다른 플러그인은 `import { dependency } from '@dokebi/websocket-hub'`를 `metadata.dependencies`에 넣고 `context.getDependency(dependency)`로 API를 얻는다.
|
|
89
|
-
- Redis 권한 스크립트는 저장소마다 `redis.script()`로 한 번 만들고 `EVALSHA`로 실행한다. 서버에 스크립트가 없을 때(`NOSCRIPT`)만 `EVAL`을 한 번 다시 보낸다.
|
|
103
|
+
## Factory and dependencies (plugin contract v2)
|
|
90
104
|
|
|
91
|
-
|
|
105
|
+
The factory input is `{ app, options? }`; `app` is required. Without options it reads `app.config['dokebi-websocket-hub']`. It takes no `dependencies` record and no Redis or NATS instance inputs ([D-0028](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0028-plugin-contract-v2.md)).
|
|
92
106
|
|
|
93
|
-
|
|
107
|
+
- Redis and NATS are wired through the `dokebi-redis` and `dokebi-nats` provider descriptors. A plugin registered under the same name is reused; otherwise it is created from the `@dokebi/redis` or `@dokebi/nats` package with `app.config`. To choose one, register it first, e.g. `app.register(createRedis({ app: app.view }))`.
|
|
108
|
+
- `@dokebi/redis` and `@dokebi/nats` are `peerDependencies`. The app installs both packages.
|
|
109
|
+
- Other plugins put `import { dependency } from '@dokebi/websocket-hub/dependency'` (the provider descriptor, the same object as the package entry's `dependency`) in `metadata.dependencies` and get the API with `context.getDependency(dependency)`. Plugins for which the hub is optional keep their own `optionalDependency`.
|
|
110
|
+
- The Redis authority script is created once per store with `redis.script()` and runs as `EVALSHA`. Only when the server lacks the script (`NOSCRIPT`) is `EVAL` sent once more.
|
|
94
111
|
|
|
95
|
-
|
|
112
|
+
With the CLI, prepare the YAML and register in the order `npx dokebi add @dokebi/redis` → `@dokebi/nats` → `@dokebi/websocket-hub`. The manifest is schemaVersion 2, and its install module (`dist/install.js`) only reports required plugins; it creates no files. Installing makes no server connections. The hub has no drain hook and closes its resources in cleanup after the app drain finished.
|
|
96
113
|
|
|
97
|
-
|
|
114
|
+
Example 18 in the repository covers the public API and a local TCP baseline; example 23 verifies two Docker servers, four workers, hub and server faults and a real DB drain. That does not mean a full port of XFramework's extra media and nonce features or full operational certification.
|
|
98
115
|
|
|
99
|
-
##
|
|
116
|
+
## Configuration fingerprint (2026-10-14)
|
|
100
117
|
|
|
101
|
-
|
|
102
|
-
|
|
118
|
+
Servers of one namespace need the same secret, lease and limit settings (the configuration fingerprint in Redis). While a server with different settings is alive, a new server fails to start at once with `CONFIG_MISMATCH` and logs an error (`code: CONFIG_MISMATCH`). Once every leader and owner with the old settings is gone (a full restart), the new settings replace the fingerprint. So rotating the secret is a full stop and start, without deleting Redis keys ([ISSUE-0005](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/issues/ISSUE-0005-playground-findings.md) PG-16).
|
|
119
|
+
|
|
120
|
+
## Shutdown cleanup and startup logs (2026-10-14)
|
|
121
|
+
|
|
122
|
+
- On a normal shutdown a node deletes its own owner record from Redis directly (`leave`: without the leader, only for the same session). Even when everything stops at once, records do not stay until `ownerTtlMs`, so a restart right after a settings change works (PG-21).
|
|
123
|
+
- Retries during startup (such as connecting to a leader TCP that is not open yet) log no warnings. Past the startup deadline (`startupTimeoutMs`) the error log `STARTUP_TIMEOUT` records the last error code (`lastError`). Fault warnings after start carry a `code` (PG-19).
|
|
124
|
+
|
|
125
|
+
## 0.4.0 shared base (D-0063)
|
|
126
|
+
|
|
127
|
+
- The options are one declaration in `src/options.ts` that is both the parser and `options.schema.json` (generated at build time; `npm run check:options` compares bytes). The connection name and host rules and defaults that only the hand parser kept are in the schema now, and `secret` counts characters and bytes together. Core logs the warn `OPTION_REMOVED` for removed options.
|
|
128
|
+
- `HubError` is a `DokebiError` (`plugin`, `fix` for codes an operator fixes). Log helpers and repeat suppression use core `diagnostics` (`describeError`, `Repeats`).
|
|
129
|
+
- The cluster worker check uses core `context.runtime.role` instead of `node:cluster` (`LOCAL_IN_CLUSTER`, `HUB_NONCES_NOT_SHARED`). The cluster error code was `HUB_LOCAL_IN_CLUSTER` before 0.4.0 (D-0071); the error log keeps that code.
|
|
130
|
+
- The hub index ([D-0068](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0068-performance-stage4.md) §3: per tenant, member, room, owner and recipient key, so selection walks the smallest matching set) and the delivery semantics are unchanged. Possible future extension hooks are listed in [D-0064](https://git.nyx-zone.com/dokebi/framework/-/blob/main/docs/decisions/D-0064-extension-contract.md) §9 (there is no extension class yet).
|
package/dist/delivery.d.ts
CHANGED
|
@@ -1,51 +1,17 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import type { HubQuery, HubRecord } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The identity of a connection in a broadcast receipt: SHA-256 (base64url) of tenant, member and
|
|
4
|
+
* clientId, or of the owner and connection when the client has no clientId.
|
|
5
|
+
*/
|
|
3
6
|
export declare function recipientKey(record: HubRecord): string;
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
readonly
|
|
7
|
-
|
|
8
|
-
readonly
|
|
9
|
-
readonly data: string;
|
|
10
|
-
readonly expiresAt: number;
|
|
11
|
-
readonly recipients: readonly string[];
|
|
12
|
-
readonly remaining: Set<string>;
|
|
13
|
-
readonly bytes: number;
|
|
14
|
-
lastSent: number;
|
|
15
|
-
}
|
|
16
|
-
interface Finished {
|
|
17
|
-
readonly producer: string;
|
|
18
|
-
readonly digest: string;
|
|
19
|
-
readonly ok: boolean;
|
|
20
|
-
readonly recipients: readonly string[];
|
|
21
|
-
readonly expiresAt: number;
|
|
22
|
-
readonly bytes: number;
|
|
7
|
+
export interface Selection {
|
|
8
|
+
/** Identities of the matching connections, once each (the receipt). */
|
|
9
|
+
readonly recipients: string[];
|
|
10
|
+
/** Matching connections by the server (owner epoch) that holds them: one frame per owner. */
|
|
11
|
+
readonly owners: Map<string, string[]>;
|
|
23
12
|
}
|
|
24
13
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* written to Redis: when the leader dies, each producer resends its unfinished broadcasts with
|
|
28
|
-
* the same id and the recipient list this leader announced (`accepted`), and the next leader
|
|
29
|
-
* carries on from there. Finished broadcasts keep a small record (no payload) until their
|
|
30
|
-
* deadline so a resent id is answered with its result instead of being delivered again.
|
|
14
|
+
* The recipients of a broadcast (D-0057): the connections that match `filter` now. Nothing is
|
|
15
|
+
* remembered; a connection that comes later, or leaves before its frame is written, misses it.
|
|
31
16
|
*/
|
|
32
|
-
export declare
|
|
33
|
-
#private;
|
|
34
|
-
readonly options: HubOptions;
|
|
35
|
-
readonly connections: () => readonly HubRecord[];
|
|
36
|
-
readonly send: (owner: string, frame: Record<string, unknown>) => boolean;
|
|
37
|
-
readonly pending: Map<string, Delivery>;
|
|
38
|
-
readonly finished: Map<string, Finished>;
|
|
39
|
-
readonly waiters: Map<string, Map<string, Reply>>;
|
|
40
|
-
constructor(options: HubOptions, connections: () => readonly HubRecord[], send: (owner: string, frame: Record<string, unknown>) => boolean);
|
|
41
|
-
/** Bytes the outbox holds now (payloads and recipient lists), bounded by deliveryBytes. */
|
|
42
|
-
get bytes(): number;
|
|
43
|
-
/**
|
|
44
|
-
* Takes a broadcast from `producer`. `accepted` is told the recipient list once, so the
|
|
45
|
-
* producer can resend it to the next leader; `reply` is told the result.
|
|
46
|
-
*/
|
|
47
|
-
accept(producer: string, frame: Record<string, unknown>, reply: Reply, accepted?: (recipients: readonly string[]) => void): void;
|
|
48
|
-
acknowledge(input: unknown, records: readonly HubRecord[]): void;
|
|
49
|
-
tick(): void;
|
|
50
|
-
}
|
|
51
|
-
export {};
|
|
17
|
+
export declare function select(records: Iterable<HubRecord>, filter: HubQuery): Selection;
|