@ignex/nova 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +313 -0
- package/docs/architecture.md +146 -0
- package/docs/publishing.md +119 -0
- package/docs/wire-format.md +170 -0
- package/index.ts +61 -0
- package/package.json +89 -0
- package/prebuilds/linux-x64/libignex_ffi.so +0 -0
- package/public/client.ts +23 -0
- package/public/nats.ts +19 -0
- package/public/server.ts +35 -0
- package/rust/Cargo.toml +19 -0
- package/rust/src/ffi.rs +135 -0
- package/rust/src/generated/backend.rs +2817 -0
- package/rust/src/generated/mod.rs +2 -0
- package/rust/src/lib.rs +9 -0
- package/rust/src/transcode/generated.rs +1352 -0
- package/rust/src/transcode/mod.rs +2 -0
- package/src/bridge/nats.ts +269 -0
- package/src/bridge/subjects.ts +30 -0
- package/src/core/auth.ts +56 -0
- package/src/core/backpressure.ts +39 -0
- package/src/core/client-heartbeat.ts +27 -0
- package/src/core/client-reconnect.ts +35 -0
- package/src/core/client-state.ts +76 -0
- package/src/core/client-wire.ts +72 -0
- package/src/core/client.ts +176 -0
- package/src/core/groups.ts +52 -0
- package/src/core/int64-guard.ts +44 -0
- package/src/core/metrics.ts +105 -0
- package/src/core/outbound.ts +76 -0
- package/src/core/replay.ts +31 -0
- package/src/core/ring.ts +85 -0
- package/src/core/rooms.ts +44 -0
- package/src/core/routing.ts +94 -0
- package/src/core/server.ts +294 -0
- package/src/core/state.ts +179 -0
- package/src/generated/direct-ser.ts +495 -0
- package/src/generated/fbs/backend.fbs +139 -0
- package/src/generated/registry.ts +341 -0
- package/src/generated/rust/backend_generated.rs +2817 -0
- package/src/generated/ts/backend.ts +25 -0
- package/src/generated/ts/big-val.ts +106 -0
- package/src/generated/ts/complex.ts +303 -0
- package/src/generated/ts/customer.ts +137 -0
- package/src/generated/ts/hello.ts +123 -0
- package/src/generated/ts/join-group.ts +78 -0
- package/src/generated/ts/leave-group.ts +78 -0
- package/src/generated/ts/order-billing.ts +137 -0
- package/src/generated/ts/order-line.ts +144 -0
- package/src/generated/ts/order.ts +236 -0
- package/src/generated/ts/ping.ts +74 -0
- package/src/generated/ts/pong.ts +74 -0
- package/src/generated/ts/portfolio-position.ts +120 -0
- package/src/generated/ts/portfolio-snapshot.ts +170 -0
- package/src/generated/ts/quote.ts +148 -0
- package/src/generated/ts/side.ts +8 -0
- package/src/generated/ts/snapshot-request.ts +78 -0
- package/src/generated/ts/subscribe.ts +78 -0
- package/src/generated/ts/tags.ts +9 -0
- package/src/generated/ts/trade.ts +135 -0
- package/src/generated/ts/unsubscribe.ts +78 -0
- package/src/generated/ts/welcome.ts +112 -0
- package/src/generated/ts-ser.ts +465 -0
- package/src/generated/wire-registry.json +20 -0
- package/src/native/codec.ts +35 -0
- package/src/native/ffi.ts +214 -0
- package/src/native/loader.ts +55 -0
- package/src/schema/index.ts +217 -0
- package/src/server.ts +87 -0
- package/src/transport/byte-buffer-pool.ts +63 -0
- package/src/transport/scratch.ts +48 -0
- package/src/transport/stats.ts +44 -0
- package/src/transport/transport.ts +106 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ignex-nova contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
# ignex-nova
|
|
2
|
+
|
|
3
|
+
TypeBox-driven **FlatBuffer transport over Bun WebSockets** with a Rust FFI
|
|
4
|
+
serializer — and a typed pub/sub API that hides all of it from developers.
|
|
5
|
+
|
|
6
|
+
```ts
|
|
7
|
+
// server (Bun)
|
|
8
|
+
import { createServer } from "ignex-nova/server";
|
|
9
|
+
const server = createServer({ port: 3000, inbound: ["chat"] });
|
|
10
|
+
server.publish("quote", { symbol: "AAPL", bid: 180.1, ask: 180.2, bidSize: 100, askSize: 200, ts: Date.now() });
|
|
11
|
+
server.publishToTopic("equities", "quote", {...}); // rooms
|
|
12
|
+
server.on("chat", (msg, ws) => server.publishTo(ws, "chatAck", { ok: true }));
|
|
13
|
+
|
|
14
|
+
// FE (browser or Bun)
|
|
15
|
+
import { createClient } from "ignex-nova/client";
|
|
16
|
+
const client = createClient("ws://localhost:3000/ws", { reconnect: true });
|
|
17
|
+
client.on("quote", (q) => console.log(q.symbol, q.bid)); // q is a plain typed object
|
|
18
|
+
client.subscribe("equities"); // rooms + last-value replay
|
|
19
|
+
client.send("chat", { text: "hi" }); // typed client→server (pure-JS encoder — works in the browser)
|
|
20
|
+
client.connect();
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
No `flatbuffers.Builder`, no `getRootAs*`, no FFI — developers only ever see
|
|
24
|
+
plain, type-checked objects. The FlatBuffer + Rust machinery is internal, and
|
|
25
|
+
the **browser can send typed frames too** (via a generated pure-JS encoder).
|
|
26
|
+
|
|
27
|
+
## Features
|
|
28
|
+
|
|
29
|
+
- **Typed pub/sub, both directions** — `publish`/`publishTo`/`publishToTopic`
|
|
30
|
+
on the server, `on`/`send`/`subscribe` on the client. Control frames
|
|
31
|
+
(hello/subscribe/ping/…) ride the same codegen, routed internally.
|
|
32
|
+
- **Rooms / topics** with optional **last-value replay** on subscribe
|
|
33
|
+
(`replay: { historySize }`).
|
|
34
|
+
- **Backpressure** — configurable slow-consumer policy (drop-oldest /
|
|
35
|
+
drop-newest / disconnect) so a hot publish loop can't balloon memory.
|
|
36
|
+
- **Auth & limits** — async `authenticate(req)` hook, origin allowlist, bearer
|
|
37
|
+
`token`, `maxConnections`, `maxMessageSize`, TLS passthrough.
|
|
38
|
+
- **Resiliency** — auto-reconnect with exponential backoff + jitter,
|
|
39
|
+
auto-resubscribe, app-level heartbeat.
|
|
40
|
+
- **Exact int64** — `Type.BigInt()` fields round-trip losslessly beyond 2^53;
|
|
41
|
+
optional `int64Guard` catches out-of-range numbers.
|
|
42
|
+
- **Observability** — `server.getMetrics()` (counters + per-event encode path),
|
|
43
|
+
JSON `/health`, graceful `drain()`.
|
|
44
|
+
- **Targeted delivery & groups** — every client has a stable id (from
|
|
45
|
+
`authenticate` metadata or an auto-UUID) so you can `publishToClient(id, …)`;
|
|
46
|
+
server-side groups (`publishToGroup(group, …)`, joined via auth metadata,
|
|
47
|
+
`joinGroup(id, group)`, or client `joinGroup` frames) target sets of clients.
|
|
48
|
+
Active clients are listed via `getClients()` / `GET /clients`.
|
|
49
|
+
- **NATS bridge (bidirectional)** — every broadcast/topic/group publish is also
|
|
50
|
+
published to NATS as the **same FlatBuffer wire frame** (`ignex.broadcast.*`,
|
|
51
|
+
`ignex.topic.*`, `ignex.group.*`) so other applications can consume it;
|
|
52
|
+
external apps push events into the hub via `ignex.inbound.>` and the server
|
|
53
|
+
forwards them to clients. Best-effort (never blocks the WS hot path),
|
|
54
|
+
observable via metrics.
|
|
55
|
+
- **Bun-only server, browser+Bun client.** Wire spec documented for independent
|
|
56
|
+
clients: [docs/wire-format.md](docs/wire-format.md).
|
|
57
|
+
|
|
58
|
+
## How it works
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
src/schema/index.ts (TypeBox — source of truth: app events + control events)
|
|
62
|
+
│ scripts/generate.ts
|
|
63
|
+
▼
|
|
64
|
+
backend.fbs ──flatc --ts──▶ src/generated/ts/ (decoders)
|
|
65
|
+
└───flatc --rust──▶ rust/src/generated/ (table builders)
|
|
66
|
+
scripts/rust-glue-gen.ts ─▶ rust/src/transcode/ (JSON glue + direct-args FFI)
|
|
67
|
+
scripts/direct-gen.ts ────▶ src/generated/direct-ser.ts (direct fast-path serde)
|
|
68
|
+
scripts/ts-ser-gen.ts ─────▶ src/generated/ts-ser.ts (pure-JS browser encoder)
|
|
69
|
+
scripts/registry-gen.ts ───▶ src/generated/registry.ts (event routing, both sides)
|
|
70
|
+
|
|
71
|
+
server: JS object ─▶ (flat event) fields as direct FFI args ─▶ Rust
|
|
72
|
+
└▶ (vector/nested) JSON.stringify → cstring ─▶ Rust
|
|
73
|
+
browser: JS object ─▶ flatc object API (ts-ser) ─▶ size-prefixed FlatBuffer
|
|
74
|
+
Rust + ts-ser both emit the same frame:
|
|
75
|
+
frame = [WIRE_VERSION:1][event_id:u32 LE][size-prefixed FlatBuffer]
|
|
76
|
+
Bun: ws.send(frame) ──▶ peer: decode via generated classes → plain object
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
**Two server serialize paths** (neither uses JSON on the hot path):
|
|
80
|
+
|
|
81
|
+
- **Direct fast path (zero-allocation)** — directable events: fields pushed
|
|
82
|
+
straight into generated `extern "C"` functions as FFI args, output written
|
|
83
|
+
into a single reusable scratch. Measured ~0 B/op.
|
|
84
|
+
- **JSON fallback** — only for nested tables-in-tables / unions (e.g. `order`),
|
|
85
|
+
or payloads with embedded NULs. Observable via `getMetrics().pathCounts`.
|
|
86
|
+
|
|
87
|
+
The **browser client** has no FFI, so outgoing frames (app + control) use the
|
|
88
|
+
generated pure-JS encoder (flatc object API over a pooled builder).
|
|
89
|
+
|
|
90
|
+
Key techniques (Bun 1.4 standard practices, following the castrum FFI guide):
|
|
91
|
+
|
|
92
|
+
- strings as pre-encoded `(buffer, usize)` via `Buffer.write` into a cached view
|
|
93
|
+
(zero JS-side text encoding, zero allocation); vectors as packed binary blobs
|
|
94
|
+
- `buffer` / `buffer_length` ABI pair for the output — the engine snapshots
|
|
95
|
+
pointer + byteLength off the same view at call time (the "peak"/pointer +
|
|
96
|
+
length pattern). `returns: 'u64_fast'` for byte counts.
|
|
97
|
+
- `ws.send(Uint8Array)` copies synchronously (verified) → the shared output
|
|
98
|
+
scratch is safe to reuse immediately.
|
|
99
|
+
- Rust `#[no_mangle] extern "C"` exports, `panic_guard`-wrapped, needed-size
|
|
100
|
+
convention, `thread_local` reused `FlatBufferBuilder` + `reset()` per call.
|
|
101
|
+
- Bind-time self-test: `fb_probe` magic + `fb_wire_version` (stale-cdylib
|
|
102
|
+
guard) + a per-symbol direct self-test that DISABLES broken symbols
|
|
103
|
+
(graceful JSON fallback instead of a hard crash).
|
|
104
|
+
- Stable event ids = FNV-1a 32-bit over the name (reorder-safe, collision-
|
|
105
|
+
checked at generate time).
|
|
106
|
+
- `flatc` guarantees wire compatibility between the Rust builders and the
|
|
107
|
+
browser decoders (both derive from the same `.fbs`).
|
|
108
|
+
|
|
109
|
+
## Prerequisites
|
|
110
|
+
|
|
111
|
+
- [Bun](https://bun.sh) ≥ 1.4
|
|
112
|
+
- Rust toolchain (`cargo`)
|
|
113
|
+
- `flatc` (FlatBuffers compiler): `brew install flatbuffers`,
|
|
114
|
+
`apt install flatbuffers-compiler`, or from <https://flatbuffers.dev>.
|
|
115
|
+
Keep `flatc` and the `flatbuffers` crate/npm versions aligned.
|
|
116
|
+
|
|
117
|
+
## Setup
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
bun install # deps: @sinclair/typebox, flatbuffers
|
|
121
|
+
bun run generate # TypeBox → .fbs → flatc --ts/--rust → Rust glue + registry
|
|
122
|
+
cargo build --release --manifest-path rust/Cargo.toml # → <platform> libignex_ffi (.so/.dylib/.dll)
|
|
123
|
+
bun run build:client # bundle the browser demo → client-dist/
|
|
124
|
+
bun test # round-trip + FFI tests (needs generate + built addon)
|
|
125
|
+
bun run lint # oxlint — FP-discipline rules (no-var, no-param-reassign, …)
|
|
126
|
+
bun run serve # demo: http://localhost:3000/ (ws: /ws)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
`bun run build` runs generate + build:rust + build:client together.
|
|
130
|
+
`bun run build:dist` emits an npm-ready JS bundle + declarations to `dist/`.
|
|
131
|
+
|
|
132
|
+
**Perf gate:** `bench/BASELINE.md` records the reference numbers. After any
|
|
133
|
+
hot-path change run `bun run bench:serialize` + `bun run bench:throughput` and
|
|
134
|
+
compare — fail-fast on >±5% drift in encode latency/throughput or any new
|
|
135
|
+
allocations on the `quote` path (must stay ~0 B/op).
|
|
136
|
+
|
|
137
|
+
## Install & use from npm
|
|
138
|
+
|
|
139
|
+
Published as **TypeScript source** (no build step needed — Bun runs `.ts`
|
|
140
|
+
natively), with subpath entrypoints. Works in any Bun ≥ 1.4 project:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
bun add ignex-nova
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
```ts
|
|
147
|
+
// server (Bun-only — needs the native addon, see below)
|
|
148
|
+
import { createServer } from "ignex-nova/server";
|
|
149
|
+
|
|
150
|
+
// client (browser + Bun) — the root entry also re-exports everything
|
|
151
|
+
import { createClient, type Events } from "ignex-nova/client";
|
|
152
|
+
|
|
153
|
+
const q: Events["quote"] = { symbol: "AAPL", bid: 180.1, ask: 180.2, bidSize: 100, askSize: 200, ts: Date.now() };
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Entrypoints:
|
|
157
|
+
|
|
158
|
+
| Import | Resolves to |
|
|
159
|
+
| --- | --- |
|
|
160
|
+
| `ignex-nova` | `index.ts` — everything (server + client + nats + schema types) |
|
|
161
|
+
| `ignex-nova/server` | `public/server.ts` — `createServer` (Bun-only) |
|
|
162
|
+
| `ignex-nova/client` | `public/client.ts` — `createClient` (browser + Bun) |
|
|
163
|
+
| `ignex-nova/nats` | `public/nats.ts` — standalone `createNatsBridge` |
|
|
164
|
+
|
|
165
|
+
**Native addon:** the tarball ships `rust/` source + `prebuilds/<platform>-<arch>/`
|
|
166
|
+
for the platforms built at release (see CI). If your platform has a prebuild it
|
|
167
|
+
just works. Otherwise either rebuild from the shipped source:
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
cargo build --release --manifest-path node_modules/ignex-nova/rust/Cargo.toml
|
|
171
|
+
# loader finds it at <pkg>/rust/target/release/…
|
|
172
|
+
# or point at any build explicitly:
|
|
173
|
+
IGNEX_FFI_PATH=/abs/path/to/libignex_ffi.so bun run your-server.ts
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
See [docs/publishing.md](docs/publishing.md) for how the package is built,
|
|
177
|
+
staged, and published to npm.
|
|
178
|
+
|
|
179
|
+
## Layout
|
|
180
|
+
|
|
181
|
+
| Path | Role |
|
|
182
|
+
| --- | --- |
|
|
183
|
+
| `src/schema/index.ts` | TypeBox schemas + `events`/`controlEvents` registries (source of truth) |
|
|
184
|
+
| `scripts/` | `generate.ts` orchestrator + `.fbs` / Rust-glue / registry / ts-ser emitters |
|
|
185
|
+
| `src/generated/` | flatc `--ts`/`--rust` output + `registry.ts` + `direct-ser.ts` + `ts-ser.ts` |
|
|
186
|
+
| `rust/` | cdylib: `ffi.rs` (C-ABI), `transcode/generated.rs` (glue) |
|
|
187
|
+
| `src/native/` | `bun:ffi` binding, self-tests, per-platform addon loader |
|
|
188
|
+
| `src/transport/` | `transport.ts` (`encodeToScratch`), `scratch.ts` (reusable zero-alloc buffer), `stats.ts` |
|
|
189
|
+
| `src/core/` | functional modules: `server.ts`/`client.ts` composition roots, `state.ts`, `auth.ts`, `rooms.ts`, `groups.ts`, `replay.ts`, `backpressure.ts`, `outbound.ts`, `routing.ts`, `metrics.ts`, `int64-guard.ts`, client-* |
|
|
190
|
+
| `src/bridge/` | optional NATS bridge: `nats.ts` (injectable transport, eager non-blocking connect), `subjects.ts` (subject naming) |
|
|
191
|
+
| `public/server.ts` | entrypoint shim: `createServer` (publish/rooms/groups/targeting/NATS/auth/backpressure/metrics/drain) |
|
|
192
|
+
| `public/client.ts` | entrypoint shim: `createClient` (on/send/subscribe/joinGroup/reconnect/heartbeat/status) |
|
|
193
|
+
| `public/nats.ts` | entrypoint shim: `createNatsBridge` standalone (`ignex-nova/nats`) |
|
|
194
|
+
| `client/` | browser demo (built to `client-dist/`) |
|
|
195
|
+
| `bench/` | serialize latency + end-to-end throughput (+ `BASELINE.md` perf gate) |
|
|
196
|
+
| `examples/` | `nats-consumer.ts` — independent NATS consumer for bridged frames |
|
|
197
|
+
| `prebuilds/` | staged native addons per platform (`<platform>-<arch>/`), built by `bun run prebuild` / CI |
|
|
198
|
+
| `docs/` | `wire-format.md`, `architecture.md`, `publishing.md` |
|
|
199
|
+
|
|
200
|
+
## Adding an event
|
|
201
|
+
|
|
202
|
+
1. Define the payload in `schema/index.ts` (TypeBox), add it to `schemas`
|
|
203
|
+
(if it's a named table) and to `events` (name → schema). For exact integer
|
|
204
|
+
values use `Type.BigInt()`.
|
|
205
|
+
2. Re-run `bun run generate`, `cargo build --release`, `bun run build:client`.
|
|
206
|
+
3. `server.publish("yourEvent", payload)`, `client.on("yourEvent", cb)`, and
|
|
207
|
+
`client.send("yourEvent", payload)` are now fully typed on both sides.
|
|
208
|
+
|
|
209
|
+
## Performance (min-of-N ns/op on this machine, `bun run bench:serialize`)
|
|
210
|
+
|
|
211
|
+
| Payload | Rust FFI (zero-alloc) | Pure JS flatc | Rust vs JS |
|
|
212
|
+
| --- | --- | --- | --- |
|
|
213
|
+
| quote | **~190–270 ns** | ~500–740 ns | **~2–3× faster** |
|
|
214
|
+
| portfolio (3 positions) | **~0.9–1.0 µs** | ~1.6 µs | **~1.5× faster** |
|
|
215
|
+
| 200-position portfolio | **~26 µs** | ~64 µs | **~2.5× faster** |
|
|
216
|
+
|
|
217
|
+
Directable events serialize with **~0 B/op** (reusable scratch, no per-call
|
|
218
|
+
allocations, no JSON). End-to-end over a real WebSocket (`bench:throughput`):
|
|
219
|
+
**~1.18M msg/s**.
|
|
220
|
+
|
|
221
|
+
## Server options (all optional)
|
|
222
|
+
|
|
223
|
+
```ts
|
|
224
|
+
createServer({
|
|
225
|
+
port: 3000,
|
|
226
|
+
path: "/ws", // websocket path
|
|
227
|
+
inbound: ["chat"], // app events clients may SEND
|
|
228
|
+
backpressure: { highWaterMark: 1 << 20, policy: "drop-oldest", maxQueue: 256 },
|
|
229
|
+
replay: { historySize: 64 }, // per-topic last-value replay on subscribe
|
|
230
|
+
authenticate: async (req) => checkToken(req.headers.get("authorization")),
|
|
231
|
+
allowedOrigins: ["http://localhost:3000"],
|
|
232
|
+
token: "shared-secret", // or (tok) => boolean
|
|
233
|
+
maxConnections: 10_000,
|
|
234
|
+
maxMessageSize: 64 * 1024,
|
|
235
|
+
int64Guard: "warn", // "off" | "throw" | "warn"
|
|
236
|
+
nats: { servers: ["nats://localhost:4222"], inbound: true }, // optional NATS bridge
|
|
237
|
+
tls: { keyFile, certFile }, // enables wss://
|
|
238
|
+
});
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## Client options (all optional)
|
|
242
|
+
|
|
243
|
+
```ts
|
|
244
|
+
createClient("ws://...", {
|
|
245
|
+
reconnect: { initialDelay: 250, maxDelay: 30_000, jitter: true }, // or true/false
|
|
246
|
+
heartbeatMs: 15_000, heartbeatMisses: 2,
|
|
247
|
+
});
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
## Targeted sends, groups & NATS
|
|
251
|
+
|
|
252
|
+
```ts
|
|
253
|
+
// identity: authenticate() can pin the id + seed groups + attach metadata
|
|
254
|
+
createServer({
|
|
255
|
+
port: 3000,
|
|
256
|
+
authenticate: async (req) => {
|
|
257
|
+
const user = await whoIs(req); // e.g. from a JWT
|
|
258
|
+
return { id: user.id, groups: user.tier ? ["premium"] : [], meta: { name: user.name } };
|
|
259
|
+
},
|
|
260
|
+
nats: { servers: ["nats://localhost:4222"], inbound: true }, // optional bridge
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
// targeted sends / groups (server)
|
|
264
|
+
server.publishToClient("user-42", "quote", {...}); // one client by id (not bridged)
|
|
265
|
+
server.joinGroup("user-42", "eu"); // server-side grouping
|
|
266
|
+
server.publishToGroup("eu", "quote", {...}); // → ignex.group.eu.quote on NATS
|
|
267
|
+
server.getClients(); // [{ id, groups, topics, meta, connectedAt, ip }]
|
|
268
|
+
|
|
269
|
+
// clients can also join groups + learn their assigned id
|
|
270
|
+
client.joinGroup("beta");
|
|
271
|
+
client.leaveGroup("beta");
|
|
272
|
+
client.onStatus(() => console.log("my id:", client.clientId));
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
NATS consumers (any language) decode bridged frames from
|
|
276
|
+
`src/generated/fbs/backend.fbs` + the FNV-1a id map in `src/generated/wire-registry.json`
|
|
277
|
+
— see [docs/wire-format.md](docs/wire-format.md) and `examples/nats-consumer.ts`.
|
|
278
|
+
|
|
279
|
+
## Publishing to npm
|
|
280
|
+
|
|
281
|
+
Publish directly from source — `bun publish` runs the release gate
|
|
282
|
+
(`generate` → typecheck → lint → test) and `prepack` stages the native addon:
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
bun run release:dry # plan only — print what would happen
|
|
286
|
+
bun run release # patch bump → verify → publish → commit/tag/push
|
|
287
|
+
bun run release minor # minor bump
|
|
288
|
+
bun run release --version 0.2.0 # explicit version
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
CI (`.github/workflows/publish.yml`) builds `prebuilds/` for
|
|
292
|
+
ubuntu/macos/macos-13, merges them, verifies, and publishes on a `v*` tag or
|
|
293
|
+
`workflow_dispatch` with `NPM_TOKEN`. Full details: [docs/publishing.md](docs/publishing.md).
|
|
294
|
+
|
|
295
|
+
## Notes / limits
|
|
296
|
+
|
|
297
|
+
- The direct fast path covers flat types AND vectors of flat tables (packed
|
|
298
|
+
bridge). Events with nested tables-in-tables or unions fall back to JSON —
|
|
299
|
+
the path is observable via `server.getMetrics().pathCounts`.
|
|
300
|
+
- Plain `number` int64 fields lose precision above ±2^53-1 — use `Type.BigInt()`
|
|
301
|
+
for exact fields, or enable `int64Guard`.
|
|
302
|
+
- The output buffer is a single reusable scratch — the returned view is only
|
|
303
|
+
valid until the next `publish`; `publish` sends immediately (Bun copies), so
|
|
304
|
+
this is safe.
|
|
305
|
+
- Event ids are stable FNV-1a hashes (not insertion order) — regenerate all
|
|
306
|
+
artifacts together.
|
|
307
|
+
- The server is **Bun-only** (`bun:ffi`, `Bun.serve`); the client runs in
|
|
308
|
+
browser + Bun. Node server support is out of scope.
|
|
309
|
+
- Full gap-based replay (per-frame sequence numbers) is a documented future
|
|
310
|
+
extension; today the server replays bounded per-topic history on subscribe.
|
|
311
|
+
- Docs: [wire-format.md](docs/wire-format.md) (incl. building an independent
|
|
312
|
+
client), [architecture.md](docs/architecture.md). MIT licensed, CI on
|
|
313
|
+
ubuntu + macos.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
ignex-nova is a TypeBox-driven **FlatBuffer transport over Bun WebSockets** with
|
|
4
|
+
a Rust FFI serializer — hidden behind a typed pub/sub API.
|
|
5
|
+
|
|
6
|
+
## Pipeline
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
src/schema/index.ts (TypeBox — single source of truth: app events + control events)
|
|
10
|
+
│ scripts/generate.ts
|
|
11
|
+
▼
|
|
12
|
+
backend.fbs ──flatc --ts──▶ src/generated/ts/ (decoders, both sides)
|
|
13
|
+
└───flatc --rust──▶ rust/src/generated/ (table builders)
|
|
14
|
+
scripts/rust-glue-gen.ts ─▶ rust/src/transcode/ (JSON glue + direct-args FFI)
|
|
15
|
+
scripts/direct-gen.ts ────▶ src/generated/direct-ser.ts (zero-alloc server encoder)
|
|
16
|
+
scripts/ts-ser-gen.ts ─────▶ src/generated/ts-ser.ts (pure-JS browser encoder)
|
|
17
|
+
scripts/registry-gen.ts ───▶ src/generated/registry.ts (event routing, both sides)
|
|
18
|
+
scripts/generate.ts ────────▶ src/generated/wire-registry.json (name→id map for external consumers)
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Functional composition
|
|
22
|
+
|
|
23
|
+
The public API is built by **functional composition over an explicit state
|
|
24
|
+
object** — no classes, no `this`. `public/server.ts` and `public/client.ts`
|
|
25
|
+
are thin re-export shims that keep the npm entrypoints (`ignex-nova/server`,
|
|
26
|
+
`ignex-nova/client`) and the `dist` build stable; the implementation lives in
|
|
27
|
+
`src/core/`.
|
|
28
|
+
|
|
29
|
+
- **Composition roots** (`src/core/server.ts`, `src/core/client.ts`) — the only
|
|
30
|
+
places that know how the pieces fit together. `createServer(options)` builds
|
|
31
|
+
the `ServerState`, wires `Bun.serve`, and returns a plain API object;
|
|
32
|
+
`createClient(url, opts)` builds the client state and returns a plain API
|
|
33
|
+
object with closures over it.
|
|
34
|
+
- **Action modules** are pure-ish functions `(state, ...) => ...` that read/
|
|
35
|
+
mutate the explicit state: `auth` (upgrade gate), `rooms` (membership +
|
|
36
|
+
fan-out), `replay` (per-topic history), `backpressure` (pure `decide` → send/
|
|
37
|
+
enqueue/drop/close), `outbound` (the only place that touches `ws.send` and
|
|
38
|
+
the per-socket queue), `routing` (inbound dispatch), and the client's
|
|
39
|
+
`client-wire` / `client-reconnect` (pure backoff math) / `client-heartbeat`.
|
|
40
|
+
- **Factories** for encapsulated mutable state: `createMetrics()`, `createScratch()`
|
|
41
|
+
(reusable zero-alloc output buffer), `createStats()` (encode-path counters).
|
|
42
|
+
`int64-guard` intentionally stays a module-global (cheap `off`-mode no-op) so
|
|
43
|
+
threading it through the generated encoders can't cost the ~200ns hot path.
|
|
44
|
+
- **Dev discipline**: `bun run lint` (oxlint with FP rules: no-var,
|
|
45
|
+
prefer-const, no-param-reassign `props:false`, prefer-arrow-callback,
|
|
46
|
+
no-else-return, consistent-return, no-nested-ternary, no-loop-func, …) runs
|
|
47
|
+
in CI after `tsc`. Perf is gated by `bench/BASELINE.md` — re-run
|
|
48
|
+
`bench:serialize` + `bench:throughput` before/after hot-path changes and
|
|
49
|
+
fail-fast on >±5% drift or new allocations.
|
|
50
|
+
|
|
51
|
+
## Encode paths (server)
|
|
52
|
+
|
|
53
|
+
- **Direct fast path (zero-alloc)** — directable events (flat types + packed
|
|
54
|
+
vectors) are pushed straight into Rust FFI args from a reusable scratch; the
|
|
55
|
+
output is written into a single reusable buffer. ~0 B/op.
|
|
56
|
+
- **JSON fallback** — nested tables/unions (e.g. `order`) serialize through
|
|
57
|
+
`JSON.stringify → fb_serialize → serde_json`. Slower + allocating; the
|
|
58
|
+
per-event encode path is observable via `server.getMetrics().pathCounts`.
|
|
59
|
+
- A payload containing an embedded NUL is routed to the JSON path (the
|
|
60
|
+
`cstring` direct path would silently truncate it).
|
|
61
|
+
|
|
62
|
+
## Encode path (client / browser)
|
|
63
|
+
|
|
64
|
+
Browsers have no Rust FFI, so outgoing frames (app events **and** control
|
|
65
|
+
frames) are encoded by `generated/ts-ser.ts` — flatc's object API (`XxxT` +
|
|
66
|
+
`pack`) over a pooled `flatbuffers.Builder`. No JSON.
|
|
67
|
+
|
|
68
|
+
## Runtime layout
|
|
69
|
+
|
|
70
|
+
- `public/server.ts`, `public/client.ts`, `public/nats.ts` — thin re-export shims
|
|
71
|
+
(npm entrypoints `ignex-nova/server` / `client` / `nats` + the `dist` build).
|
|
72
|
+
Implementation is in `src/core/` + `src/bridge/`.
|
|
73
|
+
- `src/core/server.ts` — `createServer` composition root: `Bun.serve`, client
|
|
74
|
+
registry (id → socket), rooms, groups, inbound routing, control frames,
|
|
75
|
+
auth/origin/token gates, backpressure, replay history, metrics, NATS bridge
|
|
76
|
+
hook, graceful drain, `/health` + `/clients`.
|
|
77
|
+
- `src/core/client.ts` — `createClient` composition root: typed
|
|
78
|
+
`on`/`send`/`subscribe`/`joinGroup`, reconnect with backoff, heartbeat, status
|
|
79
|
+
events, `clientId`/`groups` (from the `welcome` control frame).
|
|
80
|
+
- `src/core/{state,auth,rooms,groups,replay,backpressure,outbound,routing}.ts` —
|
|
81
|
+
server action modules over the explicit `ServerState`. `groups.ts` mirrors
|
|
82
|
+
`rooms.ts` (targeting sets, no replay); `state.clients` is the id→socket
|
|
83
|
+
registry, `state.groups` the group→members index.
|
|
84
|
+
- `src/core/{client-state,client-wire,client-reconnect,client-heartbeat}.ts` —
|
|
85
|
+
client action modules over the explicit client state.
|
|
86
|
+
- `src/bridge/{nats,subjects}.ts` — optional NATS bridge: `createNatsBridge`
|
|
87
|
+
(injectable `NatsTransport` for tests; eager non-blocking connect with retry),
|
|
88
|
+
subject builders (`ignex.broadcast.*` / `ignex.topic.*` / `ignex.group.*` /
|
|
89
|
+
inbound `ignex.inbound.>`). Outbound frames are copied from the shared
|
|
90
|
+
scratch; inbound frames are decoded via `readFrameHeader`/`decodePayload` and
|
|
91
|
+
forwarded to clients (never re-bridged).
|
|
92
|
+
- `src/core/metrics.ts`, `src/core/int64-guard.ts` — `createMetrics()` factory
|
|
93
|
+
+ the exact-int64 safety net.
|
|
94
|
+
- `src/transport/{transport,scratch,stats}.ts` — object → frame encoding:
|
|
95
|
+
`encodeToScratch` (direct/JSON), the reusable zero-alloc scratch, and
|
|
96
|
+
encode-path stats.
|
|
97
|
+
- `src/generated/` — flatc `--ts`/`--rust` output + `registry.ts` +
|
|
98
|
+
`direct-ser.ts` + `ts-ser.ts` + `wire-registry.json` (emitted; regenerated by
|
|
99
|
+
`bun run generate`).
|
|
100
|
+
- `src/schema/index.ts` — TypeBox source of truth.
|
|
101
|
+
- `src/native/*` — the only place that talks to Rust: `dlopen` binding with a
|
|
102
|
+
bind-time self-test (a failing direct symbol is disabled → JSON fallback),
|
|
103
|
+
`buffer`/`buffer_length` ABI probing, per-platform addon resolution.
|
|
104
|
+
- `rust/` — a cdylib exporting `#[no_mangle] extern "C"` symbols, `panic_guard`-
|
|
105
|
+
wrapped, `thread_local` reused `FlatBufferBuilder`.
|
|
106
|
+
|
|
107
|
+
## Client identity, targeting & the NATS bridge
|
|
108
|
+
|
|
109
|
+
- **Identity**: `authenticate(req)` may return `{ id, groups, meta }`; otherwise
|
|
110
|
+
a UUID is assigned at upgrade (`auth.ts`). Ids are de-duplicated (409 at
|
|
111
|
+
upgrade, stale-session kick at open). `state.clients: Map<id, socket>` is the
|
|
112
|
+
live registry; the server sends the `welcome` control frame so each client
|
|
113
|
+
knows its id + server-side groups.
|
|
114
|
+
- **Targeting**: `publishToClient(id, …)` (single socket, not bridged),
|
|
115
|
+
`publishToGroup(group, …)` (server-side targeting sets — from auth metadata,
|
|
116
|
+
`joinGroup(id, group)`, or client `joinGroup` control frames).
|
|
117
|
+
- **Bridge**: `createServer({ nats })` creates a `NatsBridge`. The fan-out path
|
|
118
|
+
(`fanOutAll`) encodes once and reuses that frame for WS clients AND NATS
|
|
119
|
+
(`frame.slice()` copy — the scratch is reused). Subjects are derived by
|
|
120
|
+
`src/bridge/subjects.ts`. Inbound NATS events are decoded and forwarded via
|
|
121
|
+
`fanOutAll` (no bridge call → loop prevention). All bridge counters fold into
|
|
122
|
+
`server.getMetrics()`.
|
|
123
|
+
|
|
124
|
+
## Why it's fast
|
|
125
|
+
|
|
126
|
+
- Zero-alloc server encode (reusable scratch + Bun's synchronous `ws.send`
|
|
127
|
+
copy).
|
|
128
|
+
- FlatBuffers = zero-parse reads, no JSON on the hot path.
|
|
129
|
+
- flatc guarantees the Rust builders and TS decoders agree byte-for-byte.
|
|
130
|
+
|
|
131
|
+
## Runtime & platform constraints
|
|
132
|
+
|
|
133
|
+
- **Server is Bun-only** (`bun:ffi`, `Bun.serve`). Node is not supported.
|
|
134
|
+
- The native addon is per-OS (`.so`/`.dylib`/`.dll`) — see
|
|
135
|
+
`src/native/loader.ts`. Build it with `cargo build --release`.
|
|
136
|
+
- The **client** works in Bun AND browsers (bundle with
|
|
137
|
+
`bun build --target=browser`).
|
|
138
|
+
|
|
139
|
+
## Known limits (documented, not hidden)
|
|
140
|
+
|
|
141
|
+
- Plain `number` int64 fields lose precision above ±2^53-1 — use
|
|
142
|
+
`Type.BigInt()` fields for exact values, or enable `int64Guard`.
|
|
143
|
+
- The direct fast path covers flat types + packed vectors; nested single-object
|
|
144
|
+
tables fall back to JSON (observable via metrics).
|
|
145
|
+
- Full gap-based replay (per-frame sequence numbers) is a future extension;
|
|
146
|
+
today the server replays bounded per-topic history on subscribe.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Publishing ignex-nova to npm
|
|
2
|
+
|
|
3
|
+
ignex-nova is published **from source** — the tarball contains TypeScript
|
|
4
|
+
entrypoints (Bun runs `.ts` natively, so consumers need no build step), the
|
|
5
|
+
generated artifacts, the `rust/` source, and prebuilt native addons. This
|
|
6
|
+
mirrors how the `@ignex/*` packages in the Ignex monorepo are shipped.
|
|
7
|
+
|
|
8
|
+
## Package layout (`package.json`)
|
|
9
|
+
|
|
10
|
+
| Field | Value | Why |
|
|
11
|
+
| --- | --- | --- |
|
|
12
|
+
| `main` / `module` / `types` | `./index.ts` | source entrypoint (Bun-native) |
|
|
13
|
+
| `exports` | `.` → `index.ts`, `./server` → `public/server.ts`, `./client` → `public/client.ts`, `./nats` → `public/nats.ts`, `./package.json` | typed subpath API |
|
|
14
|
+
| `files` | `index.ts`, `public`, `src`, `rust`, `prebuilds`, `docs`, `README.md`, `LICENSE` | everything consumers need, nothing they don't |
|
|
15
|
+
| `publishConfig` | `{ "access": "public" }` | unscoped package must be public |
|
|
16
|
+
| `engines` | `{ "bun": ">=1.4" }` | Bun-only runtime |
|
|
17
|
+
| `sideEffects` | `false` | safe to tree-shake / mark in bundlers |
|
|
18
|
+
|
|
19
|
+
`rust/.npmignore` keeps `rust/target/` (build output), `Cargo.lock` and the
|
|
20
|
+
dev example out of the tarball while still shipping the Rust **source** so
|
|
21
|
+
consumers can rebuild the addon on any platform.
|
|
22
|
+
|
|
23
|
+
## The release pipeline
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
bun run release ──► bump version
|
|
27
|
+
│
|
|
28
|
+
├─► verify (typecheck + lint + test)
|
|
29
|
+
├─► pack:check (tarball contents gate)
|
|
30
|
+
├─► bun publish
|
|
31
|
+
│ ├─ prepublishOnly → generate + verify
|
|
32
|
+
│ └─ prepack → prebuild (stage addon)
|
|
33
|
+
└─► git commit + tag vX.Y.Z (+ push)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### 1. Version bump
|
|
37
|
+
`scripts/release.ts` supports `patch | minor | major` (default `patch`) or an
|
|
38
|
+
explicit `--version`. Prereleases finalize on the next bump
|
|
39
|
+
(`0.2.0-beta.1` → `0.2.0`).
|
|
40
|
+
|
|
41
|
+
### 2. Verify gate
|
|
42
|
+
`bun run verify` = `typecheck` + `lint` + `test`. `prepublishOnly` first runs
|
|
43
|
+
`generate` so the TypeBox → `.fbs` → flatc → glue artifacts are always fresh
|
|
44
|
+
in the tarball (they're gitignored, so they must be regenerated at publish).
|
|
45
|
+
|
|
46
|
+
### 3. Tarball check
|
|
47
|
+
`bun run pack:check` runs `bun pm pack --dry-run --json` and asserts:
|
|
48
|
+
|
|
49
|
+
- required files present: entrypoints, `src/`, `rust/Cargo.toml`, docs, README, LICENSE
|
|
50
|
+
- nothing heavy/private leaks: `rust/target/`, `dist/`, `node_modules/`, `client-dist/`, tests, benches
|
|
51
|
+
|
|
52
|
+
### 4. Native addon staging
|
|
53
|
+
`prepack` runs `bun run prebuild` → `scripts/build-prebuild.ts`:
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
cargo build --release → rust/target/release/libignex_ffi.{so,dylib,dll}
|
|
57
|
+
cp → prebuilds/<platform>-<arch>/libignex_ffi.{so,dylib,dll}
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The loader (`src/native/loader.ts`) resolves the addon in this order:
|
|
61
|
+
|
|
62
|
+
1. `IGNEX_FFI_PATH` env override
|
|
63
|
+
2. in-repo dev build: `<repo>/rust/target/release/<lib>`
|
|
64
|
+
3. packaged layout: `<pkg>/prebuilds/<platform>-<arch>/<lib>`
|
|
65
|
+
|
|
66
|
+
So consumers on a platform with a shipped prebuild need **zero setup**; others
|
|
67
|
+
rebuild from the shipped `rust/` source or set `IGNEX_FFI_PATH`.
|
|
68
|
+
|
|
69
|
+
### 5. Publish
|
|
70
|
+
`bun publish` (equivalent to `npm publish`) with `--tag <dist-tag>`
|
|
71
|
+
(default `latest`) and `--access public`.
|
|
72
|
+
|
|
73
|
+
## Releasing
|
|
74
|
+
|
|
75
|
+
### Manual (from a checkout)
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
bun run release:dry # plan only
|
|
79
|
+
bun run release # patch bump → publish → commit + tag + push
|
|
80
|
+
bun run release minor --tag beta # minor + dist-tag `beta`
|
|
81
|
+
bun run release --version 0.2.0 --no-verify --no-check
|
|
82
|
+
bun run release --no-commit # bump + publish, no git side effects
|
|
83
|
+
bun run release --no-bump --no-verify --no-commit # retry publish as-is
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
First release from a fresh checkout needs the generated artifacts + a built
|
|
87
|
+
addon — `release` handles both via `prepublishOnly`/`prepack`, but you need a
|
|
88
|
+
local Rust toolchain + `flatc` (see README "Prerequisites").
|
|
89
|
+
|
|
90
|
+
### CI (`.github/workflows/publish.yml`)
|
|
91
|
+
|
|
92
|
+
Triggered by a `v*` tag push or `workflow_dispatch` (with optional `version`
|
|
93
|
+
and `dist_tag` inputs). It builds prebuilds for **ubuntu-latest**
|
|
94
|
+
(linux-x64), **macos-latest** (darwin-arm64) and **macos-13** (darwin-x64),
|
|
95
|
+
merges them into `prebuilds/`, sets the version from the tag/input, runs the
|
|
96
|
+
full gate, then publishes.
|
|
97
|
+
|
|
98
|
+
Set the `NPM_TOKEN` repo secret (an npm access token with publish rights) for
|
|
99
|
+
the publish step. Provenance/attestations can be enabled by using
|
|
100
|
+
`npm publish --provenance` and an `id-token: write` permission (the workflow
|
|
101
|
+
already grants it).
|
|
102
|
+
|
|
103
|
+
## Pre-publish checklist
|
|
104
|
+
|
|
105
|
+
- [ ] `bun run verify` passes locally
|
|
106
|
+
- [ ] `bun run pack:check` shows the expected files and no `rust/target/`
|
|
107
|
+
- [ ] `prebuilds/` contains the addon(s) you intend to ship
|
|
108
|
+
- [ ] npm auth works: `npm whoami`, and `NPM_TOKEN` is set for CI publishes
|
|
109
|
+
- [ ] version is correct (`bun run release --version X.Y.Z`)
|
|
110
|
+
|
|
111
|
+
## Tarball hygiene notes
|
|
112
|
+
|
|
113
|
+
- `files` is an allowlist — only the listed top-level entries are packed.
|
|
114
|
+
- Nested `rust/.npmignore` excludes `rust/target/` (platform-specific build
|
|
115
|
+
output, GB-scale) and `Cargo.lock` (library crates don't commit it).
|
|
116
|
+
- `prebuilds/` is empty (or absent) until `prepack`/CI stages addons, so a
|
|
117
|
+
source-only tarball is fine too — the loader just falls back to rebuild/`IGNEX_FFI_PATH`.
|
|
118
|
+
- The gitignored `src/generated/` artifacts ARE packed (they're under the
|
|
119
|
+
included `src/`) — that's intentional: consumers must not need `flatc`.
|