@delali/sirannon-db 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +415 -39
- package/dist/backup-scheduler/index.d.ts +2 -2
- package/dist/change-tracker-CFTQ9TSn.d.ts +89 -0
- package/dist/chunk-3MCMONVP.mjs +115 -0
- package/dist/chunk-ER7ODTDA.mjs +23 -0
- package/dist/chunk-GS7T5YMI.mjs +51 -0
- package/dist/{chunk-AX66KWBR.mjs → chunk-UTO3ZAFS.mjs} +226 -64
- package/dist/chunk-UVMVN3OT.mjs +111 -0
- package/dist/client/index.d.ts +99 -42
- package/dist/client/index.mjs +726 -26
- package/dist/core/index.d.ts +11 -108
- package/dist/core/index.mjs +134 -168
- package/dist/{sirannon-B1oTfebD.d.ts → database-BVY1GqE7.d.ts} +8 -33
- package/dist/errors-C00ed08Q.d.ts +101 -0
- package/dist/file-migrations/index.d.ts +2 -2
- package/dist/{index-hXiis3N-.d.ts → index-CLdNrcPz.d.ts} +1 -1
- package/dist/replication/coordinator/etcd.d.ts +44 -0
- package/dist/replication/coordinator/etcd.mjs +650 -0
- package/dist/replication/index.d.ts +491 -0
- package/dist/replication/index.mjs +3784 -0
- package/dist/server/index.d.ts +14 -3
- package/dist/server/index.mjs +262 -44
- package/dist/sirannon-Cd-lK6T0.d.ts +31 -0
- package/dist/transport/grpc.d.ts +316 -0
- package/dist/transport/grpc.mjs +3341 -0
- package/dist/transport/memory.d.ts +221 -0
- package/dist/transport/memory.mjs +337 -0
- package/dist/types-B2byqt0B.d.ts +273 -0
- package/dist/types-BEu1I_9_.d.ts +139 -0
- package/dist/types-BeozgNPr.d.ts +26 -0
- package/dist/{types-DtDutWRU.d.ts → types-D-74JiXb.d.ts} +78 -2
- package/package.json +54 -10
- package/dist/types-DRkJlqex.d.ts +0 -38
package/README.md
CHANGED
|
@@ -6,22 +6,22 @@
|
|
|
6
6
|
[](https://www.npmjs.com/package/@delali/sirannon-db)
|
|
7
7
|
[](https://github.com/assetcorp/sirannon-db/blob/main/LICENSE)
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
Build a networked SQLite service with connection pooling, change data capture, migrations, backups, and a client SDK. Applications reach Sirannon over HTTP or WebSocket, while Sirannon nodes replicate primary-owned changes over gRPC. Coordinator mode adds etcd-backed authority and automatic failover.
|
|
10
10
|
|
|
11
11
|
> *sirannon* means 'gate-stream' in Sindarin.
|
|
12
12
|
|
|
13
13
|
## Install
|
|
14
14
|
|
|
15
15
|
```bash
|
|
16
|
-
pnpm add @delali/sirannon-db
|
|
16
|
+
pnpm add -E @delali/sirannon-db
|
|
17
17
|
```
|
|
18
18
|
|
|
19
19
|
Then install the SQLite driver for your platform:
|
|
20
20
|
|
|
21
21
|
```bash
|
|
22
|
-
pnpm add better-sqlite3 # Node.js
|
|
23
|
-
pnpm add wa-sqlite # Browser (IndexedDB persistence)
|
|
24
|
-
pnpm add expo-sqlite # React Native (Expo)
|
|
22
|
+
pnpm add -E better-sqlite3 # Node.js
|
|
23
|
+
pnpm add -E wa-sqlite # Browser (IndexedDB persistence)
|
|
24
|
+
pnpm add -E expo-sqlite # React Native (Expo)
|
|
25
25
|
# Node 22+ built-in sqlite and Bun need no extra package
|
|
26
26
|
```
|
|
27
27
|
|
|
@@ -30,7 +30,7 @@ pnpm add expo-sqlite # React Native (Expo)
|
|
|
30
30
|
### Node.js
|
|
31
31
|
|
|
32
32
|
```bash
|
|
33
|
-
pnpm add @delali/sirannon-db better-sqlite3
|
|
33
|
+
pnpm add -E @delali/sirannon-db better-sqlite3
|
|
34
34
|
```
|
|
35
35
|
|
|
36
36
|
```ts
|
|
@@ -58,7 +58,7 @@ const driver = nodeSqlite()
|
|
|
58
58
|
### Browser
|
|
59
59
|
|
|
60
60
|
```bash
|
|
61
|
-
pnpm add @delali/sirannon-db wa-sqlite
|
|
61
|
+
pnpm add -E @delali/sirannon-db wa-sqlite
|
|
62
62
|
```
|
|
63
63
|
|
|
64
64
|
The browser driver persists data to IndexedDB through a WebAssembly SQLite build. Use `Database.create` directly since `Sirannon` registries are designed for server-side use.
|
|
@@ -82,7 +82,7 @@ const users = await db.query<{ id: number; name: string }>('SELECT * FROM users'
|
|
|
82
82
|
### React Native (Expo)
|
|
83
83
|
|
|
84
84
|
```bash
|
|
85
|
-
pnpm add @delali/sirannon-db expo-sqlite
|
|
85
|
+
pnpm add -E @delali/sirannon-db expo-sqlite
|
|
86
86
|
```
|
|
87
87
|
|
|
88
88
|
```ts
|
|
@@ -128,11 +128,11 @@ Sirannon-db separates the database engine from the library. You pick the driver
|
|
|
128
128
|
|
|
129
129
|
| Driver | Import | Runtime | Install |
|
|
130
130
|
| --- | --- | --- | --- |
|
|
131
|
-
| better-sqlite3 | `@delali/sirannon-db/driver/better-sqlite3` | Node.js | `pnpm add better-sqlite3` |
|
|
131
|
+
| better-sqlite3 | `@delali/sirannon-db/driver/better-sqlite3` | Node.js | `pnpm add -E better-sqlite3` |
|
|
132
132
|
| Node built-in | `@delali/sirannon-db/driver/node` | Node.js >= 22 | None (use `--experimental-sqlite` flag) |
|
|
133
|
-
| wa-sqlite | `@delali/sirannon-db/driver/wa-sqlite` | Browser | `pnpm add wa-sqlite` |
|
|
133
|
+
| wa-sqlite | `@delali/sirannon-db/driver/wa-sqlite` | Browser | `pnpm add -E wa-sqlite` |
|
|
134
134
|
| Bun | `@delali/sirannon-db/driver/bun` | Bun | None (uses `bun:sqlite`) |
|
|
135
|
-
| Expo | `@delali/sirannon-db/driver/expo` | React Native | `pnpm add expo-sqlite` |
|
|
135
|
+
| Expo | `@delali/sirannon-db/driver/expo` | React Native | `pnpm add -E expo-sqlite` |
|
|
136
136
|
|
|
137
137
|
```ts
|
|
138
138
|
import { betterSqlite3 } from '@delali/sirannon-db/driver/better-sqlite3'
|
|
@@ -158,6 +158,9 @@ The package ships independent exports so you only bundle what you need:
|
|
|
158
158
|
| `@delali/sirannon-db/file-migrations` | Load `.up.sql` / `.down.sql` files from a directory |
|
|
159
159
|
| `@delali/sirannon-db/server` | HTTP + WebSocket server powered by uWebSockets.js |
|
|
160
160
|
| `@delali/sirannon-db/client` | Browser/Node.js client SDK with auto-reconnect and subscription restore |
|
|
161
|
+
| `@delali/sirannon-db/replication` | Replication engine, conflict resolvers, topologies, HLC |
|
|
162
|
+
| `@delali/sirannon-db/transport/grpc` | gRPC replication transport with TLS support |
|
|
163
|
+
| `@delali/sirannon-db/transport/memory` | In-memory transport for testing |
|
|
161
164
|
|
|
162
165
|
## Core features
|
|
163
166
|
|
|
@@ -435,52 +438,391 @@ await httpDb.transaction([
|
|
|
435
438
|
httpClient.close()
|
|
436
439
|
```
|
|
437
440
|
|
|
441
|
+
## Distributed replication
|
|
442
|
+
|
|
443
|
+
Sirannon can replicate a SQLite database across multiple nodes with change propagation, new-node bootstrapping, write concerns, and coordinator-backed failover. The production path is primary-replica: one primary accepts writes, replicas serve reads and can forward writes, and coordinator mode manages authority when failover is enabled. When replication is not enabled, the replication engine does not run.
|
|
444
|
+
|
|
445
|
+
```ts
|
|
446
|
+
import { ReplicationEngine } from '@delali/sirannon-db/replication'
|
|
447
|
+
import { InMemoryTransport, MemoryBus } from '@delali/sirannon-db/transport/memory'
|
|
448
|
+
import { GrpcReplicationTransport } from '@delali/sirannon-db/transport/grpc'
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
### Client and replication transports
|
|
452
|
+
|
|
453
|
+
Sirannon has two transport interfaces with different responsibilities:
|
|
454
|
+
|
|
455
|
+
| Traffic | Interface | Built-in network transport |
|
|
456
|
+
| --- | --- | --- |
|
|
457
|
+
| Application queries, writes, and CDC subscriptions | Client `Transport` | HTTP or WebSocket |
|
|
458
|
+
| Change batches, acknowledgements, write forwarding, and first sync between Sirannon nodes | `ReplicationTransport` | gRPC |
|
|
459
|
+
|
|
460
|
+
`WebSocketTransport` conforms to the client `Transport` interface. It connects an application to the Sirannon server and does not conform to `ReplicationTransport`. Use `GrpcReplicationTransport` for production node-to-node replication, or `InMemoryTransport` for tests and single-process scenarios.
|
|
461
|
+
|
|
462
|
+
### Primary-replica setup
|
|
463
|
+
|
|
464
|
+
One node accepts writes and pushes changes to read replicas. Replicas forward writes to the primary when `writeForwarding` is enabled.
|
|
465
|
+
|
|
466
|
+
```ts
|
|
467
|
+
import { ReplicationEngine, PrimaryReplicaTopology } from '@delali/sirannon-db/replication'
|
|
468
|
+
import { GrpcReplicationTransport } from '@delali/sirannon-db/transport/grpc'
|
|
469
|
+
|
|
470
|
+
const transport = new GrpcReplicationTransport({
|
|
471
|
+
host: '0.0.0.0',
|
|
472
|
+
port: 4200,
|
|
473
|
+
tlsCert: './certs/primary.crt',
|
|
474
|
+
tlsKey: './certs/primary.key',
|
|
475
|
+
tlsCaCert: './certs/ca.crt',
|
|
476
|
+
})
|
|
477
|
+
|
|
478
|
+
const engine = new ReplicationEngine(db, writerConn, {
|
|
479
|
+
nodeId: 'primary-us-east-1',
|
|
480
|
+
topology: new PrimaryReplicaTopology('primary'),
|
|
481
|
+
transport,
|
|
482
|
+
snapshotConnectionFactory: () => driver.open(dbPath, { readonly: true }),
|
|
483
|
+
changeTracker: tracker,
|
|
484
|
+
})
|
|
485
|
+
|
|
486
|
+
await engine.start()
|
|
487
|
+
|
|
488
|
+
await engine.execute('INSERT INTO orders (id, total) VALUES (?, ?)', [1, 4999])
|
|
489
|
+
|
|
490
|
+
const rows = await engine.query<{ id: number }>('SELECT * FROM orders')
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
On the replica side:
|
|
494
|
+
|
|
495
|
+
```ts
|
|
496
|
+
const replicaEngine = new ReplicationEngine(replicaDb, replicaConn, {
|
|
497
|
+
nodeId: 'replica-eu-west-1',
|
|
498
|
+
topology: new PrimaryReplicaTopology('replica'),
|
|
499
|
+
transport: replicaTransport,
|
|
500
|
+
transportConfig: { endpoints: ['primary.example.com:4200'] },
|
|
501
|
+
writeForwarding: true,
|
|
502
|
+
changeTracker: replicaTracker,
|
|
503
|
+
})
|
|
504
|
+
|
|
505
|
+
await replicaEngine.start()
|
|
506
|
+
```
|
|
507
|
+
|
|
508
|
+
When `initialSync` is `true` (the default), a new replica automatically pulls a full snapshot from the primary before accepting reads. The replica blocks reads and writes until the sync completes and incremental catch-up reaches the configured lag threshold.
|
|
509
|
+
|
|
510
|
+
### Coordinator-backed failover
|
|
511
|
+
|
|
512
|
+
Coordinator mode stores primary authority, node sessions, replication-group state, and the in-sync set in a `ClusterCoordinator`. The TypeScript package includes an etcd adapter:
|
|
513
|
+
|
|
514
|
+
```ts
|
|
515
|
+
import { readFileSync } from 'node:fs'
|
|
516
|
+
import { createEtcdCoordinator } from '@delali/sirannon-db/replication/coordinator/etcd'
|
|
517
|
+
|
|
518
|
+
const coordinator = createEtcdCoordinator({
|
|
519
|
+
hosts: ['https://etcd-1.internal:2379', 'https://etcd-2.internal:2379'],
|
|
520
|
+
keyPrefix: '/sirannon/orders',
|
|
521
|
+
credentials: {
|
|
522
|
+
rootCertificate: readFileSync('./certs/etcd-ca.crt'),
|
|
523
|
+
privateKey: readFileSync('./certs/orders-node.key'),
|
|
524
|
+
certChain: readFileSync('./certs/orders-node.crt'),
|
|
525
|
+
},
|
|
526
|
+
})
|
|
527
|
+
|
|
528
|
+
const engine = new ReplicationEngine(db, writerConn, {
|
|
529
|
+
nodeId: 'orders-node-a',
|
|
530
|
+
topology: new PrimaryReplicaTopology('primary'),
|
|
531
|
+
transport,
|
|
532
|
+
transportConfig: {
|
|
533
|
+
endpoints: ['orders-node-b.internal:4200', 'orders-node-c.internal:4200'],
|
|
534
|
+
protocolVersion: '1',
|
|
535
|
+
},
|
|
536
|
+
changeTracker: tracker,
|
|
537
|
+
snapshotConnectionFactory: () => driver.open(dbPath, { readonly: true }),
|
|
538
|
+
writeForwarding: true,
|
|
539
|
+
coordinator: {
|
|
540
|
+
clusterId: 'commerce-production',
|
|
541
|
+
groupId: 'orders',
|
|
542
|
+
endpoint: 'https://orders-node-a.internal/db/orders',
|
|
543
|
+
coordinator,
|
|
544
|
+
votingDataBearingNodeIds: ['orders-node-a', 'orders-node-b', 'orders-node-c'],
|
|
545
|
+
controller: true,
|
|
546
|
+
},
|
|
547
|
+
})
|
|
548
|
+
```
|
|
549
|
+
|
|
550
|
+
Every coordinator-mode node needs a stable, persisted `nodeId`. `votingDataBearingNodeIds` creates the replication group when it does not exist; later nodes read the registered group from etcd. Production coordinator access requires HTTPS plus an authenticated identity. The in-memory coordinator and `allowInsecure: true` are for tests and local development.
|
|
551
|
+
|
|
552
|
+
Automatic write failover needs at least three voting data-bearing nodes. With fewer than three voters, the controller cannot prove majority authority after losing a node and keeps writes unavailable.
|
|
553
|
+
|
|
554
|
+
### Conflict resolution
|
|
555
|
+
|
|
556
|
+
Normal writes are serialised through one primary per replication group. When a receiver applies a batch and finds the target row already present, it passes the local and incoming versions to the configured resolver. This is part of normal batch application, not a separate repair command.
|
|
557
|
+
|
|
558
|
+
Three built-in strategies ship with the replication module:
|
|
559
|
+
|
|
560
|
+
| Strategy | Class | Behaviour |
|
|
561
|
+
| --- | --- | --- |
|
|
562
|
+
| Last-Writer-Wins | `LWWResolver` | Selects the change with the higher HLC timestamp. Ties break by node ID. |
|
|
563
|
+
| Field-Level Merge | `FieldMergeResolver` | Merges non-overlapping columns and uses per-column HLC metadata for overlapping columns. Falls back to whole-row LWW when column metadata is unavailable. |
|
|
564
|
+
| Primary Wins | `PrimaryWinsResolver` | Selects the version authored by a configured primary node ID. Falls back to LWW when neither version came from that node. |
|
|
565
|
+
|
|
566
|
+
Custom resolvers can be built by creating a class with a `resolve(ctx: ConflictContext): ConflictResolution` method.
|
|
567
|
+
|
|
568
|
+
Coordinator mode quarantines a returning former primary when it contains local-only writes. It does not merge that history into the current primary or expose a force-promotion or high-level repair API. An operator must rebuild, restore, or otherwise remediate the faulted node before rejoining it.
|
|
569
|
+
|
|
570
|
+
### First sync
|
|
571
|
+
|
|
572
|
+
When a new node joins a running cluster, it needs the full dataset before it can process incremental changes. The sync protocol handles this automatically:
|
|
573
|
+
|
|
574
|
+
1. The joiner connects and sends a sync request to the source
|
|
575
|
+
2. The source opens a consistent read-only snapshot and sends schema DDL (CREATE TABLE, CREATE INDEX)
|
|
576
|
+
3. The source streams table data in configurable batches (default 10,000 rows) with per-batch checksums
|
|
577
|
+
4. After all data is transferred, the source sends a manifest with row counts and primary-key hashes
|
|
578
|
+
5. The joiner verifies the manifest, transitions to catch-up mode, and applies incremental changes accumulated during the transfer
|
|
579
|
+
6. Once the replication lag drops below `maxSyncLagBeforeReady`, the joiner starts serving reads
|
|
580
|
+
|
|
581
|
+
The state machine is: `pending` -> `syncing` -> `catching-up` -> `ready`. You can monitor it via `engine.status().syncState`.
|
|
582
|
+
|
|
583
|
+
For large databases where a network transfer is impractical, the out-of-band path lets you copy the SQLite file directly and start from a known sequence:
|
|
584
|
+
|
|
585
|
+
```ts
|
|
586
|
+
const engine = new ReplicationEngine(db, writerConn, {
|
|
587
|
+
initialSync: false,
|
|
588
|
+
resumeFromSeq: 50000n,
|
|
589
|
+
// ...
|
|
590
|
+
})
|
|
591
|
+
```
|
|
592
|
+
|
|
593
|
+
### Write concerns
|
|
594
|
+
|
|
595
|
+
Control how many replicas must acknowledge a write before it returns:
|
|
596
|
+
|
|
597
|
+
```ts
|
|
598
|
+
await engine.execute(
|
|
599
|
+
'INSERT INTO orders (id, total) VALUES (?, ?)',
|
|
600
|
+
[1, 4999],
|
|
601
|
+
{ writeConcern: { level: 'majority', timeoutMs: 5000 } },
|
|
602
|
+
)
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
In static mode, omitting `writeConcern` returns after the local commit. In coordinator mode, omitting it selects `'majority'`. You can request `'local'`, `'majority'`, or `'all'` explicitly. A coordinator-mode `'local'` write is not protected against loss during automatic failover.
|
|
606
|
+
|
|
607
|
+
In coordinator mode, `majority` is calculated from configured voting data-bearing nodes in the replication group, including the primary's local durable commit. It is not calculated from the peers currently connected to this process. A successful coordinator-mode `majority` write survives automatic primary failover when only the failed primary is lost and an eligible in-sync replica remains.
|
|
608
|
+
|
|
609
|
+
### Replication FAQ
|
|
610
|
+
|
|
611
|
+
#### Is this SQLite over a shared network file system?
|
|
612
|
+
|
|
613
|
+
No. Each node owns its own SQLite database file. Sirannon moves change batches through its replication transport and exposes database operations through the server and client layers. It does not rely on many machines opening the same SQLite file over NFS or another shared network file system.
|
|
614
|
+
|
|
615
|
+
#### What is replicated?
|
|
616
|
+
|
|
617
|
+
Sirannon replicates checksummed batches of `ReplicationChange` records. Each change carries the table, operation, row ID, primary key, HLC timestamp, transaction ID, node ID, old data, new data, and optional DDL statement.
|
|
618
|
+
|
|
619
|
+
#### Is the protocol row-based, statement-based, operation-log based, or CRDT-like?
|
|
620
|
+
|
|
621
|
+
It is operation-log based at the Sirannon layer. Data changes carry row images and primary-key metadata. DDL changes carry a validated DDL statement. The current production write path is not CRDT-like; it prevents normal write conflicts with a single writable primary.
|
|
622
|
+
|
|
623
|
+
#### What ordering model does it use?
|
|
624
|
+
|
|
625
|
+
Each change carries a Hybrid Logical Clock timestamp. The HLC gives deterministic causal ordering across nodes without relying on perfectly synchronised wall clocks. Batches also carry a sequence range, checksum, and, in coordinator mode, `groupId` and `primaryTerm`.
|
|
626
|
+
|
|
627
|
+
#### What happens under partitions?
|
|
628
|
+
|
|
629
|
+
Static mode does not own failover. If the static primary is lost, writes stay unavailable until an operator or external system promotes another node and reroutes clients.
|
|
630
|
+
|
|
631
|
+
Coordinator mode uses a cluster coordinator, primary terms, leases, in-sync sets, and fail-closed write behaviour. A primary may accept writes only while it can prove current authority. Replicas reject stale batches, stale sync messages, and stale forwarded writes. Only an in-sync replica can be promoted.
|
|
632
|
+
|
|
633
|
+
#### What topology do I need for automatic write failover?
|
|
634
|
+
|
|
635
|
+
Use at least three voting data-bearing Sirannon nodes in one replication group. One node has no failover. Two nodes can replicate, but one survivor cannot prove majority authority after the other node is lost.
|
|
636
|
+
|
|
637
|
+
#### Does Sirannon support schema changes across replicas?
|
|
638
|
+
|
|
639
|
+
Yes, with a safety allowlist. Replicated DDL supports `CREATE TABLE`, `ALTER TABLE ... ADD COLUMN`, `DROP TABLE`, `CREATE INDEX`, and `DROP INDEX`. The receiver rejects multiple statements, `AS SELECT`, extension loading, `ATTACH`, dangerous file functions, and DDL outside the allowlist.
|
|
640
|
+
|
|
641
|
+
#### How do foreign keys and unique constraints behave?
|
|
642
|
+
|
|
643
|
+
SQLite enforces constraints on each node. The primary serialises normal writes, which prevents normal concurrent unique-key conflicts. First sync orders tables by foreign-key dependency, and controlled resync disables foreign keys only while wiping tables before reloading from the sync source. Incoming replicated data still has to satisfy the receiving database's constraints.
|
|
644
|
+
|
|
645
|
+
#### Are reads consistent after writes?
|
|
646
|
+
|
|
647
|
+
Read concern controls this. `local` reads the selected node's local state. `majority` reads data that has reached the replication group's majority commit point. `linearizable` reads from the current primary after it proves live authority for the current primary term. If the requested read concern cannot be satisfied, the read fails rather than returning a weaker result.
|
|
648
|
+
|
|
649
|
+
#### Is this local-first or multi-writer today?
|
|
650
|
+
|
|
651
|
+
The current production path is primary-replica. Conflict resolvers decide how a receiving node applies a change to an existing row; they do not provide local-first reconciliation or a multi-writer CRDT layer.
|
|
652
|
+
|
|
653
|
+
### Transport options
|
|
654
|
+
|
|
655
|
+
| Transport | Import | Use case |
|
|
656
|
+
| --- | --- | --- |
|
|
657
|
+
| gRPC | `@delali/sirannon-db/transport/grpc` | Production Node.js multi-node replication over the network with TLS support. |
|
|
658
|
+
| In-Memory | `@delali/sirannon-db/transport/memory` | Testing and single-process multi-node scenarios. Messages delivered via microtask scheduling. |
|
|
659
|
+
| Custom | Build your own | Any transport that satisfies the `ReplicationTransport` interface (Redis, NATS, MQTT, TCP, etc). |
|
|
660
|
+
|
|
661
|
+
`ReplicationEngine.start()` derives `TransportConfig.localRole` from `topology.role`. In coordinator mode, it also supplies the current `groupId`, `primaryTerm`, and protocol version to the transport. Set these fields yourself only when you connect a `ReplicationTransport` without `ReplicationEngine`.
|
|
662
|
+
|
|
663
|
+
`TransportConfig` accepts these fields:
|
|
664
|
+
|
|
665
|
+
| Option | Type | Description |
|
|
666
|
+
| --- | --- | --- |
|
|
667
|
+
| `endpoints` | `string[]` | Peer addresses used to establish replication connections |
|
|
668
|
+
| `localRole` | `'primary' \| 'replica'` | Local topology role; `ReplicationEngine` supplies this value |
|
|
669
|
+
| `groupId` | `string` | Replication group carried in coordinator-mode handshakes; the engine supplies it from coordinator configuration |
|
|
670
|
+
| `primaryTerm` | `bigint` | Current fencing term; the engine supplies it from coordinator state |
|
|
671
|
+
| `protocolVersion` | `string` | Replication protocol version advertised to peers |
|
|
672
|
+
| `metadata` | `Record<string, unknown>` | Optional custom transport metadata |
|
|
673
|
+
|
|
674
|
+
### Replication configuration reference
|
|
675
|
+
|
|
676
|
+
| Option | Type | Default | Description |
|
|
677
|
+
| --- | --- | --- | --- |
|
|
678
|
+
| `nodeId` | `string` | auto-generated in static mode | Unique node identifier. Coordinator mode requires a stable, persisted value. |
|
|
679
|
+
| `topology` | `Topology` | required | `PrimaryReplicaTopology` |
|
|
680
|
+
| `transport` | `ReplicationTransport` | required | Transport for inter-node communication |
|
|
681
|
+
| `transportConfig` | `TransportConfig` | `{}` | Peer endpoints and transport metadata. The engine supplies role and coordinator fencing fields when it starts. |
|
|
682
|
+
| `writeForwarding` | `boolean` | `false` | Forward writes from replicas to the primary |
|
|
683
|
+
| `defaultConflictResolver` | `ConflictResolver` | `LWWResolver` | Default conflict resolution strategy |
|
|
684
|
+
| `conflictResolvers` | `Record<string, ConflictResolver>` | - | Per-table conflict resolution overrides |
|
|
685
|
+
| `batchSize` | `number` | `100` | Changes per replication batch |
|
|
686
|
+
| `batchIntervalMs` | `number` | `100` | Sender loop interval in ms |
|
|
687
|
+
| `maxClockDriftMs` | `number` | `60000` | Maximum tolerated HLC drift before rejecting a batch |
|
|
688
|
+
| `maxPendingBatches` | `number` | `10` | In-flight batches per peer before backpressure |
|
|
689
|
+
| `maxBatchChanges` | `number` | `1000` | Maximum accepted changes in one inbound batch |
|
|
690
|
+
| `ackTimeoutMs` | `number` | `5000` | Replication batch ack timeout |
|
|
691
|
+
| `initialSync` | `boolean` | `true` | Pull a full snapshot when joining a cluster |
|
|
692
|
+
| `syncBatchSize` | `number` | `10000` | Rows per sync batch during initial sync |
|
|
693
|
+
| `maxConcurrentSyncs` | `number` | `2` | Maximum simultaneous sync sessions on the source |
|
|
694
|
+
| `maxSyncDurationMs` | `number` | `1800000` | Source aborts sync after this duration (30 min) |
|
|
695
|
+
| `maxSyncLagBeforeReady` | `number` | `100` | Catch-up lag threshold (in sequences) to transition to ready |
|
|
696
|
+
| `syncAckTimeoutMs` | `number` | `30000` | Per-batch ack timeout during sync (30s) |
|
|
697
|
+
| `catchUpDeadlineMs` | `number` | `600000` | Max time in catch-up phase before transitioning to ready (10 min) |
|
|
698
|
+
| `resumeFromSeq` | `bigint` | - | Start replication from a specific sequence (out-of-band sync) |
|
|
699
|
+
| `snapshotConnectionFactory` | `() => Promise<SQLiteConnection>` | - | Factory for read-only connections used during sync serving |
|
|
700
|
+
| `changeTracker` | `ChangeTracker` | - | CDC trigger manager, required for initial sync |
|
|
701
|
+
| `flowControl` | `{ maxLagSeconds?, onLagExceeded? }` | - | Replication lag monitoring callbacks |
|
|
702
|
+
| `onBeforeForwardedQuery` | `(sql, params?) => void` | - | Validation or authorisation hook called before the primary executes each forwarded statement |
|
|
703
|
+
| `coordinator` | `CoordinatorModeConfig` | - | Enables coordinator-backed authority and failover |
|
|
704
|
+
| `snapshotThreshold` | `number` | - | Reserved configuration field; the current engine does not read it |
|
|
705
|
+
|
|
706
|
+
### Coordinator configuration reference
|
|
707
|
+
|
|
708
|
+
| Option | Type | Default | Description |
|
|
709
|
+
| --- | --- | --- | --- |
|
|
710
|
+
| `clusterId` | `string` | required | Coordinator namespace for the Sirannon cluster |
|
|
711
|
+
| `groupId` | `string` | required | Replication group containing copies of one database |
|
|
712
|
+
| `endpoint` | `string` | - | Application endpoint advertised for client discovery |
|
|
713
|
+
| `votingDataBearingNodeIds` | `string[]` | - | Voter set used to create an unregistered group and calculate coordinator write concerns |
|
|
714
|
+
| `coordinator` | `ClusterCoordinator` | required | Coordinator adapter, such as the etcd adapter |
|
|
715
|
+
| `sessionTtlMs` | `number` | `10000` | Node-session lease lifetime |
|
|
716
|
+
| `controller` | `boolean \| CoordinatorControllerConfig` | enabled | Enables the controller loop or configures its lease holder, TTL, and tick interval |
|
|
717
|
+
| `compatibility` | `CoordinatorCompatibilityMetadata` | - | Package, specification, and protocol versions used for promotion compatibility checks |
|
|
718
|
+
|
|
719
|
+
`CoordinatorControllerConfig` accepts `enabled`, `holderId`, `leaseTtlMs`, and `tickIntervalMs`. The lease TTL defaults to 10,000 ms, and the controller tick interval defaults to 1,000 ms.
|
|
720
|
+
|
|
721
|
+
### Replication errors
|
|
722
|
+
|
|
723
|
+
| Error | Code | When |
|
|
724
|
+
| --- | --- | --- |
|
|
725
|
+
| `ReplicationError` | `REPLICATION_ERROR` | Base class for replication failures |
|
|
726
|
+
| `SyncError` | `SYNC_ERROR` | First sync failures (node not ready, timeout, integrity mismatch) |
|
|
727
|
+
| `ConflictError` | `CONFLICT_ERROR` | Unresolvable write conflict |
|
|
728
|
+
| `TransportError` | `TRANSPORT_ERROR` | Inter-node communication failure |
|
|
729
|
+
| `BatchValidationError` | `BATCH_VALIDATION_ERROR` | Checksum mismatch, clock drift, or oversized batch |
|
|
730
|
+
| `TopologyError` | `TOPOLOGY_ERROR` | Write on a read-only node without forwarding |
|
|
731
|
+
| `WriteConcernError` | `WRITE_CONCERN_ERROR` | Quorum not reached within timeout |
|
|
732
|
+
|
|
438
733
|
## Security
|
|
439
734
|
|
|
440
|
-
Sirannon-db is
|
|
735
|
+
Sirannon-db gives you secure primitives, but the network server is intentionally low-level: it can execute SQL sent by a client. Treat the server like a database endpoint, not like a public application API, unless you add an explicit application security boundary around it.
|
|
441
736
|
|
|
442
737
|
### Built-in protections
|
|
443
738
|
|
|
444
|
-
- **
|
|
445
|
-
- **Identifier validation** - CDC table and column names are validated against a strict allowlist regex (`/^[a-zA-Z_][a-zA-Z0-9_]*$/`)
|
|
446
|
-
- **Path traversal prevention** - Migration and backup paths reject null bytes, `..` segments, and control characters before
|
|
447
|
-
- **Request size limits** - HTTP bodies and WebSocket payloads are capped at 1 MB
|
|
448
|
-
- **Error isolation** -
|
|
739
|
+
- **Parameterised values** - Query and execute calls pass parameters through the driver layer. Keep user input in `params`; never concatenate user input into SQL strings.
|
|
740
|
+
- **Identifier validation** - CDC table and column names are validated against a strict allowlist regex (`/^[a-zA-Z_][a-zA-Z0-9_]*$/`) and escaped with double quotes.
|
|
741
|
+
- **Path traversal prevention** - Migration and backup paths reject null bytes, `..` segments, and control characters before filesystem access.
|
|
742
|
+
- **Request size limits** - HTTP bodies and WebSocket payloads are capped at 1 MB to reduce memory-exhaustion risk.
|
|
743
|
+
- **Error isolation** - Remote errors use a machine-readable code and message. Stack traces and internal details are not returned to clients.
|
|
449
744
|
- **Connection isolation** - Read and write operations use separate connection pools. Read-only databases enforce immutability at the connection level.
|
|
450
745
|
|
|
451
|
-
###
|
|
746
|
+
### Deployment boundary
|
|
747
|
+
|
|
748
|
+
For production, put Sirannon behind one of these boundaries:
|
|
452
749
|
|
|
453
|
-
|
|
750
|
+
- A server-side application layer that exposes domain actions, not arbitrary SQL.
|
|
751
|
+
- A private network boundary where only trusted services can reach the Sirannon server.
|
|
752
|
+
- A custom `resolveExecutionTarget` or hook layer that allows only specific statements, tenants, and tables.
|
|
753
|
+
|
|
754
|
+
Do not expose unrestricted `/db/:id/query`, `/db/:id/execute`, `/db/:id/transaction`, or `/db/:id` WebSocket routes directly to untrusted browsers.
|
|
755
|
+
|
|
756
|
+
### HTTP authentication
|
|
757
|
+
|
|
758
|
+
Use `onRequest` to authenticate HTTP database routes. The hook runs before database routes and can deny requests by returning `{ status, code, message }`. Health endpoints bypass this hook.
|
|
454
759
|
|
|
455
760
|
```ts
|
|
456
761
|
const server = createServer(sirannon, {
|
|
457
762
|
port: 9876,
|
|
458
|
-
onRequest: ({ headers
|
|
459
|
-
if (headers.authorization !== `Bearer ${process.env.
|
|
763
|
+
onRequest: ({ headers }) => {
|
|
764
|
+
if (headers.authorization !== `Bearer ${process.env.SIRANNON_API_TOKEN}`) {
|
|
460
765
|
return { status: 401, code: 'UNAUTHORIZED', message: 'Invalid or missing token' }
|
|
461
766
|
}
|
|
462
767
|
},
|
|
463
768
|
})
|
|
464
769
|
```
|
|
465
770
|
|
|
466
|
-
|
|
771
|
+
Send HTTP credentials through `headers` on the client:
|
|
467
772
|
|
|
468
|
-
|
|
773
|
+
```ts
|
|
774
|
+
const client = new SirannonClient('https://db.example.com', {
|
|
775
|
+
transport: 'http',
|
|
776
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
777
|
+
})
|
|
778
|
+
```
|
|
779
|
+
|
|
780
|
+
### WebSocket authentication
|
|
781
|
+
|
|
782
|
+
Browsers cannot attach arbitrary `Authorization` headers to `new WebSocket(...)`. For browser clients, authenticate the upgrade with a mechanism the browser can send, such as same-site cookies or a short-lived value in `Sec-WebSocket-Protocol`. Also validate the `Origin` header during the upgrade.
|
|
783
|
+
|
|
784
|
+
```ts
|
|
785
|
+
const expectedOrigin = 'https://app.example.com'
|
|
786
|
+
const expectedProtocol = process.env.SIRANNON_WS_PROTOCOL
|
|
787
|
+
|
|
788
|
+
const server = createServer(sirannon, {
|
|
789
|
+
port: 9876,
|
|
790
|
+
onRequest: ({ headers, method, path }) => {
|
|
791
|
+
const isWebSocketUpgrade = method === 'GET' && path.startsWith('/db/')
|
|
792
|
+
if (!isWebSocketUpgrade) {
|
|
793
|
+
return undefined
|
|
794
|
+
}
|
|
469
795
|
|
|
470
|
-
|
|
796
|
+
if (headers.origin !== expectedOrigin) {
|
|
797
|
+
return { status: 403, code: 'FORBIDDEN_ORIGIN', message: 'Forbidden origin' }
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
const protocols = (headers['sec-websocket-protocol'] ?? '').split(',').map(value => value.trim())
|
|
801
|
+
if (!expectedProtocol || !protocols.includes(expectedProtocol)) {
|
|
802
|
+
return { status: 401, code: 'UNAUTHORIZED', message: 'Invalid WebSocket credentials' }
|
|
803
|
+
}
|
|
804
|
+
},
|
|
805
|
+
})
|
|
806
|
+
```
|
|
471
807
|
|
|
472
|
-
|
|
808
|
+
Then pass the browser-compatible protocol through the client:
|
|
473
809
|
|
|
474
810
|
```ts
|
|
475
811
|
const client = new SirannonClient('https://db.example.com', {
|
|
476
812
|
transport: 'websocket',
|
|
477
|
-
|
|
813
|
+
webSocketProtocols: [wsProtocol],
|
|
478
814
|
})
|
|
479
815
|
```
|
|
480
816
|
|
|
481
|
-
|
|
817
|
+
Protocol values must be valid `Sec-WebSocket-Protocol` tokens. If you derive them from secrets or signed data, encode them with a URL-safe format and keep them short-lived.
|
|
818
|
+
|
|
819
|
+
### TLS and transport security
|
|
482
820
|
|
|
483
|
-
|
|
821
|
+
The built-in server binds plain HTTP and WebSocket. For any traffic outside a trusted local network, terminate TLS upstream with a reverse proxy, load balancer, or platform edge and use `https://` and `wss://` client URLs. Without TLS, credentials, SQL text, parameters, and CDC payloads travel in cleartext.
|
|
822
|
+
|
|
823
|
+
### CORS and browser access
|
|
824
|
+
|
|
825
|
+
CORS is disabled by default. Enable it only if browser clients need direct HTTP access, and restrict origins to trusted domains:
|
|
484
826
|
|
|
485
827
|
```ts
|
|
486
828
|
const server = createServer(sirannon, {
|
|
@@ -491,7 +833,36 @@ const server = createServer(sirannon, {
|
|
|
491
833
|
})
|
|
492
834
|
```
|
|
493
835
|
|
|
494
|
-
Passing `cors: true` allows all origins
|
|
836
|
+
Passing `cors: true` allows all origins and should be limited to local development. CORS does not protect WebSocket upgrades; validate WebSocket `Origin` in `onRequest`.
|
|
837
|
+
|
|
838
|
+
### SQL access control
|
|
839
|
+
|
|
840
|
+
Authentication only identifies the caller. It does not make arbitrary SQL safe. For internet-facing systems, enforce one of these patterns:
|
|
841
|
+
|
|
842
|
+
- Prefer application endpoints that perform domain operations such as `createOrder`, `reserveInventory`, or `markPaid`.
|
|
843
|
+
- If clients must use the Sirannon data API, wrap the database with `resolveExecutionTarget` and allow only known SQL statements and parameter shapes.
|
|
844
|
+
- Use hooks for additional query checks, but do not rely on naive substring matching as a SQL firewall.
|
|
845
|
+
- Keep tenant identifiers and ownership rules on the server side.
|
|
846
|
+
|
|
847
|
+
### Secrets and logging
|
|
848
|
+
|
|
849
|
+
- Do not place long-lived secrets in browser-visible configuration such as Vite `VITE_*` variables.
|
|
850
|
+
- Prefer short-lived WebSocket credentials minted by your application server.
|
|
851
|
+
- Redact `Authorization`, cookies, and WebSocket auth protocol values from access logs.
|
|
852
|
+
- Avoid logging full SQL statements when they can contain sensitive data.
|
|
853
|
+
|
|
854
|
+
### Security checklist
|
|
855
|
+
|
|
856
|
+
- Bind the server to `127.0.0.1` or a private interface unless a proxy is enforcing TLS and access control.
|
|
857
|
+
- Use HTTPS/WSS for non-local traffic.
|
|
858
|
+
- Authenticate every HTTP database route and every WebSocket upgrade.
|
|
859
|
+
- Validate WebSocket `Origin` against an explicit allowlist.
|
|
860
|
+
- Keep SQL behind application actions or a strict allowlist.
|
|
861
|
+
- Keep user input in SQL parameters, not interpolated strings.
|
|
862
|
+
- Restrict CORS to known origins.
|
|
863
|
+
- Add rate limits, audit logs, and abuse monitoring at the application or edge layer for public deployments.
|
|
864
|
+
|
|
865
|
+
Further reading: [OWASP REST Security Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html), [OWASP WebSocket Security Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/WebSocket_Security_Cheat_Sheet.html), and [MDN WebSocket constructor](https://developer.mozilla.org/en-US/docs/Web/API/WebSocket/WebSocket).
|
|
495
866
|
|
|
496
867
|
## Error handling
|
|
497
868
|
|
|
@@ -560,20 +931,21 @@ try {
|
|
|
560
931
|
| Option | Type | Default | Description |
|
|
561
932
|
| --- | --- | --- | --- |
|
|
562
933
|
| `transport` | `'websocket' \| 'http'` | `'websocket'` | Transport protocol |
|
|
563
|
-
| `headers` | `Record<string, string>` | - | Custom HTTP headers |
|
|
934
|
+
| `headers` | `Record<string, string>` | - | Custom HTTP headers; browser WebSocket handshakes do not use this option |
|
|
935
|
+
| `webSocketProtocols` | `string \| string[]` | - | WebSocket subprotocols sent during the upgrade handshake |
|
|
564
936
|
| `autoReconnect` | `boolean` | `true` | Reconnect on WebSocket disconnect |
|
|
565
937
|
| `reconnectInterval` | `number` | `1000` | Reconnect delay in ms |
|
|
566
938
|
|
|
567
939
|
## Examples
|
|
568
940
|
|
|
569
|
-
Self-contained example projects live in [`examples/`](examples/) and cover
|
|
941
|
+
Self-contained example projects live in [`examples/`](examples/) and cover the current Node.js, browser, client-server, and distributed paths:
|
|
570
942
|
|
|
571
943
|
| Example | Runtime | Driver | What it demonstrates |
|
|
572
944
|
| --- | --- | --- | --- |
|
|
573
|
-
| [`node
|
|
574
|
-
| [`node-native`](examples/node-native/) | Node.js >= 22 | built-in `node:sqlite` | Same features as above using the zero-dependency Node driver |
|
|
945
|
+
| [`node`](examples/node/) | Node.js >= 22 | better-sqlite3 or built-in `node:sqlite` | All core features: schema, migrations, CRUD, transactions, CDC, connection pools, metrics, multi-tenant, hooks, backup, shutdown |
|
|
575
946
|
| [`web-wa-sqlite`](examples/web-wa-sqlite/) | Browser (Vite) | wa-sqlite + IndexedDB | CRUD, transactions, CDC subscriptions in the browser |
|
|
576
947
|
| [`web-client`](examples/web-client/) | Browser + Node.js | better-sqlite3 (server) | Client SDK connecting to a Sirannon server over HTTP and WebSocket |
|
|
948
|
+
| [`distributed-entitlements`](examples/distributed-entitlements/) | Node.js + browser | better-sqlite3 | Three-node coordinator-backed replication over gRPC with etcd authority, local mTLS certificates, and Toxiproxy failure controls |
|
|
577
949
|
|
|
578
950
|
### Running the examples
|
|
579
951
|
|
|
@@ -587,21 +959,25 @@ pnpm --filter @delali/sirannon-db build
|
|
|
587
959
|
Then pick an example:
|
|
588
960
|
|
|
589
961
|
```bash
|
|
590
|
-
# Node.js with better-sqlite3
|
|
591
|
-
cd packages/ts/examples/node
|
|
962
|
+
# Node.js with better-sqlite3, the default driver
|
|
963
|
+
cd packages/ts/examples/node
|
|
592
964
|
pnpm start
|
|
593
965
|
|
|
594
966
|
# Node.js with built-in sqlite
|
|
595
|
-
cd packages/ts/examples/node
|
|
596
|
-
pnpm start
|
|
967
|
+
cd packages/ts/examples/node
|
|
968
|
+
pnpm run start:node-native
|
|
597
969
|
|
|
598
970
|
# Browser with wa-sqlite (opens Vite dev server)
|
|
599
971
|
cd packages/ts/examples/web-wa-sqlite
|
|
600
|
-
pnpm dev
|
|
972
|
+
pnpm run dev
|
|
601
973
|
|
|
602
974
|
# Client-server (starts both Sirannon server and Vite client)
|
|
603
975
|
cd packages/ts/examples/web-client
|
|
604
|
-
pnpm
|
|
976
|
+
pnpm run dev
|
|
977
|
+
|
|
978
|
+
# Distributed entitlements (starts the Docker cluster and dashboard)
|
|
979
|
+
cd packages/ts/examples/distributed-entitlements
|
|
980
|
+
pnpm run dev
|
|
605
981
|
```
|
|
606
982
|
|
|
607
983
|
## Benchmarks
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { a as BackupScheduler } from '../index-
|
|
2
|
-
export {
|
|
1
|
+
export { a as BackupScheduler } from '../index-CLdNrcPz.js';
|
|
2
|
+
export { m as BackupScheduleOptions } from '../types-D-74JiXb.js';
|
|
3
3
|
import '../types-BFSsG77t.js';
|