@kyneta/leveldb-store 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +137 -0
- package/dist/server.d.ts +37 -0
- package/dist/server.js +186 -0
- package/dist/server.js.map +1 -0
- package/package.json +46 -0
- package/src/__tests__/leveldb-storage.test.ts +207 -0
- package/src/server.ts +295 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025-present Duane Johnson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# @kyneta/leveldb-store
|
|
2
|
+
|
|
3
|
+
Server-side persistent storage for `@kyneta/exchange`, backed by [LevelDB](https://github.com/google/leveldb) via [`classic-level`](https://github.com/Level/classic-level).
|
|
4
|
+
|
|
5
|
+
Implements the `Store` interface — pass directly to `Exchange({ stores: [...] })` for automatic document persistence and hydration.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
pnpm add @kyneta/leveldb-store
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Quick Start
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { Exchange } from "@kyneta/exchange"
|
|
17
|
+
import { createLevelDBStore } from "@kyneta/leveldb-store/server"
|
|
18
|
+
|
|
19
|
+
const exchange = new Exchange({
|
|
20
|
+
identity: { peerId: "my-server", name: "server" },
|
|
21
|
+
stores: [createLevelDBStore("./data/exchange-db")],
|
|
22
|
+
transports: [networkTransport],
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
// Documents are automatically persisted on mutation and hydrated on restart.
|
|
26
|
+
const doc = exchange.get("my-doc", TodoDoc)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
That's it. The Exchange handles hydration (loading from storage on `get()` / `replicate()`) and persistence (saving incremental deltas via `onStateAdvanced`) — no manual save/load needed.
|
|
30
|
+
|
|
31
|
+
## API
|
|
32
|
+
|
|
33
|
+
### `createLevelDBStore(dbPath)`
|
|
34
|
+
|
|
35
|
+
Factory function that returns a `Store`. The `dbPath` is the directory where LevelDB stores its files.
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
import { createLevelDBStore } from "@kyneta/leveldb-store/server"
|
|
39
|
+
|
|
40
|
+
const store = createLevelDBStore("./data/exchange-db")
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### `LevelDBStore`
|
|
44
|
+
|
|
45
|
+
The class implementing the `Store` interface. Use `createLevelDBStore` for most cases; use the class directly if you need access to `close()` outside of the Exchange lifecycle.
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
import { LevelDBStore } from "@kyneta/leveldb-store/server"
|
|
49
|
+
|
|
50
|
+
const store = new LevelDBStore("./data/exchange-db")
|
|
51
|
+
|
|
52
|
+
// ... use with Exchange ...
|
|
53
|
+
|
|
54
|
+
await store.close() // release file handles
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Binary Codec
|
|
58
|
+
|
|
59
|
+
The module also exports `encodeStoreEntry` and `decodeStoreEntry` for the compact binary envelope format. These are pure functions — useful for debugging, migration scripts, or building custom tooling over the LevelDB data files.
|
|
60
|
+
|
|
61
|
+
```ts
|
|
62
|
+
import { encodeStoreEntry, decodeStoreEntry } from "@kyneta/leveldb-store/server"
|
|
63
|
+
|
|
64
|
+
const bytes = encodeStoreEntry(entry) // StoreEntry → Uint8Array
|
|
65
|
+
const entry = decodeStoreEntry(bytes) // Uint8Array → StoreEntry
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Design
|
|
69
|
+
|
|
70
|
+
### Key-Space
|
|
71
|
+
|
|
72
|
+
Keys follow the [FoundationDB convention](https://apple.github.io/foundationdb/developer-guide.html#key-and-value-sizes) — null-byte (`\x00`) separated prefixes:
|
|
73
|
+
|
|
74
|
+
| Key pattern | Value |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `meta\x00{docId}` | JSON-encoded `DocMetadata` |
|
|
77
|
+
| `entry\x00{docId}\x00{seqNo}` | Binary-encoded `StoreEntry` |
|
|
78
|
+
|
|
79
|
+
The `\x00` separator cannot appear in valid UTF-8 strings, so no docId validation is needed — the key-space imposes zero constraints on callers. Documents with overlapping name prefixes (e.g. `doc` vs `doc-extra`) are fully isolated.
|
|
80
|
+
|
|
81
|
+
### Sequence Numbers
|
|
82
|
+
|
|
83
|
+
Each document maintains a monotonic sequence counter for entry ordering. SeqNos are zero-padded to 10 digits, supporting up to 10 billion entries per document.
|
|
84
|
+
|
|
85
|
+
On reboot, the max seqNo for a document is lazily discovered via a single reverse-iterator seek on first `append` — no full scan needed.
|
|
86
|
+
|
|
87
|
+
### Binary Envelope
|
|
88
|
+
|
|
89
|
+
Entries are stored in a compact binary format (not JSON) for minimal overhead:
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
[1 byte flags] [4 bytes version length BE] [N bytes version UTF-8] [remaining: payload data]
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Flags byte layout:
|
|
96
|
+
- bit 0: kind (0 = entirety, 1 = since)
|
|
97
|
+
- bit 1: encoding (0 = json, 1 = binary)
|
|
98
|
+
- bit 2: data type (0 = string, 1 = Uint8Array)
|
|
99
|
+
|
|
100
|
+
### Atomicity
|
|
101
|
+
|
|
102
|
+
`replace()` uses LevelDB's batch operation to atomically delete all existing entries and write the single replacement. A concurrent reader never observes an empty intermediate state.
|
|
103
|
+
|
|
104
|
+
## Testing
|
|
105
|
+
|
|
106
|
+
The package passes the full `Store` conformance suite from `@kyneta/exchange/testing`, plus LevelDB-specific tests for close/reopen persistence and binary codec edge cases.
|
|
107
|
+
|
|
108
|
+
```sh
|
|
109
|
+
pnpm test
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
To use the conformance suite for your own `Store` implementation:
|
|
113
|
+
|
|
114
|
+
```ts
|
|
115
|
+
import { describeStore } from "@kyneta/exchange/testing"
|
|
116
|
+
|
|
117
|
+
describeStore(
|
|
118
|
+
"MyStore",
|
|
119
|
+
() => new MyStore(),
|
|
120
|
+
async (store) => { /* cleanup */ },
|
|
121
|
+
)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Peer Dependencies
|
|
125
|
+
|
|
126
|
+
```json
|
|
127
|
+
{
|
|
128
|
+
"peerDependencies": {
|
|
129
|
+
"@kyneta/exchange": "^1.1.0",
|
|
130
|
+
"@kyneta/schema": "^1.1.0"
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## License
|
|
136
|
+
|
|
137
|
+
MIT
|
package/dist/server.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { Store, StoreEntry } from '@kyneta/exchange/src/store/store.js';
|
|
2
|
+
import { DocId } from '@kyneta/exchange/src/types.js';
|
|
3
|
+
import { DocMetadata } from '@kyneta/schema';
|
|
4
|
+
|
|
5
|
+
declare function encodeStoreEntry(entry: StoreEntry): Uint8Array;
|
|
6
|
+
declare function decodeStoreEntry(bytes: Uint8Array): StoreEntry;
|
|
7
|
+
declare class LevelDBStore implements Store {
|
|
8
|
+
#private;
|
|
9
|
+
constructor(dbPath: string);
|
|
10
|
+
lookup(docId: DocId): Promise<DocMetadata | null>;
|
|
11
|
+
ensureDoc(docId: DocId, metadata: DocMetadata): Promise<void>;
|
|
12
|
+
append(docId: DocId, entry: StoreEntry): Promise<void>;
|
|
13
|
+
loadAll(docId: DocId): AsyncIterable<StoreEntry>;
|
|
14
|
+
replace(docId: DocId, entry: StoreEntry): Promise<void>;
|
|
15
|
+
delete(docId: DocId): Promise<void>;
|
|
16
|
+
listDocIds(): AsyncIterable<DocId>;
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Create a LevelDB storage backend for server-side persistence.
|
|
21
|
+
*
|
|
22
|
+
* Returns a `Store` — pass directly to `Exchange({ stores: [...] })`.
|
|
23
|
+
*
|
|
24
|
+
* @param dbPath - Directory path where LevelDB stores its files
|
|
25
|
+
*
|
|
26
|
+
* @example
|
|
27
|
+
* ```typescript
|
|
28
|
+
* import { createLevelDBStore } from "@kyneta/leveldb-store/server"
|
|
29
|
+
*
|
|
30
|
+
* const exchange = new Exchange({
|
|
31
|
+
* stores: [createLevelDBStore("./data/exchange-db")],
|
|
32
|
+
* })
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
declare function createLevelDBStore(dbPath: string): Store;
|
|
36
|
+
|
|
37
|
+
export { LevelDBStore, createLevelDBStore, decodeStoreEntry, encodeStoreEntry };
|
package/dist/server.js
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
// src/server.ts
|
|
2
|
+
import { ClassicLevel } from "classic-level";
|
|
3
|
+
var SEP = "\0";
|
|
4
|
+
var META_PREFIX = `meta${SEP}`;
|
|
5
|
+
var ENTRY_PREFIX = `entry${SEP}`;
|
|
6
|
+
var SEQ_PAD = 10;
|
|
7
|
+
var encoder = new TextEncoder();
|
|
8
|
+
var decoder = new TextDecoder();
|
|
9
|
+
function encodeStoreEntry(entry) {
|
|
10
|
+
const { payload, version } = entry;
|
|
11
|
+
let flags = 0;
|
|
12
|
+
if (payload.kind === "since") flags |= 1;
|
|
13
|
+
if (payload.encoding === "binary") flags |= 2;
|
|
14
|
+
const isDataBinary = payload.data instanceof Uint8Array;
|
|
15
|
+
if (isDataBinary) flags |= 4;
|
|
16
|
+
const versionBytes = encoder.encode(version);
|
|
17
|
+
const dataBytes = isDataBinary ? payload.data : encoder.encode(payload.data);
|
|
18
|
+
const buf = new Uint8Array(1 + 4 + versionBytes.length + dataBytes.length);
|
|
19
|
+
const view = new DataView(buf.buffer);
|
|
20
|
+
buf[0] = flags;
|
|
21
|
+
view.setUint32(1, versionBytes.length, false);
|
|
22
|
+
buf.set(versionBytes, 5);
|
|
23
|
+
buf.set(dataBytes, 5 + versionBytes.length);
|
|
24
|
+
return buf;
|
|
25
|
+
}
|
|
26
|
+
function decodeStoreEntry(bytes) {
|
|
27
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
28
|
+
const flags = bytes[0];
|
|
29
|
+
const kind = (flags & 1) !== 0 ? "since" : "entirety";
|
|
30
|
+
const encoding = (flags & 2) !== 0 ? "binary" : "json";
|
|
31
|
+
const isDataBinary = (flags & 4) !== 0;
|
|
32
|
+
const versionLen = view.getUint32(1, false);
|
|
33
|
+
const version = decoder.decode(bytes.subarray(5, 5 + versionLen));
|
|
34
|
+
const dataStart = 5 + versionLen;
|
|
35
|
+
const rawData = bytes.subarray(dataStart);
|
|
36
|
+
const data = isDataBinary ? new Uint8Array(rawData) : decoder.decode(rawData);
|
|
37
|
+
return {
|
|
38
|
+
payload: { kind, encoding, data },
|
|
39
|
+
version
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function metaKey(docId) {
|
|
43
|
+
return `${META_PREFIX}${docId}`;
|
|
44
|
+
}
|
|
45
|
+
function entryPrefix(docId) {
|
|
46
|
+
return `${ENTRY_PREFIX}${docId}${SEP}`;
|
|
47
|
+
}
|
|
48
|
+
function entryKey(docId, seqNo) {
|
|
49
|
+
return `${entryPrefix(docId)}${String(seqNo).padStart(SEQ_PAD, "0")}`;
|
|
50
|
+
}
|
|
51
|
+
function parseDocIdFromMetaKey(key) {
|
|
52
|
+
return key.slice(META_PREFIX.length);
|
|
53
|
+
}
|
|
54
|
+
function parseSeqNoFromEntryKey(key, docId) {
|
|
55
|
+
const prefix = entryPrefix(docId);
|
|
56
|
+
return Number.parseInt(key.slice(prefix.length), 10);
|
|
57
|
+
}
|
|
58
|
+
var LevelDBStore = class {
|
|
59
|
+
#db;
|
|
60
|
+
#seqNos = /* @__PURE__ */ new Map();
|
|
61
|
+
constructor(dbPath) {
|
|
62
|
+
this.#db = new ClassicLevel(dbPath, {
|
|
63
|
+
valueEncoding: "binary"
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
// -------------------------------------------------------------------------
|
|
67
|
+
// Private — seqNo management
|
|
68
|
+
// -------------------------------------------------------------------------
|
|
69
|
+
/**
|
|
70
|
+
* Get the next seqNo for a doc. On first call after reboot, discovers
|
|
71
|
+
* the max existing seqNo via a single reverse-iterator seek.
|
|
72
|
+
*/
|
|
73
|
+
async #nextSeqNo(docId) {
|
|
74
|
+
const cached = this.#seqNos.get(docId);
|
|
75
|
+
if (cached !== void 0) {
|
|
76
|
+
const next2 = cached + 1;
|
|
77
|
+
this.#seqNos.set(docId, next2);
|
|
78
|
+
return next2;
|
|
79
|
+
}
|
|
80
|
+
const prefix = entryPrefix(docId);
|
|
81
|
+
let maxSeq = -1;
|
|
82
|
+
for await (const key of this.#db.keys({
|
|
83
|
+
gte: prefix,
|
|
84
|
+
lt: `${prefix}\xFF`,
|
|
85
|
+
reverse: true,
|
|
86
|
+
limit: 1
|
|
87
|
+
})) {
|
|
88
|
+
maxSeq = parseSeqNoFromEntryKey(key, docId);
|
|
89
|
+
}
|
|
90
|
+
const next = maxSeq + 1;
|
|
91
|
+
this.#seqNos.set(docId, next);
|
|
92
|
+
return next;
|
|
93
|
+
}
|
|
94
|
+
// -------------------------------------------------------------------------
|
|
95
|
+
// Store interface
|
|
96
|
+
// -------------------------------------------------------------------------
|
|
97
|
+
async lookup(docId) {
|
|
98
|
+
try {
|
|
99
|
+
const raw = await this.#db.get(metaKey(docId));
|
|
100
|
+
return JSON.parse(decoder.decode(raw));
|
|
101
|
+
} catch (error) {
|
|
102
|
+
if (error.code === "LEVEL_NOT_FOUND") return null;
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async ensureDoc(docId, metadata) {
|
|
107
|
+
try {
|
|
108
|
+
await this.#db.get(metaKey(docId));
|
|
109
|
+
} catch (error) {
|
|
110
|
+
if (error.code === "LEVEL_NOT_FOUND") {
|
|
111
|
+
await this.#db.put(
|
|
112
|
+
metaKey(docId),
|
|
113
|
+
encoder.encode(JSON.stringify(metadata))
|
|
114
|
+
);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async append(docId, entry) {
|
|
121
|
+
const seq = await this.#nextSeqNo(docId);
|
|
122
|
+
await this.#db.put(entryKey(docId, seq), encodeStoreEntry(entry));
|
|
123
|
+
}
|
|
124
|
+
async *loadAll(docId) {
|
|
125
|
+
const prefix = entryPrefix(docId);
|
|
126
|
+
for await (const value of this.#db.values({
|
|
127
|
+
gte: prefix,
|
|
128
|
+
lt: `${prefix}\xFF`
|
|
129
|
+
})) {
|
|
130
|
+
yield decodeStoreEntry(value);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async replace(docId, entry) {
|
|
134
|
+
const prefix = entryPrefix(docId);
|
|
135
|
+
const keysToDelete = [];
|
|
136
|
+
for await (const key of this.#db.keys({
|
|
137
|
+
gte: prefix,
|
|
138
|
+
lt: `${prefix}\xFF`
|
|
139
|
+
})) {
|
|
140
|
+
keysToDelete.push(key);
|
|
141
|
+
}
|
|
142
|
+
const ops = keysToDelete.map((key) => ({ type: "del", key }));
|
|
143
|
+
ops.push({
|
|
144
|
+
type: "put",
|
|
145
|
+
key: entryKey(docId, 0),
|
|
146
|
+
value: encodeStoreEntry(entry)
|
|
147
|
+
});
|
|
148
|
+
await this.#db.batch(ops);
|
|
149
|
+
this.#seqNos.set(docId, 0);
|
|
150
|
+
}
|
|
151
|
+
async delete(docId) {
|
|
152
|
+
const prefix = entryPrefix(docId);
|
|
153
|
+
const keysToDelete = [metaKey(docId)];
|
|
154
|
+
for await (const key of this.#db.keys({
|
|
155
|
+
gte: prefix,
|
|
156
|
+
lt: `${prefix}\xFF`
|
|
157
|
+
})) {
|
|
158
|
+
keysToDelete.push(key);
|
|
159
|
+
}
|
|
160
|
+
await this.#db.batch(
|
|
161
|
+
keysToDelete.map((key) => ({ type: "del", key }))
|
|
162
|
+
);
|
|
163
|
+
this.#seqNos.delete(docId);
|
|
164
|
+
}
|
|
165
|
+
async *listDocIds() {
|
|
166
|
+
for await (const key of this.#db.keys({
|
|
167
|
+
gte: META_PREFIX,
|
|
168
|
+
lt: `${META_PREFIX}\xFF`
|
|
169
|
+
})) {
|
|
170
|
+
yield parseDocIdFromMetaKey(key);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
async close() {
|
|
174
|
+
await this.#db.close();
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
function createLevelDBStore(dbPath) {
|
|
178
|
+
return new LevelDBStore(dbPath);
|
|
179
|
+
}
|
|
180
|
+
export {
|
|
181
|
+
LevelDBStore,
|
|
182
|
+
createLevelDBStore,
|
|
183
|
+
decodeStoreEntry,
|
|
184
|
+
encodeStoreEntry
|
|
185
|
+
};
|
|
186
|
+
//# sourceMappingURL=server.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/server.ts"],"sourcesContent":["// server — LevelDB storage backend for @kyneta/exchange.\n//\n// Implements the Store interface using classic-level.\n//\n// Key-space design (FoundationDB convention — \\x00 null-byte separator):\n// meta\\x00{docId} → JSON-encoded DocMetadata\n// entry\\x00{docId}\\x00{seqNo} → binary-encoded StoreEntry\n//\n// The \\x00 separator cannot appear in valid UTF-8 strings, so no docId\n// validation is needed — the key-space imposes zero constraints on callers.\n//\n// SeqNo is a zero-padded 10-digit monotonic counter per doc, tracked in\n// memory. On reboot, the max seqNo for a doc is lazily discovered via a\n// single reverse-iterator seek on first append.\n\nimport type { Store, StoreEntry } from \"@kyneta/exchange/src/store/store.js\"\nimport type { DocId } from \"@kyneta/exchange/src/types.js\"\nimport type { DocMetadata } from \"@kyneta/schema\"\nimport { ClassicLevel } from \"classic-level\"\n\n// ---------------------------------------------------------------------------\n// Constants\n// ---------------------------------------------------------------------------\n\nconst SEP = \"\\x00\"\nconst META_PREFIX = `meta${SEP}`\nconst ENTRY_PREFIX = `entry${SEP}`\nconst SEQ_PAD = 10\n\n// ---------------------------------------------------------------------------\n// Binary envelope — pure encode/decode for StoreEntry\n// ---------------------------------------------------------------------------\n\n// Flags byte layout:\n// bit 0: kind (0 = entirety, 1 = since)\n// bit 1: encoding (0 = json, 1 = binary)\n// bit 2: data type (0 = string, 1 = Uint8Array)\n//\n// Layout: [1 byte flags] [4 bytes version length BE] [N bytes version UTF-8] [remaining: payload data]\n\nconst encoder = new TextEncoder()\nconst decoder = new TextDecoder()\n\nexport function encodeStoreEntry(entry: StoreEntry): Uint8Array {\n const { payload, version } = entry\n\n let flags = 0\n if (payload.kind === \"since\") flags |= 0x01\n if (payload.encoding === \"binary\") flags |= 0x02\n\n const isDataBinary = payload.data instanceof Uint8Array\n if (isDataBinary) flags |= 0x04\n\n const versionBytes = encoder.encode(version)\n const dataBytes = isDataBinary\n ? (payload.data as Uint8Array)\n : encoder.encode(payload.data as string)\n\n const buf = new Uint8Array(1 + 4 + versionBytes.length + dataBytes.length)\n const view = new DataView(buf.buffer)\n\n buf[0] = flags\n view.setUint32(1, versionBytes.length, false) // big-endian\n buf.set(versionBytes, 5)\n buf.set(dataBytes, 5 + versionBytes.length)\n\n return buf\n}\n\nexport function decodeStoreEntry(bytes: Uint8Array): StoreEntry {\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)\n\n const flags = bytes[0]!\n const kind = (flags & 0x01) !== 0 ? \"since\" : \"entirety\"\n const encoding = (flags & 0x02) !== 0 ? \"binary\" : \"json\"\n const isDataBinary = (flags & 0x04) !== 0\n\n const versionLen = view.getUint32(1, false)\n const version = decoder.decode(bytes.subarray(5, 5 + versionLen))\n\n const dataStart = 5 + versionLen\n const rawData = bytes.subarray(dataStart)\n\n const data: string | Uint8Array = isDataBinary\n ? new Uint8Array(rawData)\n : decoder.decode(rawData)\n\n return {\n payload: { kind, encoding, data },\n version,\n }\n}\n\n// ---------------------------------------------------------------------------\n// Key helpers\n// ---------------------------------------------------------------------------\n\nfunction metaKey(docId: DocId): string {\n return `${META_PREFIX}${docId}`\n}\n\nfunction entryPrefix(docId: DocId): string {\n return `${ENTRY_PREFIX}${docId}${SEP}`\n}\n\nfunction entryKey(docId: DocId, seqNo: number): string {\n return `${entryPrefix(docId)}${String(seqNo).padStart(SEQ_PAD, \"0\")}`\n}\n\nfunction parseDocIdFromMetaKey(key: string): DocId {\n return key.slice(META_PREFIX.length)\n}\n\nfunction parseSeqNoFromEntryKey(key: string, docId: DocId): number {\n const prefix = entryPrefix(docId)\n return Number.parseInt(key.slice(prefix.length), 10)\n}\n\n// ---------------------------------------------------------------------------\n// LevelDBStore\n// ---------------------------------------------------------------------------\n\nexport class LevelDBStore implements Store {\n readonly #db: ClassicLevel<string, Uint8Array>\n readonly #seqNos: Map<DocId, number> = new Map()\n\n constructor(dbPath: string) {\n this.#db = new ClassicLevel(dbPath, {\n valueEncoding: \"binary\",\n })\n }\n\n // -------------------------------------------------------------------------\n // Private — seqNo management\n // -------------------------------------------------------------------------\n\n /**\n * Get the next seqNo for a doc. On first call after reboot, discovers\n * the max existing seqNo via a single reverse-iterator seek.\n */\n async #nextSeqNo(docId: DocId): Promise<number> {\n const cached = this.#seqNos.get(docId)\n if (cached !== undefined) {\n const next = cached + 1\n this.#seqNos.set(docId, next)\n return next\n }\n\n // Lazy discovery: reverse iterate to find the highest existing seqNo\n const prefix = entryPrefix(docId)\n let maxSeq = -1\n for await (const key of this.#db.keys({\n gte: prefix,\n lt: `${prefix}\\xff`,\n reverse: true,\n limit: 1,\n })) {\n maxSeq = parseSeqNoFromEntryKey(key, docId)\n }\n\n const next = maxSeq + 1\n this.#seqNos.set(docId, next)\n return next\n }\n\n // -------------------------------------------------------------------------\n // Store interface\n // -------------------------------------------------------------------------\n\n async lookup(docId: DocId): Promise<DocMetadata | null> {\n try {\n const raw = await this.#db.get(metaKey(docId))\n return JSON.parse(decoder.decode(raw)) as DocMetadata\n } catch (error: any) {\n if (error.code === \"LEVEL_NOT_FOUND\") return null\n throw error\n }\n }\n\n async ensureDoc(docId: DocId, metadata: DocMetadata): Promise<void> {\n try {\n await this.#db.get(metaKey(docId))\n // Already exists — no-op (first call wins)\n } catch (error: any) {\n if (error.code === \"LEVEL_NOT_FOUND\") {\n await this.#db.put(\n metaKey(docId),\n encoder.encode(JSON.stringify(metadata)),\n )\n return\n }\n throw error\n }\n }\n\n async append(docId: DocId, entry: StoreEntry): Promise<void> {\n const seq = await this.#nextSeqNo(docId)\n await this.#db.put(entryKey(docId, seq), encodeStoreEntry(entry))\n }\n\n async *loadAll(docId: DocId): AsyncIterable<StoreEntry> {\n const prefix = entryPrefix(docId)\n for await (const value of this.#db.values({\n gte: prefix,\n lt: `${prefix}\\xff`,\n })) {\n yield decodeStoreEntry(value)\n }\n }\n\n async replace(docId: DocId, entry: StoreEntry): Promise<void> {\n const prefix = entryPrefix(docId)\n\n // Collect keys to delete\n const keysToDelete: string[] = []\n for await (const key of this.#db.keys({\n gte: prefix,\n lt: `${prefix}\\xff`,\n })) {\n keysToDelete.push(key)\n }\n\n // Atomic batch: delete all existing entries + write the single replacement\n const ops: Array<\n | { type: \"del\"; key: string }\n | { type: \"put\"; key: string; value: Uint8Array }\n > = keysToDelete.map(key => ({ type: \"del\" as const, key }))\n ops.push({\n type: \"put\",\n key: entryKey(docId, 0),\n value: encodeStoreEntry(entry),\n })\n await this.#db.batch(ops)\n\n // Reset seqNo counter\n this.#seqNos.set(docId, 0)\n }\n\n async delete(docId: DocId): Promise<void> {\n const prefix = entryPrefix(docId)\n\n // Collect entry keys to delete\n const keysToDelete: string[] = [metaKey(docId)]\n for await (const key of this.#db.keys({\n gte: prefix,\n lt: `${prefix}\\xff`,\n })) {\n keysToDelete.push(key)\n }\n\n await this.#db.batch(\n keysToDelete.map(key => ({ type: \"del\" as const, key })),\n )\n\n // Remove from in-memory seqNo tracker\n this.#seqNos.delete(docId)\n }\n\n async *listDocIds(): AsyncIterable<DocId> {\n for await (const key of this.#db.keys({\n gte: META_PREFIX,\n lt: `${META_PREFIX}\\xff`,\n })) {\n yield parseDocIdFromMetaKey(key)\n }\n }\n\n async close(): Promise<void> {\n await this.#db.close()\n }\n}\n\n// ---------------------------------------------------------------------------\n// Factory function\n// ---------------------------------------------------------------------------\n\n/**\n * Create a LevelDB storage backend for server-side persistence.\n *\n * Returns a `Store` — pass directly to `Exchange({ stores: [...] })`.\n *\n * @param dbPath - Directory path where LevelDB stores its files\n *\n * @example\n * ```typescript\n * import { createLevelDBStore } from \"@kyneta/leveldb-store/server\"\n *\n * const exchange = new Exchange({\n * stores: [createLevelDBStore(\"./data/exchange-db\")],\n * })\n * ```\n */\nexport function createLevelDBStore(dbPath: string): Store {\n return new LevelDBStore(dbPath)\n}\n"],"mappings":";AAkBA,SAAS,oBAAoB;AAM7B,IAAM,MAAM;AACZ,IAAM,cAAc,OAAO,GAAG;AAC9B,IAAM,eAAe,QAAQ,GAAG;AAChC,IAAM,UAAU;AAahB,IAAM,UAAU,IAAI,YAAY;AAChC,IAAM,UAAU,IAAI,YAAY;AAEzB,SAAS,iBAAiB,OAA+B;AAC9D,QAAM,EAAE,SAAS,QAAQ,IAAI;AAE7B,MAAI,QAAQ;AACZ,MAAI,QAAQ,SAAS,QAAS,UAAS;AACvC,MAAI,QAAQ,aAAa,SAAU,UAAS;AAE5C,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,MAAI,aAAc,UAAS;AAE3B,QAAM,eAAe,QAAQ,OAAO,OAAO;AAC3C,QAAM,YAAY,eACb,QAAQ,OACT,QAAQ,OAAO,QAAQ,IAAc;AAEzC,QAAM,MAAM,IAAI,WAAW,IAAI,IAAI,aAAa,SAAS,UAAU,MAAM;AACzE,QAAM,OAAO,IAAI,SAAS,IAAI,MAAM;AAEpC,MAAI,CAAC,IAAI;AACT,OAAK,UAAU,GAAG,aAAa,QAAQ,KAAK;AAC5C,MAAI,IAAI,cAAc,CAAC;AACvB,MAAI,IAAI,WAAW,IAAI,aAAa,MAAM;AAE1C,SAAO;AACT;AAEO,SAAS,iBAAiB,OAA+B;AAC9D,QAAM,OAAO,IAAI,SAAS,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;AAE1E,QAAM,QAAQ,MAAM,CAAC;AACrB,QAAM,QAAQ,QAAQ,OAAU,IAAI,UAAU;AAC9C,QAAM,YAAY,QAAQ,OAAU,IAAI,WAAW;AACnD,QAAM,gBAAgB,QAAQ,OAAU;AAExC,QAAM,aAAa,KAAK,UAAU,GAAG,KAAK;AAC1C,QAAM,UAAU,QAAQ,OAAO,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC;AAEhE,QAAM,YAAY,IAAI;AACtB,QAAM,UAAU,MAAM,SAAS,SAAS;AAExC,QAAM,OAA4B,eAC9B,IAAI,WAAW,OAAO,IACtB,QAAQ,OAAO,OAAO;AAE1B,SAAO;AAAA,IACL,SAAS,EAAE,MAAM,UAAU,KAAK;AAAA,IAChC;AAAA,EACF;AACF;AAMA,SAAS,QAAQ,OAAsB;AACrC,SAAO,GAAG,WAAW,GAAG,KAAK;AAC/B;AAEA,SAAS,YAAY,OAAsB;AACzC,SAAO,GAAG,YAAY,GAAG,KAAK,GAAG,GAAG;AACtC;AAEA,SAAS,SAAS,OAAc,OAAuB;AACrD,SAAO,GAAG,YAAY,KAAK,CAAC,GAAG,OAAO,KAAK,EAAE,SAAS,SAAS,GAAG,CAAC;AACrE;AAEA,SAAS,sBAAsB,KAAoB;AACjD,SAAO,IAAI,MAAM,YAAY,MAAM;AACrC;AAEA,SAAS,uBAAuB,KAAa,OAAsB;AACjE,QAAM,SAAS,YAAY,KAAK;AAChC,SAAO,OAAO,SAAS,IAAI,MAAM,OAAO,MAAM,GAAG,EAAE;AACrD;AAMO,IAAM,eAAN,MAAoC;AAAA,EAChC;AAAA,EACA,UAA8B,oBAAI,IAAI;AAAA,EAE/C,YAAY,QAAgB;AAC1B,SAAK,MAAM,IAAI,aAAa,QAAQ;AAAA,MAClC,eAAe;AAAA,IACjB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAM,WAAW,OAA+B;AAC9C,UAAM,SAAS,KAAK,QAAQ,IAAI,KAAK;AACrC,QAAI,WAAW,QAAW;AACxB,YAAMA,QAAO,SAAS;AACtB,WAAK,QAAQ,IAAI,OAAOA,KAAI;AAC5B,aAAOA;AAAA,IACT;AAGA,UAAM,SAAS,YAAY,KAAK;AAChC,QAAI,SAAS;AACb,qBAAiB,OAAO,KAAK,IAAI,KAAK;AAAA,MACpC,KAAK;AAAA,MACL,IAAI,GAAG,MAAM;AAAA,MACb,SAAS;AAAA,MACT,OAAO;AAAA,IACT,CAAC,GAAG;AACF,eAAS,uBAAuB,KAAK,KAAK;AAAA,IAC5C;AAEA,UAAM,OAAO,SAAS;AACtB,SAAK,QAAQ,IAAI,OAAO,IAAI;AAC5B,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,OAAO,OAA2C;AACtD,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,IAAI,IAAI,QAAQ,KAAK,CAAC;AAC7C,aAAO,KAAK,MAAM,QAAQ,OAAO,GAAG,CAAC;AAAA,IACvC,SAAS,OAAY;AACnB,UAAI,MAAM,SAAS,kBAAmB,QAAO;AAC7C,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,UAAU,OAAc,UAAsC;AAClE,QAAI;AACF,YAAM,KAAK,IAAI,IAAI,QAAQ,KAAK,CAAC;AAAA,IAEnC,SAAS,OAAY;AACnB,UAAI,MAAM,SAAS,mBAAmB;AACpC,cAAM,KAAK,IAAI;AAAA,UACb,QAAQ,KAAK;AAAA,UACb,QAAQ,OAAO,KAAK,UAAU,QAAQ,CAAC;AAAA,QACzC;AACA;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,OAAc,OAAkC;AAC3D,UAAM,MAAM,MAAM,KAAK,WAAW,KAAK;AACvC,UAAM,KAAK,IAAI,IAAI,SAAS,OAAO,GAAG,GAAG,iBAAiB,KAAK,CAAC;AAAA,EAClE;AAAA,EAEA,OAAO,QAAQ,OAAyC;AACtD,UAAM,SAAS,YAAY,KAAK;AAChC,qBAAiB,SAAS,KAAK,IAAI,OAAO;AAAA,MACxC,KAAK;AAAA,MACL,IAAI,GAAG,MAAM;AAAA,IACf,CAAC,GAAG;AACF,YAAM,iBAAiB,KAAK;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ,OAAc,OAAkC;AAC5D,UAAM,SAAS,YAAY,KAAK;AAGhC,UAAM,eAAyB,CAAC;AAChC,qBAAiB,OAAO,KAAK,IAAI,KAAK;AAAA,MACpC,KAAK;AAAA,MACL,IAAI,GAAG,MAAM;AAAA,IACf,CAAC,GAAG;AACF,mBAAa,KAAK,GAAG;AAAA,IACvB;AAGA,UAAM,MAGF,aAAa,IAAI,UAAQ,EAAE,MAAM,OAAgB,IAAI,EAAE;AAC3D,QAAI,KAAK;AAAA,MACP,MAAM;AAAA,MACN,KAAK,SAAS,OAAO,CAAC;AAAA,MACtB,OAAO,iBAAiB,KAAK;AAAA,IAC/B,CAAC;AACD,UAAM,KAAK,IAAI,MAAM,GAAG;AAGxB,SAAK,QAAQ,IAAI,OAAO,CAAC;AAAA,EAC3B;AAAA,EAEA,MAAM,OAAO,OAA6B;AACxC,UAAM,SAAS,YAAY,KAAK;AAGhC,UAAM,eAAyB,CAAC,QAAQ,KAAK,CAAC;AAC9C,qBAAiB,OAAO,KAAK,IAAI,KAAK;AAAA,MACpC,KAAK;AAAA,MACL,IAAI,GAAG,MAAM;AAAA,IACf,CAAC,GAAG;AACF,mBAAa,KAAK,GAAG;AAAA,IACvB;AAEA,UAAM,KAAK,IAAI;AAAA,MACb,aAAa,IAAI,UAAQ,EAAE,MAAM,OAAgB,IAAI,EAAE;AAAA,IACzD;AAGA,SAAK,QAAQ,OAAO,KAAK;AAAA,EAC3B;AAAA,EAEA,OAAO,aAAmC;AACxC,qBAAiB,OAAO,KAAK,IAAI,KAAK;AAAA,MACpC,KAAK;AAAA,MACL,IAAI,GAAG,WAAW;AAAA,IACpB,CAAC,GAAG;AACF,YAAM,sBAAsB,GAAG;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,IAAI,MAAM;AAAA,EACvB;AACF;AAsBO,SAAS,mBAAmB,QAAuB;AACxD,SAAO,IAAI,aAAa,MAAM;AAChC;","names":["next"]}
|
package/package.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@kyneta/leveldb-store",
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"description": "LevelDB storage backend for @kyneta/exchange — server-side persistent storage",
|
|
5
|
+
"author": "Duane Johnson",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "https://github.com/halecraft/kyneta",
|
|
10
|
+
"directory": "packages/exchange/storage-adapters/leveldb"
|
|
11
|
+
},
|
|
12
|
+
"publishConfig": {
|
|
13
|
+
"access": "public"
|
|
14
|
+
},
|
|
15
|
+
"type": "module",
|
|
16
|
+
"files": [
|
|
17
|
+
"dist",
|
|
18
|
+
"src"
|
|
19
|
+
],
|
|
20
|
+
"exports": {
|
|
21
|
+
"./server": {
|
|
22
|
+
"types": "./dist/server.d.ts",
|
|
23
|
+
"import": "./dist/server.js"
|
|
24
|
+
},
|
|
25
|
+
"./src/*": "./src/*"
|
|
26
|
+
},
|
|
27
|
+
"peerDependencies": {
|
|
28
|
+
"@kyneta/exchange": "^1.1.0",
|
|
29
|
+
"@kyneta/schema": "^1.1.0"
|
|
30
|
+
},
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"classic-level": "^1.4.1"
|
|
33
|
+
},
|
|
34
|
+
"devDependencies": {
|
|
35
|
+
"tsup": "^8.5.0",
|
|
36
|
+
"typescript": "^5.9.2",
|
|
37
|
+
"vitest": "^4.0.17",
|
|
38
|
+
"@kyneta/exchange": "^1.1.0",
|
|
39
|
+
"@kyneta/schema": "^1.1.0"
|
|
40
|
+
},
|
|
41
|
+
"scripts": {
|
|
42
|
+
"build": "tsup",
|
|
43
|
+
"test": "verify logic",
|
|
44
|
+
"verify": "verify"
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
// leveldb-storage — conformance + LevelDB-specific tests.
|
|
2
|
+
//
|
|
3
|
+
// Runs the reusable Store conformance suite against
|
|
4
|
+
// LevelDBStore, plus LevelDB-specific tests for
|
|
5
|
+
// close+reopen persistence and encode/decode edge cases.
|
|
6
|
+
|
|
7
|
+
import * as fs from "node:fs"
|
|
8
|
+
import * as os from "node:os"
|
|
9
|
+
import * as path from "node:path"
|
|
10
|
+
import type { StoreEntry } from "@kyneta/exchange/src/store/store.js"
|
|
11
|
+
import {
|
|
12
|
+
collectAll,
|
|
13
|
+
describeStore,
|
|
14
|
+
makeEntry,
|
|
15
|
+
plainMetadata,
|
|
16
|
+
} from "@kyneta/exchange/src/testing/store-conformance.js"
|
|
17
|
+
import { afterAll, describe, expect, it } from "vitest"
|
|
18
|
+
import { decodeStoreEntry, encodeStoreEntry, LevelDBStore } from "../server.js"
|
|
19
|
+
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
// Temp directory management
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
const tmpDirs: string[] = []
|
|
25
|
+
|
|
26
|
+
function makeTmpDir(): string {
|
|
27
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "kyneta-leveldb-test-"))
|
|
28
|
+
tmpDirs.push(dir)
|
|
29
|
+
return dir
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
afterAll(() => {
|
|
33
|
+
for (const dir of tmpDirs) {
|
|
34
|
+
try {
|
|
35
|
+
fs.rmSync(dir, { recursive: true, force: true })
|
|
36
|
+
} catch {
|
|
37
|
+
// best-effort cleanup
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
// Conformance suite — validates the full Store contract
|
|
44
|
+
// ---------------------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
describeStore(
|
|
47
|
+
"LevelDBStore",
|
|
48
|
+
() => new LevelDBStore(makeTmpDir()),
|
|
49
|
+
async backend => {
|
|
50
|
+
if (backend.close) await backend.close()
|
|
51
|
+
},
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
// LevelDB-specific tests
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
describe("LevelDBStore — close + reopen", () => {
|
|
59
|
+
it("data persists across close and reopen on the same path", async () => {
|
|
60
|
+
const dir = makeTmpDir()
|
|
61
|
+
|
|
62
|
+
// Phase 1: write data then close
|
|
63
|
+
const backend1 = new LevelDBStore(dir)
|
|
64
|
+
await backend1.ensureDoc("doc-1", plainMetadata)
|
|
65
|
+
await backend1.append("doc-1", makeEntry("entirety", "v1"))
|
|
66
|
+
await backend1.append("doc-1", makeEntry("since", "v2"))
|
|
67
|
+
await backend1.close()
|
|
68
|
+
|
|
69
|
+
// Phase 2: reopen and verify
|
|
70
|
+
const backend2 = new LevelDBStore(dir)
|
|
71
|
+
expect(await backend2.lookup("doc-1")).toEqual(plainMetadata)
|
|
72
|
+
|
|
73
|
+
const entries = await collectAll(backend2.loadAll("doc-1"))
|
|
74
|
+
expect(entries).toHaveLength(2)
|
|
75
|
+
expect(entries[0]!.version).toBe("v1")
|
|
76
|
+
expect(entries[1]!.version).toBe("v2")
|
|
77
|
+
await backend2.close()
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
it("append after reopen continues with correct seqNo ordering", async () => {
|
|
81
|
+
const dir = makeTmpDir()
|
|
82
|
+
|
|
83
|
+
const backend1 = new LevelDBStore(dir)
|
|
84
|
+
await backend1.ensureDoc("doc-1", plainMetadata)
|
|
85
|
+
await backend1.append("doc-1", makeEntry("entirety", "v1"))
|
|
86
|
+
await backend1.append("doc-1", makeEntry("since", "v2"))
|
|
87
|
+
await backend1.close()
|
|
88
|
+
|
|
89
|
+
// Reopen and append more
|
|
90
|
+
const backend2 = new LevelDBStore(dir)
|
|
91
|
+
await backend2.append("doc-1", makeEntry("since", "v3"))
|
|
92
|
+
|
|
93
|
+
const entries = await collectAll(backend2.loadAll("doc-1"))
|
|
94
|
+
expect(entries).toHaveLength(3)
|
|
95
|
+
expect(entries[0]!.version).toBe("v1")
|
|
96
|
+
expect(entries[1]!.version).toBe("v2")
|
|
97
|
+
expect(entries[2]!.version).toBe("v3")
|
|
98
|
+
await backend2.close()
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
it("replace then reopen preserves the single entry", async () => {
|
|
102
|
+
const dir = makeTmpDir()
|
|
103
|
+
|
|
104
|
+
const backend1 = new LevelDBStore(dir)
|
|
105
|
+
await backend1.ensureDoc("doc-1", plainMetadata)
|
|
106
|
+
await backend1.append("doc-1", makeEntry("since", "v1"))
|
|
107
|
+
await backend1.append("doc-1", makeEntry("since", "v2"))
|
|
108
|
+
await backend1.replace("doc-1", makeEntry("entirety", "v3"))
|
|
109
|
+
await backend1.close()
|
|
110
|
+
|
|
111
|
+
const backend2 = new LevelDBStore(dir)
|
|
112
|
+
const entries = await collectAll(backend2.loadAll("doc-1"))
|
|
113
|
+
expect(entries).toHaveLength(1)
|
|
114
|
+
expect(entries[0]!.version).toBe("v3")
|
|
115
|
+
await backend2.close()
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
it("listDocIds works after reopen", async () => {
|
|
119
|
+
const dir = makeTmpDir()
|
|
120
|
+
|
|
121
|
+
const backend1 = new LevelDBStore(dir)
|
|
122
|
+
await backend1.ensureDoc("alpha", plainMetadata)
|
|
123
|
+
await backend1.ensureDoc("beta", plainMetadata)
|
|
124
|
+
await backend1.ensureDoc("gamma", plainMetadata)
|
|
125
|
+
await backend1.close()
|
|
126
|
+
|
|
127
|
+
const backend2 = new LevelDBStore(dir)
|
|
128
|
+
const docIds = await collectAll(backend2.listDocIds())
|
|
129
|
+
expect(docIds.sort()).toEqual(["alpha", "beta", "gamma"])
|
|
130
|
+
await backend2.close()
|
|
131
|
+
})
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
// encode/decode round-trip — pure function unit tests
|
|
136
|
+
// ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
describe("encodeStoreEntry / decodeStoreEntry", () => {
|
|
139
|
+
it("round-trips a JSON string payload (entirety)", () => {
|
|
140
|
+
const entry: StoreEntry = {
|
|
141
|
+
payload: {
|
|
142
|
+
kind: "entirety",
|
|
143
|
+
encoding: "json",
|
|
144
|
+
data: '{"hello":"world"}',
|
|
145
|
+
},
|
|
146
|
+
version: "42",
|
|
147
|
+
}
|
|
148
|
+
const decoded = decodeStoreEntry(encodeStoreEntry(entry))
|
|
149
|
+
expect(decoded).toEqual(entry)
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
it("round-trips a binary Uint8Array payload (since)", () => {
|
|
153
|
+
const bytes = new Uint8Array([0x00, 0x01, 0x02, 0xff, 0xfe])
|
|
154
|
+
const entry: StoreEntry = {
|
|
155
|
+
payload: { kind: "since", encoding: "binary", data: bytes },
|
|
156
|
+
version: "v7",
|
|
157
|
+
}
|
|
158
|
+
const decoded = decodeStoreEntry(encodeStoreEntry(entry))
|
|
159
|
+
expect(decoded.version).toBe("v7")
|
|
160
|
+
expect(decoded.payload.kind).toBe("since")
|
|
161
|
+
expect(decoded.payload.encoding).toBe("binary")
|
|
162
|
+
expect(decoded.payload.data).toBeInstanceOf(Uint8Array)
|
|
163
|
+
expect(decoded.payload.data).toEqual(bytes)
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
it("handles empty string data", () => {
|
|
167
|
+
const entry: StoreEntry = {
|
|
168
|
+
payload: { kind: "entirety", encoding: "json", data: "" },
|
|
169
|
+
version: "v0",
|
|
170
|
+
}
|
|
171
|
+
const decoded = decodeStoreEntry(encodeStoreEntry(entry))
|
|
172
|
+
expect(decoded).toEqual(entry)
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
it("handles empty Uint8Array data", () => {
|
|
176
|
+
const entry: StoreEntry = {
|
|
177
|
+
payload: { kind: "since", encoding: "binary", data: new Uint8Array(0) },
|
|
178
|
+
version: "v0",
|
|
179
|
+
}
|
|
180
|
+
const decoded = decodeStoreEntry(encodeStoreEntry(entry))
|
|
181
|
+
expect(decoded.payload.data).toBeInstanceOf(Uint8Array)
|
|
182
|
+
expect((decoded.payload.data as Uint8Array).length).toBe(0)
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
it("handles empty version string", () => {
|
|
186
|
+
const entry: StoreEntry = {
|
|
187
|
+
payload: { kind: "entirety", encoding: "json", data: "{}" },
|
|
188
|
+
version: "",
|
|
189
|
+
}
|
|
190
|
+
const decoded = decodeStoreEntry(encodeStoreEntry(entry))
|
|
191
|
+
expect(decoded.version).toBe("")
|
|
192
|
+
expect(decoded.payload.data).toBe("{}")
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
it("handles large binary payload", () => {
|
|
196
|
+
const largeData = new Uint8Array(100_000)
|
|
197
|
+
for (let i = 0; i < largeData.length; i++) {
|
|
198
|
+
largeData[i] = i % 256
|
|
199
|
+
}
|
|
200
|
+
const entry: StoreEntry = {
|
|
201
|
+
payload: { kind: "entirety", encoding: "binary", data: largeData },
|
|
202
|
+
version: "large-v1",
|
|
203
|
+
}
|
|
204
|
+
const decoded = decodeStoreEntry(encodeStoreEntry(entry))
|
|
205
|
+
expect(decoded.payload.data).toEqual(largeData)
|
|
206
|
+
})
|
|
207
|
+
})
|
package/src/server.ts
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
// server — LevelDB storage backend for @kyneta/exchange.
|
|
2
|
+
//
|
|
3
|
+
// Implements the Store interface using classic-level.
|
|
4
|
+
//
|
|
5
|
+
// Key-space design (FoundationDB convention — \x00 null-byte separator):
|
|
6
|
+
// meta\x00{docId} → JSON-encoded DocMetadata
|
|
7
|
+
// entry\x00{docId}\x00{seqNo} → binary-encoded StoreEntry
|
|
8
|
+
//
|
|
9
|
+
// The \x00 separator cannot appear in valid UTF-8 strings, so no docId
|
|
10
|
+
// validation is needed — the key-space imposes zero constraints on callers.
|
|
11
|
+
//
|
|
12
|
+
// SeqNo is a zero-padded 10-digit monotonic counter per doc, tracked in
|
|
13
|
+
// memory. On reboot, the max seqNo for a doc is lazily discovered via a
|
|
14
|
+
// single reverse-iterator seek on first append.
|
|
15
|
+
|
|
16
|
+
import type { Store, StoreEntry } from "@kyneta/exchange/src/store/store.js"
|
|
17
|
+
import type { DocId } from "@kyneta/exchange/src/types.js"
|
|
18
|
+
import type { DocMetadata } from "@kyneta/schema"
|
|
19
|
+
import { ClassicLevel } from "classic-level"
|
|
20
|
+
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// Constants
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
const SEP = "\x00"
|
|
26
|
+
const META_PREFIX = `meta${SEP}`
|
|
27
|
+
const ENTRY_PREFIX = `entry${SEP}`
|
|
28
|
+
const SEQ_PAD = 10
|
|
29
|
+
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
// Binary envelope — pure encode/decode for StoreEntry
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
|
|
34
|
+
// Flags byte layout:
|
|
35
|
+
// bit 0: kind (0 = entirety, 1 = since)
|
|
36
|
+
// bit 1: encoding (0 = json, 1 = binary)
|
|
37
|
+
// bit 2: data type (0 = string, 1 = Uint8Array)
|
|
38
|
+
//
|
|
39
|
+
// Layout: [1 byte flags] [4 bytes version length BE] [N bytes version UTF-8] [remaining: payload data]
|
|
40
|
+
|
|
41
|
+
const encoder = new TextEncoder()
|
|
42
|
+
const decoder = new TextDecoder()
|
|
43
|
+
|
|
44
|
+
export function encodeStoreEntry(entry: StoreEntry): Uint8Array {
|
|
45
|
+
const { payload, version } = entry
|
|
46
|
+
|
|
47
|
+
let flags = 0
|
|
48
|
+
if (payload.kind === "since") flags |= 0x01
|
|
49
|
+
if (payload.encoding === "binary") flags |= 0x02
|
|
50
|
+
|
|
51
|
+
const isDataBinary = payload.data instanceof Uint8Array
|
|
52
|
+
if (isDataBinary) flags |= 0x04
|
|
53
|
+
|
|
54
|
+
const versionBytes = encoder.encode(version)
|
|
55
|
+
const dataBytes = isDataBinary
|
|
56
|
+
? (payload.data as Uint8Array)
|
|
57
|
+
: encoder.encode(payload.data as string)
|
|
58
|
+
|
|
59
|
+
const buf = new Uint8Array(1 + 4 + versionBytes.length + dataBytes.length)
|
|
60
|
+
const view = new DataView(buf.buffer)
|
|
61
|
+
|
|
62
|
+
buf[0] = flags
|
|
63
|
+
view.setUint32(1, versionBytes.length, false) // big-endian
|
|
64
|
+
buf.set(versionBytes, 5)
|
|
65
|
+
buf.set(dataBytes, 5 + versionBytes.length)
|
|
66
|
+
|
|
67
|
+
return buf
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function decodeStoreEntry(bytes: Uint8Array): StoreEntry {
|
|
71
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
72
|
+
|
|
73
|
+
const flags = bytes[0]!
|
|
74
|
+
const kind = (flags & 0x01) !== 0 ? "since" : "entirety"
|
|
75
|
+
const encoding = (flags & 0x02) !== 0 ? "binary" : "json"
|
|
76
|
+
const isDataBinary = (flags & 0x04) !== 0
|
|
77
|
+
|
|
78
|
+
const versionLen = view.getUint32(1, false)
|
|
79
|
+
const version = decoder.decode(bytes.subarray(5, 5 + versionLen))
|
|
80
|
+
|
|
81
|
+
const dataStart = 5 + versionLen
|
|
82
|
+
const rawData = bytes.subarray(dataStart)
|
|
83
|
+
|
|
84
|
+
const data: string | Uint8Array = isDataBinary
|
|
85
|
+
? new Uint8Array(rawData)
|
|
86
|
+
: decoder.decode(rawData)
|
|
87
|
+
|
|
88
|
+
return {
|
|
89
|
+
payload: { kind, encoding, data },
|
|
90
|
+
version,
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// Key helpers
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
function metaKey(docId: DocId): string {
|
|
99
|
+
return `${META_PREFIX}${docId}`
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function entryPrefix(docId: DocId): string {
|
|
103
|
+
return `${ENTRY_PREFIX}${docId}${SEP}`
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function entryKey(docId: DocId, seqNo: number): string {
|
|
107
|
+
return `${entryPrefix(docId)}${String(seqNo).padStart(SEQ_PAD, "0")}`
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function parseDocIdFromMetaKey(key: string): DocId {
|
|
111
|
+
return key.slice(META_PREFIX.length)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function parseSeqNoFromEntryKey(key: string, docId: DocId): number {
|
|
115
|
+
const prefix = entryPrefix(docId)
|
|
116
|
+
return Number.parseInt(key.slice(prefix.length), 10)
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
// LevelDBStore
|
|
121
|
+
// ---------------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
export class LevelDBStore implements Store {
|
|
124
|
+
readonly #db: ClassicLevel<string, Uint8Array>
|
|
125
|
+
readonly #seqNos: Map<DocId, number> = new Map()
|
|
126
|
+
|
|
127
|
+
constructor(dbPath: string) {
|
|
128
|
+
this.#db = new ClassicLevel(dbPath, {
|
|
129
|
+
valueEncoding: "binary",
|
|
130
|
+
})
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// -------------------------------------------------------------------------
|
|
134
|
+
// Private — seqNo management
|
|
135
|
+
// -------------------------------------------------------------------------
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Get the next seqNo for a doc. On first call after reboot, discovers
|
|
139
|
+
* the max existing seqNo via a single reverse-iterator seek.
|
|
140
|
+
*/
|
|
141
|
+
async #nextSeqNo(docId: DocId): Promise<number> {
|
|
142
|
+
const cached = this.#seqNos.get(docId)
|
|
143
|
+
if (cached !== undefined) {
|
|
144
|
+
const next = cached + 1
|
|
145
|
+
this.#seqNos.set(docId, next)
|
|
146
|
+
return next
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Lazy discovery: reverse iterate to find the highest existing seqNo
|
|
150
|
+
const prefix = entryPrefix(docId)
|
|
151
|
+
let maxSeq = -1
|
|
152
|
+
for await (const key of this.#db.keys({
|
|
153
|
+
gte: prefix,
|
|
154
|
+
lt: `${prefix}\xff`,
|
|
155
|
+
reverse: true,
|
|
156
|
+
limit: 1,
|
|
157
|
+
})) {
|
|
158
|
+
maxSeq = parseSeqNoFromEntryKey(key, docId)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const next = maxSeq + 1
|
|
162
|
+
this.#seqNos.set(docId, next)
|
|
163
|
+
return next
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// -------------------------------------------------------------------------
|
|
167
|
+
// Store interface
|
|
168
|
+
// -------------------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
async lookup(docId: DocId): Promise<DocMetadata | null> {
|
|
171
|
+
try {
|
|
172
|
+
const raw = await this.#db.get(metaKey(docId))
|
|
173
|
+
return JSON.parse(decoder.decode(raw)) as DocMetadata
|
|
174
|
+
} catch (error: any) {
|
|
175
|
+
if (error.code === "LEVEL_NOT_FOUND") return null
|
|
176
|
+
throw error
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async ensureDoc(docId: DocId, metadata: DocMetadata): Promise<void> {
|
|
181
|
+
try {
|
|
182
|
+
await this.#db.get(metaKey(docId))
|
|
183
|
+
// Already exists — no-op (first call wins)
|
|
184
|
+
} catch (error: any) {
|
|
185
|
+
if (error.code === "LEVEL_NOT_FOUND") {
|
|
186
|
+
await this.#db.put(
|
|
187
|
+
metaKey(docId),
|
|
188
|
+
encoder.encode(JSON.stringify(metadata)),
|
|
189
|
+
)
|
|
190
|
+
return
|
|
191
|
+
}
|
|
192
|
+
throw error
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async append(docId: DocId, entry: StoreEntry): Promise<void> {
|
|
197
|
+
const seq = await this.#nextSeqNo(docId)
|
|
198
|
+
await this.#db.put(entryKey(docId, seq), encodeStoreEntry(entry))
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async *loadAll(docId: DocId): AsyncIterable<StoreEntry> {
|
|
202
|
+
const prefix = entryPrefix(docId)
|
|
203
|
+
for await (const value of this.#db.values({
|
|
204
|
+
gte: prefix,
|
|
205
|
+
lt: `${prefix}\xff`,
|
|
206
|
+
})) {
|
|
207
|
+
yield decodeStoreEntry(value)
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
async replace(docId: DocId, entry: StoreEntry): Promise<void> {
|
|
212
|
+
const prefix = entryPrefix(docId)
|
|
213
|
+
|
|
214
|
+
// Collect keys to delete
|
|
215
|
+
const keysToDelete: string[] = []
|
|
216
|
+
for await (const key of this.#db.keys({
|
|
217
|
+
gte: prefix,
|
|
218
|
+
lt: `${prefix}\xff`,
|
|
219
|
+
})) {
|
|
220
|
+
keysToDelete.push(key)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Atomic batch: delete all existing entries + write the single replacement
|
|
224
|
+
const ops: Array<
|
|
225
|
+
| { type: "del"; key: string }
|
|
226
|
+
| { type: "put"; key: string; value: Uint8Array }
|
|
227
|
+
> = keysToDelete.map(key => ({ type: "del" as const, key }))
|
|
228
|
+
ops.push({
|
|
229
|
+
type: "put",
|
|
230
|
+
key: entryKey(docId, 0),
|
|
231
|
+
value: encodeStoreEntry(entry),
|
|
232
|
+
})
|
|
233
|
+
await this.#db.batch(ops)
|
|
234
|
+
|
|
235
|
+
// Reset seqNo counter
|
|
236
|
+
this.#seqNos.set(docId, 0)
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async delete(docId: DocId): Promise<void> {
|
|
240
|
+
const prefix = entryPrefix(docId)
|
|
241
|
+
|
|
242
|
+
// Collect entry keys to delete
|
|
243
|
+
const keysToDelete: string[] = [metaKey(docId)]
|
|
244
|
+
for await (const key of this.#db.keys({
|
|
245
|
+
gte: prefix,
|
|
246
|
+
lt: `${prefix}\xff`,
|
|
247
|
+
})) {
|
|
248
|
+
keysToDelete.push(key)
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
await this.#db.batch(
|
|
252
|
+
keysToDelete.map(key => ({ type: "del" as const, key })),
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
// Remove from in-memory seqNo tracker
|
|
256
|
+
this.#seqNos.delete(docId)
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
async *listDocIds(): AsyncIterable<DocId> {
|
|
260
|
+
for await (const key of this.#db.keys({
|
|
261
|
+
gte: META_PREFIX,
|
|
262
|
+
lt: `${META_PREFIX}\xff`,
|
|
263
|
+
})) {
|
|
264
|
+
yield parseDocIdFromMetaKey(key)
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async close(): Promise<void> {
|
|
269
|
+
await this.#db.close()
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// ---------------------------------------------------------------------------
|
|
274
|
+
// Factory function
|
|
275
|
+
// ---------------------------------------------------------------------------
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Create a LevelDB storage backend for server-side persistence.
|
|
279
|
+
*
|
|
280
|
+
* Returns a `Store` — pass directly to `Exchange({ stores: [...] })`.
|
|
281
|
+
*
|
|
282
|
+
* @param dbPath - Directory path where LevelDB stores its files
|
|
283
|
+
*
|
|
284
|
+
* @example
|
|
285
|
+
* ```typescript
|
|
286
|
+
* import { createLevelDBStore } from "@kyneta/leveldb-store/server"
|
|
287
|
+
*
|
|
288
|
+
* const exchange = new Exchange({
|
|
289
|
+
* stores: [createLevelDBStore("./data/exchange-db")],
|
|
290
|
+
* })
|
|
291
|
+
* ```
|
|
292
|
+
*/
|
|
293
|
+
export function createLevelDBStore(dbPath: string): Store {
|
|
294
|
+
return new LevelDBStore(dbPath)
|
|
295
|
+
}
|