@typeberry/lib 0.8.4 → 0.9.0-959889e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -4
- package/packages/configs/index.d.ts +30 -1
- package/packages/configs/index.d.ts.map +1 -1
- package/packages/configs/index.js +4 -2
- package/packages/configs/typeberry-dev-full.json +29 -0
- package/packages/core/bytes/bytes.d.ts +1 -0
- package/packages/core/bytes/bytes.d.ts.map +1 -1
- package/packages/core/bytes/bytes.js +8 -0
- package/packages/core/utils/debug.d.ts +4 -2
- package/packages/core/utils/debug.d.ts.map +1 -1
- package/packages/core/utils/debug.js +18 -13
- package/packages/core/utils/debug.test.js +12 -6
- package/packages/core/utils/index.d.ts +1 -0
- package/packages/core/utils/index.d.ts.map +1 -1
- package/packages/core/utils/index.js +1 -0
- package/packages/core/utils/shutdown.d.ts +18 -0
- package/packages/core/utils/shutdown.d.ts.map +1 -0
- package/packages/core/utils/shutdown.js +80 -0
- package/packages/core/utils/shutdown.test.d.ts +2 -0
- package/packages/core/utils/shutdown.test.d.ts.map +1 -0
- package/packages/core/utils/shutdown.test.js +155 -0
- package/packages/extensions/ipc/server.d.ts.map +1 -1
- package/packages/extensions/ipc/server.js +7 -0
- package/packages/jam/config-node/node-config.d.ts +2 -1
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +8 -3
- package/packages/jam/config-node/node-config.test.js +3 -3
- package/packages/jam/database-fjall/hybrid-states.d.ts +82 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.js +169 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts +2 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.test.js +113 -0
- package/packages/jam/database-fjall/index.d.ts +3 -0
- package/packages/jam/database-fjall/index.d.ts.map +1 -0
- package/packages/jam/database-fjall/index.js +2 -0
- package/packages/jam/database-fjall/root.d.ts +58 -0
- package/packages/jam/database-fjall/root.d.ts.map +1 -0
- package/packages/jam/database-fjall/root.js +89 -0
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts +18 -10
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts.map +1 -1
- package/packages/jam/jamnp-s/tasks/ticket-distribution.js +44 -68
- package/packages/jam/jamnp-s/tasks/ticket-distribution.test.js +30 -8
- package/packages/jam/node/main-fuzz.d.ts +4 -1
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +89 -16
- package/packages/jam/node/main-importer.d.ts +13 -3
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +5 -3
- package/packages/jam/safrole/bandersnatch-vrf.d.ts +24 -4
- package/packages/jam/safrole/bandersnatch-vrf.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-vrf.js +63 -26
- package/packages/jam/safrole/bandersnatch-vrf.test.js +12 -9
- package/packages/jam/safrole/bandersnatch-wasm.d.ts +10 -0
- package/packages/jam/safrole/bandersnatch-wasm.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-wasm.js +12 -0
- package/packages/jam/safrole/safrole.js +5 -5
- package/packages/jam/safrole/safrole.test.js +13 -13
- package/packages/jam/ticket-pool/index.d.ts +4 -0
- package/packages/jam/ticket-pool/index.d.ts.map +1 -0
- package/packages/jam/ticket-pool/index.js +3 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts +30 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.js +56 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.js +67 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts +47 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.js +34 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts +2 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.test.js +35 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts +26 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.js +41 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.js +54 -0
- package/packages/workers/api-node/config.d.ts +21 -6
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +26 -19
- package/packages/workers/api-node/config.test.js +38 -1
- package/packages/workers/block-authorship/{generator.d.ts → block-generator.d.ts} +5 -5
- package/packages/workers/block-authorship/block-generator.d.ts.map +1 -0
- package/packages/workers/block-authorship/{generator.js → block-generator.js} +3 -3
- package/packages/workers/block-authorship/block-generator.test.d.ts +2 -0
- package/packages/workers/block-authorship/block-generator.test.d.ts.map +1 -0
- package/packages/workers/block-authorship/{generator.test.js → block-generator.test.js} +8 -8
- package/packages/workers/block-authorship/epoch-authoring-slots.d.ts +35 -0
- package/packages/workers/block-authorship/epoch-authoring-slots.d.ts.map +1 -0
- package/packages/workers/block-authorship/epoch-authoring-slots.js +86 -0
- package/packages/workers/block-authorship/epoch-tracker.d.ts +29 -0
- package/packages/workers/block-authorship/epoch-tracker.d.ts.map +1 -0
- package/packages/workers/block-authorship/epoch-tracker.js +80 -0
- package/packages/workers/block-authorship/index.d.ts.map +1 -1
- package/packages/workers/block-authorship/index.js +1 -1
- package/packages/workers/block-authorship/main.d.ts +3 -0
- package/packages/workers/block-authorship/main.d.ts.map +1 -1
- package/packages/workers/block-authorship/main.js +197 -315
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts +2 -0
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.js +23 -0
- package/packages/workers/block-authorship/ticket-generator/index.d.ts +16 -0
- package/packages/workers/block-authorship/ticket-generator/index.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/index.js +62 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.d.ts +50 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.js +54 -0
- package/packages/workers/block-authorship/{ticket-generator.d.ts → ticket-generator/ticket-generator.d.ts} +4 -0
- package/packages/workers/block-authorship/ticket-generator/ticket-generator.d.ts.map +1 -0
- package/packages/workers/block-authorship/{ticket-generator.js → ticket-generator/ticket-generator.js} +19 -9
- package/packages/workers/block-authorship/ticket-generator/ticket-generator.test.d.ts.map +1 -0
- package/packages/workers/block-authorship/{ticket-generator.test.js → ticket-generator/ticket-generator.test.js} +13 -9
- package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts +36 -0
- package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/worker-pool.js +111 -0
- package/packages/workers/block-authorship/ticket-validator.d.ts +31 -0
- package/packages/workers/block-authorship/ticket-validator.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-validator.js +59 -0
- package/packages/workers/comms-authorship-network/protocol.d.ts +14 -4
- package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/protocol.js +12 -6
- package/packages/workers/comms-authorship-network/tickets-message.d.ts +0 -14
- package/packages/workers/comms-authorship-network/tickets-message.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/tickets-message.js +0 -17
- package/packages/workers/importer/importer.d.ts +2 -2
- package/packages/workers/importer/importer.d.ts.map +1 -1
- package/packages/workers/importer/importer.js +5 -5
- package/packages/workers/importer/stats.d.ts +1 -3
- package/packages/workers/importer/stats.d.ts.map +1 -1
- package/packages/workers/importer/stats.js +12 -12
- package/packages/workers/jam-network/main.d.ts.map +1 -1
- package/packages/workers/jam-network/main.js +25 -4
- package/packages/workers/block-authorship/generator.d.ts.map +0 -1
- package/packages/workers/block-authorship/generator.test.d.ts +0 -2
- package/packages/workers/block-authorship/generator.test.d.ts.map +0 -1
- package/packages/workers/block-authorship/ticket-generator.d.ts.map +0 -1
- package/packages/workers/block-authorship/ticket-generator.test.d.ts.map +0 -1
- /package/packages/configs/{typeberry-dev.json → typeberry-dev-tiny.json} +0 -0
- /package/packages/workers/block-authorship/{ticket-generator.test.d.ts → ticket-generator/ticket-generator.test.d.ts} +0 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { HashDictionary, SortedSet } from "#@typeberry/collections";
|
|
2
|
+
import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
|
|
3
|
+
import { Logger } from "#@typeberry/logger";
|
|
4
|
+
import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
|
|
5
|
+
import { leafComparator } from "#@typeberry/trie";
|
|
6
|
+
import { OK, Result } from "#@typeberry/utils";
|
|
7
|
+
import { FjallRoot, toUint8Array } from "./root.js";
|
|
8
|
+
const logger = Logger.new(import.meta.filename, "db");
|
|
9
|
+
/**
|
|
10
|
+
* One open fjall keyspace together with its content-addressed `values`
|
|
11
|
+
* partition.
|
|
12
|
+
*
|
|
13
|
+
* Opening the keyspace is the slow part, so the fuzz target opens one session
|
|
14
|
+
* per run and reuses it for every reset (see `HybridSerializedStates.fromSession`).
|
|
15
|
+
* The values partition is immutable - the key is the hash of the value - so it
|
|
16
|
+
* is fine that values pile up across resets, the unreferenced ones just sit
|
|
17
|
+
* there unused.
|
|
18
|
+
*/
|
|
19
|
+
export class FjallValuesSession {
|
|
20
|
+
root;
|
|
21
|
+
values;
|
|
22
|
+
constructor(root,
|
|
23
|
+
/** Shared content-addressed values partition, reused across resets. */
|
|
24
|
+
values) {
|
|
25
|
+
this.root = root;
|
|
26
|
+
this.values = values;
|
|
27
|
+
}
|
|
28
|
+
/** Open (or create) the keyspace at `dbPath` and its `values` partition. */
|
|
29
|
+
static async open(dbPath, options = {}) {
|
|
30
|
+
const root = await FjallRoot.open(dbPath, options);
|
|
31
|
+
const values = await root.partition("values");
|
|
32
|
+
return new FjallValuesSession(root, values);
|
|
33
|
+
}
|
|
34
|
+
/** Flush the journal to disk (a no-op for ephemeral keyspaces). */
|
|
35
|
+
async persist() {
|
|
36
|
+
await this.root.persist();
|
|
37
|
+
}
|
|
38
|
+
/** Size of the keyspace directory on disk, in bytes. */
|
|
39
|
+
sizeInBytes() {
|
|
40
|
+
return this.root.sizeInBytes();
|
|
41
|
+
}
|
|
42
|
+
/** Release the keyspace handle (skips the sync-all fsync when ephemeral). */
|
|
43
|
+
async close() {
|
|
44
|
+
await this.root.close();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Hybrid serialized-states db (fjall variant).
|
|
49
|
+
*
|
|
50
|
+
* States (leafs) are kept in memory, only the large values go to fjall on disk.
|
|
51
|
+
* Reads hit fjall directly, which keeps its own bounded block cache. Meant for
|
|
52
|
+
* long fuzzing, used together with pruning so the heap stays bounded.
|
|
53
|
+
*
|
|
54
|
+
* Construction is async, and value writes are flushed explicitly, because fjall
|
|
55
|
+
* has no transaction primitive.
|
|
56
|
+
*/
|
|
57
|
+
export class HybridSerializedStates {
|
|
58
|
+
spec;
|
|
59
|
+
blake2b;
|
|
60
|
+
session;
|
|
61
|
+
ownsSession;
|
|
62
|
+
static async new({ spec, blake2b, dbPath, ephemeral, cacheSizeBytes, }) {
|
|
63
|
+
const session = await FjallValuesSession.open(dbPath, { ephemeral, cacheSizeBytes });
|
|
64
|
+
// This instance owns the session it just opened, so its `close()` closes it.
|
|
65
|
+
return new HybridSerializedStates(spec, blake2b, session, true);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Wrap an already-open `FjallValuesSession` and reuse its keyspace.
|
|
69
|
+
*
|
|
70
|
+
* The new instance starts with its own empty in-memory leaf sets but shares
|
|
71
|
+
* the values partition on disk. Its `close()` does not close the session, the
|
|
72
|
+
* session owner closes it once. The fuzz target uses this to keep one keyspace
|
|
73
|
+
* across resets and only rebuild the in-memory state for each vector.
|
|
74
|
+
*/
|
|
75
|
+
static fromSession(spec, blake2b, session) {
|
|
76
|
+
return new HybridSerializedStates(spec, blake2b, session, false);
|
|
77
|
+
}
|
|
78
|
+
inMemStates = HashDictionary.new();
|
|
79
|
+
// A single shared values accessor reused by every `LeafDb` we hand out.
|
|
80
|
+
valuesDb;
|
|
81
|
+
/** Shared content-addressed values partition (owned by `session`). */
|
|
82
|
+
values;
|
|
83
|
+
constructor(spec, blake2b, session,
|
|
84
|
+
/** Whether `close()` should close the underlying session. */
|
|
85
|
+
ownsSession) {
|
|
86
|
+
this.spec = spec;
|
|
87
|
+
this.blake2b = blake2b;
|
|
88
|
+
this.session = session;
|
|
89
|
+
this.ownsSession = ownsSession;
|
|
90
|
+
this.values = session.values;
|
|
91
|
+
this.valuesDb = { get: (key) => this.readValue(key) };
|
|
92
|
+
}
|
|
93
|
+
async insertInitialState(headerHash, entries) {
|
|
94
|
+
const { values, leafs } = updateLeafs(SortedSet.fromArray(leafComparator, []), this.blake2b, Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
|
|
95
|
+
const res = await this.writeValues(values);
|
|
96
|
+
if (res.isError) {
|
|
97
|
+
return res;
|
|
98
|
+
}
|
|
99
|
+
this.inMemStates.set(headerHash, leafs);
|
|
100
|
+
return Result.ok(OK);
|
|
101
|
+
}
|
|
102
|
+
async updateAndSetState(header, state, update) {
|
|
103
|
+
const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
|
|
104
|
+
// Clone the leaf set before mutating: the previous state keeps using its own.
|
|
105
|
+
const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
|
|
106
|
+
const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
|
|
107
|
+
const res = await this.writeValues(values);
|
|
108
|
+
if (res.isError) {
|
|
109
|
+
// Leave the caller's state untouched: its new leaves would reference
|
|
110
|
+
// values that never reached disk.
|
|
111
|
+
return res;
|
|
112
|
+
}
|
|
113
|
+
// Re-create the lookup with the shared values accessor only once the new
|
|
114
|
+
// values are durably written.
|
|
115
|
+
state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
|
|
116
|
+
this.inMemStates.set(header, leafs);
|
|
117
|
+
return Result.ok(OK);
|
|
118
|
+
}
|
|
119
|
+
async getStateRoot(state) {
|
|
120
|
+
return state.backend.getStateRoot(this.blake2b);
|
|
121
|
+
}
|
|
122
|
+
getState(header) {
|
|
123
|
+
const leafs = this.inMemStates.get(header);
|
|
124
|
+
if (leafs === undefined) {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
|
|
128
|
+
return SerializedState.new(this.spec, this.blake2b, leafDb);
|
|
129
|
+
}
|
|
130
|
+
markUnused(header) {
|
|
131
|
+
// We only remove the state from memory - values are not pruned at all,
|
|
132
|
+
// but since they are stored on disk we should be safe.
|
|
133
|
+
this.inMemStates.delete(header);
|
|
134
|
+
}
|
|
135
|
+
diskSizeInBytes() {
|
|
136
|
+
return this.session.sizeInBytes();
|
|
137
|
+
}
|
|
138
|
+
async close() {
|
|
139
|
+
// Instances backed by a shared session (fuzz reset reuse) keep the keyspace
|
|
140
|
+
// open for the next reset. The session owner closes it once.
|
|
141
|
+
if (this.ownsSession) {
|
|
142
|
+
await this.session.close();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/** Write new large values to fjall in a single batch, then flush. */
|
|
146
|
+
async writeValues(values) {
|
|
147
|
+
if (values.length === 0) {
|
|
148
|
+
return Result.ok(OK);
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
const entries = values.map(([hash, val]) => ({ key: hash.raw, value: val.raw }));
|
|
152
|
+
await this.values.insertBatch(entries);
|
|
153
|
+
await this.session.persist();
|
|
154
|
+
}
|
|
155
|
+
catch (e) {
|
|
156
|
+
logger.error `${e}`;
|
|
157
|
+
return Result.error(StateUpdateError.Commit, () => `Failed to commit values: ${e}`);
|
|
158
|
+
}
|
|
159
|
+
return Result.ok(OK);
|
|
160
|
+
}
|
|
161
|
+
/** Read a value from fjall. */
|
|
162
|
+
readValue(key) {
|
|
163
|
+
const val = toUint8Array(this.values.get(key.raw));
|
|
164
|
+
if (val === null) {
|
|
165
|
+
throw new Error(`Missing value at key: ${key}`);
|
|
166
|
+
}
|
|
167
|
+
return val;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hybrid-states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/hybrid-states.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { afterEach, before, beforeEach, describe, it } from "node:test";
|
|
4
|
+
import { Bytes, BytesBlob } from "#@typeberry/bytes";
|
|
5
|
+
import { tinyChainSpec } from "#@typeberry/config";
|
|
6
|
+
import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
|
|
7
|
+
import { InMemoryState } from "#@typeberry/state";
|
|
8
|
+
import { StateEntries } from "#@typeberry/state-merkleization";
|
|
9
|
+
import { deepEqual, OK, Result } from "#@typeberry/utils";
|
|
10
|
+
import { FjallValuesSession, HybridSerializedStates } from "./hybrid-states.js";
|
|
11
|
+
let blake2b;
|
|
12
|
+
before(async () => {
|
|
13
|
+
blake2b = await Blake2b.createHasher();
|
|
14
|
+
});
|
|
15
|
+
function createTempDir(suffix = "fjall-hybrid") {
|
|
16
|
+
return fs.mkdtempSync(`typeberry-${suffix}`);
|
|
17
|
+
}
|
|
18
|
+
describe("Fjall hybrid serialized states", () => {
|
|
19
|
+
const spec = tinyChainSpec;
|
|
20
|
+
const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
21
|
+
let dbPath = "";
|
|
22
|
+
beforeEach(() => {
|
|
23
|
+
dbPath = createTempDir();
|
|
24
|
+
});
|
|
25
|
+
afterEach(() => {
|
|
26
|
+
fs.rmSync(dbPath, { recursive: true });
|
|
27
|
+
});
|
|
28
|
+
it("round-trips an initial state through the on-disk values store", async () => {
|
|
29
|
+
const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
|
|
30
|
+
try {
|
|
31
|
+
const empty = InMemoryState.empty(spec);
|
|
32
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
|
|
33
|
+
const expectedRoot = serialized.getRootHash(blake2b);
|
|
34
|
+
const res = await states.insertInitialState(headerHash, serialized);
|
|
35
|
+
deepEqual(res, Result.ok(OK));
|
|
36
|
+
const state = states.getState(headerHash);
|
|
37
|
+
assert.ok(state !== null);
|
|
38
|
+
const stateRoot = await states.getStateRoot(state);
|
|
39
|
+
assert.strictEqual(`${stateRoot}`, `${expectedRoot}`);
|
|
40
|
+
deepEqual(InMemoryState.copyFrom(spec, state, new Map()), empty);
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await states.close();
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
it("reads large values back from disk", async () => {
|
|
47
|
+
const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
|
|
48
|
+
try {
|
|
49
|
+
// > 32 bytes => stored in the values db (not embedded in the leaf).
|
|
50
|
+
const big1 = BytesBlob.blobFromString("x".repeat(100));
|
|
51
|
+
const big2 = BytesBlob.blobFromString("y".repeat(100));
|
|
52
|
+
const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
|
|
53
|
+
const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
|
|
54
|
+
const entries = StateEntries.fromEntriesUnsafe([
|
|
55
|
+
[key1, big1],
|
|
56
|
+
[key2, big2],
|
|
57
|
+
]);
|
|
58
|
+
const res = await states.insertInitialState(headerHash, entries);
|
|
59
|
+
deepEqual(res, Result.ok(OK));
|
|
60
|
+
const state = states.getState(headerHash);
|
|
61
|
+
assert.ok(state !== null);
|
|
62
|
+
assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
|
|
63
|
+
assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
await states.close();
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
it("shares an open values session across resets without closing it", async () => {
|
|
70
|
+
// The fuzz reset path opens the values keyspace once per session and reuses
|
|
71
|
+
// it: each "reset" builds a fresh states instance sharing that session, and
|
|
72
|
+
// closing a session-backed states must NOT close the shared keyspace.
|
|
73
|
+
const session = await FjallValuesSession.open(dbPath);
|
|
74
|
+
try {
|
|
75
|
+
const big = BytesBlob.blobFromString("z".repeat(100));
|
|
76
|
+
const key = Bytes.fill(HASH_SIZE, 7).asOpaque();
|
|
77
|
+
const entries = StateEntries.fromEntriesUnsafe([[key, big]]);
|
|
78
|
+
// First "reset": write values through a states instance, then close it.
|
|
79
|
+
const first = HybridSerializedStates.fromSession(spec, blake2b, session);
|
|
80
|
+
const res = await first.insertInitialState(headerHash, entries);
|
|
81
|
+
deepEqual(res, Result.ok(OK));
|
|
82
|
+
await first.close();
|
|
83
|
+
// Second "reset": a fresh states sharing the same session. Its in-memory
|
|
84
|
+
// leaf set is independent (empty until it inserts)...
|
|
85
|
+
const second = HybridSerializedStates.fromSession(spec, blake2b, session);
|
|
86
|
+
assert.strictEqual(second.getState(headerHash), null);
|
|
87
|
+
// ...but the on-disk values store is the same one, still open and usable
|
|
88
|
+
// (a closed keyspace would throw here).
|
|
89
|
+
await second.insertInitialState(headerHash, entries);
|
|
90
|
+
const state = second.getState(headerHash);
|
|
91
|
+
assert.ok(state !== null);
|
|
92
|
+
assert.strictEqual(`${state.backend.get(key)}`, `${big}`);
|
|
93
|
+
await second.close();
|
|
94
|
+
}
|
|
95
|
+
finally {
|
|
96
|
+
await session.close();
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
it("drops the leaf set on markUnused while values stay on disk", async () => {
|
|
100
|
+
const states = await HybridSerializedStates.new({ spec, blake2b, dbPath });
|
|
101
|
+
try {
|
|
102
|
+
const empty = InMemoryState.empty(spec);
|
|
103
|
+
const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
|
|
104
|
+
await states.insertInitialState(headerHash, serialized);
|
|
105
|
+
assert.ok(states.getState(headerHash) !== null);
|
|
106
|
+
states.markUnused(headerHash);
|
|
107
|
+
assert.strictEqual(states.getState(headerHash), null);
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
await states.close();
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { type Partition } from "@fjall-js/fjall";
|
|
2
|
+
export type { Partition };
|
|
3
|
+
/**
|
|
4
|
+
* Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
|
|
5
|
+
*/
|
|
6
|
+
export declare function toUint8Array(value: Buffer | null): Uint8Array | null;
|
|
7
|
+
export type FjallRootOptions = {
|
|
8
|
+
/**
|
|
9
|
+
* When set, we skip the durability flush (`persist`) and `close()` does not
|
|
10
|
+
* do the sync-all fsync.
|
|
11
|
+
*
|
|
12
|
+
* Only safe for throwaway databases, like the fuzz target that wipes on every
|
|
13
|
+
* reset.
|
|
14
|
+
*/
|
|
15
|
+
ephemeral?: boolean;
|
|
16
|
+
/**
|
|
17
|
+
* Cache size in bytes, shared by all partitions of the keyspace. fjall reads
|
|
18
|
+
* through this cache, so it bounds how much we keep in memory. When not set,
|
|
19
|
+
* fjall uses its own default.
|
|
20
|
+
*/
|
|
21
|
+
cacheSizeBytes?: number;
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Thin wrapper over the fjall keyspace.
|
|
25
|
+
*
|
|
26
|
+
* fjall is an LSM-tree: it reads and writes through normal file i/o and keeps
|
|
27
|
+
* only a bounded block cache in memory, so the resident set stays bounded even
|
|
28
|
+
* when the store on disk is big.
|
|
29
|
+
*/
|
|
30
|
+
export declare class FjallRoot {
|
|
31
|
+
private readonly keyspace;
|
|
32
|
+
/** Path of the underlying keyspace directory, used to report on-disk usage. */
|
|
33
|
+
private readonly dbPath;
|
|
34
|
+
private readonly options;
|
|
35
|
+
private constructor();
|
|
36
|
+
/** Open (or create) a fjall keyspace at the given path. */
|
|
37
|
+
static open(dbPath: string, options: FjallRootOptions): Promise<FjallRoot>;
|
|
38
|
+
/** Open (or create) a partition under this keyspace. */
|
|
39
|
+
partition(name: string): Promise<Partition>;
|
|
40
|
+
/**
|
|
41
|
+
* Flush the journal to disk so prior writes survive a crash.
|
|
42
|
+
*
|
|
43
|
+
* Call after a logically-complete unit of work (one block, one state commit).
|
|
44
|
+
* A no-op for ephemeral databases.
|
|
45
|
+
*/
|
|
46
|
+
persist(): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Size of the keyspace directory on disk, in bytes.
|
|
49
|
+
*
|
|
50
|
+
* Returns `null` when the directory cannot be walked (e.g. not created yet).
|
|
51
|
+
* A fjall keyspace is a directory of partition and journal files, so we sum
|
|
52
|
+
* them recursively.
|
|
53
|
+
*/
|
|
54
|
+
sizeInBytes(): number | null;
|
|
55
|
+
/** Persist with `sync-all` and release the keyspace handle. */
|
|
56
|
+
close(): Promise<void>;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=root.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-fjall/root.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEtE,YAAY,EAAE,SAAS,EAAE,CAAC;AAE1B;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,CAKpE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,SAAS;IAElB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO;IAOP,2DAA2D;WAC9C,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC;IAUhF,wDAAwD;IAClD,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAIjD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;;;;;OAMG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,+DAA+D;IACzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { open } from "@fjall-js/fjall";
|
|
4
|
+
/**
|
|
5
|
+
* Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
|
|
6
|
+
*/
|
|
7
|
+
export function toUint8Array(value) {
|
|
8
|
+
if (value === null) {
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Thin wrapper over the fjall keyspace.
|
|
15
|
+
*
|
|
16
|
+
* fjall is an LSM-tree: it reads and writes through normal file i/o and keeps
|
|
17
|
+
* only a bounded block cache in memory, so the resident set stays bounded even
|
|
18
|
+
* when the store on disk is big.
|
|
19
|
+
*/
|
|
20
|
+
export class FjallRoot {
|
|
21
|
+
keyspace;
|
|
22
|
+
dbPath;
|
|
23
|
+
options;
|
|
24
|
+
constructor(keyspace,
|
|
25
|
+
/** Path of the underlying keyspace directory, used to report on-disk usage. */
|
|
26
|
+
dbPath, options) {
|
|
27
|
+
this.keyspace = keyspace;
|
|
28
|
+
this.dbPath = dbPath;
|
|
29
|
+
this.options = options;
|
|
30
|
+
}
|
|
31
|
+
/** Open (or create) a fjall keyspace at the given path. */
|
|
32
|
+
static async open(dbPath, options) {
|
|
33
|
+
// Forward our options to the binding: `ephemeral` makes `close()` skip the
|
|
34
|
+
// sync-all fsync, `cacheSizeBytes` bounds how much we keep in memory.
|
|
35
|
+
const keyspace = await open(dbPath, {
|
|
36
|
+
ephemeral: options.ephemeral,
|
|
37
|
+
cacheSizeBytes: options.cacheSizeBytes,
|
|
38
|
+
});
|
|
39
|
+
return new FjallRoot(keyspace, dbPath, options);
|
|
40
|
+
}
|
|
41
|
+
/** Open (or create) a partition under this keyspace. */
|
|
42
|
+
async partition(name) {
|
|
43
|
+
return this.keyspace.partition(name);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Flush the journal to disk so prior writes survive a crash.
|
|
47
|
+
*
|
|
48
|
+
* Call after a logically-complete unit of work (one block, one state commit).
|
|
49
|
+
* A no-op for ephemeral databases.
|
|
50
|
+
*/
|
|
51
|
+
async persist() {
|
|
52
|
+
if (this.options.ephemeral === true) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
await this.keyspace.persist();
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Size of the keyspace directory on disk, in bytes.
|
|
59
|
+
*
|
|
60
|
+
* Returns `null` when the directory cannot be walked (e.g. not created yet).
|
|
61
|
+
* A fjall keyspace is a directory of partition and journal files, so we sum
|
|
62
|
+
* them recursively.
|
|
63
|
+
*/
|
|
64
|
+
sizeInBytes() {
|
|
65
|
+
try {
|
|
66
|
+
return dirSizeInBytes(this.dbPath);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/** Persist with `sync-all` and release the keyspace handle. */
|
|
73
|
+
async close() {
|
|
74
|
+
await this.keyspace.close();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function dirSizeInBytes(dir) {
|
|
78
|
+
let total = 0;
|
|
79
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
80
|
+
const full = path.join(dir, entry.name);
|
|
81
|
+
if (entry.isDirectory()) {
|
|
82
|
+
total += dirSizeInBytes(full);
|
|
83
|
+
}
|
|
84
|
+
else if (entry.isFile()) {
|
|
85
|
+
total += fs.statSync(full).size;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return total;
|
|
89
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Epoch } from "#@typeberry/block";
|
|
2
2
|
import type { SignedTicket } from "#@typeberry/block/tickets.js";
|
|
3
3
|
import type { ChainSpec } from "#@typeberry/config";
|
|
4
|
+
import { type TicketValidator } from "#@typeberry/ticket-pool";
|
|
4
5
|
import type { Connections } from "../peers.js";
|
|
5
6
|
import type { StreamManager } from "../stream-manager.js";
|
|
6
7
|
/**
|
|
@@ -9,33 +10,40 @@ import type { StreamManager } from "../stream-manager.js";
|
|
|
9
10
|
* Uses CE-132 (proxy-to-all) for direct broadcast to all peers.
|
|
10
11
|
* Implements a maintain pattern similar to SyncTask: tickets are collected
|
|
11
12
|
* and periodically distributed to peers that haven't received them yet.
|
|
13
|
+
*
|
|
14
|
+
* Incoming tickets from peers are first run through a {@link TicketValidator};
|
|
15
|
+
* only validated tickets are added to the redistribution pool. The default
|
|
16
|
+
* validator denies everything, so callers must wire a real one via
|
|
17
|
+
* {@link setTicketValidator} before any networked ticket can be redistributed.
|
|
12
18
|
*/
|
|
13
19
|
export declare class TicketDistributionTask {
|
|
14
20
|
private readonly streamManager;
|
|
15
21
|
private readonly connections;
|
|
16
22
|
static start(streamManager: StreamManager, connections: Connections, chainSpec: ChainSpec): TicketDistributionTask;
|
|
17
|
-
|
|
18
|
-
private
|
|
19
|
-
/** Current epoch being tracked (cleared when epoch changes) */
|
|
20
|
-
private currentEpoch;
|
|
23
|
+
private readonly pool;
|
|
24
|
+
private validator;
|
|
21
25
|
private constructor();
|
|
22
26
|
/**
|
|
23
27
|
* Should be called periodically to distribute pending tickets to connected peers.
|
|
24
28
|
*/
|
|
25
29
|
maintainDistribution(): void;
|
|
26
30
|
/**
|
|
27
|
-
* Add a ticket to the
|
|
31
|
+
* Add a ticket to the redistribution pool.
|
|
28
32
|
* Clears pending tickets when epoch changes.
|
|
29
33
|
* Deduplicates tickets based on signature.
|
|
30
34
|
*/
|
|
31
35
|
addTicket(epochIndex: Epoch, ticket: SignedTicket): void;
|
|
32
|
-
private onTicketReceivedCallback;
|
|
33
36
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
|
|
37
|
+
* Replace the redistribution pool for the given epoch with the supplied tickets.
|
|
38
|
+
* Used when the authorship worker dumps the authoritative pool on an epoch boundary.
|
|
39
|
+
*/
|
|
40
|
+
replacePool(epochIndex: Epoch, tickets: readonly SignedTicket[]): void;
|
|
41
|
+
/**
|
|
42
|
+
* Register the validator that decides whether tickets received from peers should be
|
|
43
|
+
* accepted (and therefore redistributed). The default is {@link DenyTicketsValidator},
|
|
44
|
+
* so the caller must install a real validator for any peer ticket to make it through.
|
|
37
45
|
*/
|
|
38
|
-
|
|
46
|
+
setTicketValidator(validator: TicketValidator): void;
|
|
39
47
|
private onTicketReceived;
|
|
40
48
|
}
|
|
41
49
|
//# sourceMappingURL=ticket-distribution.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAA2C,KAAK,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEvG,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAqB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IArB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA2B;IAChD,OAAO,CAAC,SAAS,CAA+C;IAEhE,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAgDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAIjD;;;OAGG;IACH,WAAW,CAAC,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,YAAY,EAAE;IAI/D;;;;OAIG;IACH,kBAAkB,CAAC,SAAS,EAAE,eAAe;IAI7C,OAAO,CAAC,gBAAgB;CAkBzB"}
|