@typeberry/lib 0.8.3 → 0.8.4-70b1490
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -4
- package/packages/configs/index.d.ts +30 -1
- package/packages/configs/index.d.ts.map +1 -1
- package/packages/configs/index.js +4 -2
- package/packages/configs/typeberry-dev-full.json +29 -0
- package/packages/core/bytes/bytes.d.ts +1 -0
- package/packages/core/bytes/bytes.d.ts.map +1 -1
- package/packages/core/bytes/bytes.js +8 -0
- package/packages/core/utils/debug.d.ts +16 -4
- package/packages/core/utils/debug.d.ts.map +1 -1
- package/packages/core/utils/debug.js +39 -17
- package/packages/core/utils/debug.test.js +12 -6
- package/packages/jam/config-node/node-config.d.ts +2 -1
- package/packages/jam/config-node/node-config.d.ts.map +1 -1
- package/packages/jam/config-node/node-config.js +8 -3
- package/packages/jam/config-node/node-config.test.js +3 -3
- package/packages/jam/database/states.d.ts +7 -0
- package/packages/jam/database/states.d.ts.map +1 -1
- package/packages/jam/database-fjall/hybrid-states.d.ts +45 -0
- package/packages/jam/database-fjall/hybrid-states.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.js +113 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts +2 -0
- package/packages/jam/database-fjall/hybrid-states.test.d.ts.map +1 -0
- package/packages/jam/database-fjall/hybrid-states.test.js +83 -0
- package/packages/jam/database-fjall/index.d.ts +3 -0
- package/packages/jam/database-fjall/index.d.ts.map +1 -0
- package/packages/jam/database-fjall/index.js +2 -0
- package/packages/jam/database-fjall/root.d.ts +52 -0
- package/packages/jam/database-fjall/root.d.ts.map +1 -0
- package/packages/jam/database-fjall/root.js +85 -0
- package/packages/jam/database-lmdb/hybrid-states.d.ts +3 -1
- package/packages/jam/database-lmdb/hybrid-states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/hybrid-states.js +5 -2
- package/packages/jam/database-lmdb/root.d.ts +14 -1
- package/packages/jam/database-lmdb/root.d.ts.map +1 -1
- package/packages/jam/database-lmdb/root.js +25 -5
- package/packages/jam/database-lmdb/states.d.ts +1 -0
- package/packages/jam/database-lmdb/states.d.ts.map +1 -1
- package/packages/jam/database-lmdb/states.js +3 -0
- package/packages/jam/database-lmdb/states.test.js +4 -4
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts +18 -10
- package/packages/jam/jamnp-s/tasks/ticket-distribution.d.ts.map +1 -1
- package/packages/jam/jamnp-s/tasks/ticket-distribution.js +44 -68
- package/packages/jam/jamnp-s/tasks/ticket-distribution.test.js +30 -8
- package/packages/jam/node/main-fuzz.d.ts.map +1 -1
- package/packages/jam/node/main-fuzz.js +21 -4
- package/packages/jam/node/main-importer.d.ts +7 -4
- package/packages/jam/node/main-importer.d.ts.map +1 -1
- package/packages/jam/node/main-importer.js +10 -4
- package/packages/jam/safrole/bandersnatch-vrf.d.ts +24 -4
- package/packages/jam/safrole/bandersnatch-vrf.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-vrf.js +92 -40
- package/packages/jam/safrole/bandersnatch-vrf.test.js +12 -9
- package/packages/jam/safrole/bandersnatch-wasm.d.ts +10 -0
- package/packages/jam/safrole/bandersnatch-wasm.d.ts.map +1 -1
- package/packages/jam/safrole/bandersnatch-wasm.js +12 -0
- package/packages/jam/safrole/safrole.js +5 -5
- package/packages/jam/safrole/safrole.test.js +13 -13
- package/packages/jam/ticket-pool/index.d.ts +4 -0
- package/packages/jam/ticket-pool/index.d.ts.map +1 -0
- package/packages/jam/ticket-pool/index.js +3 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts +30 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.js +56 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/pending-ticket-pool.test.js +67 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts +47 -0
- package/packages/jam/ticket-pool/ticket-validator.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.js +34 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts +2 -0
- package/packages/jam/ticket-pool/ticket-validator.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/ticket-validator.test.js +35 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts +26 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.js +41 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts +2 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.d.ts.map +1 -0
- package/packages/jam/ticket-pool/verified-ticket-pool.test.js +54 -0
- package/packages/jam/transition/chain-stf.d.ts +2 -1
- package/packages/jam/transition/chain-stf.d.ts.map +1 -1
- package/packages/jam/transition/chain-stf.js +15 -3
- package/packages/workers/api-node/config.d.ts +14 -5
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +29 -20
- package/packages/workers/api-node/config.test.js +38 -1
- package/packages/workers/block-authorship/{generator.d.ts → block-generator.d.ts} +5 -5
- package/packages/workers/block-authorship/block-generator.d.ts.map +1 -0
- package/packages/workers/block-authorship/{generator.js → block-generator.js} +3 -3
- package/packages/workers/block-authorship/block-generator.test.d.ts +2 -0
- package/packages/workers/block-authorship/block-generator.test.d.ts.map +1 -0
- package/packages/workers/block-authorship/{generator.test.js → block-generator.test.js} +8 -8
- package/packages/workers/block-authorship/epoch-authoring-slots.d.ts +35 -0
- package/packages/workers/block-authorship/epoch-authoring-slots.d.ts.map +1 -0
- package/packages/workers/block-authorship/epoch-authoring-slots.js +86 -0
- package/packages/workers/block-authorship/epoch-tracker.d.ts +29 -0
- package/packages/workers/block-authorship/epoch-tracker.d.ts.map +1 -0
- package/packages/workers/block-authorship/epoch-tracker.js +80 -0
- package/packages/workers/block-authorship/index.d.ts.map +1 -1
- package/packages/workers/block-authorship/index.js +1 -1
- package/packages/workers/block-authorship/main.d.ts +3 -0
- package/packages/workers/block-authorship/main.d.ts.map +1 -1
- package/packages/workers/block-authorship/main.js +197 -315
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts +2 -0
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/bootstrap-main.js +23 -0
- package/packages/workers/block-authorship/ticket-generator/index.d.ts +16 -0
- package/packages/workers/block-authorship/ticket-generator/index.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/index.js +62 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.d.ts +50 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/protocol.js +54 -0
- package/packages/workers/block-authorship/{ticket-generator.d.ts → ticket-generator/ticket-generator.d.ts} +4 -0
- package/packages/workers/block-authorship/ticket-generator/ticket-generator.d.ts.map +1 -0
- package/packages/workers/block-authorship/{ticket-generator.js → ticket-generator/ticket-generator.js} +19 -9
- package/packages/workers/block-authorship/ticket-generator/ticket-generator.test.d.ts.map +1 -0
- package/packages/workers/block-authorship/{ticket-generator.test.js → ticket-generator/ticket-generator.test.js} +13 -9
- package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts +36 -0
- package/packages/workers/block-authorship/ticket-generator/worker-pool.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-generator/worker-pool.js +111 -0
- package/packages/workers/block-authorship/ticket-validator.d.ts +31 -0
- package/packages/workers/block-authorship/ticket-validator.d.ts.map +1 -0
- package/packages/workers/block-authorship/ticket-validator.js +59 -0
- package/packages/workers/comms-authorship-network/protocol.d.ts +14 -4
- package/packages/workers/comms-authorship-network/protocol.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/protocol.js +12 -6
- package/packages/workers/comms-authorship-network/tickets-message.d.ts +0 -14
- package/packages/workers/comms-authorship-network/tickets-message.d.ts.map +1 -1
- package/packages/workers/comms-authorship-network/tickets-message.js +0 -17
- package/packages/workers/importer/importer.d.ts +5 -3
- package/packages/workers/importer/importer.d.ts.map +1 -1
- package/packages/workers/importer/importer.js +43 -35
- package/packages/workers/importer/stats.d.ts +36 -0
- package/packages/workers/importer/stats.d.ts.map +1 -0
- package/packages/workers/importer/stats.js +69 -0
- package/packages/workers/jam-network/main.d.ts.map +1 -1
- package/packages/workers/jam-network/main.js +25 -4
- package/packages/workers/block-authorship/generator.d.ts.map +0 -1
- package/packages/workers/block-authorship/generator.test.d.ts +0 -2
- package/packages/workers/block-authorship/generator.test.d.ts.map +0 -1
- package/packages/workers/block-authorship/ticket-generator.d.ts.map +0 -1
- package/packages/workers/block-authorship/ticket-generator.test.d.ts.map +0 -1
- /package/packages/configs/{typeberry-dev.json → typeberry-dev-tiny.json} +0 -0
- /package/packages/workers/block-authorship/{ticket-generator.test.d.ts → ticket-generator/ticket-generator.test.d.ts} +0 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { open } from "@fjall-js/fjall";
|
|
4
|
+
/**
|
|
5
|
+
* Normalize a value read from fjall (a Node `Buffer`) into a plain `Uint8Array`.
|
|
6
|
+
*/
|
|
7
|
+
export function toUint8Array(value) {
|
|
8
|
+
if (value === null) {
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A thin abstraction over the fjall keyspace.
|
|
15
|
+
*
|
|
16
|
+
* Unlike LMDB (a memory-mapped B-tree), fjall is an LSM-tree that reads/writes
|
|
17
|
+
* through regular file I/O, so its working set is bounded by an explicit block
|
|
18
|
+
* cache rather than the whole mmap. LMDB has been causing oom issues because
|
|
19
|
+
* of that.
|
|
20
|
+
*/
|
|
21
|
+
export class FjallRoot {
|
|
22
|
+
keyspace;
|
|
23
|
+
dbPath;
|
|
24
|
+
options;
|
|
25
|
+
constructor(keyspace,
|
|
26
|
+
/** Path of the underlying keyspace directory, used to report on-disk usage. */
|
|
27
|
+
dbPath, options) {
|
|
28
|
+
this.keyspace = keyspace;
|
|
29
|
+
this.dbPath = dbPath;
|
|
30
|
+
this.options = options;
|
|
31
|
+
}
|
|
32
|
+
/** Open (or create) a fjall keyspace at the given path. */
|
|
33
|
+
static async open(dbPath, options) {
|
|
34
|
+
const keyspace = await open(dbPath);
|
|
35
|
+
return new FjallRoot(keyspace, dbPath, options);
|
|
36
|
+
}
|
|
37
|
+
/** Open (or create) a partition under this keyspace. */
|
|
38
|
+
async partition(name) {
|
|
39
|
+
return this.keyspace.partition(name);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Flush the journal to disk so prior writes survive a crash.
|
|
43
|
+
*
|
|
44
|
+
* Call after a logically-complete unit of work (one block, one state commit).
|
|
45
|
+
* A no-op for ephemeral databases.
|
|
46
|
+
*/
|
|
47
|
+
async persist() {
|
|
48
|
+
if (this.options.ephemeral === true) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
await this.keyspace.persist();
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Apparent on-disk size of the keyspace directory, in bytes.
|
|
55
|
+
*
|
|
56
|
+
* Returns `null` if the directory cannot be walked (e.g. not yet created).
|
|
57
|
+
* Unlike LMDB's single `data.mdb`, a fjall keyspace is a directory of
|
|
58
|
+
* partition and journal files, so we sum it recursively.
|
|
59
|
+
*/
|
|
60
|
+
sizeInBytes() {
|
|
61
|
+
try {
|
|
62
|
+
return dirSizeInBytes(this.dbPath);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** Persist with `sync-all` and release the keyspace handle. */
|
|
69
|
+
async close() {
|
|
70
|
+
await this.keyspace.close();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function dirSizeInBytes(dir) {
|
|
74
|
+
let total = 0;
|
|
75
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
76
|
+
const full = path.join(dir, entry.name);
|
|
77
|
+
if (entry.isDirectory()) {
|
|
78
|
+
total += dirSizeInBytes(full);
|
|
79
|
+
}
|
|
80
|
+
else if (entry.isFile()) {
|
|
81
|
+
total += fs.statSync(full).size;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return total;
|
|
85
|
+
}
|
|
@@ -20,12 +20,13 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
20
20
|
private readonly inMemStates;
|
|
21
21
|
private readonly lmdbValues;
|
|
22
22
|
private readonly valuesDb;
|
|
23
|
-
static new({ spec, blake2b, dbPath, readOnly, ephemeral, }: {
|
|
23
|
+
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
|
|
24
24
|
spec: ChainSpec;
|
|
25
25
|
blake2b: Blake2b;
|
|
26
26
|
dbPath: string;
|
|
27
27
|
readOnly?: boolean;
|
|
28
28
|
ephemeral?: boolean;
|
|
29
|
+
compression?: boolean;
|
|
29
30
|
}): HybridSerializedStates;
|
|
30
31
|
private constructor();
|
|
31
32
|
insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
|
|
@@ -33,6 +34,7 @@ export declare class HybridSerializedStates implements StatesDb<SerializedState<
|
|
|
33
34
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
34
35
|
getState(header: HeaderHash): SerializedState<LeafDb> | null;
|
|
35
36
|
markUnused(header: HeaderHash): void;
|
|
37
|
+
diskSizeInBytes(): number | null;
|
|
36
38
|
close(): Promise<void>;
|
|
37
39
|
/** Write new large values to LMDB in one transaction. */
|
|
38
40
|
private writeValues;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA0BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA3BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAMpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAKX,yDAAyD;YAC3C,WAAW;IAgBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -20,8 +20,8 @@ export class HybridSerializedStates {
|
|
|
20
20
|
lmdbValues;
|
|
21
21
|
// A single shared values accessor reused by every `LeafDb` we hand out.
|
|
22
22
|
valuesDb;
|
|
23
|
-
static new({ spec, blake2b, dbPath, readOnly, ephemeral, }) {
|
|
24
|
-
const root = LmdbRoot.new(dbPath, readOnly, ephemeral);
|
|
23
|
+
static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
|
|
24
|
+
const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
|
|
25
25
|
return new HybridSerializedStates(spec, blake2b, root);
|
|
26
26
|
}
|
|
27
27
|
constructor(spec, blake2b, root) {
|
|
@@ -73,6 +73,9 @@ export class HybridSerializedStates {
|
|
|
73
73
|
// but since they are stored on disk we should be safe.
|
|
74
74
|
this.inMemStates.delete(header);
|
|
75
75
|
}
|
|
76
|
+
diskSizeInBytes() {
|
|
77
|
+
return this.root.sizeInBytes();
|
|
78
|
+
}
|
|
76
79
|
async close() {
|
|
77
80
|
await this.lmdbValues.close();
|
|
78
81
|
await this.root.close();
|
|
@@ -1,12 +1,25 @@
|
|
|
1
1
|
import * as lmdb from "lmdb";
|
|
2
2
|
export type SubDb = lmdb.Database<Uint8Array, lmdb.Key>;
|
|
3
|
+
export type LmdbRootOptions = {
|
|
4
|
+
readOnly?: boolean;
|
|
5
|
+
ephemeral?: boolean;
|
|
6
|
+
compression?: boolean;
|
|
7
|
+
};
|
|
3
8
|
/** A thin abstraction over lmdb database interface. */
|
|
4
9
|
export declare class LmdbRoot {
|
|
5
10
|
readonly db: lmdb.RootDatabase<Uint8Array, lmdb.Key>;
|
|
6
|
-
|
|
11
|
+
/** Path of the underlying LMDB data file, used to report on-disk usage. */
|
|
12
|
+
private readonly dataFilePath;
|
|
13
|
+
static new(dbPath: string, options: LmdbRootOptions): LmdbRoot;
|
|
7
14
|
private constructor();
|
|
8
15
|
/** Open a sub-database under the same path. */
|
|
9
16
|
subDb(name: string): SubDb;
|
|
17
|
+
/**
|
|
18
|
+
* Apparent on-disk size of the LMDB data file, in bytes.
|
|
19
|
+
*
|
|
20
|
+
* Returns `null` if the file cannot be `stat`-ed (e.g. not yet created).
|
|
21
|
+
*/
|
|
22
|
+
sizeInBytes(): number | null;
|
|
10
23
|
/** Close the database and all sub-databases. */
|
|
11
24
|
close(): Promise<void>;
|
|
12
25
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/root.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"root.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/root.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;AAExD,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,uDAAuD;AACvD,qBAAa,QAAQ;IACnB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACrD,2EAA2E;IAC3E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe;IAInD,OAAO;IAwBP,+CAA+C;IAC/C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK;IAI1B;;;;OAIG;IACH,WAAW,IAAI,MAAM,GAAG,IAAI;IAQ5B,gDAAgD;IAC1C,KAAK;CAGZ"}
|
|
@@ -1,11 +1,18 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
1
2
|
import * as lmdb from "lmdb";
|
|
2
3
|
/** A thin abstraction over lmdb database interface. */
|
|
3
4
|
export class LmdbRoot {
|
|
4
5
|
db;
|
|
5
|
-
|
|
6
|
-
|
|
6
|
+
/** Path of the underlying LMDB data file, used to report on-disk usage. */
|
|
7
|
+
dataFilePath;
|
|
8
|
+
static new(dbPath, options) {
|
|
9
|
+
return new LmdbRoot(dbPath, options);
|
|
7
10
|
}
|
|
8
|
-
constructor(dbPath, readOnly
|
|
11
|
+
constructor(dbPath, { readOnly, ephemeral, compression }) {
|
|
12
|
+
// `lmdb.open` treats an extension-less path as a directory and stores the
|
|
13
|
+
// environment in `<dbPath>/data.mdb` (next to `lock.mdb`).
|
|
14
|
+
this.dataFilePath = `${dbPath}/data.mdb`;
|
|
15
|
+
const isEphemeral = ephemeral ?? false;
|
|
9
16
|
this.db = lmdb.open(dbPath, {
|
|
10
17
|
// experimental options
|
|
11
18
|
noMemInit: true,
|
|
@@ -15,8 +22,8 @@ export class LmdbRoot {
|
|
|
15
22
|
// durability is pointless, so we skip fsync and skip compressing the large
|
|
16
23
|
// per-block leaf blobs. Both are pure overhead there and dominate the cost.
|
|
17
24
|
// This trades disk space (uncompressed) and crash-durability for speed.
|
|
18
|
-
compression: !
|
|
19
|
-
noSync:
|
|
25
|
+
compression: compression ?? !isEphemeral,
|
|
26
|
+
noSync: isEphemeral,
|
|
20
27
|
keyEncoding: "binary",
|
|
21
28
|
encoding: "binary",
|
|
22
29
|
readOnly,
|
|
@@ -26,6 +33,19 @@ export class LmdbRoot {
|
|
|
26
33
|
subDb(name) {
|
|
27
34
|
return this.db.openDB({ name });
|
|
28
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Apparent on-disk size of the LMDB data file, in bytes.
|
|
38
|
+
*
|
|
39
|
+
* Returns `null` if the file cannot be `stat`-ed (e.g. not yet created).
|
|
40
|
+
*/
|
|
41
|
+
sizeInBytes() {
|
|
42
|
+
try {
|
|
43
|
+
return fs.statSync(this.dataFilePath).size;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
29
49
|
/** Close the database and all sub-databases. */
|
|
30
50
|
async close() {
|
|
31
51
|
await this.db.close();
|
|
@@ -69,6 +69,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
|
|
|
69
69
|
getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
|
|
70
70
|
getState(root: HeaderHash): SerializedState<LeafDb> | null;
|
|
71
71
|
markUnused(header: HeaderHash): void;
|
|
72
|
+
diskSizeInBytes(): number | null;
|
|
72
73
|
close(): Promise<void>;
|
|
73
74
|
}
|
|
74
75
|
//# sourceMappingURL=states.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
|
|
@@ -131,6 +131,9 @@ export class LmdbStates {
|
|
|
131
131
|
markUnused(header) {
|
|
132
132
|
this.states.removeSync(header.raw);
|
|
133
133
|
}
|
|
134
|
+
diskSizeInBytes() {
|
|
135
|
+
return this.root.sizeInBytes();
|
|
136
|
+
}
|
|
134
137
|
async close() {
|
|
135
138
|
await Promise.all([this.states.close(), this.values.close()]);
|
|
136
139
|
}
|
|
@@ -37,7 +37,7 @@ describe("LMDB States database", () => {
|
|
|
37
37
|
const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
|
|
38
38
|
const spec = tinyChainSpec;
|
|
39
39
|
it("should import state and read state", async () => {
|
|
40
|
-
const root = LmdbRoot.new(tmpDir);
|
|
40
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
41
41
|
const states = LmdbStates.new(spec, blake2b, root);
|
|
42
42
|
try {
|
|
43
43
|
const emptyState = InMemoryState.empty(spec);
|
|
@@ -58,7 +58,7 @@ describe("LMDB States database", () => {
|
|
|
58
58
|
}
|
|
59
59
|
});
|
|
60
60
|
it("should update the state", async () => {
|
|
61
|
-
const root = LmdbRoot.new(tmpDir);
|
|
61
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
62
62
|
const states = LmdbStates.new(spec, blake2b, root);
|
|
63
63
|
try {
|
|
64
64
|
const state = InMemoryState.empty(spec);
|
|
@@ -142,7 +142,7 @@ describe("LMDB States database", () => {
|
|
|
142
142
|
deepEqual(Array.from(set), Array.from(SortedSet.fromArray(leafComparator, Array.from(trie.nodes.leaves()))));
|
|
143
143
|
});
|
|
144
144
|
it("should import more complex state", async () => {
|
|
145
|
-
const root = LmdbRoot.new(tmpDir);
|
|
145
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
146
146
|
const states = LmdbStates.new(spec, blake2b, root);
|
|
147
147
|
try {
|
|
148
148
|
const initialState = testState();
|
|
@@ -167,7 +167,7 @@ describe("LMDB States database", () => {
|
|
|
167
167
|
}
|
|
168
168
|
});
|
|
169
169
|
it("should update more complex entries", async () => {
|
|
170
|
-
const root = LmdbRoot.new(tmpDir);
|
|
170
|
+
const root = LmdbRoot.new(tmpDir, {});
|
|
171
171
|
const states = LmdbStates.new(spec, blake2b, root);
|
|
172
172
|
try {
|
|
173
173
|
const state = testState();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Epoch } from "#@typeberry/block";
|
|
2
2
|
import type { SignedTicket } from "#@typeberry/block/tickets.js";
|
|
3
3
|
import type { ChainSpec } from "#@typeberry/config";
|
|
4
|
+
import { type TicketValidator } from "#@typeberry/ticket-pool";
|
|
4
5
|
import type { Connections } from "../peers.js";
|
|
5
6
|
import type { StreamManager } from "../stream-manager.js";
|
|
6
7
|
/**
|
|
@@ -9,33 +10,40 @@ import type { StreamManager } from "../stream-manager.js";
|
|
|
9
10
|
* Uses CE-132 (proxy-to-all) for direct broadcast to all peers.
|
|
10
11
|
* Implements a maintain pattern similar to SyncTask: tickets are collected
|
|
11
12
|
* and periodically distributed to peers that haven't received them yet.
|
|
13
|
+
*
|
|
14
|
+
* Incoming tickets from peers are first run through a {@link TicketValidator};
|
|
15
|
+
* only validated tickets are added to the redistribution pool. The default
|
|
16
|
+
* validator denies everything, so callers must wire a real one via
|
|
17
|
+
* {@link setTicketValidator} before any networked ticket can be redistributed.
|
|
12
18
|
*/
|
|
13
19
|
export declare class TicketDistributionTask {
|
|
14
20
|
private readonly streamManager;
|
|
15
21
|
private readonly connections;
|
|
16
22
|
static start(streamManager: StreamManager, connections: Connections, chainSpec: ChainSpec): TicketDistributionTask;
|
|
17
|
-
|
|
18
|
-
private
|
|
19
|
-
/** Current epoch being tracked (cleared when epoch changes) */
|
|
20
|
-
private currentEpoch;
|
|
23
|
+
private readonly pool;
|
|
24
|
+
private validator;
|
|
21
25
|
private constructor();
|
|
22
26
|
/**
|
|
23
27
|
* Should be called periodically to distribute pending tickets to connected peers.
|
|
24
28
|
*/
|
|
25
29
|
maintainDistribution(): void;
|
|
26
30
|
/**
|
|
27
|
-
* Add a ticket to the
|
|
31
|
+
* Add a ticket to the redistribution pool.
|
|
28
32
|
* Clears pending tickets when epoch changes.
|
|
29
33
|
* Deduplicates tickets based on signature.
|
|
30
34
|
*/
|
|
31
35
|
addTicket(epochIndex: Epoch, ticket: SignedTicket): void;
|
|
32
|
-
private onTicketReceivedCallback;
|
|
33
36
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
|
|
37
|
+
* Replace the redistribution pool for the given epoch with the supplied tickets.
|
|
38
|
+
* Used when the authorship worker dumps the authoritative pool on an epoch boundary.
|
|
39
|
+
*/
|
|
40
|
+
replacePool(epochIndex: Epoch, tickets: readonly SignedTicket[]): void;
|
|
41
|
+
/**
|
|
42
|
+
* Register the validator that decides whether tickets received from peers should be
|
|
43
|
+
* accepted (and therefore redistributed). The default is {@link DenyTicketsValidator},
|
|
44
|
+
* so the caller must install a real validator for any peer ticket to make it through.
|
|
37
45
|
*/
|
|
38
|
-
|
|
46
|
+
setTicketValidator(validator: TicketValidator): void;
|
|
39
47
|
private onTicketReceived;
|
|
40
48
|
}
|
|
41
49
|
//# sourceMappingURL=ticket-distribution.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"ticket-distribution.d.ts","sourceRoot":"","sources":["../../../../../../packages/jam/jamnp-s/tasks/ticket-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAA2C,KAAK,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEvG,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAe1D;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAqB/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IArB9B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS;IAgBzF,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA2B;IAChD,OAAO,CAAC,SAAS,CAA+C;IAEhE,OAAO;IAKP;;OAEG;IACH,oBAAoB;IAgDpB;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAIjD;;;OAGG;IACH,WAAW,CAAC,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,YAAY,EAAE;IAI/D;;;;OAIG;IACH,kBAAkB,CAAC,SAAS,EAAE,eAAe;IAI7C,OAAO,CAAC,gBAAgB;CAkBzB"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Logger } from "#@typeberry/logger";
|
|
2
|
+
import { DenyTicketsValidator, PendingTicketPool } from "#@typeberry/ticket-pool";
|
|
2
3
|
import { OK } from "#@typeberry/utils";
|
|
3
4
|
import { ce131 } from "../protocol/index.js";
|
|
4
5
|
const logger = Logger.new(import.meta.filename, "net:tickets");
|
|
@@ -12,6 +13,11 @@ const TICKET_AUX = {
|
|
|
12
13
|
* Uses CE-132 (proxy-to-all) for direct broadcast to all peers.
|
|
13
14
|
* Implements a maintain pattern similar to SyncTask: tickets are collected
|
|
14
15
|
* and periodically distributed to peers that haven't received them yet.
|
|
16
|
+
*
|
|
17
|
+
* Incoming tickets from peers are first run through a {@link TicketValidator};
|
|
18
|
+
* only validated tickets are added to the redistribution pool. The default
|
|
19
|
+
* validator denies everything, so callers must wire a real one via
|
|
20
|
+
* {@link setTicketValidator} before any networked ticket can be redistributed.
|
|
15
21
|
*/
|
|
16
22
|
export class TicketDistributionTask {
|
|
17
23
|
streamManager;
|
|
@@ -26,10 +32,8 @@ export class TicketDistributionTask {
|
|
|
26
32
|
streamManager.registerOutgoingHandlers(ce131.ClientHandler.new(chainSpec, ce131.STREAM_KIND_PROXY_TO_ALL));
|
|
27
33
|
return task;
|
|
28
34
|
}
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
/** Current epoch being tracked (cleared when epoch changes) */
|
|
32
|
-
currentEpoch = null;
|
|
35
|
+
pool = new PendingTicketPool();
|
|
36
|
+
validator = new DenyTicketsValidator();
|
|
33
37
|
constructor(streamManager, connections) {
|
|
34
38
|
this.streamManager = streamManager;
|
|
35
39
|
this.connections = connections;
|
|
@@ -38,14 +42,13 @@ export class TicketDistributionTask {
|
|
|
38
42
|
* Should be called periodically to distribute pending tickets to connected peers.
|
|
39
43
|
*/
|
|
40
44
|
maintainDistribution() {
|
|
41
|
-
|
|
42
|
-
|
|
45
|
+
const currentEpoch = this.pool.currentEpoch;
|
|
46
|
+
if (currentEpoch === null) {
|
|
47
|
+
return;
|
|
43
48
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
for (let ticketIdx = 0; ticketIdx < this.pendingTickets.length; ticketIdx++) {
|
|
48
|
-
const { epochIndex, ticket } = this.pendingTickets[ticketIdx];
|
|
49
|
+
const tickets = this.pool.getTickets();
|
|
50
|
+
for (let ticketIdx = 0; ticketIdx < tickets.length; ticketIdx++) {
|
|
51
|
+
const { epochIndex, ticket } = tickets[ticketIdx];
|
|
49
52
|
// Try to send to each connected peer
|
|
50
53
|
for (const peerInfo of this.connections.getConnectedPeers()) {
|
|
51
54
|
this.connections.withAuxData(peerInfo.peerId, TICKET_AUX, (maybeAux) => {
|
|
@@ -77,72 +80,45 @@ export class TicketDistributionTask {
|
|
|
77
80
|
}
|
|
78
81
|
}
|
|
79
82
|
/**
|
|
80
|
-
* Add a ticket to the
|
|
83
|
+
* Add a ticket to the redistribution pool.
|
|
81
84
|
* Clears pending tickets when epoch changes.
|
|
82
85
|
* Deduplicates tickets based on signature.
|
|
83
86
|
*/
|
|
84
87
|
addTicket(epochIndex, ticket) {
|
|
85
|
-
|
|
86
|
-
// after the epoch has already advanced — accepting it would roll back currentEpoch).
|
|
87
|
-
if (this.currentEpoch !== null && epochIndex < this.currentEpoch) {
|
|
88
|
-
return;
|
|
89
|
-
}
|
|
90
|
-
// Epoch advanced — clear old tickets
|
|
91
|
-
if (this.currentEpoch !== null && epochIndex > this.currentEpoch) {
|
|
92
|
-
logger.log `[addTicket] Epoch changed from ${this.currentEpoch} to ${epochIndex}, clearing ${this.pendingTickets.length} old tickets`;
|
|
93
|
-
this.pendingTickets = [];
|
|
94
|
-
// Note: We don't need to clear aux data for all peers here.
|
|
95
|
-
// The aux data contains the epoch, so maintainDistribution will lazily
|
|
96
|
-
// reset it when it detects an epoch mismatch. This handles both connected
|
|
97
|
-
// and disconnected peers correctly.
|
|
98
|
-
}
|
|
99
|
-
this.currentEpoch = epochIndex;
|
|
100
|
-
/**
|
|
101
|
-
* Deduplicate: check if a ticket with the same signature already exists
|
|
102
|
-
*
|
|
103
|
-
* Here we are risking "poisoning" the local pendingTickets - i.e:
|
|
104
|
-
* 1. The adversary sees a signature and swaps the ticket attempt to something different.
|
|
105
|
-
* 2. This creates an invalid ticket, but prevents a valid ticket with the same signature from being included and distributed.
|
|
106
|
-
*
|
|
107
|
-
* TODO [MaSi]: The poisoning risk should be fixed during implementation of ticket validation.
|
|
108
|
-
*/
|
|
109
|
-
const isDuplicate = this.pendingTickets.some((pending) => pending.epochIndex === epochIndex && pending.ticket.signature.isEqualTo(ticket.signature));
|
|
110
|
-
if (!isDuplicate) {
|
|
111
|
-
this.pendingTickets.push({ epochIndex, ticket });
|
|
112
|
-
logger.info `[addTicket] Added ticket for epoch ${epochIndex}, total: ${this.pendingTickets.length}`;
|
|
113
|
-
}
|
|
88
|
+
this.pool.addTicket(epochIndex, ticket);
|
|
114
89
|
}
|
|
115
|
-
onTicketReceivedCallback = null;
|
|
116
90
|
/**
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* This prevents redistribution of invalid tickets (e.g. those with a tampered `attempt` field).
|
|
91
|
+
* Replace the redistribution pool for the given epoch with the supplied tickets.
|
|
92
|
+
* Used when the authorship worker dumps the authoritative pool on an epoch boundary.
|
|
120
93
|
*/
|
|
121
|
-
|
|
122
|
-
this.
|
|
94
|
+
replacePool(epochIndex, tickets) {
|
|
95
|
+
this.pool.replace(epochIndex, tickets);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Register the validator that decides whether tickets received from peers should be
|
|
99
|
+
* accepted (and therefore redistributed). The default is {@link DenyTicketsValidator},
|
|
100
|
+
* so the caller must install a real validator for any peer ticket to make it through.
|
|
101
|
+
*/
|
|
102
|
+
setTicketValidator(validator) {
|
|
103
|
+
this.validator = validator;
|
|
123
104
|
}
|
|
124
105
|
onTicketReceived(epochIndex, ticket) {
|
|
125
106
|
logger.trace `Received ticket for epoch ${epochIndex}, attempt ${ticket.attempt}`;
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
});
|
|
143
|
-
}
|
|
144
|
-
else {
|
|
145
|
-
this.addTicket(epochIndex, ticket);
|
|
146
|
-
}
|
|
107
|
+
const validator = this.validator;
|
|
108
|
+
// Wrap with Promise.resolve().then() so a synchronous throw inside the validator
|
|
109
|
+
// funnels into the same .catch() as an async rejection.
|
|
110
|
+
Promise.resolve()
|
|
111
|
+
.then(() => validator.validate(epochIndex, [ticket]))
|
|
112
|
+
.then((result) => {
|
|
113
|
+
if (result.isOk) {
|
|
114
|
+
this.addTicket(epochIndex, ticket);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
logger.trace `Dropping ticket for epoch ${epochIndex}: ${result.error}`;
|
|
118
|
+
}
|
|
119
|
+
})
|
|
120
|
+
.catch((error) => {
|
|
121
|
+
logger.error `Error validating ticket for epoch ${epochIndex}, attempt ${ticket.attempt}: ${error}`;
|
|
122
|
+
});
|
|
147
123
|
}
|
|
148
124
|
}
|
|
@@ -8,7 +8,8 @@ import { tinyChainSpec } from "#@typeberry/config";
|
|
|
8
8
|
import { BANDERSNATCH_PROOF_BYTES } from "#@typeberry/crypto";
|
|
9
9
|
import { Logger } from "#@typeberry/logger";
|
|
10
10
|
import { createTestPeerPair, MockNetwork } from "#@typeberry/networking/testing.js";
|
|
11
|
-
import {
|
|
11
|
+
import { AcceptTicketsValidator, ValidationError } from "#@typeberry/ticket-pool";
|
|
12
|
+
import { OK, Result } from "#@typeberry/utils";
|
|
12
13
|
import { Connections } from "../peers.js";
|
|
13
14
|
import { StreamManager } from "../stream-manager.js";
|
|
14
15
|
import { TicketDistributionTask } from "./ticket-distribution.js";
|
|
@@ -34,6 +35,9 @@ describe("TicketDistributionTask", () => {
|
|
|
34
35
|
const receivedTickets = [];
|
|
35
36
|
// Use real TicketDistributionTask
|
|
36
37
|
const ticketTask = TicketDistributionTask.start(streamManager, connections, tinyChainSpec);
|
|
38
|
+
// Default validator accepts every ticket so the test asserts purely on distribution
|
|
39
|
+
// behaviour. Tests that exercise the rejection path overwrite this.
|
|
40
|
+
ticketTask.setTicketValidator(new AcceptTicketsValidator());
|
|
37
41
|
// Intercept received tickets by wrapping onTicketReceived behavior
|
|
38
42
|
// The task already adds received tickets to pending queue via addTicket,
|
|
39
43
|
// so we can track them by checking the pending queue growth or by
|
|
@@ -217,7 +221,7 @@ describe("TicketDistributionTask", () => {
|
|
|
217
221
|
assert.strictEqual(peer2.receivedTickets.length, 1);
|
|
218
222
|
assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
|
|
219
223
|
});
|
|
220
|
-
it("should NOT redistribute ticket if
|
|
224
|
+
it("should NOT redistribute ticket if validator rejects", async () => {
|
|
221
225
|
const self = await init("self");
|
|
222
226
|
const peer1 = await init("peer1");
|
|
223
227
|
const peer2 = await init("peer2");
|
|
@@ -225,35 +229,53 @@ describe("TicketDistributionTask", () => {
|
|
|
225
229
|
self.openConnection(peer2);
|
|
226
230
|
await tick();
|
|
227
231
|
// Validation always rejects
|
|
228
|
-
self.ticketTask.
|
|
232
|
+
self.ticketTask.setTicketValidator({
|
|
233
|
+
validate: async () => Result.error(ValidationError.InvalidProof, () => "rejected"),
|
|
234
|
+
});
|
|
229
235
|
const ticket = createTestTicket(0);
|
|
230
236
|
peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
|
|
231
237
|
peer1.ticketTask.maintainDistribution();
|
|
232
238
|
await tick();
|
|
233
|
-
// self.addTicket was NOT called (
|
|
239
|
+
// self.addTicket was NOT called (validator rejected), so nothing to redistribute
|
|
234
240
|
assert.strictEqual(self.receivedTickets.length, 0);
|
|
235
241
|
self.ticketTask.maintainDistribution();
|
|
236
242
|
await tick();
|
|
237
243
|
assert.strictEqual(peer2.receivedTickets.length, 0);
|
|
238
244
|
});
|
|
239
|
-
it("should redistribute ticket if
|
|
245
|
+
it("should redistribute ticket if validator accepts", async () => {
|
|
240
246
|
const self = await init("self");
|
|
241
247
|
const peer1 = await init("peer1");
|
|
242
248
|
const peer2 = await init("peer2");
|
|
243
249
|
self.openConnection(peer1);
|
|
244
250
|
self.openConnection(peer2);
|
|
245
251
|
await tick();
|
|
246
|
-
//
|
|
247
|
-
self.ticketTask.setOnTicketReceived(async () => true);
|
|
252
|
+
// Default init() already wires an AcceptTicketsValidator
|
|
248
253
|
const ticket = createTestTicket(0);
|
|
249
254
|
peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
|
|
250
255
|
peer1.ticketTask.maintainDistribution();
|
|
251
256
|
await tick();
|
|
252
|
-
// self.addTicket WAS called
|
|
257
|
+
// self.addTicket WAS called
|
|
253
258
|
assert.strictEqual(self.receivedTickets.length, 1);
|
|
254
259
|
self.ticketTask.maintainDistribution();
|
|
255
260
|
await tick();
|
|
256
261
|
assert.strictEqual(peer2.receivedTickets.length, 1);
|
|
257
262
|
assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
|
|
258
263
|
});
|
|
264
|
+
it("replacePool overwrites the redistribution pool", async () => {
|
|
265
|
+
const self = await init("self");
|
|
266
|
+
const peer1 = await init("peer1");
|
|
267
|
+
self.openConnection(peer1);
|
|
268
|
+
await tick();
|
|
269
|
+
// Locally added tickets first
|
|
270
|
+
self.ticketTask.addTicket(TEST_EPOCH, createTestTicket(0));
|
|
271
|
+
self.ticketTask.addTicket(TEST_EPOCH, createTestTicket(1));
|
|
272
|
+
// Pool dump replaces with a different set
|
|
273
|
+
const dump = [createTestTicket(2), createTestTicket(3)];
|
|
274
|
+
self.ticketTask.replacePool(TEST_EPOCH, dump);
|
|
275
|
+
self.ticketTask.maintainDistribution();
|
|
276
|
+
await tick();
|
|
277
|
+
assert.strictEqual(peer1.receivedTickets.length, 2);
|
|
278
|
+
assert.deepStrictEqual(peer1.receivedTickets[0].ticket, dump[0]);
|
|
279
|
+
assert.deepStrictEqual(peer1.receivedTickets[1].ticket, dump[1]);
|
|
280
|
+
});
|
|
259
281
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;
|
|
1
|
+
{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAWF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,uBAqHxF"}
|