@lodestar/beacon-node 1.51.0-dev.e95d4a80f2 → 1.51.0-dev.ec1447da07
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/chain/archiveStore/utils/archiveBlocks.d.ts +4 -0
- package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
- package/lib/chain/archiveStore/utils/archiveBlocks.js +7 -1
- package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
- package/lib/chain/blocks/importBlock.d.ts.map +1 -1
- package/lib/chain/blocks/importBlock.js +6 -5
- package/lib/chain/blocks/importBlock.js.map +1 -1
- package/lib/chain/chain.js +4 -4
- package/lib/chain/chain.js.map +1 -1
- package/lib/db/beacon.d.ts +7 -0
- package/lib/db/beacon.d.ts.map +1 -1
- package/lib/db/beacon.js +32 -0
- package/lib/db/beacon.js.map +1 -1
- package/lib/db/dataColumnStore.d.ts +1 -1
- package/lib/db/dataColumnStore.d.ts.map +1 -1
- package/lib/db/dataColumnStore.js +1 -4
- package/lib/db/dataColumnStore.js.map +1 -1
- package/lib/db/interface.d.ts +1 -0
- package/lib/db/interface.d.ts.map +1 -1
- package/lib/db/repositories/dataColumnSidecarArchive.d.ts +6 -0
- package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
- package/lib/db/repositories/dataColumnSidecarArchive.js +19 -0
- package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
- package/lib/node/nodejs.d.ts.map +1 -1
- package/lib/node/nodejs.js +9 -1
- package/lib/node/nodejs.js.map +1 -1
- package/package.json +15 -15
- package/src/chain/archiveStore/utils/archiveBlocks.ts +12 -4
- package/src/chain/blocks/importBlock.ts +8 -7
- package/src/chain/chain.ts +4 -4
- package/src/db/beacon.ts +39 -0
- package/src/db/dataColumnStore.ts +2 -4
- package/src/db/interface.ts +2 -0
- package/src/db/repositories/dataColumnSidecarArchive.ts +20 -0
- package/src/node/nodejs.ts +18 -1
package/lib/db/beacon.d.ts
CHANGED
|
@@ -38,10 +38,17 @@ export declare class BeaconDb implements IBeaconDb {
|
|
|
38
38
|
readonly dataColumns: IDataColumnStore;
|
|
39
39
|
lastLegacyArchiveSlot: Slot | null;
|
|
40
40
|
private readonly flatFileStore;
|
|
41
|
+
private readonly logger;
|
|
41
42
|
constructor(config: ChainForkConfig, db: Db, opts: BeaconDbOpts);
|
|
42
43
|
init(): Promise<void>;
|
|
43
44
|
close(): Promise<void>;
|
|
44
45
|
setMetrics(metrics: LevelDbControllerMetrics, flatFileStoreMetrics?: FlatFileStoreMetrics | null): void;
|
|
46
|
+
/**
|
|
47
|
+
* Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
|
|
48
|
+
* rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
|
|
49
|
+
* makes the compaction overlap those files and rewrite them.
|
|
50
|
+
*/
|
|
51
|
+
compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
|
|
45
52
|
pruneHotDb(): Promise<void>;
|
|
46
53
|
deleteDeprecatedEth1Data(): Promise<void>;
|
|
47
54
|
private deleteBucketData;
|
package/lib/db/beacon.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"beacon.d.ts","sourceRoot":"","sources":["../../src/db/beacon.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAC,EAAE,EAAE,wBAAwB,EAAY,MAAM,cAAc,CAAC;AACrE,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAC,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAEvC,OAAO,EAAC,KAAK,gBAAgB,EAAwB,MAAM,sBAAsB,CAAC;AAElF,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAC,+BAA+B,EAAC,MAAM,yCAAyC,CAAC;AACxF,OAAO,EACL,0BAA0B,EAC1B,8BAA8B,EAC9B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,0BAA0B,EAC1B,kCAAkC,EAClC,2BAA2B,EAC3B,yCAAyC,EACzC,kCAAkC,EAClC,6BAA6B,EAC7B,0BAA0B,EAC1B,sBAAsB,EACtB,uBAAuB,EACvB,8BAA8B,EAC9B,uBAAuB,EACxB,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"beacon.d.ts","sourceRoot":"","sources":["../../src/db/beacon.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAC,EAAE,EAAE,wBAAwB,EAAY,MAAM,cAAc,CAAC;AACrE,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAC,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAEvC,OAAO,EAAC,KAAK,gBAAgB,EAAwB,MAAM,sBAAsB,CAAC;AAElF,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAC,+BAA+B,EAAC,MAAM,yCAAyC,CAAC;AACxF,OAAO,EACL,0BAA0B,EAC1B,8BAA8B,EAC9B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,0BAA0B,EAC1B,kCAAkC,EAClC,2BAA2B,EAC3B,yCAAyC,EACzC,kCAAkC,EAClC,6BAA6B,EAC7B,0BAA0B,EAC1B,sBAAsB,EACtB,uBAAuB,EACvB,8BAA8B,EAC9B,uBAAuB,EACxB,MAAM,yBAAyB,CAAC;AAKjC,MAAM,MAAM,YAAY,GAAG;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACvC,CAAC;AAEF,qBAAa,QAAS,YAAW,SAAS;IAmCtC,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE;IAlC3B,KAAK,EAAE,eAAe,CAAC;IACvB,YAAY,EAAE,sBAAsB,CAAC;IAErC,YAAY,EAAE,sBAAsB,CAAC;IACrC,mBAAmB,EAAE,6BAA6B,CAAC;IACnD,iBAAiB,EAAE,2BAA2B,CAAC;IAC/C,wBAAwB,EAAE,kCAAkC,CAAC;IAE7D,wBAAwB,EAAE,kCAAkC,CAAC;IAC7D,+BAA+B,EAAE,yCAAyC,CAAC;IAE3E,YAAY,EAAE,sBAAsB,CAAC;IACrC,eAAe,EAAE,yBAAyB,CAAC;IAC3C,qBAAqB,EAAE,+BAA+B,CAAC;IAEvD,aAAa,EAAE,uBAAuB,CAAC;IACvC,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,mBAAmB,EAAE,6BAA6B,CAAC;IACnD,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,oBAAoB,EAAE,8BAA8B,CAAC;IAGrD,qBAAqB,EAAE,+BAA+B,CAAC;IACvD,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,aAAa,EAAE,uBAAuB,CAAC;IACvC,oBAAoB,EAAE,8BAA8B,CAAC;IAErD,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAC;IACvC,qBAAqB,EAAE,IAAI,GAAG,IAAI,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAEhC,YACE,MAAM,EAAE,eAAe,EACJ,EAAE,EAAE,EAAE,EACzB,IAAI,EAAE,YAAY,EAqCnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAG3B;IAED,UAAU,CAAC,OAAO,EAAE,wBAAwB,EAAE,oBAAoB,GAAE,oBAAoB,GAAG,IAAW,GAAG,IAAI,CAG5G;IAED;;;;OAIG;IACG,wBAAwB,CAC5B,gBAAgB,EAAE,IAAI,EACtB,YAAY,SAA2C,GACtD,OAAO,CAAC,IAAI,CAAC,CAwBf;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAahC;IAEK,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC,CAa9C;YAEa,gBAAgB;CAoB/B"}
|
package/lib/db/beacon.js
CHANGED
|
@@ -5,6 +5,8 @@ import { FlatFileStore } from "./flatFileStore/flatFileStore.js";
|
|
|
5
5
|
import { CheckpointStateRepository } from "./repositories/checkpointState.js";
|
|
6
6
|
import { EarliestAvailableSlotRepository } from "./repositories/earliestAvailableSlot.js";
|
|
7
7
|
import { AttesterSlashingRepository, BLSToExecutionChangeRepository, BestLightClientUpdateRepository, BlobSidecarsArchiveRepository, BlobSidecarsRepository, BlockArchiveRepository, BlockRepository, CheckpointHeaderRepository, DataColumnSidecarArchiveRepository, DataColumnSidecarRepository, ExecutionPayloadEnvelopeArchiveRepository, ExecutionPayloadEnvelopeRepository, ProposerPreferencesRepository, ProposerSlashingRepository, StateArchiveRepository, SyncCommitteeRepository, SyncCommitteeWitnessRepository, VoluntaryExitRepository, } from "./repositories/index.js";
|
|
8
|
+
/** Below this size, seeks over leftover legacy column data stay cheap */
|
|
9
|
+
const LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES = 1024 ** 3;
|
|
8
10
|
export class BeaconDb {
|
|
9
11
|
db;
|
|
10
12
|
block;
|
|
@@ -31,6 +33,7 @@ export class BeaconDb {
|
|
|
31
33
|
dataColumns;
|
|
32
34
|
lastLegacyArchiveSlot = null;
|
|
33
35
|
flatFileStore;
|
|
36
|
+
logger;
|
|
34
37
|
constructor(config, db, opts) {
|
|
35
38
|
this.db = db;
|
|
36
39
|
// Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' -
|
|
@@ -55,6 +58,7 @@ export class BeaconDb {
|
|
|
55
58
|
this.checkpointHeader = new CheckpointHeaderRepository(config, db);
|
|
56
59
|
this.syncCommittee = new SyncCommitteeRepository(config, db);
|
|
57
60
|
this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
|
|
61
|
+
this.logger = opts.logger;
|
|
58
62
|
this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
|
|
59
63
|
this.dataColumns = new LegacyDataColumnStore(this.flatFileStore, this.dataColumnSidecar, this.dataColumnSidecarArchive, this.blockArchive);
|
|
60
64
|
}
|
|
@@ -71,6 +75,34 @@ export class BeaconDb {
|
|
|
71
75
|
this.db.setMetrics(metrics);
|
|
72
76
|
this.flatFileStore.setMetrics(flatFileStoreMetrics);
|
|
73
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
|
|
80
|
+
* rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
|
|
81
|
+
* makes the compaction overlap those files and rewrite them.
|
|
82
|
+
*/
|
|
83
|
+
async compactLegacyDataColumns(prunedBeforeSlot, minSizeBytes = LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES) {
|
|
84
|
+
const archiveBucket = Bucket.allForks_dataColumnSidecarsArchive;
|
|
85
|
+
const start = encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0));
|
|
86
|
+
const end = encodeKey(archiveBucket, this.dataColumnSidecarArchive.getMinKeyRaw(prunedBeforeSlot));
|
|
87
|
+
const sizeBytes = await this.db.approximateSize(start, end);
|
|
88
|
+
if (sizeBytes < minSizeBytes)
|
|
89
|
+
return;
|
|
90
|
+
// Column indices stay below 0xffff and hot column keys are shorter, so no sentinel matches a real key
|
|
91
|
+
const upperSentinel = prunedBeforeSlot > 0
|
|
92
|
+
? encodeKey(archiveBucket, this.dataColumnSidecarArchive.encodeKeyRaw(prunedBeforeSlot - 1, 0xffff))
|
|
93
|
+
: encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(35, 0xff));
|
|
94
|
+
const sentinels = [start, upperSentinel];
|
|
95
|
+
this.logger.info("Compacting legacy data columns, this may take a while", { sizeBytes });
|
|
96
|
+
const startTime = Date.now();
|
|
97
|
+
await this.db.batchPut(sentinels.map((key) => ({ key, value: new Uint8Array() })));
|
|
98
|
+
await this.db.compactRange(start, end);
|
|
99
|
+
await this.db.batchDelete(sentinels);
|
|
100
|
+
await this.db.compactRange(start, end);
|
|
101
|
+
this.logger.verbose("Compacted legacy data columns", {
|
|
102
|
+
durationMs: Date.now() - startTime,
|
|
103
|
+
sizeBytes: await this.db.approximateSize(start, end),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
74
106
|
async pruneHotDb() {
|
|
75
107
|
// Prune all hot blobs
|
|
76
108
|
await this.blobSidecars.batchDelete(await this.blobSidecars.keys());
|
package/lib/db/beacon.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"beacon.js","sourceRoot":"","sources":["../../src/db/beacon.ts"],"names":[],"mappings":"AACA,OAAO,EAA+B,SAAS,EAAC,MAAM,cAAc,CAAC;AAGrE,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAC;AACpC,OAAO,EAAwB,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,kCAAkC,CAAC;AAG/D,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAC,+BAA+B,EAAC,MAAM,yCAAyC,CAAC;AACxF,OAAO,EACL,0BAA0B,EAC1B,8BAA8B,EAC9B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,0BAA0B,EAC1B,kCAAkC,EAClC,2BAA2B,EAC3B,yCAAyC,EACzC,kCAAkC,EAClC,6BAA6B,EAC7B,0BAA0B,EAC1B,sBAAsB,EACtB,uBAAuB,EACvB,8BAA8B,EAC9B,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"beacon.js","sourceRoot":"","sources":["../../src/db/beacon.ts"],"names":[],"mappings":"AACA,OAAO,EAA+B,SAAS,EAAC,MAAM,cAAc,CAAC;AAGrE,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAC;AACpC,OAAO,EAAwB,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,kCAAkC,CAAC;AAG/D,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAC,+BAA+B,EAAC,MAAM,yCAAyC,CAAC;AACxF,OAAO,EACL,0BAA0B,EAC1B,8BAA8B,EAC9B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,0BAA0B,EAC1B,kCAAkC,EAClC,2BAA2B,EAC3B,yCAAyC,EACzC,kCAAkC,EAClC,6BAA6B,EAC7B,0BAA0B,EAC1B,sBAAsB,EACtB,uBAAuB,EACvB,8BAA8B,EAC9B,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;AAEjC,yEAAyE;AACzE,MAAM,wCAAwC,GAAG,IAAI,IAAI,CAAC,CAAC;AAQ3D,MAAM,OAAO,QAAQ;IAmCE,EAAE;IAlCvB,KAAK,CAAkB;IACvB,YAAY,CAAyB;IAErC,YAAY,CAAyB;IACrC,mBAAmB,CAAgC;IACnD,iBAAiB,CAA8B;IAC/C,wBAAwB,CAAqC;IAE7D,wBAAwB,CAAqC;IAC7D,+BAA+B,CAA4C;IAE3E,YAAY,CAAyB;IACrC,eAAe,CAA4B;IAC3C,qBAAqB,CAAkC;IAEvD,aAAa,CAA0B;IACvC,gBAAgB,CAA6B;IAC7C,mBAAmB,CAAgC;IACnD,gBAAgB,CAA6B;IAC7C,oBAAoB,CAAiC;IAErD,cAAc;IACd,qBAAqB,CAAkC;IACvD,gBAAgB,CAA6B;IAC7C,aAAa,CAA0B;IACvC,oBAAoB,CAAiC;IAE5C,WAAW,CAAmB;IACvC,qBAAqB,GAAgB,IAAI,CAAC;IACzB,aAAa,CAAgB;IAC7B,MAAM,CAAS;IAEhC,YACE,MAAuB,EACJ,EAAM,EACzB,IAAkB;kBADC,EAAE;QAGrB,4IAA4I;QAC5I,IAAI,CAAC,KAAK,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC,YAAY,GAAG,IAAI,sBAAsB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE3D,IAAI,CAAC,YAAY,GAAG,IAAI,sBAAsB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,IAAI,6BAA6B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,iBAAiB,GAAG,IAAI,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrE,IAAI,CAAC,wBAAwB,GAAG,IAAI,kCAAkC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAEnF,IAAI,CAAC,wBAAwB,GAAG,IAAI,kCAAkC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACnF,IAAI,CAAC,+BAA+B,GAAG,IAAI,yCAAyC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAEjG,IAAI,CAAC,YAAY,GAAG,IAAI,sBAAsB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,GAAG,IAAI,yBAAyB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,qBAAqB,GAAG,IAAI,+BAA+B,CAAC,EAAE,CAAC,CAAC;QACrE,IAAI,CAAC,aAAa,GAAG,IAAI,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,oBAAoB,GAAG,IAAI,8BAA8B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,gBAAgB,GAAG,IAAI,0BAA0B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,mBAAmB,GAAG,IAAI,6BAA6B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,gBAAgB,GAAG,IAAI,0BAA0B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAEnE,cAAc;QACd,IAAI,CAAC,qBAAqB,GAAG,IAAI,+BAA+B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,gBAAgB,GAAG,IAAI,0BAA0B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,aAAa,GAAG,IAAI,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,oBAAoB,GAAG,IAAI,8BAA8B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9F,IAAI,CAAC,WAAW,GAAG,IAAI,qBAAqB,CAC1C,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,YAAY,CAClB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,EAAC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAC,CAAC,CAAC;QAC5F,IAAI,CAAC,qBAAqB,GAAG,aAAa,EAAE,MAAM,IAAI,IAAI,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,UAAU,CAAC,OAAiC,EAAE,oBAAoB,GAAgC,IAAI;QACpG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,wBAAwB,CAC5B,gBAAsB,EACtB,YAAY,GAAG,wCAAwC;QAEvD,MAAM,aAAa,GAAG,MAAM,CAAC,kCAAkC,CAAC;QAChE,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,2BAA2B,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,wBAAwB,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACnG,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5D,IAAI,SAAS,GAAG,YAAY;YAAE,OAAO;QAErC,sGAAsG;QACtG,MAAM,aAAa,GACjB,gBAAgB,GAAG,CAAC;YAClB,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,wBAAwB,CAAC,YAAY,CAAC,gBAAgB,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;YACpG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,2BAA2B,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,EAAE,EAAC,SAAS,EAAC,CAAC,CAAC;QACvF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAC,GAAG,EAAE,KAAK,EAAE,IAAI,UAAU,EAAE,EAAC,CAAC,CAAC,CAAC,CAAC;QACjF,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,EAAE;YACnD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,SAAS,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,sBAAsB;QACtB,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QACpE,yFAAyF;QACzF,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAChE,yEAAyE;QACzE,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CACxB,SAAS,CAAC,MAAM,CAAC,2BAA2B,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAC9D,SAAS,CAAC,MAAM,CAAC,2BAA2B,GAAG,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACnE,CAAC;QACF,uBAAuB;QACvB,qCAAqC;QACrC,yDAAyD;IAC3D,CAAC;IAED,KAAK,CAAC,wBAAwB;QAC5B,MAAM,iBAAiB,GAAG;YACxB,MAAM,CAAC,eAAe;YACtB,MAAM,CAAC,qBAAqB;YAC5B,MAAM,CAAC,kBAAkB;YACzB,MAAM,CAAC,mBAAmB;YAC1B,MAAM,CAAC,sBAAsB;YAC7B,MAAM,CAAC,wCAAwC;SAChD,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;YACvC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,MAAc;QAC3C,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtD,6DAA6D;QAC7D,MAAM,iBAAiB,GAAG,IAAI,CAAC;QAC/B,IAAI,SAAS,GAAiB,EAAE,CAAC;QAEjC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,EAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAC,CAAC,EAAE,CAAC;YACtE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,IAAI,SAAS,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;gBAC1C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBACrC,SAAS,GAAG,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -10,7 +10,7 @@ export type IndexedDataColumnBytes = {
|
|
|
10
10
|
data: Uint8Array;
|
|
11
11
|
};
|
|
12
12
|
type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
|
|
13
|
-
type LegacyArchiveDataColumnStore = Pick<DataColumnSidecarArchiveRepository, "values" | "getManyBinary" | "
|
|
13
|
+
type LegacyArchiveDataColumnStore = Pick<DataColumnSidecarArchiveRepository, "values" | "getManyBinary" | "pruneBefore">;
|
|
14
14
|
type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
|
|
15
15
|
export interface IDataColumnStore {
|
|
16
16
|
getAll(key: DataColumnKey): Promise<DataColumnSidecar[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataColumnStore.d.ts","sourceRoot":"","sources":["../../src/db/dataColumnStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAE,iBAAiB,EAAE,OAAO,EAAE,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAE9E,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AACjE,OAAO,KAAK,EACV,sBAAsB,EACtB,kCAAkC,EAClC,2BAA2B,EAC5B,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,aAAa,GAAG;IAAC,IAAI,EAAE,IAAI,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAC,CAAC;AAC7D,MAAM,MAAM,sBAAsB,GAAG;IAAC,KAAK,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAC,CAAC;AAE5E,KAAK,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,EAAE,QAAQ,GAAG,eAAe,GAAG,YAAY,CAAC,CAAC;AAC7G,KAAK,4BAA4B,GAAG,IAAI,CACtC,kCAAkC,EAClC,QAAQ,GAAG,eAAe,GAAG,
|
|
1
|
+
{"version":3,"file":"dataColumnStore.d.ts","sourceRoot":"","sources":["../../src/db/dataColumnStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAE,iBAAiB,EAAE,OAAO,EAAE,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAE9E,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AACjE,OAAO,KAAK,EACV,sBAAsB,EACtB,kCAAkC,EAClC,2BAA2B,EAC5B,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,aAAa,GAAG;IAAC,IAAI,EAAE,IAAI,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAC,CAAC;AAC7D,MAAM,MAAM,sBAAsB,GAAG;IAAC,KAAK,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAC,CAAC;AAE5E,KAAK,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,EAAE,QAAQ,GAAG,eAAe,GAAG,YAAY,CAAC,CAAC;AAC7G,KAAK,4BAA4B,GAAG,IAAI,CACtC,kCAAkC,EAClC,QAAQ,GAAG,eAAe,GAAG,aAAa,CAC3C,CAAC;AACF,KAAK,cAAc,GAAG,IAAI,CAAC,sBAAsB,EAAE,eAAe,CAAC,CAAC;AAEpE,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACzD,aAAa,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC/F,aAAa,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpF,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD,WAAW,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CAC1C;AAED;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAE1D,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAJ/B,YACmB,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,wBAAwB,EACnC,aAAa,EAAE,4BAA4B,EAC3C,YAAY,EAAE,cAAc,EAC3C;IAEE,MAAM,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAiB3E;IAEK,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,CAAC,CAiBjH;IAEK,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtG;IAEK,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAGrD;IAEK,WAAW,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAI7C;CACF"}
|
|
@@ -56,10 +56,7 @@ export class LegacyDataColumnStore {
|
|
|
56
56
|
}
|
|
57
57
|
async pruneBefore(slot) {
|
|
58
58
|
const flatFileSlots = await this.flatFiles.pruneBefore(slot);
|
|
59
|
-
const
|
|
60
|
-
const slots = [...new Set(prefixedKeys.map(({ prefix }) => prefix))];
|
|
61
|
-
if (slots.length > 0)
|
|
62
|
-
await this.legacyArchive.deleteMany(slots);
|
|
59
|
+
const slots = await this.legacyArchive.pruneBefore(slot);
|
|
63
60
|
return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
|
|
64
61
|
}
|
|
65
62
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataColumnStore.js","sourceRoot":"","sources":["../../src/db/dataColumnStore.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AA0BxC;;;GAGG;AACH,MAAM,OAAO,qBAAqB;IAEb,SAAS;IACT,SAAS;IACT,aAAa;IACb,YAAY;IAJ/B,YACmB,SAAyB,EACzB,SAAmC,EACnC,aAA2C,EAC3C,YAA4B;yBAH5B,SAAS;yBACT,SAAS;6BACT,aAAa;4BACb,YAAY;IAC5B,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB;QAC3C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,QAAQ,CAAC;QAEzC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkC,CAAC;QAClE,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3D,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB,EAAE,OAAsB;QAC1E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7F,IAAI,gBAAgB,KAAK,IAAI;YAAE,OAAO,gBAAgB,CAAC;QACvD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAEjE,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC;QAE3G,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,aAAa,CAC7D,IAAI,EACJ,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACtD,CAAC;QACF,WAAW,CAAC,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;QACxD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB,EAAE,OAAiC;QACrF,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACtE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAqB;QACpC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAC,SAAS,EAAC,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAU;QAC1B,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,
|
|
1
|
+
{"version":3,"file":"dataColumnStore.js","sourceRoot":"","sources":["../../src/db/dataColumnStore.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AA0BxC;;;GAGG;AACH,MAAM,OAAO,qBAAqB;IAEb,SAAS;IACT,SAAS;IACT,aAAa;IACb,YAAY;IAJ/B,YACmB,SAAyB,EACzB,SAAmC,EACnC,aAA2C,EAC3C,YAA4B;yBAH5B,SAAS;yBACT,SAAS;6BACT,aAAa;4BACb,YAAY;IAC5B,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB;QAC3C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,QAAQ,CAAC;QAEzC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkC,CAAC;QAClE,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3D,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB,EAAE,OAAsB;QAC1E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7F,IAAI,gBAAgB,KAAK,IAAI;YAAE,OAAO,gBAAgB,CAAC;QACvD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAEjE,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC;QAE3G,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,aAAa,CAC7D,IAAI,EACJ,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACtD,CAAC;QACF,WAAW,CAAC,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;QACxD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB,EAAE,OAAiC;QACrF,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACtE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAqB;QACpC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAC,SAAS,EAAC,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAU;QAC1B,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF;AAED,SAAS,mBAAmB,CAAC,QAAoC;IAC/D,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAClB,MAAkC,EAClC,gBAA0B,EAC1B,QAA2C;IAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;IAC7D,CAAC;AACH,CAAC"}
|
package/lib/db/interface.d.ts
CHANGED
|
@@ -35,6 +35,7 @@ export interface IBeaconDb {
|
|
|
35
35
|
/** Highest legacy column slot at startup; later archival writes the block-root index first. */
|
|
36
36
|
readonly lastLegacyArchiveSlot: Slot | null;
|
|
37
37
|
init(): Promise<void>;
|
|
38
|
+
compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
|
|
38
39
|
pruneHotDb(): Promise<void>;
|
|
39
40
|
deleteDeprecatedEth1Data(): Promise<void>;
|
|
40
41
|
/** Close the connection to the db instance and close the db store. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/db/interface.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,wBAAwB,EAAC,MAAM,cAAc,CAAC;AACtD,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAC;AACrC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAC,+BAA+B,EAAC,MAAM,yCAAyC,CAAC;AACxF,OAAO,EACL,0BAA0B,EAC1B,8BAA8B,EAC9B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,0BAA0B,EAC1B,kCAAkC,EAClC,2BAA2B,EAC3B,yCAAyC,EACzC,kCAAkC,EAClC,6BAA6B,EAC7B,0BAA0B,EAC1B,sBAAsB,EACtB,uBAAuB,EACvB,8BAA8B,EAC9B,uBAAuB,EACxB,MAAM,yBAAyB,CAAC;AAEjC;;;;GAIG;AACH,MAAM,WAAW,SAAS;IAExB,KAAK,EAAE,eAAe,CAAC;IAEvB,YAAY,EAAE,sBAAsB,CAAC;IAErC,YAAY,EAAE,sBAAsB,CAAC;IACrC,mBAAmB,EAAE,6BAA6B,CAAC;IACnD,iBAAiB,EAAE,2BAA2B,CAAC;IAC/C,wBAAwB,EAAE,kCAAkC,CAAC;IAE7D,wBAAwB,EAAE,kCAAkC,CAAC;IAC7D,+BAA+B,EAAE,yCAAyC,CAAC;IAG3E,YAAY,EAAE,sBAAsB,CAAC;IAErC,eAAe,EAAE,yBAAyB,CAAC;IAC3C,qBAAqB,EAAE,+BAA+B,CAAC;IAGvD,aAAa,EAAE,uBAAuB,CAAC;IACvC,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,mBAAmB,EAAE,6BAA6B,CAAC;IACnD,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,oBAAoB,EAAE,8BAA8B,CAAC;IAGrD,qBAAqB,EAAE,+BAA+B,CAAC;IACvD,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,aAAa,EAAE,uBAAuB,CAAC;IACvC,oBAAoB,EAAE,8BAA8B,CAAC;IAErD,WAAW,EAAE,gBAAgB,CAAC;IAC9B,+FAA+F;IAC/F,QAAQ,CAAC,qBAAqB,EAAE,IAAI,GAAG,IAAI,CAAC;IAC5C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1C,uEAAuE;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,iDAAiD;IACjD,UAAU,CAAC,OAAO,EAAE,wBAAwB,EAAE,oBAAoB,CAAC,EAAE,oBAAoB,GAAG,IAAI,GAAG,IAAI,CAAC;CACzG"}
|
|
1
|
+
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/db/interface.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,wBAAwB,EAAC,MAAM,cAAc,CAAC;AACtD,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAC;AACrC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAC,+BAA+B,EAAC,MAAM,yCAAyC,CAAC;AACxF,OAAO,EACL,0BAA0B,EAC1B,8BAA8B,EAC9B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,0BAA0B,EAC1B,kCAAkC,EAClC,2BAA2B,EAC3B,yCAAyC,EACzC,kCAAkC,EAClC,6BAA6B,EAC7B,0BAA0B,EAC1B,sBAAsB,EACtB,uBAAuB,EACvB,8BAA8B,EAC9B,uBAAuB,EACxB,MAAM,yBAAyB,CAAC;AAEjC;;;;GAIG;AACH,MAAM,WAAW,SAAS;IAExB,KAAK,EAAE,eAAe,CAAC;IAEvB,YAAY,EAAE,sBAAsB,CAAC;IAErC,YAAY,EAAE,sBAAsB,CAAC;IACrC,mBAAmB,EAAE,6BAA6B,CAAC;IACnD,iBAAiB,EAAE,2BAA2B,CAAC;IAC/C,wBAAwB,EAAE,kCAAkC,CAAC;IAE7D,wBAAwB,EAAE,kCAAkC,CAAC;IAC7D,+BAA+B,EAAE,yCAAyC,CAAC;IAG3E,YAAY,EAAE,sBAAsB,CAAC;IAErC,eAAe,EAAE,yBAAyB,CAAC;IAC3C,qBAAqB,EAAE,+BAA+B,CAAC;IAGvD,aAAa,EAAE,uBAAuB,CAAC;IACvC,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,mBAAmB,EAAE,6BAA6B,CAAC;IACnD,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,oBAAoB,EAAE,8BAA8B,CAAC;IAGrD,qBAAqB,EAAE,+BAA+B,CAAC;IACvD,gBAAgB,EAAE,0BAA0B,CAAC;IAC7C,aAAa,EAAE,uBAAuB,CAAC;IACvC,oBAAoB,EAAE,8BAA8B,CAAC;IAErD,WAAW,EAAE,gBAAgB,CAAC;IAC9B,+FAA+F;IAC/F,QAAQ,CAAC,qBAAqB,EAAE,IAAI,GAAG,IAAI,CAAC;IAC5C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB,wBAAwB,CAAC,gBAAgB,EAAE,IAAI,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvF,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1C,uEAAuE;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,iDAAiD;IACjD,UAAU,CAAC,OAAO,EAAE,wBAAwB,EAAE,oBAAoB,CAAC,EAAE,oBAAoB,GAAG,IAAI,GAAG,IAAI,CAAC;CACzG"}
|
|
@@ -8,6 +8,7 @@ import { ColumnIndex, DataColumnSidecar, Slot } from "@lodestar/types";
|
|
|
8
8
|
* Indexed data by `slot` + `columnIndex`
|
|
9
9
|
*/
|
|
10
10
|
export declare class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
|
|
11
|
+
private prunedBeforeSlot;
|
|
11
12
|
constructor(config: ChainForkConfig, db: Db);
|
|
12
13
|
/**
|
|
13
14
|
* Id is hashTreeRoot of unsigned BeaconBlock
|
|
@@ -22,5 +23,10 @@ export declare class DataColumnSidecarArchiveRepository extends PrefixedReposito
|
|
|
22
23
|
};
|
|
23
24
|
getMaxKeyRaw(prefix: Slot): Uint8Array;
|
|
24
25
|
getMinKeyRaw(prefix: Slot): Uint8Array;
|
|
26
|
+
/**
|
|
27
|
+
* Deletes all entries before `slot`. Prunes only delete from the start of the bucket, so the first prune
|
|
28
|
+
* compacts the range to drop tombstones of earlier runs and later prunes resume where the previous one ended.
|
|
29
|
+
*/
|
|
30
|
+
pruneBefore(slot: Slot): Promise<Slot[]>;
|
|
25
31
|
}
|
|
26
32
|
//# sourceMappingURL=dataColumnSidecarArchive.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataColumnSidecarArchive.d.ts","sourceRoot":"","sources":["../../../src/db/repositories/dataColumnSidecarArchive.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAC,EAAE,EAAE,kBAAkB,EAA6C,MAAM,cAAc,CAAC;AAEhG,OAAO,EAAC,WAAW,EAAE,iBAAiB,EAAE,IAAI,EAAgC,MAAM,iBAAiB,CAAC;AAOpG;;;;;GAKG;AACH,qBAAa,kCAAmC,SAAQ,kBAAkB,CAAC,IAAI,EAAE,WAAW,EAAE,iBAAiB,CAAC;IAC9G,YAAY,MAAM,EAAE,eAAe,EAAE,EAAE,EAAE,EAAE,EAK1C;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,iBAAiB,GAAG,WAAW,CAE3C;IAED,WAAW,CAAC,KAAK,EAAE,iBAAiB,GAAG,UAAU,CAKhD;IAED,WAAW,CAAC,IAAI,EAAE,UAAU,GAAG,iBAAiB,CAK/C;IAED,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,GAAG,UAAU,CAKtD;IAED,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG;QAAC,MAAM,EAAE,IAAI,CAAC;QAAC,EAAE,EAAE,WAAW,CAAA;KAAC,CAK7D;IAED,YAAY,CAAC,MAAM,EAAE,IAAI,GAAG,UAAU,CAKrC;IAED,YAAY,CAAC,MAAM,EAAE,IAAI,GAAG,UAAU,CAKrC;CACF"}
|
|
1
|
+
{"version":3,"file":"dataColumnSidecarArchive.d.ts","sourceRoot":"","sources":["../../../src/db/repositories/dataColumnSidecarArchive.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAC,EAAE,EAAE,kBAAkB,EAA6C,MAAM,cAAc,CAAC;AAEhG,OAAO,EAAC,WAAW,EAAE,iBAAiB,EAAE,IAAI,EAAgC,MAAM,iBAAiB,CAAC;AAOpG;;;;;GAKG;AACH,qBAAa,kCAAmC,SAAQ,kBAAkB,CAAC,IAAI,EAAE,WAAW,EAAE,iBAAiB,CAAC;IAC9G,OAAO,CAAC,gBAAgB,CAAqB;IAE7C,YAAY,MAAM,EAAE,eAAe,EAAE,EAAE,EAAE,EAAE,EAK1C;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,iBAAiB,GAAG,WAAW,CAE3C;IAED,WAAW,CAAC,KAAK,EAAE,iBAAiB,GAAG,UAAU,CAKhD;IAED,WAAW,CAAC,IAAI,EAAE,UAAU,GAAG,iBAAiB,CAK/C;IAED,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,GAAG,UAAU,CAKtD;IAED,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG;QAAC,MAAM,EAAE,IAAI,CAAC;QAAC,EAAE,EAAE,WAAW,CAAA;KAAC,CAK7D;IAED,YAAY,CAAC,MAAM,EAAE,IAAI,GAAG,UAAU,CAKrC;IAED,YAAY,CAAC,MAAM,EAAE,IAAI,GAAG,UAAU,CAKrC;IAED;;;OAGG;IACG,WAAW,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAY7C;CACF"}
|
|
@@ -12,6 +12,7 @@ const SLOT_BYTE_SIZE = 8;
|
|
|
12
12
|
* Indexed data by `slot` + `columnIndex`
|
|
13
13
|
*/
|
|
14
14
|
export class DataColumnSidecarArchiveRepository extends PrefixedRepository {
|
|
15
|
+
prunedBeforeSlot = null;
|
|
15
16
|
constructor(config, db) {
|
|
16
17
|
const bucket = Bucket.allForks_dataColumnSidecarsArchive;
|
|
17
18
|
// Type won't be used since we select it dynamically based on fork
|
|
@@ -60,5 +61,23 @@ export class DataColumnSidecarArchiveRepository extends PrefixedRepository {
|
|
|
60
61
|
encodeNumberForDbKey(0, COLUMN_INDEX_BYTE_SIZE),
|
|
61
62
|
]);
|
|
62
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Deletes all entries before `slot`. Prunes only delete from the start of the bucket, so the first prune
|
|
66
|
+
* compacts the range to drop tombstones of earlier runs and later prunes resume where the previous one ended.
|
|
67
|
+
*/
|
|
68
|
+
async pruneBefore(slot) {
|
|
69
|
+
if (this.prunedBeforeSlot === null) {
|
|
70
|
+
await this.db.compactRange(this.wrapKey(this.getMinKeyRaw(0)), this.wrapKey(this.getMinKeyRaw(slot)));
|
|
71
|
+
}
|
|
72
|
+
else if (slot <= this.prunedBeforeSlot) {
|
|
73
|
+
return [];
|
|
74
|
+
}
|
|
75
|
+
const keys = await this.keys({ gte: { prefix: this.prunedBeforeSlot ?? 0, id: 0 }, lt: { prefix: slot, id: 0 } });
|
|
76
|
+
const slots = [...new Set(keys.map(({ prefix }) => prefix))];
|
|
77
|
+
if (slots.length > 0)
|
|
78
|
+
await this.deleteMany(slots);
|
|
79
|
+
this.prunedBeforeSlot = slot;
|
|
80
|
+
return slots;
|
|
81
|
+
}
|
|
63
82
|
}
|
|
64
83
|
//# sourceMappingURL=dataColumnSidecarArchive.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataColumnSidecarArchive.js","sourceRoot":"","sources":["../../../src/db/repositories/dataColumnSidecarArchive.ts"],"names":[],"mappings":"AACA,OAAO,EAAK,kBAAkB,EAAE,oBAAoB,EAAE,oBAAoB,EAAC,MAAM,cAAc,CAAC;AAChG,OAAO,EAAC,iBAAiB,EAAC,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAuC,wBAAwB,EAAE,GAAG,EAAC,MAAM,iBAAiB,CAAC;AACpG,OAAO,EAAC,6BAA6B,EAAC,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAC,MAAM,EAAE,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAE3D,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB;;;;;GAKG;AACH,MAAM,OAAO,kCAAmC,SAAQ,kBAAwD;
|
|
1
|
+
{"version":3,"file":"dataColumnSidecarArchive.js","sourceRoot":"","sources":["../../../src/db/repositories/dataColumnSidecarArchive.ts"],"names":[],"mappings":"AACA,OAAO,EAAK,kBAAkB,EAAE,oBAAoB,EAAE,oBAAoB,EAAC,MAAM,cAAc,CAAC;AAChG,OAAO,EAAC,iBAAiB,EAAC,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAuC,wBAAwB,EAAE,GAAG,EAAC,MAAM,iBAAiB,CAAC;AACpG,OAAO,EAAC,6BAA6B,EAAC,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAC,MAAM,EAAE,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAE3D,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB;;;;;GAKG;AACH,MAAM,OAAO,kCAAmC,SAAQ,kBAAwD;IACtG,gBAAgB,GAAgB,IAAI,CAAC;IAE7C,YAAY,MAAuB,EAAE,EAAM;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,kCAAkC,CAAC;QACzD,kEAAkE;QAClE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACxC,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAwB;QAC5B,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAED,WAAW,CAAC,KAAwB;QAClC,IAAI,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;YACpC,OAAO,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACrD,CAAC;IAED,WAAW,CAAC,IAAgB;QAC1B,IAAI,6BAA6B,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC;IAED,YAAY,CAAC,MAAY,EAAE,EAAe;QACxC,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,oBAAoB,CAAC,MAAM,EAAE,cAAc,CAAC;YAC5C,oBAAoB,CAAC,EAAE,EAAE,sBAAsB,CAAC;SACjD,CAAC,CAAC;IACL,CAAC;IAED,YAAY,CAAC,GAAe;QAC1B,OAAO;YACL,MAAM,EAAE,oBAAoB,CAAC,GAAG,EAAE,cAAc,CAAS;YACzD,EAAE,EAAE,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,sBAAsB,CAAgB;SAC3F,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,MAAY;QACvB,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,oBAAoB,CAAC,MAAM,EAAE,cAAc,CAAC;YAC5C,oBAAoB,CAAC,iBAAiB,GAAG,CAAC,EAAE,sBAAsB,CAAC;SACpE,CAAC,CAAC;IACL,CAAC;IAED,YAAY,CAAC,MAAY;QACvB,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,oBAAoB,CAAC,MAAM,EAAE,cAAc,CAAC;YAC5C,oBAAoB,CAAC,CAAC,EAAE,sBAAsB,CAAC;SAChD,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,IAAU;QAC1B,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxG,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAC,GAAG,EAAE,EAAC,MAAM,EAAE,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,EAAC,EAAE,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAC,EAAC,CAAC,CAAC;QAC5G,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAC,MAAM,EAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACnD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
package/lib/node/nodejs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nodejs.d.ts","sourceRoot":"","sources":["../../src/node/nodejs.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAOrC,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAE/D,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAC;AAEtD,OAAO,
|
|
1
|
+
{"version":3,"file":"nodejs.d.ts","sourceRoot":"","sources":["../../src/node/nodejs.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAOrC,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAE/D,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAC9C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAC;AAEtD,OAAO,EACL,gBAAgB,EAIjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAErC,OAAO,EAAC,uBAAuB,EAAC,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAC,mBAAmB,EAAS,MAAM,iBAAiB,CAAC;AAE5D,OAAO,EAAc,YAAY,EAAoB,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAC,gBAAgB,EAAyB,MAAM,8BAA8B,CAAC;AACtF,OAAO,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AAEzC,OAAO,EAAC,iBAAiB,EAAE,OAAO,EAAsC,MAAM,qBAAqB,CAAC;AACpG,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,OAAO,EAAqB,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAa,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAIzD,OAAO,EAAC,kBAAkB,EAAC,MAAM,cAAc,CAAC;AAEhD,cAAc,cAAc,CAAC;AAE7B,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,YAAY,CAAC;IACrB,EAAE,EAAE,SAAS,CAAC;IACd,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,gBAAgB,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC1C,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,YAAY,CAAC;IACpB,GAAG,EAAE,gBAAgB,CAAC;IACtB,IAAI,EAAE,WAAW,CAAC;IAClB,aAAa,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACxC,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACrC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,YAAY,CAAC;IACrB,WAAW,EAAE,WAAW,CAAC;IACzB,EAAE,EAAE,SAAS,CAAC;IACd,MAAM,EAAE,UAAU,CAAC;IACnB,uBAAuB,EAAE,uBAAuB,CAAC;IACjD,UAAU,EAAE,UAAU,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,gBAAgB,CAAC;IAC9B,sBAAsB,EAAE,OAAO,CAAC;IAChC,qBAAqB,EAAE,IAAI,CAAC;IAC5B,iBAAiB,CAAC,EAAE,QAAQ,EAAE,CAAC;CAChC,CAAC;AAEF,oBAAY,gBAAgB;IAC1B,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,MAAM,WAAW;CAClB;AAqBD;;;GAGG;AACH,qBAAa,UAAU;IACrB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,YAAY,CAAC;IACrB,EAAE,EAAE,SAAS,CAAC;IACd,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,aAAa,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACxC,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACrC,gBAAgB,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC1C,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,YAAY,CAAC;IACpB,GAAG,EAAE,gBAAgB,CAAC;IACtB,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,IAAI,EAAE,WAAW,CAAC;IAElB,MAAM,EAAE,gBAAgB,CAAC;IACzB,OAAO,CAAC,UAAU,CAAC,CAAkB;IAErC,YAAY,EACV,IAAI,EACJ,MAAM,EACN,EAAE,EACF,OAAO,EACP,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,IAAI,EACJ,UAAU,GACX,EAAE,iBAAiB,EAgBnB;IAED;;;OAGG;IACH,OAAa,IAAI,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EACnD,IAAI,EACJ,MAAM,EACN,WAAW,EACX,EAAE,EACF,MAAM,EACN,uBAAuB,EACvB,UAAU,EACV,OAAO,EACP,aAAa,EACb,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,qBAAqB,EACrB,iBAAsB,GACvB,EAAE,qBAAqB,GAAG,OAAO,CAAC,CAAC,CAAC,CA0NpC;IAED;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAkB3B;CACF"}
|
package/lib/node/nodejs.js
CHANGED
|
@@ -2,9 +2,10 @@ import { setMaxListeners } from "node:events";
|
|
|
2
2
|
import { init as initNativeMetrics, registerLocalValidator as registerNativeLocalValidator, scrapeMetrics as scrapeNativeMetrics, unregisterLocalValidator as unregisterNativeLocalValidator, } from "@chainsafe/lodestar-z/metrics";
|
|
3
3
|
import { hasher } from "@chainsafe/persistent-merkle-tree";
|
|
4
4
|
import { ZERO_HASH_HEX } from "@lodestar/params";
|
|
5
|
-
import { isStatePostBellatrix, isStatePostGloas } from "@lodestar/state-transition";
|
|
5
|
+
import { computeStartSlotAtEpoch, isStatePostBellatrix, isStatePostGloas, } from "@lodestar/state-transition";
|
|
6
6
|
import { sleep, toRootHex } from "@lodestar/utils";
|
|
7
7
|
import { BeaconRestApiServer, getApi } from "../api/index.js";
|
|
8
|
+
import { getDataColumnSidecarsMinEpoch } from "../chain/archiveStore/utils/archiveBlocks.js";
|
|
8
9
|
import { BeaconChain, initBeaconMetrics } from "../chain/index.js";
|
|
9
10
|
import { createValidatorMonitor } from "../chain/validatorMonitor.js";
|
|
10
11
|
import { initializeExecutionBuilder, initializeExecutionEngine } from "../execution/index.js";
|
|
@@ -112,6 +113,13 @@ export class BeaconNode {
|
|
|
112
113
|
: null)
|
|
113
114
|
: null;
|
|
114
115
|
const clock = new Clock({ config, genesisTime: anchorState.genesisTime, signal });
|
|
116
|
+
// Before any lookup into the legacy column buckets, which can be slow until compacted
|
|
117
|
+
const columnsMinEpoch = opts.chain.archiveDataEpochs === Infinity
|
|
118
|
+
? null
|
|
119
|
+
: getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
|
|
120
|
+
await db.compactLegacyDataColumns(columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
|
|
121
|
+
? computeStartSlotAtEpoch(columnsMinEpoch)
|
|
122
|
+
: 0);
|
|
115
123
|
await db.init();
|
|
116
124
|
// Prune hot db repos
|
|
117
125
|
// TODO: Should this call be awaited?
|
package/lib/node/nodejs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nodejs.js","sourceRoot":"","sources":["../../src/node/nodejs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,aAAa,CAAC;AAG5C,OAAO,EACL,IAAI,IAAI,iBAAiB,EACzB,sBAAsB,IAAI,4BAA4B,EACtD,aAAa,IAAI,mBAAmB,EACpC,wBAAwB,IAAI,8BAA8B,GAC3D,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAC,MAAM,EAAC,MAAM,mCAAmC,CAAC;AAIzD,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,
|
|
1
|
+
{"version":3,"file":"nodejs.js","sourceRoot":"","sources":["../../src/node/nodejs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,aAAa,CAAC;AAG5C,OAAO,EACL,IAAI,IAAI,iBAAiB,EACzB,sBAAsB,IAAI,4BAA4B,EACtD,aAAa,IAAI,mBAAmB,EACpC,wBAAwB,IAAI,8BAA8B,GAC3D,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAC,MAAM,EAAC,MAAM,mCAAmC,CAAC;AAIzD,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAEL,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAC,KAAK,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAC,mBAAmB,EAAE,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAC,6BAA6B,EAAC,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAC,WAAW,EAAgB,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAmB,sBAAsB,EAAC,MAAM,8BAA8B,CAAC;AAEtF,OAAO,EAAC,0BAA0B,EAAE,yBAAyB,EAAC,MAAM,uBAAuB,CAAC;AAC5F,OAAO,EAA6B,aAAa,EAAE,oBAAoB,EAAC,MAAM,qBAAqB,CAAC;AACpG,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,OAAO,EAAE,kBAAkB,EAAC,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAC,UAAU,EAAc,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAC,KAAK,EAAC,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAC,mCAAmC,EAAC,MAAM,qCAAqC,CAAC;AACxF,OAAO,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAG9C,cAAc,cAAc,CAAC;AAmC7B,MAAM,CAAN,IAAY,gBAIX;AAJD,WAAY,gBAAgB;IAC1B,uCAAmB,CAAA;IACnB,uCAAmB,CAAA;IACnB,qCAAiB,CAAA;AACnB,CAAC,EAJW,gBAAgB,KAAhB,gBAAgB,QAI3B;AAED,IAAK,YAWJ;AAXD,WAAK,YAAY;IACf,2BAAW,CAAA;IACX,+BAAe,CAAA;IACf,uCAAuB,CAAA;IACvB,mCAAmB,CAAA;IACnB,yCAAyB,CAAA;IACzB,mCAAmB,CAAA;IACnB,wBAAwB;IACxB,6BAAa,CAAA;IACb,6BAAa,CAAA;IACb,6BAAa,CAAA;AACf,CAAC,EAXI,YAAY,KAAZ,YAAY,QAWhB;AAED;;;GAGG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAEvC;;;GAGG;AACH,MAAM,OAAO,UAAU;IACrB,IAAI,CAAqB;IACzB,MAAM,CAAe;IACrB,EAAE,CAAY;IACd,OAAO,CAAiB;IACxB,aAAa,CAA2B;IACxC,UAAU,CAA2B;IACrC,gBAAgB,CAA0B;IAC1C,OAAO,CAAU;IACjB,KAAK,CAAe;IACpB,GAAG,CAAmB;IACtB,OAAO,CAAuB;IAC9B,IAAI,CAAc;IAElB,MAAM,CAAmB;IACjB,UAAU,CAAmB;IAErC,YAAY,EACV,IAAI,EACJ,MAAM,EACN,EAAE,EACF,OAAO,EACP,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,IAAI,EACJ,UAAU,GACQ;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAE7B,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAoC,EACnD,IAAI,EACJ,MAAM,EACN,WAAW,EACX,EAAE,EACF,MAAM,EACN,uBAAuB,EACvB,UAAU,EACV,OAAO,EACP,aAAa,EACb,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,GAAG,EAAE,GACA;QACtB,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,2CAA2C,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,8FAA8F;QAC9F,oDAAoD;QACpD,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;QAEjC,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IACE,IAAI,CAAC,OAAO,CAAC,OAAO;YACpB,oCAAoC;YACpC,IAAI,CAAC,UAAU,CAAC,QAAQ,EACxB,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC;gBACrC,iBAAiB,EAAE,CAAC;YACtB,CAAC;YAED,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,WAAW,EAAE,iBAAiB,EAAE;gBAChF,6BAA6B,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB;aACjE,CAAC,CAAC;YACH,iBAAiB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YACxC,yFAAyF;YACzF,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,gBAAgB,GACpB,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,gBAAgB,CAAC,oBAAoB;YAChE,CAAC,CAAC,sBAAsB,CACpB,OAAO,EAAE,QAAQ,IAAI,IAAI,EACzB,MAAM,EACN,WAAW,CAAC,WAAW,EACvB,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,IAAI,EAAC,CAAC,EACzC,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBAC9B,CAAC,CAAC;oBACE,sBAAsB,EAAE,4BAA4B;oBACpD,wBAAwB,EAAE,8BAA8B;iBACzD;gBACH,CAAC,CAAC,IAAI,CACT;YACH,CAAC,CAAC,IAAI,CAAC;QAEX,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,EAAC,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE,MAAM,EAAC,CAAC,CAAC;QAEhF,sFAAsF;QACtF,MAAM,eAAe,GACnB,IAAI,CAAC,KAAK,CAAC,iBAAiB,KAAK,QAAQ;YACvC,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,6BAA6B,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC9F,MAAM,EAAE,CAAC,wBAAwB,CAC/B,eAAe,KAAK,IAAI,IAAI,eAAe,IAAI,MAAM,CAAC,eAAe;YACnE,CAAC,CAAC,uBAAuB,CAAC,eAAe,CAAC;YAC1C,CAAC,CAAC,CAAC,CACN,CAAC;QAEF,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QAEhB,qBAAqB;QACrB,qCAAqC;QACrC,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QAEtB,8DAA8D;QAC9D,MAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,EAAE,CAAC,wBAAwB,EAAE;aAC1B,IAAI,CAAC,GAAG,EAAE;YACT,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAC,CAAC,CAAC;QACrF,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEL,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ;YACzC,CAAC,CAAC,IAAI,iBAAiB,CACnB,QAAQ,EACR,EAAC,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAC,EACxD,EAAC,QAAQ,EAAG,OAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,UAAU,EAAC,CAAC,EAAC,CACnG;YACH,CAAC,CAAC,IAAI,CAAC;QAET,IAAI,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAC/C,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzC,MAAM,mBAAmB,GACvB,oBAAoB,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,oBAAoB;gBACnE,CAAC,CAAC,gBAAgB,CAAC,WAAW,CAAC;oBAC7B,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,eAAe,CAAC;oBACxC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,4BAA4B,CAAC,SAAS,CAAC;gBACjE,CAAC,CAAC,SAAS,CAAC;YAChB,mBAAmB,GAAG;gBACpB,GAAG,IAAI,CAAC,eAAe;gBACvB,gBAAgB,EAAE,aAAa;gBAC/B,aAAa,EAAE,IAAI,CAAC,eAAe,CAAC,aAAa,IAAI,mBAAmB;gBACxE,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,MAAM;aACP,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE;YACxC,UAAU;YACV,MAAM;YACN,KAAK;YACL,WAAW;YACX,OAAO;YACP,aAAa;YACb,EAAE;YACF,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI;YACpB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,KAAK,EAAC,CAAC;YAClD,uBAAuB;YACvB,OAAO;YACP,gBAAgB;YAChB,WAAW;YACX,sBAAsB;YACtB,qBAAqB;YACrB,eAAe,EAAE,yBAAyB,CAAC,mBAAmB,EAAE;gBAC9D,OAAO;gBACP,MAAM;gBACN,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,SAAS,EAAC,CAAC;aACvD,CAAC;YACF,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO;gBAC7C,CAAC,CAAC,0BAA0B,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;gBAC5E,CAAC,CAAC,SAAS;YACb,oBAAoB,EAAE;gBACpB,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO;gBACtC,2EAA2E;gBAC3E,SAAS,EAAE,IAAI,CAAC,gBAAgB,CAAC,SAAS;aAC3C;SACF,CAAC,CAAC;QAEH,oDAAoD;QACpD,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,kDAAkD;QAClD,wDAAwD;QACxD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACjC,IAAI,EAAE,IAAI,CAAC,OAAO;YAClB,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,OAAO,EAAC,CAAC;YACpD,OAAO;YACP,KAAK;YACL,EAAE;YACF,UAAU;YACV,YAAY;YACZ,iBAAiB,EAAE,kBAAkB,CAAC,EAAC,EAAE,EAAE,KAAK,EAAC,CAAC;SACnD,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE;YACrC,MAAM;YACN,EAAE;YACF,KAAK;YACL,OAAO;YACP,OAAO;YACP,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,IAAI,EAAC,CAAC;SAClD,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,GAAG,EAAC,CAAC;YAChD,EAAE;YACF,IAAI;YACJ,OAAO;YACP,KAAK;YACL,OAAO;SACR,CAAC,CAAC;QAEH,sDAAsD;QACtD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;YACxC,CAAC,CAAC,MAAM,oBAAoB,CAAC,IAAI,CAAC,OAAO,EAAE;gBACvC,QAAQ,EAAG,OAAmB,CAAC,QAAQ;gBACvC,eAAe,EAAE,KAAK,IAAI,EAAE;oBAC1B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;oBACtG,IAAI,IAAI,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC;wBACrC,IAAI,CAAC;4BACH,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;wBAC3C,CAAC;wBAAC,OAAO,CAAC,EAAE,CAAC;4BACX,MAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE,EAAE,EAAE,CAAU,CAAC,CAAC;wBAClF,CAAC;oBACH,CAAC;oBACD,OAAO,YAAY,CAAC;gBACtB,CAAC;gBACD,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,OAAO,EAAC,CAAC;aACrD,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;YACrD,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,MAAM,EAAE,YAAY,CAAC,IAAI,EAAC,CAAC;YACjD,GAAG;YACH,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;SAC1C,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/C,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;QACzB,CAAC;QAED,KAAK,eAAe,CAAC,EAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAC,CAAC,CAAC;QAErE,mCAAmC,CAAC,EAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAC,CAAC,CAAC;QAEtE,OAAO,IAAI,UAAU,CAAC;YACpB,IAAI;YACJ,MAAM;YACN,EAAE;YACF,OAAO;YACP,aAAa;YACb,UAAU;YACV,gBAAgB;YAChB,OAAO;YACP,KAAK;YACL,GAAG;YACH,OAAO;YACP,IAAI;YACJ,UAAU;SACX,CAAM,CAAC;IACV,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,KAAK,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC;YACvC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,aAAa;gBAAE,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;YACzD,IAAI,IAAI,CAAC,UAAU;gBAAE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACnD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACzB,4EAA4E;YAC5E,gFAAgF;YAChF,yFAAyF;YACzF,IAAI,IAAI,CAAC,UAAU;gBAAE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,KAAK,CAAC,0BAA0B,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;QACxC,CAAC;IACH,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"bugs": {
|
|
12
12
|
"url": "https://github.com/ChainSafe/lodestar/issues"
|
|
13
13
|
},
|
|
14
|
-
"version": "1.51.0-dev.
|
|
14
|
+
"version": "1.51.0-dev.ec1447da07",
|
|
15
15
|
"type": "module",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": {
|
|
@@ -127,17 +127,17 @@
|
|
|
127
127
|
"@libp2p/prometheus-metrics": "^5.0.31",
|
|
128
128
|
"@libp2p/tcp": "^11.0.30",
|
|
129
129
|
"@libp2p/utils": "^7.4.3",
|
|
130
|
-
"@lodestar/api": "^1.51.0-dev.
|
|
131
|
-
"@lodestar/config": "^1.51.0-dev.
|
|
132
|
-
"@lodestar/db": "^1.51.0-dev.
|
|
133
|
-
"@lodestar/fork-choice": "^1.51.0-dev.
|
|
134
|
-
"@lodestar/logger": "^1.51.0-dev.
|
|
135
|
-
"@lodestar/params": "^1.51.0-dev.
|
|
136
|
-
"@lodestar/reqresp": "^1.51.0-dev.
|
|
137
|
-
"@lodestar/state-transition": "^1.51.0-dev.
|
|
138
|
-
"@lodestar/types": "^1.51.0-dev.
|
|
139
|
-
"@lodestar/utils": "^1.51.0-dev.
|
|
140
|
-
"@lodestar/validator": "^1.51.0-dev.
|
|
130
|
+
"@lodestar/api": "^1.51.0-dev.ec1447da07",
|
|
131
|
+
"@lodestar/config": "^1.51.0-dev.ec1447da07",
|
|
132
|
+
"@lodestar/db": "^1.51.0-dev.ec1447da07",
|
|
133
|
+
"@lodestar/fork-choice": "^1.51.0-dev.ec1447da07",
|
|
134
|
+
"@lodestar/logger": "^1.51.0-dev.ec1447da07",
|
|
135
|
+
"@lodestar/params": "^1.51.0-dev.ec1447da07",
|
|
136
|
+
"@lodestar/reqresp": "^1.51.0-dev.ec1447da07",
|
|
137
|
+
"@lodestar/state-transition": "^1.51.0-dev.ec1447da07",
|
|
138
|
+
"@lodestar/types": "^1.51.0-dev.ec1447da07",
|
|
139
|
+
"@lodestar/utils": "^1.51.0-dev.ec1447da07",
|
|
140
|
+
"@lodestar/validator": "^1.51.0-dev.ec1447da07",
|
|
141
141
|
"@multiformats/multiaddr": "^13.0.1",
|
|
142
142
|
"datastore-core": "^12.0.1",
|
|
143
143
|
"datastore-level": "^13.0.1",
|
|
@@ -159,8 +159,8 @@
|
|
|
159
159
|
"@ethereumjs/rlp": "^4.0.1",
|
|
160
160
|
"@libp2p/interface-internal": "^3.2.0",
|
|
161
161
|
"@libp2p/logger": "^6.2.15",
|
|
162
|
-
"@lodestar/spec-test-util": "^1.51.0-dev.
|
|
163
|
-
"@lodestar/test-utils": "^1.51.0-dev.
|
|
162
|
+
"@lodestar/spec-test-util": "^1.51.0-dev.ec1447da07",
|
|
163
|
+
"@lodestar/test-utils": "^1.51.0-dev.ec1447da07",
|
|
164
164
|
"@types/js-yaml": "^4.0.5",
|
|
165
165
|
"@types/qs": "^6.9.7",
|
|
166
166
|
"@types/tmp": "^0.2.3",
|
|
@@ -175,5 +175,5 @@
|
|
|
175
175
|
"beacon",
|
|
176
176
|
"blockchain"
|
|
177
177
|
],
|
|
178
|
-
"gitHead": "
|
|
178
|
+
"gitHead": "091ae08d4c37038451f8fcd05fb0e9ca98fdfd4a"
|
|
179
179
|
}
|
|
@@ -56,6 +56,17 @@ async function persistOrphanedBlock(
|
|
|
56
56
|
await writeIfNotExist(filepath, bytes);
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
/**
|
|
60
|
+
* Oldest epoch for which data column sidecars are kept
|
|
61
|
+
*/
|
|
62
|
+
export function getDataColumnSidecarsMinEpoch(
|
|
63
|
+
config: ChainForkConfig,
|
|
64
|
+
currentEpoch: Epoch,
|
|
65
|
+
archiveDataEpochs?: number
|
|
66
|
+
): Epoch {
|
|
67
|
+
return Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
|
|
68
|
+
}
|
|
69
|
+
|
|
59
70
|
/**
|
|
60
71
|
* Archives finalized blocks from active bucket to archive bucket.
|
|
61
72
|
*
|
|
@@ -260,10 +271,7 @@ export async function archiveBlocks(
|
|
|
260
271
|
// Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
|
|
261
272
|
if (finalizedPostFulu) {
|
|
262
273
|
if (archiveDataEpochs !== Infinity) {
|
|
263
|
-
const dataColumnSidecarsMinEpoch =
|
|
264
|
-
getBlobDataRetentionStartEpoch(config, currentEpoch),
|
|
265
|
-
currentEpoch - (archiveDataEpochs ?? 0)
|
|
266
|
-
);
|
|
274
|
+
const dataColumnSidecarsMinEpoch = getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs);
|
|
267
275
|
if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
|
|
268
276
|
const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
|
|
269
277
|
const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
IBeaconStateView,
|
|
22
22
|
RootCache,
|
|
23
23
|
computeEpochAtSlot,
|
|
24
|
-
|
|
24
|
+
computeTimeAtSlotMs,
|
|
25
25
|
getBeaconCommittees,
|
|
26
26
|
isStatePostAltair,
|
|
27
27
|
isStatePostBellatrix,
|
|
@@ -105,8 +105,9 @@ export async function importBlock(
|
|
|
105
105
|
const currentEpoch = computeEpochAtSlot(currentSlot);
|
|
106
106
|
const blockEpoch = computeEpochAtSlot(blockSlot);
|
|
107
107
|
const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
|
|
108
|
-
const
|
|
109
|
-
fullyVerifiedBlock.seenTimestampSec
|
|
108
|
+
const receiveDelayMs =
|
|
109
|
+
Math.round(fullyVerifiedBlock.seenTimestampSec * 1000) -
|
|
110
|
+
computeTimeAtSlotMs(this.config, blockSlot, postState.genesisTime * 1000);
|
|
110
111
|
const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
|
|
111
112
|
const fork = this.config.getForkSeq(blockSlot);
|
|
112
113
|
|
|
@@ -148,12 +149,12 @@ export async function importBlock(
|
|
|
148
149
|
executionStatus = parentBlock.executionStatus;
|
|
149
150
|
}
|
|
150
151
|
|
|
151
|
-
const
|
|
152
|
+
const importDelayMs = this.clock.msFromSlot(blockSlot);
|
|
152
153
|
const blockSummary = this.forkChoice.onBlock(
|
|
153
154
|
block.message,
|
|
154
155
|
postState,
|
|
155
|
-
|
|
156
|
-
|
|
156
|
+
receiveDelayMs,
|
|
157
|
+
importDelayMs,
|
|
157
158
|
currentSlot,
|
|
158
159
|
executionStatus,
|
|
159
160
|
dataAvailabilityStatus
|
|
@@ -417,7 +418,7 @@ export async function importBlock(
|
|
|
417
418
|
// 4) Block meets the criteria of being re-orged out (this is also checked inside the function)
|
|
418
419
|
const result = this.forkChoice.shouldOverrideForkChoiceUpdate(
|
|
419
420
|
blockSummary,
|
|
420
|
-
this.clock.
|
|
421
|
+
this.clock.msFromSlot(currentSlot),
|
|
421
422
|
currentSlot
|
|
422
423
|
);
|
|
423
424
|
shouldOverrideFcu = result.shouldOverrideFcu;
|
package/src/chain/chain.ts
CHANGED
|
@@ -1486,10 +1486,10 @@ export class BeaconChain implements IBeaconChain {
|
|
|
1486
1486
|
this.metrics?.forkChoice.requests.inc();
|
|
1487
1487
|
const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.predictProposerHead});
|
|
1488
1488
|
const slot = this.clock.currentSlot;
|
|
1489
|
-
const
|
|
1489
|
+
const msFromSlot = this.clock.msFromSlot(slot);
|
|
1490
1490
|
|
|
1491
1491
|
try {
|
|
1492
|
-
return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead,
|
|
1492
|
+
return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead, msFromSlot, slot}).head;
|
|
1493
1493
|
} catch (e) {
|
|
1494
1494
|
this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetPredictedProposerHead});
|
|
1495
1495
|
throw e;
|
|
@@ -1501,13 +1501,13 @@ export class BeaconChain implements IBeaconChain {
|
|
|
1501
1501
|
getProposerHead(slot: Slot): ProtoBlock {
|
|
1502
1502
|
this.metrics?.forkChoice.requests.inc();
|
|
1503
1503
|
const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.getProposerHead});
|
|
1504
|
-
const
|
|
1504
|
+
const msFromSlot = this.clock.msFromSlot(slot);
|
|
1505
1505
|
|
|
1506
1506
|
try {
|
|
1507
1507
|
// Do not emit head event here, when proposing we rely on the one emitted when importing our own block
|
|
1508
1508
|
const {head, isHeadTimely, notReorgedReason} = this.forkChoice.updateAndGetHead({
|
|
1509
1509
|
mode: UpdateHeadOpt.GetProposerHead,
|
|
1510
|
-
|
|
1510
|
+
msFromSlot,
|
|
1511
1511
|
slot,
|
|
1512
1512
|
});
|
|
1513
1513
|
|
package/src/db/beacon.ts
CHANGED
|
@@ -30,6 +30,9 @@ import {
|
|
|
30
30
|
VoluntaryExitRepository,
|
|
31
31
|
} from "./repositories/index.js";
|
|
32
32
|
|
|
33
|
+
/** Below this size, seeks over leftover legacy column data stay cheap */
|
|
34
|
+
const LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES = 1024 ** 3;
|
|
35
|
+
|
|
33
36
|
export type BeaconDbOpts = {
|
|
34
37
|
dataColumnDir: string;
|
|
35
38
|
logger: Logger;
|
|
@@ -67,6 +70,7 @@ export class BeaconDb implements IBeaconDb {
|
|
|
67
70
|
readonly dataColumns: IDataColumnStore;
|
|
68
71
|
lastLegacyArchiveSlot: Slot | null = null;
|
|
69
72
|
private readonly flatFileStore: FlatFileStore;
|
|
73
|
+
private readonly logger: Logger;
|
|
70
74
|
|
|
71
75
|
constructor(
|
|
72
76
|
config: ChainForkConfig,
|
|
@@ -100,6 +104,7 @@ export class BeaconDb implements IBeaconDb {
|
|
|
100
104
|
this.syncCommittee = new SyncCommitteeRepository(config, db);
|
|
101
105
|
this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
|
|
102
106
|
|
|
107
|
+
this.logger = opts.logger;
|
|
103
108
|
this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
|
|
104
109
|
this.dataColumns = new LegacyDataColumnStore(
|
|
105
110
|
this.flatFileStore,
|
|
@@ -125,6 +130,40 @@ export class BeaconDb implements IBeaconDb {
|
|
|
125
130
|
this.flatFileStore.setMetrics(flatFileStoreMetrics);
|
|
126
131
|
}
|
|
127
132
|
|
|
133
|
+
/**
|
|
134
|
+
* Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
|
|
135
|
+
* rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
|
|
136
|
+
* makes the compaction overlap those files and rewrite them.
|
|
137
|
+
*/
|
|
138
|
+
async compactLegacyDataColumns(
|
|
139
|
+
prunedBeforeSlot: Slot,
|
|
140
|
+
minSizeBytes = LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES
|
|
141
|
+
): Promise<void> {
|
|
142
|
+
const archiveBucket = Bucket.allForks_dataColumnSidecarsArchive;
|
|
143
|
+
const start = encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0));
|
|
144
|
+
const end = encodeKey(archiveBucket, this.dataColumnSidecarArchive.getMinKeyRaw(prunedBeforeSlot));
|
|
145
|
+
const sizeBytes = await this.db.approximateSize(start, end);
|
|
146
|
+
if (sizeBytes < minSizeBytes) return;
|
|
147
|
+
|
|
148
|
+
// Column indices stay below 0xffff and hot column keys are shorter, so no sentinel matches a real key
|
|
149
|
+
const upperSentinel =
|
|
150
|
+
prunedBeforeSlot > 0
|
|
151
|
+
? encodeKey(archiveBucket, this.dataColumnSidecarArchive.encodeKeyRaw(prunedBeforeSlot - 1, 0xffff))
|
|
152
|
+
: encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(35, 0xff));
|
|
153
|
+
const sentinels = [start, upperSentinel];
|
|
154
|
+
|
|
155
|
+
this.logger.info("Compacting legacy data columns, this may take a while", {sizeBytes});
|
|
156
|
+
const startTime = Date.now();
|
|
157
|
+
await this.db.batchPut(sentinels.map((key) => ({key, value: new Uint8Array()})));
|
|
158
|
+
await this.db.compactRange(start, end);
|
|
159
|
+
await this.db.batchDelete(sentinels);
|
|
160
|
+
await this.db.compactRange(start, end);
|
|
161
|
+
this.logger.verbose("Compacted legacy data columns", {
|
|
162
|
+
durationMs: Date.now() - startTime,
|
|
163
|
+
sizeBytes: await this.db.approximateSize(start, end),
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
128
167
|
async pruneHotDb(): Promise<void> {
|
|
129
168
|
// Prune all hot blobs
|
|
130
169
|
await this.blobSidecars.batchDelete(await this.blobSidecars.keys());
|
|
@@ -13,7 +13,7 @@ export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
|
|
|
13
13
|
type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
|
|
14
14
|
type LegacyArchiveDataColumnStore = Pick<
|
|
15
15
|
DataColumnSidecarArchiveRepository,
|
|
16
|
-
"values" | "getManyBinary" | "
|
|
16
|
+
"values" | "getManyBinary" | "pruneBefore"
|
|
17
17
|
>;
|
|
18
18
|
type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
|
|
19
19
|
|
|
@@ -86,9 +86,7 @@ export class LegacyDataColumnStore implements IDataColumnStore {
|
|
|
86
86
|
|
|
87
87
|
async pruneBefore(slot: Slot): Promise<Slot[]> {
|
|
88
88
|
const flatFileSlots = await this.flatFiles.pruneBefore(slot);
|
|
89
|
-
const
|
|
90
|
-
const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
|
|
91
|
-
if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
|
|
89
|
+
const slots = await this.legacyArchive.pruneBefore(slot);
|
|
92
90
|
return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
|
|
93
91
|
}
|
|
94
92
|
}
|
package/src/db/interface.ts
CHANGED
|
@@ -68,6 +68,8 @@ export interface IBeaconDb {
|
|
|
68
68
|
readonly lastLegacyArchiveSlot: Slot | null;
|
|
69
69
|
init(): Promise<void>;
|
|
70
70
|
|
|
71
|
+
compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
|
|
72
|
+
|
|
71
73
|
pruneHotDb(): Promise<void>;
|
|
72
74
|
|
|
73
75
|
deleteDeprecatedEth1Data(): Promise<void>;
|