envio 3.13.0 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +10 -1
- package/index.d.ts +11 -0
- package/package.json +8 -20
- package/rescript.json +1 -4
- package/src/Config.res +69 -100
- package/src/Config.res.mjs +50 -63
- package/src/Core.res +46 -6
- package/src/Core.res.mjs +41 -4
- package/src/EventConfigBuilder.res +4 -10
- package/src/EventConfigBuilder.res.mjs +1 -3
- package/src/ExitOnCaughtUp.res +5 -1
- package/src/ExitOnCaughtUp.res.mjs +8 -3
- package/src/HandlerLoader.res +74 -32
- package/src/HandlerLoader.res.mjs +69 -23
- package/src/HandlerTypeCheck.mjs +14 -0
- package/src/HandlerTypeCheckWorker.mjs +68 -0
- package/src/Internal.res +1 -1
- package/src/Main.res +17 -12
- package/src/Main.res.mjs +9 -3
- package/src/NodeVersion.mjs +13 -0
- package/src/Persistence.res +79 -38
- package/src/Persistence.res.mjs +41 -10
- package/src/PgStorage.res +62 -34
- package/src/PgStorage.res.mjs +65 -31
- package/src/ResumePlan.res +172 -0
- package/src/ResumePlan.res.mjs +122 -0
- package/src/Supervisor.res +8 -8
- package/src/Supervisor.res.mjs +6 -1
- package/src/TestIndexer.res +14 -4
- package/src/TestIndexer.res.mjs +3 -2
- package/src/TsModuleHooks.mjs +121 -0
- package/src/Utils.res +0 -8
- package/src/Worker.res +8 -0
- package/src/Worker.res.mjs +11 -0
- package/src/bindings/NodeJs.res +8 -7
- package/src/bindings/NodeJs.res.mjs +0 -3
- package/src/bindings/Pino.res +2 -2
- package/src/db/InternalTable.res +52 -2
- package/src/db/InternalTable.res.mjs +32 -3
- package/src/sources/FuelHyperSync.res +2 -0
- package/src/sources/FuelHyperSync.res.mjs +1 -0
- package/src/sources/FuelHyperSync.resi +1 -0
- package/src/sources/FuelHyperSyncClient.res +25 -8
- package/src/sources/FuelHyperSyncClient.res.mjs +8 -2
- package/src/sources/FuelHyperSyncSource.res +24 -22
- package/src/sources/FuelHyperSyncSource.res.mjs +18 -13
- package/src/tui/InitApi.res +85 -0
- package/src/tui/{components/CustomHooks.res.mjs → InitApi.res.mjs} +11 -64
- package/src/tui/Tui.res +142 -249
- package/src/tui/Tui.res.mjs +123 -332
- package/src/Js.shim.ts +0 -11
- package/src/bindings/DateFns.res +0 -74
- package/src/bindings/DateFns.res.mjs +0 -22
- package/src/sources/FuelSDK.res +0 -38
- package/src/sources/FuelSDK.res.mjs +0 -27
- package/src/sources/vendored-fuel-abi-coder.js +0 -1792
- package/src/tui/bindings/Ink.res +0 -369
- package/src/tui/bindings/Ink.res.mjs +0 -66
- package/src/tui/bindings/Style.res +0 -123
- package/src/tui/bindings/Style.res.mjs +0 -2
- package/src/tui/components/BufferedProgressBar.res +0 -40
- package/src/tui/components/BufferedProgressBar.res.mjs +0 -57
- package/src/tui/components/CustomHooks.res +0 -122
- package/src/tui/components/Messages.res +0 -40
- package/src/tui/components/Messages.res.mjs +0 -71
- package/src/tui/components/SyncETA.res +0 -179
- package/src/tui/components/SyncETA.res.mjs +0 -249
- package/src/tui/components/TuiData.res +0 -98
- package/src/tui/components/TuiData.res.mjs +0 -73
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
const MINIMUM = [22, 15, 0];
|
|
2
|
+
|
|
3
|
+
// The message for a Node.js older than envio supports, or undefined.
|
|
4
|
+
export const unsupportedNodeMessage = () => {
|
|
5
|
+
const current = process.versions.node;
|
|
6
|
+
const parts = current.split(".").map(Number);
|
|
7
|
+
const index = MINIMUM.findIndex((minimum, i) => parts[i] !== minimum);
|
|
8
|
+
if (index === -1 || parts[index] > MINIMUM[index]) return undefined;
|
|
9
|
+
return [
|
|
10
|
+
`envio needs Node.js ${MINIMUM.join(".")} or newer, and this is Node.js ${current}.`,
|
|
11
|
+
"Install a newer one from https://nodejs.org/en/download, or with your version manager, for example: nvm install --lts",
|
|
12
|
+
].join("\n");
|
|
13
|
+
};
|
package/src/Persistence.res
CHANGED
|
@@ -39,9 +39,6 @@ type initialState = {
|
|
|
39
39
|
// On a resume this is what the database holds, not what the config would
|
|
40
40
|
// derive — the ids must never reshuffle under stored rows.
|
|
41
41
|
contractMapping: ContractMapping.t,
|
|
42
|
-
// Public config snapshot, restored with the address rows. None when
|
|
43
|
-
// envio_info or envio_contracts is missing.
|
|
44
|
-
envioInfo: option<JSON.t>,
|
|
45
42
|
cache: dict<effectCacheRecord>,
|
|
46
43
|
chains: array<initialChainState>,
|
|
47
44
|
// Where each chain's checkpoint sequence stands in the database.
|
|
@@ -123,20 +120,25 @@ type storage = {
|
|
|
123
120
|
~contractMapping: ContractMapping.t,
|
|
124
121
|
~envioInfo: JSON.t,
|
|
125
122
|
) => promise<initialState>,
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
|
|
123
|
+
// Read before anything is resumed, so a config the stored one rules out is
|
|
124
|
+
// reported as such rather than as a sink tripping over tables it never
|
|
125
|
+
// created.
|
|
126
|
+
readStoredConfig: unit => promise<ResumePlan.stored>,
|
|
127
|
+
// Brings an initialized storage a chain it doesn't have yet: its row, its
|
|
128
|
+
// partitions and its config addresses, as `initialize` would have created
|
|
129
|
+
// them. The chains already there are left untouched.
|
|
130
|
+
addChain: (
|
|
131
|
+
~chainConfig: Config.chain,
|
|
132
|
+
~entities: array<Internal.entityConfig>,
|
|
133
|
+
~contractMapping: ContractMapping.t,
|
|
134
|
+
) => promise<unit>,
|
|
130
135
|
// `chainIds` is what this run drives. An isolated run resumes a subset of
|
|
131
136
|
// the stored chains, and the sink's resume trims past each resumed chain's
|
|
132
137
|
// checkpoint, so chains a sibling process is still writing must stay out.
|
|
133
138
|
resumeInitialState: (
|
|
134
139
|
~entities: array<Internal.entityConfig>,
|
|
135
140
|
~chainIds: array<ChainId.t>,
|
|
136
|
-
~
|
|
137
|
-
~storedEnvioInfo: option<JSON.t>,
|
|
138
|
-
~storedContractMapping: ContractMapping.t,
|
|
139
|
-
) => unit,
|
|
141
|
+
~contractMapping: ContractMapping.t,
|
|
140
142
|
) => promise<initialState>,
|
|
141
143
|
// Returns rows matching the filter.
|
|
142
144
|
// Field values are serialized and rows parsed with the table's field schemas.
|
|
@@ -243,6 +245,10 @@ type t = {
|
|
|
243
245
|
|
|
244
246
|
exception StorageError({message: string, reason: exn})
|
|
245
247
|
|
|
248
|
+
// A start the operator has to act on, not a failure: what it says is all they
|
|
249
|
+
// need, so it's printed once, without a stack trace.
|
|
250
|
+
exception Refused(string)
|
|
251
|
+
|
|
246
252
|
let make = (
|
|
247
253
|
~userEntities,
|
|
248
254
|
// TODO: Should only pass userEnums and create internal config in runtime
|
|
@@ -271,10 +277,11 @@ let init = {
|
|
|
271
277
|
~runCommand,
|
|
272
278
|
~reset=false,
|
|
273
279
|
~lowercaseAddresses=false,
|
|
274
|
-
// An
|
|
275
|
-
// would create rows for this process's chains only,
|
|
276
|
-
// skipped with no state for their own processes to
|
|
277
|
-
|
|
280
|
+
// An `envio start --chain` run. It needs the schema to exist already:
|
|
281
|
+
// initializing under it would create rows for this process's chains only,
|
|
282
|
+
// leaving the ones it skipped with no state for their own processes to
|
|
283
|
+
// resume. It is also the one run that may add its chain to the schema.
|
|
284
|
+
~isolated=false,
|
|
278
285
|
// Whether this process is the one that tells the operator the run resumed.
|
|
279
286
|
// A supervisor says it once for the whole run, so the workers it forked
|
|
280
287
|
// keep it to their own log files.
|
|
@@ -297,9 +304,11 @@ let init = {
|
|
|
297
304
|
})
|
|
298
305
|
persistence.storageStatus = Initializing(promise)
|
|
299
306
|
if reset || !(await persistence.storage.isInitialized()) {
|
|
300
|
-
if
|
|
301
|
-
|
|
302
|
-
|
|
307
|
+
if isolated {
|
|
308
|
+
throw(
|
|
309
|
+
Refused(
|
|
310
|
+
"`envio start --chain` needs a database that is already set up. Run `envio local db-migrate up` once with the full config, then start a process per chain.",
|
|
311
|
+
),
|
|
303
312
|
)
|
|
304
313
|
}
|
|
305
314
|
Logging.info(`Initializing the indexer storage...`)
|
|
@@ -330,18 +339,51 @@ let init = {
|
|
|
330
339
|
) {
|
|
331
340
|
let logResume = announceResume ? Logging.info : Logging.trace
|
|
332
341
|
logResume(`Found existing indexer storage. Resuming indexing state...`)
|
|
342
|
+
let stored = await persistence.storage.readStoredConfig()
|
|
343
|
+
switch ResumePlan.make(
|
|
344
|
+
~stored,
|
|
345
|
+
~envioInfo,
|
|
346
|
+
~chainConfigs,
|
|
347
|
+
~contractMapping,
|
|
348
|
+
~lowercaseAddresses,
|
|
349
|
+
~isolated,
|
|
350
|
+
) {
|
|
351
|
+
| Resume => ()
|
|
352
|
+
| Incompatible(changedPaths) =>
|
|
353
|
+
throw(
|
|
354
|
+
Refused(
|
|
355
|
+
ResumePlan.incompatibleMessage(
|
|
356
|
+
changedPaths,
|
|
357
|
+
~envioInfo,
|
|
358
|
+
~resetCommand,
|
|
359
|
+
~runCommand,
|
|
360
|
+
),
|
|
361
|
+
),
|
|
362
|
+
)
|
|
363
|
+
| AddChain(chainConfig) =>
|
|
364
|
+
Logging.info({
|
|
365
|
+
"msg": `Adding the chain to the existing indexer storage...`,
|
|
366
|
+
"chainId": chainConfig.id,
|
|
367
|
+
})
|
|
368
|
+
// The same resolution `initialize` runs, and for the same reason:
|
|
369
|
+
// this is the one time the chain's row is written.
|
|
370
|
+
let chainConfig =
|
|
371
|
+
(
|
|
372
|
+
await [chainConfig]->StartBlockResolver.resolveAllOrThrow(
|
|
373
|
+
~lowercaseAddresses,
|
|
374
|
+
~retry=startBlockRetry,
|
|
375
|
+
)
|
|
376
|
+
)->Array.getUnsafe(0)
|
|
377
|
+
await persistence.storage.addChain(
|
|
378
|
+
~chainConfig,
|
|
379
|
+
~entities=persistence.allEntities,
|
|
380
|
+
~contractMapping=stored.contractMapping,
|
|
381
|
+
)
|
|
382
|
+
}
|
|
333
383
|
let initialState = await persistence.storage.resumeInitialState(
|
|
334
384
|
~entities=persistence.allEntities,
|
|
335
385
|
~chainIds=chainConfigs->Array.map(chain => chain.id),
|
|
336
|
-
~
|
|
337
|
-
Config.throwIfResumeIncompatible(
|
|
338
|
-
~storedEnvioInfo,
|
|
339
|
-
~storedContractMapping,
|
|
340
|
-
~envioInfo,
|
|
341
|
-
~contractMapping,
|
|
342
|
-
~resetCommand,
|
|
343
|
-
~runCommand,
|
|
344
|
-
),
|
|
386
|
+
~contractMapping=stored.contractMapping,
|
|
345
387
|
)
|
|
346
388
|
persistence.storageStatus = Ready(initialState)
|
|
347
389
|
let progress = Dict.make()
|
|
@@ -356,6 +398,7 @@ let init = {
|
|
|
356
398
|
resolveRef.contents()
|
|
357
399
|
}
|
|
358
400
|
} catch {
|
|
401
|
+
| Refused(message) => JsError.throwWithMessage(message)
|
|
359
402
|
| exn => exn->ErrorHandling.mkLogAndRaise(~msg=`Failed to initialize the indexer storage.`)
|
|
360
403
|
}
|
|
361
404
|
}
|
|
@@ -365,23 +408,21 @@ let init = {
|
|
|
365
408
|
// a migration command: what a config change prints names the command the
|
|
366
409
|
// operator ran, and an unreachable chain is waited on rather than reported,
|
|
367
410
|
// since somebody is watching the run come up.
|
|
368
|
-
let initForRun = (
|
|
369
|
-
persistence,
|
|
370
|
-
~config: Config.t,
|
|
371
|
-
~reset,
|
|
372
|
-
~isDevelopmentMode,
|
|
373
|
-
~requireInitialized,
|
|
374
|
-
) =>
|
|
411
|
+
let initForRun = (persistence, ~config: Config.t, ~reset, ~isDevelopmentMode) =>
|
|
375
412
|
persistence->init(
|
|
376
413
|
~announceResume=!Worker.isEnabled,
|
|
377
414
|
~reset,
|
|
378
415
|
~chainConfigs=config.chainMap->ChainMap.values,
|
|
379
416
|
~contractMapping=config.contractMapping,
|
|
380
|
-
~envioInfo=
|
|
381
|
-
~resetCommand=isDevelopmentMode ? "envio dev -r" : "envio start -r"
|
|
382
|
-
|
|
417
|
+
~envioInfo=config.envioInfo,
|
|
418
|
+
~resetCommand=(isDevelopmentMode ? "envio dev -r" : "envio start -r")->Config.withProjectFlags(
|
|
419
|
+
~config,
|
|
420
|
+
),
|
|
421
|
+
~runCommand=Some(
|
|
422
|
+
(isDevelopmentMode ? "envio dev" : "envio start")->Config.withProjectFlags(~config),
|
|
423
|
+
),
|
|
383
424
|
~lowercaseAddresses=config.lowercaseAddresses,
|
|
384
|
-
~
|
|
425
|
+
~isolated=config.isolated,
|
|
385
426
|
)
|
|
386
427
|
|
|
387
428
|
let getInitializedStorageOrThrow = persistence => {
|
package/src/Persistence.res.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import * as Worker from "./Worker.res.mjs";
|
|
|
6
6
|
import * as Logging from "./Logging.res.mjs";
|
|
7
7
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
8
8
|
import * as Frontier from "./db/Frontier.res.mjs";
|
|
9
|
+
import * as ResumePlan from "./ResumePlan.res.mjs";
|
|
9
10
|
import * as EntityHistory from "./db/EntityHistory.res.mjs";
|
|
10
11
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
11
12
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
@@ -22,6 +23,8 @@ function writtenFrontier(batch, rollback) {
|
|
|
22
23
|
|
|
23
24
|
let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
|
|
24
25
|
|
|
26
|
+
let Refused = /* @__PURE__ */Primitive_exceptions.create("Persistence.Refused");
|
|
27
|
+
|
|
25
28
|
function make(userEntities, allEnums, storage) {
|
|
26
29
|
let allEnums$1 = allEnums.concat([EntityHistory.RowAction.config]);
|
|
27
30
|
return {
|
|
@@ -33,10 +36,10 @@ function make(userEntities, allEnums, storage) {
|
|
|
33
36
|
};
|
|
34
37
|
}
|
|
35
38
|
|
|
36
|
-
async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt, lowercaseAddressesOpt,
|
|
39
|
+
async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt, lowercaseAddressesOpt, isolatedOpt, announceResumeOpt, startBlockRetryOpt) {
|
|
37
40
|
let reset = resetOpt !== undefined ? resetOpt : false;
|
|
38
41
|
let lowercaseAddresses = lowercaseAddressesOpt !== undefined ? lowercaseAddressesOpt : false;
|
|
39
|
-
let
|
|
42
|
+
let isolated = isolatedOpt !== undefined ? isolatedOpt : false;
|
|
40
43
|
let announceResume = announceResumeOpt !== undefined ? announceResumeOpt : true;
|
|
41
44
|
let startBlockRetry = startBlockRetryOpt !== undefined ? startBlockRetryOpt : "UntilItAnswers";
|
|
42
45
|
try {
|
|
@@ -64,8 +67,12 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
|
|
|
64
67
|
_0: promise$1
|
|
65
68
|
};
|
|
66
69
|
if (reset || !await persistence.storage.isInitialized()) {
|
|
67
|
-
if (
|
|
68
|
-
|
|
70
|
+
if (isolated) {
|
|
71
|
+
throw {
|
|
72
|
+
RE_EXN_ID: Refused,
|
|
73
|
+
_1: "`envio start --chain` needs a database that is already set up. Run `envio local db-migrate up` once with the full config, then start a process per chain.",
|
|
74
|
+
Error: new Error()
|
|
75
|
+
};
|
|
69
76
|
}
|
|
70
77
|
Logging.info(`Initializing the indexer storage...`);
|
|
71
78
|
let chainConfigs$1 = await StartBlockResolver.resolveAllOrThrow(chainConfigs, lowercaseAddresses, startBlockRetry, undefined, undefined);
|
|
@@ -82,7 +89,26 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
|
|
|
82
89
|
if (tmp) {
|
|
83
90
|
let logResume = announceResume ? Logging.info : Logging.trace;
|
|
84
91
|
logResume(`Found existing indexer storage. Resuming indexing state...`);
|
|
85
|
-
let
|
|
92
|
+
let stored = await persistence.storage.readStoredConfig();
|
|
93
|
+
let changedPaths = ResumePlan.make(stored, envioInfo, chainConfigs, contractMapping, lowercaseAddresses, isolated);
|
|
94
|
+
if (typeof changedPaths === "object") {
|
|
95
|
+
if (changedPaths.TAG === "AddChain") {
|
|
96
|
+
let chainConfig = changedPaths._0;
|
|
97
|
+
Logging.info({
|
|
98
|
+
msg: `Adding the chain to the existing indexer storage...`,
|
|
99
|
+
chainId: chainConfig.id
|
|
100
|
+
});
|
|
101
|
+
let chainConfig$1 = (await StartBlockResolver.resolveAllOrThrow([chainConfig], lowercaseAddresses, startBlockRetry, undefined, undefined))[0];
|
|
102
|
+
await persistence.storage.addChain(chainConfig$1, persistence.allEntities, stored.contractMapping);
|
|
103
|
+
} else {
|
|
104
|
+
throw {
|
|
105
|
+
RE_EXN_ID: Refused,
|
|
106
|
+
_1: ResumePlan.incompatibleMessage(changedPaths._0, envioInfo, resetCommand, runCommand),
|
|
107
|
+
Error: new Error()
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, chainConfigs.map(chain => chain.id), stored.contractMapping);
|
|
86
112
|
persistence.storageStatus = {
|
|
87
113
|
TAG: "Ready",
|
|
88
114
|
_0: initialState$1
|
|
@@ -98,14 +124,18 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
|
|
|
98
124
|
}
|
|
99
125
|
}
|
|
100
126
|
return resolveRef.contents();
|
|
101
|
-
} catch (
|
|
102
|
-
let
|
|
103
|
-
|
|
127
|
+
} catch (raw_message) {
|
|
128
|
+
let message = Primitive_exceptions.internalToException(raw_message);
|
|
129
|
+
if (message.RE_EXN_ID === Refused) {
|
|
130
|
+
return Stdlib_JsError.throwWithMessage(message._1);
|
|
131
|
+
} else {
|
|
132
|
+
return ErrorHandling.mkLogAndRaise(undefined, `Failed to initialize the indexer storage.`, message);
|
|
133
|
+
}
|
|
104
134
|
}
|
|
105
135
|
}
|
|
106
136
|
|
|
107
|
-
function initForRun(persistence, config, reset, isDevelopmentMode
|
|
108
|
-
return init(persistence, ChainMap.values(config.chainMap), config.contractMapping,
|
|
137
|
+
function initForRun(persistence, config, reset, isDevelopmentMode) {
|
|
138
|
+
return init(persistence, ChainMap.values(config.chainMap), config.contractMapping, config.envioInfo, Config.withProjectFlags(isDevelopmentMode ? "envio dev -r" : "envio start -r", config), Config.withProjectFlags(isDevelopmentMode ? "envio dev" : "envio start", config), reset, config.lowercaseAddresses, config.isolated, !Worker.isEnabled, undefined);
|
|
109
139
|
}
|
|
110
140
|
|
|
111
141
|
function getInitializedStorageOrThrow(persistence) {
|
|
@@ -129,6 +159,7 @@ function getInitializedState(persistence) {
|
|
|
129
159
|
export {
|
|
130
160
|
writtenFrontier,
|
|
131
161
|
StorageError,
|
|
162
|
+
Refused,
|
|
132
163
|
make,
|
|
133
164
|
init,
|
|
134
165
|
initForRun,
|
package/src/PgStorage.res
CHANGED
|
@@ -270,11 +270,16 @@ let getEntityHistory = (~entityConfig: Internal.entityConfig): EntityHistory.pgE
|
|
|
270
270
|
}
|
|
271
271
|
}
|
|
272
272
|
|
|
273
|
+
let makeCreatePartitionQuery = (entityConfig: Internal.entityConfig, ~pgSchema, ~chainId) =>
|
|
274
|
+
`CREATE TABLE IF NOT EXISTS "${pgSchema}"."${partitionTableName(
|
|
275
|
+
~entityConfig,
|
|
276
|
+
~chainId,
|
|
277
|
+
)}" PARTITION OF "${pgSchema}"."${entityConfig.table.tableName}" FOR VALUES IN (${chainId->ChainId.toString});`
|
|
278
|
+
|
|
273
279
|
// Every table an entity needs: its own, one partition per chain when it's
|
|
274
|
-
// per-chain, and its history table.
|
|
275
|
-
//
|
|
276
|
-
//
|
|
277
|
-
// here, at init.
|
|
280
|
+
// per-chain, and its history table. A chain the schema gains later gets its
|
|
281
|
+
// partitions from `addChain`, which the resume compat check allows only for an
|
|
282
|
+
// `envio start --chain` process naming it.
|
|
278
283
|
//
|
|
279
284
|
// History stays unpartitioned: it is only ever read by checkpoint, never by
|
|
280
285
|
// chain, so partitioning it would route every write and prune nothing.
|
|
@@ -294,12 +299,7 @@ let makeCreateEntityTableQueries = (
|
|
|
294
299
|
[
|
|
295
300
|
entityConfig.table->createTable(~partitionByColumn=chainIdField->Table.getPgDbFieldName),
|
|
296
301
|
]->Array.concat(
|
|
297
|
-
chainIds->Array.map(chainId =>
|
|
298
|
-
`CREATE TABLE IF NOT EXISTS "${pgSchema}"."${partitionTableName(
|
|
299
|
-
~entityConfig,
|
|
300
|
-
~chainId,
|
|
301
|
-
)}" PARTITION OF "${pgSchema}"."${entityConfig.table.tableName}" FOR VALUES IN (${chainId->ChainId.toString});`
|
|
302
|
-
),
|
|
302
|
+
chainIds->Array.map(chainId => entityConfig->makeCreatePartitionQuery(~pgSchema, ~chainId)),
|
|
303
303
|
)
|
|
304
304
|
}->Array.concat([getEntityHistory(~entityConfig).table->createTable])
|
|
305
305
|
}
|
|
@@ -2009,7 +2009,6 @@ let make = (
|
|
|
2009
2009
|
cache,
|
|
2010
2010
|
reorgCheckpoints: [],
|
|
2011
2011
|
contractMapping,
|
|
2012
|
-
envioInfo: Some(envioInfo),
|
|
2013
2012
|
chains: chainConfigs->Array.mapWithIndex((
|
|
2014
2013
|
chainConfig,
|
|
2015
2014
|
idx,
|
|
@@ -2437,17 +2436,59 @@ let make = (
|
|
|
2437
2436
|
}
|
|
2438
2437
|
}
|
|
2439
2438
|
|
|
2439
|
+
let readStoredConfig = async (): ResumePlan.stored => {
|
|
2440
|
+
let (envioInfo, contractNames) = await Promise.all2((
|
|
2441
|
+
InternalTable.EnvioInfo.read(sql, ~pgSchema),
|
|
2442
|
+
InternalTable.EnvioContracts.read(sql, ~pgSchema),
|
|
2443
|
+
))
|
|
2444
|
+
// Both tables join the schema in one transaction. A missing mapping means
|
|
2445
|
+
// an older envio wrote this schema, so treat the record as unreadable
|
|
2446
|
+
// rather than decoding address rows against ids nothing assigned.
|
|
2447
|
+
switch (envioInfo, contractNames) {
|
|
2448
|
+
| (Some(envioInfo), Some(contractNames)) => {
|
|
2449
|
+
envioInfo: Some(envioInfo),
|
|
2450
|
+
chains: await InternalTable.Chains.readStoredChains(sql, ~pgSchema),
|
|
2451
|
+
contractMapping: ContractMapping.fromStoredNames(contractNames),
|
|
2452
|
+
}
|
|
2453
|
+
| _ => {envioInfo: None, chains: [], contractMapping: ContractMapping.empty}
|
|
2454
|
+
}
|
|
2455
|
+
}
|
|
2456
|
+
|
|
2457
|
+
let addChain = async (~chainConfig: Config.chain, ~entities, ~contractMapping) => {
|
|
2458
|
+
let partitionQueries =
|
|
2459
|
+
entities
|
|
2460
|
+
->Array.filter((entityConfig: Internal.entityConfig) =>
|
|
2461
|
+
entityConfig.storage.postgres && entityConfig.table->Table.getChainIdField->Option.isSome
|
|
2462
|
+
)
|
|
2463
|
+
->Array.map(entityConfig =>
|
|
2464
|
+
entityConfig->makeCreatePartitionQuery(~pgSchema, ~chainId=chainConfig.id)
|
|
2465
|
+
)
|
|
2466
|
+
let addressRows = chainConfig->ChainState.configStorageRows(~ecosystem, ~contractMapping)
|
|
2467
|
+
// No conflict clauses: two processes adding the same chain are two
|
|
2468
|
+
// processes driving it, and the second one fails on a primary key instead
|
|
2469
|
+
// of indexing alongside the first.
|
|
2470
|
+
let _ = await sql->Postgres.beginSql(async sql => {
|
|
2471
|
+
for idx in 0 to partitionQueries->Array.length - 1 {
|
|
2472
|
+
await sql
|
|
2473
|
+
->Postgres.unsafe(partitionQueries->Array.getUnsafe(idx))
|
|
2474
|
+
->Utils.Promise.ignoreValue
|
|
2475
|
+
}
|
|
2476
|
+
if addressRows->Utils.Array.notEmpty {
|
|
2477
|
+
await InternalTable.EnvioAddresses.insert(sql, ~pgSchema, ~rows=addressRows, ~chainIdMode)
|
|
2478
|
+
}
|
|
2479
|
+
switch InternalTable.Chains.makeInitialValuesQuery(~pgSchema, ~chainConfigs=[chainConfig]) {
|
|
2480
|
+
| Some(query) => await sql->Postgres.unsafe(query)->Utils.Promise.ignoreValue
|
|
2481
|
+
| None => ()
|
|
2482
|
+
}
|
|
2483
|
+
})
|
|
2484
|
+
}
|
|
2485
|
+
|
|
2440
2486
|
let resumeInitialState = async (
|
|
2441
2487
|
~entities,
|
|
2442
2488
|
~chainIds,
|
|
2443
|
-
~
|
|
2489
|
+
~contractMapping,
|
|
2444
2490
|
): Persistence.initialState => {
|
|
2445
|
-
let (
|
|
2446
|
-
cache,
|
|
2447
|
-
(chains, checkpointFrontier),
|
|
2448
|
-
reorgCheckpoints,
|
|
2449
|
-
(storedEnvioInfo, storedContractMapping),
|
|
2450
|
-
) = await Promise.all4((
|
|
2491
|
+
let (cache, (chains, checkpointFrontier), reorgCheckpoints) = await Promise.all3((
|
|
2451
2492
|
restoreEffectCache(~withUpload=false),
|
|
2452
2493
|
InternalTable.Chains.getInitialState(
|
|
2453
2494
|
sql,
|
|
@@ -2491,22 +2532,8 @@ let make = (
|
|
|
2491
2532
|
}>,
|
|
2492
2533
|
>
|
|
2493
2534
|
),
|
|
2494
|
-
Promise.all2((
|
|
2495
|
-
InternalTable.EnvioInfo.read(sql, ~pgSchema),
|
|
2496
|
-
InternalTable.EnvioContracts.read(sql, ~pgSchema),
|
|
2497
|
-
))->Promise.thenResolve(((info, names)) =>
|
|
2498
|
-
// Both tables join the schema in one transaction. A missing mapping
|
|
2499
|
-
// means an older envio wrote this schema, so treat the snapshot as
|
|
2500
|
-
// unreadable rather than decoding address rows against ids nothing assigned.
|
|
2501
|
-
switch (info, names) {
|
|
2502
|
-
| (Some(info), Some(names)) => (Some(info), ContractMapping.fromStoredNames(names))
|
|
2503
|
-
| _ => (None, ContractMapping.empty)
|
|
2504
|
-
}
|
|
2505
|
-
),
|
|
2506
2535
|
))
|
|
2507
2536
|
|
|
2508
|
-
throwIfIncompatible(~storedEnvioInfo, ~storedContractMapping)
|
|
2509
|
-
|
|
2510
2537
|
await reloadIndexCatalog()
|
|
2511
2538
|
|
|
2512
2539
|
// Convert string checkpoint IDs from DB to bigint
|
|
@@ -2529,8 +2556,7 @@ let make = (
|
|
|
2529
2556
|
cache,
|
|
2530
2557
|
chains,
|
|
2531
2558
|
checkpointFrontier,
|
|
2532
|
-
contractMapping
|
|
2533
|
-
envioInfo: storedEnvioInfo,
|
|
2559
|
+
contractMapping,
|
|
2534
2560
|
}
|
|
2535
2561
|
}
|
|
2536
2562
|
|
|
@@ -2687,6 +2713,8 @@ let make = (
|
|
|
2687
2713
|
name: storageName,
|
|
2688
2714
|
isInitialized,
|
|
2689
2715
|
initialize,
|
|
2716
|
+
readStoredConfig,
|
|
2717
|
+
addChain,
|
|
2690
2718
|
resumeInitialState,
|
|
2691
2719
|
loadOrThrow,
|
|
2692
2720
|
ensureQueryIndexes,
|
package/src/PgStorage.res.mjs
CHANGED
|
@@ -244,12 +244,16 @@ function getEntityHistory(entityConfig) {
|
|
|
244
244
|
return cache$1;
|
|
245
245
|
}
|
|
246
246
|
|
|
247
|
+
function makeCreatePartitionQuery(entityConfig, pgSchema, chainId) {
|
|
248
|
+
return `CREATE TABLE IF NOT EXISTS "` + pgSchema + `"."` + partitionTableName(entityConfig, chainId) + `" PARTITION OF "` + pgSchema + `"."` + entityConfig.table.tableName + `" FOR VALUES IN (` + ChainId.toString(chainId) + `);`;
|
|
249
|
+
}
|
|
250
|
+
|
|
247
251
|
function makeCreateEntityTableQueries(entityConfig, pgSchema, isNumericArrayAsText, chainIdModeOpt, chainIds) {
|
|
248
252
|
let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
|
|
249
253
|
let createTable = (table, partitionByColumn) => makeCreateTableQuery(table, pgSchema, isNumericArrayAsText, chainIdMode, partitionByColumn);
|
|
250
254
|
let chainIdField = Table.getChainIdField(entityConfig.table);
|
|
251
255
|
return (
|
|
252
|
-
chainIdField !== undefined ? [createTable(entityConfig.table, Table.getPgDbFieldName(chainIdField))].concat(chainIds.map(chainId =>
|
|
256
|
+
chainIdField !== undefined ? [createTable(entityConfig.table, Table.getPgDbFieldName(chainIdField))].concat(chainIds.map(chainId => makeCreatePartitionQuery(entityConfig, pgSchema, chainId))) : [createTable(entityConfig.table, undefined)]
|
|
253
257
|
).concat([createTable(getEntityHistory(entityConfig).table, undefined)]);
|
|
254
258
|
}
|
|
255
259
|
|
|
@@ -1234,7 +1238,6 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1234
1238
|
return {
|
|
1235
1239
|
cleanRun: true,
|
|
1236
1240
|
contractMapping: contractMapping,
|
|
1237
|
-
envioInfo: envioInfo,
|
|
1238
1241
|
cache: cache,
|
|
1239
1242
|
chains: chainConfigs.map((chainConfig, idx) => ({
|
|
1240
1243
|
id: chainConfig.id,
|
|
@@ -1500,7 +1503,58 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1500
1503
|
return Logging.errorWithExn(Utils.prettifyExn(exn$1), `Failed to dump cache.`);
|
|
1501
1504
|
}
|
|
1502
1505
|
};
|
|
1503
|
-
let
|
|
1506
|
+
let readStoredConfig = async () => {
|
|
1507
|
+
let match = await Promise.all([
|
|
1508
|
+
InternalTable.EnvioInfo.read(sql, pgSchema),
|
|
1509
|
+
InternalTable.EnvioContracts.read(sql, pgSchema)
|
|
1510
|
+
]);
|
|
1511
|
+
let contractNames = match[1];
|
|
1512
|
+
let envioInfo = match[0];
|
|
1513
|
+
if (envioInfo !== undefined) {
|
|
1514
|
+
if (contractNames !== undefined) {
|
|
1515
|
+
return {
|
|
1516
|
+
envioInfo: envioInfo,
|
|
1517
|
+
chains: await InternalTable.Chains.readStoredChains(sql, pgSchema),
|
|
1518
|
+
contractMapping: ContractMapping.fromStoredNames(contractNames)
|
|
1519
|
+
};
|
|
1520
|
+
} else {
|
|
1521
|
+
return {
|
|
1522
|
+
envioInfo: undefined,
|
|
1523
|
+
chains: [],
|
|
1524
|
+
contractMapping: ContractMapping.empty
|
|
1525
|
+
};
|
|
1526
|
+
}
|
|
1527
|
+
} else {
|
|
1528
|
+
return {
|
|
1529
|
+
envioInfo: undefined,
|
|
1530
|
+
chains: [],
|
|
1531
|
+
contractMapping: ContractMapping.empty
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
};
|
|
1535
|
+
let addChain = async (chainConfig, entities, contractMapping) => {
|
|
1536
|
+
let partitionQueries = entities.filter(entityConfig => {
|
|
1537
|
+
if (entityConfig.storage.postgres) {
|
|
1538
|
+
return Stdlib_Option.isSome(Table.getChainIdField(entityConfig.table));
|
|
1539
|
+
} else {
|
|
1540
|
+
return false;
|
|
1541
|
+
}
|
|
1542
|
+
}).map(entityConfig => makeCreatePartitionQuery(entityConfig, pgSchema, chainConfig.id));
|
|
1543
|
+
let addressRows = ChainState.configStorageRows(chainConfig, ecosystem, contractMapping);
|
|
1544
|
+
await sql.begin(async sql => {
|
|
1545
|
+
for (let idx = 0, idx_finish = partitionQueries.length; idx < idx_finish; ++idx) {
|
|
1546
|
+
await sql.unsafe(partitionQueries[idx]);
|
|
1547
|
+
}
|
|
1548
|
+
if (Utils.$$Array.notEmpty(addressRows)) {
|
|
1549
|
+
await InternalTable.EnvioAddresses.insert(sql, pgSchema, addressRows, chainIdMode);
|
|
1550
|
+
}
|
|
1551
|
+
let query = InternalTable.Chains.makeInitialValuesQuery(pgSchema, [chainConfig]);
|
|
1552
|
+
if (query !== undefined) {
|
|
1553
|
+
return await sql.unsafe(query);
|
|
1554
|
+
}
|
|
1555
|
+
});
|
|
1556
|
+
};
|
|
1557
|
+
let resumeInitialState = async (entities, chainIds, contractMapping) => {
|
|
1504
1558
|
let match = await Promise.all([
|
|
1505
1559
|
restoreEffectCache(false),
|
|
1506
1560
|
InternalTable.Chains.getInitialState(sql, pgSchema).then(rawInitialStates => {
|
|
@@ -1525,33 +1579,11 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1525
1579
|
]))
|
|
1526
1580
|
];
|
|
1527
1581
|
}),
|
|
1528
|
-
sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema))
|
|
1529
|
-
Promise.all([
|
|
1530
|
-
InternalTable.EnvioInfo.read(sql, pgSchema),
|
|
1531
|
-
InternalTable.EnvioContracts.read(sql, pgSchema)
|
|
1532
|
-
]).then(param => {
|
|
1533
|
-
let names = param[1];
|
|
1534
|
-
let info = param[0];
|
|
1535
|
-
if (info !== undefined && names !== undefined) {
|
|
1536
|
-
return [
|
|
1537
|
-
info,
|
|
1538
|
-
ContractMapping.fromStoredNames(names)
|
|
1539
|
-
];
|
|
1540
|
-
} else {
|
|
1541
|
-
return [
|
|
1542
|
-
undefined,
|
|
1543
|
-
ContractMapping.empty
|
|
1544
|
-
];
|
|
1545
|
-
}
|
|
1546
|
-
})
|
|
1582
|
+
sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema))
|
|
1547
1583
|
]);
|
|
1548
|
-
let match$1 = match[
|
|
1549
|
-
let
|
|
1550
|
-
let
|
|
1551
|
-
let match$2 = match[1];
|
|
1552
|
-
let checkpointFrontier = match$2[1];
|
|
1553
|
-
let chains = match$2[0];
|
|
1554
|
-
throwIfIncompatible(storedEnvioInfo, storedContractMapping);
|
|
1584
|
+
let match$1 = match[1];
|
|
1585
|
+
let checkpointFrontier = match$1[1];
|
|
1586
|
+
let chains = match$1[0];
|
|
1555
1587
|
await reloadIndexCatalog();
|
|
1556
1588
|
let reorgCheckpoints = match[2].map(raw => ({
|
|
1557
1589
|
id: BigInt(raw.id),
|
|
@@ -1564,8 +1596,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1564
1596
|
}
|
|
1565
1597
|
return {
|
|
1566
1598
|
cleanRun: false,
|
|
1567
|
-
contractMapping:
|
|
1568
|
-
envioInfo: storedEnvioInfo,
|
|
1599
|
+
contractMapping: contractMapping,
|
|
1569
1600
|
cache: match[0],
|
|
1570
1601
|
chains: chains,
|
|
1571
1602
|
checkpointFrontier: checkpointFrontier,
|
|
@@ -1647,6 +1678,8 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1647
1678
|
name: storageName,
|
|
1648
1679
|
isInitialized: isInitialized,
|
|
1649
1680
|
initialize: initialize,
|
|
1681
|
+
readStoredConfig: readStoredConfig,
|
|
1682
|
+
addChain: addChain,
|
|
1650
1683
|
resumeInitialState: resumeInitialState,
|
|
1651
1684
|
loadOrThrow: loadOrThrow,
|
|
1652
1685
|
ensureQueryIndexes: ensureQueryIndexes,
|
|
@@ -1720,6 +1753,7 @@ export {
|
|
|
1720
1753
|
getRowSchema,
|
|
1721
1754
|
entityHistoryCache,
|
|
1722
1755
|
getEntityHistory,
|
|
1756
|
+
makeCreatePartitionQuery,
|
|
1723
1757
|
makeCreateEntityTableQueries,
|
|
1724
1758
|
makeInitializeTransaction,
|
|
1725
1759
|
makeLoadQuery,
|