envio 3.3.0-alpha.8 → 3.3.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/licenses/CLA.md +35 -0
- package/licenses/EULA.md +67 -0
- package/licenses/LICENSE.md +45 -0
- package/licenses/README.md +35 -0
- package/package.json +9 -8
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +0 -7
- package/src/BatchProcessing.res.mjs +1 -8
- package/src/ChainFetching.res +30 -18
- package/src/ChainFetching.res.mjs +23 -13
- package/src/ChainState.res +205 -59
- package/src/ChainState.res.mjs +130 -27
- package/src/ChainState.resi +16 -1
- package/src/Config.res +9 -5
- package/src/Config.res.mjs +2 -2
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +12 -0
- package/src/CrossChainState.res +147 -39
- package/src/CrossChainState.res.mjs +42 -10
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +30 -56
- package/src/EventConfigBuilder.res.mjs +24 -30
- package/src/EventProcessing.res +8 -5
- package/src/EventProcessing.res.mjs +2 -1
- package/src/FetchState.res +842 -474
- package/src/FetchState.res.mjs +545 -356
- package/src/HandlerLoader.res +1 -1
- package/src/HandlerLoader.res.mjs +1 -1
- package/src/HandlerRegister.res +478 -305
- package/src/HandlerRegister.res.mjs +277 -209
- package/src/HandlerRegister.resi +11 -6
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +15 -39
- package/src/IndexerState.res.mjs +18 -29
- package/src/IndexerState.resi +2 -10
- package/src/Internal.res +121 -18
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/LogSelection.res +92 -217
- package/src/LogSelection.res.mjs +95 -184
- package/src/Main.res +29 -143
- package/src/Main.res.mjs +25 -88
- package/src/Metrics.res +1 -1
- package/src/Metrics.res.mjs +1 -1
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/PruneStaleHistory.res +146 -35
- package/src/PruneStaleHistory.res.mjs +114 -20
- package/src/RawEvent.res +2 -2
- package/src/Rollback.res +32 -13
- package/src/Rollback.res.mjs +24 -11
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateDeadInputTracker.res +1 -1
- package/src/SimulateItems.res +39 -7
- package/src/SimulateItems.res.mjs +28 -9
- package/src/TestIndexer.res +2 -2
- package/src/TestIndexer.res.mjs +2 -2
- package/src/TopicFilter.res +1 -25
- package/src/TopicFilter.res.mjs +0 -74
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +60 -21
- package/src/Writing.res.mjs +27 -9
- package/src/bindings/NodeJs.res +5 -0
- package/src/bindings/Viem.res +0 -5
- package/src/sources/EventRouter.res +0 -21
- package/src/sources/EventRouter.res.mjs +0 -12
- package/src/sources/Evm.res +4 -1
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/EvmChain.res +2 -27
- package/src/sources/EvmChain.res.mjs +2 -23
- package/src/sources/EvmRpcClient.res +58 -23
- package/src/sources/EvmRpcClient.res.mjs +11 -5
- package/src/sources/Fuel.res +3 -1
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/HyperSync.res +9 -38
- package/src/sources/HyperSync.res.mjs +16 -28
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +88 -11
- package/src/sources/HyperSyncClient.res.mjs +39 -6
- package/src/sources/HyperSyncSource.res +18 -210
- package/src/sources/HyperSyncSource.res.mjs +9 -137
- package/src/sources/Rpc.res +0 -32
- package/src/sources/Rpc.res.mjs +1 -46
- package/src/sources/RpcSource.res +129 -533
- package/src/sources/RpcSource.res.mjs +161 -379
- package/src/sources/SimulateSource.res +37 -19
- package/src/sources/SimulateSource.res.mjs +27 -10
- package/src/sources/SourceManager.res +4 -14
- package/src/sources/SourceManager.res.mjs +3 -9
- package/src/sources/SourceManager.resi +0 -2
- package/src/sources/SvmHyperSyncSource.res +13 -3
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
package/src/Main.res.mjs
CHANGED
|
@@ -13,6 +13,7 @@ import * as ChainMap from "./ChainMap.res.mjs";
|
|
|
13
13
|
import * as PgStorage from "./PgStorage.res.mjs";
|
|
14
14
|
import * as ChainState from "./ChainState.res.mjs";
|
|
15
15
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
16
|
+
import * as EnvioGlobal from "./EnvioGlobal.res.mjs";
|
|
16
17
|
import * as IndexerLoop from "./IndexerLoop.res.mjs";
|
|
17
18
|
import * as Persistence from "./Persistence.res.mjs";
|
|
18
19
|
import * as PromClient from "prom-client";
|
|
@@ -61,28 +62,24 @@ let stateSchema = S$RescriptSchema.union([
|
|
|
61
62
|
}))
|
|
62
63
|
]);
|
|
63
64
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
_every: s.f("_every", S$RescriptSchema.Option.getOr(S$RescriptSchema.option(S$RescriptSchema.intMin(S$RescriptSchema.int, 1, undefined)), 1))
|
|
68
|
-
})));
|
|
65
|
+
function getIndexerState() {
|
|
66
|
+
return EnvioGlobal.value.indexerState;
|
|
67
|
+
}
|
|
69
68
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
_every: 1
|
|
74
|
-
};
|
|
69
|
+
function setIndexerState(state) {
|
|
70
|
+
EnvioGlobal.value.indexerState = Primitive_option.some(state);
|
|
71
|
+
}
|
|
75
72
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
}
|
|
73
|
+
function getGlobalPersistence() {
|
|
74
|
+
return EnvioGlobal.value.persistence;
|
|
75
|
+
}
|
|
79
76
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
}
|
|
77
|
+
function setGlobalPersistence(persistence) {
|
|
78
|
+
EnvioGlobal.value.persistence = Primitive_option.some(persistence);
|
|
79
|
+
}
|
|
83
80
|
|
|
84
81
|
function getInitialChainState(chainId) {
|
|
85
|
-
let persistence =
|
|
82
|
+
let persistence = EnvioGlobal.value.persistence;
|
|
86
83
|
if (persistence === undefined) {
|
|
87
84
|
return;
|
|
88
85
|
}
|
|
@@ -134,7 +131,7 @@ function buildChainsObject(config) {
|
|
|
134
131
|
}), "isRealtime", {
|
|
135
132
|
enumerable: true,
|
|
136
133
|
get: () => {
|
|
137
|
-
let state =
|
|
134
|
+
let state = EnvioGlobal.value.indexerState;
|
|
138
135
|
if (state !== undefined) {
|
|
139
136
|
return IndexerState.isRealtime(Primitive_option.valFromOption(state));
|
|
140
137
|
} else if (Env.updateSyncTimeOnRestart) {
|
|
@@ -162,7 +159,7 @@ function buildChainsObject(config) {
|
|
|
162
159
|
}), "addresses", {
|
|
163
160
|
enumerable: true,
|
|
164
161
|
get: () => {
|
|
165
|
-
let state =
|
|
162
|
+
let state = EnvioGlobal.value.indexerState;
|
|
166
163
|
if (state !== undefined) {
|
|
167
164
|
let chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
|
|
168
165
|
let chainState = IndexerState.getChainState(Primitive_option.valFromOption(state), chain);
|
|
@@ -278,69 +275,9 @@ function getGlobalIndexer() {
|
|
|
278
275
|
HandlerRegister.throwIfFinishedRegistration("~internalAndWillBeRemovedSoon_onRollbackCommit");
|
|
279
276
|
RollbackCommit.register(callback);
|
|
280
277
|
};
|
|
281
|
-
let extractRange = (filter, name, ecosystem) => {
|
|
282
|
-
try {
|
|
283
|
-
let inner = S$RescriptSchema.parseOrThrow(filter, ecosystem.onBlockFilterSchema);
|
|
284
|
-
if (inner !== undefined) {
|
|
285
|
-
return S$RescriptSchema.parseOrThrow(Primitive_option.valFromOption(inner), blockRangeSchema);
|
|
286
|
-
} else {
|
|
287
|
-
return defaultBlockRange;
|
|
288
|
-
}
|
|
289
|
-
} catch (raw_exn) {
|
|
290
|
-
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
291
|
-
if (exn.RE_EXN_ID === S$RescriptSchema.Raised) {
|
|
292
|
-
return Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` returned an invalid filter: ` + Utils.prettifyExn(exn._1));
|
|
293
|
-
}
|
|
294
|
-
throw exn;
|
|
295
|
-
}
|
|
296
|
-
};
|
|
297
278
|
let onBlockFn = (rawOptions, handler) => {
|
|
298
279
|
HandlerRegister.throwIfFinishedRegistration("onBlock");
|
|
299
|
-
|
|
300
|
-
let ecosystem = config.ecosystem;
|
|
301
|
-
let match = buildChainsObject(config);
|
|
302
|
-
let chains = match[0];
|
|
303
|
-
let name = rawOptions.name;
|
|
304
|
-
let logger = Logging.createChild({
|
|
305
|
-
onBlock: name
|
|
306
|
-
});
|
|
307
|
-
let w = rawOptions.where;
|
|
308
|
-
let where = w === undefined || w === null ? undefined : (
|
|
309
|
-
typeof w === "function" ? rawOptions.where : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` expected \`where\` to be a function or omitted, but got ` + typeof w + `.`)
|
|
310
|
-
);
|
|
311
|
-
let matchedAny = {
|
|
312
|
-
contents: false
|
|
313
|
-
};
|
|
314
|
-
ChainMap.values(config.chainMap).forEach(chainConfig => {
|
|
315
|
-
let chainId = chainConfig.id;
|
|
316
|
-
let chainObj = chains[chainId.toString()];
|
|
317
|
-
let result = where !== undefined ? where({
|
|
318
|
-
chain: chainObj
|
|
319
|
-
}) : true;
|
|
320
|
-
let match = result === true ? [
|
|
321
|
-
true,
|
|
322
|
-
defaultBlockRange
|
|
323
|
-
] : (
|
|
324
|
-
result === false ? [
|
|
325
|
-
false,
|
|
326
|
-
defaultBlockRange
|
|
327
|
-
] : (
|
|
328
|
-
typeof result === "object" && !Array.isArray(result) && result !== null ? [
|
|
329
|
-
true,
|
|
330
|
-
extractRange(result, name, ecosystem)
|
|
331
|
-
] : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` predicate returned an invalid value of type ` + typeof result + `. Expected boolean or a filter object.`)
|
|
332
|
-
)
|
|
333
|
-
);
|
|
334
|
-
if (!match[0]) {
|
|
335
|
-
return;
|
|
336
|
-
}
|
|
337
|
-
let range = match[1];
|
|
338
|
-
matchedAny.contents = true;
|
|
339
|
-
HandlerRegister.registerOnBlock(name, chainId, range._every, range._gte, range._lte, handler);
|
|
340
|
-
});
|
|
341
|
-
if (!matchedAny.contents) {
|
|
342
|
-
return Logging.childWarn(logger, `\`indexer.` + ecosystem.onBlockMethodName + `\` matched 0 chains. Check the \`where\` predicate.`);
|
|
343
|
-
}
|
|
280
|
+
HandlerRegister.registerOnBlock(rawOptions.name, rawOptions.where, handler, config => buildChainsObject(config)[0]);
|
|
344
281
|
};
|
|
345
282
|
let keysMemo = {
|
|
346
283
|
contents: undefined
|
|
@@ -478,7 +415,7 @@ function startServer(getState, persistence, isDevelopmentMode) {
|
|
|
478
415
|
});
|
|
479
416
|
app.get("/metrics", (_req, res) => {
|
|
480
417
|
res.set("Content-Type", PromClient.register.contentType);
|
|
481
|
-
Metrics.collect(
|
|
418
|
+
Metrics.collect(EnvioGlobal.value.indexerState).then(metrics => res.end(metrics));
|
|
482
419
|
});
|
|
483
420
|
app.get("/metrics/runtime", (_req, res) => {
|
|
484
421
|
res.set("Content-Type", runtimeRegistry.contentType);
|
|
@@ -529,7 +466,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
529
466
|
let config = Config.load();
|
|
530
467
|
let isDevelopmentMode = !isTest && config.isDev;
|
|
531
468
|
let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(config, undefined);
|
|
532
|
-
|
|
469
|
+
EnvioGlobal.value.persistence = Primitive_option.some(persistence$1);
|
|
533
470
|
await Persistence.init(persistence$1, ChainMap.values(config.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset);
|
|
534
471
|
let registrationsByChainId = await HandlerLoader.registerAllHandlers(config);
|
|
535
472
|
let config$1 = isTest ? ({
|
|
@@ -573,7 +510,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
573
510
|
Prometheus.RollbackEnabled.set(config$2.shouldRollbackOnReorg);
|
|
574
511
|
if (!isTest) {
|
|
575
512
|
startServer(() => {
|
|
576
|
-
let state =
|
|
513
|
+
let state = EnvioGlobal.value.indexerState;
|
|
577
514
|
if (state === undefined) {
|
|
578
515
|
return {
|
|
579
516
|
status: "initializing"
|
|
@@ -595,7 +532,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
595
532
|
if (shouldUseTui) {
|
|
596
533
|
Tui.start(() => state);
|
|
597
534
|
}
|
|
598
|
-
|
|
535
|
+
EnvioGlobal.value.indexerState = Primitive_option.some(state);
|
|
599
536
|
IndexerLoop.start(state);
|
|
600
537
|
return await runUntilFatalError;
|
|
601
538
|
}
|
|
@@ -603,10 +540,10 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
603
540
|
export {
|
|
604
541
|
chainDataSchema,
|
|
605
542
|
stateSchema,
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
543
|
+
getIndexerState,
|
|
544
|
+
setIndexerState,
|
|
545
|
+
getGlobalPersistence,
|
|
546
|
+
setGlobalPersistence,
|
|
610
547
|
getInitialChainState,
|
|
611
548
|
buildChainsObject,
|
|
612
549
|
getGlobalIndexer,
|
package/src/Metrics.res
CHANGED
|
@@ -47,7 +47,7 @@ let renderSourceRequests = (~samples: array<SourceManager.requestStatSample>) =>
|
|
|
47
47
|
if sample.seconds !== 0. {
|
|
48
48
|
secondsOut :=
|
|
49
49
|
secondsOut.contents ++
|
|
50
|
-
`\n${sourceRequestSecondsTotalName}${labels} ${sample.seconds->Float.
|
|
50
|
+
`\n${sourceRequestSecondsTotalName}${labels} ${sample.seconds->Float.toFixed(~digits=3)}`
|
|
51
51
|
}
|
|
52
52
|
})
|
|
53
53
|
countOut.contents ++ secondsOut.contents
|
package/src/Metrics.res.mjs
CHANGED
|
@@ -44,7 +44,7 @@ function renderSourceRequests(samples) {
|
|
|
44
44
|
let labels = `{source="` + sample.sourceName + `",chainId="` + sample.chainId.toString() + `",method="` + sample.method + `"}`;
|
|
45
45
|
countOut.contents = countOut.contents + (`\n` + sourceRequestTotalName + labels + ` ` + sample.count.toString());
|
|
46
46
|
if (sample.seconds !== 0) {
|
|
47
|
-
secondsOut.contents = secondsOut.contents + (`\n` + sourceRequestSecondsTotalName + labels + ` ` + sample.seconds.
|
|
47
|
+
secondsOut.contents = secondsOut.contents + (`\n` + sourceRequestSecondsTotalName + labels + ` ` + sample.seconds.toFixed(3));
|
|
48
48
|
return;
|
|
49
49
|
}
|
|
50
50
|
});
|
package/src/Persistence.res
CHANGED
|
@@ -6,9 +6,14 @@
|
|
|
6
6
|
// DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
|
|
7
7
|
|
|
8
8
|
// The type reflects an cache table in the db
|
|
9
|
-
// It might be present even if the effect is not used in the application
|
|
9
|
+
// It might be present even if the effect is not used in the application.
|
|
10
|
+
// `initialState.cache` is keyed by `tableName` (the full cache address), so a
|
|
11
|
+
// cross-chain and a chain-scoped cache for the same effect are tracked
|
|
12
|
+
// independently.
|
|
10
13
|
type effectCacheRecord = {
|
|
11
14
|
effectName: string,
|
|
15
|
+
scope: Internal.chainScope,
|
|
16
|
+
tableName: string,
|
|
12
17
|
// Number of rows in the table
|
|
13
18
|
mutable count: int,
|
|
14
19
|
}
|
|
@@ -40,8 +45,12 @@ type initialState = {
|
|
|
40
45
|
envioInfo: option<JSON.t>,
|
|
41
46
|
}
|
|
42
47
|
|
|
48
|
+
// Carries the already-resolved cache address (`table`) rather than an effect +
|
|
49
|
+
// scope: the scope is contextual (resolved per call from the handler's chain),
|
|
50
|
+
// so the write layer only needs the concrete table it targets.
|
|
43
51
|
type updatedEffectCache = {
|
|
44
|
-
|
|
52
|
+
table: Table.table,
|
|
53
|
+
itemSchema: S.t<Internal.effectCacheItem>,
|
|
45
54
|
items: array<Internal.effectCacheItem>,
|
|
46
55
|
shouldInitialize: bool,
|
|
47
56
|
}
|
|
@@ -112,7 +121,7 @@ type storage = {
|
|
|
112
121
|
getRollbackData: (
|
|
113
122
|
~entityConfig: Internal.entityConfig,
|
|
114
123
|
~rollbackTargetCheckpointId: Internal.checkpointId,
|
|
115
|
-
) => promise<(array<
|
|
124
|
+
) => promise<(array<string>, array<unknown>)>,
|
|
116
125
|
// Write batch to storage
|
|
117
126
|
writeBatch: (
|
|
118
127
|
~batch: Batch.t,
|
package/src/PgStorage.res
CHANGED
|
@@ -654,8 +654,6 @@ let makeSchemaTableNamesQuery = (~pgSchema) => {
|
|
|
654
654
|
`SELECT table_name FROM information_schema.tables WHERE table_schema = '${pgSchema}';`
|
|
655
655
|
}
|
|
656
656
|
|
|
657
|
-
let cacheTablePrefixLength = Internal.cacheTablePrefix->String.length
|
|
658
|
-
|
|
659
657
|
type schemaCacheTableInfo = {
|
|
660
658
|
@as("table_name")
|
|
661
659
|
tableName: string,
|
|
@@ -663,13 +661,24 @@ type schemaCacheTableInfo = {
|
|
|
663
661
|
count: int,
|
|
664
662
|
}
|
|
665
663
|
|
|
664
|
+
// Matches both the cross-chain (`envio_effect_<name>`) and chain-scoped
|
|
665
|
+
// (`envio_<chainId>_effect_<name>`) cache-table formats. Kept in sync with
|
|
666
|
+
// Internal.EffectCache.
|
|
666
667
|
let makeSchemaCacheTableInfoQuery = (~pgSchema) => {
|
|
667
|
-
`
|
|
668
|
+
// The column guard requires the effect-cache shape (exactly an `id` + `output`
|
|
669
|
+
// pair) so a user entity table that happens to match the name pattern is never
|
|
670
|
+
// mistaken for an effect cache.
|
|
671
|
+
`SELECT
|
|
668
672
|
t.table_name,
|
|
669
673
|
${getCacheRowCountFnName}(t.table_name) as count
|
|
670
674
|
FROM information_schema.tables t
|
|
671
|
-
WHERE t.table_schema = '${pgSchema}'
|
|
672
|
-
AND t.table_name
|
|
675
|
+
WHERE t.table_schema = '${pgSchema}'
|
|
676
|
+
AND t.table_name ~ '^envio_([0-9]+_)?effect_.+'
|
|
677
|
+
AND (
|
|
678
|
+
SELECT array_agg(c.column_name::text ORDER BY c.column_name::text)
|
|
679
|
+
FROM information_schema.columns c
|
|
680
|
+
WHERE c.table_schema = t.table_schema AND c.table_name = t.table_name
|
|
681
|
+
) = ARRAY['id', 'output'];`
|
|
673
682
|
}
|
|
674
683
|
|
|
675
684
|
type psqlExecState =
|
|
@@ -1111,8 +1120,10 @@ let rec writeBatch = async (
|
|
|
1111
1120
|
// Since effect cache currently doesn't support rollback,
|
|
1112
1121
|
// we can run it outside of the transaction for simplicity.
|
|
1113
1122
|
updatedEffectsCache
|
|
1114
|
-
->Array.map((
|
|
1115
|
-
|
|
1123
|
+
->Array.map((
|
|
1124
|
+
{table, itemSchema, items, shouldInitialize}: Persistence.updatedEffectCache,
|
|
1125
|
+
) => {
|
|
1126
|
+
setEffectCacheOrThrow(~table, ~itemSchema, ~items, ~initialize=shouldInitialize)
|
|
1116
1127
|
})
|
|
1117
1128
|
->Promise.all,
|
|
1118
1129
|
))
|
|
@@ -1157,9 +1168,9 @@ let rec writeBatch = async (
|
|
|
1157
1168
|
}
|
|
1158
1169
|
}
|
|
1159
1170
|
|
|
1160
|
-
// Returns the most recent
|
|
1161
|
-
//
|
|
1162
|
-
let
|
|
1171
|
+
// Returns the most recent history row at or before the rollback target for IDs changed after it.
|
|
1172
|
+
// envio_change is included so ReScript can turn SET rows into restores and DELETE rows into removals.
|
|
1173
|
+
let makeGetRollbackPreTargetRowsQuery = (~entityConfig: Internal.entityConfig, ~pgSchema) => {
|
|
1163
1174
|
let dataFieldNames = entityConfig.table.fields->Array.filterMap(fieldOrDerived =>
|
|
1164
1175
|
switch fieldOrDerived {
|
|
1165
1176
|
| Field(field) => field->Table.getPgDbFieldName->Some
|
|
@@ -1175,36 +1186,41 @@ let makeGetRollbackRestoredEntitiesQuery = (~entityConfig: Internal.entityConfig
|
|
|
1175
1186
|
~entityIndex=entityConfig.index,
|
|
1176
1187
|
)
|
|
1177
1188
|
|
|
1178
|
-
`SELECT DISTINCT ON (${Table.idFieldName}) ${dataFieldsCommaSeparated}
|
|
1179
|
-
FROM "${pgSchema}"."${historyTableName}"
|
|
1180
|
-
WHERE "${EntityHistory.checkpointIdFieldName}" <= $1
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
ORDER BY ${Table.idFieldName}, "${EntityHistory.checkpointIdFieldName}" DESC`
|
|
1189
|
+
`SELECT DISTINCT ON ("${Table.idFieldName}") ${dataFieldsCommaSeparated}, "${EntityHistory.changeFieldName}"
|
|
1190
|
+
FROM "${pgSchema}"."${historyTableName}"
|
|
1191
|
+
WHERE "${EntityHistory.checkpointIdFieldName}" <= $1
|
|
1192
|
+
AND EXISTS (
|
|
1193
|
+
SELECT 1
|
|
1194
|
+
FROM "${pgSchema}"."${historyTableName}" h
|
|
1195
|
+
WHERE h."${Table.idFieldName}" = "${historyTableName}"."${Table.idFieldName}"
|
|
1196
|
+
AND h."${EntityHistory.checkpointIdFieldName}" > $1
|
|
1197
|
+
)
|
|
1198
|
+
ORDER BY "${Table.idFieldName}", "${EntityHistory.checkpointIdFieldName}" DESC`
|
|
1188
1199
|
}
|
|
1189
1200
|
|
|
1190
1201
|
// Returns entity IDs that were created after the rollback target and have no history before it.
|
|
1191
|
-
//
|
|
1202
|
+
// DELETE rows at or before the target are returned by the restore query and classified in ReScript.
|
|
1192
1203
|
let makeGetRollbackRemovedIdsQuery = (~entityConfig: Internal.entityConfig, ~pgSchema) => {
|
|
1193
1204
|
let historyTableName = EntityHistory.historyTableName(
|
|
1194
1205
|
~entityName=entityConfig.name,
|
|
1195
1206
|
~entityIndex=entityConfig.index,
|
|
1196
1207
|
)
|
|
1197
|
-
`SELECT DISTINCT ${Table.idFieldName}
|
|
1198
|
-
FROM "${pgSchema}"."${historyTableName}"
|
|
1199
|
-
WHERE "${EntityHistory.checkpointIdFieldName}" > $1
|
|
1200
|
-
AND NOT EXISTS (
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
)`
|
|
1208
|
+
`SELECT DISTINCT "${Table.idFieldName}"
|
|
1209
|
+
FROM "${pgSchema}"."${historyTableName}"
|
|
1210
|
+
WHERE "${EntityHistory.checkpointIdFieldName}" > $1
|
|
1211
|
+
AND NOT EXISTS (
|
|
1212
|
+
SELECT 1
|
|
1213
|
+
FROM "${pgSchema}"."${historyTableName}" h
|
|
1214
|
+
WHERE h."${Table.idFieldName}" = "${historyTableName}"."${Table.idFieldName}"
|
|
1215
|
+
AND h."${EntityHistory.checkpointIdFieldName}" <= $1
|
|
1216
|
+
)`
|
|
1206
1217
|
}
|
|
1207
1218
|
|
|
1219
|
+
let rollbackRowStateSchema = S.object(s => (
|
|
1220
|
+
s.field(Table.idFieldName, S.string),
|
|
1221
|
+
s.field(EntityHistory.changeFieldName, EntityHistory.RowAction.schema),
|
|
1222
|
+
))
|
|
1223
|
+
|
|
1208
1224
|
let make = (
|
|
1209
1225
|
~sql: Postgres.sql,
|
|
1210
1226
|
~pgHost,
|
|
@@ -1238,32 +1254,66 @@ let make = (
|
|
|
1238
1254
|
envioTables->Utils.Array.notEmpty
|
|
1239
1255
|
}
|
|
1240
1256
|
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1257
|
+
// Scans .envio/cache into a list of (cache table, absolute TSV path). Flat
|
|
1258
|
+
// `<name>.tsv` files map to cross-chain caches; a numeric subdirectory
|
|
1259
|
+
// `<chainId>/<name>.tsv` maps to a chain-scoped cache. Exactly one directory
|
|
1260
|
+
// level is supported. A non-numeric directory that contains TSVs is rejected.
|
|
1261
|
+
// Returns [] when .envio/cache doesn't exist.
|
|
1262
|
+
let scanCacheDir = async () => {
|
|
1263
|
+
let topEntries = try {
|
|
1264
|
+
await NodeJs.Fs.Promises.readdir(cacheDirPath)
|
|
1265
|
+
} catch {
|
|
1266
|
+
| _ => []
|
|
1267
|
+
}
|
|
1268
|
+
let result = []
|
|
1269
|
+
let _ = await topEntries
|
|
1270
|
+
->Array.map(async entry => {
|
|
1271
|
+
let entryPath = NodeJs.Path.join(cacheDirPath, entry)
|
|
1272
|
+
let isDir = (await NodeJs.Fs.Promises.stat(entryPath))->NodeJs.Fs.Promises.statsIsDirectory
|
|
1273
|
+
if isDir {
|
|
1274
|
+
let subEntries = await NodeJs.Fs.Promises.readdir(entryPath)
|
|
1275
|
+
let tsvs = subEntries->Array.filter(sub => sub->String.endsWith(".tsv"))
|
|
1276
|
+
switch Internal.EffectCache.parseChainId(entry) {
|
|
1277
|
+
| Some(chainId) =>
|
|
1278
|
+
tsvs->Array.forEach(sub => {
|
|
1279
|
+
let effectName = sub->String.slice(~start=0, ~end=-4)
|
|
1280
|
+
let table = Internal.makeCacheTable(~effectName, ~scope=Chain(chainId))
|
|
1281
|
+
result
|
|
1282
|
+
->Array.push((table, NodeJs.Path.join(entryPath, sub)->NodeJs.Path.toString))
|
|
1283
|
+
->ignore
|
|
1255
1284
|
})
|
|
1285
|
+
| None =>
|
|
1286
|
+
if tsvs->Utils.Array.notEmpty {
|
|
1287
|
+
JsError.throwWithMessage(
|
|
1288
|
+
`Invalid effect cache directory ".envio/cache/${entry}". Chain cache directories must be named by a numeric chain id (e.g. "1"). Found cache files: ${tsvs->Array.joinUnsafe(
|
|
1289
|
+
", ",
|
|
1290
|
+
)}.`,
|
|
1291
|
+
)
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
} else if entry->String.endsWith(".tsv") {
|
|
1295
|
+
let effectName = entry->String.slice(~start=0, ~end=-4)
|
|
1296
|
+
let table = Internal.makeCacheTable(~effectName, ~scope=CrossChain)
|
|
1297
|
+
result->Array.push((table, entryPath->NodeJs.Path.toString))->ignore
|
|
1298
|
+
}
|
|
1299
|
+
})
|
|
1300
|
+
->Promise.all
|
|
1301
|
+
result
|
|
1302
|
+
}
|
|
1256
1303
|
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1304
|
+
let restoreEffectCache = async (~withUpload) => {
|
|
1305
|
+
if withUpload {
|
|
1306
|
+
// Try to restore cache tables from the .envio/cache TSV files
|
|
1307
|
+
switch await scanCacheDir() {
|
|
1308
|
+
| [] => Logging.info("No cache found to upload.")
|
|
1309
|
+
| entries =>
|
|
1310
|
+
switch await getConnectedPsqlExec(~pgUser, ~pgHost, ~pgDatabase, ~pgPort, ~containerName) {
|
|
1311
|
+
| Ok(psqlExec) =>
|
|
1312
|
+
let _ = await entries
|
|
1313
|
+
->Array.map(((table, inputFile)) => {
|
|
1262
1314
|
sql
|
|
1263
1315
|
->Postgres.unsafe(makeCreateTableQuery(table, ~pgSchema, ~isNumericArrayAsText=false))
|
|
1264
1316
|
->Promise.then(() => {
|
|
1265
|
-
let inputFile = NodeJs.Path.join(cacheDirPath, entry)->NodeJs.Path.toString
|
|
1266
|
-
|
|
1267
1317
|
let command = `${psqlExec} -c 'COPY "${pgSchema}"."${table.tableName}" FROM STDIN WITH (FORMAT text, HEADER);' < ${inputFile}`
|
|
1268
1318
|
|
|
1269
1319
|
Promise.make(
|
|
@@ -1283,14 +1333,8 @@ let make = (
|
|
|
1283
1333
|
})
|
|
1284
1334
|
})
|
|
1285
1335
|
->Promise.all
|
|
1286
|
-
|
|
1287
1336
|
Logging.info("Successfully uploaded cache.")
|
|
1288
|
-
|
|
1289
|
-
| (Error(message), _)
|
|
1290
|
-
| (_, Error(message)) =>
|
|
1291
|
-
if message === nothingToUploadErrorMessage {
|
|
1292
|
-
Logging.info("No cache found to upload.")
|
|
1293
|
-
} else {
|
|
1337
|
+
| Error(message) =>
|
|
1294
1338
|
Logging.error(`Failed to upload cache, continuing without it. ${message}`)
|
|
1295
1339
|
}
|
|
1296
1340
|
}
|
|
@@ -1315,8 +1359,14 @@ let make = (
|
|
|
1315
1359
|
|
|
1316
1360
|
let cache = Dict.make()
|
|
1317
1361
|
cacheTableInfo->Array.forEach(({tableName, count}) => {
|
|
1318
|
-
|
|
1319
|
-
|
|
1362
|
+
switch Internal.EffectCache.fromTableName(tableName) {
|
|
1363
|
+
| Some((effectName, scope)) =>
|
|
1364
|
+
cache->Dict.set(
|
|
1365
|
+
tableName,
|
|
1366
|
+
({effectName, scope, tableName, count}: Persistence.effectCacheRecord),
|
|
1367
|
+
)
|
|
1368
|
+
| None => ()
|
|
1369
|
+
}
|
|
1320
1370
|
})
|
|
1321
1371
|
cache
|
|
1322
1372
|
}
|
|
@@ -1480,12 +1530,11 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1480
1530
|
}
|
|
1481
1531
|
|
|
1482
1532
|
let setEffectCacheOrThrow = async (
|
|
1483
|
-
~
|
|
1533
|
+
~table: Table.table,
|
|
1534
|
+
~itemSchema,
|
|
1484
1535
|
~items: array<Internal.effectCacheItem>,
|
|
1485
1536
|
~initialize: bool,
|
|
1486
1537
|
) => {
|
|
1487
|
-
let {table, itemSchema} = effect.storageMeta
|
|
1488
|
-
|
|
1489
1538
|
if initialize {
|
|
1490
1539
|
let _ = await sql->Postgres.unsafe(
|
|
1491
1540
|
makeCreateTableQuery(table, ~pgSchema, ~isNumericArrayAsText=false),
|
|
@@ -1531,24 +1580,36 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1531
1580
|
)
|
|
1532
1581
|
|
|
1533
1582
|
let promises = cacheTableInfo->Array.map(async ({tableName}) => {
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1583
|
+
switch Internal.EffectCache.fromTableName(tableName) {
|
|
1584
|
+
| Some((effectName, scope)) =>
|
|
1585
|
+
// Reverse mapping: chain-scoped caches dump into a per-chain
|
|
1586
|
+
// subdirectory, created here if needed.
|
|
1587
|
+
let outputPath = NodeJs.Path.join(
|
|
1588
|
+
cacheDirPath,
|
|
1589
|
+
Internal.EffectCache.toCachePath(~effectName, ~scope),
|
|
1590
|
+
)
|
|
1591
|
+
let _ = await NodeJs.Fs.Promises.mkdir(
|
|
1592
|
+
~path=NodeJs.Path.dirname(outputPath->NodeJs.Path.toString),
|
|
1593
|
+
~options={recursive: true},
|
|
1594
|
+
)
|
|
1595
|
+
let outputFile = outputPath->NodeJs.Path.toString
|
|
1537
1596
|
|
|
1538
|
-
|
|
1597
|
+
let command = `${psqlExec} -c 'COPY "${pgSchema}"."${tableName}" TO STDOUT WITH (FORMAT text, HEADER);' > ${outputFile}`
|
|
1539
1598
|
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1599
|
+
await Promise.make((resolve, reject) => {
|
|
1600
|
+
NodeJs.ChildProcess.execWithOptions(
|
|
1601
|
+
command,
|
|
1602
|
+
psqlExecOptions,
|
|
1603
|
+
(~error, ~stdout, ~stderr as _) => {
|
|
1604
|
+
switch error {
|
|
1605
|
+
| Value(error) => reject(error)
|
|
1606
|
+
| Null => resolve(stdout)
|
|
1607
|
+
}
|
|
1608
|
+
},
|
|
1609
|
+
)
|
|
1610
|
+
})
|
|
1611
|
+
| None => ""
|
|
1612
|
+
}
|
|
1552
1613
|
})
|
|
1553
1614
|
|
|
1554
1615
|
let _ = await promises->Promise.all
|
|
@@ -1663,7 +1724,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1663
1724
|
~entityConfig: Internal.entityConfig,
|
|
1664
1725
|
~rollbackTargetCheckpointId,
|
|
1665
1726
|
) => {
|
|
1666
|
-
await Promise.all2((
|
|
1727
|
+
let (removedIdRows, rollbackRows) = await Promise.all2((
|
|
1667
1728
|
// Get IDs of entities that should be deleted (created after rollback target with no prior history)
|
|
1668
1729
|
sql
|
|
1669
1730
|
->Postgres.preparedUnsafe(
|
|
@@ -1671,14 +1732,26 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1671
1732
|
[rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
|
|
1672
1733
|
)
|
|
1673
1734
|
->(Utils.magic: promise<unknown> => promise<array<{"id": string}>>),
|
|
1674
|
-
// Get
|
|
1735
|
+
// Get the latest pre-target row, including its SET or DELETE action.
|
|
1675
1736
|
sql
|
|
1676
1737
|
->Postgres.preparedUnsafe(
|
|
1677
|
-
|
|
1738
|
+
makeGetRollbackPreTargetRowsQuery(~entityConfig, ~pgSchema),
|
|
1678
1739
|
[rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
|
|
1679
1740
|
)
|
|
1680
1741
|
->(Utils.magic: promise<unknown> => promise<array<unknown>>),
|
|
1681
1742
|
))
|
|
1743
|
+
|
|
1744
|
+
let removedIds = removedIdRows->Array.map(row => row["id"])
|
|
1745
|
+
let restoredEntitiesResult = []
|
|
1746
|
+
rollbackRows->Array.forEach(row => {
|
|
1747
|
+
let (entityId, action) = row->S.parseOrThrow(rollbackRowStateSchema)
|
|
1748
|
+
switch action {
|
|
1749
|
+
| SET => restoredEntitiesResult->Array.push(row)->ignore
|
|
1750
|
+
| DELETE => removedIds->Array.push(entityId)->ignore
|
|
1751
|
+
}
|
|
1752
|
+
})
|
|
1753
|
+
|
|
1754
|
+
(removedIds, restoredEntitiesResult)
|
|
1682
1755
|
}
|
|
1683
1756
|
|
|
1684
1757
|
let writeBatchMethod = async (
|