envio 3.12.0 → 3.13.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/BatchProcessing.res +9 -9
- package/src/BatchProcessing.res.mjs +5 -5
- package/src/Bin.res +24 -17
- package/src/Bin.res.mjs +5 -0
- package/src/ChainFetching.res +31 -13
- package/src/ChainFetching.res.mjs +9 -4
- package/src/ChainState.res +55 -3
- package/src/ChainState.res.mjs +46 -3
- package/src/ChainState.resi +3 -1
- package/src/Config.res +35 -0
- package/src/Config.res.mjs +39 -0
- package/src/Core.res +4 -0
- package/src/Core.res.mjs +4 -0
- package/src/CrossChainState.res +61 -4
- package/src/CrossChainState.res.mjs +39 -5
- package/src/CrossChainState.resi +10 -1
- package/src/Env.res +4 -0
- package/src/FetchState.res +51 -46
- package/src/FetchState.res.mjs +54 -36
- package/src/InMemoryTable.res +155 -67
- package/src/InMemoryTable.res.mjs +151 -60
- package/src/IndexerLoop.res +2 -0
- package/src/IndexerLoop.res.mjs +1 -0
- package/src/IndexerState.res +41 -1
- package/src/IndexerState.res.mjs +43 -4
- package/src/IndexerState.resi +10 -0
- package/src/LoadLayer.res +57 -34
- package/src/LoadLayer.res.mjs +45 -62
- package/src/LoadLayer.resi +1 -1
- package/src/Logging.res +38 -6
- package/src/Logging.res.mjs +32 -5
- package/src/Main.res +131 -268
- package/src/Main.res.mjs +28 -152
- package/src/Metrics.res +263 -102
- package/src/Metrics.res.mjs +227 -48
- package/src/Persistence.res +27 -2
- package/src/Persistence.res.mjs +9 -2
- package/src/PgStorage.res +109 -71
- package/src/PgStorage.res.mjs +88 -58
- package/src/Server.res +181 -0
- package/src/Server.res.mjs +143 -0
- package/src/Supervisor.res +415 -0
- package/src/Supervisor.res.mjs +325 -0
- package/src/TestIndexer.res +9 -25
- package/src/TestIndexer.res.mjs +5 -4
- package/src/UserContext.res +13 -32
- package/src/UserContext.res.mjs +1 -7
- package/src/Utils.res +1 -4
- package/src/Utils.res.mjs +7 -16
- package/src/Worker.res +95 -0
- package/src/Worker.res.mjs +80 -0
- package/src/bindings/NodeJs.res +41 -0
- package/src/db/EntityFilter.res +487 -275
- package/src/db/EntityFilter.res.mjs +557 -309
- package/src/db/InternalTable.res +8 -1
- package/src/db/InternalTable.res.mjs +5 -1
- package/src/db/Table.res +21 -6
- package/src/db/Table.res.mjs +13 -4
- package/src/sources/BlockStore.res +7 -2
- package/src/sources/EvmHyperSyncSource.res +2 -0
- package/src/sources/EvmHyperSyncSource.res.mjs +2 -2
- package/src/sources/FuelHyperSyncSource.res +1 -0
- package/src/sources/FuelHyperSyncSource.res.mjs +1 -1
- package/src/sources/HyperSync.res +4 -0
- package/src/sources/HyperSync.res.mjs +4 -2
- package/src/sources/HyperSync.resi +1 -0
- package/src/sources/HyperSyncClient.res +3 -0
- package/src/sources/HyperSyncSSE.res +1 -1
- package/src/sources/HyperSyncSSE.res.mjs +4 -10
- package/src/sources/RpcSource.res +1 -0
- package/src/sources/RpcSource.res.mjs +1 -1
- package/src/sources/SimulateSource.res +1 -0
- package/src/sources/SimulateSource.res.mjs +1 -1
- package/src/sources/Source.res +7 -0
- package/src/sources/SourceManager.res +4 -3
- package/src/sources/SourceManager.res.mjs +2 -2
- package/src/sources/SvmHyperSyncClient.res +5 -0
- package/src/sources/SvmHyperSyncSource.res +3 -0
- package/src/sources/SvmHyperSyncSource.res.mjs +4 -2
- package/src/tui/Tui.res +24 -0
- package/src/tui/Tui.res.mjs +18 -0
- package/src/tui/components/SyncETA.res +12 -6
- package/src/tui/components/SyncETA.res.mjs +12 -8
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Env from "./Env.res.mjs";
|
|
4
|
+
import * as Tui from "./tui/Tui.res.mjs";
|
|
5
|
+
import * as Utils from "./Utils.res.mjs";
|
|
6
|
+
import * as Config from "./Config.res.mjs";
|
|
7
|
+
import * as Server from "./Server.res.mjs";
|
|
8
|
+
import * as Worker from "./Worker.res.mjs";
|
|
9
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
10
|
+
import * as Logging from "./Logging.res.mjs";
|
|
11
|
+
import * as Metrics from "./Metrics.res.mjs";
|
|
12
|
+
import * as Process from "process";
|
|
13
|
+
import * as ChainMap from "./ChainMap.res.mjs";
|
|
14
|
+
import * as PgStorage from "./PgStorage.res.mjs";
|
|
15
|
+
import * as Performance from "./bindings/Performance.res.mjs";
|
|
16
|
+
import * as Persistence from "./Persistence.res.mjs";
|
|
17
|
+
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
18
|
+
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
19
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
20
|
+
import * as Child_process from "child_process";
|
|
21
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
22
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
23
|
+
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
24
|
+
|
|
25
|
+
function plan(chainIds, maxConnections) {
|
|
26
|
+
let workerCount = Primitive_int.min(chainIds.length, maxConnections / 2 | 0);
|
|
27
|
+
if (workerCount < 2) {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
let evenShare = Primitive_int.div(maxConnections, workerCount);
|
|
31
|
+
let remainder = Primitive_int.mod_(maxConnections, workerCount);
|
|
32
|
+
return Stdlib_Array.fromInitializer(workerCount, workerIndex => ({
|
|
33
|
+
chainIds: chainIds.filter((param, dealIndex) => {
|
|
34
|
+
let position = Primitive_int.mod_(dealIndex, workerCount);
|
|
35
|
+
let isReversePass = Primitive_int.div(dealIndex, workerCount) % 2 === 1;
|
|
36
|
+
return (
|
|
37
|
+
isReversePass ? (workerCount - 1 | 0) - position | 0 : position
|
|
38
|
+
) === workerIndex;
|
|
39
|
+
}),
|
|
40
|
+
maxConnections: evenShare + (
|
|
41
|
+
workerIndex < remainder ? 1 : 0
|
|
42
|
+
) | 0
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function planForRun(config, maxConnectionsOpt) {
|
|
47
|
+
let maxConnections = maxConnectionsOpt !== undefined ? maxConnectionsOpt : Env.Db.maxConnections;
|
|
48
|
+
if (config.isolated || !Config.isPerChain(config)) {
|
|
49
|
+
return;
|
|
50
|
+
} else {
|
|
51
|
+
return plan(ChainMap.values(config.chainMap).map(chain => chain.id), maxConnections);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function name(worker) {
|
|
56
|
+
return worker.chainIds.map(ChainId.toString).join(";");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function label(worker) {
|
|
60
|
+
return `[chain ` + name(worker) + `]`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function logFilePath(workerIndex, pathOpt) {
|
|
64
|
+
let path = pathOpt !== undefined ? pathOpt : Env.logFilePath;
|
|
65
|
+
let suffix = `.worker-` + workerIndex.toString();
|
|
66
|
+
let dot = path.lastIndexOf(".");
|
|
67
|
+
if (dot > path.lastIndexOf("/")) {
|
|
68
|
+
return path.slice(0, dot) + suffix + path.slice(dot, path.length);
|
|
69
|
+
} else {
|
|
70
|
+
return path + suffix;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function readLines(onLine) {
|
|
75
|
+
let pending = {
|
|
76
|
+
contents: ""
|
|
77
|
+
};
|
|
78
|
+
let read = chunk => {
|
|
79
|
+
let parts = (pending.contents + chunk).split("\n");
|
|
80
|
+
pending.contents = Stdlib_Option.getOr(parts.pop(), "");
|
|
81
|
+
parts.forEach(onLine);
|
|
82
|
+
};
|
|
83
|
+
let flush = () => {
|
|
84
|
+
let line = pending.contents;
|
|
85
|
+
if (line === "") {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
pending.contents = "";
|
|
89
|
+
onLine(line);
|
|
90
|
+
};
|
|
91
|
+
return [
|
|
92
|
+
read,
|
|
93
|
+
flush
|
|
94
|
+
];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function fork(worker, workerIndex, holdRealtime, entryPathOpt, pipeOutputOpt, onOutputOpt) {
|
|
98
|
+
let entryPath = entryPathOpt !== undefined ? entryPathOpt : process.argv[1];
|
|
99
|
+
let pipeOutput = pipeOutputOpt !== undefined ? pipeOutputOpt : false;
|
|
100
|
+
let onOutput = onOutputOpt !== undefined ? onOutputOpt : prim => {
|
|
101
|
+
console.log(prim);
|
|
102
|
+
};
|
|
103
|
+
let env = Object.assign({}, Process.env);
|
|
104
|
+
env[Worker.envVar] = S$RescriptSchema.reverseConvertToJsonStringOrThrow({
|
|
105
|
+
chainIds: worker.chainIds,
|
|
106
|
+
holdRealtime: holdRealtime
|
|
107
|
+
}, Worker.configSchema, undefined);
|
|
108
|
+
env["ENVIO_PG_MAX_CONNECTIONS"] = worker.maxConnections.toString();
|
|
109
|
+
env["LOG_FILE"] = logFilePath(workerIndex, undefined);
|
|
110
|
+
if (pipeOutput && Stdlib_Option.getOr((process.stdout.isTTY == null) ? undefined : Primitive_option.some(process.stdout.isTTY), false)) {
|
|
111
|
+
env["FORCE_COLOR"] = "1";
|
|
112
|
+
}
|
|
113
|
+
let child = Child_process.fork(entryPath, [], {
|
|
114
|
+
env: env,
|
|
115
|
+
serialization: "advanced",
|
|
116
|
+
stdio: pipeOutput ? [
|
|
117
|
+
"inherit",
|
|
118
|
+
"pipe",
|
|
119
|
+
"pipe",
|
|
120
|
+
"ipc"
|
|
121
|
+
] : [
|
|
122
|
+
"inherit",
|
|
123
|
+
"inherit",
|
|
124
|
+
"inherit",
|
|
125
|
+
"ipc"
|
|
126
|
+
]
|
|
127
|
+
});
|
|
128
|
+
if (pipeOutput) {
|
|
129
|
+
[
|
|
130
|
+
child.stdout,
|
|
131
|
+
child.stderr
|
|
132
|
+
].forEach(stream => {
|
|
133
|
+
if (stream === null) {
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
let match = readLines(onOutput);
|
|
137
|
+
stream.setEncoding("utf8");
|
|
138
|
+
stream.on("data", match[0]);
|
|
139
|
+
stream.on("end", match[1]);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
let running = {
|
|
143
|
+
worker: worker,
|
|
144
|
+
child: child,
|
|
145
|
+
snapshot: undefined,
|
|
146
|
+
runtime: undefined,
|
|
147
|
+
settled: false
|
|
148
|
+
};
|
|
149
|
+
child.on("message", message => {
|
|
150
|
+
running.snapshot = message.metrics;
|
|
151
|
+
running.runtime = message.runtime;
|
|
152
|
+
});
|
|
153
|
+
return running;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function stop(group) {
|
|
157
|
+
group.stopping = true;
|
|
158
|
+
group.running.forEach(r => {
|
|
159
|
+
r.child.kill("SIGTERM");
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
let inFlight = {
|
|
164
|
+
contents: undefined
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
function syncCache(dump) {
|
|
168
|
+
let dumping = inFlight.contents;
|
|
169
|
+
if (dumping !== undefined) {
|
|
170
|
+
return dumping;
|
|
171
|
+
}
|
|
172
|
+
let dumping$1 = dump().finally(() => {
|
|
173
|
+
inFlight.contents = undefined;
|
|
174
|
+
});
|
|
175
|
+
inFlight.contents = dumping$1;
|
|
176
|
+
return dumping$1;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function dumpCache(config) {
|
|
180
|
+
let storage = PgStorage.makeStorageFromEnv(config, Primitive_option.some(PgStorage.makeClient(1)), undefined, undefined);
|
|
181
|
+
return storage.dumpEffectCache().finally(() => {
|
|
182
|
+
storage.close();
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function awaitExit(group) {
|
|
187
|
+
let failed = {
|
|
188
|
+
contents: false
|
|
189
|
+
};
|
|
190
|
+
let alive = {
|
|
191
|
+
contents: group.running.length
|
|
192
|
+
};
|
|
193
|
+
await new Promise((resolve, param) => {
|
|
194
|
+
let onGone = (r, failure) => {
|
|
195
|
+
if (!r.settled) {
|
|
196
|
+
r.settled = true;
|
|
197
|
+
if (failure) {
|
|
198
|
+
failed.contents = true;
|
|
199
|
+
if (!group.stopping) {
|
|
200
|
+
stop(group);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
alive.contents = alive.contents - 1 | 0;
|
|
204
|
+
if (alive.contents === 0) {
|
|
205
|
+
return resolve();
|
|
206
|
+
} else {
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
group.running.forEach(r => {
|
|
212
|
+
r.child.on("exit", (code, _signal) => onGone(r, !group.stopping && (
|
|
213
|
+
code === null ? undefined : Primitive_option.some(code)
|
|
214
|
+
) !== 0));
|
|
215
|
+
r.child.on("error", exn => {
|
|
216
|
+
Logging.errorWithExn(exn, label(r.worker) + ` failed to start`);
|
|
217
|
+
onGone(r, true);
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
});
|
|
221
|
+
if (failed.contents) {
|
|
222
|
+
Stdlib_JsError.throwWithMessage("An indexer process exited with a failure. Stopped the others.");
|
|
223
|
+
}
|
|
224
|
+
if (group.stopping) {
|
|
225
|
+
return "Stopped";
|
|
226
|
+
} else {
|
|
227
|
+
return "Finished";
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function isRunAtHead(snapshots, workerCount) {
|
|
232
|
+
if (snapshots.length === workerCount) {
|
|
233
|
+
return snapshots.every(snapshot => snapshot.hasArrivedAtHead);
|
|
234
|
+
} else {
|
|
235
|
+
return false;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async function run(config, workers, reset) {
|
|
240
|
+
let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
|
|
241
|
+
await Persistence.initForRun(persistence, config, reset, config.isDev, false);
|
|
242
|
+
await persistence.storage.close();
|
|
243
|
+
let startTime = new Date();
|
|
244
|
+
let startTimeRef = Performance.now();
|
|
245
|
+
Logging.info(`Splitting ` + ChainMap.values(config.chainMap).length.toString() + ` chains across ` + workers.length.toString() + ` processes, from a budget of ` + Env.Db.maxConnections.toString() + ` database connections.`);
|
|
246
|
+
let shouldUseTui = Tui.shouldUse(undefined);
|
|
247
|
+
let holdRealtime = Persistence.getInitializedState(persistence).chains.some(chain => Stdlib_Option.isNone(chain.timestampCaughtUpToHeadOrEndblock));
|
|
248
|
+
let group = {
|
|
249
|
+
running: workers.map((worker, workerIndex) => fork(worker, workerIndex, holdRealtime, undefined, shouldUseTui, undefined)),
|
|
250
|
+
stopping: false
|
|
251
|
+
};
|
|
252
|
+
let reported = () => Stdlib_Array.filterMap(group.running, r => r.snapshot);
|
|
253
|
+
let merge = snapshots => Metrics.merge(snapshots, startTime, new Date(), Performance.secondsSince(startTimeRef));
|
|
254
|
+
Server.startServer(() => {
|
|
255
|
+
let snapshots = reported();
|
|
256
|
+
if (snapshots.length !== 0) {
|
|
257
|
+
return merge(snapshots);
|
|
258
|
+
}
|
|
259
|
+
}, Utils.EnvioPackage.value.version, () => syncCache(() => dumpCache(config)), () => Metrics.renderRuntime(Stdlib_Array.filterMap(group.running, r => Stdlib_Option.map(r.runtime, runtime => [
|
|
260
|
+
`worker="` + name(r.worker) + `"`,
|
|
261
|
+
runtime
|
|
262
|
+
]))), config.isDev);
|
|
263
|
+
let releaseCheck = {
|
|
264
|
+
contents: undefined
|
|
265
|
+
};
|
|
266
|
+
let stopReleaseCheck = () => {
|
|
267
|
+
Stdlib_Option.forEach(releaseCheck.contents, prim => {
|
|
268
|
+
clearInterval(prim);
|
|
269
|
+
});
|
|
270
|
+
releaseCheck.contents = undefined;
|
|
271
|
+
};
|
|
272
|
+
if (holdRealtime) {
|
|
273
|
+
releaseCheck.contents = Primitive_option.some(setInterval(() => {
|
|
274
|
+
if (isRunAtHead(reported(), group.running.length)) {
|
|
275
|
+
stopReleaseCheck();
|
|
276
|
+
group.running.forEach(r => {
|
|
277
|
+
r.child.send("release-realtime");
|
|
278
|
+
});
|
|
279
|
+
return Logging.info("Every chain has reached the head. Switching the run to realtime.");
|
|
280
|
+
}
|
|
281
|
+
}, 500));
|
|
282
|
+
}
|
|
283
|
+
if (shouldUseTui) {
|
|
284
|
+
Tui.start(config, () => merge(reported()));
|
|
285
|
+
}
|
|
286
|
+
process.on("SIGTERM", () => stop(group));
|
|
287
|
+
process.on("SIGINT", () => stop(group));
|
|
288
|
+
let outcome = await awaitExit(group);
|
|
289
|
+
stopReleaseCheck();
|
|
290
|
+
if (outcome === "Finished") {
|
|
291
|
+
if (shouldUseTui) {
|
|
292
|
+
process.on("SIGTERM", () => {
|
|
293
|
+
Process.exit(0);
|
|
294
|
+
});
|
|
295
|
+
process.on("SIGINT", () => {
|
|
296
|
+
Process.exit(0);
|
|
297
|
+
});
|
|
298
|
+
} else {
|
|
299
|
+
Logging.info("Exiting with success");
|
|
300
|
+
Process.exit(0);
|
|
301
|
+
}
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
Process.exit(0);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
let minConnectionsPerWorker = 2;
|
|
308
|
+
|
|
309
|
+
export {
|
|
310
|
+
minConnectionsPerWorker,
|
|
311
|
+
plan,
|
|
312
|
+
planForRun,
|
|
313
|
+
name,
|
|
314
|
+
label,
|
|
315
|
+
logFilePath,
|
|
316
|
+
readLines,
|
|
317
|
+
fork,
|
|
318
|
+
stop,
|
|
319
|
+
syncCache,
|
|
320
|
+
dumpCache,
|
|
321
|
+
awaitExit,
|
|
322
|
+
isRunAtHead,
|
|
323
|
+
run,
|
|
324
|
+
}
|
|
325
|
+
/* Env Not a pure module */
|
package/src/TestIndexer.res
CHANGED
|
@@ -75,15 +75,10 @@ let handleLoad = (state: testIndexerState, ~tableName: string, ~filter: EntityFi
|
|
|
75
75
|
| None => []
|
|
76
76
|
| Some(entityConfig) =>
|
|
77
77
|
let entityDict = state.entities->Dict.get(tableName)->Option.getOr(Dict.make())
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
// The store holds decoded entities and the filter carries decoded values,
|
|
83
|
-
// so compare directly (same approach as InMemoryTable) — no JSON round-trip.
|
|
84
|
-
let entityAsDict = entity->(Utils.magic: Internal.entity => dict<EntityFilter.FieldValue.t>)
|
|
85
|
-
filter->EntityFilter.matches(~entity=entityAsDict)
|
|
86
|
-
})
|
|
78
|
+
// The store holds decoded entities and the filter carries decoded values,
|
|
79
|
+
// so compare directly (same approach as InMemoryTable) — no JSON round-trip.
|
|
80
|
+
let matcher = filter->EntityFilter.makeMatcher(~table=entityConfig.table)
|
|
81
|
+
let matched = entityDict->Dict.valuesToArray->Array.filter(matcher)
|
|
87
82
|
// The chain is already fixed by the scope the load ran for, so the loaded
|
|
88
83
|
// entity is handed back in the shape the handlers see.
|
|
89
84
|
switch entityConfig.table->Table.getChainIdField {
|
|
@@ -544,23 +539,12 @@ let makeEntityGetWhere = (~state: testIndexerState, ~entityConfig: Internal.enti
|
|
|
544
539
|
`Cannot call ${entityConfig.name}.getWhere() while indexer.process() is running. ` ++ "Wait for process() to complete before accessing entities directly.",
|
|
545
540
|
)
|
|
546
541
|
}
|
|
547
|
-
let
|
|
548
|
-
filter
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
)
|
|
542
|
+
let matcher =
|
|
543
|
+
filter
|
|
544
|
+
->EntityFilter.parseOrThrow(~entityName=entityConfig.name, ~table=entityConfig.table)
|
|
545
|
+
->EntityFilter.makeMatcher(~table=entityConfig.table)
|
|
552
546
|
let entityDict = state.entities->Dict.get(entityConfig.name)->Option.getOr(Dict.make())
|
|
553
|
-
|
|
554
|
-
// matches are disjoint, so the union needs no dedup.
|
|
555
|
-
Promise.resolve(
|
|
556
|
-
entityDict
|
|
557
|
-
->Dict.valuesToArray
|
|
558
|
-
->Array.filter(entity => {
|
|
559
|
-
let entityAsDict = entity->(Utils.magic: Internal.entity => dict<EntityFilter.FieldValue.t>)
|
|
560
|
-
filters->Array.some(filter => filter->EntityFilter.matches(~entity=entityAsDict))
|
|
561
|
-
})
|
|
562
|
-
->Array.map(copyEntity),
|
|
563
|
-
)
|
|
547
|
+
Promise.resolve(entityDict->Dict.valuesToArray->Array.filter(matcher)->Array.map(copyEntity))
|
|
564
548
|
}
|
|
565
549
|
}
|
|
566
550
|
|
package/src/TestIndexer.res.mjs
CHANGED
|
@@ -56,7 +56,8 @@ function handleLoad(state, tableName, filter) {
|
|
|
56
56
|
return [];
|
|
57
57
|
}
|
|
58
58
|
let entityDict = Stdlib_Option.getOr(state.entities[tableName], {});
|
|
59
|
-
let
|
|
59
|
+
let matcher = EntityFilter.makeMatcher(filter, entityConfig.table);
|
|
60
|
+
let matched = Object.values(entityDict).filter(matcher);
|
|
60
61
|
let field = Table.getChainIdField(entityConfig.table);
|
|
61
62
|
if (field !== undefined) {
|
|
62
63
|
return matched.map(entity => {
|
|
@@ -376,9 +377,9 @@ function makeEntityGetWhere(state, entityConfig) {
|
|
|
376
377
|
if (state.processInProgress) {
|
|
377
378
|
Stdlib_JsError.throwWithMessage(`Cannot call ` + entityConfig.name + `.getWhere() while indexer.process() is running. ` + "Wait for process() to complete before accessing entities directly.");
|
|
378
379
|
}
|
|
379
|
-
let
|
|
380
|
+
let matcher = EntityFilter.makeMatcher(EntityFilter.parseOrThrow(filter, entityConfig.name, entityConfig.table), entityConfig.table);
|
|
380
381
|
let entityDict = Stdlib_Option.getOr(state.entities[entityConfig.name], {});
|
|
381
|
-
return Promise.resolve(Object.values(entityDict).filter(
|
|
382
|
+
return Promise.resolve(Object.values(entityDict).filter(matcher).map(copyEntity));
|
|
382
383
|
};
|
|
383
384
|
}
|
|
384
385
|
|
|
@@ -638,7 +639,7 @@ function createTestIndexer() {
|
|
|
638
639
|
};
|
|
639
640
|
try {
|
|
640
641
|
await new Promise((resolve, reject) => {
|
|
641
|
-
let indexerState = IndexerState.makeFromDbState(runConfig, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, errHandler => {
|
|
642
|
+
let indexerState = IndexerState.makeFromDbState(runConfig, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, undefined, errHandler => {
|
|
642
643
|
ErrorHandling.log(errHandler);
|
|
643
644
|
reject(Utils.prettifyExn(errHandler.exn));
|
|
644
645
|
}, () => resolve());
|
package/src/UserContext.res
CHANGED
|
@@ -122,39 +122,20 @@ type entityContextParams = {
|
|
|
122
122
|
// The handler context is always chain-scoped, so a per-chain entity resolves to
|
|
123
123
|
// the chain the handler runs on and a cross-chain one to the shared partition.
|
|
124
124
|
let entityScope = (params: entityContextParams) =>
|
|
125
|
-
params.entityConfig->InMemoryStore.entityScope(
|
|
126
|
-
~chainId=params.item->Internal.getItemChainId,
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
let getWhereHandler = (params: entityContextParams, filter: dict<dict<unknown>>) => {
|
|
130
|
-
let entityConfig = params.entityConfig
|
|
131
|
-
|
|
132
|
-
@inline
|
|
133
|
-
let loadWithFilter = filter =>
|
|
134
|
-
LoadLayer.loadByFilter(
|
|
135
|
-
~loadManager=params.loadManager,
|
|
136
|
-
~persistence=params.persistence,
|
|
137
|
-
~entityConfig,
|
|
138
|
-
~scope=params->entityScope,
|
|
139
|
-
~indexerState=params.indexerState,
|
|
140
|
-
~shouldGroup=params.isPreload,
|
|
141
|
-
~item=params.item,
|
|
142
|
-
~ecosystem=params.config.ecosystem,
|
|
143
|
-
~filter,
|
|
144
|
-
)
|
|
125
|
+
params.entityConfig->InMemoryStore.entityScope(~chainId=params.item->Internal.getItemChainId)
|
|
145
126
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
~
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
127
|
+
let getWhereHandler = (params: entityContextParams, filter: dict<dict<unknown>>) =>
|
|
128
|
+
LoadLayer.loadByFilter(
|
|
129
|
+
~loadManager=params.loadManager,
|
|
130
|
+
~persistence=params.persistence,
|
|
131
|
+
~entityConfig=params.entityConfig,
|
|
132
|
+
~scope=params->entityScope,
|
|
133
|
+
~indexerState=params.indexerState,
|
|
134
|
+
~shouldGroup=params.isPreload,
|
|
135
|
+
~item=params.item,
|
|
136
|
+
~ecosystem=params.config.ecosystem,
|
|
137
|
+
~filter,
|
|
138
|
+
)
|
|
158
139
|
|
|
159
140
|
let noopSet = (_entity: Internal.entity) => ()
|
|
160
141
|
let noopDeleteUnsafe = (_entityId: EntityId.t) => ()
|
package/src/UserContext.res.mjs
CHANGED
|
@@ -77,13 +77,7 @@ function entityScope(params) {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
function getWhereHandler(params, filter) {
|
|
80
|
-
|
|
81
|
-
let filters = EntityFilter.parseGetWhereOrThrow(filter, entityConfig.name, entityConfig.table);
|
|
82
|
-
if (filters.length !== 1) {
|
|
83
|
-
return Promise.all(filters.map(filter => LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter))).then(results => results.flat());
|
|
84
|
-
} else {
|
|
85
|
-
return LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filters[0]);
|
|
86
|
-
}
|
|
80
|
+
return LoadLayer.loadByFilter(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter);
|
|
87
81
|
}
|
|
88
82
|
|
|
89
83
|
function noopSet(_entity) {
|
package/src/Utils.res
CHANGED
|
@@ -273,7 +273,7 @@ module Dict = {
|
|
|
273
273
|
}
|
|
274
274
|
|
|
275
275
|
module Math = {
|
|
276
|
-
let minOptInt = (a
|
|
276
|
+
let minOptInt = (a: option<int>, b: option<int>) =>
|
|
277
277
|
switch (a, b) {
|
|
278
278
|
| (Some(a), Some(b)) => Some(a < b ? a : b)
|
|
279
279
|
| (Some(a), None) => Some(a)
|
|
@@ -374,9 +374,6 @@ module Array = {
|
|
|
374
374
|
/**
|
|
375
375
|
Helper to check if a value exists in an array
|
|
376
376
|
*/
|
|
377
|
-
let includes = (arr: array<'a>, val: 'a) =>
|
|
378
|
-
arr->Array.find(item => item == val)->Stdlib.Option.isSome
|
|
379
|
-
|
|
380
377
|
let isEmpty = (arr: array<_>) =>
|
|
381
378
|
switch arr {
|
|
382
379
|
| [] => true
|
package/src/Utils.res.mjs
CHANGED
|
@@ -6,11 +6,9 @@ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
|
6
6
|
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
7
7
|
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
8
8
|
import * as Stdlib_BigInt from "@rescript/runtime/lib/es6/Stdlib_BigInt.js";
|
|
9
|
-
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
10
9
|
import * as Stdlib_String from "@rescript/runtime/lib/es6/Stdlib_String.js";
|
|
11
10
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
12
11
|
import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
|
|
13
|
-
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
14
12
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
15
13
|
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
16
14
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
@@ -221,19 +219,17 @@ let Dict = {
|
|
|
221
219
|
};
|
|
222
220
|
|
|
223
221
|
function minOptInt(a, b) {
|
|
224
|
-
if (a
|
|
222
|
+
if (a !== undefined) {
|
|
225
223
|
if (b !== undefined) {
|
|
226
|
-
return
|
|
224
|
+
return a < b ? a : b;
|
|
227
225
|
} else {
|
|
228
|
-
return;
|
|
226
|
+
return a;
|
|
229
227
|
}
|
|
228
|
+
} else if (b !== undefined) {
|
|
229
|
+
return b;
|
|
230
|
+
} else {
|
|
231
|
+
return;
|
|
230
232
|
}
|
|
231
|
-
let a$1 = Primitive_option.valFromOption(a);
|
|
232
|
-
if (b === undefined) {
|
|
233
|
-
return Primitive_option.some(a$1);
|
|
234
|
-
}
|
|
235
|
-
let b$1 = Primitive_option.valFromOption(b);
|
|
236
|
-
return Primitive_option.some(Primitive_object.lessthan(a$1, b$1) ? a$1 : b$1);
|
|
237
233
|
}
|
|
238
234
|
|
|
239
235
|
let $$Math = {
|
|
@@ -324,10 +320,6 @@ function transposeResults(results) {
|
|
|
324
320
|
};
|
|
325
321
|
}
|
|
326
322
|
|
|
327
|
-
function includes(arr, val) {
|
|
328
|
-
return Stdlib_Option.isSome(arr.find(item => Primitive_object.equal(item, val)));
|
|
329
|
-
}
|
|
330
|
-
|
|
331
323
|
function isEmpty$1(arr) {
|
|
332
324
|
return arr.length === 0;
|
|
333
325
|
}
|
|
@@ -435,7 +427,6 @@ let $$Array$1 = {
|
|
|
435
427
|
clearInPlace: clearInPlace$1,
|
|
436
428
|
setIndexImmutable: setIndexImmutable,
|
|
437
429
|
transposeResults: transposeResults,
|
|
438
|
-
includes: includes,
|
|
439
430
|
isEmpty: isEmpty$1,
|
|
440
431
|
notEmpty: notEmpty,
|
|
441
432
|
awaitEach: awaitEach,
|
package/src/Worker.res
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// The worker side of a supervised run: a process the supervisor forked to drive
|
|
2
|
+
// a subset of the chains. It has no server and no TUI of its own — it reports
|
|
3
|
+
// through the IPC channel, and the supervisor is the one operational surface.
|
|
4
|
+
|
|
5
|
+
// Set by a supervisor in the environment of the workers it forks. Internal:
|
|
6
|
+
// it counts only together with the fork's own channel, so a copy left in a
|
|
7
|
+
// shell starts nothing, and an indexer a user starts themselves takes every
|
|
8
|
+
// path it takes today.
|
|
9
|
+
let envVar = "ENVIO_INTERNAL_WORKER"
|
|
10
|
+
|
|
11
|
+
// What the supervisor decided about this worker, handed over in the spawn
|
|
12
|
+
// environment rather than over the channel: it is settled before the process
|
|
13
|
+
// starts, and the worker needs it before it can load its own config.
|
|
14
|
+
type config = {
|
|
15
|
+
chainIds: array<ChainId.t>,
|
|
16
|
+
// The chains this worker drives may reach the head while chains in another
|
|
17
|
+
// process are still backfilling, and an indexer goes realtime as a whole or
|
|
18
|
+
// not at all. Cleared by the supervisor's `ReleaseRealtime`.
|
|
19
|
+
holdRealtime: bool,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
let configSchema = S.object((s): config => {
|
|
23
|
+
chainIds: s.field("chainIds", S.array(ChainId.schema)),
|
|
24
|
+
holdRealtime: s.fieldOr("holdRealtime", S.bool, false),
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
// Read as this module loads, which is before anything that could catch a bare
|
|
28
|
+
// schema error and say where it came from.
|
|
29
|
+
let detect = (~env: dict<string>, ~hasChannel) =>
|
|
30
|
+
switch (hasChannel, env->Dict.get(envVar)) {
|
|
31
|
+
| (true, Some(json)) =>
|
|
32
|
+
switch json->S.parseJsonStringOrThrow(configSchema) {
|
|
33
|
+
| config => Some(config)
|
|
34
|
+
| exception S.Raised(error) =>
|
|
35
|
+
JsError.throwWithMessage(
|
|
36
|
+
`Invalid ${envVar}: ${error->S.Error.message}. It is set by an indexer supervisor for the processes it forks, and isn't meant to be set by hand.`,
|
|
37
|
+
)
|
|
38
|
+
}
|
|
39
|
+
| _ => None
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
let config = detect(
|
|
43
|
+
~env=NodeJs.Process.process.env,
|
|
44
|
+
~hasChannel=NodeJs.Process.channel->Nullable.toOption->Option.isSome,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
let isEnabled = config->Option.isSome
|
|
48
|
+
|
|
49
|
+
@tag("kind")
|
|
50
|
+
type parentMessage =
|
|
51
|
+
// Every chain in the run has reached the head, so this worker may enter the
|
|
52
|
+
// reorg threshold and switch to realtime with the rest of them.
|
|
53
|
+
| @as("release-realtime") ReleaseRealtime
|
|
54
|
+
|
|
55
|
+
@tag("kind")
|
|
56
|
+
type workerMessage =
|
|
57
|
+
| @as("snapshot") Snapshot({metrics: Metrics.t, runtime: Metrics.runtimeSample})
|
|
58
|
+
|
|
59
|
+
// How often a worker reports. Matches the TUI's own refresh, so the supervised
|
|
60
|
+
// display moves at the same rate an unsplit run's does.
|
|
61
|
+
%%private(let snapshotIntervalMillis = 500)
|
|
62
|
+
|
|
63
|
+
// The supervisor is the one that stops a worker, and the one whose absence
|
|
64
|
+
// ends it. A terminal's interrupt reaches the whole group at once, so the
|
|
65
|
+
// worker leaves it to the supervisor, which stops every worker in turn; without
|
|
66
|
+
// that, a worker gone on its own would read as a failure to the supervisor
|
|
67
|
+
// still deciding what the interrupt meant. A supervisor that dies can't tear
|
|
68
|
+
// the group down, so losing the channel is what ends the worker then.
|
|
69
|
+
let bindToSupervisor = () => {
|
|
70
|
+
NodeJs.Process.onSignal("SIGINT", () => ())
|
|
71
|
+
NodeJs.Process.onDisconnect(() => {
|
|
72
|
+
Logging.error("The indexer supervisor is gone. Stopping this chain's process.")
|
|
73
|
+
NodeJs.process->NodeJs.exitWithCode(Failure)
|
|
74
|
+
})
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
%%private(let send = (message: workerMessage) => NodeJs.Process.sendToParent(message)->ignore)
|
|
78
|
+
|
|
79
|
+
// Reports this process's chains and its own runtime for as long as it runs, so
|
|
80
|
+
// the supervisor can merge every worker's into the one snapshot the run serves,
|
|
81
|
+
// and listens for the one decision the supervisor makes on the run's behalf.
|
|
82
|
+
// Does nothing in a process nobody forked.
|
|
83
|
+
let bindRun = (~getMetrics: unit => Metrics.t, ~onReleaseRealtime: unit => unit) =>
|
|
84
|
+
if isEnabled {
|
|
85
|
+
Metrics.startRuntimeCollectors()
|
|
86
|
+
let _intervalId = setInterval(
|
|
87
|
+
() => send(Snapshot({metrics: getMetrics(), runtime: Metrics.sampleRuntime()})),
|
|
88
|
+
snapshotIntervalMillis,
|
|
89
|
+
)
|
|
90
|
+
NodeJs.Process.onMessage((message: parentMessage) =>
|
|
91
|
+
switch message {
|
|
92
|
+
| ReleaseRealtime => onReleaseRealtime()
|
|
93
|
+
}
|
|
94
|
+
)
|
|
95
|
+
}
|