envio 3.5.1 → 3.6.1-subgraph
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +7 -0
- package/fuel.schema.json +7 -0
- package/index.d.ts +33 -10
- package/package.json +6 -6
- package/src/ChainState.res +22 -4
- package/src/ChainState.res.mjs +20 -3
- package/src/ChainState.resi +3 -0
- package/src/Config.res +72 -2
- package/src/Config.res.mjs +57 -15
- package/src/Core.res +15 -0
- package/src/Core.res.mjs +11 -0
- package/src/EffectState.res +2 -2
- package/src/EffectState.res.mjs +2 -2
- package/src/EntityTables.res +32 -0
- package/src/EntityTables.res.mjs +44 -0
- package/src/Envio.res +7 -7
- package/src/Envio.res.mjs +5 -6
- package/src/EventProcessing.res +6 -6
- package/src/EventProcessing.res.mjs +8 -6
- package/src/HandlerLoader.res +20 -8
- package/src/HandlerLoader.res.mjs +11 -2
- package/src/Hasura.res +37 -3
- package/src/Hasura.res.mjs +23 -5
- package/src/InMemoryStore.res +48 -8
- package/src/InMemoryStore.res.mjs +37 -7
- package/src/IndexerState.res +30 -24
- package/src/IndexerState.res.mjs +20 -35
- package/src/IndexerState.resi +8 -6
- package/src/Internal.res +29 -11
- package/src/Internal.res.mjs +27 -10
- package/src/LoadLayer.res +39 -8
- package/src/LoadLayer.res.mjs +36 -9
- package/src/LoadLayer.resi +2 -0
- package/src/Metrics.res +5 -5
- package/src/Metrics.res.mjs +5 -1
- package/src/Persistence.res +12 -1
- package/src/PgStorage.res +183 -37
- package/src/PgStorage.res.mjs +149 -34
- package/src/PruneStaleHistory.res +1 -0
- package/src/PruneStaleHistory.res.mjs +2 -1
- package/src/Sink.res +3 -3
- package/src/Sink.res.mjs +2 -2
- package/src/TestIndexer.res +128 -17
- package/src/TestIndexer.res.mjs +75 -9
- package/src/UserContext.res +375 -53
- package/src/UserContext.res.mjs +281 -39
- package/src/Writing.res +8 -16
- package/src/Writing.res.mjs +10 -10
- package/src/bindings/ClickHouse.res +40 -4
- package/src/bindings/ClickHouse.res.mjs +37 -5
- package/src/db/EntityHistory.res +64 -21
- package/src/db/EntityHistory.res.mjs +43 -22
- package/src/db/InternalTable.res.mjs +31 -31
- package/src/db/Table.res +24 -1
- package/src/db/Table.res.mjs +26 -4
- package/src/sources/SourceManager.res +6 -12
- package/src/sources/SourceManager.res.mjs +8 -5
- package/src/subgraph/blocks.ts +176 -0
- package/src/subgraph/calls.ts +213 -0
- package/src/subgraph/conformance.ts +99 -0
- package/src/subgraph/division.ts +100 -0
- package/src/subgraph/errors.ts +90 -0
- package/src/subgraph/graph-ts-types/VERSION +2 -0
- package/src/subgraph/graph-ts-types/chain/arweave.d.ts +70 -0
- package/src/subgraph/graph-ts-types/chain/cosmos.d.ts +327 -0
- package/src/subgraph/graph-ts-types/chain/ethereum.d.ts +233 -0
- package/src/subgraph/graph-ts-types/chain/near.d.ts +253 -0
- package/src/subgraph/graph-ts-types/chain/starknet.d.ts +32 -0
- package/src/subgraph/graph-ts-types/common/collections.d.ts +136 -0
- package/src/subgraph/graph-ts-types/common/conversion.d.ts +11 -0
- package/src/subgraph/graph-ts-types/common/datasource.d.ts +30 -0
- package/src/subgraph/graph-ts-types/common/eager-offset.d.ts +0 -0
- package/src/subgraph/graph-ts-types/common/json.d.ts +17 -0
- package/src/subgraph/graph-ts-types/common/numbers.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/value.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/yaml.d.ts +90 -0
- package/src/subgraph/graph-ts-types/global/global.d.ts +194 -0
- package/src/subgraph/graph-ts-types/helper-functions.d.ts +22 -0
- package/src/subgraph/graph-ts-types/index.d.ts +102 -0
- package/src/subgraph/graph-ts.ts +1695 -0
- package/src/subgraph/hosts.ts +148 -0
- package/src/subgraph/runtime.ts +827 -0
- package/src/subgraph/scope.ts +63 -0
- package/svm.schema.json +7 -0
package/src/UserContext.res.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
4
5
|
import * as Logging from "./Logging.res.mjs";
|
|
5
6
|
import * as Internal from "./Internal.res.mjs";
|
|
6
7
|
import * as Ecosystem from "./Ecosystem.res.mjs";
|
|
@@ -12,8 +13,70 @@ import * as InMemoryStore from "./InMemoryStore.res.mjs";
|
|
|
12
13
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
13
14
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
14
15
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
16
|
+
import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
|
|
15
17
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
16
18
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
19
|
+
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
20
|
+
|
|
21
|
+
let Suspend = /* @__PURE__ */Primitive_exceptions.create("UserContext.Suspend");
|
|
22
|
+
|
|
23
|
+
function isSuspend(exn) {
|
|
24
|
+
return exn.RE_EXN_ID === Suspend;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function makeSyncState() {
|
|
28
|
+
return {
|
|
29
|
+
status: "Active",
|
|
30
|
+
pending: undefined,
|
|
31
|
+
memo: undefined
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function getPending(sync) {
|
|
36
|
+
let pending = sync.pending;
|
|
37
|
+
if (pending !== undefined) {
|
|
38
|
+
return pending;
|
|
39
|
+
}
|
|
40
|
+
let pending$1 = [];
|
|
41
|
+
sync.pending = pending$1;
|
|
42
|
+
return pending$1;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function getMemo(sync) {
|
|
46
|
+
let memo = sync.memo;
|
|
47
|
+
if (memo !== undefined) {
|
|
48
|
+
return memo;
|
|
49
|
+
}
|
|
50
|
+
let memo$1 = {};
|
|
51
|
+
sync.memo = memo$1;
|
|
52
|
+
return memo$1;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function checkStatusOrThrow(params, access) {
|
|
56
|
+
let exn = params.sync.status;
|
|
57
|
+
if (typeof exn !== "object") {
|
|
58
|
+
if (exn === "Active") {
|
|
59
|
+
return;
|
|
60
|
+
} else {
|
|
61
|
+
return ErrorHandling.mkLogAndRaise(Ecosystem.getItemLogger(params.item, params.config.ecosystem), undefined, new Error(`Impossible to access ` + access + ` after the handler is resolved. Make sure you didn't miss an await in the handler.`));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
throw exn._0;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function scheduleAndSuspend(params, promise) {
|
|
68
|
+
getPending(params.sync).push(promise);
|
|
69
|
+
params.sync.status = {
|
|
70
|
+
TAG: "Aborted",
|
|
71
|
+
_0: {
|
|
72
|
+
RE_EXN_ID: Suspend
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
throw {
|
|
76
|
+
RE_EXN_ID: Suspend,
|
|
77
|
+
Error: new Error()
|
|
78
|
+
};
|
|
79
|
+
}
|
|
17
80
|
|
|
18
81
|
let paramsByThis = new WeakMap();
|
|
19
82
|
|
|
@@ -47,36 +110,82 @@ var EffectContext = function(params, chainId, effectName, defaultShouldCache, ca
|
|
|
47
110
|
EffectContext.prototype = effectContextPrototype;
|
|
48
111
|
;
|
|
49
112
|
|
|
113
|
+
function prepareEffectCall(params, effect, input, caller) {
|
|
114
|
+
let isCrossChain = effect => Stdlib_Option.getOr(effect.crossChain, params.config.defaultCrossChain);
|
|
115
|
+
let scope = isCrossChain(effect) ? "crossChain" : Internal.getItemChainId(params.item);
|
|
116
|
+
if (caller !== undefined && isCrossChain(caller) && !isCrossChain(effect)) {
|
|
117
|
+
Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
|
|
118
|
+
}
|
|
119
|
+
let tmp;
|
|
120
|
+
tmp = scope === "crossChain" ? undefined : Primitive_option.some(scope);
|
|
121
|
+
let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, (nested, nestedInput) => callEffectAsync(params, nested, nestedInput, effect));
|
|
122
|
+
let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
|
|
123
|
+
let effectArgs_checkpointId = params.checkpointId;
|
|
124
|
+
let effectArgs = {
|
|
125
|
+
input: input,
|
|
126
|
+
context: effectContext,
|
|
127
|
+
cacheKey: effectArgs_cacheKey,
|
|
128
|
+
checkpointId: effectArgs_checkpointId
|
|
129
|
+
};
|
|
130
|
+
return [
|
|
131
|
+
scope,
|
|
132
|
+
effectArgs
|
|
133
|
+
];
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function callEffectAsync(params, effect, input, caller) {
|
|
137
|
+
let match = prepareEffectCall(params, effect, input, caller);
|
|
138
|
+
return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, match[1], match[0], params.indexerState, params.isPreload, params.item, params.config.ecosystem);
|
|
139
|
+
}
|
|
140
|
+
|
|
50
141
|
function initEffect(params) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
142
|
+
return (effect, input) => {
|
|
143
|
+
checkStatusOrThrow(params, "context.effect");
|
|
144
|
+
return callEffectAsync(params, effect, input, undefined);
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function effectMemoKey(effect, scope, cacheKey) {
|
|
149
|
+
if (scope === "crossChain") {
|
|
150
|
+
return effect.name + `.` + cacheKey;
|
|
151
|
+
} else {
|
|
152
|
+
return effect.name + `.` + ChainId.toString(scope) + `.` + cacheKey;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function initEffectSync(params) {
|
|
157
|
+
return (effect, input) => {
|
|
158
|
+
checkStatusOrThrow(params, "context.effectSync");
|
|
159
|
+
let match = prepareEffectCall(params, effect, input, undefined);
|
|
160
|
+
let effectArgs = match[1];
|
|
161
|
+
let scope = match[0];
|
|
162
|
+
let memo = getMemo(params.sync);
|
|
163
|
+
let memoKey = effectMemoKey(effect, scope, effectArgs.cacheKey);
|
|
164
|
+
let output = memo[memoKey];
|
|
165
|
+
if (output !== undefined) {
|
|
166
|
+
return Primitive_option.valFromOption(output);
|
|
56
167
|
}
|
|
57
|
-
let
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
let
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
});
|
|
70
|
-
return makeCaller(undefined);
|
|
168
|
+
let inMemTable = InMemoryStore.getEffectInMemTable(params.indexerState, effect, scope);
|
|
169
|
+
if (!InMemoryStore.hasEffectOutput(inMemTable, effectArgs.cacheKey)) {
|
|
170
|
+
return scheduleAndSuspend(params, LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem));
|
|
171
|
+
}
|
|
172
|
+
let output$1 = InMemoryStore.getEffectOutputUnsafe(inMemTable, effectArgs.cacheKey);
|
|
173
|
+
memo[memoKey] = output$1;
|
|
174
|
+
return output$1;
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function entityScope(params) {
|
|
179
|
+
return InMemoryStore.entityScope(params.entityConfig, Internal.getItemChainId(params.item));
|
|
71
180
|
}
|
|
72
181
|
|
|
73
182
|
function getWhereHandler(params, filter) {
|
|
74
183
|
let entityConfig = params.entityConfig;
|
|
75
184
|
let filters = EntityFilter.parseGetWhereOrThrow(filter, entityConfig.name, entityConfig.table);
|
|
76
185
|
if (filters.length !== 1) {
|
|
77
|
-
return Promise.all(filters.map(filter => LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter))).then(results => results.flat());
|
|
186
|
+
return Promise.all(filters.map(filter => LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter))).then(results => results.flat());
|
|
78
187
|
} else {
|
|
79
|
-
return LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, params.indexerState, params.isPreload, params.item, params.config.ecosystem, filters[0]);
|
|
188
|
+
return LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filters[0]);
|
|
80
189
|
}
|
|
81
190
|
}
|
|
82
191
|
|
|
@@ -92,36 +201,97 @@ function throwClickHouseReadOnly(entityConfig, op) {
|
|
|
92
201
|
return Stdlib_JsError.throwWithMessage(`context.` + entityConfig.name + `.` + op + `() is unavailable: ClickHouse storage is currently write-only. Follow Envio releases to be notified when ClickHouse supports both reads and writes from handlers.`);
|
|
93
202
|
}
|
|
94
203
|
|
|
204
|
+
function getSyncHandler(params, entityId) {
|
|
205
|
+
let inMemTable = InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params));
|
|
206
|
+
if (entityId in inMemTable.latestEntityChangeById) {
|
|
207
|
+
return InMemoryTable.Entity.getUnsafe(inMemTable)(entityId);
|
|
208
|
+
} else {
|
|
209
|
+
return scheduleAndSuspend(params, LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId));
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function getWhereSyncHandler(params, filter) {
|
|
214
|
+
let entityConfig = params.entityConfig;
|
|
215
|
+
let inMemTable = InMemoryStore.getInMemTable(params.indexerState, entityConfig, entityScope(params));
|
|
216
|
+
let hasIndex = InMemoryTable.Entity.hasIndex(inMemTable);
|
|
217
|
+
let getOnIndex = InMemoryTable.Entity.getUnsafeOnIndex(inMemTable);
|
|
218
|
+
let filters = EntityFilter.parseGetWhereOrThrow(filter, entityConfig.name, entityConfig.table);
|
|
219
|
+
let missing = [];
|
|
220
|
+
let entities = [];
|
|
221
|
+
filters.forEach(filter => {
|
|
222
|
+
let filterKey = EntityFilter.toString(filter);
|
|
223
|
+
if (hasIndex(filterKey)) {
|
|
224
|
+
entities.push(...getOnIndex(filterKey));
|
|
225
|
+
} else {
|
|
226
|
+
missing.push(LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter));
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
if (Utils.$$Array.notEmpty(missing)) {
|
|
230
|
+
let pending = getPending(params.sync);
|
|
231
|
+
missing.forEach(promise => {
|
|
232
|
+
pending.push(promise);
|
|
233
|
+
});
|
|
234
|
+
params.sync.status = {
|
|
235
|
+
TAG: "Aborted",
|
|
236
|
+
_0: {
|
|
237
|
+
RE_EXN_ID: Suspend
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
throw {
|
|
241
|
+
RE_EXN_ID: Suspend,
|
|
242
|
+
Error: new Error()
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
return entities;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function getInBlockSyncHandler(params, entityId) {
|
|
249
|
+
let inMemTable = InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params));
|
|
250
|
+
let change = inMemTable.latestEntityChangeById[entityId];
|
|
251
|
+
if (change !== undefined && change.checkpointId === params.checkpointId) {
|
|
252
|
+
return InMemoryTable.Entity.mapChangeToEntity(change);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
95
256
|
let entityTraps_get = (params, prop) => {
|
|
257
|
+
checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.` + prop);
|
|
96
258
|
let isClickHouseOnly = !params.entityConfig.storage.postgres;
|
|
97
|
-
let set = params.isPreload ? noopSet : entity =>
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
259
|
+
let set = params.isPreload ? noopSet : entity => {
|
|
260
|
+
checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.set`);
|
|
261
|
+
InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
|
|
262
|
+
type: "SET",
|
|
263
|
+
entityId: entity.id,
|
|
264
|
+
entity: entity,
|
|
265
|
+
checkpointId: params.checkpointId
|
|
266
|
+
});
|
|
267
|
+
};
|
|
103
268
|
switch (prop) {
|
|
104
269
|
case "deleteUnsafe" :
|
|
105
270
|
if (params.isPreload) {
|
|
106
271
|
return noopDeleteUnsafe;
|
|
107
272
|
} else {
|
|
108
|
-
return entityId =>
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
273
|
+
return entityId => {
|
|
274
|
+
checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.deleteUnsafe`);
|
|
275
|
+
InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
|
|
276
|
+
type: "DELETE",
|
|
277
|
+
entityId: entityId,
|
|
278
|
+
checkpointId: params.checkpointId
|
|
279
|
+
});
|
|
280
|
+
};
|
|
113
281
|
}
|
|
114
282
|
case "get" :
|
|
115
283
|
if (isClickHouseOnly) {
|
|
116
284
|
return _entityId => throwClickHouseReadOnly(params.entityConfig, "get");
|
|
117
285
|
} else {
|
|
118
|
-
return entityId => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId);
|
|
286
|
+
return entityId => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId);
|
|
119
287
|
}
|
|
288
|
+
case "getInBlockSync" :
|
|
289
|
+
return entityId => getInBlockSyncHandler(params, entityId);
|
|
120
290
|
case "getOrCreate" :
|
|
121
291
|
if (isClickHouseOnly) {
|
|
122
292
|
return _entity => throwClickHouseReadOnly(params.entityConfig, "getOrCreate");
|
|
123
293
|
} else {
|
|
124
|
-
return entity => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, params.indexerState, params.isPreload, params.item, params.config.ecosystem, entity.id).then(storageEntity => {
|
|
294
|
+
return entity => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entity.id).then(storageEntity => {
|
|
125
295
|
if (storageEntity !== undefined) {
|
|
126
296
|
return storageEntity;
|
|
127
297
|
} else {
|
|
@@ -134,7 +304,7 @@ let entityTraps_get = (params, prop) => {
|
|
|
134
304
|
if (isClickHouseOnly) {
|
|
135
305
|
return (_entityId, param) => throwClickHouseReadOnly(params.entityConfig, "getOrThrow");
|
|
136
306
|
} else {
|
|
137
|
-
return (entityId, message) => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId).then(entity => {
|
|
307
|
+
return (entityId, message) => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId).then(entity => {
|
|
138
308
|
if (entity !== undefined) {
|
|
139
309
|
return entity;
|
|
140
310
|
} else {
|
|
@@ -142,12 +312,16 @@ let entityTraps_get = (params, prop) => {
|
|
|
142
312
|
}
|
|
143
313
|
});
|
|
144
314
|
}
|
|
315
|
+
case "getSync" :
|
|
316
|
+
return entityId => getSyncHandler(params, entityId);
|
|
145
317
|
case "getWhere" :
|
|
146
318
|
if (isClickHouseOnly) {
|
|
147
319
|
return _filter => throwClickHouseReadOnly(params.entityConfig, "getWhere");
|
|
148
320
|
} else {
|
|
149
321
|
return filter => getWhereHandler(params, filter);
|
|
150
322
|
}
|
|
323
|
+
case "getWhereSync" :
|
|
324
|
+
return filter => getWhereSyncHandler(params, filter);
|
|
151
325
|
case "set" :
|
|
152
326
|
return set;
|
|
153
327
|
default:
|
|
@@ -159,16 +333,63 @@ let entityTraps = {
|
|
|
159
333
|
get: entityTraps_get
|
|
160
334
|
};
|
|
161
335
|
|
|
162
|
-
|
|
163
|
-
if (
|
|
164
|
-
|
|
336
|
+
async function runSyncRound(params, fn, round) {
|
|
337
|
+
if (round > 10000) {
|
|
338
|
+
Stdlib_JsError.throwWithMessage(`The handler suspended on a synchronous read too many times: gave up after ` + (10000).toString() + ` rounds. This usually means the code isn't deterministic across reruns.`);
|
|
339
|
+
}
|
|
340
|
+
params.sync.status = "Active";
|
|
341
|
+
params.sync.pending = undefined;
|
|
342
|
+
let suspended;
|
|
343
|
+
try {
|
|
344
|
+
fn();
|
|
345
|
+
suspended = false;
|
|
346
|
+
} catch (raw_exn) {
|
|
347
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
348
|
+
if (exn.RE_EXN_ID === Suspend) {
|
|
349
|
+
suspended = true;
|
|
350
|
+
} else {
|
|
351
|
+
let pending = params.sync.pending;
|
|
352
|
+
if (pending !== undefined) {
|
|
353
|
+
params.sync.pending = undefined;
|
|
354
|
+
pending.forEach(promise => {
|
|
355
|
+
Utils.$$Promise.silentCatch(promise);
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
throw exn;
|
|
359
|
+
}
|
|
165
360
|
}
|
|
361
|
+
let pending$1 = params.sync.pending;
|
|
362
|
+
if (pending$1 === undefined) {
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
params.sync.pending = undefined;
|
|
366
|
+
if (suspended) {
|
|
367
|
+
let errors = [];
|
|
368
|
+
await Promise.all(pending$1.map(promise => Stdlib_Promise.$$catch(promise, exn => {
|
|
369
|
+
errors.push(exn);
|
|
370
|
+
return Promise.resolve();
|
|
371
|
+
})));
|
|
372
|
+
let exn$1 = errors[0];
|
|
373
|
+
if (exn$1 !== undefined) {
|
|
374
|
+
throw exn$1;
|
|
375
|
+
}
|
|
376
|
+
return await runSyncRound(params, fn, round + 1 | 0);
|
|
377
|
+
}
|
|
378
|
+
pending$1.forEach(promise => {
|
|
379
|
+
Utils.$$Promise.silentCatch(promise);
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
let handlerTraps_get = (params, prop) => {
|
|
384
|
+
checkStatusOrThrow(params, `context.` + prop);
|
|
166
385
|
switch (prop) {
|
|
167
386
|
case "chain" :
|
|
168
387
|
let chainId = Internal.getItemChainId(params.item);
|
|
169
388
|
return params.chains[chainId];
|
|
170
389
|
case "effect" :
|
|
171
390
|
return initEffect(params);
|
|
391
|
+
case "effectSync" :
|
|
392
|
+
return initEffectSync(params);
|
|
172
393
|
case "isPreload" :
|
|
173
394
|
return params.isPreload;
|
|
174
395
|
case "log" :
|
|
@@ -177,6 +398,8 @@ let handlerTraps_get = (params, prop) => {
|
|
|
177
398
|
} else {
|
|
178
399
|
return Ecosystem.getItemUserLogger(params.item, params.config.ecosystem);
|
|
179
400
|
}
|
|
401
|
+
case "runSync" :
|
|
402
|
+
return fn => runSyncRound(params, fn, 1);
|
|
180
403
|
default:
|
|
181
404
|
let entityConfig = params.config.userEntitiesByName[prop];
|
|
182
405
|
if (entityConfig !== undefined) {
|
|
@@ -189,7 +412,7 @@ let handlerTraps_get = (params, prop) => {
|
|
|
189
412
|
isPreload: params.isPreload,
|
|
190
413
|
chains: params.chains,
|
|
191
414
|
config: params.config,
|
|
192
|
-
|
|
415
|
+
sync: params.sync,
|
|
193
416
|
entityConfig: entityConfig
|
|
194
417
|
}, entityTraps);
|
|
195
418
|
} else {
|
|
@@ -206,15 +429,34 @@ function getHandlerContext(params) {
|
|
|
206
429
|
return new Proxy(params, handlerTraps);
|
|
207
430
|
}
|
|
208
431
|
|
|
432
|
+
let maxSyncRounds = 10000;
|
|
433
|
+
|
|
209
434
|
export {
|
|
435
|
+
Suspend,
|
|
436
|
+
isSuspend,
|
|
437
|
+
makeSyncState,
|
|
438
|
+
getPending,
|
|
439
|
+
getMemo,
|
|
440
|
+
checkStatusOrThrow,
|
|
441
|
+
scheduleAndSuspend,
|
|
210
442
|
paramsByThis,
|
|
211
443
|
effectContextPrototype,
|
|
444
|
+
prepareEffectCall,
|
|
445
|
+
callEffectAsync,
|
|
212
446
|
initEffect,
|
|
447
|
+
effectMemoKey,
|
|
448
|
+
initEffectSync,
|
|
449
|
+
entityScope,
|
|
213
450
|
getWhereHandler,
|
|
214
451
|
noopSet,
|
|
215
452
|
noopDeleteUnsafe,
|
|
216
453
|
throwClickHouseReadOnly,
|
|
454
|
+
getSyncHandler,
|
|
455
|
+
getWhereSyncHandler,
|
|
456
|
+
getInBlockSyncHandler,
|
|
217
457
|
entityTraps,
|
|
458
|
+
maxSyncRounds,
|
|
459
|
+
runSyncRound,
|
|
218
460
|
handlerTraps,
|
|
219
461
|
getHandlerContext,
|
|
220
462
|
}
|
package/src/Writing.res
CHANGED
|
@@ -10,10 +10,8 @@ let keepLatestChangesLimit = Env.inMemoryObjectsTarget
|
|
|
10
10
|
|
|
11
11
|
let getChangesCount = (state: IndexerState.t) => {
|
|
12
12
|
let total = ref(0.)
|
|
13
|
-
state
|
|
14
|
-
|
|
15
|
-
->Array.forEach(entityConfig => {
|
|
16
|
-
total := total.contents +. (state->InMemoryStore.getInMemTable(~entityConfig)).changesCount
|
|
13
|
+
state->IndexerState.eachEntityTable((~entityConfig as _, ~scope as _, ~table) => {
|
|
14
|
+
total := total.contents +. table.changesCount
|
|
17
15
|
})
|
|
18
16
|
state
|
|
19
17
|
->IndexerState.effectState
|
|
@@ -113,14 +111,12 @@ let runOneWrite = async (state: IndexerState.t) => {
|
|
|
113
111
|
|
|
114
112
|
let rollback = state->IndexerState.takeRollback
|
|
115
113
|
|
|
116
|
-
let updatedEntities =
|
|
117
|
-
|
|
114
|
+
let updatedEntities = []
|
|
115
|
+
state->IndexerState.eachEntityTable((~entityConfig, ~scope, ~table) => {
|
|
118
116
|
let changes =
|
|
119
117
|
table->InMemoryTable.Entity.snapshotChanges(~committedCheckpointId, ~upToCheckpointId)
|
|
120
|
-
if changes->Utils.Array.
|
|
121
|
-
|
|
122
|
-
} else {
|
|
123
|
-
Some(({entityConfig, changes}: Persistence.updatedEntity))
|
|
118
|
+
if changes->Utils.Array.notEmpty {
|
|
119
|
+
updatedEntities->Array.push(({entityConfig, scope, changes}: Persistence.updatedEntity))
|
|
124
120
|
}
|
|
125
121
|
})
|
|
126
122
|
let updatedEffectsCache = snapshotEffects(state, ~cache)
|
|
@@ -221,12 +217,8 @@ let commitBatch = (state: IndexerState.t, ~batch: Batch.t) => {
|
|
|
221
217
|
// keepLoadedFromDb, entries seeded from a db read are spared.
|
|
222
218
|
let dropCommitted = (state: IndexerState.t, ~keepLoadedFromDb) => {
|
|
223
219
|
let committedCheckpointId = state->IndexerState.committedCheckpointId
|
|
224
|
-
state
|
|
225
|
-
|
|
226
|
-
->Array.forEach(entityConfig =>
|
|
227
|
-
state
|
|
228
|
-
->InMemoryStore.getInMemTable(~entityConfig)
|
|
229
|
-
->InMemoryTable.Entity.dropCommittedChanges(~committedCheckpointId, ~keepLoadedFromDb)
|
|
220
|
+
state->IndexerState.eachEntityTable((~entityConfig as _, ~scope as _, ~table) =>
|
|
221
|
+
table->InMemoryTable.Entity.dropCommittedChanges(~committedCheckpointId, ~keepLoadedFromDb)
|
|
230
222
|
)
|
|
231
223
|
state
|
|
232
224
|
->IndexerState.effectState
|
package/src/Writing.res.mjs
CHANGED
|
@@ -19,8 +19,8 @@ function getChangesCount(state) {
|
|
|
19
19
|
let total = {
|
|
20
20
|
contents: 0
|
|
21
21
|
};
|
|
22
|
-
IndexerState.
|
|
23
|
-
total.contents = total.contents +
|
|
22
|
+
IndexerState.eachEntityTable(state, (param, param$1, table) => {
|
|
23
|
+
total.contents = total.contents + table.changesCount;
|
|
24
24
|
});
|
|
25
25
|
EffectState.forEach(IndexerState.effectState(state), inMemTable => {
|
|
26
26
|
total.contents = total.contents + inMemTable.changesCount;
|
|
@@ -105,16 +105,16 @@ async function runOneWrite(state) {
|
|
|
105
105
|
let checkpointId = Utils.$$Array.last(batch.checkpointIds);
|
|
106
106
|
let upToCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
|
|
107
107
|
let rollback = IndexerState.takeRollback(state);
|
|
108
|
-
let updatedEntities =
|
|
109
|
-
|
|
108
|
+
let updatedEntities = [];
|
|
109
|
+
IndexerState.eachEntityTable(state, (entityConfig, scope, table) => {
|
|
110
110
|
let changes = InMemoryTable.Entity.snapshotChanges(table, committedCheckpointId, upToCheckpointId);
|
|
111
|
-
if (Utils.$$Array.
|
|
112
|
-
|
|
113
|
-
} else {
|
|
114
|
-
return {
|
|
111
|
+
if (Utils.$$Array.notEmpty(changes)) {
|
|
112
|
+
updatedEntities.push({
|
|
115
113
|
entityConfig: entityConfig,
|
|
114
|
+
scope: scope,
|
|
116
115
|
changes: changes
|
|
117
|
-
};
|
|
116
|
+
});
|
|
117
|
+
return;
|
|
118
118
|
}
|
|
119
119
|
});
|
|
120
120
|
let updatedEffectsCache = snapshotEffects(state, cache);
|
|
@@ -179,7 +179,7 @@ function commitBatch(state, batch) {
|
|
|
179
179
|
|
|
180
180
|
function dropCommitted(state, keepLoadedFromDb) {
|
|
181
181
|
let committedCheckpointId = IndexerState.committedCheckpointId(state);
|
|
182
|
-
IndexerState.
|
|
182
|
+
IndexerState.eachEntityTable(state, (param, param$1, table) => InMemoryTable.Entity.dropCommittedChanges(table, committedCheckpointId, keepLoadedFromDb));
|
|
183
183
|
EffectState.forEach(IndexerState.effectState(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
|
|
184
184
|
}
|
|
185
185
|
|
|
@@ -248,15 +248,21 @@ type setUpdatesCache = {
|
|
|
248
248
|
|
|
249
249
|
let setUpdatesOrThrow = async (
|
|
250
250
|
client,
|
|
251
|
-
~cache:
|
|
251
|
+
~cache: dict<setUpdatesCache>,
|
|
252
252
|
~changes: array<Change.t<Internal.entity>>,
|
|
253
253
|
~entityConfig: Internal.entityConfig,
|
|
254
|
+
~scope: Internal.chainScope,
|
|
254
255
|
~database: string,
|
|
255
256
|
) => {
|
|
256
257
|
if changes->Array.length === 0 {
|
|
257
258
|
()
|
|
258
259
|
} else {
|
|
259
|
-
let
|
|
260
|
+
let chainIdField = entityConfig.table->Table.getChainIdField
|
|
261
|
+
let scopeChainId = scope->Internal.chainScopeChainId
|
|
262
|
+
// A delete row carries no entity to stamp, so the chain id is baked into
|
|
263
|
+
// the schema instead — which is why the cache is keyed per scope.
|
|
264
|
+
let cacheKey = `${entityConfig.name}|${scope->Internal.chainScopeToString}`
|
|
265
|
+
let {convertOrThrow, tableName} = switch cache->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
|
|
260
266
|
| Some(cached) => cached
|
|
261
267
|
| None =>
|
|
262
268
|
let cached: setUpdatesCache = {
|
|
@@ -273,6 +279,11 @@ let setUpdatesOrThrow = async (
|
|
|
273
279
|
),
|
|
274
280
|
S.object(s => {
|
|
275
281
|
s.tag(EntityHistory.changeFieldName, EntityHistory.RowAction.DELETE)
|
|
282
|
+
switch (chainIdField, scopeChainId) {
|
|
283
|
+
| (Some(field), Some(chainId)) =>
|
|
284
|
+
s.tag(field->Table.getClickHouseDbFieldName, chainId)
|
|
285
|
+
| _ => ()
|
|
286
|
+
}
|
|
276
287
|
Change.Delete({
|
|
277
288
|
entityId: s.field(Table.idFieldName, entityConfig.table->Table.getIdSchema),
|
|
278
289
|
checkpointId: s.field(
|
|
@@ -294,10 +305,25 @@ let setUpdatesOrThrow = async (
|
|
|
294
305
|
),
|
|
295
306
|
}
|
|
296
307
|
|
|
297
|
-
cache->
|
|
308
|
+
cache->Dict.set(cacheKey, cached)
|
|
298
309
|
cached
|
|
299
310
|
}
|
|
300
311
|
|
|
312
|
+
let changes = switch (chainIdField, scopeChainId) {
|
|
313
|
+
| (Some(field), Some(chainId)) =>
|
|
314
|
+
changes->Array.map(change =>
|
|
315
|
+
switch change {
|
|
316
|
+
| Change.Set(set) =>
|
|
317
|
+
Change.Set({
|
|
318
|
+
...set,
|
|
319
|
+
entity: set.entity->Internal.stampChainId(~fieldName=field.fieldName, ~chainId),
|
|
320
|
+
})
|
|
321
|
+
| Delete(_) => change
|
|
322
|
+
}
|
|
323
|
+
)
|
|
324
|
+
| _ => changes
|
|
325
|
+
}
|
|
326
|
+
|
|
301
327
|
try {
|
|
302
328
|
// The entity history table is the source of truth for ClickHouse, so every
|
|
303
329
|
// intermediate change must be persisted, not only the current value.
|
|
@@ -505,6 +531,16 @@ let makeCreateViewQuery = (
|
|
|
505
531
|
~entityIndex=entityConfig.index,
|
|
506
532
|
)
|
|
507
533
|
|
|
534
|
+
// A per-chain entity's rows are only comparable within a chain, so the
|
|
535
|
+
// current-state dedup keys on (id, chain id).
|
|
536
|
+
let dedupKey =
|
|
537
|
+
switch entityConfig.table->Table.getChainIdField {
|
|
538
|
+
| Some(field) => [Table.idFieldName, field->Table.getClickHouseDbFieldName]
|
|
539
|
+
| None => [Table.idFieldName]
|
|
540
|
+
}
|
|
541
|
+
->Array.map(name => `\`${name}\``)
|
|
542
|
+
->Array.joinUnsafe(", ")
|
|
543
|
+
|
|
508
544
|
let checkpointsTableName = InternalTable.Checkpoints.table.tableName
|
|
509
545
|
let checkpointIdField = (#id: InternalTable.Checkpoints.field :> string)
|
|
510
546
|
|
|
@@ -528,7 +564,7 @@ FROM (
|
|
|
528
564
|
FROM ${database}.\`${historyTableName}\`
|
|
529
565
|
WHERE \`${EntityHistory.checkpointIdFieldName}\` <= (SELECT max(${checkpointIdField}) FROM ${database}.\`${checkpointsTableName}\`)
|
|
530
566
|
ORDER BY \`${EntityHistory.checkpointIdFieldName}\` DESC
|
|
531
|
-
LIMIT 1 BY
|
|
567
|
+
LIMIT 1 BY ${dedupKey}
|
|
532
568
|
)
|
|
533
569
|
WHERE \`${EntityHistory.changeFieldName}\` = '${(EntityHistory.RowAction.SET :> string)}'`
|
|
534
570
|
}
|