envio 3.3.0-alpha.9 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/index.d.ts +19 -0
  2. package/licenses/CLA.md +35 -0
  3. package/licenses/EULA.md +67 -0
  4. package/licenses/LICENSE.md +45 -0
  5. package/licenses/README.md +35 -0
  6. package/package.json +9 -8
  7. package/src/Batch.res.mjs +1 -1
  8. package/src/BatchProcessing.res +0 -7
  9. package/src/BatchProcessing.res.mjs +1 -8
  10. package/src/ChainFetching.res +30 -18
  11. package/src/ChainFetching.res.mjs +23 -13
  12. package/src/ChainState.res +134 -55
  13. package/src/ChainState.res.mjs +78 -36
  14. package/src/ChainState.resi +14 -2
  15. package/src/Config.res +9 -5
  16. package/src/Config.res.mjs +2 -2
  17. package/src/Core.res +20 -0
  18. package/src/Core.res.mjs +12 -0
  19. package/src/CrossChainState.res +115 -34
  20. package/src/CrossChainState.res.mjs +37 -14
  21. package/src/EffectState.res +100 -0
  22. package/src/EffectState.res.mjs +74 -0
  23. package/src/EffectState.resi +32 -0
  24. package/src/Envio.res +25 -7
  25. package/src/Envio.res.mjs +15 -19
  26. package/src/EventConfigBuilder.res +21 -46
  27. package/src/EventConfigBuilder.res.mjs +18 -25
  28. package/src/EventProcessing.res +8 -5
  29. package/src/EventProcessing.res.mjs +2 -1
  30. package/src/FetchState.res +814 -467
  31. package/src/FetchState.res.mjs +541 -369
  32. package/src/HandlerRegister.res +3 -1
  33. package/src/HandlerRegister.res.mjs +3 -2
  34. package/src/InMemoryStore.res +14 -26
  35. package/src/InMemoryStore.res.mjs +6 -19
  36. package/src/IndexerState.res +15 -39
  37. package/src/IndexerState.res.mjs +18 -29
  38. package/src/IndexerState.resi +2 -10
  39. package/src/Internal.res +98 -14
  40. package/src/Internal.res.mjs +96 -2
  41. package/src/LoadLayer.res +44 -24
  42. package/src/LoadLayer.res.mjs +43 -20
  43. package/src/LoadLayer.resi +1 -0
  44. package/src/LogSelection.res +0 -62
  45. package/src/LogSelection.res.mjs +0 -74
  46. package/src/Metrics.res +1 -1
  47. package/src/Metrics.res.mjs +1 -1
  48. package/src/Persistence.res +12 -3
  49. package/src/PgStorage.res +155 -82
  50. package/src/PgStorage.res.mjs +130 -77
  51. package/src/Prometheus.res +20 -10
  52. package/src/Prometheus.res.mjs +22 -10
  53. package/src/PruneStaleHistory.res +146 -35
  54. package/src/PruneStaleHistory.res.mjs +114 -20
  55. package/src/RawEvent.res +2 -2
  56. package/src/Rollback.res +32 -13
  57. package/src/Rollback.res.mjs +24 -11
  58. package/src/SimulateDeadInputTracker.res +1 -1
  59. package/src/SimulateItems.res +38 -6
  60. package/src/SimulateItems.res.mjs +28 -9
  61. package/src/TestIndexer.res +2 -2
  62. package/src/TestIndexer.res.mjs +2 -2
  63. package/src/TopicFilter.res +1 -25
  64. package/src/TopicFilter.res.mjs +0 -74
  65. package/src/UserContext.res +62 -23
  66. package/src/UserContext.res.mjs +26 -6
  67. package/src/Writing.res +60 -21
  68. package/src/Writing.res.mjs +27 -9
  69. package/src/bindings/NodeJs.res +5 -0
  70. package/src/bindings/Viem.res +0 -5
  71. package/src/sources/EventRouter.res +0 -21
  72. package/src/sources/EventRouter.res.mjs +0 -12
  73. package/src/sources/EvmChain.res +2 -27
  74. package/src/sources/EvmChain.res.mjs +2 -23
  75. package/src/sources/EvmRpcClient.res +58 -23
  76. package/src/sources/EvmRpcClient.res.mjs +11 -5
  77. package/src/sources/HyperSync.res +9 -38
  78. package/src/sources/HyperSync.res.mjs +16 -28
  79. package/src/sources/HyperSync.resi +2 -2
  80. package/src/sources/HyperSyncClient.res +88 -11
  81. package/src/sources/HyperSyncClient.res.mjs +39 -6
  82. package/src/sources/HyperSyncSource.res +18 -199
  83. package/src/sources/HyperSyncSource.res.mjs +9 -128
  84. package/src/sources/Rpc.res +0 -32
  85. package/src/sources/Rpc.res.mjs +1 -46
  86. package/src/sources/RpcSource.res +129 -522
  87. package/src/sources/RpcSource.res.mjs +161 -366
  88. package/src/sources/SimulateSource.res +37 -19
  89. package/src/sources/SimulateSource.res.mjs +27 -10
  90. package/src/sources/SourceManager.res +3 -7
  91. package/src/sources/SourceManager.res.mjs +2 -7
  92. package/src/sources/SourceManager.resi +0 -2
  93. package/src/sources/SvmHyperSyncSource.res +13 -3
  94. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
@@ -679,7 +679,9 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => {
679
679
  config.ecosystem.name === Evm &&
680
680
  (registration->getResolvedWhere).topicSelections->Utils.Array.isEmpty
681
681
  if !isDroppedByWhere {
682
- onEventRegistrations->Array.push(registration)
682
+ onEventRegistrations
683
+ ->Array.push({...registration, index: onEventRegistrations->Array.length})
684
+ ->ignore
683
685
  }
684
686
  | None => ()
685
687
  }
@@ -388,10 +388,11 @@ function finishRegistration(config) {
388
388
  return;
389
389
  }
390
390
  let isDroppedByWhere = config.ecosystem.name === "evm" && Utils.$$Array.isEmpty(registration$1.resolvedWhere.topicSelections);
391
- if (!isDroppedByWhere) {
392
- onEventRegistrations.push(registration$1);
391
+ if (isDroppedByWhere) {
393
392
  return;
394
393
  }
394
+ let newrecord = {...registration$1};
395
+ onEventRegistrations.push((newrecord.index = onEventRegistrations.length, newrecord));
395
396
  });
396
397
  });
397
398
  registrationsByChainId[key] = {
@@ -8,25 +8,13 @@ let getInMemTable = (
8
8
  ): InMemoryTable.Entity.t =>
9
9
  state->IndexerState.entities->IndexerState.EntityTables.get(~entityName=entityConfig.name)
10
10
 
11
- let getEffectInMemTable = (state: IndexerState.t, ~effect: Internal.effect) => {
12
- let key = effect.name
13
- let effects = state->IndexerState.effects
14
- switch effects->Utils.Dict.dangerouslyGetNonOption(key) {
15
- | Some(table) => table
16
- | None =>
17
- let table: IndexerState.effectCacheInMemTable = {
18
- idsToStore: [],
19
- dict: Dict.make(),
20
- changesCount: 0.,
21
- invalidationsCount: 0,
22
- effect,
23
- }
24
- effects->Dict.set(key, table)
25
- table
26
- }
27
- }
11
+ let getEffectInMemTable = (
12
+ state: IndexerState.t,
13
+ ~effect: Internal.effect,
14
+ ~scope: Internal.chainScope,
15
+ ) => state->IndexerState.effectState->EffectState.getTable(~effect, ~scope)
28
16
 
29
- let hasEffectOutput = (inMemTable: IndexerState.effectCacheInMemTable, key) =>
17
+ let hasEffectOutput = (inMemTable: EffectState.effectCacheInMemTable, key) =>
30
18
  switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
31
19
  | Some(Set(_)) => true
32
20
  | Some(Delete(_)) | None => false
@@ -36,7 +24,7 @@ let hasEffectOutput = (inMemTable: IndexerState.effectCacheInMemTable, key) =>
36
24
  // optional output, so it must never be wrapped in another option here: Some(None)
37
25
  // is encoded as the nested-option sentinel and would leak to the handler.
38
26
  let getEffectOutputUnsafe = (
39
- inMemTable: IndexerState.effectCacheInMemTable,
27
+ inMemTable: EffectState.effectCacheInMemTable,
40
28
  key,
41
29
  ): Internal.effectOutput =>
42
30
  switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
@@ -47,7 +35,7 @@ let getEffectOutputUnsafe = (
47
35
  // Records a handler output. Persisted on the next write only when shouldCache;
48
36
  // otherwise kept in memory (re-run on a later miss) but never written to the db.
49
37
  let setEffectOutput = (
50
- inMemTable: IndexerState.effectCacheInMemTable,
38
+ inMemTable: EffectState.effectCacheInMemTable,
51
39
  ~checkpointId,
52
40
  ~cacheKey,
53
41
  ~output,
@@ -65,7 +53,7 @@ let setEffectOutput = (
65
53
 
66
54
  // Seeds an entry from a db read. Stamped with loadedFromDbCheckpointId so it's
67
55
  // always droppable (re-readable from the db) and never re-persisted.
68
- let initEffectOutputFromDb = (inMemTable: IndexerState.effectCacheInMemTable, ~cacheKey, ~output) =>
56
+ let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~cacheKey, ~output) =>
69
57
  if inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(cacheKey)->Option.isNone {
70
58
  inMemTable.changesCount = inMemTable.changesCount +. 1.
71
59
  inMemTable.dict->Dict.set(
@@ -78,7 +66,7 @@ let initEffectOutputFromDb = (inMemTable: IndexerState.effectCacheInMemTable, ~c
78
66
  // cache:false). Uncommitted entries stay warm. With keepLoadedFromDb, entries
79
67
  // seeded from a db read are spared. Mirrors entity dropCommittedChanges.
80
68
  let dropCommittedEffects = (
81
- inMemTable: IndexerState.effectCacheInMemTable,
69
+ inMemTable: EffectState.effectCacheInMemTable,
82
70
  ~committedCheckpointId,
83
71
  ~keepLoadedFromDb,
84
72
  ) => {
@@ -117,17 +105,17 @@ let prepareRollbackDiff = async (
117
105
  ->Array.map(async entityConfig => {
118
106
  let entityTable = state->getInMemTable(~entityConfig)
119
107
 
120
- let (removedIdsResult, restoredEntitiesResult) = await persistence.storage.getRollbackData(
108
+ let (removedIds, restoredEntitiesResult) = await persistence.storage.getRollbackData(
121
109
  ~entityConfig,
122
110
  ~rollbackTargetCheckpointId,
123
111
  )
124
112
 
125
- removedIdsResult->Array.forEach(data => {
126
- deletedEntities->Utils.Dict.push(entityConfig.name, data["id"])
113
+ removedIds->Array.forEach(entityId => {
114
+ deletedEntities->Utils.Dict.push(entityConfig.name, entityId)
127
115
  entityTable->InMemoryTable.Entity.set(
128
116
  ~committedCheckpointId,
129
117
  Delete({
130
- entityId: data["id"],
118
+ entityId,
131
119
  checkpointId: rollbackDiffCheckpointId,
132
120
  }),
133
121
  )
@@ -4,6 +4,7 @@ import * as Table from "./db/Table.res.mjs";
4
4
  import * as Utils from "./Utils.res.mjs";
5
5
  import * as Config from "./Config.res.mjs";
6
6
  import * as Internal from "./Internal.res.mjs";
7
+ import * as EffectState from "./EffectState.res.mjs";
7
8
  import * as IndexerState from "./IndexerState.res.mjs";
8
9
  import * as InMemoryTable from "./InMemoryTable.res.mjs";
9
10
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
@@ -13,22 +14,8 @@ function getInMemTable(state, entityConfig) {
13
14
  return IndexerState.EntityTables.get(IndexerState.entities(state), entityConfig.name);
14
15
  }
15
16
 
16
- function getEffectInMemTable(state, effect) {
17
- let key = effect.name;
18
- let effects = IndexerState.effects(state);
19
- let table = effects[key];
20
- if (table !== undefined) {
21
- return table;
22
- }
23
- let table$1 = {
24
- idsToStore: [],
25
- invalidationsCount: 0,
26
- dict: {},
27
- changesCount: 0,
28
- effect: effect
29
- };
30
- effects[key] = table$1;
31
- return table$1;
17
+ function getEffectInMemTable(state, effect, scope) {
18
+ return EffectState.getTable(IndexerState.effectState(state), effect, scope);
32
19
  }
33
20
 
34
21
  function hasEffectOutput(inMemTable, key) {
@@ -102,11 +89,11 @@ async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDi
102
89
  await Promise.all(persistence.allEntities.map(async entityConfig => {
103
90
  let entityTable = getInMemTable(state, entityConfig);
104
91
  let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
105
- match[0].forEach(data => {
106
- Utils.Dict.push(deletedEntities, entityConfig.name, data.id);
92
+ match[0].forEach(entityId => {
93
+ Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
107
94
  InMemoryTable.Entity.set(entityTable, committedCheckpointId, {
108
95
  type: "DELETE",
109
- entityId: data.id,
96
+ entityId: entityId,
110
97
  checkpointId: rollbackDiffCheckpointId
111
98
  });
112
99
  });
@@ -6,23 +6,6 @@ type rollbackState =
6
6
  | FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
7
7
  | RollbackReady({eventsProcessedDiffByChain: dict<float>})
8
8
 
9
- module WriteThrottlers = {
10
- type t = {pruneStaleEntityHistory: Throttler.t}
11
- let make = (): t => {
12
- let pruneStaleEntityHistory = {
13
- let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis
14
- let logger = Logging.createChild(
15
- ~params={
16
- "context": "Throttler for pruning stale entity history data",
17
- "intervalMillis": intervalMillis,
18
- },
19
- )
20
- Throttler.make(~intervalMillis, ~logger)
21
- }
22
- {pruneStaleEntityHistory: pruneStaleEntityHistory}
23
- }
24
- }
25
-
26
9
  module EntityTables = {
27
10
  type t = dict<InMemoryTable.Entity.t>
28
11
  exception UndefinedEntity({entityName: string})
@@ -45,26 +28,13 @@ module EntityTables = {
45
28
  }
46
29
  }
47
30
 
48
- type effectCacheInMemTable = {
49
- // Cache keys whose handler output is persisted on the next write. Drained
50
- // each write; eviction is driven by the per-entry checkpointId instead.
51
- mutable idsToStore: array<string>,
52
- mutable invalidationsCount: int,
53
- // Each entry is stamped with the checkpoint that referenced it (or
54
- // loadedFromDbCheckpointId for db reads), so committed entries can be
55
- // dropped once persisted/re-derivable, mirroring entity changes.
56
- mutable dict: dict<Change.t<Internal.effectOutput>>,
57
- mutable changesCount: float,
58
- effect: Internal.effect,
59
- }
60
-
61
31
  type t = {
62
32
  config: Config.t,
63
33
  persistence: Persistence.t,
64
34
  // --- In-memory store: entity/effect tables and the pending-write queue. ---
65
35
  allEntities: array<Internal.entityConfig>,
66
36
  mutable entities: EntityTables.t,
67
- mutable effects: dict<effectCacheInMemTable>,
37
+ effectState: EffectState.t,
68
38
  mutable rollback: option<Persistence.rollback>,
69
39
  // Last checkpoint persisted to the db.
70
40
  mutable committedCheckpointId: Internal.checkpointId,
@@ -94,7 +64,9 @@ type t = {
94
64
  crossChainState: CrossChainState.t,
95
65
  mutable rollbackState: rollbackState,
96
66
  indexerStartTime: Date.t,
97
- writeThrottlers: WriteThrottlers.t,
67
+ // When an entity's history was last pruned. No key = never pruned yet,
68
+ // which counts as overdue.
69
+ lastPrunedAtMillis: dict<float>,
98
70
  loadManager: LoadManager.t,
99
71
  keepProcessAlive: bool,
100
72
  exitAfterFirstEventBlock: bool,
@@ -143,7 +115,7 @@ let make = (
143
115
  persistence,
144
116
  allEntities: persistence.allEntities,
145
117
  entities: EntityTables.make(persistence.allEntities),
146
- effects: Dict.make(),
118
+ effectState: EffectState.make(),
147
119
  rollback: None,
148
120
  committedCheckpointId,
149
121
  processedCheckpointId: committedCheckpointId,
@@ -164,7 +136,7 @@ let make = (
164
136
  ),
165
137
  indexerStartTime: Date.make(),
166
138
  rollbackState: NoRollback,
167
- writeThrottlers: WriteThrottlers.make(),
139
+ lastPrunedAtMillis: Dict.make(),
168
140
  loadManager: LoadManager.make(),
169
141
  keepProcessAlive: isDevelopmentMode || shouldUseTui,
170
142
  exitAfterFirstEventBlock,
@@ -211,8 +183,12 @@ let makeFromDbState = (
211
183
 
212
184
  Prometheus.ProcessingMaxBatchSize.set(~maxBatchSize=config.batchSize)
213
185
  Prometheus.ReorgThreshold.set(~isInReorgThreshold)
214
- initialState.cache->Utils.Dict.forEach(({effectName, count}) => {
215
- Prometheus.EffectCacheCount.set(~count, ~effectName)
186
+ initialState.cache->Utils.Dict.forEach(({effectName, count, scope}) => {
187
+ Prometheus.EffectCacheCount.set(
188
+ ~count,
189
+ ~effectName,
190
+ ~scope=scope->Internal.EffectCache.scopeToString,
191
+ )
216
192
  })
217
193
 
218
194
  // updateSyncTimeOnRestart wipes the saved timestamp so a restart re-enters
@@ -385,7 +361,7 @@ let config = (state: t) => state.config
385
361
  let persistence = (state: t) => state.persistence
386
362
  let allEntities = (state: t) => state.allEntities
387
363
  let entities = (state: t) => state.entities
388
- let effects = (state: t) => state.effects
364
+ let effectState = (state: t) => state.effectState
389
365
  let committedCheckpointId = (state: t) => state.committedCheckpointId
390
366
  let processedCheckpointId = (state: t) => state.processedCheckpointId
391
367
  let processedBatches = (state: t) => state.processedBatches
@@ -405,7 +381,7 @@ let keepProcessAlive = (state: t) => state.keepProcessAlive
405
381
  let exitAfterFirstEventBlock = (state: t) => state.exitAfterFirstEventBlock
406
382
  let isStopped = (state: t) => state.isStopped
407
383
  let epoch = (state: t) => state.epoch
408
- let pruneStaleEntityHistoryThrottler = (state: t) => state.writeThrottlers.pruneStaleEntityHistory
384
+ let lastPrunedAtMillis = (state: t) => state.lastPrunedAtMillis
409
385
  let simulateDeadInputTracker = (state: t) => state.simulateDeadInputTracker
410
386
 
411
387
  // --- Store domain operations. ---
@@ -485,7 +461,7 @@ let beginRollbackDiff = (
485
461
  ~progressBlockNumberByChainId,
486
462
  ) => {
487
463
  state.entities = EntityTables.make(state.allEntities)
488
- state.effects = Dict.make()
464
+ state.effectState->EffectState.resetForRollback
489
465
  state.rollback = Some({
490
466
  targetCheckpointId,
491
467
  diffCheckpointId,
@@ -9,6 +9,7 @@ import * as Internal from "./Internal.res.mjs";
9
9
  import * as Throttler from "./Throttler.res.mjs";
10
10
  import * as ChainState from "./ChainState.res.mjs";
11
11
  import * as Prometheus from "./Prometheus.res.mjs";
12
+ import * as EffectState from "./EffectState.res.mjs";
12
13
  import * as LoadManager from "./LoadManager.res.mjs";
13
14
  import * as ErrorHandling from "./ErrorHandling.res.mjs";
14
15
  import * as InMemoryTable from "./InMemoryTable.res.mjs";
@@ -20,21 +21,9 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
20
21
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
21
22
  import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
22
23
 
23
- function make() {
24
- let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
25
- let logger = Logging.createChild({
26
- context: "Throttler for pruning stale entity history data",
27
- intervalMillis: intervalMillis
28
- });
29
- let pruneStaleEntityHistory = Throttler.make(intervalMillis, logger);
30
- return {
31
- pruneStaleEntityHistory: pruneStaleEntityHistory
32
- };
33
- }
34
-
35
24
  let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("IndexerState.EntityTables.UndefinedEntity");
36
25
 
37
- function make$1(entities) {
26
+ function make(entities) {
38
27
  let init = {};
39
28
  entities.forEach(entityConfig => {
40
29
  init[entityConfig.name] = InMemoryTable.Entity.make();
@@ -54,7 +43,7 @@ function get(self, entityName) {
54
43
  }
55
44
  }
56
45
 
57
- function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError) {
46
+ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError) {
58
47
  let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
59
48
  let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
60
49
  let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
@@ -69,8 +58,8 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
69
58
  config: config,
70
59
  persistence: persistence,
71
60
  allEntities: persistence.allEntities,
72
- entities: make$1(persistence.allEntities),
73
- effects: {},
61
+ entities: make(persistence.allEntities),
62
+ effectState: EffectState.make(),
74
63
  rollback: undefined,
75
64
  committedCheckpointId: committedCheckpointId,
76
65
  processedCheckpointId: committedCheckpointId,
@@ -86,7 +75,7 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
86
75
  crossChainState: CrossChainState.make(chainStates, isInReorgThreshold, isRealtime, targetBufferSize),
87
76
  rollbackState: "NoRollback",
88
77
  indexerStartTime: new Date(),
89
- writeThrottlers: make(),
78
+ lastPrunedAtMillis: {},
90
79
  loadManager: LoadManager.make(),
91
80
  keepProcessAlive: isDevelopmentMode || shouldUseTui,
92
81
  exitAfterFirstEventBlock: exitAfterFirstEventBlock,
@@ -116,7 +105,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
116
105
  Prometheus.ReorgThreshold.set(isInReorgThreshold);
117
106
  Utils.Dict.forEach(initialState.cache, param => {
118
107
  let count = param.count;
119
- Prometheus.EffectCacheCount.set(count, param.effectName);
108
+ Prometheus.EffectCacheCount.set(count, param.effectName, Internal.EffectCache.scopeToString(param.scope));
120
109
  });
121
110
  let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
122
111
  let chainStates = {};
@@ -142,7 +131,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
142
131
  if (allChainsReady.contents) {
143
132
  Prometheus.ProgressReady.setAllReady();
144
133
  }
145
- return make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
134
+ return make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
146
135
  }
147
136
 
148
137
  function isStale(state, stateId) {
@@ -266,8 +255,8 @@ function entities(state) {
266
255
  return state.entities;
267
256
  }
268
257
 
269
- function effects(state) {
270
- return state.effects;
258
+ function effectState(state) {
259
+ return state.effectState;
271
260
  }
272
261
 
273
262
  function committedCheckpointId(state) {
@@ -346,8 +335,8 @@ function epoch(state) {
346
335
  return state.epoch;
347
336
  }
348
337
 
349
- function pruneStaleEntityHistoryThrottler(state) {
350
- return state.writeThrottlers.pruneStaleEntityHistory;
338
+ function lastPrunedAtMillis(state) {
339
+ return state.lastPrunedAtMillis;
351
340
  }
352
341
 
353
342
  function simulateDeadInputTracker(state) {
@@ -418,8 +407,8 @@ function markCommitted(state, upToCheckpointId) {
418
407
  }
419
408
 
420
409
  function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
421
- state.entities = make$1(state.allEntities);
422
- state.effects = {};
410
+ state.entities = make(state.allEntities);
411
+ EffectState.resetForRollback(state.effectState);
423
412
  state.rollback = {
424
413
  targetCheckpointId: targetCheckpointId,
425
414
  diffCheckpointId: diffCheckpointId,
@@ -478,7 +467,7 @@ function takeChainMetaSnapshot(state) {
478
467
  }
479
468
 
480
469
  let EntityTables = {
481
- make: make$1,
470
+ make: make,
482
471
  get: get
483
472
  };
484
473
 
@@ -486,7 +475,7 @@ let unexpectedErrorMsg = "Indexer has failed with an unexpected error";
486
475
 
487
476
  export {
488
477
  EntityTables,
489
- make$2 as make,
478
+ make$1 as make,
490
479
  makeFromDbState,
491
480
  unexpectedErrorMsg,
492
481
  isStale,
@@ -512,7 +501,7 @@ export {
512
501
  persistence,
513
502
  allEntities,
514
503
  entities,
515
- effects,
504
+ effectState,
516
505
  committedCheckpointId,
517
506
  processedCheckpointId,
518
507
  processedBatches,
@@ -532,7 +521,7 @@ export {
532
521
  exitAfterFirstEventBlock,
533
522
  isStopped,
534
523
  epoch,
535
- pruneStaleEntityHistoryThrottler,
524
+ lastPrunedAtMillis,
536
525
  simulateDeadInputTracker,
537
526
  queueProcessedBatch,
538
527
  drainBatchRun,
@@ -16,14 +16,6 @@ module EntityTables: {
16
16
  let get: (t, ~entityName: string) => InMemoryTable.Entity.t
17
17
  }
18
18
 
19
- type effectCacheInMemTable = {
20
- mutable idsToStore: array<string>,
21
- mutable invalidationsCount: int,
22
- mutable dict: dict<Change.t<Internal.effectOutput>>,
23
- mutable changesCount: float,
24
- effect: Internal.effect,
25
- }
26
-
27
19
  type t
28
20
 
29
21
  let make: (
@@ -86,7 +78,7 @@ let config: t => Config.t
86
78
  let persistence: t => Persistence.t
87
79
  let allEntities: t => array<Internal.entityConfig>
88
80
  let entities: t => EntityTables.t
89
- let effects: t => dict<effectCacheInMemTable>
81
+ let effectState: t => EffectState.t
90
82
  let committedCheckpointId: t => Internal.checkpointId
91
83
  let processedCheckpointId: t => Internal.checkpointId
92
84
  let processedBatches: t => array<Batch.t>
@@ -106,7 +98,7 @@ let keepProcessAlive: t => bool
106
98
  let exitAfterFirstEventBlock: t => bool
107
99
  let isStopped: t => bool
108
100
  let epoch: t => int
109
- let pruneStaleEntityHistoryThrottler: t => Throttler.t
101
+ let lastPrunedAtMillis: t => dict<float>
110
102
  let simulateDeadInputTracker: t => option<SimulateDeadInputTracker.t>
111
103
 
112
104
  // Store domain operations.
package/src/Internal.res CHANGED
@@ -330,6 +330,9 @@ type eventPayload
330
330
  @get external getPayloadBlock: eventPayload => Nullable.t<eventBlock> = "block"
331
331
  @set external setPayloadBlock: (eventPayload, eventBlock) => unit = "block"
332
332
 
333
+ // The log's emitting address (EVM/Fuel; the program id carries it for SVM).
334
+ @get external getPayloadSrcAddress: eventPayload => Address.t = "srcAddress"
335
+
333
336
  type genericLoaderArgs<'event, 'context> = {
334
337
  event: 'event,
335
338
  context: 'context,
@@ -555,6 +558,11 @@ type svmInstructionEventConfig = {
555
558
  // must stay directly constructable), and the evm→base cast in sources is sound
556
559
  // by ecosystem homogeneity — an EVM chain only ever holds `evmOnEventRegistration`s.
557
560
  type onEventRegistration = {
561
+ // Chain-scoped sequential index — the registration's position in the
562
+ // chain's onEventRegistrations array, assigned when registration finishes
563
+ // (-1 until then). Native-routed items reference their registration by this
564
+ // index across the napi boundary; sources resolve it before creating an item.
565
+ index: int,
558
566
  eventConfig: eventConfig,
559
567
  handler: option<handler>,
560
568
  contractRegister: option<contractRegister>,
@@ -601,8 +609,9 @@ type indexingAddress = {
601
609
 
602
610
  type dcs = array<indexingAddress>
603
611
 
604
- // Duplicate the type from item
605
- // to make item properly unboxed
612
+ // Duplicate the type from item to keep item properly unboxed. Runtime event
613
+ // items carry the registration their source already resolved from the
614
+ // ChainState-owned registration array.
606
615
  type eventItem = private {
607
616
  kind: [#0],
608
617
  onEventRegistration: onEventRegistration,
@@ -743,33 +752,108 @@ type effectCacheItem = {id: string, output: effectOutput}
743
752
  type effectCacheStorageMeta = {
744
753
  itemSchema: S.t<effectCacheItem>,
745
754
  outputSchema: S.t<effectOutput>,
746
- table: Table.table,
747
755
  }
748
- type rateLimitState = {
756
+ type rateLimitOptions = {
749
757
  callsPerDuration: int,
750
758
  durationMs: int,
751
- mutable availableCalls: int,
752
- mutable windowStartTime: float,
753
- mutable queueCount: int,
754
- mutable nextWindowPromise: option<promise<unit>>,
755
759
  }
756
760
  type effect = {
757
761
  name: string,
758
762
  handler: effectArgs => promise<effectOutput>,
759
763
  storageMeta: effectCacheStorageMeta,
760
764
  defaultShouldCache: bool,
765
+ // When true (the default) a single cache is shared across every chain and the
766
+ // handler must not read context.chain. When false the cache is isolated per
767
+ // chain and context.chain.id is available.
768
+ crossChain: bool,
761
769
  output: S.t<effectOutput>,
762
770
  input: S.t<effectInput>,
763
- // The number of functions that are currently running.
764
- mutable activeCallsCount: int,
765
- mutable prevCallStartTimerRef: Performance.timeRef,
766
- rateLimit: option<rateLimitState>,
771
+ rateLimit: option<rateLimitOptions>,
767
772
  }
773
+
774
+ // Whether some piece of data (currently an effect cache; entities in a future
775
+ // version) is shared across every chain or isolated to a single chain. Unboxed:
776
+ // `CrossChain` is the string "crossChain" and `Chain(id)` is the raw chain id,
777
+ // discriminated by runtime type.
778
+ @unboxed
779
+ type chainScope =
780
+ | @as("crossChain") CrossChain
781
+ | Chain(int)
782
+
768
783
  let cacheTablePrefix = "envio_effect_"
784
+
785
+ // The single reversible mapping between an effect's (name, scope) and its
786
+ // canonical Postgres cache-table name and .envio/cache file path. Everything
787
+ // that needs a cache address goes through here instead of slicing prefixes.
788
+ // CrossChain -> envio_effect_<name> <name>.tsv
789
+ // Chain(1) -> envio_1_effect_<name> 1/<name>.tsv
790
+ // Chain(137) -> envio_137_effect_<name> 137/<name>.tsv
791
+ module EffectCache = {
792
+ let toTableName = (~effectName, ~scope) =>
793
+ switch scope {
794
+ | CrossChain => cacheTablePrefix ++ effectName
795
+ | Chain(chainId) => `envio_${chainId->Int.toString}_effect_${effectName}`
796
+ }
797
+
798
+ // "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
799
+ let scopeToString = scope =>
800
+ switch scope {
801
+ | CrossChain => "crossChain"
802
+ | Chain(chainId) => chainId->Int.toString
803
+ }
804
+
805
+ // Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
806
+ // Int.fromString alone follows parseInt semantics and accepts both.
807
+ let parseChainId = str =>
808
+ switch Int.fromString(str) {
809
+ | Some(chainId) if chainId >= 0 && chainId->Int.toString === str => Some(chainId)
810
+ | _ => None
811
+ }
812
+
813
+ let chainScopedRe = /^envio_([0-9]+)_effect_(.+)$/
814
+ let crossChainRe = /^envio_effect_(.+)$/
815
+
816
+ // Inverse of toTableName. Returns None for any table name that isn't a cache
817
+ // table. Chain-scoped is tried first: the `_effect_` separator keeps effect
818
+ // names that themselves start with digits unambiguous.
819
+ let fromTableName = (tableName): option<(string, chainScope)> =>
820
+ switch RegExp.exec(chainScopedRe, tableName) {
821
+ | Some(result) =>
822
+ switch (
823
+ RegExp.Result.matches(result)->Array.get(0),
824
+ RegExp.Result.matches(result)->Array.get(1),
825
+ ) {
826
+ | (Some(Some(chainIdStr)), Some(Some(effectName))) =>
827
+ switch parseChainId(chainIdStr) {
828
+ | Some(chainId) => Some((effectName, Chain(chainId)))
829
+ | None => None
830
+ }
831
+ | _ => None
832
+ }
833
+ | None =>
834
+ switch RegExp.exec(crossChainRe, tableName) {
835
+ | Some(result) =>
836
+ switch RegExp.Result.matches(result)->Array.get(0) {
837
+ | Some(Some(effectName)) => Some((effectName, CrossChain))
838
+ | _ => None
839
+ }
840
+ | None => None
841
+ }
842
+ }
843
+
844
+ // Relative posix path within .envio/cache. Chain-scoped caches live one
845
+ // directory level deep, named by chain id.
846
+ let toCachePath = (~effectName, ~scope) =>
847
+ switch scope {
848
+ | CrossChain => effectName ++ ".tsv"
849
+ | Chain(chainId) => `${chainId->Int.toString}/${effectName}.tsv`
850
+ }
851
+ }
852
+
769
853
  let cacheOutputSchema = S.json(~validate=false)->(Utils.magic: S.t<JSON.t> => S.t<effectOutput>)
770
- let makeCacheTable = (~effectName) => {
854
+ let makeCacheTable = (~effectName, ~scope) => {
771
855
  Table.mkTable(
772
- cacheTablePrefix ++ effectName,
856
+ EffectCache.toTableName(~effectName, ~scope),
773
857
  ~fields=[
774
858
  Table.mkField("id", String, ~fieldSchema=S.string, ~isPrimaryKey=true),
775
859
  Table.mkField("output", Json, ~fieldSchema=cacheOutputSchema, ~isNullable=true),