envio 3.3.0 → 3.3.1-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/package.json +6 -7
  2. package/src/ChainFetching.res +4 -32
  3. package/src/ChainFetching.res.mjs +2 -12
  4. package/src/ChainState.res +73 -53
  5. package/src/ChainState.res.mjs +61 -30
  6. package/src/ChainState.resi +13 -17
  7. package/src/Config.res +5 -0
  8. package/src/Config.res.mjs +1 -0
  9. package/src/CrossChainState.res +2 -8
  10. package/src/CrossChainState.res.mjs +6 -8
  11. package/src/CrossChainState.resi +1 -0
  12. package/src/EffectState.res +183 -44
  13. package/src/EffectState.res.mjs +134 -26
  14. package/src/EffectState.resi +26 -3
  15. package/src/EventProcessing.res +14 -9
  16. package/src/EventProcessing.res.mjs +7 -7
  17. package/src/FetchState.res +18 -29
  18. package/src/FetchState.res.mjs +20 -37
  19. package/src/IndexerState.res +280 -31
  20. package/src/IndexerState.res.mjs +234 -25
  21. package/src/IndexerState.resi +22 -0
  22. package/src/LoadLayer.res +19 -65
  23. package/src/LoadLayer.res.mjs +13 -55
  24. package/src/Main.res +42 -23
  25. package/src/Main.res.mjs +32 -35
  26. package/src/Metrics.res +943 -66
  27. package/src/Metrics.res.mjs +367 -50
  28. package/src/Persistence.res +3 -0
  29. package/src/PgStorage.res +3 -8
  30. package/src/PgStorage.res.mjs +3 -4
  31. package/src/PruneStaleHistory.res +1 -1
  32. package/src/PruneStaleHistory.res.mjs +1 -2
  33. package/src/Rollback.res +4 -5
  34. package/src/Rollback.res.mjs +2 -3
  35. package/src/SimulateItems.res.mjs +3 -21
  36. package/src/TestIndexer.res.mjs +4 -23
  37. package/src/TestIndexerProxyStorage.res +1 -0
  38. package/src/TestIndexerProxyStorage.res.mjs +1 -1
  39. package/src/Writing.res +3 -5
  40. package/src/Writing.res.mjs +3 -6
  41. package/src/bindings/ClickHouse.res +101 -17
  42. package/src/bindings/ClickHouse.res.mjs +58 -19
  43. package/src/bindings/NodeJs.res +64 -0
  44. package/src/bindings/NodeJs.res.mjs +6 -0
  45. package/src/sources/HyperSyncClient.res +5 -11
  46. package/src/sources/SourceManager.res +72 -27
  47. package/src/sources/SourceManager.res.mjs +81 -11
  48. package/src/sources/SourceManager.resi +14 -0
  49. package/src/tui/Tui.res +1 -1
  50. package/src/tui/Tui.res.mjs +2 -2
  51. package/src/Prometheus.res +0 -789
  52. package/src/Prometheus.res.mjs +0 -847
  53. package/src/bindings/PromClient.res +0 -72
  54. package/src/bindings/PromClient.res.mjs +0 -38
@@ -28,15 +28,13 @@ let make = (
28
28
  ~isRealtime,
29
29
  ~targetBufferSize=calculateTargetBufferSize(),
30
30
  ): t => {
31
- let crossChainState = {
31
+ {
32
32
  chainStates,
33
33
  chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
34
34
  isRealtime,
35
35
  isInReorgThreshold,
36
36
  targetBufferSize,
37
37
  }
38
- Prometheus.IndexingTargetBufferSize.set(~targetBufferSize)
39
- crossChainState
40
38
  }
41
39
 
42
40
  // Resolve a chain's state by id. The id always comes from `chainIds`, which is
@@ -49,6 +47,7 @@ let getChainState = (crossChainState: t, chainId) =>
49
47
  let chainStates = (crossChainState: t) => crossChainState.chainStates
50
48
  let isRealtime = (crossChainState: t) => crossChainState.isRealtime
51
49
  let isInReorgThreshold = (crossChainState: t) => crossChainState.isInReorgThreshold
50
+ let targetBufferSize = (crossChainState: t) => crossChainState.targetBufferSize
52
51
 
53
52
  // Ready-to-process items across every chain — the live draw against
54
53
  // targetBufferSize, which is a budget of processable events (items stuck behind
@@ -120,7 +119,6 @@ let createBatch = (
120
119
  // blockLag and flip the flag.
121
120
  let enterReorgThreshold = (crossChainState: t) => {
122
121
  Logging.info("Reorg threshold reached")
123
- Prometheus.ReorgThreshold.set(~isInReorgThreshold=true)
124
122
 
125
123
  for i in 0 to crossChainState.chainIds->Array.length - 1 {
126
124
  crossChainState
@@ -160,10 +158,6 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
160
158
  }
161
159
  }
162
160
 
163
- if allChainsReady.contents {
164
- Prometheus.ProgressReady.setAllReady()
165
- }
166
-
167
161
  crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
168
162
  }
169
163
 
@@ -7,7 +7,6 @@ import * as Logging from "./Logging.res.mjs";
7
7
  import * as ChainMap from "./ChainMap.res.mjs";
8
8
  import * as ChainState from "./ChainState.res.mjs";
9
9
  import * as FetchState from "./FetchState.res.mjs";
10
- import * as Prometheus from "./Prometheus.res.mjs";
11
10
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
12
11
  import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
13
12
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
@@ -26,15 +25,13 @@ function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt)
26
25
  let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : (
27
26
  Env.targetBufferSize !== undefined ? Env.targetBufferSize : 100000
28
27
  );
29
- let crossChainState = {
28
+ return {
30
29
  chainStates: chainStates,
31
30
  chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
32
31
  isRealtime: isRealtime,
33
32
  isInReorgThreshold: isInReorgThreshold,
34
33
  targetBufferSize: targetBufferSize
35
34
  };
36
- Prometheus.IndexingTargetBufferSize.set(targetBufferSize);
37
- return crossChainState;
38
35
  }
39
36
 
40
37
  function chainStates(crossChainState) {
@@ -49,6 +46,10 @@ function isInReorgThreshold(crossChainState) {
49
46
  return crossChainState.isInReorgThreshold;
50
47
  }
51
48
 
49
+ function targetBufferSize(crossChainState) {
50
+ return crossChainState.targetBufferSize;
51
+ }
52
+
52
53
  function totalReadyCount(crossChainState) {
53
54
  let total = 0;
54
55
  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
@@ -89,7 +90,6 @@ function createBatch(crossChainState, processedCheckpointId, batchSizeTarget, is
89
90
 
90
91
  function enterReorgThreshold(crossChainState) {
91
92
  Logging.info("Reorg threshold reached");
92
- Prometheus.ReorgThreshold.set(true);
93
93
  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
94
94
  ChainState.enterReorgThreshold(crossChainState.chainStates[crossChainState.chainIds[i]]);
95
95
  }
@@ -117,9 +117,6 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
117
117
  allChainsReady = false;
118
118
  }
119
119
  }
120
- if (allChainsReady) {
121
- Prometheus.ProgressReady.setAllReady();
122
- }
123
120
  crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady;
124
121
  }
125
122
 
@@ -230,6 +227,7 @@ export {
230
227
  chainStates,
231
228
  isRealtime,
232
229
  isInReorgThreshold,
230
+ targetBufferSize,
233
231
  nextItemIsNone,
234
232
  getSafeCheckpointId,
235
233
  createBatch,
@@ -16,6 +16,7 @@ let make: (
16
16
  let chainStates: t => dict<ChainState.t>
17
17
  let isRealtime: t => bool
18
18
  let isInReorgThreshold: t => bool
19
+ let targetBufferSize: t => int
19
20
 
20
21
  // Derived (pure).
21
22
  let nextItemIsNone: t => bool
@@ -1,6 +1,7 @@
1
- // Owns all per-(effect, scope) runtime state and its lifecycle. The two maps
2
- // have different rollback semantics, so keeping them together behind an explicit
3
- // `resetForRollback` makes the invariant enforced rather than remembered.
1
+ // Owns all per-(effect, scope) runtime state and its lifecycle. A single store
2
+ // of tables holds the cache alongside the counters and rate-limit window that
3
+ // must outlive a rollback; `resetForRollback` clears only the cache in place so
4
+ // that survival is structural rather than remembered.
4
5
 
5
6
  // Per-scope rate-limit window and queue. A chain-scoped effect gets one of
6
7
  // these per chain, so each chain's throughput is independent.
@@ -9,10 +10,32 @@ type effectRateLimitState = {
9
10
  durationMs: int,
10
11
  mutable availableCalls: int,
11
12
  mutable windowStartTime: float,
12
- mutable queueCount: int,
13
13
  mutable nextWindowPromise: option<promise<unit>>,
14
14
  }
15
15
 
16
+ // Per-(effect, scope) counters rendered into the envio_effect_* metrics. Live
17
+ // on the cache table and survive a rollback because the table does, keeping the
18
+ // prometheus counters monotonic across a reorg.
19
+ type effectStats = {
20
+ effectName: string,
21
+ scope: Internal.chainScope,
22
+ // Wall-clock time with at least one call in flight.
23
+ mutable callSeconds: float,
24
+ // Cumulative per-call time; exceeds callSeconds under parallel execution.
25
+ mutable callSecondsTotal: float,
26
+ mutable callCount: float,
27
+ mutable activeCallsCount: int,
28
+ mutable prevCallStartTimerRef: Performance.timeRef,
29
+ mutable queueCount: int,
30
+ mutable queueWaitSeconds: float,
31
+ mutable invalidationsCount: float,
32
+ // Number of persisted cache rows; seeded from the db on restart. hasCache
33
+ // marks that the effect persists at all, so an empty table still gets a
34
+ // zero-valued gauge sample.
35
+ mutable cacheCount: int,
36
+ mutable hasCache: bool,
37
+ }
38
+
16
39
  type effectCacheInMemTable = {
17
40
  // Cache keys whose handler output is persisted on the next write. Drained
18
41
  // each write; eviction is driven by the per-entry checkpointId instead.
@@ -30,52 +53,162 @@ type effectCacheInMemTable = {
30
53
  scope: Internal.chainScope,
31
54
  table: Table.table,
32
55
  rateLimitState: option<effectRateLimitState>,
33
- // Per-scope active-call bookkeeping, surfaced through the Effect metrics.
34
- mutable activeCallsCount: int,
35
- mutable prevCallStartTimerRef: Performance.timeRef,
56
+ stats: effectStats,
57
+ }
58
+
59
+ // Cache-row count loaded from the db on restart for an effect whose in-mem
60
+ // table hasn't been created yet this session, keyed by cache table name. Enough
61
+ // to render envio_effect_cache until the effect runs; consumed (and removed) at
62
+ // table creation so the table becomes the sole owner.
63
+ type unregisteredCacheCount = {
64
+ effectName: string,
65
+ scope: Internal.chainScope,
66
+ count: int,
36
67
  }
37
68
 
38
69
  type t = {
39
- // Cache tables keyed by cache table name. Dropped on rollback the cache
40
- // is re-derivable from the db.
41
- mutable tables: dict<effectCacheInMemTable>,
42
- // Rate-limit windows keyed by the same name. Survive rollback: rate limiting
43
- // reflects real API throughput, not indexing progress, so a reorg must not
44
- // refill an effect's budget.
45
- rateLimits: dict<effectRateLimitState>,
70
+ // The single per-(effect, scope) store. Cache-derived fields are cleared on
71
+ // rollback; stats and the rate-limit window survive because the table does.
72
+ tables: dict<effectCacheInMemTable>,
73
+ unregisteredCacheCounts: dict<unregisteredCacheCount>,
46
74
  }
47
75
 
48
- let make = (): t => {tables: Dict.make(), rateLimits: Dict.make()}
49
-
50
- let getRateLimitState = (self: t, ~tableName, ~effect: Internal.effect) =>
51
- switch effect.rateLimit {
52
- | None => None
53
- | Some({callsPerDuration, durationMs}) =>
54
- Some(
55
- switch self.rateLimits->Utils.Dict.dangerouslyGetNonOption(tableName) {
56
- | Some(existing) => existing
57
- | None =>
58
- let created: effectRateLimitState = {
59
- callsPerDuration,
60
- durationMs,
61
- availableCalls: callsPerDuration,
62
- windowStartTime: Date.now(),
63
- queueCount: 0,
64
- nextWindowPromise: None,
65
- }
66
- self.rateLimits->Dict.set(tableName, created)
67
- created
68
- },
69
- )
76
+ let make = (): t => {tables: Dict.make(), unregisteredCacheCounts: Dict.make()}
77
+
78
+ let setUnregisteredCacheCount = (self: t, ~effectName, ~scope, ~count) => {
79
+ let tableName = Internal.EffectCache.toTableName(~effectName, ~scope)
80
+ self.unregisteredCacheCounts->Dict.set(tableName, {effectName, scope, count})
81
+ }
82
+
83
+ // --- Metric mutations. The stats records are opaque outside this module, so
84
+ // every counter change goes through these. ---
85
+
86
+ // Track a call starting at timerRef: bump the active count and extend the
87
+ // wall-clock callSeconds, which counts overlapping calls once.
88
+ let startCall = (stats: effectStats, ~timerRef) => {
89
+ let hadActiveCalls = stats.activeCallsCount > 0
90
+ stats.activeCallsCount = stats.activeCallsCount + 1
91
+ if hadActiveCalls {
92
+ let elapsed = Performance.secondsBetween(~from=stats.prevCallStartTimerRef, ~to=timerRef)
93
+ if elapsed > 0. {
94
+ stats.callSeconds = stats.callSeconds +. elapsed
95
+ }
70
96
  }
97
+ stats.prevCallStartTimerRef = timerRef
98
+ }
99
+
100
+ // Finish a call started at startTimerRef: close the wall-clock interval and
101
+ // record the call's own cumulative duration.
102
+ let endCall = (stats: effectStats, ~startTimerRef) => {
103
+ stats.activeCallsCount = stats.activeCallsCount - 1
104
+ let newTimer = Performance.now()
105
+ stats.callSeconds =
106
+ stats.callSeconds +. Performance.secondsBetween(~from=stats.prevCallStartTimerRef, ~to=newTimer)
107
+ stats.prevCallStartTimerRef = newTimer
108
+
109
+ stats.callCount = stats.callCount +. 1.
110
+ stats.callSecondsTotal = stats.callSecondsTotal +. startTimerRef->Performance.secondsSince
111
+ }
71
112
 
72
- // Get, or lazily create, the in-mem table for an (effect, scope). A recreated
73
- // table (e.g. after a rollback) reuses the surviving rate-limit window.
113
+ let queueEnqueued = (stats: effectStats, ~count) => stats.queueCount = stats.queueCount + count
114
+
115
+ let queueDequeued = (stats: effectStats, ~count) => stats.queueCount = stats.queueCount - count
116
+
117
+ let addQueueWaitSeconds = (stats: effectStats, ~seconds) =>
118
+ stats.queueWaitSeconds = stats.queueWaitSeconds +. seconds
119
+
120
+ // Bumps both the per-write invalidation count (consumed by the cache
121
+ // persistence math) and the monotonic metric counter.
122
+ let recordInvalidation = (inMemTable: effectCacheInMemTable) => {
123
+ inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1
124
+ inMemTable.stats.invalidationsCount = inMemTable.stats.invalidationsCount +. 1.
125
+ }
126
+
127
+ let commitCacheCount = (inMemTable: effectCacheInMemTable, ~count) => {
128
+ inMemTable.stats.cacheCount = count
129
+ inMemTable.stats.hasCache = true
130
+ }
131
+
132
+ let statsToMetrics = (stats: effectStats): Metrics.effectMetrics => {
133
+ Metrics.effect: stats.effectName,
134
+ scope: stats.scope->Internal.EffectCache.scopeToString,
135
+ callSeconds: stats.callSeconds,
136
+ callSecondsTotal: stats.callSecondsTotal,
137
+ callCount: stats.callCount,
138
+ activeCallsCount: stats.activeCallsCount,
139
+ queueCount: stats.queueCount,
140
+ queueWaitSeconds: stats.queueWaitSeconds,
141
+ invalidationsCount: stats.invalidationsCount,
142
+ cacheCount: stats.hasCache ? Some(stats.cacheCount) : None,
143
+ }
144
+
145
+ // Full per-effect metrics for every live table, plus a cache-only entry for
146
+ // each effect that hasn't run this session (envio_effect_cache only). Such an
147
+ // entry is removed once its table is created, so the two never double-count the
148
+ // same effect.
149
+ let toMetrics = (self: t): array<Metrics.effectMetrics> => {
150
+ let metrics = self.tables->Utils.Dict.mapValuesToArray(t => t.stats->statsToMetrics)
151
+ self.unregisteredCacheCounts->Utils.Dict.forEach(({effectName, scope, count}) => {
152
+ metrics
153
+ ->Array.push({
154
+ Metrics.effect: effectName,
155
+ scope: scope->Internal.EffectCache.scopeToString,
156
+ callSeconds: 0.,
157
+ callSecondsTotal: 0.,
158
+ callCount: 0.,
159
+ activeCallsCount: 0,
160
+ queueCount: 0,
161
+ queueWaitSeconds: 0.,
162
+ invalidationsCount: 0.,
163
+ cacheCount: Some(count),
164
+ })
165
+ ->ignore
166
+ })
167
+ metrics
168
+ }
169
+
170
+ // Get, or lazily create, the in-mem table for an (effect, scope). On first
171
+ // creation the rate-limit window is built from the effect config and any
172
+ // matching unregistered cache count is consumed. A table recreated after a
173
+ // rollback keeps its surviving stats and rate-limit window because the table
174
+ // object itself survives — only its cache is cleared.
74
175
  let getTable = (self: t, ~effect: Internal.effect, ~scope: Internal.chainScope) => {
75
176
  let tableName = Internal.EffectCache.toTableName(~effectName=effect.name, ~scope)
76
177
  switch self.tables->Utils.Dict.dangerouslyGetNonOption(tableName) {
77
178
  | Some(inMemTable) => inMemTable
78
179
  | None =>
180
+ let stats: effectStats = {
181
+ effectName: effect.name,
182
+ scope,
183
+ callSeconds: 0.,
184
+ callSecondsTotal: 0.,
185
+ callCount: 0.,
186
+ activeCallsCount: 0,
187
+ prevCallStartTimerRef: %raw(`null`),
188
+ queueCount: 0,
189
+ queueWaitSeconds: 0.,
190
+ invalidationsCount: 0.,
191
+ cacheCount: 0,
192
+ hasCache: false,
193
+ }
194
+ switch self.unregisteredCacheCounts->Utils.Dict.dangerouslyGetNonOption(tableName) {
195
+ | Some({count}) =>
196
+ stats.cacheCount = count
197
+ stats.hasCache = true
198
+ self.unregisteredCacheCounts->Utils.Dict.deleteInPlace(tableName)
199
+ | None => ()
200
+ }
201
+ let rateLimitState = switch effect.rateLimit {
202
+ | None => None
203
+ | Some({callsPerDuration, durationMs}) =>
204
+ Some({
205
+ callsPerDuration,
206
+ durationMs,
207
+ availableCalls: callsPerDuration,
208
+ windowStartTime: Date.now(),
209
+ nextWindowPromise: None,
210
+ })
211
+ }
79
212
  let inMemTable: effectCacheInMemTable = {
80
213
  idsToStore: [],
81
214
  dict: Dict.make(),
@@ -84,9 +217,8 @@ let getTable = (self: t, ~effect: Internal.effect, ~scope: Internal.chainScope)
84
217
  effect,
85
218
  scope,
86
219
  table: Internal.makeCacheTable(~effectName=effect.name, ~scope),
87
- rateLimitState: self->getRateLimitState(~tableName, ~effect),
88
- activeCallsCount: 0,
89
- prevCallStartTimerRef: %raw(`null`),
220
+ rateLimitState,
221
+ stats,
90
222
  }
91
223
  self.tables->Dict.set(tableName, inMemTable)
92
224
  inMemTable
@@ -95,6 +227,13 @@ let getTable = (self: t, ~effect: Internal.effect, ~scope: Internal.chainScope)
95
227
 
96
228
  let forEach = (self: t, fn) => self.tables->Utils.Dict.forEach(fn)
97
229
 
98
- // Drop the cache tables (re-derivable from the db) but keep the rate-limit
99
- // windows so a reorg doesn't hand an effect a fresh budget.
100
- let resetForRollback = (self: t) => self.tables = Dict.make()
230
+ // Clear the cache-derived fields on every table in place. The cache is
231
+ // re-derivable from the db, but stats and the rate-limit window must survive a
232
+ // reorg so the table object (their owner) is kept and only its cache reset.
233
+ let resetForRollback = (self: t) =>
234
+ self.tables->Utils.Dict.forEach(t => {
235
+ t.dict = Dict.make()
236
+ t.idsToStore = []
237
+ t.changesCount = 0.
238
+ t.invalidationsCount = 0
239
+ })
@@ -2,37 +2,98 @@
2
2
 
3
3
  import * as Utils from "./Utils.res.mjs";
4
4
  import * as Internal from "./Internal.res.mjs";
5
+ import * as Performance from "./bindings/Performance.res.mjs";
5
6
 
6
7
  function make() {
7
8
  return {
8
9
  tables: {},
9
- rateLimits: {}
10
+ unregisteredCacheCounts: {}
10
11
  };
11
12
  }
12
13
 
13
- function getRateLimitState(self, tableName, effect) {
14
- let match = effect.rateLimit;
15
- if (match === undefined) {
16
- return;
14
+ function setUnregisteredCacheCount(self, effectName, scope, count) {
15
+ let tableName = Internal.EffectCache.toTableName(effectName, scope);
16
+ self.unregisteredCacheCounts[tableName] = {
17
+ effectName: effectName,
18
+ scope: scope,
19
+ count: count
20
+ };
21
+ }
22
+
23
+ function startCall(stats, timerRef) {
24
+ let hadActiveCalls = stats.activeCallsCount > 0;
25
+ stats.activeCallsCount = stats.activeCallsCount + 1 | 0;
26
+ if (hadActiveCalls) {
27
+ let elapsed = Performance.secondsBetween(stats.prevCallStartTimerRef, timerRef);
28
+ if (elapsed > 0) {
29
+ stats.callSeconds = stats.callSeconds + elapsed;
30
+ }
17
31
  }
18
- let callsPerDuration = match.callsPerDuration;
19
- let existing = self.rateLimits[tableName];
20
- let tmp;
21
- if (existing !== undefined) {
22
- tmp = existing;
23
- } else {
24
- let created = {
25
- callsPerDuration: callsPerDuration,
26
- durationMs: match.durationMs,
27
- availableCalls: callsPerDuration,
28
- windowStartTime: Date.now(),
29
- queueCount: 0,
30
- nextWindowPromise: undefined
32
+ stats.prevCallStartTimerRef = timerRef;
33
+ }
34
+
35
+ function endCall(stats, startTimerRef) {
36
+ stats.activeCallsCount = stats.activeCallsCount - 1 | 0;
37
+ let newTimer = Performance.now();
38
+ stats.callSeconds = stats.callSeconds + Performance.secondsBetween(stats.prevCallStartTimerRef, newTimer);
39
+ stats.prevCallStartTimerRef = newTimer;
40
+ stats.callCount = stats.callCount + 1;
41
+ stats.callSecondsTotal = stats.callSecondsTotal + Performance.secondsSince(startTimerRef);
42
+ }
43
+
44
+ function queueEnqueued(stats, count) {
45
+ stats.queueCount = stats.queueCount + count | 0;
46
+ }
47
+
48
+ function queueDequeued(stats, count) {
49
+ stats.queueCount = stats.queueCount - count | 0;
50
+ }
51
+
52
+ function addQueueWaitSeconds(stats, seconds) {
53
+ stats.queueWaitSeconds = stats.queueWaitSeconds + seconds;
54
+ }
55
+
56
+ function recordInvalidation(inMemTable) {
57
+ inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1 | 0;
58
+ inMemTable.stats.invalidationsCount = inMemTable.stats.invalidationsCount + 1;
59
+ }
60
+
61
+ function commitCacheCount(inMemTable, count) {
62
+ inMemTable.stats.cacheCount = count;
63
+ inMemTable.stats.hasCache = true;
64
+ }
65
+
66
+ function toMetrics(self) {
67
+ let metrics = Utils.Dict.mapValuesToArray(self.tables, t => {
68
+ let stats = t.stats;
69
+ return {
70
+ effect: stats.effectName,
71
+ scope: Internal.EffectCache.scopeToString(stats.scope),
72
+ callSeconds: stats.callSeconds,
73
+ callSecondsTotal: stats.callSecondsTotal,
74
+ callCount: stats.callCount,
75
+ activeCallsCount: stats.activeCallsCount,
76
+ queueCount: stats.queueCount,
77
+ queueWaitSeconds: stats.queueWaitSeconds,
78
+ invalidationsCount: stats.invalidationsCount,
79
+ cacheCount: stats.hasCache ? stats.cacheCount : undefined
31
80
  };
32
- self.rateLimits[tableName] = created;
33
- tmp = created;
34
- }
35
- return tmp;
81
+ });
82
+ Utils.Dict.forEach(self.unregisteredCacheCounts, param => {
83
+ metrics.push({
84
+ effect: param.effectName,
85
+ scope: Internal.EffectCache.scopeToString(param.scope),
86
+ callSeconds: 0,
87
+ callSecondsTotal: 0,
88
+ callCount: 0,
89
+ activeCallsCount: 0,
90
+ queueCount: 0,
91
+ queueWaitSeconds: 0,
92
+ invalidationsCount: 0,
93
+ cacheCount: param.count
94
+ });
95
+ });
96
+ return metrics;
36
97
  }
37
98
 
38
99
  function getTable(self, effect, scope) {
@@ -41,6 +102,40 @@ function getTable(self, effect, scope) {
41
102
  if (inMemTable !== undefined) {
42
103
  return inMemTable;
43
104
  }
105
+ let stats = {
106
+ effectName: effect.name,
107
+ scope: scope,
108
+ callSeconds: 0,
109
+ callSecondsTotal: 0,
110
+ callCount: 0,
111
+ activeCallsCount: 0,
112
+ prevCallStartTimerRef: null,
113
+ queueCount: 0,
114
+ queueWaitSeconds: 0,
115
+ invalidationsCount: 0,
116
+ cacheCount: 0,
117
+ hasCache: false
118
+ };
119
+ let match = self.unregisteredCacheCounts[tableName];
120
+ if (match !== undefined) {
121
+ stats.cacheCount = match.count;
122
+ stats.hasCache = true;
123
+ Utils.Dict.deleteInPlace(self.unregisteredCacheCounts, tableName);
124
+ }
125
+ let match$1 = effect.rateLimit;
126
+ let rateLimitState;
127
+ if (match$1 !== undefined) {
128
+ let callsPerDuration = match$1.callsPerDuration;
129
+ rateLimitState = {
130
+ callsPerDuration: callsPerDuration,
131
+ durationMs: match$1.durationMs,
132
+ availableCalls: callsPerDuration,
133
+ windowStartTime: Date.now(),
134
+ nextWindowPromise: undefined
135
+ };
136
+ } else {
137
+ rateLimitState = undefined;
138
+ }
44
139
  let inMemTable$1 = {
45
140
  idsToStore: [],
46
141
  invalidationsCount: 0,
@@ -49,9 +144,8 @@ function getTable(self, effect, scope) {
49
144
  effect: effect,
50
145
  scope: scope,
51
146
  table: Internal.makeCacheTable(effect.name, scope),
52
- rateLimitState: getRateLimitState(self, tableName, effect),
53
- activeCallsCount: 0,
54
- prevCallStartTimerRef: null
147
+ rateLimitState: rateLimitState,
148
+ stats: stats
55
149
  };
56
150
  self.tables[tableName] = inMemTable$1;
57
151
  return inMemTable$1;
@@ -62,7 +156,12 @@ function forEach(self, fn) {
62
156
  }
63
157
 
64
158
  function resetForRollback(self) {
65
- self.tables = {};
159
+ Utils.Dict.forEach(self.tables, t => {
160
+ t.dict = {};
161
+ t.idsToStore = [];
162
+ t.changesCount = 0;
163
+ t.invalidationsCount = 0;
164
+ });
66
165
  }
67
166
 
68
167
  export {
@@ -70,5 +169,14 @@ export {
70
169
  getTable,
71
170
  forEach,
72
171
  resetForRollback,
172
+ startCall,
173
+ endCall,
174
+ queueEnqueued,
175
+ queueDequeued,
176
+ addQueueWaitSeconds,
177
+ recordInvalidation,
178
+ commitCacheCount,
179
+ setUnregisteredCacheCount,
180
+ toMetrics,
73
181
  }
74
182
  /* Utils Not a pure module */
@@ -3,10 +3,14 @@ type effectRateLimitState = {
3
3
  durationMs: int,
4
4
  mutable availableCalls: int,
5
5
  mutable windowStartTime: float,
6
- mutable queueCount: int,
7
6
  mutable nextWindowPromise: option<promise<unit>>,
8
7
  }
9
8
 
9
+ // Per-(effect, scope) metric counters. Survive rollback so they stay monotonic.
10
+ // Opaque: mutation goes through the metric operations below, reads through
11
+ // toMetrics.
12
+ type effectStats
13
+
10
14
  type effectCacheInMemTable = {
11
15
  mutable idsToStore: array<string>,
12
16
  mutable invalidationsCount: int,
@@ -16,8 +20,7 @@ type effectCacheInMemTable = {
16
20
  scope: Internal.chainScope,
17
21
  table: Table.table,
18
22
  rateLimitState: option<effectRateLimitState>,
19
- mutable activeCallsCount: int,
20
- mutable prevCallStartTimerRef: Performance.timeRef,
23
+ stats: effectStats,
21
24
  }
22
25
 
23
26
  // Opaque store of all per-(effect, scope) runtime state.
@@ -30,3 +33,23 @@ let getTable: (t, ~effect: Internal.effect, ~scope: Internal.chainScope) => effe
30
33
  let forEach: (t, effectCacheInMemTable => unit) => unit
31
34
  // Drop the cache tables but keep the rate-limit windows across a rollback.
32
35
  let resetForRollback: t => unit
36
+
37
+ let startCall: (effectStats, ~timerRef: Performance.timeRef) => unit
38
+ let endCall: (effectStats, ~startTimerRef: Performance.timeRef) => unit
39
+ let queueEnqueued: (effectStats, ~count: int) => unit
40
+ let queueDequeued: (effectStats, ~count: int) => unit
41
+ let addQueueWaitSeconds: (effectStats, ~seconds: float) => unit
42
+ // Bumps both the per-write invalidation count (consumed by the cache
43
+ // persistence math) and the monotonic metric counter.
44
+ let recordInvalidation: effectCacheInMemTable => unit
45
+ let commitCacheCount: (effectCacheInMemTable, ~count: int) => unit
46
+ // Record the db cache-row count for an effect not yet created in memory this
47
+ // session (loaded on restart). Consumed when the effect's table is created.
48
+ let setUnregisteredCacheCount: (
49
+ t,
50
+ ~effectName: string,
51
+ ~scope: Internal.chainScope,
52
+ ~count: int,
53
+ ) => unit
54
+
55
+ let toMetrics: t => array<Metrics.effectMetrics>