envio 2.32.0-alpha.0 → 2.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +32 -0
- package/package.json +5 -5
- package/src/Batch.res +2 -2
- package/src/Config.res +124 -0
- package/src/Config.res.js +84 -0
- package/src/Envio.gen.ts +21 -0
- package/src/Envio.res +77 -13
- package/src/Envio.res.js +51 -15
- package/src/EventRegister.res +2 -2
- package/src/EventRegister.resi +2 -2
- package/src/InMemoryTable.res +356 -0
- package/src/InMemoryTable.res.js +401 -0
- package/src/Indexer.res +5 -0
- package/src/Internal.res +13 -3
- package/src/Persistence.res +7 -2
- package/src/Persistence.res.js +3 -2
- package/src/PgStorage.res +1 -7
- package/src/PgStorage.res.js +1 -2
- package/src/Prometheus.res +12 -0
- package/src/Prometheus.res.js +12 -0
- package/src/ReorgDetection.res +8 -22
- package/src/ReorgDetection.res.js +11 -21
- package/src/TableIndices.res +110 -0
- package/src/TableIndices.res.js +143 -0
- package/src/Types.ts +5 -0
- package/src/bindings/Lodash.res +3 -0
- package/src/bindings/Lodash.res.js +11 -0
- package/src/bindings/vendored-lodash-fns.js +1441 -0
- package/src/db/InternalTable.res +2 -2
- package/src/sources/RpcSource.res +25 -21
- package/src/sources/RpcSource.res.js +21 -15
- package/src/InternalConfig.res +0 -48
- /package/src/{InternalConfig.res.js → Indexer.res.js} +0 -0
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
open Belt
|
|
2
|
+
|
|
3
|
+
type t<'key, 'val> = {
|
|
4
|
+
dict: dict<'val>,
|
|
5
|
+
hash: 'key => string,
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
let make = (~hash): t<'key, 'val> => {
|
|
9
|
+
dict: Js.Dict.empty(),
|
|
10
|
+
hash,
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
let set = (self: t<'key, 'val>, key, value) => self.dict->Js.Dict.set(key->self.hash, value)
|
|
14
|
+
|
|
15
|
+
let setByHash = (self: t<'key, 'val>, hash, value) => self.dict->Js.Dict.set(hash, value)
|
|
16
|
+
|
|
17
|
+
let hasByHash = (self: t<'key, 'val>, hash) => {
|
|
18
|
+
self.dict->Utils.Dict.has(hash)
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let getUnsafeByHash = (self: t<'key, 'val>, hash) => {
|
|
22
|
+
self.dict->Js.Dict.unsafeGet(hash)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
let get = (self: t<'key, 'val>, key: 'key) =>
|
|
26
|
+
self.dict->Utils.Dict.dangerouslyGetNonOption(key->self.hash)
|
|
27
|
+
|
|
28
|
+
let values = (self: t<'key, 'val>) => self.dict->Js.Dict.values
|
|
29
|
+
|
|
30
|
+
let clone = (self: t<'key, 'val>) => {
|
|
31
|
+
...self,
|
|
32
|
+
dict: self.dict->Lodash.cloneDeep,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
module Entity = {
|
|
36
|
+
type relatedEntityId = string
|
|
37
|
+
type indexWithRelatedIds = (TableIndices.Index.t, Utils.Set.t<relatedEntityId>)
|
|
38
|
+
type indicesSerializedToValue = t<TableIndices.Index.t, indexWithRelatedIds>
|
|
39
|
+
type indexFieldNameToIndices = t<TableIndices.Index.t, indicesSerializedToValue>
|
|
40
|
+
|
|
41
|
+
type entityWithIndices<'entity> = {
|
|
42
|
+
entityRow: Internal.inMemoryStoreRowEntity<'entity>,
|
|
43
|
+
entityIndices: Utils.Set.t<TableIndices.Index.t>,
|
|
44
|
+
}
|
|
45
|
+
type t<'entity> = {
|
|
46
|
+
table: t<string, entityWithIndices<'entity>>,
|
|
47
|
+
fieldNameIndices: indexFieldNameToIndices,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Helper to extract entity ID from any entity
|
|
51
|
+
exception UnexpectedIdNotDefinedOnEntity
|
|
52
|
+
let getEntityIdUnsafe = (entity: 'entity): string =>
|
|
53
|
+
switch Utils.magic(entity)["id"] {
|
|
54
|
+
| Some(id) => id
|
|
55
|
+
| None =>
|
|
56
|
+
UnexpectedIdNotDefinedOnEntity->ErrorHandling.mkLogAndRaise(
|
|
57
|
+
~msg="Property 'id' does not exist on expected entity object",
|
|
58
|
+
)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let makeIndicesSerializedToValue = (
|
|
62
|
+
~index,
|
|
63
|
+
~relatedEntityIds=Utils.Set.make(),
|
|
64
|
+
): indicesSerializedToValue => {
|
|
65
|
+
let empty = make(~hash=TableIndices.Index.toString)
|
|
66
|
+
empty->set(index, (index, relatedEntityIds))
|
|
67
|
+
empty
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
let make = (): t<'entity> => {
|
|
71
|
+
table: make(~hash=str => str),
|
|
72
|
+
fieldNameIndices: make(~hash=TableIndices.Index.getFieldName),
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
exception UndefinedKey(string)
|
|
76
|
+
let updateIndices = (
|
|
77
|
+
self: t<'entity>,
|
|
78
|
+
~entity: 'entity,
|
|
79
|
+
~entityIndices: Utils.Set.t<TableIndices.Index.t>,
|
|
80
|
+
) => {
|
|
81
|
+
//Remove any invalid indices on entity
|
|
82
|
+
entityIndices->Utils.Set.forEach(index => {
|
|
83
|
+
let fieldName = index->TableIndices.Index.getFieldName
|
|
84
|
+
let fieldValue =
|
|
85
|
+
entity
|
|
86
|
+
->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
|
|
87
|
+
->Js.Dict.get(fieldName)
|
|
88
|
+
->Option.getUnsafe
|
|
89
|
+
if !(index->TableIndices.Index.evaluate(~fieldName, ~fieldValue)) {
|
|
90
|
+
entityIndices->Utils.Set.delete(index)->ignore
|
|
91
|
+
}
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
self.fieldNameIndices.dict
|
|
95
|
+
->Js.Dict.keys
|
|
96
|
+
->Array.forEach(fieldName => {
|
|
97
|
+
switch (
|
|
98
|
+
entity
|
|
99
|
+
->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
|
|
100
|
+
->Js.Dict.get(fieldName),
|
|
101
|
+
self.fieldNameIndices.dict->Js.Dict.get(fieldName),
|
|
102
|
+
) {
|
|
103
|
+
| (Some(fieldValue), Some(indices)) =>
|
|
104
|
+
indices
|
|
105
|
+
->values
|
|
106
|
+
->Array.forEach(((index, relatedEntityIds)) => {
|
|
107
|
+
if index->TableIndices.Index.evaluate(~fieldName, ~fieldValue) {
|
|
108
|
+
//Add entity id to indices and add index to entity indicies
|
|
109
|
+
relatedEntityIds->Utils.Set.add(getEntityIdUnsafe(entity))->ignore
|
|
110
|
+
entityIndices->Utils.Set.add(index)->ignore
|
|
111
|
+
} else {
|
|
112
|
+
relatedEntityIds->Utils.Set.delete(getEntityIdUnsafe(entity))->ignore
|
|
113
|
+
}
|
|
114
|
+
})
|
|
115
|
+
| _ =>
|
|
116
|
+
UndefinedKey(fieldName)->ErrorHandling.mkLogAndRaise(
|
|
117
|
+
~msg="Expected field name to exist on the referenced index and the provided entity",
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
})
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
let deleteEntityFromIndices = (self: t<'entity>, ~entityId: string, ~entityIndices) =>
|
|
124
|
+
entityIndices->Utils.Set.forEach(index => {
|
|
125
|
+
switch self.fieldNameIndices
|
|
126
|
+
->get(index)
|
|
127
|
+
->Option.flatMap(get(_, index)) {
|
|
128
|
+
| Some((_index, relatedEntityIds)) =>
|
|
129
|
+
let _wasRemoved = relatedEntityIds->Utils.Set.delete(entityId)
|
|
130
|
+
| None => () //Unexpected index should exist if it is entityIndices
|
|
131
|
+
}
|
|
132
|
+
let _wasRemoved = entityIndices->Utils.Set.delete(index)
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
let initValue = (
|
|
136
|
+
inMemTable: t<'entity>,
|
|
137
|
+
~key: string,
|
|
138
|
+
~entity: option<'entity>,
|
|
139
|
+
// NOTE: This value is only set to true in the internals of the test framework to create the mockDb.
|
|
140
|
+
~allowOverWriteEntity=false,
|
|
141
|
+
) => {
|
|
142
|
+
let shouldWriteEntity =
|
|
143
|
+
allowOverWriteEntity ||
|
|
144
|
+
inMemTable.table.dict->Js.Dict.get(key->inMemTable.table.hash)->Option.isNone
|
|
145
|
+
|
|
146
|
+
//Only initialize a row in the case where it is none
|
|
147
|
+
//or if allowOverWriteEntity is true (used for mockDb in test helpers)
|
|
148
|
+
if shouldWriteEntity {
|
|
149
|
+
let entityIndices = Utils.Set.make()
|
|
150
|
+
let initialStoreRow: Internal.inMemoryStoreRowEntity<'entity> = switch entity {
|
|
151
|
+
| Some(entity) =>
|
|
152
|
+
//update table indices in the case where there
|
|
153
|
+
//is an already set entity
|
|
154
|
+
inMemTable->updateIndices(~entity, ~entityIndices)
|
|
155
|
+
InitialReadFromDb(AlreadySet(entity))
|
|
156
|
+
|
|
157
|
+
| None => InitialReadFromDb(NotSet)
|
|
158
|
+
}
|
|
159
|
+
inMemTable.table.dict->Js.Dict.set(
|
|
160
|
+
key->inMemTable.table.hash,
|
|
161
|
+
{entityRow: initialStoreRow, entityIndices},
|
|
162
|
+
)
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
let setRow = set
|
|
167
|
+
let set = (
|
|
168
|
+
inMemTable: t<'entity>,
|
|
169
|
+
entityUpdate: EntityHistory.entityUpdate<'entity>,
|
|
170
|
+
~shouldSaveHistory,
|
|
171
|
+
~containsRollbackDiffChange=false,
|
|
172
|
+
) => {
|
|
173
|
+
//New entity row with only the latest update
|
|
174
|
+
@inline
|
|
175
|
+
let newEntityRow = () => Internal.Updated({
|
|
176
|
+
latest: entityUpdate,
|
|
177
|
+
history: shouldSaveHistory ? [entityUpdate] : [],
|
|
178
|
+
containsRollbackDiffChange,
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
let {entityRow, entityIndices} = switch inMemTable.table->get(entityUpdate.entityId) {
|
|
182
|
+
| None => {entityRow: newEntityRow(), entityIndices: Utils.Set.make()}
|
|
183
|
+
| Some({entityRow: InitialReadFromDb(_), entityIndices}) => {
|
|
184
|
+
entityRow: newEntityRow(),
|
|
185
|
+
entityIndices,
|
|
186
|
+
}
|
|
187
|
+
| Some({entityRow: Updated(previous_values), entityIndices}) =>
|
|
188
|
+
let entityRow = Internal.Updated({
|
|
189
|
+
latest: entityUpdate,
|
|
190
|
+
history: switch shouldSaveHistory {
|
|
191
|
+
// This prevents two db actions in the same event on the same entity from being recorded to the history table.
|
|
192
|
+
| true if previous_values.latest.checkpointId === entityUpdate.checkpointId =>
|
|
193
|
+
previous_values.history->Utils.Array.setIndexImmutable(
|
|
194
|
+
previous_values.history->Array.length - 1,
|
|
195
|
+
entityUpdate,
|
|
196
|
+
)
|
|
197
|
+
| true => [...previous_values.history, entityUpdate]
|
|
198
|
+
| false => previous_values.history
|
|
199
|
+
},
|
|
200
|
+
containsRollbackDiffChange: previous_values.containsRollbackDiffChange,
|
|
201
|
+
})
|
|
202
|
+
{entityRow, entityIndices}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
switch entityUpdate.entityUpdateAction {
|
|
206
|
+
| Set(entity) => inMemTable->updateIndices(~entity, ~entityIndices)
|
|
207
|
+
| Delete => inMemTable->deleteEntityFromIndices(~entityId=entityUpdate.entityId, ~entityIndices)
|
|
208
|
+
}
|
|
209
|
+
inMemTable.table->setRow(entityUpdate.entityId, {entityRow, entityIndices})
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
let rowToEntity = row =>
|
|
213
|
+
switch row.entityRow {
|
|
214
|
+
| Internal.Updated({latest: {entityUpdateAction: Set(entity)}}) => Some(entity)
|
|
215
|
+
| Updated({latest: {entityUpdateAction: Delete}}) => None
|
|
216
|
+
| InitialReadFromDb(AlreadySet(entity)) => Some(entity)
|
|
217
|
+
| InitialReadFromDb(NotSet) => None
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
let getRow = get
|
|
221
|
+
|
|
222
|
+
/** It returns option<option<'entity>> where the first option means
|
|
223
|
+
that the entity is not set to the in memory store,
|
|
224
|
+
and the second option means that the entity doesn't esist/deleted.
|
|
225
|
+
It's needed to prevent an additional round trips to the database for deleted entities. */
|
|
226
|
+
let getUnsafe = (inMemTable: t<'entity>) => (key: string) =>
|
|
227
|
+
inMemTable.table.dict
|
|
228
|
+
->Js.Dict.unsafeGet(key)
|
|
229
|
+
->rowToEntity
|
|
230
|
+
|
|
231
|
+
let hasIndex = (
|
|
232
|
+
inMemTable: t<'entity>,
|
|
233
|
+
~fieldName,
|
|
234
|
+
~operator: TableIndices.Operator.t,
|
|
235
|
+
) => fieldValueHash => {
|
|
236
|
+
switch inMemTable.fieldNameIndices.dict->Utils.Dict.dangerouslyGetNonOption(fieldName) {
|
|
237
|
+
| None => false
|
|
238
|
+
| Some(indicesSerializedToValue) => {
|
|
239
|
+
// Should match TableIndices.toString logic
|
|
240
|
+
let key = `${fieldName}:${(operator :> string)}:${fieldValueHash}`
|
|
241
|
+
indicesSerializedToValue.dict->Utils.Dict.dangerouslyGetNonOption(key) !== None
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
let getUnsafeOnIndex = (
|
|
247
|
+
inMemTable: t<'entity>,
|
|
248
|
+
~fieldName,
|
|
249
|
+
~operator: TableIndices.Operator.t,
|
|
250
|
+
) => {
|
|
251
|
+
let getEntity = inMemTable->getUnsafe
|
|
252
|
+
fieldValueHash => {
|
|
253
|
+
switch inMemTable.fieldNameIndices.dict->Utils.Dict.dangerouslyGetNonOption(fieldName) {
|
|
254
|
+
| None => Js.Exn.raiseError(`Unexpected error. Must have an index on field ${fieldName}`)
|
|
255
|
+
| Some(indicesSerializedToValue) => {
|
|
256
|
+
// Should match TableIndices.toString logic
|
|
257
|
+
let key = `${fieldName}:${(operator :> string)}:${fieldValueHash}`
|
|
258
|
+
switch indicesSerializedToValue.dict->Utils.Dict.dangerouslyGetNonOption(key) {
|
|
259
|
+
| None =>
|
|
260
|
+
Js.Exn.raiseError(
|
|
261
|
+
`Unexpected error. Must have an index for the value ${fieldValueHash} on field ${fieldName}`,
|
|
262
|
+
)
|
|
263
|
+
| Some((_index, relatedEntityIds)) => {
|
|
264
|
+
let res =
|
|
265
|
+
relatedEntityIds
|
|
266
|
+
->Utils.Set.toArray
|
|
267
|
+
->Array.keepMap(entityId => {
|
|
268
|
+
switch hasByHash(inMemTable.table, entityId) {
|
|
269
|
+
| true => getEntity(entityId)
|
|
270
|
+
| false => None
|
|
271
|
+
}
|
|
272
|
+
})
|
|
273
|
+
res
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
let addEmptyIndex = (inMemTable: t<'entity>, ~index) => {
|
|
282
|
+
let fieldName = index->TableIndices.Index.getFieldName
|
|
283
|
+
let relatedEntityIds = Utils.Set.make()
|
|
284
|
+
|
|
285
|
+
inMemTable.table
|
|
286
|
+
->values
|
|
287
|
+
->Array.forEach(row => {
|
|
288
|
+
switch row->rowToEntity {
|
|
289
|
+
| Some(entity) =>
|
|
290
|
+
let fieldValue =
|
|
291
|
+
entity
|
|
292
|
+
->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
|
|
293
|
+
->Js.Dict.unsafeGet(fieldName)
|
|
294
|
+
if index->TableIndices.Index.evaluate(~fieldName, ~fieldValue) {
|
|
295
|
+
let _ = row.entityIndices->Utils.Set.add(index)
|
|
296
|
+
let _ = relatedEntityIds->Utils.Set.add(entity->getEntityIdUnsafe)
|
|
297
|
+
}
|
|
298
|
+
| None => ()
|
|
299
|
+
}
|
|
300
|
+
})
|
|
301
|
+
switch inMemTable.fieldNameIndices->getRow(index) {
|
|
302
|
+
| None =>
|
|
303
|
+
inMemTable.fieldNameIndices->setRow(
|
|
304
|
+
index,
|
|
305
|
+
makeIndicesSerializedToValue(~index, ~relatedEntityIds),
|
|
306
|
+
)
|
|
307
|
+
| Some(indicesSerializedToValue) =>
|
|
308
|
+
switch indicesSerializedToValue->getRow(index) {
|
|
309
|
+
| None => indicesSerializedToValue->setRow(index, (index, relatedEntityIds))
|
|
310
|
+
| Some(_) => () //Should not happen, this means the index already exists
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
let addIdToIndex = (inMemTable: t<'entity>, ~index, ~entityId) =>
|
|
316
|
+
switch inMemTable.fieldNameIndices->getRow(index) {
|
|
317
|
+
| None =>
|
|
318
|
+
inMemTable.fieldNameIndices->setRow(
|
|
319
|
+
index,
|
|
320
|
+
makeIndicesSerializedToValue(
|
|
321
|
+
~index,
|
|
322
|
+
~relatedEntityIds=Utils.Set.make()->Utils.Set.add(entityId),
|
|
323
|
+
),
|
|
324
|
+
)
|
|
325
|
+
| Some(indicesSerializedToValue) =>
|
|
326
|
+
switch indicesSerializedToValue->getRow(index) {
|
|
327
|
+
| None =>
|
|
328
|
+
indicesSerializedToValue->setRow(index, (index, Utils.Set.make()->Utils.Set.add(entityId)))
|
|
329
|
+
| Some((_index, relatedEntityIds)) => relatedEntityIds->Utils.Set.add(entityId)->ignore
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
let rows = (inMemTable: t<'entity>) => {
|
|
334
|
+
inMemTable.table
|
|
335
|
+
->values
|
|
336
|
+
->Array.map(v => v.entityRow)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
let values = (inMemTable: t<'entity>) => {
|
|
340
|
+
inMemTable.table
|
|
341
|
+
->values
|
|
342
|
+
->Array.keepMap(rowToEntity)
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
let clone = ({table, fieldNameIndices}: t<'entity>) => {
|
|
346
|
+
table: table->clone,
|
|
347
|
+
fieldNameIndices: {
|
|
348
|
+
...fieldNameIndices,
|
|
349
|
+
dict: fieldNameIndices.dict
|
|
350
|
+
->Js.Dict.entries
|
|
351
|
+
->Array.map(((k, v)) => (k, v->clone))
|
|
352
|
+
->Js.Dict.fromArray,
|
|
353
|
+
},
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|