envio 3.0.2 → 3.1.0-rc.1

package/src/InMemoryTable.res

@@ -1,55 +1,27 @@
-type t<'key, 'val> = {
-  dict: dict<'val>,
-  hash: 'key => string,
-}
-
-let make = (~hash): t<'key, 'val> => {
-  dict: Dict.make(),
-  hash,
-}
-
-let set = (self: t<'key, 'val>, key, value) => self.dict->Dict.set(key->self.hash, value)
-
-let setByHash = (self: t<'key, 'val>, hash, value) => self.dict->Dict.set(hash, value)
-
-let hasByHash = (self: t<'key, 'val>, hash) => {
-  self.dict->Utils.Dict.has(hash)
-}
-
-let getUnsafeByHash = (self: t<'key, 'val>, hash) => {
-  self.dict->Dict.getUnsafe(hash)
-}
-
-let get = (self: t<'key, 'val>, key: 'key) =>
-  self.dict->Utils.Dict.dangerouslyGetNonOption(key->self.hash)
-
-let values = (self: t<'key, 'val>) => self.dict->Dict.valuesToArray
-
-let clone = (self: t<'key, 'val>) => {
-  ...self,
-  dict: self.dict->Lodash.cloneDeep,
-}
-
 module Entity = {
   type relatedEntityId = string
   type indexWithRelatedIds = (TableIndices.Index.t, Utils.Set.t<relatedEntityId>)
-  type indicesSerializedToValue = t<TableIndices.Index.t, indexWithRelatedIds>
-  type indexFieldNameToIndices = t<TableIndices.Index.t, indicesSerializedToValue>
-
-  type entityWithIndices<'entity> = {
-    latest: option<'entity>,
-    status: Internal.inMemoryStoreEntityStatus<'entity>,
-    entityIndices: Utils.Set.t<TableIndices.Index.t>,
-  }
-  type t<'entity> = {
-    table: t<string, entityWithIndices<'entity>>,
+  // Keyed by TableIndices.Index.toString
+  type indicesSerializedToValue = dict<indexWithRelatedIds>
+  // Keyed by TableIndices.Index.getFieldName
+  type indexFieldNameToIndices = dict<indicesSerializedToValue>
+
+  type entityIndices = Utils.Set.t<TableIndices.Index.t>
+  type t = {
+    latestEntityChangeById: dict<Change.t<Internal.entity>>,
+    // Counts every recorded change (new latest ids and pushes to
+    // prevEntityChanges), kept in sync manually so InMemoryStore can gauge the
+    // store size without scanning every dict.
+    mutable changesCount: float,
+    prevEntityChanges: array<Change.t<Internal.entity>>,
+    indicesByEntityId: dict<entityIndices>,
     fieldNameIndices: indexFieldNameToIndices,
   }
 
   // Helper to extract entity ID from any entity
   exception UnexpectedIdNotDefinedOnEntity
-  let getEntityIdUnsafe = (entity: 'entity): string =>
-    switch (entity->(Utils.magic: 'entity => {"id": option<string>}))["id"] {
+  let getEntityIdUnsafe = (entity: Internal.entity): string =>
+    switch (entity->(Utils.magic: Internal.entity => {"id": option<string>}))["id"] {
     | Some(id) => id
     | None =>
       UnexpectedIdNotDefinedOnEntity->ErrorHandling.mkLogAndRaise(
@@ -57,207 +29,192 @@ module Entity = {
       )
     }
 
+  let getOrCreateEntityIndices = (self: t, ~entityId) =>
+    switch self.indicesByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
+    | Some(s) => s
+    | None =>
+      let s = Utils.Set.make()
+      self.indicesByEntityId->Dict.set(entityId, s)
+      s
+    }
+
   let makeIndicesSerializedToValue = (
     ~index,
     ~relatedEntityIds=Utils.Set.make(),
   ): indicesSerializedToValue => {
-    let empty = make(~hash=TableIndices.Index.toString)
-    empty->set(index, (index, relatedEntityIds))
+    let empty = Dict.make()
+    empty->Dict.set(index->TableIndices.Index.toString, (index, relatedEntityIds))
     empty
   }
 
-  let make = (): t<'entity> => {
-    table: make(~hash=str => str),
-    fieldNameIndices: make(~hash=TableIndices.Index.getFieldName),
+  let make = (): t => {
+    latestEntityChangeById: Dict.make(),
+    changesCount: 0.,
+    prevEntityChanges: [],
+    indicesByEntityId: Dict.make(),
+    fieldNameIndices: Dict.make(),
   }
 
-  let updateIndices = (
-    self: t<'entity>,
-    ~entity: 'entity,
-    ~entityIndices: Utils.Set.t<TableIndices.Index.t>,
-  ) => {
-    //Remove any invalid indices on entity
-    entityIndices->Utils.Set.forEach(index => {
-      let fieldName = index->TableIndices.Index.getFieldName
-      let fieldValue =
-        entity
-        ->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
-        ->Dict.getUnsafe(fieldName)
-      if !(index->TableIndices.Index.evaluate(~fieldName, ~fieldValue)) {
-        entityIndices->Utils.Set.delete(index)->ignore
+  // Drops the per-batch index state and rollback history, but keeps the
+  // already committed entities so the next batch can read them without
+  // hitting the database.
+  let resetButKeepLatestChanges = (self: t): t => {
+    ...make(),
+    latestEntityChangeById: self.latestEntityChangeById,
+    // writeBatch already mutated this to subtract the dropped prevEntityChanges.
+    changesCount: self.changesCount,
+  }
+
+  // Like resetButKeepLatestChanges, but only keeps entities loaded from the db
+  // (changes carrying loadedFromDbCheckpointId), dropping everything written in
+  // a batch. The kept count is exposed through the table's changesCount.
+  let resetButKeepLoadedFromDbChanges = (self: t): t => {
+    let latestEntityChangeById = Dict.make()
+    let keptCount = ref(0.)
+    self.latestEntityChangeById->Utils.Dict.forEachWithKey((change, key) =>
+      if change->Change.getCheckpointId === Internal.loadedFromDbCheckpointId {
+        latestEntityChangeById->Dict.set(key, change)
+        keptCount := keptCount.contents +. 1.
       }
-    })
+    )
+    {
+      ...make(),
+      latestEntityChangeById,
+      changesCount: keptCount.contents,
+    }
+  }
+
+  let updateIndices = (self: t, ~entity: Internal.entity) => {
+    let entityId = entity->getEntityIdUnsafe
+    //Remove any invalid indices on entity
+    switch self.indicesByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
+    | None => ()
+    | Some(entityIndices) =>
+      entityIndices->Utils.Set.forEach(index => {
+        let fieldName = index->TableIndices.Index.getFieldName
+        let fieldValue =
+          entity
+          ->(Utils.magic: Internal.entity => dict<TableIndices.FieldValue.t>)
+          ->Dict.getUnsafe(fieldName)
+        if !(index->TableIndices.Index.evaluate(~fieldName, ~fieldValue)) {
+          entityIndices->Utils.Set.delete(index)->ignore
+        }
+      })
+    }
 
-    self.fieldNameIndices.dict
-    ->Dict.keysToArray
-    ->Array.forEach(fieldName => {
-      let indices = self.fieldNameIndices.dict->Dict.getUnsafe(fieldName)
+    self.fieldNameIndices->Utils.Dict.forEachWithKey((indices, fieldName) => {
       // A missing key reads as `undefined`, which matches the `None` arm of
       // `FieldValue.t` (`option<...>`). Mirror `addEmptyIndex` so nullable
       // FK columns that were omitted on the set entity don't crash.
       let fieldValue =
         entity
-        ->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
+        ->(Utils.magic: Internal.entity => dict<TableIndices.FieldValue.t>)
         ->Dict.getUnsafe(fieldName)
-      indices
-      ->values
-      ->Array.forEach(((index, relatedEntityIds)) => {
+      indices->Utils.Dict.forEach(((index, relatedEntityIds)) => {
         if index->TableIndices.Index.evaluate(~fieldName, ~fieldValue) {
           //Add entity id to indices and add index to entity indicies
-          relatedEntityIds->Utils.Set.add(getEntityIdUnsafe(entity))->ignore
-          entityIndices->Utils.Set.add(index)->ignore
+          relatedEntityIds->Utils.Set.add(entityId)->ignore
+          self->getOrCreateEntityIndices(~entityId)->Utils.Set.add(index)->ignore
         } else {
-          relatedEntityIds->Utils.Set.delete(getEntityIdUnsafe(entity))->ignore
+          relatedEntityIds->Utils.Set.delete(entityId)->ignore
         }
       })
     })
   }
 
-  let deleteEntityFromIndices = (self: t<'entity>, ~entityId: string, ~entityIndices) =>
-    entityIndices->Utils.Set.forEach(index => {
-      switch self.fieldNameIndices
-      ->get(index)
-      ->Option.flatMap(get(_, index)) {
-      | Some((_index, relatedEntityIds)) =>
-        let _wasRemoved = relatedEntityIds->Utils.Set.delete(entityId)
-      | None => () //Unexpected index should exist if it is entityIndices
-      }
-      let _wasRemoved = entityIndices->Utils.Set.delete(index)
-    })
-
-  let initValue = (
-    inMemTable: t<'entity>,
-    ~key: string,
-    ~entity: option<'entity>,
-    // NOTE: This value is only set to true in the internals of the test framework to create the mockDb.
-    ~allowOverWriteEntity=false,
-  ) => {
-    let shouldWriteEntity =
-      allowOverWriteEntity ||
-      inMemTable.table.dict->Dict.get(key->inMemTable.table.hash)->Option.isNone
+  let deleteEntityFromIndices = (self: t, ~entityId: string) =>
+    switch self.indicesByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
+    | None => ()
+    | Some(entityIndices) =>
+      entityIndices->Utils.Set.forEach(index => {
+        switch self.fieldNameIndices
+        ->Utils.Dict.dangerouslyGetNonOption(index->TableIndices.Index.getFieldName)
+        ->Option.flatMap(indices =>
+          indices->Utils.Dict.dangerouslyGetNonOption(index->TableIndices.Index.toString)
+        ) {
+        | Some((_index, relatedEntityIds)) =>
+          let _wasRemoved = relatedEntityIds->Utils.Set.delete(entityId)
+        | None => () //Unexpected index should exist if it is entityIndices
+        }
+        let _wasRemoved = entityIndices->Utils.Set.delete(index)
+      })
+    }
 
-    //Only initialize a row in the case where it is none
-    //or if allowOverWriteEntity is true (used for mockDb in test helpers)
-    if shouldWriteEntity {
-      let entityIndices = Utils.Set.make()
-      switch entity {
-      | Some(entity) =>
-        //update table indices in the case where there
-        //is an already set entity
-        inMemTable->updateIndices(~entity, ~entityIndices)
-      | None => ()
+  let set = (inMemTable: t, ~committedCheckpointId, change: Change.t<Internal.entity>) => {
+    let entityId = change->Change.getEntityId
+    switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(entityId) {
+    | Some(prev) =>
+      let prevCheckpointId = prev->Change.getCheckpointId
+      if (
+        prevCheckpointId > committedCheckpointId &&
+          prevCheckpointId < change->Change.getCheckpointId
+      ) {
+        inMemTable.prevEntityChanges->Array.push(prev)
+        inMemTable.changesCount = inMemTable.changesCount +. 1.
       }
-      inMemTable.table.dict->Dict.set(
-        key->inMemTable.table.hash,
-        {
-          latest: entity,
-          status: Loaded,
-          entityIndices,
-        },
-      )
+    | None => inMemTable.changesCount = inMemTable.changesCount +. 1.
     }
-  }
 
-  let setRow = set
-  let set = (
-    inMemTable: t<'entity>,
-    change: Change.t<'entity>,
-    ~shouldSaveHistory,
-    ~containsRollbackDiffChange=false,
-  ) => {
-    //New entity row with only the latest update
-    @inline
-    let newStatus = () => Internal.Updated({
-      latestChange: change,
-      history: shouldSaveHistory
-        ? [change]
-        : Utils.Array.immutableEmpty->(Utils.magic: array<unknown> => array<Change.t<'entity>>),
-      containsRollbackDiffChange,
-    })
-    let latest = switch change {
-    | Set({entity}) => Some(entity)
-    | Delete(_) => None
+    switch change {
+    | Set({entity}) => inMemTable->updateIndices(~entity)
+    | Delete({entityId}) => inMemTable->deleteEntityFromIndices(~entityId)
     }
+    inMemTable.latestEntityChangeById->Dict.set(entityId, change)
+  }
 
-    let updatedEntityRecord = switch inMemTable.table->get(change->Change.getEntityId) {
-    | None => {latest, status: newStatus(), entityIndices: Utils.Set.make()}
-    | Some({status: Loaded, entityIndices}) => {
-        latest,
-        status: newStatus(),
-        entityIndices,
+  // Only writes when the id isn't already present, so set always takes its
+  // None branch here (committedCheckpointId is never read).
+  let initValue = (
+    inMemTable: t,
+    ~committedCheckpointId,
+    ~key: string,
+    ~entity: option<Internal.entity>,
+  ) =>
+    if inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(key)->Option.isNone {
+      let change: Change.t<Internal.entity> = switch entity {
+      | Some(entity) =>
+        Set({entityId: key, entity, checkpointId: Internal.loadedFromDbCheckpointId})
+      | None => Delete({entityId: key, checkpointId: Internal.loadedFromDbCheckpointId})
       }
-    | Some({status: Updated(previous_values), entityIndices}) =>
-      let newStatus = Internal.Updated({
-        latestChange: change,
-        history: switch shouldSaveHistory {
-        // This prevents two db actions in the same event on the same entity from being recorded to the history table.
-        | true
-          if previous_values.latestChange->Change.getCheckpointId ===
-            change->Change.getCheckpointId =>
-          previous_values.history->Utils.Array.setIndexImmutable(
-            previous_values.history->Array.length - 1,
-            change,
-          )
-        | true => [...previous_values.history, change]
-        | false => previous_values.history
-        },
-        containsRollbackDiffChange: previous_values.containsRollbackDiffChange,
-      })
-      {latest, status: newStatus, entityIndices}
+      inMemTable->set(~committedCheckpointId, change)
     }
 
+  let mapChangeToEntity = (change: Change.t<Internal.entity>) =>
     switch change {
-    | Set({entity}) =>
-      inMemTable->updateIndices(~entity, ~entityIndices=updatedEntityRecord.entityIndices)
-    | Delete({entityId}) =>
-      inMemTable->deleteEntityFromIndices(
-        ~entityId,
-        ~entityIndices=updatedEntityRecord.entityIndices,
-      )
+    | Set({entity}) => Some(entity)
+    | Delete(_) => None
     }
-    inMemTable.table->setRow(change->Change.getEntityId, updatedEntityRecord)
-  }
-
-  let rowToEntity = row => row.latest
-
-  let getRow = get
 
   /** It returns option<option<'entity>> where the first option means
   that the entity is not set to the in memory store,
   and the second option means that the entity doesn't esist/deleted.
   It's needed to prevent an additional round trips to the database for deleted entities. */
-  let getUnsafe = (inMemTable: t<'entity>) =>
+  let getUnsafe = (inMemTable: t) =>
     (key: string) =>
-      inMemTable.table.dict
+      inMemTable.latestEntityChangeById
       ->Dict.getUnsafe(key)
-      ->rowToEntity
+      ->mapChangeToEntity
 
-  let hasIndex = (inMemTable: t<'entity>, ~fieldName, ~operator: TableIndices.Operator.t) =>
+  let hasIndex = (inMemTable: t, ~fieldName, ~operator: TableIndices.Operator.t) =>
     fieldValueHash => {
-      switch inMemTable.fieldNameIndices.dict->Utils.Dict.dangerouslyGetNonOption(fieldName) {
+      switch inMemTable.fieldNameIndices->Utils.Dict.dangerouslyGetNonOption(fieldName) {
       | None => false
       | Some(indicesSerializedToValue) => {
-          // Should match TableIndices.toString logic
-          let key = `${fieldName}:${(operator :> string)}:${fieldValueHash}`
-          indicesSerializedToValue.dict->Utils.Dict.dangerouslyGetNonOption(key) !== None
+          let key = TableIndices.Index.toStringByParts(~fieldName, ~operator, ~fieldValueHash)
+          indicesSerializedToValue->Utils.Dict.dangerouslyGetNonOption(key) !== None
         }
       }
     }
 
-  let getUnsafeOnIndex = (
-    inMemTable: t<'entity>,
-    ~fieldName,
-    ~operator: TableIndices.Operator.t,
-  ) => {
+  let getUnsafeOnIndex = (inMemTable: t, ~fieldName, ~operator: TableIndices.Operator.t) => {
     let getEntity = inMemTable->getUnsafe
     fieldValueHash => {
-      switch inMemTable.fieldNameIndices.dict->Utils.Dict.dangerouslyGetNonOption(fieldName) {
+      switch inMemTable.fieldNameIndices->Utils.Dict.dangerouslyGetNonOption(fieldName) {
       | None =>
         JsError.throwWithMessage(`Unexpected error. Must have an index on field ${fieldName}`)
       | Some(indicesSerializedToValue) => {
-          // Should match TableIndices.toString logic
-          let key = `${fieldName}:${(operator :> string)}:${fieldValueHash}`
-          switch indicesSerializedToValue.dict->Utils.Dict.dangerouslyGetNonOption(key) {
+          let key = TableIndices.Index.toStringByParts(~fieldName, ~operator, ~fieldValueHash)
+          switch indicesSerializedToValue->Utils.Dict.dangerouslyGetNonOption(key) {
           | None =>
             JsError.throwWithMessage(
               `Unexpected error. Must have an index for the value ${fieldValueHash} on field ${fieldName}`,
@@ -267,7 +224,7 @@ module Entity = {
                 relatedEntityIds
                 ->Utils.Set.toArray
                 ->Array.filterMap(entityId => {
-                  switch hasByHash(inMemTable.table, entityId) {
+                  switch inMemTable.latestEntityChangeById->Dict.has(entityId) {
                   | true => getEntity(entityId)
                   | false => None
                   }
@@ -280,83 +237,42 @@ module Entity = {
     }
   }
 
-  let addEmptyIndex = (inMemTable: t<'entity>, ~index) => {
+  let addEmptyIndex = (inMemTable: t, ~index) => {
     let fieldName = index->TableIndices.Index.getFieldName
     let relatedEntityIds = Utils.Set.make()
 
-    inMemTable.table
-    ->values
-    ->Array.forEach(row => {
-      switch row->rowToEntity {
+    inMemTable.latestEntityChangeById->Utils.Dict.forEach(change => {
+      switch change->mapChangeToEntity {
       | Some(entity) =>
         let fieldValue =
           entity
-          ->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
+          ->(Utils.magic: Internal.entity => dict<TableIndices.FieldValue.t>)
           ->Dict.getUnsafe(fieldName)
         if index->TableIndices.Index.evaluate(~fieldName, ~fieldValue) {
-          let _ = row.entityIndices->Utils.Set.add(index)
-          let _ = relatedEntityIds->Utils.Set.add(entity->getEntityIdUnsafe)
+          let entityId = entity->getEntityIdUnsafe
+          let _ = inMemTable->getOrCreateEntityIndices(~entityId)->Utils.Set.add(index)
+          let _ = relatedEntityIds->Utils.Set.add(entityId)
         }
       | None => ()
       }
     })
-    switch inMemTable.fieldNameIndices->getRow(index) {
+    switch inMemTable.fieldNameIndices->Utils.Dict.dangerouslyGetNonOption(fieldName) {
     | None =>
-      inMemTable.fieldNameIndices->setRow(
-        index,
+      inMemTable.fieldNameIndices->Dict.set(
+        fieldName,
         makeIndicesSerializedToValue(~index, ~relatedEntityIds),
       )
     | Some(indicesSerializedToValue) =>
-      switch indicesSerializedToValue->getRow(index) {
-      | None => indicesSerializedToValue->setRow(index, (index, relatedEntityIds))
-      | Some(_) => () //Should not happen, this means the index already exists
-      }
-    }
-  }
-
-  let addIdToIndex = (inMemTable: t<'entity>, ~index, ~entityId) =>
-    switch inMemTable.fieldNameIndices->getRow(index) {
-    | None =>
-      inMemTable.fieldNameIndices->setRow(
-        index,
-        makeIndicesSerializedToValue(
-          ~index,
-          ~relatedEntityIds=Utils.Set.make()->Utils.Set.add(entityId),
-        ),
-      )
-    | Some(indicesSerializedToValue) =>
-      switch indicesSerializedToValue->getRow(index) {
+      switch indicesSerializedToValue->Utils.Dict.dangerouslyGetNonOption(
+        index->TableIndices.Index.toString,
+      ) {
       | None =>
-        indicesSerializedToValue->setRow(index, (index, Utils.Set.make()->Utils.Set.add(entityId)))
-      | Some((_index, relatedEntityIds)) => relatedEntityIds->Utils.Set.add(entityId)->ignore
+        indicesSerializedToValue->Dict.set(
+          index->TableIndices.Index.toString,
+          (index, relatedEntityIds),
+        )
+      | Some(_) => () //Should not happen, this means the index already exists
       }
     }
-
-  let updates = (inMemTable: t<'entity>) => {
-    inMemTable.table
-    ->values
-    ->Array.filterMap(v =>
-      switch v.status {
-      | Updated(update) => Some(update)
-      | Loaded => None
-      }
-    )
-  }
-
-  let values = (inMemTable: t<'entity>) => {
-    inMemTable.table
-    ->values
-    ->Array.filterMap(rowToEntity)
-  }
-
-  let clone = ({table, fieldNameIndices}: t<'entity>) => {
-    table: table->clone,
-    fieldNameIndices: {
-      ...fieldNameIndices,
-      dict: fieldNameIndices.dict
-      ->Dict.toArray
-      ->Array.map(((k, v)) => (k, v->clone))
-      ->Dict.fromArray,
-    },
   }
 }