envio 3.12.0 → 3.12.1

package/src/InMemoryTable.res

@@ -1,10 +1,26 @@
 module Entity = {
   type relatedEntityId = string
-  type filterWithRelatedIds = (EntityFilter.t, Utils.Set.t<relatedEntityId>)
-  // Keyed by EntityFilter.toString
-  type filterIndexes = dict<filterWithRelatedIds>
+  // The matcher is specialized from the filter once at index creation and
+  // reused for every entity write against this index.
+  type index = {
+    matcher: EntityFilter.matcher,
+    ids: Utils.Set.t<relatedEntityId>,
+  }
+
+  // Indexes on one field that a single value can resolve — _eq, and _in under
+  // each of its values — found by the field's value rather than by running
+  // their matchers, so a write costs a lookup per indexed field instead of a
+  // pass over every registered index. Several indexes can share a value: an
+  // _in and an _eq naming it, or two _in filters overlapping on it.
+  type eqBucket = {
+    keyOf: unknown => unknown,
+    byValue: Utils.Map.t<unknown, array<index>>,
+    // Values a completed query has already loaded every row for. A later
+    // filter naming only these is answered from the table instead of a
+    // second round trip for rows that are already in it.
+    loadedValues: Utils.Set.t<unknown>,
+  }
 
-  type entityFilters = Utils.Set.t<EntityFilter.t>
   type t = {
     latestEntityChangeById: dict<Change.t<Internal.entity>>,
     // Recorded changes (new latest ids + prevEntityChanges pushes), tracked
@@ -13,8 +29,14 @@ module Entity = {
     // Swapped out when a write starts so processing keeps appending while the
     // previous changes persist in the background.
     mutable prevEntityChanges: array<Change.t<Internal.entity>>,
-    mutable filtersByEntityId: dict<entityFilters>,
-    mutable filterIndexes: filterIndexes,
+    // Every index an entity currently belongs to, so a write can drop it from
+    // the ones it stopped matching without consulting the others.
+    mutable indexesByEntityId: dict<Utils.Set.t<index>>,
+    // Keyed by EntityFilter.toString, for lookups coming from the load manager.
+    mutable indexesByKey: dict<index>,
+    mutable eqBucketsByField: dict<eqBucket>,
+    // Ranges and composites, which no single field value can resolve.
+    mutable scanIndexes: array<index>,
   }
 
   // Helper to extract an entity's id as a dict key. The raw id may be a
@@ -30,21 +52,22 @@ module Entity = {
       )
     }
 
-  let getOrCreateEntityFilters = (self: t, ~entityId) =>
-    switch self.filtersByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
-    | Some(s) => s
-    | None =>
-      let s = Utils.Set.make()
-      self.filtersByEntityId->Dict.set(entityId, s)
-      s
+  let addToIndex = (self: t, ~index: index, ~entityId) => {
+    index.ids->Utils.Set.add(entityId)->ignore
+    switch self.indexesByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
+    | Some(indexes) => indexes->Utils.Set.add(index)->ignore
+    | None => self.indexesByEntityId->Dict.set(entityId, Utils.Set.fromArray([index]))
     }
+  }
 
   let make = (): t => {
     latestEntityChangeById: Dict.make(),
     changesCount: 0.,
     prevEntityChanges: [],
-    filtersByEntityId: Dict.make(),
-    filterIndexes: Dict.make(),
+    indexesByEntityId: Dict.make(),
+    indexesByKey: Dict.make(),
+    eqBucketsByField: Dict.make(),
+    scanIndexes: [],
   }
 
   // Changes to persist for checkpoints in (committedCheckpointId, upToCheckpointId].
@@ -97,50 +120,52 @@ module Entity = {
     })
     keysToDelete->Array.forEach(key => self.latestEntityChangeById->Utils.Dict.deleteInPlace(key))
     self.changesCount = self.changesCount -. keysToDelete->Array.length->Int.toFloat
-    self.filtersByEntityId = Dict.make()
-    self.filterIndexes = Dict.make()
+    self.indexesByEntityId = Dict.make()
+    self.indexesByKey = Dict.make()
+    self.eqBucketsByField = Dict.make()
+    self.scanIndexes = []
   }
 
   let updateIndexes = (self: t, ~entity: Internal.entity) => {
     let entityId = entity->getEntityIdUnsafe
-    let entityAsDict = entity->(Utils.magic: Internal.entity => dict<EntityFilter.FieldValue.t>)
 
-    //Remove any invalid filters on entity
-    switch self.filtersByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
+    switch self.indexesByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
     | None => ()
-    | Some(entityFilters) =>
-      entityFilters->Utils.Set.forEach(filter => {
-        if !(filter->EntityFilter.matches(~entity=entityAsDict)) {
-          entityFilters->Utils.Set.delete(filter)->ignore
+    | Some(indexes) =>
+      indexes->Utils.Set.forEach(index =>
+        if !index.matcher(entity) {
+          index.ids->Utils.Set.delete(entityId)->ignore
+          indexes->Utils.Set.delete(index)->ignore
         }
-      })
+      )
     }
 
-    self.filterIndexes->Utils.Dict.forEach(((filter, relatedEntityIds)) => {
-      if filter->EntityFilter.matches(~entity=entityAsDict) {
-        //Add entity id to the filter index and the filter to entity filters
-        relatedEntityIds->Utils.Set.add(entityId)->ignore
-        self->getOrCreateEntityFilters(~entityId)->Utils.Set.add(filter)->ignore
-      } else {
-        relatedEntityIds->Utils.Set.delete(entityId)->ignore
+    self.eqBucketsByField->Utils.Dict.forEachWithKey((bucket, fieldName) => {
+      let fieldValue = entity->EntityFilter.getField(fieldName)
+
+      // A nullish column matches no equality index, and the key projections
+      // would throw on it.
+      if !(fieldValue->EntityFilter.nullish) {
+        switch bucket.byValue->Utils.Map.get(bucket.keyOf(fieldValue)) {
+        | Some(indexes) => indexes->Array.forEach(index => self->addToIndex(~index, ~entityId))
+        | None => ()
+        }
       }
     })
+
+    self.scanIndexes->Array.forEach(index =>
+      if index.matcher(entity) {
+        self->addToIndex(~index, ~entityId)
+      }
+    )
   }
 
   let deleteEntityFromIndexes = (self: t, ~entityId: string) =>
-    switch self.filtersByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
+    switch self.indexesByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
     | None => ()
-    | Some(entityFilters) =>
-      entityFilters->Utils.Set.forEach(filter => {
-        switch self.filterIndexes->Utils.Dict.dangerouslyGetNonOption(
-          filter->EntityFilter.toString,
-        ) {
-        | Some((_filter, relatedEntityIds)) =>
-          let _wasRemoved = relatedEntityIds->Utils.Set.delete(entityId)
-        | None => () //Unexpected filter index should exist if it is in entityFilters
-        }
-        let _wasRemoved = entityFilters->Utils.Set.delete(filter)
-      })
+    | Some(indexes) =>
+      indexes->Utils.Set.forEach(index => index.ids->Utils.Set.delete(entityId)->ignore)
+      self.indexesByEntityId->Utils.Dict.deleteInPlace(entityId)
     }
 
   let set = (inMemTable: t, ~committedCheckpointId, change: Change.t<Internal.entity>) => {
@@ -200,47 +225,110 @@ module Entity = {
 
   let hasIndex = (inMemTable: t) =>
     (filterKey: string) =>
-      inMemTable.filterIndexes->Utils.Dict.dangerouslyGetNonOption(filterKey) !== None
+      inMemTable.indexesByKey->Utils.Dict.dangerouslyGetNonOption(filterKey) !== None
 
-  let getUnsafeOnIndex = (inMemTable: t) => {
-    let getEntity = inMemTable->getUnsafe
+  let getUnsafeOnIndex = (inMemTable: t) =>
     (filterKey: string) => {
-      switch inMemTable.filterIndexes->Utils.Dict.dangerouslyGetNonOption(filterKey) {
+      switch inMemTable.indexesByKey->Utils.Dict.dangerouslyGetNonOption(filterKey) {
       | None =>
         JsError.throwWithMessage(`Unexpected error. Must have an index for the filter ${filterKey}`)
-      | Some((_filter, relatedEntityIds)) =>
-        relatedEntityIds
+      | Some({ids}) =>
+        ids
         ->Utils.Set.toArray
-        ->Array.filterMap(entityId => {
-          switch inMemTable.latestEntityChangeById->Dict.has(entityId) {
-          | true => getEntity(entityId)
-          | false => None
+        ->Array.filterMap(entityId =>
+          switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(entityId) {
+          | Some(change) => change->mapChangeToEntity
+          | None => None
           }
-        })
+        )
       }
     }
-  }
 
-  let addEmptyIndex = (inMemTable: t, ~filter: EntityFilter.t) => {
-    let filterKey = filter->EntityFilter.toString
-    switch inMemTable.filterIndexes->Utils.Dict.dangerouslyGetNonOption(filterKey) {
+  let getOrCreateEqBucket = (inMemTable: t, ~table: Table.table, ~fieldName) =>
+    switch inMemTable.eqBucketsByField->Utils.Dict.dangerouslyGetNonOption(fieldName) {
+    | Some(bucket) => bucket
+    | None =>
+      let bucket = {
+        keyOf: EntityFilter.makeValueKey(~table, ~fieldName),
+        byValue: Utils.Map.make(),
+        loadedValues: Utils.Set.make(),
+      }
+      inMemTable.eqBucketsByField->Dict.set(fieldName, bucket)
+      bucket
+    }
+
+  // The values a filter constrains a single field to exactly, which is what a
+  // bucket and the loaded set are keyed by. Everything else — ranges, several
+  // fields, several operators — has no such list.
+  let singleFieldValues = (filter: EntityFilter.t) =>
+    switch filter->EntityFilter.asSingleOperator {
+    | Some((fieldName, "_eq", fieldValue)) => Some((fieldName, [fieldValue]))
+    | Some((fieldName, "_in", fieldValue)) => Some((fieldName, fieldValue->EntityFilter.asArray))
+    | Some(_) | None => None
+    }
+
+  let addEmptyIndex = (inMemTable: t, ~filter: EntityFilter.t, ~table: Table.table) => {
+    let filterKey = filter->EntityFilter.toString(~table)
+    switch inMemTable.indexesByKey->Utils.Dict.dangerouslyGetNonOption(filterKey) {
     | Some(_) => () //Should not happen, this means the index already exists
     | None =>
-      let relatedEntityIds = Utils.Set.make()
+      let index = {matcher: filter->EntityFilter.makeMatcher(~table), ids: Utils.Set.make()}
+      inMemTable.indexesByKey->Dict.set(filterKey, index)
+
+      switch filter->singleFieldValues {
+      | Some((fieldName, fieldValues)) =>
+        let bucket = inMemTable->getOrCreateEqBucket(~table, ~fieldName)
+        fieldValues->Array.forEach(fieldValue => {
+          let valueKey = bucket.keyOf(fieldValue)
+          switch bucket.byValue->Utils.Map.get(valueKey) {
+          | Some(indexes) => indexes->Array.push(index)->ignore
+          | None => bucket.byValue->Utils.Map.set(valueKey, [index])->ignore
+          }
+        })
+      | None => inMemTable.scanIndexes->Array.push(index)->ignore
+      }
+
       inMemTable.latestEntityChangeById->Utils.Dict.forEach(change => {
         switch change->mapChangeToEntity {
         | Some(entity) =>
-          let entityAsDict =
-            entity->(Utils.magic: Internal.entity => dict<EntityFilter.FieldValue.t>)
-          if filter->EntityFilter.matches(~entity=entityAsDict) {
-            let entityId = entity->getEntityIdUnsafe
-            let _ = inMemTable->getOrCreateEntityFilters(~entityId)->Utils.Set.add(filter)
-            let _ = relatedEntityIds->Utils.Set.add(entityId)
+          if index.matcher(entity) {
+            inMemTable->addToIndex(~index, ~entityId=entity->getEntityIdUnsafe)
           }
         | None => ()
         }
       })
-      inMemTable.filterIndexes->Dict.set(filterKey, (filter, relatedEntityIds))
     }
   }
+
+  // Called once a query's rows are in the table, so every row the filter's
+  // values could match is now in memory.
+  let recordLoadedValues = (inMemTable: t, ~filter: EntityFilter.t, ~table: Table.table) =>
+    switch filter->singleFieldValues {
+    | Some((fieldName, fieldValues)) =>
+      let bucket = inMemTable->getOrCreateEqBucket(~table, ~fieldName)
+      fieldValues->Array.forEach(fieldValue =>
+        bucket.loadedValues->Utils.Set.add(bucket.keyOf(fieldValue))->ignore
+      )
+    | None => ()
+    }
+
+  let isFullyLoaded = (inMemTable: t, ~filter: EntityFilter.t) =>
+    switch filter->singleFieldValues {
+    | Some((fieldName, fieldValues)) =>
+      switch inMemTable.eqBucketsByField->Utils.Dict.dangerouslyGetNonOption(fieldName) {
+      | Some(bucket) =>
+        fieldValues->Array.every(fieldValue =>
+          bucket.loadedValues->Utils.Set.has(bucket.keyOf(fieldValue))
+        )
+      | None => false
+      }
+    | None => false
+    }
+
+  // Builds the index for a filter whose every value an earlier query already
+  // loaded, so the call is served from the table instead of going to storage.
+  let tryIndexFromLoadedValues = (inMemTable: t, ~filter: EntityFilter.t, ~table: Table.table) =>
+    if inMemTable->isFullyLoaded(~filter) {
+      inMemTable->addEmptyIndex(~filter, ~table)
+    }
 }

package/src/InMemoryTable.res.mjs

@@ -24,14 +24,14 @@ function getEntityIdUnsafe(entity) {
   }
 }
 
-function getOrCreateEntityFilters(self, entityId) {
-  let s = self.filtersByEntityId[entityId];
-  if (s !== undefined) {
-    return Primitive_option.valFromOption(s);
+function addToIndex(self, index, entityId) {
+  index.ids.add(entityId);
+  let indexes = self.indexesByEntityId[entityId];
+  if (indexes !== undefined) {
+    Primitive_option.valFromOption(indexes).add(index);
+  } else {
+    self.indexesByEntityId[entityId] = new Set([index]);
   }
-  let s$1 = new Set();
-  self.filtersByEntityId[entityId] = s$1;
-  return s$1;
 }
 
 function make() {
@@ -39,8 +39,10 @@ function make() {
     latestEntityChangeById: {},
     changesCount: 0,
     prevEntityChanges: [],
-    filtersByEntityId: {},
-    filterIndexes: {}
+    indexesByEntityId: {},
+    indexesByKey: {},
+    eqBucketsByField: {},
+    scanIndexes: []
   };
 }
 
@@ -83,47 +85,51 @@ function dropCommittedChanges(self, committedCheckpointId, keepLoadedFromDb) {
   });
   keysToDelete.forEach(key => Utils.Dict.deleteInPlace(self.latestEntityChangeById, key));
   self.changesCount = self.changesCount - keysToDelete.length;
-  self.filtersByEntityId = {};
-  self.filterIndexes = {};
+  self.indexesByEntityId = {};
+  self.indexesByKey = {};
+  self.eqBucketsByField = {};
+  self.scanIndexes = [];
 }
 
 function updateIndexes(self, entity) {
   let entityId = getEntityIdUnsafe(entity);
-  let entityFilters = self.filtersByEntityId[entityId];
-  if (entityFilters !== undefined) {
-    let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
-    entityFilters$1.forEach(filter => {
-      if (!EntityFilter.matches(filter, entity)) {
-        entityFilters$1.delete(filter);
+  let indexes = self.indexesByEntityId[entityId];
+  if (indexes !== undefined) {
+    let indexes$1 = Primitive_option.valFromOption(indexes);
+    indexes$1.forEach(index => {
+      if (!index.matcher(entity)) {
+        index.ids.delete(entityId);
+        indexes$1.delete(index);
         return;
       }
     });
   }
-  Utils.Dict.forEach(self.filterIndexes, param => {
-    let relatedEntityIds = param[1];
-    let filter = param[0];
-    if (EntityFilter.matches(filter, entity)) {
-      relatedEntityIds.add(entityId);
-      getOrCreateEntityFilters(self, entityId).add(filter);
-    } else {
-      relatedEntityIds.delete(entityId);
+  Utils.Dict.forEachWithKey(self.eqBucketsByField, (bucket, fieldName) => {
+    let fieldValue = entity[fieldName];
+    if (EntityFilter.nullish(fieldValue)) {
+      return;
+    }
+    let indexes = bucket.byValue.get(bucket.keyOf(fieldValue));
+    if (indexes !== undefined) {
+      indexes.forEach(index => addToIndex(self, index, entityId));
+      return;
+    }
+  });
+  self.scanIndexes.forEach(index => {
+    if (index.matcher(entity)) {
+      return addToIndex(self, index, entityId);
     }
   });
 }
 
 function deleteEntityFromIndexes(self, entityId) {
-  let entityFilters = self.filtersByEntityId[entityId];
-  if (entityFilters === undefined) {
-    return;
+  let indexes = self.indexesByEntityId[entityId];
+  if (indexes !== undefined) {
+    Primitive_option.valFromOption(indexes).forEach(index => {
+      index.ids.delete(entityId);
+    });
+    return Utils.Dict.deleteInPlace(self.indexesByEntityId, entityId);
   }
-  let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
-  entityFilters$1.forEach(filter => {
-    let match = self.filterIndexes[EntityFilter.toString(filter)];
-    if (match !== undefined) {
-      match[1].delete(entityId);
-    }
-    entityFilters$1.delete(filter);
-  });
 }
 
 function set(inMemTable, committedCheckpointId, change) {
@@ -174,17 +180,17 @@ function getUnsafe(inMemTable) {
 }
 
 function hasIndex(inMemTable) {
-  return filterKey => inMemTable.filterIndexes[filterKey] !== undefined;
+  return filterKey => inMemTable.indexesByKey[filterKey] !== undefined;
 }
 
 function getUnsafeOnIndex(inMemTable) {
-  let getEntity = getUnsafe(inMemTable);
   return filterKey => {
-    let match = inMemTable.filterIndexes[filterKey];
+    let match = inMemTable.indexesByKey[filterKey];
     if (match !== undefined) {
-      return Stdlib_Array.filterMap(Array.from(match[1]), entityId => {
-        if (entityId in inMemTable.latestEntityChangeById) {
-          return getEntity(entityId);
+      return Stdlib_Array.filterMap(Array.from(match.ids), entityId => {
+        let change = inMemTable.latestEntityChangeById[entityId];
+        if (change !== undefined) {
+          return mapChangeToEntity(change);
         }
       });
     } else {
@@ -193,35 +199,115 @@ function getUnsafeOnIndex(inMemTable) {
   };
 }
 
-function addEmptyIndex(inMemTable, filter) {
-  let filterKey = EntityFilter.toString(filter);
-  let match = inMemTable.filterIndexes[filterKey];
+function getOrCreateEqBucket(inMemTable, table, fieldName) {
+  let bucket = inMemTable.eqBucketsByField[fieldName];
+  if (bucket !== undefined) {
+    return bucket;
+  }
+  let bucket_keyOf = EntityFilter.makeValueKey(table, fieldName);
+  let bucket_byValue = new Map();
+  let bucket_loadedValues = new Set();
+  let bucket$1 = {
+    keyOf: bucket_keyOf,
+    byValue: bucket_byValue,
+    loadedValues: bucket_loadedValues
+  };
+  inMemTable.eqBucketsByField[fieldName] = bucket$1;
+  return bucket$1;
+}
+
+function singleFieldValues(filter) {
+  let match = EntityFilter.asSingleOperator(filter);
+  if (match === undefined) {
+    return;
+  }
+  let fieldName = match[0];
+  switch (match[1]) {
+    case "_eq" :
+      return [
+        fieldName,
+        [match[2]]
+      ];
+    case "_in" :
+      return [
+        fieldName,
+        EntityFilter.asArray(match[2])
+      ];
+    default:
+      return;
+  }
+}
+
+function addEmptyIndex(inMemTable, filter, table) {
+  let filterKey = EntityFilter.toString(filter, table);
+  let match = inMemTable.indexesByKey[filterKey];
   if (match !== undefined) {
     return;
   }
-  let relatedEntityIds = new Set();
+  let index_matcher = EntityFilter.makeMatcher(filter, table);
+  let index_ids = new Set();
+  let index = {
+    matcher: index_matcher,
+    ids: index_ids
+  };
+  inMemTable.indexesByKey[filterKey] = index;
+  let match$1 = singleFieldValues(filter);
+  if (match$1 !== undefined) {
+    let bucket = getOrCreateEqBucket(inMemTable, table, match$1[0]);
+    match$1[1].forEach(fieldValue => {
+      let valueKey = bucket.keyOf(fieldValue);
+      let indexes = bucket.byValue.get(valueKey);
+      if (indexes !== undefined) {
+        indexes.push(index);
+      } else {
+        bucket.byValue.set(valueKey, [index]);
+      }
+    });
+  } else {
+    inMemTable.scanIndexes.push(index);
+  }
   Utils.Dict.forEach(inMemTable.latestEntityChangeById, change => {
     let entity = mapChangeToEntity(change);
-    if (entity === undefined) {
-      return;
+    if (entity !== undefined && index_matcher(entity)) {
+      return addToIndex(inMemTable, index, getEntityIdUnsafe(entity));
     }
-    if (!EntityFilter.matches(filter, entity)) {
-      return;
-    }
-    let entityId = getEntityIdUnsafe(entity);
-    getOrCreateEntityFilters(inMemTable, entityId).add(filter);
-    relatedEntityIds.add(entityId);
   });
-  inMemTable.filterIndexes[filterKey] = [
-    filter,
-    relatedEntityIds
-  ];
+}
+
+function recordLoadedValues(inMemTable, filter, table) {
+  let match = singleFieldValues(filter);
+  if (match === undefined) {
+    return;
+  }
+  let bucket = getOrCreateEqBucket(inMemTable, table, match[0]);
+  match[1].forEach(fieldValue => {
+    bucket.loadedValues.add(bucket.keyOf(fieldValue));
+  });
+}
+
+function isFullyLoaded(inMemTable, filter) {
+  let match = singleFieldValues(filter);
+  if (match === undefined) {
+    return false;
+  }
+  let bucket = inMemTable.eqBucketsByField[match[0]];
+  if (bucket !== undefined) {
+    return match[1].every(fieldValue => bucket.loadedValues.has(bucket.keyOf(fieldValue)));
+  } else {
+    return false;
+  }
+}
+
+function tryIndexFromLoadedValues(inMemTable, filter, table) {
+  if (isFullyLoaded(inMemTable, filter)) {
+    return addEmptyIndex(inMemTable, filter, table);
+  }
 }
 
 let Entity = {
   UnexpectedIdNotDefinedOnEntity: UnexpectedIdNotDefinedOnEntity,
   getEntityIdUnsafe: getEntityIdUnsafe,
-  getOrCreateEntityFilters: getOrCreateEntityFilters,
+  addToIndex: addToIndex,
   make: make,
   snapshotChanges: snapshotChanges,
   dropCommittedChanges: dropCommittedChanges,
@@ -233,7 +319,12 @@ let Entity = {
   getUnsafe: getUnsafe,
   hasIndex: hasIndex,
   getUnsafeOnIndex: getUnsafeOnIndex,
-  addEmptyIndex: addEmptyIndex
+  getOrCreateEqBucket: getOrCreateEqBucket,
+  singleFieldValues: singleFieldValues,
+  addEmptyIndex: addEmptyIndex,
+  recordLoadedValues: recordLoadedValues,
+  isFullyLoaded: isFullyLoaded,
+  tryIndexFromLoadedValues: tryIndexFromLoadedValues
 };
 
 export {