envio 3.12.0 → 3.12.1

package/src/LoadLayer.res

@@ -6,23 +6,6 @@ let scopeKeySuffix = (scope: Internal.chainScope) =>
   | Chain(chainId) => `.${chainId->ChainId.toString}`
   }
 
-// Narrows a query to the scope's chain. Cross-chain entities have no chain-id
-// column, so their filter is left untouched.
-let scopeFilter = (filter: EntityFilter.t, ~table: Table.table, ~scope: Internal.chainScope) =>
-  switch (scope, table->Table.getChainIdField) {
-  | (Chain(chainId), Some(field)) =>
-    EntityFilter.And({
-      filters: [
-        filter,
-        Eq({
-          fieldName: field.fieldName,
-          fieldValue: chainId->(Utils.magic: ChainId.t => unknown),
-        }),
-      ],
-    })
-  | _ => filter
-  }
-
 let loadById = (
   ~loadManager,
   ~persistence: Persistence.t,
@@ -48,10 +31,10 @@ let loadById = (
       (
         await storage.loadOrThrow(
           ~table=entityConfig.table,
-          ~filter=EntityFilter.In({
-            fieldName: Table.idFieldName,
-            fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
-          })->scopeFilter(~table=entityConfig.table, ~scope),
+          ~filter=EntityFilter.byIds(idsToLoad)->EntityFilter.scoped(
+            ~table=entityConfig.table,
+            ~scope,
+          ),
         )
       )->(Utils.magic: array<unknown> => array<Internal.entity>)
     } catch {
@@ -273,15 +256,9 @@ let loadEffect = (
       let {outputSchema} = effect.storageMeta
 
       let dbEntities = try {
-        (
-          await storage.loadOrThrow(
-            ~table,
-            ~filter=EntityFilter.In({
-              fieldName: Table.idFieldName,
-              fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
-            }),
-          )
-        )->(Utils.magic: array<unknown> => array<Internal.effectCacheItem>)
+        (await storage.loadOrThrow(~table, ~filter=EntityFilter.byIds(idsToLoad)))->(
+          Utils.magic: array<unknown> => array<Internal.effectCacheItem>
+        )
       } catch {
       | exn =>
         Ecosystem.getItemLogger(item, ~ecosystem)->Logging.childWarn({
@@ -355,7 +332,7 @@ let loadEffect = (
   )
 }
 
-let loadByFilter = (
+let loadByParsedFilter = (
   ~loadManager,
   ~persistence: Persistence.t,
   ~entityConfig: Internal.entityConfig,
@@ -378,7 +355,9 @@ let loadByFilter = (
 
     let size = ref(0)
 
-    filters->Array.forEach(filter => inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter))
+    filters->Array.forEach(filter =>
+      inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter, ~table=entityConfig.table)
+    )
 
     // Any non-derived field can be filtered on, so the columns this query reads
     // are indexed on demand before it runs rather than promised by the schema.
@@ -399,7 +378,7 @@ let loadByFilter = (
           (
             await storage.loadOrThrow(
               ~table=entityConfig.table,
-              ~filter=filter->scopeFilter(~table=entityConfig.table, ~scope),
+              ~filter=filter->EntityFilter.scoped(~table=entityConfig.table, ~scope),
             )
           )->(Utils.magic: array<unknown> => array<Internal.entity>)
 
@@ -412,6 +391,10 @@ let loadByFilter = (
           )
         })
 
+        // Every row this filter's values could match is now in the table, so a
+        // later getWhere naming any of them needs no round trip of its own.
+        inMemTable->InMemoryTable.Entity.recordLoadedValues(~filter, ~table=entityConfig.table)
+
         size := size.contents + entities->Array.length
       } catch {
       | Persistence.StorageError({message, reason}) =>
@@ -442,13 +425,53 @@ let loadByFilter = (
     )
   }
 
+  // Keying an _in walks every value, so it's computed once here and handed to
+  // the load manager rather than recomputed by the hasher.
+  let filterKey = filter->EntityFilter.toString(~table=entityConfig.table)
+
+  if !(inMemTable->InMemoryTable.Entity.hasIndex)(filterKey) {
+    inMemTable->InMemoryTable.Entity.tryIndexFromLoadedValues(~filter, ~table=entityConfig.table)
+  }
+
   loadManager->LoadManager.call(
     ~key,
     ~load,
     ~input=filter,
     ~shouldGroup,
-    ~hasher=EntityFilter.toString,
+    ~hasher=_ => filterKey,
     ~getUnsafeInMemory=inMemTable->InMemoryTable.Entity.getUnsafeOnIndex,
     ~hasInMemory=inMemTable->InMemoryTable.Entity.hasIndex,
   )
 }
+
+let loadByFilter = (
+  ~loadManager,
+  ~persistence,
+  ~entityConfig: Internal.entityConfig,
+  ~scope,
+  ~indexerState,
+  ~shouldGroup,
+  ~item,
+  ~ecosystem,
+  ~filter: dict<dict<unknown>>,
+) =>
+  // Rejecting rather than throwing keeps a bad filter failing only its own
+  // call, even when the caller batches several with Promise.all.
+  try {
+    loadByParsedFilter(
+      ~loadManager,
+      ~persistence,
+      ~entityConfig,
+      ~scope,
+      ~indexerState,
+      ~shouldGroup,
+      ~item,
+      ~ecosystem,
+      ~filter=filter->EntityFilter.parseOrThrow(
+        ~entityName=entityConfig.name,
+        ~table=entityConfig.table,
+      ),
+    )
+  } catch {
+  | exn => Promise.reject(exn->Utils.prettifyExn)
+  }

package/src/LoadLayer.res.mjs

@@ -1,6 +1,5 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
@@ -27,25 +26,6 @@ function scopeKeySuffix(scope) {
   }
 }
 
-function scopeFilter(filter, table, scope) {
-  let match = Table.getChainIdField(table);
-  if (scope === "crossChain" || match === undefined) {
-    return filter;
-  } else {
-    return {
-      operator: "and",
-      filters: [
-        filter,
-        {
-          operator: "=",
-          fieldName: match.fieldName,
-          fieldValue: scope
-        }
-      ]
-    };
-  }
-}
-
 function loadById(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, entityId) {
   let key = entityConfig.name + `.get` + scopeKeySuffix(scope);
   let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
@@ -54,11 +34,7 @@ function loadById(loadManager, persistence, entityConfig, scope, indexerState, s
     let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
     let dbEntities;
     try {
-      dbEntities = await storage.loadOrThrow(scopeFilter({
-        operator: "in",
-        fieldName: Table.idFieldName,
-        fieldValue: idsToLoad
-      }, entityConfig.table, scope), entityConfig.table);
+      dbEntities = await storage.loadOrThrow(EntityFilter.scoped(EntityFilter.byIds(idsToLoad), entityConfig.table, scope), entityConfig.table);
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       if (exn.RE_EXN_ID === Persistence.StorageError) {
@@ -157,11 +133,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
       let outputSchema = match$1.outputSchema;
       let dbEntities;
       try {
-        dbEntities = await storage.loadOrThrow({
-          operator: "in",
-          fieldName: Table.idFieldName,
-          fieldValue: idsToLoad
-        }, table);
+        dbEntities = await storage.loadOrThrow(EntityFilter.byIds(idsToLoad), table);
       } catch (raw_exn) {
         let exn = Primitive_exceptions.internalToException(raw_exn);
         Logging.childWarn(Ecosystem.getItemLogger(item, ecosystem), {
@@ -211,38 +183,49 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
 }
 
 function loadByFilter(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, filter) {
-  let key = EntityFilter.toOperationKey(filter, entityConfig.name) + scopeKeySuffix(scope);
-  let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
-  let load = async (filters, param) => {
-    let storage = Persistence.getInitializedStorageOrThrow(persistence);
-    let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
-    let size = {
-      contents: 0
-    };
-    filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter));
-    await storage.ensureQueryIndexes(entityConfig, scope, filters);
-    let queries = EntityFilter.merge(filters);
-    await Promise.all(queries.map(async filter => {
-      try {
-        let entities = await storage.loadOrThrow(scopeFilter(filter, entityConfig.table, scope), entityConfig.table);
-        let committedCheckpointId = IndexerState.committedCheckpointIdFor(indexerState, scope);
-        entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entity.id, entity));
-        size.contents = size.contents + entities.length | 0;
-        return;
-      } catch (raw_exn) {
-        let exn = Primitive_exceptions.internalToException(raw_exn);
-        if (exn.RE_EXN_ID === Persistence.StorageError) {
-          return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Ecosystem.getItemLogger(item, ecosystem), {
-            operation: key,
-            params: EntityFilter.getParams(filter)
-          }), exn.message, exn.reason);
+  try {
+    let filter$1 = EntityFilter.parseOrThrow(filter, entityConfig.name, entityConfig.table);
+    let key = EntityFilter.toOperationKey(filter$1, entityConfig.name) + scopeKeySuffix(scope);
+    let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
+    let load = async (filters, param) => {
+      let storage = Persistence.getInitializedStorageOrThrow(persistence);
+      let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
+      let size = {
+        contents: 0
+      };
+      filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter, entityConfig.table));
+      await storage.ensureQueryIndexes(entityConfig, scope, filters);
+      let queries = EntityFilter.merge(filters);
+      await Promise.all(queries.map(async filter => {
+        try {
+          let entities = await storage.loadOrThrow(EntityFilter.scoped(filter, entityConfig.table, scope), entityConfig.table);
+          let committedCheckpointId = IndexerState.committedCheckpointIdFor(indexerState, scope);
+          entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entity.id, entity));
+          InMemoryTable.Entity.recordLoadedValues(inMemTable, filter, entityConfig.table);
+          size.contents = size.contents + entities.length | 0;
+          return;
+        } catch (raw_exn) {
+          let exn = Primitive_exceptions.internalToException(raw_exn);
+          if (exn.RE_EXN_ID === Persistence.StorageError) {
+            return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Ecosystem.getItemLogger(item, ecosystem), {
+              operation: key,
+              params: EntityFilter.getParams(filter)
+            }), exn.message, exn.reason);
+          }
+          throw exn;
         }
-        throw exn;
-      }
-    }));
-    return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
-  };
-  return LoadManager.call(loadManager, filter, key, load, EntityFilter.toString, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
+      }));
+      return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
+    };
+    let filterKey = EntityFilter.toString(filter$1, entityConfig.table);
+    if (!InMemoryTable.Entity.hasIndex(inMemTable)(filterKey)) {
+      InMemoryTable.Entity.tryIndexFromLoadedValues(inMemTable, filter$1, entityConfig.table);
+    }
+    return LoadManager.call(loadManager, filter$1, key, load, param => filterKey, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    return Promise.reject(Utils.prettifyExn(exn));
+  }
 }
 
 export {
@@ -250,4 +233,4 @@ export {
   loadByFilter,
   loadEffect,
 }
-/* Table Not a pure module */
+/* Utils Not a pure module */

package/src/LoadLayer.resi

@@ -19,7 +19,7 @@ let loadByFilter: (
   ~shouldGroup: bool,
   ~item: Internal.item,
   ~ecosystem: Ecosystem.t,
-  ~filter: EntityFilter.t,
+  ~filter: dict<dict<unknown>>,
 ) => promise<array<Internal.entity>>
 
 let loadEffect: (

package/src/PgStorage.res

@@ -409,9 +409,10 @@ let makeLoadQuery = (~pgSchema, ~tableName, ~condition) => {
 // Field names are spliced as quoted identifiers only after the queryFields
 // lookup proves they exist on the table (and they originate from
 // codegen-validated schemas), so the interpolation can't be abused.
-let rec makeFilterCondition = (
+let makeFilterCondition = (
   ~filter: EntityFilter.t,
   ~table: Table.table,
+  ~pgSchema,
   ~params: array<unknown>,
 ) => {
   // Filters reference fields by API name, while the SQL references columns
@@ -447,60 +448,89 @@ let rec makeFilterCondition = (
     params->Array.push(param)->ignore
     `$${params->Array.length->Int.toString}`
   }
-  let scalarCondition = (~fieldName, ~fieldValue, ~op) => {
+
+  let condition = ref("")
+  filter
+  ->EntityFilter.entries
+  ->Utils.Dict.forEachWithKey((operators, fieldName) => {
     let queryField = getQueryFieldOrThrow(fieldName)
-    `"${queryField.pgDbFieldName}" ${op} ${serializeParamOrThrow(
-        ~queryField,
-        ~fieldName,
-        ~fieldValue,
-        ~isArray=false,
-      )}`
-  }
-  switch filter {
-  // A per-chain entity's table is partitioned by its chain-id column, and
-  // Postgres can only prune a plan it caches when that column is a constant in
-  // the SQL. Bound, the cached plan has to keep every partition, and the
-  // planner ends up throwing it away and re-planning on every execution
-  // instead — measured at 315us per load against 218us with the id written in,
-  // on 30 chains.
-  //
-  // The cost is that each chain gets its own query text, so Postgres caches a
-  // prepared statement per (entity, chain, filter shape) rather than per
-  // (entity, filter shape). Measured at ~8KB of plan cache each, which is ~10MB
-  // per connection for 40 entities across 30 chains — accepted, since the
-  // alternative is a cached plan that can't prune.
-  //
-  // `LoadLayer.scopeFilter` is what puts this filter here, and the value is
-  // range-checked to a non-negative safe integer, so it can carry nothing but
-  // digits.
-  | Eq({fieldName, fieldValue}) if getQueryFieldOrThrow(fieldName).isChainId =>
-    `"${getQueryFieldOrThrow(fieldName).pgDbFieldName}" = ${fieldValue
-      ->ChainId.normalizeOrThrow
-      ->ChainId.toString}`
-  | Eq({fieldName, fieldValue}) => scalarCondition(~fieldName, ~fieldValue, ~op="=")
-  | Gt({fieldName, fieldValue}) => scalarCondition(~fieldName, ~fieldValue, ~op=">")
-  | Lt({fieldName, fieldValue}) => scalarCondition(~fieldName, ~fieldValue, ~op="<")
-  | In({fieldName, fieldValue}) => {
-      let queryField = getQueryFieldOrThrow(fieldName)
-      `"${queryField.pgDbFieldName}" = ANY(${serializeParamOrThrow(
-          ~queryField,
-          ~fieldName,
-          ~fieldValue=fieldValue->(Utils.magic: array<unknown> => unknown),
-          ~isArray=true,
-        )})`
-    }
-  | And({filters: []}) =>
-    throw(
-      Persistence.StorageError({
-        message: `Failed loading "${table.tableName}" from storage. The "and" filter must contain at least one nested filter.`,
-        reason: Utils.Error.make(`Empty "and" filter`),
-      }),
-    )
-  | And({filters}) =>
-    `(${filters
-      ->Array.map(filter => makeFilterCondition(~filter, ~table, ~params))
-      ->Array.join(" AND ")})`
-  }
+    operators->Utils.Dict.forEachWithKey((fieldValue, operator) => {
+      let column = `"${queryField.pgDbFieldName}"`
+      let part = switch operator {
+      // A per-chain entity's table is partitioned by its chain-id column, and
+      // Postgres can only prune a plan it caches when that column is a constant
+      // in the SQL. Bound, the cached plan has to keep every partition, and the
+      // planner ends up throwing it away and re-planning on every execution
+      // instead — measured at 315us per load against 218us with the id written
+      // in, on 30 chains.
+      //
+      // The cost is that each chain gets its own query text, so Postgres caches
+      // a prepared statement per (entity, chain, filter shape) rather than per
+      // (entity, filter shape). Measured at ~8KB of plan cache each, which is
+      // ~10MB per connection for 40 entities across 30 chains — accepted, since
+      // the alternative is a cached plan that can't prune.
+      //
+      // `EntityFilter.scoped` is what puts this filter here, and the value is
+      // range-checked to a non-negative safe integer, so it can carry nothing
+      // but digits.
+      | "_eq" if queryField.isChainId =>
+        `${column} = ${fieldValue->ChainId.normalizeOrThrow->ChainId.toString}`
+      // Postgres arrays are rectangular, so candidates for a list column can't
+      // be bound as one array unless they all have the same length, and
+      // postgres.js can't bind a boolean array at all
+      // (https://github.com/porsager/postgres/issues/471). One equality per
+      // candidate has neither problem.
+      | "_in" if queryField.isArray || queryField.fieldType === Boolean =>
+        switch fieldValue->EntityFilter.asArray {
+        | [] => "FALSE"
+        | candidates =>
+          `(${candidates
+            ->Array.map(
+              candidate =>
+                `${column} = ${serializeParamOrThrow(
+                    ~queryField,
+                    ~fieldName,
+                    ~fieldValue=candidate,
+                    ~isArray=false,
+                  )}`,
+            )
+            ->Array.join(" OR ")})`
+        }
+      | "_in" =>
+        let param = serializeParamOrThrow(~queryField, ~fieldName, ~fieldValue, ~isArray=true)
+        switch queryField.fieldType {
+        // A bound array of strings is text[], which has no equality with an
+        // enum. The insert casts the same way.
+        | Enum({config}) => `${column} = ANY(${param}::TEXT[]::"${pgSchema}".${config.name}[])`
+        | _ => `${column} = ANY(${param})`
+        }
+      | _ =>
+        let sqlOperator = switch operator {
+        | "_eq" => "="
+        | "_gt" => ">"
+        | "_lt" => "<"
+        | "_gte" => ">="
+        | "_lte" => "<="
+        | _ =>
+          throw(
+            Persistence.StorageError({
+              message: `Failed loading "${table.tableName}" from storage. Unknown filter operator "${operator}".`,
+              reason: Utils.Error.make(`Unknown filter operator "${operator}"`),
+            }),
+          )
+        }
+        `${column} ${sqlOperator} ${serializeParamOrThrow(
+            ~queryField,
+            ~fieldName,
+            ~fieldValue,
+            ~isArray=false,
+          )}`
+      }
+      condition := (condition.contents === "" ? part : condition.contents ++ " AND " ++ part)
+    })
+  })
+
+  condition.contents
 }
 
 // The chain-id predicate a per-chain entity's row-level SQL needs, already
@@ -1709,7 +1739,7 @@ let make = (
   // Must match PG_CONTAINER in packages/cli/src/docker_env.rs
   let containerName = "envio-postgres"
   let psqlExecOptions: NodeJs.ChildProcess.execOptions = {
-    env: Dict.fromArray([("PGPASSWORD", pgPassword), ("PATH", %raw(`process.env.PATH`))]),
+    env: dict{"PGPASSWORD": pgPassword, "PATH": %raw(`process.env.PATH`)},
   }
 
   let cacheDirPath = NodeJs.Path.resolve([
@@ -2002,7 +2032,7 @@ let make = (
 
   let loadOrThrow = async (~filter: EntityFilter.t, ~table: Table.table) => {
     let params = []
-    let condition = makeFilterCondition(~filter, ~table, ~params)
+    let condition = makeFilterCondition(~filter, ~table, ~pgSchema, ~params)
     switch await sql->Postgres.preparedUnsafe(
       makeLoadQuery(~pgSchema, ~tableName=table.tableName, ~condition),
       params->Obj.magic,
@@ -2033,9 +2063,10 @@ let make = (
     let queryFields = table->Table.queryFields
     let columns = []
     let seen = Utils.Set.make()
-    let rec collect = (filter: EntityFilter.t) =>
-      switch filter {
-      | Eq({fieldName}) | Gt({fieldName}) | Lt({fieldName}) | In({fieldName}) =>
+    filters->Array.forEach(filter =>
+      filter
+      ->EntityFilter.entries
+      ->Utils.Dict.forEachWithKey((_, fieldName) =>
         switch queryFields->Utils.Dict.dangerouslyGetNonOption(fieldName) {
         | Some({pgDbFieldName}) =>
           if !(seen->Utils.Set.has(pgDbFieldName)) {
@@ -2044,9 +2075,8 @@ let make = (
           }
         | None => ()
         }
-      | And({filters}) => filters->Array.forEach(collect)
-      }
-    filters->Array.forEach(collect)
+      )
+    )
     columns
   }