envio 2.22.3 → 2.24.0

package/evm.schema.json

@@ -33,6 +33,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/fuel.schema.json

@@ -26,6 +26,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.22.3",
+  "version": "v2.24.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.22.3",
-    "envio-linux-arm64": "v2.22.3",
-    "envio-darwin-x64": "v2.22.3",
-    "envio-darwin-arm64": "v2.22.3"
+    "envio-linux-x64": "v2.24.0",
+    "envio-linux-arm64": "v2.24.0",
+    "envio-darwin-x64": "v2.24.0",
+    "envio-darwin-arm64": "v2.24.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Logging.res

@@ -168,6 +168,14 @@ let logForItem = (eventItem, level: Pino.logLevel, message: string, ~params=?) =
   (eventItem->getEventLogger->Utils.magic->Js.Dict.unsafeGet((level :> string)))(params, message)
 }
 
+let noopLogger: Envio.logger = {
+  info: (_message: string, ~params as _=?) => (),
+  debug: (_message: string, ~params as _=?) => (),
+  warn: (_message: string, ~params as _=?) => (),
+  error: (_message: string, ~params as _=?) => (),
+  errorWithExn: (_message: string, _exn) => (),
+}
+
 let getUserLogger = (eventItem): Envio.logger => {
   info: (message: string, ~params=?) => eventItem->logForItem(#uinfo, message, ~params?),
   debug: (message: string, ~params=?) => eventItem->logForItem(#udebug, message, ~params?),

package/src/Logging.res.js

@@ -197,6 +197,34 @@ function logForItem(eventItem, level, message, params) {
   return getEventLogger(eventItem)[level](params, message);
 }
 
+function noopLogger_debug(_message, param) {
+  
+}
+
+function noopLogger_info(_message, param) {
+  
+}
+
+function noopLogger_warn(_message, param) {
+  
+}
+
+function noopLogger_error(_message, param) {
+  
+}
+
+function noopLogger_errorWithExn(_message, _exn) {
+  
+}
+
+var noopLogger = {
+  debug: noopLogger_debug,
+  info: noopLogger_info,
+  warn: noopLogger_warn,
+  error: noopLogger_error,
+  errorWithExn: noopLogger_errorWithExn
+};
+
 function getUserLogger(eventItem) {
   return {
           debug: (function (message, params) {
@@ -243,5 +271,6 @@ exports.createChild = createChild;
 exports.createChildFrom = createChildFrom;
 exports.getEventLogger = getEventLogger;
 exports.logForItem = logForItem;
+exports.noopLogger = noopLogger;
 exports.getUserLogger = getUserLogger;
 /* logLevels Not a pure module */

package/src/Persistence.res

@@ -12,14 +12,28 @@ type storage = {
   // Should initialize the storage so we can start interacting with it
   // Eg create connection, schema, tables, etc.
   initialize: (
-    ~entities: array<Internal.entityConfig>,
-    ~staticTables: array<Table.table>,
-    ~enums: array<Internal.enumConfig<Internal.enum>>,
+    ~entities: array<Internal.entityConfig>=?,
+    ~generalTables: array<Table.table>=?,
+    ~enums: array<Internal.enumConfig<Internal.enum>>=?,
     // If true, the storage should clear existing data
-    ~cleanRun: bool,
+    ~cleanRun: bool=?,
+  ) => promise<unit>,
+  @raises("StorageError")
+  loadByIdsOrThrow: 'item. (
+    ~ids: array<string>,
+    ~table: Table.table,
+    ~rowsSchema: S.t<array<'item>>,
+  ) => promise<array<'item>>,
+  @raises("StorageError")
+  setOrThrow: 'item. (
+    ~items: array<'item>,
+    ~table: Table.table,
+    ~itemSchema: S.t<'item>,
   ) => promise<unit>,
 }
 
+exception StorageError({message: string, reason: exn})
+
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
@@ -93,7 +107,7 @@ let init = async (
       } else {
         let _ = await persistence.storage.initialize(
           ~entities=persistence.allEntities,
-          ~staticTables=persistence.staticTables,
+          ~generalTables=persistence.staticTables,
           ~enums=persistence.allEnums,
           ~cleanRun=reset || !skipIsInitializedCheck,
         )
@@ -109,3 +123,12 @@ let init = async (
   | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
+
+let getInitializedStorageOrThrow = persistence => {
+  switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
+  | Ready(_) => persistence.storage
+  }
+}

package/src/Persistence.res.js

@@ -1,10 +1,14 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Js_exn = require("rescript/lib/js/js_exn.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
+var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
+var StorageError = /* @__PURE__ */Caml_exceptions.create("Persistence.StorageError");
+
 var entityHistoryActionEnumConfig_name = EntityHistory.RowAction.name;
 
 var entityHistoryActionEnumConfig_variants = EntityHistory.RowAction.variants;
@@ -84,7 +88,18 @@ async function init(persistence, skipIsInitializedCheckOpt, resetOpt) {
   }
 }
 
+function getInitializedStorageOrThrow(persistence) {
+  var match = persistence.storageStatus;
+  if (typeof match !== "object" || match.TAG === "Initializing") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  } else {
+    return persistence.storage;
+  }
+}
+
+exports.StorageError = StorageError;
 exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
+exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
 /* EntityHistory Not a pure module */

package/src/PgStorage.res

@@ -1,16 +1,16 @@
-let makeCreateIndexSqlUnsafe = (~tableName, ~indexFields, ~pgSchema) => {
+let makeCreateIndexSql = (~tableName, ~indexFields, ~pgSchema) => {
   let indexName = tableName ++ "_" ++ indexFields->Js.Array2.joinWith("_")
   let index = indexFields->Belt.Array.map(idx => `"${idx}"`)->Js.Array2.joinWith(", ")
   `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${pgSchema}"."${tableName}"(${index});`
 }
 
-let makeCreateTableIndicesSqlUnsafe = (table: Table.table, ~pgSchema) => {
+let makeCreateTableIndicesSql = (table: Table.table, ~pgSchema) => {
   open Belt
   let tableName = table.tableName
   let createIndex = indexField =>
-    makeCreateIndexSqlUnsafe(~tableName, ~indexFields=[indexField], ~pgSchema)
+    makeCreateIndexSql(~tableName, ~indexFields=[indexField], ~pgSchema)
   let createCompositeIndex = indexFields => {
-    makeCreateIndexSqlUnsafe(~tableName, ~indexFields, ~pgSchema)
+    makeCreateIndexSql(~tableName, ~indexFields, ~pgSchema)
   }
 
   let singleIndices = table->Table.getSingleIndices
@@ -20,7 +20,7 @@ let makeCreateTableIndicesSqlUnsafe = (table: Table.table, ~pgSchema) => {
     compositeIndices->Array.map(createCompositeIndex)->Js.Array2.joinWith("\n")
 }
 
-let makeCreateTableSqlUnsafe = (table: Table.table, ~pgSchema) => {
+let makeCreateTableSql = (table: Table.table, ~pgSchema) => {
   open Belt
   let fieldsMapped =
     table
@@ -55,12 +55,12 @@ let makeCreateTableSqlUnsafe = (table: Table.table, ~pgSchema) => {
 let makeInitializeTransaction = (
   ~pgSchema,
   ~pgUser,
-  ~staticTables,
-  ~entities,
-  ~enums,
-  ~cleanRun,
+  ~generalTables=[],
+  ~entities=[],
+  ~enums=[],
+  ~cleanRun=false,
 ) => {
-  let allTables = staticTables->Array.copy
+  let allTables = generalTables->Array.copy
   let allEntityTables = []
   entities->Js.Array2.forEach((entity: Internal.entityConfig) => {
     allEntityTables->Js.Array2.push(entity.table)->ignore
@@ -106,12 +106,12 @@ END IF;`
 
   // Batch all table creation first (optimal for PostgreSQL)
   allTables->Js.Array2.forEach((table: Table.table) => {
-    query := query.contents ++ "\n" ++ makeCreateTableSqlUnsafe(table, ~pgSchema)
+    query := query.contents ++ "\n" ++ makeCreateTableSql(table, ~pgSchema)
   })
 
   // Then batch all indices (better performance when tables exist)
   allTables->Js.Array2.forEach((table: Table.table) => {
-    let indices = makeCreateTableIndicesSqlUnsafe(table, ~pgSchema)
+    let indices = makeCreateTableIndicesSql(table, ~pgSchema)
     if indices !== "" {
       query := query.contents ++ "\n" ++ indices
     }
@@ -131,7 +131,7 @@ END IF;`
       query :=
         query.contents ++
         "\n" ++
-        makeCreateIndexSqlUnsafe(
+        makeCreateIndexSql(
           ~tableName=derivedFromField.derivedFromEntity,
           ~indexFields=[indexField],
           ~pgSchema,
@@ -143,11 +143,221 @@ END IF;`
     // Return optimized queries - main DDL in DO block, functions separate
     // Note: DO $$ BEGIN wrapper is only needed for PL/pgSQL conditionals (IF NOT EXISTS)
     // Reset case uses direct DDL (faster), non-cleanRun case uses conditionals (safer)
-    cleanRun ? query.contents : `DO $$ BEGIN ${query.contents} END $$;`,
+    cleanRun || enums->Utils.Array.isEmpty
+      ? query.contents
+      : `DO $$ BEGIN ${query.contents} END $$;`,
     // Functions query (separate as they can't be in DO block)
   ]->Js.Array2.concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : [])
 }
 
+let makeLoadByIdSql = (~pgSchema, ~tableName) => {
+  `SELECT * FROM "${pgSchema}"."${tableName}" WHERE id = $1 LIMIT 1;`
+}
+
+let makeLoadByIdsSql = (~pgSchema, ~tableName) => {
+  `SELECT * FROM "${pgSchema}"."${tableName}" WHERE id = ANY($1::text[]);`
+}
+
+let makeInsertUnnestSetSql = (~pgSchema, ~table: Table.table, ~itemSchema, ~isRawEvents) => {
+  let {quotedFieldNames, quotedNonPrimaryFieldNames, arrayFieldTypes} =
+    table->Table.toSqlParams(~schema=itemSchema)
+
+  let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table)
+
+  `INSERT INTO "${pgSchema}"."${table.tableName}" (${quotedFieldNames->Js.Array2.joinWith(", ")})
+SELECT * FROM unnest(${arrayFieldTypes
+    ->Js.Array2.mapi((arrayFieldType, idx) => {
+      `$${(idx + 1)->Js.Int.toString}::${arrayFieldType}`
+    })
+    ->Js.Array2.joinWith(",")})` ++
+  switch (isRawEvents, primaryKeyFieldNames) {
+  | (true, _)
+  | (_, []) => ``
+  | (false, primaryKeyFieldNames) =>
+    `ON CONFLICT(${primaryKeyFieldNames
+      ->Js.Array2.map(fieldName => `"${fieldName}"`)
+      ->Js.Array2.joinWith(",")}) DO ` ++ (
+      quotedNonPrimaryFieldNames->Utils.Array.isEmpty
+        ? `NOTHING`
+        : `UPDATE SET ${quotedNonPrimaryFieldNames
+            ->Js.Array2.map(fieldName => {
+              `${fieldName} = EXCLUDED.${fieldName}`
+            })
+            ->Js.Array2.joinWith(",")}`
+    )
+  } ++ ";"
+}
+
+let makeInsertValuesSetSql = (~pgSchema, ~table: Table.table, ~itemSchema, ~itemsCount) => {
+  let {quotedFieldNames, quotedNonPrimaryFieldNames} = table->Table.toSqlParams(~schema=itemSchema)
+
+  let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table)
+  let fieldsCount = quotedFieldNames->Array.length
+
+  // Create placeholder variables for the VALUES clause - using $1, $2, etc.
+  let placeholders = ref("")
+  for idx in 1 to itemsCount {
+    if idx > 1 {
+      placeholders := placeholders.contents ++ ","
+    }
+    placeholders := placeholders.contents ++ "("
+    for fieldIdx in 0 to fieldsCount - 1 {
+      if fieldIdx > 0 {
+        placeholders := placeholders.contents ++ ","
+      }
+      placeholders := placeholders.contents ++ `$${(fieldIdx * itemsCount + idx)->Js.Int.toString}`
+    }
+    placeholders := placeholders.contents ++ ")"
+  }
+
+  `INSERT INTO "${pgSchema}"."${table.tableName}" (${quotedFieldNames->Js.Array2.joinWith(", ")})
+VALUES${placeholders.contents}` ++
+  switch primaryKeyFieldNames {
+  | [] => ``
+  | primaryKeyFieldNames =>
+    `ON CONFLICT(${primaryKeyFieldNames
+      ->Js.Array2.map(fieldName => `"${fieldName}"`)
+      ->Js.Array2.joinWith(",")}) DO ` ++ (
+      quotedNonPrimaryFieldNames->Utils.Array.isEmpty
+        ? `NOTHING`
+        : `UPDATE SET ${quotedNonPrimaryFieldNames
+            ->Js.Array2.map(fieldName => {
+              `${fieldName} = EXCLUDED.${fieldName}`
+            })
+            ->Js.Array2.joinWith(",")}`
+    )
+  } ++ ";"
+}
+
+// Should move this to a better place
+// We need it for the isRawEvents check in makeTableBatchSet
+// to always apply the unnest optimization.
+// This is needed, because even though it has JSON fields,
+// they are always guaranteed to be an object.
+// FIXME what about Fuel params?
+let rawEventsTableName = "raw_events"
+
+// Constants for chunking
+let maxItemsPerQuery = 500
+
+let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'item>) => {
+  let {dbSchema, hasArrayField} = table->Table.toSqlParams(~schema=itemSchema)
+  let isRawEvents = table.tableName === rawEventsTableName
+
+  // Should experiment how much it'll affect performance
+  // Although, it should be fine not to perform the validation check,
+  // since the values are validated by type system.
+  // As an alternative, we can only run Sury validation only when
+  // db write fails to show a better user error.
+  let typeValidation = false
+
+  if isRawEvents || !hasArrayField {
+    {
+      "sql": makeInsertUnnestSetSql(~pgSchema, ~table, ~itemSchema, ~isRawEvents),
+      "convertOrThrow": S.compile(
+        S.unnest(dbSchema),
+        ~input=Value,
+        ~output=Unknown,
+        ~mode=Sync,
+        ~typeValidation,
+      ),
+      "isInsertValues": false,
+    }
+  } else {
+    {
+      "sql": makeInsertValuesSetSql(~pgSchema, ~table, ~itemSchema, ~itemsCount=maxItemsPerQuery),
+      "convertOrThrow": S.compile(
+        S.unnest(itemSchema)->S.preprocess(_ => {
+          serializer: Utils.Array.flatten->Utils.magic,
+        }),
+        ~input=Value,
+        ~output=Unknown,
+        ~mode=Sync,
+        ~typeValidation,
+      ),
+      "isInsertValues": true,
+    }
+  }
+}
+
+let chunkArray = (arr: array<'a>, ~chunkSize) => {
+  let chunks = []
+  let i = ref(0)
+  while i.contents < arr->Array.length {
+    let chunk = arr->Js.Array2.slice(~start=i.contents, ~end_=i.contents + chunkSize)
+    chunks->Js.Array2.push(chunk)->ignore
+    i := i.contents + chunkSize
+  }
+  chunks
+}
+
+// WeakMap for caching table batch set queries
+let setQueryCache = Utils.WeakMap.make()
+let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema) => {
+  if items->Array.length === 0 {
+    ()
+  } else {
+    // Get or create cached query for this table
+    let query = switch setQueryCache->Utils.WeakMap.get(table) {
+    | Some(cached) => cached
+    | None => {
+        let newQuery = makeTableBatchSetQuery(
+          ~pgSchema,
+          ~table,
+          ~itemSchema=itemSchema->S.toUnknown,
+        )
+        setQueryCache->Utils.WeakMap.set(table, newQuery)->ignore
+        newQuery
+      }
+    }
+
+    let sqlQuery = query["sql"]
+
+    try {
+      if query["isInsertValues"] {
+        let chunks = chunkArray(items, ~chunkSize=maxItemsPerQuery)
+        let responses = []
+        chunks->Js.Array2.forEach(chunk => {
+          let chunkSize = chunk->Array.length
+          let isFullChunk = chunkSize === maxItemsPerQuery
+
+          let response = sql->Postgres.preparedUnsafe(
+            // Either use the sql query for full chunks from cache
+            // or create a new one for partial chunks on the fly.
+            isFullChunk
+              ? sqlQuery
+              : makeInsertValuesSetSql(~pgSchema, ~table, ~itemSchema, ~itemsCount=chunkSize),
+            query["convertOrThrow"](chunk->(Utils.magic: array<'item> => array<unknown>)),
+          )
+          responses->Js.Array2.push(response)->ignore
+        })
+        let _ = await Promise.all(responses)
+      } else {
+        // Use UNNEST approach for single query
+        await sql->Postgres.preparedUnsafe(
+          sqlQuery,
+          query["convertOrThrow"](items->(Utils.magic: array<'item> => array<unknown>)),
+        )
+      }
+    } catch {
+    | S.Raised(_) as exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed to convert items for table "${table.tableName}"`,
+          reason: exn,
+        }),
+      )
+    | exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed to insert items into table "${table.tableName}"`,
+          reason: exn,
+        }),
+      )
+    }
+  }
+}
+
 let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
   let isInitialized = async () => {
     let schemas =
@@ -157,11 +367,11 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
     schemas->Utils.Array.notEmpty
   }
 
-  let initialize = async (~entities, ~staticTables, ~enums, ~cleanRun) => {
+  let initialize = async (~entities=[], ~generalTables=[], ~enums=[], ~cleanRun=false) => {
     let queries = makeInitializeTransaction(
       ~pgSchema,
       ~pgUser,
-      ~staticTables,
+      ~generalTables,
       ~entities,
       ~enums,
       ~cleanRun,
@@ -172,8 +382,60 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
     })
   }
 
+  let loadByIdsOrThrow = async (~ids, ~table: Table.table, ~rowsSchema) => {
+    switch await (
+      switch ids {
+      | [_] =>
+        sql->Postgres.preparedUnsafe(
+          makeLoadByIdSql(~pgSchema, ~tableName=table.tableName),
+          ids->Obj.magic,
+        )
+      | _ =>
+        sql->Postgres.preparedUnsafe(
+          makeLoadByIdsSql(~pgSchema, ~tableName=table.tableName),
+          [ids]->Obj.magic,
+        )
+      }
+    ) {
+    | exception exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed loading "${table.tableName}" from storage by ids`,
+          reason: exn,
+        }),
+      )
+    | rows =>
+      try rows->S.parseOrThrow(rowsSchema) catch {
+      | exn =>
+        raise(
+          Persistence.StorageError({
+            message: `Failed to parse "${table.tableName}" loaded from storage by ids`,
+            reason: exn,
+          }),
+        )
+      }
+    }
+  }
+
+  let setOrThrow = (
+    type item,
+    ~items: array<item>,
+    ~table: Table.table,
+    ~itemSchema: S.t<item>,
+  ) => {
+    setOrThrow(
+      sql,
+      ~items=items->(Utils.magic: array<item> => array<unknown>),
+      ~table,
+      ~itemSchema=itemSchema->S.toUnknown,
+      ~pgSchema,
+    )
+  }
+
   {
     isInitialized,
     initialize,
+    loadByIdsOrThrow,
+    setOrThrow,
   }
 }

package/src/PgStorage.res.js

@@ -6,8 +6,12 @@ var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
 var Schema = require("./db/Schema.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Caml_option = require("rescript/lib/js/caml_option.js");
+var Persistence = require("./Persistence.res.js");
+var S$RescriptSchema = require("rescript-schema/src/S.res.js");
+var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
-function makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema) {
+function makeCreateIndexSql(tableName, indexFields, pgSchema) {
   var indexName = tableName + "_" + indexFields.join("_");
   var index = Belt_Array.map(indexFields, (function (idx) {
             return "\"" + idx + "\"";
@@ -15,20 +19,20 @@ function makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema) {
   return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
 }
 
-function makeCreateTableIndicesSqlUnsafe(table, pgSchema) {
+function makeCreateTableIndicesSql(table, pgSchema) {
   var tableName = table.tableName;
   var createIndex = function (indexField) {
-    return makeCreateIndexSqlUnsafe(tableName, [indexField], pgSchema);
+    return makeCreateIndexSql(tableName, [indexField], pgSchema);
   };
   var createCompositeIndex = function (indexFields) {
-    return makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema);
+    return makeCreateIndexSql(tableName, indexFields, pgSchema);
   };
   var singleIndices = Table.getSingleIndices(table);
   var compositeIndices = Table.getCompositeIndices(table);
   return Belt_Array.map(singleIndices, createIndex).join("\n") + Belt_Array.map(compositeIndices, createCompositeIndex).join("\n");
 }
 
-function makeCreateTableSqlUnsafe(table, pgSchema) {
+function makeCreateTableSql(table, pgSchema) {
   var fieldsMapped = Belt_Array.map(Table.getFields(table), (function (field) {
             var defaultValue = field.defaultValue;
             var fieldType = field.fieldType;
@@ -52,8 +56,12 @@ function makeCreateTableSqlUnsafe(table, pgSchema) {
         ) + ");";
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enums, cleanRun) {
-  var allTables = $$Array.copy(staticTables);
+function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesOpt, enumsOpt, cleanRunOpt) {
+  var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
+  var entities = entitiesOpt !== undefined ? entitiesOpt : [];
+  var enums = enumsOpt !== undefined ? enumsOpt : [];
+  var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
+  var allTables = $$Array.copy(generalTables);
   var allEntityTables = [];
   entities.forEach(function (entity) {
         allEntityTables.push(entity.table);
@@ -75,10 +83,10 @@ function makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enu
         );
       });
   allTables.forEach(function (table) {
-        query.contents = query.contents + "\n" + makeCreateTableSqlUnsafe(table, pgSchema);
+        query.contents = query.contents + "\n" + makeCreateTableSql(table, pgSchema);
       });
   allTables.forEach(function (table) {
-        var indices = makeCreateTableIndicesSqlUnsafe(table, pgSchema);
+        var indices = makeCreateTableIndicesSql(table, pgSchema);
         if (indices !== "") {
           query.contents = query.contents + "\n" + indices;
           return ;
@@ -92,10 +100,153 @@ function makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enu
         functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.createInsertFnQuery;
         Table.getDerivedFromFields(entity.table).forEach(function (derivedFromField) {
               var indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
-              query.contents = query.contents + "\n" + makeCreateIndexSqlUnsafe(derivedFromField.derivedFromEntity, [indexField], pgSchema);
+              query.contents = query.contents + "\n" + makeCreateIndexSql(derivedFromField.derivedFromEntity, [indexField], pgSchema);
             });
       });
-  return [cleanRun ? query.contents : "DO $$ BEGIN " + query.contents + " END $$;"].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+  return [cleanRun || Utils.$$Array.isEmpty(enums) ? query.contents : "DO $$ BEGIN " + query.contents + " END $$;"].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+}
+
+function makeLoadByIdSql(pgSchema, tableName) {
+  return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = $1 LIMIT 1;";
+}
+
+function makeLoadByIdsSql(pgSchema, tableName) {
+  return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = ANY($1::text[]);";
+}
+
+function makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents) {
+  var match = Table.toSqlParams(table, itemSchema);
+  var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
+  var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
+  return "INSERT INTO \"" + pgSchema + "\".\"" + table.tableName + "\" (" + match.quotedFieldNames.join(", ") + ")\nSELECT * FROM unnest(" + match.arrayFieldTypes.map(function (arrayFieldType, idx) {
+                return "$" + (idx + 1 | 0).toString() + "::" + arrayFieldType;
+              }).join(",") + ")" + (
+          isRawEvents || primaryKeyFieldNames.length === 0 ? "" : "ON CONFLICT(" + primaryKeyFieldNames.map(function (fieldName) {
+                    return "\"" + fieldName + "\"";
+                  }).join(",") + ") DO " + (
+              Utils.$$Array.isEmpty(quotedNonPrimaryFieldNames) ? "NOTHING" : "UPDATE SET " + quotedNonPrimaryFieldNames.map(function (fieldName) {
+                        return fieldName + " = EXCLUDED." + fieldName;
+                      }).join(",")
+            )
+        ) + ";";
+}
+
+function makeInsertValuesSetSql(pgSchema, table, itemSchema, itemsCount) {
+  var match = Table.toSqlParams(table, itemSchema);
+  var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
+  var quotedFieldNames = match.quotedFieldNames;
+  var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
+  var fieldsCount = quotedFieldNames.length;
+  var placeholders = "";
+  for(var idx = 1; idx <= itemsCount; ++idx){
+    if (idx > 1) {
+      placeholders = placeholders + ",";
+    }
+    placeholders = placeholders + "(";
+    for(var fieldIdx = 0; fieldIdx < fieldsCount; ++fieldIdx){
+      if (fieldIdx > 0) {
+        placeholders = placeholders + ",";
+      }
+      placeholders = placeholders + ("$" + (Math.imul(fieldIdx, itemsCount) + idx | 0).toString());
+    }
+    placeholders = placeholders + ")";
+  }
+  return "INSERT INTO \"" + pgSchema + "\".\"" + table.tableName + "\" (" + quotedFieldNames.join(", ") + ")\nVALUES" + placeholders + (
+          primaryKeyFieldNames.length !== 0 ? "ON CONFLICT(" + primaryKeyFieldNames.map(function (fieldName) {
+                    return "\"" + fieldName + "\"";
+                  }).join(",") + ") DO " + (
+              Utils.$$Array.isEmpty(quotedNonPrimaryFieldNames) ? "NOTHING" : "UPDATE SET " + quotedNonPrimaryFieldNames.map(function (fieldName) {
+                        return fieldName + " = EXCLUDED." + fieldName;
+                      }).join(",")
+            ) : ""
+        ) + ";";
+}
+
+var rawEventsTableName = "raw_events";
+
+function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
+  var match = Table.toSqlParams(table, itemSchema);
+  var isRawEvents = table.tableName === rawEventsTableName;
+  if (isRawEvents || !match.hasArrayField) {
+    return {
+            sql: makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents),
+            convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
+            isInsertValues: false
+          };
+  } else {
+    return {
+            sql: makeInsertValuesSetSql(pgSchema, table, itemSchema, 500),
+            convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), (function (param) {
+                        return {
+                                s: (function (prim) {
+                                    return prim.flat(1);
+                                  })
+                              };
+                      })), "Output", "Input", "Sync", false),
+            isInsertValues: true
+          };
+  }
+}
+
+function chunkArray(arr, chunkSize) {
+  var chunks = [];
+  var i = 0;
+  while(i < arr.length) {
+    var chunk = arr.slice(i, i + chunkSize | 0);
+    chunks.push(chunk);
+    i = i + chunkSize | 0;
+  };
+  return chunks;
+}
+
+var setQueryCache = new WeakMap();
+
+async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
+  if (items.length === 0) {
+    return ;
+  }
+  var cached = setQueryCache.get(table);
+  var query;
+  if (cached !== undefined) {
+    query = Caml_option.valFromOption(cached);
+  } else {
+    var newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema);
+    setQueryCache.set(table, newQuery);
+    query = newQuery;
+  }
+  var sqlQuery = query.sql;
+  try {
+    if (!query.isInsertValues) {
+      return await sql.unsafe(sqlQuery, query.convertOrThrow(items), {prepare: true});
+    }
+    var chunks = chunkArray(items, 500);
+    var responses = [];
+    chunks.forEach(function (chunk) {
+          var chunkSize = chunk.length;
+          var isFullChunk = chunkSize === 500;
+          var response = sql.unsafe(isFullChunk ? sqlQuery : makeInsertValuesSetSql(pgSchema, table, itemSchema, chunkSize), query.convertOrThrow(chunk), {prepare: true});
+          responses.push(response);
+        });
+    await Promise.all(responses);
+    return ;
+  }
+  catch (raw_exn){
+    var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+    if (exn.RE_EXN_ID === S$RescriptSchema.Raised) {
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed to convert items for table \"" + table.tableName + "\"",
+            reason: exn,
+            Error: new Error()
+          };
+    }
+    throw {
+          RE_EXN_ID: Persistence.StorageError,
+          message: "Failed to insert items into table \"" + table.tableName + "\"",
+          reason: exn,
+          Error: new Error()
+        };
+  }
 }
 
 function make(sql, pgSchema, pgUser) {
@@ -103,23 +254,73 @@ function make(sql, pgSchema, pgUser) {
     var schemas = await sql.unsafe("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '" + pgSchema + "';");
     return Utils.$$Array.notEmpty(schemas);
   };
-  var initialize = async function (entities, staticTables, enums, cleanRun) {
-    var queries = makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enums, cleanRun);
+  var initialize = async function (entitiesOpt, generalTablesOpt, enumsOpt, cleanRunOpt) {
+    var entities = entitiesOpt !== undefined ? entitiesOpt : [];
+    var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
+    var enums = enumsOpt !== undefined ? enumsOpt : [];
+    var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
+    var queries = makeInitializeTransaction(pgSchema, pgUser, generalTables, entities, enums, cleanRun);
     await sql.begin(function (sql) {
           return queries.map(function (query) {
                       return sql.unsafe(query);
                     });
         });
   };
+  var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
+    var rows;
+    try {
+      rows = await (
+        ids.length !== 1 ? sql.unsafe(makeLoadByIdsSql(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeLoadByIdSql(pgSchema, table.tableName), ids, {prepare: true})
+      );
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed loading \"" + table.tableName + "\" from storage by ids",
+            reason: exn,
+            Error: new Error()
+          };
+    }
+    try {
+      return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
+    }
+    catch (raw_exn$1){
+      var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed to parse \"" + table.tableName + "\" loaded from storage by ids",
+            reason: exn$1,
+            Error: new Error()
+          };
+    }
+  };
+  var setOrThrow$1 = function (items, table, itemSchema) {
+    return setOrThrow(sql, items, table, itemSchema, pgSchema);
+  };
   return {
           isInitialized: isInitialized,
-          initialize: initialize
+          initialize: initialize,
+          loadByIdsOrThrow: loadByIdsOrThrow,
+          setOrThrow: setOrThrow$1
         };
 }
 
-exports.makeCreateIndexSqlUnsafe = makeCreateIndexSqlUnsafe;
-exports.makeCreateTableIndicesSqlUnsafe = makeCreateTableIndicesSqlUnsafe;
-exports.makeCreateTableSqlUnsafe = makeCreateTableSqlUnsafe;
+var maxItemsPerQuery = 500;
+
+exports.makeCreateIndexSql = makeCreateIndexSql;
+exports.makeCreateTableIndicesSql = makeCreateTableIndicesSql;
+exports.makeCreateTableSql = makeCreateTableSql;
 exports.makeInitializeTransaction = makeInitializeTransaction;
+exports.makeLoadByIdSql = makeLoadByIdSql;
+exports.makeLoadByIdsSql = makeLoadByIdsSql;
+exports.makeInsertUnnestSetSql = makeInsertUnnestSetSql;
+exports.makeInsertValuesSetSql = makeInsertValuesSetSql;
+exports.rawEventsTableName = rawEventsTableName;
+exports.maxItemsPerQuery = maxItemsPerQuery;
+exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
+exports.chunkArray = chunkArray;
+exports.setQueryCache = setQueryCache;
+exports.setOrThrow = setOrThrow;
 exports.make = make;
-/* Table Not a pure module */
+/* setQueryCache Not a pure module */

package/src/db/EntityHistory.res

@@ -173,8 +173,9 @@ let batchInsertRows = (self: t<'entity>, ~sql, ~rows: array<historyRow<'entity>>
 type entityInternal
 
 external castInternal: t<'entity> => t<entityInternal> = "%identity"
+external eval: string => 'a = "eval"
 
-let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
+let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> => {
   let entity_history_block_timestamp = "entity_history_block_timestamp"
   let entity_history_chain_id = "entity_history_chain_id"
   let entity_history_block_number = "entity_history_block_number"
@@ -235,12 +236,10 @@ let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
   let dataFieldNames = dataFields->Belt.Array.map(field => field->getFieldName)
 
   let originTableName = table.tableName
-  let originSchemaName = table.schemaName
   let historyTableName = originTableName ++ "_history"
   //ignore composite indices
   let table = mkTable(
     historyTableName,
-    ~schemaName=originSchemaName,
     ~fields=Belt.Array.concatMany([
       currentHistoryFields,
       previousHistoryFields,
@@ -251,8 +250,8 @@ let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
 
   let insertFnName = `"insert_${table.tableName}"`
   let historyRowArg = "history_row"
-  let historyTablePath = `"${originSchemaName}"."${historyTableName}"`
-  let originTablePath = `"${originSchemaName}"."${originTableName}"`
+  let historyTablePath = `"${pgSchema}"."${historyTableName}"`
+  let originTablePath = `"${pgSchema}"."${originTableName}"`
 
   let previousHistoryFieldsAreNullStr =
     previousChangeFieldNames
@@ -335,7 +334,7 @@ let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
     \${shouldCopyCurrentEntity});\``
 
   let insertFn: (Postgres.sql, Js.Json.t, ~shouldCopyCurrentEntity: bool) => promise<unit> =
-    insertFnString->Table.PostgresInterop.eval
+    insertFnString->eval
 
   let schema = makeHistoryRowSchema(schema)
 

package/src/db/EntityHistory.res.js

@@ -173,7 +173,7 @@ function batchInsertRows(self, sql, rows) {
             });
 }
 
-function fromTable(table, schema) {
+function fromTable(table, pgSchema, schema) {
   var currentChangeFieldNames = [
     "entity_history_block_timestamp",
     "entity_history_chain_id",
@@ -238,9 +238,8 @@ function fromTable(table, schema) {
           return Table.getFieldName(field);
         }));
   var originTableName = table.tableName;
-  var originSchemaName = table.schemaName;
   var historyTableName = originTableName + "_history";
-  var table$1 = Table.mkTable(historyTableName, originSchemaName, undefined, Belt_Array.concatMany([
+  var table$1 = Table.mkTable(historyTableName, undefined, Belt_Array.concatMany([
             currentHistoryFields,
             previousHistoryFields,
             dataFields,
@@ -251,8 +250,8 @@ function fromTable(table, schema) {
           ]));
   var insertFnName = "\"insert_" + table$1.tableName + "\"";
   var historyRowArg = "history_row";
-  var historyTablePath = "\"" + originSchemaName + "\".\"" + historyTableName + "\"";
-  var originTablePath = "\"" + originSchemaName + "\".\"" + originTableName + "\"";
+  var historyTablePath = "\"" + pgSchema + "\".\"" + historyTableName + "\"";
+  var originTablePath = "\"" + pgSchema + "\".\"" + originTableName + "\"";
   var previousHistoryFieldsAreNullStr = Belt_Array.map(previousChangeFieldNames, (function (fieldName) {
             return historyRowArg + "." + fieldName + " IS NULL";
           })).join(" OR ");

package/src/db/Table.res

@@ -89,14 +89,12 @@ let getFieldType = (field: field) => {
 
 type table = {
   tableName: string,
-  schemaName: string,
   fields: array<fieldOrDerived>,
   compositeIndices: array<array<string>>,
 }
 
-let mkTable = (tableName, ~schemaName, ~compositeIndices=[], ~fields) => {
+let mkTable = (tableName, ~compositeIndices=[], ~fields) => {
   tableName,
-  schemaName,
   fields,
   compositeIndices,
 }
@@ -299,59 +297,3 @@ let getCompositeIndices = (table): array<array<string>> => {
   ->getUnfilteredCompositeIndicesUnsafe
   ->Array.keep(ind => ind->Array.length > 1)
 }
-
-module PostgresInterop = {
-  type pgFn<'payload, 'return> = (Postgres.sql, 'payload) => promise<'return>
-  type batchSetFn<'a> = (Postgres.sql, array<'a>) => promise<unit>
-  external eval: string => 'a = "eval"
-
-  let makeBatchSetFnString = (table: table) => {
-    let fieldNamesInQuotes =
-      table->getNonDefaultFieldNames->Array.map(fieldName => `"${fieldName}"`)
-    `(sql, rows) => {
-      return sql\`
-        INSERT INTO "${table.schemaName}"."${table.tableName}"
-        \${sql(rows, ${fieldNamesInQuotes->Js.Array2.joinWith(", ")})}
-        ON CONFLICT(${table->getPrimaryKeyFieldNames->Js.Array2.joinWith(", ")}) DO UPDATE
-        SET
-        ${fieldNamesInQuotes
-      ->Array.map(fieldNameInQuotes => `${fieldNameInQuotes} = EXCLUDED.${fieldNameInQuotes}`)
-      ->Js.Array2.joinWith(", ")};\`
-    }`
-  }
-
-  let chunkBatchQuery = (
-    sql,
-    entityDataArray: array<'entity>,
-    queryToExecute: pgFn<array<'entity>, 'return>,
-    ~maxItemsPerQuery=500,
-  ): promise<array<'return>> => {
-    let responses = []
-    let i = ref(0)
-    let shouldContinue = () => i.contents < entityDataArray->Array.length
-    // Split entityDataArray into chunks of maxItemsPerQuery
-    while shouldContinue() {
-      let chunk =
-        entityDataArray->Js.Array2.slice(~start=i.contents, ~end_=i.contents + maxItemsPerQuery)
-      let response = queryToExecute(sql, chunk)
-      responses->Js.Array2.push(response)->ignore
-      i := i.contents + maxItemsPerQuery
-    }
-    Promise.all(responses)
-  }
-
-  let makeBatchSetFn = (~table, ~schema: S.t<'a>): batchSetFn<'a> => {
-    let batchSetFn: pgFn<array<Js.Json.t>, unit> = table->makeBatchSetFnString->eval
-    let parseOrThrow = S.compile(
-      S.array(schema),
-      ~input=Value,
-      ~output=Json,
-      ~mode=Sync,
-      ~typeValidation=true,
-    )
-    async (sql, rows) => {
-      let rowsJson = rows->parseOrThrow->(Utils.magic: Js.Json.t => array<Js.Json.t>)
-      let _res = await chunkBatchQuery(sql, rowsJson, batchSetFn)
-    }
-  }
-}

package/src/db/Table.res.js

@@ -72,11 +72,10 @@ function getFieldType(field) {
         );
 }
 
-function mkTable(tableName, schemaName, compositeIndicesOpt, fields) {
+function mkTable(tableName, compositeIndicesOpt, fields) {
   var compositeIndices = compositeIndicesOpt !== undefined ? compositeIndicesOpt : [];
   return {
           tableName: tableName,
-          schemaName: schemaName,
           fields: fields,
           compositeIndices: compositeIndices
         };
@@ -302,43 +301,6 @@ function getCompositeIndices(table) {
               }));
 }
 
-function makeBatchSetFnString(table) {
-  var fieldNamesInQuotes = Belt_Array.map(getNonDefaultFieldNames(table), (function (fieldName) {
-          return "\"" + fieldName + "\"";
-        }));
-  return "(sql, rows) => {\n      return sql\`\n        INSERT INTO \"" + table.schemaName + "\".\"" + table.tableName + "\"\n        \${sql(rows, " + fieldNamesInQuotes.join(", ") + ")}\n        ON CONFLICT(" + getPrimaryKeyFieldNames(table).join(", ") + ") DO UPDATE\n        SET\n        " + Belt_Array.map(fieldNamesInQuotes, (function (fieldNameInQuotes) {
-                  return fieldNameInQuotes + " = EXCLUDED." + fieldNameInQuotes;
-                })).join(", ") + ";\`\n    }";
-}
-
-function chunkBatchQuery(sql, entityDataArray, queryToExecute, maxItemsPerQueryOpt) {
-  var maxItemsPerQuery = maxItemsPerQueryOpt !== undefined ? maxItemsPerQueryOpt : 500;
-  var responses = [];
-  var i = 0;
-  while(i < entityDataArray.length) {
-    var chunk = entityDataArray.slice(i, i + maxItemsPerQuery | 0);
-    var response = queryToExecute(sql, chunk);
-    responses.push(response);
-    i = i + maxItemsPerQuery | 0;
-  };
-  return Promise.all(responses);
-}
-
-function makeBatchSetFn(table, schema) {
-  var batchSetFn = eval(makeBatchSetFnString(table));
-  var parseOrThrow = S$RescriptSchema.compile(S$RescriptSchema.array(schema), "Output", "Json", "Sync", true);
-  return async function (sql, rows) {
-    var rowsJson = parseOrThrow(rows);
-    await chunkBatchQuery(sql, rowsJson, batchSetFn, undefined);
-  };
-}
-
-var PostgresInterop = {
-  makeBatchSetFnString: makeBatchSetFnString,
-  chunkBatchQuery: chunkBatchQuery,
-  makeBatchSetFn: makeBatchSetFn
-};
-
 exports.mkField = mkField;
 exports.mkDerivedFromField = mkDerivedFromField;
 exports.getUserDefinedFieldName = getUserDefinedFieldName;
@@ -361,5 +323,4 @@ exports.getUnfilteredCompositeIndicesUnsafe = getUnfilteredCompositeIndicesUnsaf
 exports.toSqlParams = toSqlParams;
 exports.getSingleIndices = getSingleIndices;
 exports.getCompositeIndices = getCompositeIndices;
-exports.PostgresInterop = PostgresInterop;
 /* BigInt Not a pure module */