envio 2.22.2 → 2.23.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.22.2",
+  "version": "v2.23.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.22.2",
-    "envio-linux-arm64": "v2.22.2",
-    "envio-darwin-x64": "v2.22.2",
-    "envio-darwin-arm64": "v2.22.2"
+    "envio-linux-x64": "v2.23.0",
+    "envio-linux-arm64": "v2.23.0",
+    "envio-darwin-x64": "v2.23.0",
+    "envio-darwin-arm64": "v2.23.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Logging.res

@@ -168,6 +168,14 @@ let logForItem = (eventItem, level: Pino.logLevel, message: string, ~params=?) =
   (eventItem->getEventLogger->Utils.magic->Js.Dict.unsafeGet((level :> string)))(params, message)
 }
 
+let noopLogger: Envio.logger = {
+  info: (_message: string, ~params as _=?) => (),
+  debug: (_message: string, ~params as _=?) => (),
+  warn: (_message: string, ~params as _=?) => (),
+  error: (_message: string, ~params as _=?) => (),
+  errorWithExn: (_message: string, _exn) => (),
+}
+
 let getUserLogger = (eventItem): Envio.logger => {
   info: (message: string, ~params=?) => eventItem->logForItem(#uinfo, message, ~params?),
   debug: (message: string, ~params=?) => eventItem->logForItem(#udebug, message, ~params?),

package/src/Logging.res.js

@@ -197,6 +197,34 @@ function logForItem(eventItem, level, message, params) {
   return getEventLogger(eventItem)[level](params, message);
 }
 
+function noopLogger_debug(_message, param) {
+  
+}
+
+function noopLogger_info(_message, param) {
+  
+}
+
+function noopLogger_warn(_message, param) {
+  
+}
+
+function noopLogger_error(_message, param) {
+  
+}
+
+function noopLogger_errorWithExn(_message, _exn) {
+  
+}
+
+var noopLogger = {
+  debug: noopLogger_debug,
+  info: noopLogger_info,
+  warn: noopLogger_warn,
+  error: noopLogger_error,
+  errorWithExn: noopLogger_errorWithExn
+};
+
 function getUserLogger(eventItem) {
   return {
           debug: (function (message, params) {
@@ -243,5 +271,6 @@ exports.createChild = createChild;
 exports.createChildFrom = createChildFrom;
 exports.getEventLogger = getEventLogger;
 exports.logForItem = logForItem;
+exports.noopLogger = noopLogger;
 exports.getUserLogger = getUserLogger;
 /* logLevels Not a pure module */

package/src/Persistence.res

@@ -12,14 +12,28 @@ type storage = {
   // Should initialize the storage so we can start interacting with it
   // Eg create connection, schema, tables, etc.
   initialize: (
-    ~entities: array<Internal.entityConfig>,
-    ~staticTables: array<Table.table>,
-    ~enums: array<Internal.enumConfig<Internal.enum>>,
+    ~entities: array<Internal.entityConfig>=?,
+    ~generalTables: array<Table.table>=?,
+    ~enums: array<Internal.enumConfig<Internal.enum>>=?,
     // If true, the storage should clear existing data
-    ~cleanRun: bool,
+    ~cleanRun: bool=?,
+  ) => promise<unit>,
+  @raises("StorageError")
+  loadByIdsOrThrow: 'item. (
+    ~ids: array<string>,
+    ~table: Table.table,
+    ~rowsSchema: S.t<array<'item>>,
+  ) => promise<array<'item>>,
+  @raises("StorageError")
+  setOrThrow: 'item. (
+    ~items: array<'item>,
+    ~table: Table.table,
+    ~itemSchema: S.t<'item>,
   ) => promise<unit>,
 }
 
+exception StorageError({message: string, reason: exn})
+
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
@@ -33,6 +47,7 @@ type t = {
   mutable storageStatus: storageStatus,
   storage: storage,
   onStorageInitialize: option<unit => promise<unit>>,
+  cacheStorage: storage,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -49,6 +64,7 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
+  ~cacheStorage,
   ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
@@ -62,6 +78,7 @@ let make = (
     storageStatus: Unknown,
     storage,
     onStorageInitialize,
+    cacheStorage,
   }
 }
 
@@ -93,7 +110,7 @@ let init = async (
       } else {
         let _ = await persistence.storage.initialize(
           ~entities=persistence.allEntities,
-          ~staticTables=persistence.staticTables,
+          ~generalTables=persistence.staticTables,
           ~enums=persistence.allEnums,
           ~cleanRun=reset || !skipIsInitializedCheck,
         )
@@ -109,3 +126,12 @@ let init = async (
   | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
+
+let getInitializedStorageOrThrow = persistence => {
+  switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
+  | Ready(_) => persistence.storage
+  }
+}

package/src/Persistence.res.js

@@ -1,10 +1,14 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Js_exn = require("rescript/lib/js/js_exn.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
+var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
+var StorageError = /* @__PURE__ */Caml_exceptions.create("Persistence.StorageError");
+
 var entityHistoryActionEnumConfig_name = EntityHistory.RowAction.name;
 
 var entityHistoryActionEnumConfig_variants = EntityHistory.RowAction.variants;
@@ -18,7 +22,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, onStorageInitialize) {
+function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, cacheStorage, onStorageInitialize) {
   var allEntities = userEntities.concat([dcRegistryEntityConfig]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -28,7 +32,8 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
           allEnums: allEnums$1,
           storageStatus: "Unknown",
           storage: storage,
-          onStorageInitialize: onStorageInitialize
+          onStorageInitialize: onStorageInitialize,
+          cacheStorage: cacheStorage
         };
 }
 
@@ -84,7 +89,18 @@ async function init(persistence, skipIsInitializedCheckOpt, resetOpt) {
   }
 }
 
+function getInitializedStorageOrThrow(persistence) {
+  var match = persistence.storageStatus;
+  if (typeof match !== "object" || match.TAG === "Initializing") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  } else {
+    return persistence.storage;
+  }
+}
+
+exports.StorageError = StorageError;
 exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
+exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
 /* EntityHistory Not a pure module */

package/src/PgStorage.res

@@ -1,16 +1,16 @@
-let makeCreateIndexSqlUnsafe = (~tableName, ~indexFields, ~pgSchema) => {
+let makeCreateIndexSql = (~tableName, ~indexFields, ~pgSchema) => {
   let indexName = tableName ++ "_" ++ indexFields->Js.Array2.joinWith("_")
   let index = indexFields->Belt.Array.map(idx => `"${idx}"`)->Js.Array2.joinWith(", ")
   `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${pgSchema}"."${tableName}"(${index});`
 }
 
-let makeCreateTableIndicesSqlUnsafe = (table: Table.table, ~pgSchema) => {
+let makeCreateTableIndicesSql = (table: Table.table, ~pgSchema) => {
   open Belt
   let tableName = table.tableName
   let createIndex = indexField =>
-    makeCreateIndexSqlUnsafe(~tableName, ~indexFields=[indexField], ~pgSchema)
+    makeCreateIndexSql(~tableName, ~indexFields=[indexField], ~pgSchema)
   let createCompositeIndex = indexFields => {
-    makeCreateIndexSqlUnsafe(~tableName, ~indexFields, ~pgSchema)
+    makeCreateIndexSql(~tableName, ~indexFields, ~pgSchema)
   }
 
   let singleIndices = table->Table.getSingleIndices
@@ -20,7 +20,7 @@ let makeCreateTableIndicesSqlUnsafe = (table: Table.table, ~pgSchema) => {
     compositeIndices->Array.map(createCompositeIndex)->Js.Array2.joinWith("\n")
 }
 
-let makeCreateTableSqlUnsafe = (table: Table.table, ~pgSchema) => {
+let makeCreateTableSql = (table: Table.table, ~pgSchema) => {
   open Belt
   let fieldsMapped =
     table
@@ -55,12 +55,12 @@ let makeCreateTableSqlUnsafe = (table: Table.table, ~pgSchema) => {
 let makeInitializeTransaction = (
   ~pgSchema,
   ~pgUser,
-  ~staticTables,
-  ~entities,
-  ~enums,
-  ~cleanRun,
+  ~generalTables=[],
+  ~entities=[],
+  ~enums=[],
+  ~cleanRun=false,
 ) => {
-  let allTables = staticTables->Array.copy
+  let allTables = generalTables->Array.copy
   let allEntityTables = []
   entities->Js.Array2.forEach((entity: Internal.entityConfig) => {
     allEntityTables->Js.Array2.push(entity.table)->ignore
@@ -76,7 +76,7 @@ let makeInitializeTransaction = (
 CREATE SCHEMA "${pgSchema}";`
         : `CREATE SCHEMA IF NOT EXISTS "${pgSchema}";`
     ) ++
-    `GRANT ALL ON SCHEMA "${pgSchema}" TO ${pgUser};
+    `GRANT ALL ON SCHEMA "${pgSchema}" TO "${pgUser}";
 GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
   )
 
@@ -106,12 +106,12 @@ END IF;`
 
   // Batch all table creation first (optimal for PostgreSQL)
   allTables->Js.Array2.forEach((table: Table.table) => {
-    query := query.contents ++ "\n" ++ makeCreateTableSqlUnsafe(table, ~pgSchema)
+    query := query.contents ++ "\n" ++ makeCreateTableSql(table, ~pgSchema)
   })
 
   // Then batch all indices (better performance when tables exist)
   allTables->Js.Array2.forEach((table: Table.table) => {
-    let indices = makeCreateTableIndicesSqlUnsafe(table, ~pgSchema)
+    let indices = makeCreateTableIndicesSql(table, ~pgSchema)
     if indices !== "" {
       query := query.contents ++ "\n" ++ indices
     }
@@ -131,7 +131,7 @@ END IF;`
       query :=
         query.contents ++
         "\n" ++
-        makeCreateIndexSqlUnsafe(
+        makeCreateIndexSql(
           ~tableName=derivedFromField.derivedFromEntity,
           ~indexFields=[indexField],
           ~pgSchema,
@@ -143,11 +143,238 @@ END IF;`
     // Return optimized queries - main DDL in DO block, functions separate
     // Note: DO $$ BEGIN wrapper is only needed for PL/pgSQL conditionals (IF NOT EXISTS)
     // Reset case uses direct DDL (faster), non-cleanRun case uses conditionals (safer)
-    cleanRun ? query.contents : `DO $$ BEGIN ${query.contents} END $$;`,
+    cleanRun || enums->Utils.Array.isEmpty
+      ? query.contents
+      : `DO $$ BEGIN ${query.contents} END $$;`,
     // Functions query (separate as they can't be in DO block)
   ]->Js.Array2.concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : [])
 }
 
+let makeLoadByIdSql = (~pgSchema, ~tableName) => {
+  `SELECT * FROM "${pgSchema}"."${tableName}" WHERE id = $1 LIMIT 1;`
+}
+
+let makeLoadByIdsSql = (~pgSchema, ~tableName) => {
+  `SELECT * FROM "${pgSchema}"."${tableName}" WHERE id = ANY($1::text[]);`
+}
+
+let makeInsertUnnestSetSql = (~pgSchema, ~table: Table.table, ~itemSchema, ~isRawEvents) => {
+  let {quotedFieldNames, quotedNonPrimaryFieldNames, arrayFieldTypes} =
+    table->Table.toSqlParams(~schema=itemSchema)
+
+  let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table)
+
+  `INSERT INTO "${pgSchema}"."${table.tableName}" (${quotedFieldNames->Js.Array2.joinWith(", ")})
+SELECT * FROM unnest(${arrayFieldTypes
+    ->Js.Array2.mapi((arrayFieldType, idx) => {
+      `$${(idx + 1)->Js.Int.toString}::${arrayFieldType}`
+    })
+    ->Js.Array2.joinWith(",")})` ++
+  switch (isRawEvents, primaryKeyFieldNames) {
+  | (true, _)
+  | (_, []) => ``
+  | (false, primaryKeyFieldNames) =>
+    `ON CONFLICT(${primaryKeyFieldNames
+      ->Js.Array2.map(fieldName => `"${fieldName}"`)
+      ->Js.Array2.joinWith(",")}) DO ` ++ (
+      quotedNonPrimaryFieldNames->Utils.Array.isEmpty
+        ? `NOTHING`
+        : `UPDATE SET ${quotedNonPrimaryFieldNames
+            ->Js.Array2.map(fieldName => {
+              `${fieldName} = EXCLUDED.${fieldName}`
+            })
+            ->Js.Array2.joinWith(",")}`
+    )
+  } ++ ";"
+}
+
+let makeInsertValuesSetSql = (~pgSchema, ~table: Table.table, ~itemSchema, ~itemsCount) => {
+  let {quotedFieldNames, quotedNonPrimaryFieldNames} = table->Table.toSqlParams(~schema=itemSchema)
+
+  let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table)
+  let fieldsCount = quotedFieldNames->Array.length
+
+  // Create placeholder variables for the VALUES clause - using $1, $2, etc.
+  let placeholders = ref("")
+  for idx in 1 to itemsCount {
+    if idx > 1 {
+      placeholders := placeholders.contents ++ ","
+    }
+    placeholders := placeholders.contents ++ "("
+    for fieldIdx in 0 to fieldsCount - 1 {
+      if fieldIdx > 0 {
+        placeholders := placeholders.contents ++ ","
+      }
+      placeholders := placeholders.contents ++ `$${(fieldIdx * itemsCount + idx)->Js.Int.toString}`
+    }
+    placeholders := placeholders.contents ++ ")"
+  }
+
+  `INSERT INTO "${pgSchema}"."${table.tableName}" (${quotedFieldNames->Js.Array2.joinWith(", ")})
+VALUES${placeholders.contents}` ++
+  switch primaryKeyFieldNames {
+  | [] => ``
+  | primaryKeyFieldNames =>
+    `ON CONFLICT(${primaryKeyFieldNames
+      ->Js.Array2.map(fieldName => `"${fieldName}"`)
+      ->Js.Array2.joinWith(",")}) DO ` ++ (
+      quotedNonPrimaryFieldNames->Utils.Array.isEmpty
+        ? `NOTHING`
+        : `UPDATE SET ${quotedNonPrimaryFieldNames
+            ->Js.Array2.map(fieldName => {
+              `${fieldName} = EXCLUDED.${fieldName}`
+            })
+            ->Js.Array2.joinWith(",")}`
+    )
+  } ++ ";"
+}
+
+// Should move this to a better place
+// We need it for the isRawEvents check in makeTableBatchSet
+// to always apply the unnest optimization.
+// This is needed, because even though it has JSON fields,
+// they are always guaranteed to be an object.
+// FIXME what about Fuel params?
+let rawEventsTableName = "raw_events"
+
+// Constants for chunking
+let maxItemsPerQuery = 500
+
+let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'item>) => {
+  let {dbSchema, hasArrayField} = table->Table.toSqlParams(~schema=itemSchema)
+  let isRawEvents = table.tableName === rawEventsTableName
+
+  // Should experiment how much it'll affect performance
+  // Although, it should be fine not to perform the validation check,
+  // since the values are validated by type system.
+  // As an alternative, we can only run Sury validation only when
+  // db write fails to show a better user error.
+  let typeValidation = false
+
+  if isRawEvents || !hasArrayField {
+    {
+      "sql": makeInsertUnnestSetSql(~pgSchema, ~table, ~itemSchema, ~isRawEvents),
+      "convertOrThrow": S.compile(
+        S.unnest(dbSchema),
+        ~input=Value,
+        ~output=Unknown,
+        ~mode=Sync,
+        ~typeValidation,
+      ),
+      "isInsertValues": false,
+    }
+  } else {
+    {
+      "sql": makeInsertValuesSetSql(~pgSchema, ~table, ~itemSchema, ~itemsCount=maxItemsPerQuery),
+      "convertOrThrow": S.compile(
+        S.unnest(itemSchema)->S.preprocess(_ => {
+          serializer: Utils.Array.flatten->Utils.magic,
+        }),
+        ~input=Value,
+        ~output=Unknown,
+        ~mode=Sync,
+        ~typeValidation,
+      ),
+      "isInsertValues": true,
+    }
+  }
+}
+
+let chunkArray = (arr: array<'a>, ~chunkSize) => {
+  let chunks = []
+  let i = ref(0)
+  while i.contents < arr->Array.length {
+    let chunk = arr->Js.Array2.slice(~start=i.contents, ~end_=i.contents + chunkSize)
+    chunks->Js.Array2.push(chunk)->ignore
+    i := i.contents + chunkSize
+  }
+  chunks
+}
+
+// WeakMap for caching table batch set queries
+let setQueryCache = Utils.WeakMap.make()
+let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema) => {
+  if items->Array.length === 0 {
+    ()
+  } else {
+    // Get or create cached query for this table
+    let query = switch setQueryCache->Utils.WeakMap.get(table) {
+    | Some(cached) => cached
+    | None => {
+        let newQuery = makeTableBatchSetQuery(
+          ~pgSchema,
+          ~table,
+          ~itemSchema=itemSchema->S.toUnknown,
+        )
+        setQueryCache->Utils.WeakMap.set(table, newQuery)->ignore
+        newQuery
+      }
+    }
+
+    let sqlQuery = query["sql"]
+
+    try {
+      let payload =
+        query["convertOrThrow"](items->(Utils.magic: array<'item> => array<unknown>))->(
+          Utils.magic: unknown => array<unknown>
+        )
+
+      if query["isInsertValues"] {
+        let fieldsCount = switch itemSchema->S.classify {
+        | S.Object({items}) => items->Array.length
+        | _ => Js.Exn.raiseError("Expected an object schema for table")
+        }
+
+        // Chunk the items for VALUES-based queries
+        // We need to multiply by fields count,
+        // because we flattened our entity values with S.unnest
+        // to optimize the query execution.
+        let maxChunkSize = maxItemsPerQuery * fieldsCount
+        let chunks = chunkArray(payload, ~chunkSize=maxChunkSize)
+        let responses = []
+        chunks->Js.Array2.forEach(chunk => {
+          let chunkSize = chunk->Array.length
+          let isFullChunk = chunkSize === maxChunkSize
+
+          let response = sql->Postgres.preparedUnsafe(
+            // Either use the sql query for full chunks from cache
+            // or create a new one for partial chunks on the fly.
+            isFullChunk
+              ? sqlQuery
+              : makeInsertValuesSetSql(
+                  ~pgSchema,
+                  ~table,
+                  ~itemSchema,
+                  ~itemsCount=chunkSize / fieldsCount,
+                ),
+            chunk->Utils.magic,
+          )
+          responses->Js.Array2.push(response)->ignore
+        })
+        let _ = await Promise.all(responses)
+      } else {
+        // Use UNNEST approach for single query
+        await sql->Postgres.preparedUnsafe(sqlQuery, payload->Obj.magic)
+      }
+    } catch {
+    | S.Raised(_) as exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed to convert items for table "${table.tableName}"`,
+          reason: exn,
+        }),
+      )
+    | exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed to insert items into table "${table.tableName}"`,
+          reason: exn,
+        }),
+      )
+    }
+  }
+}
+
 let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
   let isInitialized = async () => {
     let schemas =
@@ -157,11 +384,11 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
     schemas->Utils.Array.notEmpty
   }
 
-  let initialize = async (~entities, ~staticTables, ~enums, ~cleanRun) => {
+  let initialize = async (~entities=[], ~generalTables=[], ~enums=[], ~cleanRun=false) => {
     let queries = makeInitializeTransaction(
       ~pgSchema,
       ~pgUser,
-      ~staticTables,
+      ~generalTables,
       ~entities,
       ~enums,
       ~cleanRun,
@@ -172,8 +399,60 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
     })
   }
 
+  let loadByIdsOrThrow = async (~ids, ~table: Table.table, ~rowsSchema) => {
+    switch await (
+      switch ids {
+      | [_] =>
+        sql->Postgres.preparedUnsafe(
+          makeLoadByIdSql(~pgSchema, ~tableName=table.tableName),
+          ids->Obj.magic,
+        )
+      | _ =>
+        sql->Postgres.preparedUnsafe(
+          makeLoadByIdsSql(~pgSchema, ~tableName=table.tableName),
+          [ids]->Obj.magic,
+        )
+      }
+    ) {
+    | exception exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed loading "${table.tableName}" from storage by ids`,
+          reason: exn,
+        }),
+      )
+    | rows =>
+      try rows->S.parseOrThrow(rowsSchema) catch {
+      | exn =>
+        raise(
+          Persistence.StorageError({
+            message: `Failed to parse "${table.tableName}" loaded from storage by ids`,
+            reason: exn,
+          }),
+        )
+      }
+    }
+  }
+
+  let setOrThrow = (
+    type item,
+    ~items: array<item>,
+    ~table: Table.table,
+    ~itemSchema: S.t<item>,
+  ) => {
+    setOrThrow(
+      sql,
+      ~items=items->(Utils.magic: array<item> => array<unknown>),
+      ~table,
+      ~itemSchema=itemSchema->S.toUnknown,
+      ~pgSchema,
+    )
+  }
+
   {
     isInitialized,
     initialize,
+    loadByIdsOrThrow,
+    setOrThrow,
   }
 }

package/src/PgStorage.res.js

@@ -4,10 +4,16 @@
 var $$Array = require("rescript/lib/js/array.js");
 var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
+var Js_exn = require("rescript/lib/js/js_exn.js");
 var Schema = require("./db/Schema.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Caml_int32 = require("rescript/lib/js/caml_int32.js");
+var Caml_option = require("rescript/lib/js/caml_option.js");
+var Persistence = require("./Persistence.res.js");
+var S$RescriptSchema = require("rescript-schema/src/S.res.js");
+var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
-function makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema) {
+function makeCreateIndexSql(tableName, indexFields, pgSchema) {
   var indexName = tableName + "_" + indexFields.join("_");
   var index = Belt_Array.map(indexFields, (function (idx) {
             return "\"" + idx + "\"";
@@ -15,20 +21,20 @@ function makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema) {
   return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
 }
 
-function makeCreateTableIndicesSqlUnsafe(table, pgSchema) {
+function makeCreateTableIndicesSql(table, pgSchema) {
   var tableName = table.tableName;
   var createIndex = function (indexField) {
-    return makeCreateIndexSqlUnsafe(tableName, [indexField], pgSchema);
+    return makeCreateIndexSql(tableName, [indexField], pgSchema);
   };
   var createCompositeIndex = function (indexFields) {
-    return makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema);
+    return makeCreateIndexSql(tableName, indexFields, pgSchema);
   };
   var singleIndices = Table.getSingleIndices(table);
   var compositeIndices = Table.getCompositeIndices(table);
   return Belt_Array.map(singleIndices, createIndex).join("\n") + Belt_Array.map(compositeIndices, createCompositeIndex).join("\n");
 }
 
-function makeCreateTableSqlUnsafe(table, pgSchema) {
+function makeCreateTableSql(table, pgSchema) {
   var fieldsMapped = Belt_Array.map(Table.getFields(table), (function (field) {
             var defaultValue = field.defaultValue;
             var fieldType = field.fieldType;
@@ -52,8 +58,12 @@ function makeCreateTableSqlUnsafe(table, pgSchema) {
         ) + ");";
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enums, cleanRun) {
-  var allTables = $$Array.copy(staticTables);
+function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesOpt, enumsOpt, cleanRunOpt) {
+  var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
+  var entities = entitiesOpt !== undefined ? entitiesOpt : [];
+  var enums = enumsOpt !== undefined ? enumsOpt : [];
+  var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
+  var allTables = $$Array.copy(generalTables);
   var allEntityTables = [];
   entities.forEach(function (entity) {
         allEntityTables.push(entity.table);
@@ -64,7 +74,7 @@ function makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enu
   var query = {
     contents: (
       cleanRun ? "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";" : "CREATE SCHEMA IF NOT EXISTS \"" + pgSchema + "\";"
-    ) + ("GRANT ALL ON SCHEMA \"" + pgSchema + "\" TO " + pgUser + ";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;")
+    ) + ("GRANT ALL ON SCHEMA \"" + pgSchema + "\" TO \"" + pgUser + "\";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;")
   };
   enums.forEach(function (enumConfig) {
         var enumCreateQuery = "CREATE TYPE \"" + pgSchema + "\"." + enumConfig.name + " AS ENUM(" + enumConfig.variants.map(function (v) {
@@ -75,10 +85,10 @@ function makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enu
         );
       });
   allTables.forEach(function (table) {
-        query.contents = query.contents + "\n" + makeCreateTableSqlUnsafe(table, pgSchema);
+        query.contents = query.contents + "\n" + makeCreateTableSql(table, pgSchema);
       });
   allTables.forEach(function (table) {
-        var indices = makeCreateTableIndicesSqlUnsafe(table, pgSchema);
+        var indices = makeCreateTableIndicesSql(table, pgSchema);
         if (indices !== "") {
           query.contents = query.contents + "\n" + indices;
           return ;
@@ -92,10 +102,158 @@ function makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enu
         functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.createInsertFnQuery;
         Table.getDerivedFromFields(entity.table).forEach(function (derivedFromField) {
               var indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
-              query.contents = query.contents + "\n" + makeCreateIndexSqlUnsafe(derivedFromField.derivedFromEntity, [indexField], pgSchema);
+              query.contents = query.contents + "\n" + makeCreateIndexSql(derivedFromField.derivedFromEntity, [indexField], pgSchema);
             });
       });
-  return [cleanRun ? query.contents : "DO $$ BEGIN " + query.contents + " END $$;"].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+  return [cleanRun || Utils.$$Array.isEmpty(enums) ? query.contents : "DO $$ BEGIN " + query.contents + " END $$;"].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+}
+
+function makeLoadByIdSql(pgSchema, tableName) {
+  return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = $1 LIMIT 1;";
+}
+
+function makeLoadByIdsSql(pgSchema, tableName) {
+  return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = ANY($1::text[]);";
+}
+
+function makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents) {
+  var match = Table.toSqlParams(table, itemSchema);
+  var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
+  var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
+  return "INSERT INTO \"" + pgSchema + "\".\"" + table.tableName + "\" (" + match.quotedFieldNames.join(", ") + ")\nSELECT * FROM unnest(" + match.arrayFieldTypes.map(function (arrayFieldType, idx) {
+                return "$" + (idx + 1 | 0).toString() + "::" + arrayFieldType;
+              }).join(",") + ")" + (
+          isRawEvents || primaryKeyFieldNames.length === 0 ? "" : "ON CONFLICT(" + primaryKeyFieldNames.map(function (fieldName) {
+                    return "\"" + fieldName + "\"";
+                  }).join(",") + ") DO " + (
+              Utils.$$Array.isEmpty(quotedNonPrimaryFieldNames) ? "NOTHING" : "UPDATE SET " + quotedNonPrimaryFieldNames.map(function (fieldName) {
+                        return fieldName + " = EXCLUDED." + fieldName;
+                      }).join(",")
+            )
+        ) + ";";
+}
+
+function makeInsertValuesSetSql(pgSchema, table, itemSchema, itemsCount) {
+  var match = Table.toSqlParams(table, itemSchema);
+  var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
+  var quotedFieldNames = match.quotedFieldNames;
+  var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
+  var fieldsCount = quotedFieldNames.length;
+  var placeholders = "";
+  for(var idx = 1; idx <= itemsCount; ++idx){
+    if (idx > 1) {
+      placeholders = placeholders + ",";
+    }
+    placeholders = placeholders + "(";
+    for(var fieldIdx = 0; fieldIdx < fieldsCount; ++fieldIdx){
+      if (fieldIdx > 0) {
+        placeholders = placeholders + ",";
+      }
+      placeholders = placeholders + ("$" + (Math.imul(fieldIdx, itemsCount) + idx | 0).toString());
+    }
+    placeholders = placeholders + ")";
+  }
+  return "INSERT INTO \"" + pgSchema + "\".\"" + table.tableName + "\" (" + quotedFieldNames.join(", ") + ")\nVALUES" + placeholders + (
+          primaryKeyFieldNames.length !== 0 ? "ON CONFLICT(" + primaryKeyFieldNames.map(function (fieldName) {
+                    return "\"" + fieldName + "\"";
+                  }).join(",") + ") DO " + (
+              Utils.$$Array.isEmpty(quotedNonPrimaryFieldNames) ? "NOTHING" : "UPDATE SET " + quotedNonPrimaryFieldNames.map(function (fieldName) {
+                        return fieldName + " = EXCLUDED." + fieldName;
+                      }).join(",")
+            ) : ""
+        ) + ";";
+}
+
+var rawEventsTableName = "raw_events";
+
+function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
+  var match = Table.toSqlParams(table, itemSchema);
+  var isRawEvents = table.tableName === rawEventsTableName;
+  if (isRawEvents || !match.hasArrayField) {
+    return {
+            sql: makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents),
+            convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
+            isInsertValues: false
+          };
+  } else {
+    return {
+            sql: makeInsertValuesSetSql(pgSchema, table, itemSchema, 500),
+            convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), (function (param) {
+                        return {
+                                s: (function (prim) {
+                                    return prim.flat(1);
+                                  })
+                              };
+                      })), "Output", "Input", "Sync", false),
+            isInsertValues: true
+          };
+  }
+}
+
+function chunkArray(arr, chunkSize) {
+  var chunks = [];
+  var i = 0;
+  while(i < arr.length) {
+    var chunk = arr.slice(i, i + chunkSize | 0);
+    chunks.push(chunk);
+    i = i + chunkSize | 0;
+  };
+  return chunks;
+}
+
+var setQueryCache = new WeakMap();
+
+async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
+  if (items.length === 0) {
+    return ;
+  }
+  var cached = setQueryCache.get(table);
+  var query;
+  if (cached !== undefined) {
+    query = Caml_option.valFromOption(cached);
+  } else {
+    var newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema);
+    setQueryCache.set(table, newQuery);
+    query = newQuery;
+  }
+  var sqlQuery = query.sql;
+  try {
+    var payload = query.convertOrThrow(items);
+    if (!query.isInsertValues) {
+      return await sql.unsafe(sqlQuery, payload, {prepare: true});
+    }
+    var match = itemSchema.t;
+    var fieldsCount;
+    fieldsCount = typeof match !== "object" || match.TAG !== "object" ? Js_exn.raiseError("Expected an object schema for table") : match.items.length;
+    var maxChunkSize = Math.imul(500, fieldsCount);
+    var chunks = chunkArray(payload, maxChunkSize);
+    var responses = [];
+    chunks.forEach(function (chunk) {
+          var chunkSize = chunk.length;
+          var isFullChunk = chunkSize === maxChunkSize;
+          var response = sql.unsafe(isFullChunk ? sqlQuery : makeInsertValuesSetSql(pgSchema, table, itemSchema, Caml_int32.div(chunkSize, fieldsCount)), chunk, {prepare: true});
+          responses.push(response);
+        });
+    await Promise.all(responses);
+    return ;
+  }
+  catch (raw_exn){
+    var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+    if (exn.RE_EXN_ID === S$RescriptSchema.Raised) {
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed to convert items for table \"" + table.tableName + "\"",
+            reason: exn,
+            Error: new Error()
+          };
+    }
+    throw {
+          RE_EXN_ID: Persistence.StorageError,
+          message: "Failed to insert items into table \"" + table.tableName + "\"",
+          reason: exn,
+          Error: new Error()
+        };
+  }
 }
 
 function make(sql, pgSchema, pgUser) {
@@ -103,23 +261,73 @@ function make(sql, pgSchema, pgUser) {
     var schemas = await sql.unsafe("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '" + pgSchema + "';");
     return Utils.$$Array.notEmpty(schemas);
   };
-  var initialize = async function (entities, staticTables, enums, cleanRun) {
-    var queries = makeInitializeTransaction(pgSchema, pgUser, staticTables, entities, enums, cleanRun);
+  var initialize = async function (entitiesOpt, generalTablesOpt, enumsOpt, cleanRunOpt) {
+    var entities = entitiesOpt !== undefined ? entitiesOpt : [];
+    var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
+    var enums = enumsOpt !== undefined ? enumsOpt : [];
+    var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
+    var queries = makeInitializeTransaction(pgSchema, pgUser, generalTables, entities, enums, cleanRun);
     await sql.begin(function (sql) {
           return queries.map(function (query) {
                       return sql.unsafe(query);
                     });
         });
   };
+  var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
+    var rows;
+    try {
+      rows = await (
+        ids.length !== 1 ? sql.unsafe(makeLoadByIdsSql(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeLoadByIdSql(pgSchema, table.tableName), ids, {prepare: true})
+      );
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed loading \"" + table.tableName + "\" from storage by ids",
+            reason: exn,
+            Error: new Error()
+          };
+    }
+    try {
+      return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
+    }
+    catch (raw_exn$1){
+      var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed to parse \"" + table.tableName + "\" loaded from storage by ids",
+            reason: exn$1,
+            Error: new Error()
+          };
+    }
+  };
+  var setOrThrow$1 = function (items, table, itemSchema) {
+    return setOrThrow(sql, items, table, itemSchema, pgSchema);
+  };
   return {
           isInitialized: isInitialized,
-          initialize: initialize
+          initialize: initialize,
+          loadByIdsOrThrow: loadByIdsOrThrow,
+          setOrThrow: setOrThrow$1
         };
 }
 
-exports.makeCreateIndexSqlUnsafe = makeCreateIndexSqlUnsafe;
-exports.makeCreateTableIndicesSqlUnsafe = makeCreateTableIndicesSqlUnsafe;
-exports.makeCreateTableSqlUnsafe = makeCreateTableSqlUnsafe;
+var maxItemsPerQuery = 500;
+
+exports.makeCreateIndexSql = makeCreateIndexSql;
+exports.makeCreateTableIndicesSql = makeCreateTableIndicesSql;
+exports.makeCreateTableSql = makeCreateTableSql;
 exports.makeInitializeTransaction = makeInitializeTransaction;
+exports.makeLoadByIdSql = makeLoadByIdSql;
+exports.makeLoadByIdsSql = makeLoadByIdsSql;
+exports.makeInsertUnnestSetSql = makeInsertUnnestSetSql;
+exports.makeInsertValuesSetSql = makeInsertValuesSetSql;
+exports.rawEventsTableName = rawEventsTableName;
+exports.maxItemsPerQuery = maxItemsPerQuery;
+exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
+exports.chunkArray = chunkArray;
+exports.setQueryCache = setQueryCache;
+exports.setOrThrow = setOrThrow;
 exports.make = make;
-/* Table Not a pure module */
+/* setQueryCache Not a pure module */