envio 2.25.0 → 2.25.1

package/index.d.ts

@@ -105,6 +105,8 @@ export function experimental_createEffect<
     readonly input: IS;
     /** The output schema of the effect. */
     readonly output: OS;
+    /** Whether the effect should be cached. */
+    readonly cache?: boolean;
   },
   handler: (args: EffectArgs<I>) => Promise<R>
 ): Effect<I, O>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.25.0",
+  "version": "v2.25.1",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.25.0",
-    "envio-linux-arm64": "v2.25.0",
-    "envio-darwin-x64": "v2.25.0",
-    "envio-darwin-arm64": "v2.25.0"
+    "envio-linux-x64": "v2.25.1",
+    "envio-linux-arm64": "v2.25.1",
+    "envio-darwin-x64": "v2.25.1",
+    "envio-darwin-arm64": "v2.25.1"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.gen.ts

@@ -21,7 +21,9 @@ export type effectOptions<input,output> = {
   /** The input schema of the effect. */
   readonly input: RescriptSchema_S_t<input>; 
   /** The output schema of the effect. */
-  readonly output: RescriptSchema_S_t<output>
+  readonly output: RescriptSchema_S_t<output>; 
+  /** Whether the effect should be cached. */
+  readonly cache?: boolean
 };
 
 export type effectContext = $$effectContext;

package/src/Envio.res

@@ -22,6 +22,8 @@ and effectOptions<'input, 'output> = {
   input: S.t<'input>,
   /** The output schema of the effect. */
   output: S.t<'output>,
+  /** Whether the effect should be cached. */
+  cache?: bool,
 }
 @genType.import(("./Types.ts", "EffectContext"))
 and effectContext = {
@@ -48,5 +50,6 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
+    cache: options.cache->Belt.Option.getWithDefault(false),
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -2,12 +2,14 @@
 'use strict';
 
 var Prometheus = require("./Prometheus.res.js");
+var Belt_Option = require("rescript/lib/js/belt_Option.js");
 
 function experimental_createEffect(options, handler) {
   Prometheus.EffectCallsCount.set(0, options.name);
   return {
           name: options.name,
           handler: handler,
+          cache: Belt_Option.getWithDefault(options.cache, false),
           callsCount: 0
         };
 }

package/src/Internal.res

@@ -197,6 +197,7 @@ type effectArgs = {
 type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
+  cache: bool,
   mutable callsCount: int,
 }
 

package/src/Persistence.res

@@ -5,6 +5,18 @@
 // Currently there are quite many code spread across
 // DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
 
+// The type reflects an effect cache table in the db
+// It might be present even if the effect is not used in the application
+type effectCache = {
+  name: string,
+  // Number of rows in the table
+  mutable size: int,
+  // Lazily attached table definition when effect is used in the application
+  mutable table: option<Table.table>,
+}
+
+type operator = [#">" | #"="]
+
 type storage = {
   // Should return true if we already have persisted data
   // and we can skip initialization
@@ -15,9 +27,8 @@ type storage = {
     ~entities: array<Internal.entityConfig>=?,
     ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
-    // If true, the storage should clear existing data
-    ~cleanRun: bool=?,
   ) => promise<unit>,
+  loadEffectCaches: unit => promise<array<effectCache>>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
     ~ids: array<string>,
@@ -25,6 +36,15 @@ type storage = {
     ~rowsSchema: S.t<array<'item>>,
   ) => promise<array<'item>>,
   @raises("StorageError")
+  loadByFieldOrThrow: 'item 'value. (
+    ~fieldName: string,
+    ~fieldSchema: S.t<'value>,
+    ~fieldValue: 'value,
+    ~operator: operator,
+    ~table: Table.table,
+    ~rowsSchema: S.t<array<'item>>,
+  ) => promise<array<'item>>,
+  @raises("StorageError")
   setOrThrow: 'item. (
     ~items: array<'item>,
     ~table: Table.table,
@@ -37,7 +57,7 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool})
+  | Ready({cleanRun: bool, effectCaches: dict<effectCache>})
 
 type t = {
   userEntities: array<Internal.entityConfig>,
@@ -79,14 +99,7 @@ let make = (
   }
 }
 
-let init = async (
-  persistence,
-  // There are not much sense in the option,
-  // but this is how the runUpMigration used to work
-  // and we want to keep the upsert behavior without breaking changes.
-  ~skipIsInitializedCheck=false,
-  ~reset=false,
-) => {
+let init = async (persistence, ~reset=false) => {
   try {
     let shouldRun = switch persistence.storageStatus {
     | Unknown => true
@@ -102,20 +115,37 @@ let init = async (
         resolveRef := resolve
       })
       persistence.storageStatus = Initializing(promise)
-      if !(reset || skipIsInitializedCheck) && (await persistence.storage.isInitialized()) {
-        persistence.storageStatus = Ready({cleanRun: false})
-      } else {
+      if reset || !(await persistence.storage.isInitialized()) {
         let _ = await persistence.storage.initialize(
           ~entities=persistence.allEntities,
           ~generalTables=persistence.staticTables,
           ~enums=persistence.allEnums,
-          ~cleanRun=reset || !skipIsInitializedCheck,
         )
-        persistence.storageStatus = Ready({cleanRun: true})
+
+        persistence.storageStatus = Ready({
+          cleanRun: true,
+          effectCaches: Js.Dict.empty(),
+        })
         switch persistence.onStorageInitialize {
         | Some(onStorageInitialize) => await onStorageInitialize()
         | None => ()
         }
+      } else if (
+        // In case of a race condition,
+        // we want to set the initial status to Ready only once.
+        switch persistence.storageStatus {
+        | Initializing(_) => true
+        | _ => false
+        }
+      ) {
+        let effectCaches = Js.Dict.empty()
+        (await persistence.storage.loadEffectCaches())->Js.Array2.forEach(effectCache => {
+          effectCaches->Js.Dict.set(effectCache.name, effectCache)
+        })
+        persistence.storageStatus = Ready({
+          cleanRun: false,
+          effectCaches,
+        })
       }
       resolveRef.contents()
     }

package/src/Persistence.res.js

@@ -36,8 +36,7 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
         };
 }
 
-async function init(persistence, skipIsInitializedCheckOpt, resetOpt) {
-  var skipIsInitializedCheck = skipIsInitializedCheckOpt !== undefined ? skipIsInitializedCheckOpt : false;
+async function init(persistence, resetOpt) {
   var reset = resetOpt !== undefined ? resetOpt : false;
   try {
     var promise = persistence.storageStatus;
@@ -63,22 +62,34 @@ async function init(persistence, skipIsInitializedCheckOpt, resetOpt) {
       TAG: "Initializing",
       _0: promise$1
     };
-    if (!(reset || skipIsInitializedCheck) && await persistence.storage.isInitialized()) {
+    if (reset || !await persistence.storage.isInitialized()) {
+      await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums);
       persistence.storageStatus = {
         TAG: "Ready",
-        cleanRun: false
-      };
-    } else {
-      await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums, reset || !skipIsInitializedCheck);
-      persistence.storageStatus = {
-        TAG: "Ready",
-        cleanRun: true
+        cleanRun: true,
+        effectCaches: {}
       };
       var onStorageInitialize = persistence.onStorageInitialize;
       if (onStorageInitialize !== undefined) {
         await onStorageInitialize();
       }
       
+    } else {
+      var match = persistence.storageStatus;
+      var tmp;
+      tmp = typeof match !== "object" || match.TAG !== "Initializing" ? false : true;
+      if (tmp) {
+        var effectCaches = {};
+        (await persistence.storage.loadEffectCaches()).forEach(function (effectCache) {
+              effectCaches[effectCache.name] = effectCache;
+            });
+        persistence.storageStatus = {
+          TAG: "Ready",
+          cleanRun: false,
+          effectCaches: effectCaches
+        };
+      }
+      
     }
     return resolveRef.contents();
   }

package/src/PgStorage.res

@@ -1,16 +1,16 @@
-let makeCreateIndexSql = (~tableName, ~indexFields, ~pgSchema) => {
+let makeCreateIndexQuery = (~tableName, ~indexFields, ~pgSchema) => {
   let indexName = tableName ++ "_" ++ indexFields->Js.Array2.joinWith("_")
   let index = indexFields->Belt.Array.map(idx => `"${idx}"`)->Js.Array2.joinWith(", ")
   `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${pgSchema}"."${tableName}"(${index});`
 }
 
-let makeCreateTableIndicesSql = (table: Table.table, ~pgSchema) => {
+let makeCreateTableIndicesQuery = (table: Table.table, ~pgSchema) => {
   open Belt
   let tableName = table.tableName
   let createIndex = indexField =>
-    makeCreateIndexSql(~tableName, ~indexFields=[indexField], ~pgSchema)
+    makeCreateIndexQuery(~tableName, ~indexFields=[indexField], ~pgSchema)
   let createCompositeIndex = indexFields => {
-    makeCreateIndexSql(~tableName, ~indexFields, ~pgSchema)
+    makeCreateIndexQuery(~tableName, ~indexFields, ~pgSchema)
   }
 
   let singleIndices = table->Table.getSingleIndices
@@ -20,7 +20,7 @@ let makeCreateTableIndicesSql = (table: Table.table, ~pgSchema) => {
     compositeIndices->Array.map(createCompositeIndex)->Js.Array2.joinWith("\n")
 }
 
-let makeCreateTableSql = (table: Table.table, ~pgSchema) => {
+let makeCreateTableQuery = (table: Table.table, ~pgSchema) => {
   open Belt
   let fieldsMapped =
     table
@@ -58,7 +58,6 @@ let makeInitializeTransaction = (
   ~generalTables=[],
   ~entities=[],
   ~enums=[],
-  ~cleanRun=false,
 ) => {
   let allTables = generalTables->Array.copy
   let allEntityTables = []
@@ -70,13 +69,9 @@ let makeInitializeTransaction = (
   let derivedSchema = Schema.make(allEntityTables)
 
   let query = ref(
-    (
-      cleanRun
-        ? `DROP SCHEMA IF EXISTS "${pgSchema}" CASCADE;
-CREATE SCHEMA "${pgSchema}";`
-        : `CREATE SCHEMA IF NOT EXISTS "${pgSchema}";`
-    ) ++
-    `GRANT ALL ON SCHEMA "${pgSchema}" TO "${pgUser}";
+    `DROP SCHEMA IF EXISTS "${pgSchema}" CASCADE;
+CREATE SCHEMA "${pgSchema}";
+GRANT ALL ON SCHEMA "${pgSchema}" TO "${pgUser}";
 GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
   )
 
@@ -87,31 +82,17 @@ GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
       ->Js.Array2.map(v => `'${v->(Utils.magic: Internal.enum => string)}'`)
       ->Js.Array2.joinWith(", ")});`
 
-    query :=
-      query.contents ++
-      "\n" ++ if cleanRun {
-        // Direct creation when cleanRunting (faster)
-        enumCreateQuery
-      } else {
-        // Wrap with conditional check only when not cleanRunting
-        `IF NOT EXISTS (
-  SELECT 1 FROM pg_type 
-  WHERE typname = '${enumConfig.name->Js.String2.toLowerCase}' 
-  AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${pgSchema}')
-) THEN 
-    ${enumCreateQuery}
-END IF;`
-      }
+    query := query.contents ++ "\n" ++ enumCreateQuery
   })
 
   // Batch all table creation first (optimal for PostgreSQL)
   allTables->Js.Array2.forEach((table: Table.table) => {
-    query := query.contents ++ "\n" ++ makeCreateTableSql(table, ~pgSchema)
+    query := query.contents ++ "\n" ++ makeCreateTableQuery(table, ~pgSchema)
   })
 
   // Then batch all indices (better performance when tables exist)
   allTables->Js.Array2.forEach((table: Table.table) => {
-    let indices = makeCreateTableIndicesSql(table, ~pgSchema)
+    let indices = makeCreateTableIndicesQuery(table, ~pgSchema)
     if indices !== "" {
       query := query.contents ++ "\n" ++ indices
     }
@@ -131,7 +112,7 @@ END IF;`
       query :=
         query.contents ++
         "\n" ++
-        makeCreateIndexSql(
+        makeCreateIndexQuery(
           ~tableName=derivedFromField.derivedFromEntity,
           ~indexFields=[indexField],
           ~pgSchema,
@@ -139,28 +120,26 @@ END IF;`
     })
   })
 
-  [
-    // Return optimized queries - main DDL in DO block, functions separate
-    // Note: DO $$ BEGIN wrapper is only needed for PL/pgSQL conditionals (IF NOT EXISTS)
-    // Reset case uses direct DDL (faster), non-cleanRun case uses conditionals (safer)
-    cleanRun || enums->Utils.Array.isEmpty
-      ? query.contents
-      : `DO $$ BEGIN ${query.contents} END $$;`,
-    // Functions query (separate as they can't be in DO block)
-  ]->Js.Array2.concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : [])
+  [query.contents]->Js.Array2.concat(
+    functionsQuery.contents !== "" ? [functionsQuery.contents] : [],
+  )
 }
 
-let makeLoadByIdSql = (~pgSchema, ~tableName) => {
+let makeLoadByIdQuery = (~pgSchema, ~tableName) => {
   `SELECT * FROM "${pgSchema}"."${tableName}" WHERE id = $1 LIMIT 1;`
 }
 
-let makeLoadByIdsSql = (~pgSchema, ~tableName) => {
+let makeLoadByFieldQuery = (~pgSchema, ~tableName, ~fieldName, ~operator) => {
+  `SELECT * FROM "${pgSchema}"."${tableName}" WHERE "${fieldName}" ${operator} $1;`
+}
+
+let makeLoadByIdsQuery = (~pgSchema, ~tableName) => {
   `SELECT * FROM "${pgSchema}"."${tableName}" WHERE id = ANY($1::text[]);`
 }
 
-let makeInsertUnnestSetSql = (~pgSchema, ~table: Table.table, ~itemSchema, ~isRawEvents) => {
+let makeInsertUnnestSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema, ~isRawEvents) => {
   let {quotedFieldNames, quotedNonPrimaryFieldNames, arrayFieldTypes} =
-    table->Table.toSqlParams(~schema=itemSchema)
+    table->Table.toSqlParams(~schema=itemSchema, ~pgSchema)
 
   let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table)
 
@@ -188,8 +167,9 @@ SELECT * FROM unnest(${arrayFieldTypes
   } ++ ";"
 }
 
-let makeInsertValuesSetSql = (~pgSchema, ~table: Table.table, ~itemSchema, ~itemsCount) => {
-  let {quotedFieldNames, quotedNonPrimaryFieldNames} = table->Table.toSqlParams(~schema=itemSchema)
+let makeInsertValuesSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema, ~itemsCount) => {
+  let {quotedFieldNames, quotedNonPrimaryFieldNames} =
+    table->Table.toSqlParams(~schema=itemSchema, ~pgSchema)
 
   let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table)
   let fieldsCount = quotedFieldNames->Array.length
@@ -236,12 +216,13 @@ VALUES${placeholders.contents}` ++
 // they are always guaranteed to be an object.
 // FIXME what about Fuel params?
 let rawEventsTableName = "raw_events"
+let eventSyncStateTableName = "event_sync_state"
 
 // Constants for chunking
 let maxItemsPerQuery = 500
 
 let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'item>) => {
-  let {dbSchema, hasArrayField} = table->Table.toSqlParams(~schema=itemSchema)
+  let {dbSchema, hasArrayField} = table->Table.toSqlParams(~schema=itemSchema, ~pgSchema)
   let isRawEvents = table.tableName === rawEventsTableName
 
   // Should experiment how much it'll affect performance
@@ -253,7 +234,7 @@ let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'
 
   if isRawEvents || !hasArrayField {
     {
-      "sql": makeInsertUnnestSetSql(~pgSchema, ~table, ~itemSchema, ~isRawEvents),
+      "query": makeInsertUnnestSetQuery(~pgSchema, ~table, ~itemSchema, ~isRawEvents),
       "convertOrThrow": S.compile(
         S.unnest(dbSchema),
         ~input=Value,
@@ -265,7 +246,12 @@ let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'
     }
   } else {
     {
-      "sql": makeInsertValuesSetSql(~pgSchema, ~table, ~itemSchema, ~itemsCount=maxItemsPerQuery),
+      "query": makeInsertValuesSetQuery(
+        ~pgSchema,
+        ~table,
+        ~itemSchema,
+        ~itemsCount=maxItemsPerQuery,
+      ),
       "convertOrThrow": S.compile(
         S.unnest(itemSchema)->S.preprocess(_ => {
           serializer: Utils.Array.flatten->Utils.magic,
@@ -291,6 +277,35 @@ let chunkArray = (arr: array<'a>, ~chunkSize) => {
   chunks
 }
 
+let removeInvalidUtf8InPlace = entities =>
+  entities->Js.Array2.forEach(item => {
+    let dict = item->(Utils.magic: 'a => dict<unknown>)
+    dict->Utils.Dict.forEachWithKey((key, value) => {
+      if value->Js.typeof === "string" {
+        let value = value->(Utils.magic: unknown => string)
+        // We mutate here, since we don't care
+        // about the original value with \x00 anyways.
+        //
+        // This is unsafe, but we rely that it'll use
+        // the mutated reference on retry.
+        // TODO: Test it properly after we start using
+        // in-memory PGLite for indexer test framework.
+        dict->Js.Dict.set(
+          key,
+          value
+          ->Utils.String.replaceAll("\x00", "")
+          ->(Utils.magic: string => unknown),
+        )
+      }
+    })
+  })
+
+let pgEncodingErrorSchema = S.object(s =>
+  s.tag("message", `invalid byte sequence for encoding "UTF8": 0x00`)
+)
+
+exception PgEncodingError({table: Table.table})
+
 // WeakMap for caching table batch set queries
 let setQueryCache = Utils.WeakMap.make()
 let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema) => {
@@ -298,7 +313,7 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
     ()
   } else {
     // Get or create cached query for this table
-    let query = switch setQueryCache->Utils.WeakMap.get(table) {
+    let data = switch setQueryCache->Utils.WeakMap.get(table) {
     | Some(cached) => cached
     | None => {
         let newQuery = makeTableBatchSetQuery(
@@ -311,10 +326,8 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
       }
     }
 
-    let sqlQuery = query["sql"]
-
     try {
-      if query["isInsertValues"] {
+      if data["isInsertValues"] {
         let chunks = chunkArray(items, ~chunkSize=maxItemsPerQuery)
         let responses = []
         chunks->Js.Array2.forEach(chunk => {
@@ -325,9 +338,9 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
             // Either use the sql query for full chunks from cache
             // or create a new one for partial chunks on the fly.
             isFullChunk
-              ? sqlQuery
-              : makeInsertValuesSetSql(~pgSchema, ~table, ~itemSchema, ~itemsCount=chunkSize),
-            query["convertOrThrow"](chunk->(Utils.magic: array<'item> => array<unknown>)),
+              ? data["query"]
+              : makeInsertValuesSetQuery(~pgSchema, ~table, ~itemSchema, ~itemsCount=chunkSize),
+            data["convertOrThrow"](chunk->(Utils.magic: array<'item> => array<unknown>)),
           )
           responses->Js.Array2.push(response)->ignore
         })
@@ -335,8 +348,8 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
       } else {
         // Use UNNEST approach for single query
         await sql->Postgres.preparedUnsafe(
-          sqlQuery,
-          query["convertOrThrow"](items->(Utils.magic: array<'item> => array<unknown>)),
+          data["query"],
+          data["convertOrThrow"](items->(Utils.magic: array<'item> => array<unknown>)),
         )
       }
     } catch {
@@ -358,41 +371,112 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
   }
 }
 
+let setEntityHistoryOrThrow = (
+  sql,
+  ~entityHistory: EntityHistory.t<'entity>,
+  ~rows: array<EntityHistory.historyRow<'entity>>,
+  ~shouldCopyCurrentEntity=?,
+  ~shouldRemoveInvalidUtf8=false,
+) => {
+  rows
+  ->Belt.Array.map(historyRow => {
+    let row = historyRow->S.reverseConvertToJsonOrThrow(entityHistory.schema)
+    if shouldRemoveInvalidUtf8 {
+      [row]->removeInvalidUtf8InPlace
+    }
+    entityHistory.insertFn(
+      sql,
+      row,
+      ~shouldCopyCurrentEntity=switch shouldCopyCurrentEntity {
+      | Some(v) => v
+      | None => {
+          let containsRollbackDiffChange =
+            historyRow.containsRollbackDiffChange->Belt.Option.getWithDefault(false)
+          !containsRollbackDiffChange
+        }
+      },
+    )
+  })
+  ->Promise.all
+  ->(Utils.magic: promise<array<unit>> => promise<unit>)
+}
+
+type schemaTableName = {
+  @as("table_name")
+  tableName: string,
+}
+
+let makeSchemaTableNamesQuery = (~pgSchema) => {
+  `SELECT table_name FROM information_schema.tables WHERE table_schema = '${pgSchema}';`
+}
+
 let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
   let isInitialized = async () => {
-    let schemas =
+    let envioTables =
       await sql->Postgres.unsafe(
-        `SELECT schema_name FROM information_schema.schemata WHERE schema_name = '${pgSchema}';`,
+        `SELECT table_schema FROM information_schema.tables WHERE table_schema = '${pgSchema}' AND table_name = '${eventSyncStateTableName}';`,
       )
-    schemas->Utils.Array.notEmpty
+    envioTables->Utils.Array.notEmpty
   }
 
-  let initialize = async (~entities=[], ~generalTables=[], ~enums=[], ~cleanRun=false) => {
-    let queries = makeInitializeTransaction(
-      ~pgSchema,
-      ~pgUser,
-      ~generalTables,
-      ~entities,
-      ~enums,
-      ~cleanRun,
-    )
+  let initialize = async (~entities=[], ~generalTables=[], ~enums=[]) => {
+    let schemaTableNames: array<schemaTableName> =
+      await sql->Postgres.unsafe(makeSchemaTableNamesQuery(~pgSchema))
+
+    // The initialization query will completely drop the schema and recreate it from scratch.
+    // So we need to check if the schema is not used for anything else than envio.
+    if (
+      // Should pass with existing schema with no tables
+      // This might happen when used with public schema
+      // which is automatically created by postgres.
+      schemaTableNames->Utils.Array.notEmpty &&
+        // Otherwise should throw if there's a table, but no envio specific one
+        // This means that the schema is used for something else than envio.
+        !(schemaTableNames->Js.Array2.some(table => table.tableName === eventSyncStateTableName))
+    ) {
+      Js.Exn.raiseError(
+        `Cannot run Envio migrations on PostgreSQL schema "${pgSchema}" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: "pnpm envio local db-migrate down"\n2. Or specify a different schema name by setting the "ENVIO_PG_PUBLIC_SCHEMA" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.`,
+      )
+    }
+
+    let queries = makeInitializeTransaction(~pgSchema, ~pgUser, ~generalTables, ~entities, ~enums)
     // Execute all queries within a single transaction for integrity
     let _ = await sql->Postgres.beginSql(sql => {
       queries->Js.Array2.map(query => sql->Postgres.unsafe(query))
     })
   }
 
+  let loadEffectCaches = async () => {
+    let schemaTableNames: array<schemaTableName> =
+      await sql->Postgres.unsafe(makeSchemaTableNamesQuery(~pgSchema))
+    schemaTableNames->Belt.Array.keepMapU(schemaTableName => {
+      if schemaTableName.tableName->Js.String2.startsWith("effect_cache_") {
+        Some(
+          (
+            {
+              name: schemaTableName.tableName,
+              size: 0,
+              table: None,
+            }: Persistence.effectCache
+          ),
+        )
+      } else {
+        None
+      }
+    })
+  }
+
   let loadByIdsOrThrow = async (~ids, ~table: Table.table, ~rowsSchema) => {
     switch await (
       switch ids {
       | [_] =>
         sql->Postgres.preparedUnsafe(
-          makeLoadByIdSql(~pgSchema, ~tableName=table.tableName),
+          makeLoadByIdQuery(~pgSchema, ~tableName=table.tableName),
           ids->Obj.magic,
         )
       | _ =>
         sql->Postgres.preparedUnsafe(
-          makeLoadByIdsSql(~pgSchema, ~tableName=table.tableName),
+          makeLoadByIdsQuery(~pgSchema, ~tableName=table.tableName),
           [ids]->Obj.magic,
         )
       }
@@ -417,6 +501,52 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
     }
   }
 
+  let loadByFieldOrThrow = async (
+    ~fieldName: string,
+    ~fieldSchema,
+    ~fieldValue,
+    ~operator: Persistence.operator,
+    ~table: Table.table,
+    ~rowsSchema,
+  ) => {
+    let params = try [fieldValue->S.reverseConvertToJsonOrThrow(fieldSchema)]->Obj.magic catch {
+    | exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed loading "${table.tableName}" from storage by field "${fieldName}". Couldn't serialize provided value.`,
+          reason: exn,
+        }),
+      )
+    }
+    switch await sql->Postgres.preparedUnsafe(
+      makeLoadByFieldQuery(
+        ~pgSchema,
+        ~tableName=table.tableName,
+        ~fieldName,
+        ~operator=(operator :> string),
+      ),
+      params,
+    ) {
+    | exception exn =>
+      raise(
+        Persistence.StorageError({
+          message: `Failed loading "${table.tableName}" from storage by field "${fieldName}"`,
+          reason: exn,
+        }),
+      )
+    | rows =>
+      try rows->S.parseOrThrow(rowsSchema) catch {
+      | exn =>
+        raise(
+          Persistence.StorageError({
+            message: `Failed to parse "${table.tableName}" loaded from storage by ids`,
+            reason: exn,
+          }),
+        )
+      }
+    }
+  }
+
   let setOrThrow = (
     type item,
     ~items: array<item>,
@@ -435,6 +565,8 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
   {
     isInitialized,
     initialize,
+    loadByFieldOrThrow,
+    loadEffectCaches,
     loadByIdsOrThrow,
     setOrThrow,
   }

package/src/PgStorage.res.js

@@ -4,14 +4,17 @@
 var $$Array = require("rescript/lib/js/array.js");
 var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
+var Js_exn = require("rescript/lib/js/js_exn.js");
 var Schema = require("./db/Schema.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
+var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
-function makeCreateIndexSql(tableName, indexFields, pgSchema) {
+function makeCreateIndexQuery(tableName, indexFields, pgSchema) {
   var indexName = tableName + "_" + indexFields.join("_");
   var index = Belt_Array.map(indexFields, (function (idx) {
             return "\"" + idx + "\"";
@@ -19,20 +22,20 @@ function makeCreateIndexSql(tableName, indexFields, pgSchema) {
   return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
 }
 
-function makeCreateTableIndicesSql(table, pgSchema) {
+function makeCreateTableIndicesQuery(table, pgSchema) {
   var tableName = table.tableName;
   var createIndex = function (indexField) {
-    return makeCreateIndexSql(tableName, [indexField], pgSchema);
+    return makeCreateIndexQuery(tableName, [indexField], pgSchema);
   };
   var createCompositeIndex = function (indexFields) {
-    return makeCreateIndexSql(tableName, indexFields, pgSchema);
+    return makeCreateIndexQuery(tableName, indexFields, pgSchema);
   };
   var singleIndices = Table.getSingleIndices(table);
   var compositeIndices = Table.getCompositeIndices(table);
   return Belt_Array.map(singleIndices, createIndex).join("\n") + Belt_Array.map(compositeIndices, createCompositeIndex).join("\n");
 }
 
-function makeCreateTableSql(table, pgSchema) {
+function makeCreateTableQuery(table, pgSchema) {
   var fieldsMapped = Belt_Array.map(Table.getFields(table), (function (field) {
             var defaultValue = field.defaultValue;
             var fieldType = field.fieldType;
@@ -56,11 +59,10 @@ function makeCreateTableSql(table, pgSchema) {
         ) + ");";
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesOpt, enumsOpt, cleanRunOpt) {
+function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesOpt, enumsOpt) {
   var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
   var entities = entitiesOpt !== undefined ? entitiesOpt : [];
   var enums = enumsOpt !== undefined ? enumsOpt : [];
-  var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
   var allTables = $$Array.copy(generalTables);
   var allEntityTables = [];
   entities.forEach(function (entity) {
@@ -70,23 +72,19 @@ function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesO
       });
   var derivedSchema = Schema.make(allEntityTables);
   var query = {
-    contents: (
-      cleanRun ? "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";" : "CREATE SCHEMA IF NOT EXISTS \"" + pgSchema + "\";"
-    ) + ("GRANT ALL ON SCHEMA \"" + pgSchema + "\" TO \"" + pgUser + "\";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;")
+    contents: "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO \"" + pgUser + "\";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;"
   };
   enums.forEach(function (enumConfig) {
         var enumCreateQuery = "CREATE TYPE \"" + pgSchema + "\"." + enumConfig.name + " AS ENUM(" + enumConfig.variants.map(function (v) {
                 return "'" + v + "'";
               }).join(", ") + ");";
-        query.contents = query.contents + "\n" + (
-          cleanRun ? enumCreateQuery : "IF NOT EXISTS (\n  SELECT 1 FROM pg_type \n  WHERE typname = '" + enumConfig.name.toLowerCase() + "' \n  AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '" + pgSchema + "')\n) THEN \n    " + enumCreateQuery + "\nEND IF;"
-        );
+        query.contents = query.contents + "\n" + enumCreateQuery;
       });
   allTables.forEach(function (table) {
-        query.contents = query.contents + "\n" + makeCreateTableSql(table, pgSchema);
+        query.contents = query.contents + "\n" + makeCreateTableQuery(table, pgSchema);
       });
   allTables.forEach(function (table) {
-        var indices = makeCreateTableIndicesSql(table, pgSchema);
+        var indices = makeCreateTableIndicesQuery(table, pgSchema);
         if (indices !== "") {
           query.contents = query.contents + "\n" + indices;
           return ;
@@ -100,22 +98,26 @@ function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesO
         functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.createInsertFnQuery;
         Table.getDerivedFromFields(entity.table).forEach(function (derivedFromField) {
               var indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
-              query.contents = query.contents + "\n" + makeCreateIndexSql(derivedFromField.derivedFromEntity, [indexField], pgSchema);
+              query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
             });
       });
-  return [cleanRun || Utils.$$Array.isEmpty(enums) ? query.contents : "DO $$ BEGIN " + query.contents + " END $$;"].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+  return [query.contents].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
 }
 
-function makeLoadByIdSql(pgSchema, tableName) {
+function makeLoadByIdQuery(pgSchema, tableName) {
   return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = $1 LIMIT 1;";
 }
 
-function makeLoadByIdsSql(pgSchema, tableName) {
+function makeLoadByFieldQuery(pgSchema, tableName, fieldName, operator) {
+  return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE \"" + fieldName + "\" " + operator + " $1;";
+}
+
+function makeLoadByIdsQuery(pgSchema, tableName) {
   return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = ANY($1::text[]);";
 }
 
-function makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents) {
-  var match = Table.toSqlParams(table, itemSchema);
+function makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents) {
+  var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
   var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
   return "INSERT INTO \"" + pgSchema + "\".\"" + table.tableName + "\" (" + match.quotedFieldNames.join(", ") + ")\nSELECT * FROM unnest(" + match.arrayFieldTypes.map(function (arrayFieldType, idx) {
@@ -131,8 +133,8 @@ function makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents) {
         ) + ";";
 }
 
-function makeInsertValuesSetSql(pgSchema, table, itemSchema, itemsCount) {
-  var match = Table.toSqlParams(table, itemSchema);
+function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount) {
+  var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
   var quotedFieldNames = match.quotedFieldNames;
   var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
@@ -164,18 +166,20 @@ function makeInsertValuesSetSql(pgSchema, table, itemSchema, itemsCount) {
 
 var rawEventsTableName = "raw_events";
 
+var eventSyncStateTableName = "event_sync_state";
+
 function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
-  var match = Table.toSqlParams(table, itemSchema);
+  var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var isRawEvents = table.tableName === rawEventsTableName;
   if (isRawEvents || !match.hasArrayField) {
     return {
-            sql: makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents),
+            query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
             convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
             isInsertValues: false
           };
   } else {
     return {
-            sql: makeInsertValuesSetSql(pgSchema, table, itemSchema, 500),
+            query: makeInsertValuesSetQuery(pgSchema, table, itemSchema, 500),
             convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), (function (param) {
                         return {
                                 s: (function (prim) {
@@ -199,6 +203,24 @@ function chunkArray(arr, chunkSize) {
   return chunks;
 }
 
+function removeInvalidUtf8InPlace(entities) {
+  entities.forEach(function (item) {
+        Utils.Dict.forEachWithKey(item, (function (key, value) {
+                if (typeof value === "string") {
+                  item[key] = value.replaceAll("\x00", "");
+                  return ;
+                }
+                
+              }));
+      });
+}
+
+var pgEncodingErrorSchema = S$RescriptSchema.object(function (s) {
+      s.tag("message", "invalid byte sequence for encoding \"UTF8\": 0x00");
+    });
+
+var PgEncodingError = /* @__PURE__ */Caml_exceptions.create("PgStorage.PgEncodingError");
+
 var setQueryCache = new WeakMap();
 
 async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
@@ -206,25 +228,24 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
     return ;
   }
   var cached = setQueryCache.get(table);
-  var query;
+  var data;
   if (cached !== undefined) {
-    query = Caml_option.valFromOption(cached);
+    data = Caml_option.valFromOption(cached);
   } else {
     var newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema);
     setQueryCache.set(table, newQuery);
-    query = newQuery;
+    data = newQuery;
   }
-  var sqlQuery = query.sql;
   try {
-    if (!query.isInsertValues) {
-      return await sql.unsafe(sqlQuery, query.convertOrThrow(items), {prepare: true});
+    if (!data.isInsertValues) {
+      return await sql.unsafe(data.query, data.convertOrThrow(items), {prepare: true});
     }
     var chunks = chunkArray(items, 500);
     var responses = [];
     chunks.forEach(function (chunk) {
           var chunkSize = chunk.length;
           var isFullChunk = chunkSize === 500;
-          var response = sql.unsafe(isFullChunk ? sqlQuery : makeInsertValuesSetSql(pgSchema, table, itemSchema, chunkSize), query.convertOrThrow(chunk), {prepare: true});
+          var response = sql.unsafe(isFullChunk ? data.query : makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize), data.convertOrThrow(chunk), {prepare: true});
           responses.push(response);
         });
     await Promise.all(responses);
@@ -249,28 +270,61 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
   }
 }
 
+function setEntityHistoryOrThrow(sql, entityHistory, rows, shouldCopyCurrentEntity, shouldRemoveInvalidUtf8Opt) {
+  var shouldRemoveInvalidUtf8 = shouldRemoveInvalidUtf8Opt !== undefined ? shouldRemoveInvalidUtf8Opt : false;
+  return Promise.all(Belt_Array.map(rows, (function (historyRow) {
+                    var row = S$RescriptSchema.reverseConvertToJsonOrThrow(historyRow, entityHistory.schema);
+                    if (shouldRemoveInvalidUtf8) {
+                      removeInvalidUtf8InPlace([row]);
+                    }
+                    return entityHistory.insertFn(sql, row, shouldCopyCurrentEntity !== undefined ? shouldCopyCurrentEntity : !Belt_Option.getWithDefault(historyRow.containsRollbackDiffChange, false));
+                  })));
+}
+
+function makeSchemaTableNamesQuery(pgSchema) {
+  return "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + pgSchema + "';";
+}
+
 function make(sql, pgSchema, pgUser) {
   var isInitialized = async function () {
-    var schemas = await sql.unsafe("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '" + pgSchema + "';");
-    return Utils.$$Array.notEmpty(schemas);
+    var envioTables = await sql.unsafe("SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND table_name = '" + eventSyncStateTableName + "';");
+    return Utils.$$Array.notEmpty(envioTables);
   };
-  var initialize = async function (entitiesOpt, generalTablesOpt, enumsOpt, cleanRunOpt) {
+  var initialize = async function (entitiesOpt, generalTablesOpt, enumsOpt) {
     var entities = entitiesOpt !== undefined ? entitiesOpt : [];
     var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
     var enums = enumsOpt !== undefined ? enumsOpt : [];
-    var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
-    var queries = makeInitializeTransaction(pgSchema, pgUser, generalTables, entities, enums, cleanRun);
+    var schemaTableNames = await sql.unsafe(makeSchemaTableNamesQuery(pgSchema));
+    if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(function (table) {
+            return table.table_name === eventSyncStateTableName;
+          })) {
+      Js_exn.raiseError("Cannot run Envio migrations on PostgreSQL schema \"" + pgSchema + "\" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: \"pnpm envio local db-migrate down\"\n2. Or specify a different schema name by setting the \"ENVIO_PG_PUBLIC_SCHEMA\" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.");
+    }
+    var queries = makeInitializeTransaction(pgSchema, pgUser, generalTables, entities, enums);
     await sql.begin(function (sql) {
           return queries.map(function (query) {
                       return sql.unsafe(query);
                     });
         });
   };
+  var loadEffectCaches = async function () {
+    var schemaTableNames = await sql.unsafe(makeSchemaTableNamesQuery(pgSchema));
+    return Belt_Array.keepMapU(schemaTableNames, (function (schemaTableName) {
+                  if (schemaTableName.table_name.startsWith("effect_cache_")) {
+                    return {
+                            name: schemaTableName.table_name,
+                            size: 0,
+                            table: undefined
+                          };
+                  }
+                  
+                }));
+  };
   var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
     var rows;
     try {
       rows = await (
-        ids.length !== 1 ? sql.unsafe(makeLoadByIdsSql(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeLoadByIdSql(pgSchema, table.tableName), ids, {prepare: true})
+        ids.length !== 1 ? sql.unsafe(makeLoadByIdsQuery(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeLoadByIdQuery(pgSchema, table.tableName), ids, {prepare: true})
       );
     }
     catch (raw_exn){
@@ -295,32 +349,81 @@ function make(sql, pgSchema, pgUser) {
           };
     }
   };
+  var loadByFieldOrThrow = async function (fieldName, fieldSchema, fieldValue, operator, table, rowsSchema) {
+    var params;
+    try {
+      params = [S$RescriptSchema.reverseConvertToJsonOrThrow(fieldValue, fieldSchema)];
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed loading \"" + table.tableName + "\" from storage by field \"" + fieldName + "\". Couldn't serialize provided value.",
+            reason: exn,
+            Error: new Error()
+          };
+    }
+    var rows;
+    try {
+      rows = await sql.unsafe(makeLoadByFieldQuery(pgSchema, table.tableName, fieldName, operator), params, {prepare: true});
+    }
+    catch (raw_exn$1){
+      var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed loading \"" + table.tableName + "\" from storage by field \"" + fieldName + "\"",
+            reason: exn$1,
+            Error: new Error()
+          };
+    }
+    try {
+      return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
+    }
+    catch (raw_exn$2){
+      var exn$2 = Caml_js_exceptions.internalToOCamlException(raw_exn$2);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed to parse \"" + table.tableName + "\" loaded from storage by ids",
+            reason: exn$2,
+            Error: new Error()
+          };
+    }
+  };
   var setOrThrow$1 = function (items, table, itemSchema) {
     return setOrThrow(sql, items, table, itemSchema, pgSchema);
   };
   return {
           isInitialized: isInitialized,
           initialize: initialize,
+          loadEffectCaches: loadEffectCaches,
           loadByIdsOrThrow: loadByIdsOrThrow,
+          loadByFieldOrThrow: loadByFieldOrThrow,
           setOrThrow: setOrThrow$1
         };
 }
 
 var maxItemsPerQuery = 500;
 
-exports.makeCreateIndexSql = makeCreateIndexSql;
-exports.makeCreateTableIndicesSql = makeCreateTableIndicesSql;
-exports.makeCreateTableSql = makeCreateTableSql;
+exports.makeCreateIndexQuery = makeCreateIndexQuery;
+exports.makeCreateTableIndicesQuery = makeCreateTableIndicesQuery;
+exports.makeCreateTableQuery = makeCreateTableQuery;
 exports.makeInitializeTransaction = makeInitializeTransaction;
-exports.makeLoadByIdSql = makeLoadByIdSql;
-exports.makeLoadByIdsSql = makeLoadByIdsSql;
-exports.makeInsertUnnestSetSql = makeInsertUnnestSetSql;
-exports.makeInsertValuesSetSql = makeInsertValuesSetSql;
+exports.makeLoadByIdQuery = makeLoadByIdQuery;
+exports.makeLoadByFieldQuery = makeLoadByFieldQuery;
+exports.makeLoadByIdsQuery = makeLoadByIdsQuery;
+exports.makeInsertUnnestSetQuery = makeInsertUnnestSetQuery;
+exports.makeInsertValuesSetQuery = makeInsertValuesSetQuery;
 exports.rawEventsTableName = rawEventsTableName;
+exports.eventSyncStateTableName = eventSyncStateTableName;
 exports.maxItemsPerQuery = maxItemsPerQuery;
 exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
 exports.chunkArray = chunkArray;
+exports.removeInvalidUtf8InPlace = removeInvalidUtf8InPlace;
+exports.pgEncodingErrorSchema = pgEncodingErrorSchema;
+exports.PgEncodingError = PgEncodingError;
 exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
+exports.setEntityHistoryOrThrow = setEntityHistoryOrThrow;
+exports.makeSchemaTableNamesQuery = makeSchemaTableNamesQuery;
 exports.make = make;
-/* setQueryCache Not a pure module */
+/* pgEncodingErrorSchema Not a pure module */

package/src/Utils.res

@@ -301,6 +301,23 @@ module String = {
     str->Js.String2.slice(~from=0, ~to_=1)->Js.String.toUpperCase ++
       str->Js.String2.sliceToEnd(~from=1)
   }
+
+  /**
+`replaceAll(str, substr, newSubstr)` returns a new `string` which is
+identical to `str` except with all matching instances of `substr` replaced
+by `newSubstr`. `substr` is treated as a verbatim string to match, not a
+regular expression.
+See [`String.replaceAll`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replaceAll) on MDN.
+
+## Examples
+
+```rescript
+String.replaceAll("old old string", "old", "new") == "new new string"
+String.replaceAll("the cat and the dog", "the", "this") == "this cat and this dog"
+```
+*/
+  @send
+  external replaceAll: (string, string, string) => string = "replaceAll"
 }
 
 module Result = {

package/src/db/EntityHistory.res

@@ -148,28 +148,6 @@ type t<'entity> = {
   insertFn: (Postgres.sql, Js.Json.t, ~shouldCopyCurrentEntity: bool) => promise<unit>,
 }
 
-let insertRow = (
-  self: t<'entity>,
-  ~sql,
-  ~historyRow: historyRow<'entity>,
-  ~shouldCopyCurrentEntity,
-) => {
-  let row = historyRow->S.reverseConvertToJsonOrThrow(self.schema)
-  self.insertFn(sql, row, ~shouldCopyCurrentEntity)
-}
-
-let batchInsertRows = (self: t<'entity>, ~sql, ~rows: array<historyRow<'entity>>) => {
-  rows
-  ->Belt.Array.map(historyRow => {
-    let containsRollbackDiffChange =
-      historyRow.containsRollbackDiffChange->Belt.Option.getWithDefault(false)
-    let shouldCopyCurrentEntity = !containsRollbackDiffChange
-    self->insertRow(~sql, ~historyRow, ~shouldCopyCurrentEntity)
-  })
-  ->Promise.all
-  ->Promise.thenResolve(_ => ())
-}
-
 type entityInternal
 
 external castInternal: t<'entity> => t<entityInternal> = "%identity"

package/src/db/EntityHistory.res.js

@@ -4,7 +4,6 @@
 var Table = require("./Table.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 var variants = [
@@ -158,21 +157,6 @@ function makeHistoryRowSchema(entitySchema) {
               }));
 }
 
-function insertRow(self, sql, historyRow, shouldCopyCurrentEntity) {
-  var row = S$RescriptSchema.reverseConvertToJsonOrThrow(historyRow, self.schema);
-  return self.insertFn(sql, row, shouldCopyCurrentEntity);
-}
-
-function batchInsertRows(self, sql, rows) {
-  return Promise.all(Belt_Array.map(rows, (function (historyRow) {
-                      var containsRollbackDiffChange = Belt_Option.getWithDefault(historyRow.containsRollbackDiffChange, false);
-                      var shouldCopyCurrentEntity = !containsRollbackDiffChange;
-                      return insertRow(self, sql, historyRow, shouldCopyCurrentEntity);
-                    }))).then(function (param) {
-              
-            });
-}
-
 function fromTable(table, pgSchema, schema) {
   var currentChangeFieldNames = [
     "entity_history_block_timestamp",
@@ -300,7 +284,5 @@ exports.entityIdOnlySchema = entityIdOnlySchema;
 exports.previousHistoryFieldsSchema = previousHistoryFieldsSchema;
 exports.currentHistoryFieldsSchema = currentHistoryFieldsSchema;
 exports.makeHistoryRowSchema = makeHistoryRowSchema;
-exports.insertRow = insertRow;
-exports.batchInsertRows = batchInsertRows;
 exports.fromTable = fromTable;
 /* schema Not a pure module */

package/src/db/Table.res

@@ -185,7 +185,7 @@ type sqlParams<'entity> = {
   hasArrayField: bool,
 }
 
-let toSqlParams = (table: table, ~schema) => {
+let toSqlParams = (table: table, ~schema, ~pgSchema) => {
   let quotedFieldNames = []
   let quotedNonPrimaryFieldNames = []
   let arrayFieldTypes = []
@@ -240,7 +240,7 @@ let toSqlParams = (table: table, ~schema) => {
           switch field {
           | Field(f) =>
             switch f.fieldType {
-            | Custom(fieldType) => `${(Text :> string)}[]::${(fieldType :> string)}`
+            | Custom(fieldType) => `${(Text :> string)}[]::"${pgSchema}".${(fieldType :> string)}`
             | Boolean => `${(Integer :> string)}[]::${(f.fieldType :> string)}`
             | fieldType => (fieldType :> string)
             }

package/src/db/Table.res.js

@@ -183,7 +183,7 @@ function getUnfilteredCompositeIndicesUnsafe(table) {
               }));
 }
 
-function toSqlParams(table, schema) {
+function toSqlParams(table, schema, pgSchema) {
   var quotedFieldNames = [];
   var quotedNonPrimaryFieldNames = [];
   var arrayFieldTypes = [];
@@ -261,7 +261,7 @@ function toSqlParams(table, schema) {
                   var fieldType = f.fieldType;
                   tmp = fieldType === "TIMESTAMP" || fieldType === "TIMESTAMP WITH TIME ZONE" || fieldType === "JSONB" || fieldType === "SERIAL" || fieldType === "TEXT" || fieldType === "DOUBLE PRECISION" || fieldType === "NUMERIC" || fieldType === "BOOLEAN" || fieldType === "INTEGER" || fieldType === "TIMESTAMP WITH TIME ZONE NULL" ? (
                       fieldType === "BOOLEAN" ? "INTEGER[]::" + f.fieldType : fieldType
-                    ) : "TEXT[]::" + fieldType;
+                    ) : "TEXT[]::\"" + pgSchema + "\"." + fieldType;
                 } else {
                   tmp = "TEXT";
                 }