envio 2.23.0 → 2.24.0

package/evm.schema.json

@@ -33,6 +33,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/fuel.schema.json

@@ -26,6 +26,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.23.0",
+  "version": "v2.24.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.23.0",
-    "envio-linux-arm64": "v2.23.0",
-    "envio-darwin-x64": "v2.23.0",
-    "envio-darwin-arm64": "v2.23.0"
+    "envio-linux-x64": "v2.24.0",
+    "envio-linux-arm64": "v2.24.0",
+    "envio-darwin-x64": "v2.24.0",
+    "envio-darwin-arm64": "v2.24.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Persistence.res

@@ -47,7 +47,6 @@ type t = {
   mutable storageStatus: storageStatus,
   storage: storage,
   onStorageInitialize: option<unit => promise<unit>>,
-  cacheStorage: storage,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -64,7 +63,6 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
-  ~cacheStorage,
   ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
@@ -78,7 +76,6 @@ let make = (
     storageStatus: Unknown,
     storage,
     onStorageInitialize,
-    cacheStorage,
   }
 }
 

package/src/Persistence.res.js

@@ -22,7 +22,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, cacheStorage, onStorageInitialize) {
+function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, onStorageInitialize) {
   var allEntities = userEntities.concat([dcRegistryEntityConfig]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -32,8 +32,7 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
           allEnums: allEnums$1,
           storageStatus: "Unknown",
           storage: storage,
-          onStorageInitialize: onStorageInitialize,
-          cacheStorage: cacheStorage
+          onStorageInitialize: onStorageInitialize
         };
 }
 

package/src/PgStorage.res

@@ -314,47 +314,30 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
     let sqlQuery = query["sql"]
 
     try {
-      let payload =
-        query["convertOrThrow"](items->(Utils.magic: array<'item> => array<unknown>))->(
-          Utils.magic: unknown => array<unknown>
-        )
-
       if query["isInsertValues"] {
-        let fieldsCount = switch itemSchema->S.classify {
-        | S.Object({items}) => items->Array.length
-        | _ => Js.Exn.raiseError("Expected an object schema for table")
-        }
-
-        // Chunk the items for VALUES-based queries
-        // We need to multiply by fields count,
-        // because we flattened our entity values with S.unnest
-        // to optimize the query execution.
-        let maxChunkSize = maxItemsPerQuery * fieldsCount
-        let chunks = chunkArray(payload, ~chunkSize=maxChunkSize)
+        let chunks = chunkArray(items, ~chunkSize=maxItemsPerQuery)
         let responses = []
         chunks->Js.Array2.forEach(chunk => {
           let chunkSize = chunk->Array.length
-          let isFullChunk = chunkSize === maxChunkSize
+          let isFullChunk = chunkSize === maxItemsPerQuery
 
           let response = sql->Postgres.preparedUnsafe(
             // Either use the sql query for full chunks from cache
             // or create a new one for partial chunks on the fly.
             isFullChunk
               ? sqlQuery
-              : makeInsertValuesSetSql(
-                  ~pgSchema,
-                  ~table,
-                  ~itemSchema,
-                  ~itemsCount=chunkSize / fieldsCount,
-                ),
-            chunk->Utils.magic,
+              : makeInsertValuesSetSql(~pgSchema, ~table, ~itemSchema, ~itemsCount=chunkSize),
+            query["convertOrThrow"](chunk->(Utils.magic: array<'item> => array<unknown>)),
           )
           responses->Js.Array2.push(response)->ignore
         })
         let _ = await Promise.all(responses)
       } else {
         // Use UNNEST approach for single query
-        await sql->Postgres.preparedUnsafe(sqlQuery, payload->Obj.magic)
+        await sql->Postgres.preparedUnsafe(
+          sqlQuery,
+          query["convertOrThrow"](items->(Utils.magic: array<'item> => array<unknown>)),
+        )
       }
     } catch {
     | S.Raised(_) as exn =>

package/src/PgStorage.res.js

@@ -4,10 +4,8 @@
 var $$Array = require("rescript/lib/js/array.js");
 var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
-var Js_exn = require("rescript/lib/js/js_exn.js");
 var Schema = require("./db/Schema.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Caml_int32 = require("rescript/lib/js/caml_int32.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
@@ -218,20 +216,15 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
   }
   var sqlQuery = query.sql;
   try {
-    var payload = query.convertOrThrow(items);
     if (!query.isInsertValues) {
-      return await sql.unsafe(sqlQuery, payload, {prepare: true});
+      return await sql.unsafe(sqlQuery, query.convertOrThrow(items), {prepare: true});
     }
-    var match = itemSchema.t;
-    var fieldsCount;
-    fieldsCount = typeof match !== "object" || match.TAG !== "object" ? Js_exn.raiseError("Expected an object schema for table") : match.items.length;
-    var maxChunkSize = Math.imul(500, fieldsCount);
-    var chunks = chunkArray(payload, maxChunkSize);
+    var chunks = chunkArray(items, 500);
     var responses = [];
     chunks.forEach(function (chunk) {
           var chunkSize = chunk.length;
-          var isFullChunk = chunkSize === maxChunkSize;
-          var response = sql.unsafe(isFullChunk ? sqlQuery : makeInsertValuesSetSql(pgSchema, table, itemSchema, Caml_int32.div(chunkSize, fieldsCount)), chunk, {prepare: true});
+          var isFullChunk = chunkSize === 500;
+          var response = sql.unsafe(isFullChunk ? sqlQuery : makeInsertValuesSetSql(pgSchema, table, itemSchema, chunkSize), query.convertOrThrow(chunk), {prepare: true});
           responses.push(response);
         });
     await Promise.all(responses);

package/src/db/EntityHistory.res

@@ -173,8 +173,9 @@ let batchInsertRows = (self: t<'entity>, ~sql, ~rows: array<historyRow<'entity>>
 type entityInternal
 
 external castInternal: t<'entity> => t<entityInternal> = "%identity"
+external eval: string => 'a = "eval"
 
-let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
+let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> => {
   let entity_history_block_timestamp = "entity_history_block_timestamp"
   let entity_history_chain_id = "entity_history_chain_id"
   let entity_history_block_number = "entity_history_block_number"
@@ -235,12 +236,10 @@ let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
   let dataFieldNames = dataFields->Belt.Array.map(field => field->getFieldName)
 
   let originTableName = table.tableName
-  let originSchemaName = table.schemaName
   let historyTableName = originTableName ++ "_history"
   //ignore composite indices
   let table = mkTable(
     historyTableName,
-    ~schemaName=originSchemaName,
     ~fields=Belt.Array.concatMany([
       currentHistoryFields,
       previousHistoryFields,
@@ -251,8 +250,8 @@ let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
 
   let insertFnName = `"insert_${table.tableName}"`
   let historyRowArg = "history_row"
-  let historyTablePath = `"${originSchemaName}"."${historyTableName}"`
-  let originTablePath = `"${originSchemaName}"."${originTableName}"`
+  let historyTablePath = `"${pgSchema}"."${historyTableName}"`
+  let originTablePath = `"${pgSchema}"."${originTableName}"`
 
   let previousHistoryFieldsAreNullStr =
     previousChangeFieldNames
@@ -335,7 +334,7 @@ let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
     \${shouldCopyCurrentEntity});\``
 
   let insertFn: (Postgres.sql, Js.Json.t, ~shouldCopyCurrentEntity: bool) => promise<unit> =
-    insertFnString->Table.PostgresInterop.eval
+    insertFnString->eval
 
   let schema = makeHistoryRowSchema(schema)
 

package/src/db/EntityHistory.res.js

@@ -173,7 +173,7 @@ function batchInsertRows(self, sql, rows) {
             });
 }
 
-function fromTable(table, schema) {
+function fromTable(table, pgSchema, schema) {
   var currentChangeFieldNames = [
     "entity_history_block_timestamp",
     "entity_history_chain_id",
@@ -238,9 +238,8 @@ function fromTable(table, schema) {
           return Table.getFieldName(field);
         }));
   var originTableName = table.tableName;
-  var originSchemaName = table.schemaName;
   var historyTableName = originTableName + "_history";
-  var table$1 = Table.mkTable(historyTableName, originSchemaName, undefined, Belt_Array.concatMany([
+  var table$1 = Table.mkTable(historyTableName, undefined, Belt_Array.concatMany([
             currentHistoryFields,
             previousHistoryFields,
             dataFields,
@@ -251,8 +250,8 @@ function fromTable(table, schema) {
           ]));
   var insertFnName = "\"insert_" + table$1.tableName + "\"";
   var historyRowArg = "history_row";
-  var historyTablePath = "\"" + originSchemaName + "\".\"" + historyTableName + "\"";
-  var originTablePath = "\"" + originSchemaName + "\".\"" + originTableName + "\"";
+  var historyTablePath = "\"" + pgSchema + "\".\"" + historyTableName + "\"";
+  var originTablePath = "\"" + pgSchema + "\".\"" + originTableName + "\"";
   var previousHistoryFieldsAreNullStr = Belt_Array.map(previousChangeFieldNames, (function (fieldName) {
             return historyRowArg + "." + fieldName + " IS NULL";
           })).join(" OR ");

package/src/db/Table.res

@@ -89,14 +89,12 @@ let getFieldType = (field: field) => {
 
 type table = {
   tableName: string,
-  schemaName: string,
   fields: array<fieldOrDerived>,
   compositeIndices: array<array<string>>,
 }
 
-let mkTable = (tableName, ~schemaName, ~compositeIndices=[], ~fields) => {
+let mkTable = (tableName, ~compositeIndices=[], ~fields) => {
   tableName,
-  schemaName,
   fields,
   compositeIndices,
 }
@@ -299,59 +297,3 @@ let getCompositeIndices = (table): array<array<string>> => {
   ->getUnfilteredCompositeIndicesUnsafe
   ->Array.keep(ind => ind->Array.length > 1)
 }
-
-module PostgresInterop = {
-  type pgFn<'payload, 'return> = (Postgres.sql, 'payload) => promise<'return>
-  type batchSetFn<'a> = (Postgres.sql, array<'a>) => promise<unit>
-  external eval: string => 'a = "eval"
-
-  let makeBatchSetFnString = (table: table) => {
-    let fieldNamesInQuotes =
-      table->getNonDefaultFieldNames->Array.map(fieldName => `"${fieldName}"`)
-    `(sql, rows) => {
-      return sql\`
-        INSERT INTO "${table.schemaName}"."${table.tableName}"
-        \${sql(rows, ${fieldNamesInQuotes->Js.Array2.joinWith(", ")})}
-        ON CONFLICT(${table->getPrimaryKeyFieldNames->Js.Array2.joinWith(", ")}) DO UPDATE
-        SET
-        ${fieldNamesInQuotes
-      ->Array.map(fieldNameInQuotes => `${fieldNameInQuotes} = EXCLUDED.${fieldNameInQuotes}`)
-      ->Js.Array2.joinWith(", ")};\`
-    }`
-  }
-
-  let chunkBatchQuery = (
-    sql,
-    entityDataArray: array<'entity>,
-    queryToExecute: pgFn<array<'entity>, 'return>,
-    ~maxItemsPerQuery=500,
-  ): promise<array<'return>> => {
-    let responses = []
-    let i = ref(0)
-    let shouldContinue = () => i.contents < entityDataArray->Array.length
-    // Split entityDataArray into chunks of maxItemsPerQuery
-    while shouldContinue() {
-      let chunk =
-        entityDataArray->Js.Array2.slice(~start=i.contents, ~end_=i.contents + maxItemsPerQuery)
-      let response = queryToExecute(sql, chunk)
-      responses->Js.Array2.push(response)->ignore
-      i := i.contents + maxItemsPerQuery
-    }
-    Promise.all(responses)
-  }
-
-  let makeBatchSetFn = (~table, ~schema: S.t<'a>): batchSetFn<'a> => {
-    let batchSetFn: pgFn<array<Js.Json.t>, unit> = table->makeBatchSetFnString->eval
-    let parseOrThrow = S.compile(
-      S.array(schema),
-      ~input=Value,
-      ~output=Json,
-      ~mode=Sync,
-      ~typeValidation=true,
-    )
-    async (sql, rows) => {
-      let rowsJson = rows->parseOrThrow->(Utils.magic: Js.Json.t => array<Js.Json.t>)
-      let _res = await chunkBatchQuery(sql, rowsJson, batchSetFn)
-    }
-  }
-}

package/src/db/Table.res.js

@@ -72,11 +72,10 @@ function getFieldType(field) {
         );
 }
 
-function mkTable(tableName, schemaName, compositeIndicesOpt, fields) {
+function mkTable(tableName, compositeIndicesOpt, fields) {
   var compositeIndices = compositeIndicesOpt !== undefined ? compositeIndicesOpt : [];
   return {
           tableName: tableName,
-          schemaName: schemaName,
           fields: fields,
           compositeIndices: compositeIndices
         };
@@ -302,43 +301,6 @@ function getCompositeIndices(table) {
               }));
 }
 
-function makeBatchSetFnString(table) {
-  var fieldNamesInQuotes = Belt_Array.map(getNonDefaultFieldNames(table), (function (fieldName) {
-          return "\"" + fieldName + "\"";
-        }));
-  return "(sql, rows) => {\n      return sql\`\n        INSERT INTO \"" + table.schemaName + "\".\"" + table.tableName + "\"\n        \${sql(rows, " + fieldNamesInQuotes.join(", ") + ")}\n        ON CONFLICT(" + getPrimaryKeyFieldNames(table).join(", ") + ") DO UPDATE\n        SET\n        " + Belt_Array.map(fieldNamesInQuotes, (function (fieldNameInQuotes) {
-                  return fieldNameInQuotes + " = EXCLUDED." + fieldNameInQuotes;
-                })).join(", ") + ";\`\n    }";
-}
-
-function chunkBatchQuery(sql, entityDataArray, queryToExecute, maxItemsPerQueryOpt) {
-  var maxItemsPerQuery = maxItemsPerQueryOpt !== undefined ? maxItemsPerQueryOpt : 500;
-  var responses = [];
-  var i = 0;
-  while(i < entityDataArray.length) {
-    var chunk = entityDataArray.slice(i, i + maxItemsPerQuery | 0);
-    var response = queryToExecute(sql, chunk);
-    responses.push(response);
-    i = i + maxItemsPerQuery | 0;
-  };
-  return Promise.all(responses);
-}
-
-function makeBatchSetFn(table, schema) {
-  var batchSetFn = eval(makeBatchSetFnString(table));
-  var parseOrThrow = S$RescriptSchema.compile(S$RescriptSchema.array(schema), "Output", "Json", "Sync", true);
-  return async function (sql, rows) {
-    var rowsJson = parseOrThrow(rows);
-    await chunkBatchQuery(sql, rowsJson, batchSetFn, undefined);
-  };
-}
-
-var PostgresInterop = {
-  makeBatchSetFnString: makeBatchSetFnString,
-  chunkBatchQuery: chunkBatchQuery,
-  makeBatchSetFn: makeBatchSetFn
-};
-
 exports.mkField = mkField;
 exports.mkDerivedFromField = mkDerivedFromField;
 exports.getUserDefinedFieldName = getUserDefinedFieldName;
@@ -361,5 +323,4 @@ exports.getUnfilteredCompositeIndicesUnsafe = getUnfilteredCompositeIndicesUnsaf
 exports.toSqlParams = toSqlParams;
 exports.getSingleIndices = getSingleIndices;
 exports.getCompositeIndices = getCompositeIndices;
-exports.PostgresInterop = PostgresInterop;
 /* BigInt Not a pure module */