envio 2.12.0 → 2.12.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.12.0",
+  "version": "v2.12.1",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "repository": {
@@ -23,15 +23,15 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.12.0",
-    "envio-linux-arm64": "v2.12.0",
-    "envio-darwin-x64": "v2.12.0",
-    "envio-darwin-arm64": "v2.12.0"
+    "envio-linux-x64": "v2.12.1",
+    "envio-linux-arm64": "v2.12.1",
+    "envio-darwin-x64": "v2.12.1",
+    "envio-darwin-arm64": "v2.12.1"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.3",
     "rescript": "11.1.3",
-    "rescript-schema": "8.1.0",
+    "rescript-schema": "9.1.0",
     "viem": "2.21.0"
   },
   "files": [

package/src/Enum.res

@@ -9,7 +9,7 @@ type enum<'a> = {
 let make = (~name, ~variants) => {
   name,
   variants,
-  schema: Utils.Schema.enum(variants),
+  schema: S.enum(variants),
   default: switch variants->Belt.Array.get(0) {
   | Some(v) => v
   | None => Js.Exn.raiseError("No variants defined for enum " ++ name)

package/src/Internal.res

@@ -123,3 +123,5 @@ let fuelTransferParamsSchema = S.schema(s => {
   assetId: s.matches(S.string),
   amount: s.matches(BigInt.schema),
 })
+
+type entity = private {id: string}

package/src/ReorgDetection.res

@@ -72,7 +72,7 @@ module LastBlockScannedHashes: {
     int,
   >
 
-  let getAllBlockNumbers: t => Belt.Array.t<int>
+  let getThresholdBlockNumbers: (t, ~currentBlockHeight: int) => array<int>
 
   let hasReorgOccurred: (t, ~reorgGuard: reorgGuard) => bool
 
@@ -399,10 +399,16 @@ module LastBlockScannedHashes: {
     }
   }
 
-  let getAllBlockNumbers = (self: t) =>
-    self.lastBlockScannedDataList->Belt.List.reduceReverse([], (acc, v) => {
-      Belt.Array.concat(acc, [v.blockNumber])
+  let getThresholdBlockNumbers = (self: t, ~currentBlockHeight) => {
+    let blockNumbers = []
+    let thresholdBlocknumber = currentBlockHeight - self.confirmedBlockThreshold
+    self.lastBlockScannedDataList->Belt.List.forEach(v => {
+      if v.blockNumber >= thresholdBlocknumber {
+        blockNumbers->Belt.Array.push(v.blockNumber)
+      }
     })
+    blockNumbers
+  }
 
   /**
   Checks whether reorg has occured by comparing the parent hash with the last saved block hash.

package/src/Utils.res

@@ -266,14 +266,7 @@ let unwrapResultExn = res =>
 external queueMicrotask: (unit => unit) => unit = "queueMicrotask"
 
 module Schema = {
-  let enum = items => S.union(items->Belt.Array.mapU(S.literal))
-
-  // A hot fix after we use the version where it's supported
-  // https://github.com/DZakh/rescript-schema/blob/v8.4.0/docs/rescript-usage.md#removetypevalidation
-  let removeTypeValidationInPlace = schema => {
-    // The variables input is guaranteed to be an object, so we reset the rescript-schema type filter here
-    (schema->Obj.magic)["f"] = ()
-  }
+  let enum = S.enum
 
   let getNonOptionalFieldNames = schema => {
     let acc = []
@@ -300,6 +293,13 @@ module Schema = {
     }
   }
 
+  // Don't use S.unknown, since it's not serializable to json
+  // In a nutshell, this is completely unsafe.
+  let dbDate =
+    S.json(~validate=false)
+    ->(magic: S.t<Js.Json.t> => S.t<Js.Date.t>)
+    ->S.preprocess(_ => {serializer: date => date->magic->Js.Date.toISOString})
+
   // When trying to serialize data to Json pg type, it will fail with
   // PostgresError: column "params" is of type json but expression is of type boolean
   // If there's bool or null on the root level. It works fine as object field values.

package/src/bindings/BigInt.res

@@ -56,14 +56,4 @@ let schema =
     serializer: bigint => bigint->toString,
   })
 
-let nativeSchema: S.t<bigint> = S.custom("BigInt", s => {
-  {
-    parser: unknown => {
-      if Js.typeof(unknown) !== "bigint" {
-        s.fail("Expected bigint")
-      } else {
-        unknown->Obj.magic
-      }
-    },
-  }
-})
+let nativeSchema = S.bigint

package/src/bindings/Ethers.res

@@ -185,15 +185,6 @@ module JsonRpcProvider = {
     fields->Obj.magic
   }
 
-  type listenerEvent = [#block]
-  @send external onEventListener: (t, listenerEvent, int => unit) => unit = "on"
-
-  @send external offAllEventListeners: (t, listenerEvent) => unit = "off"
-
-  let onBlock = (t, callback: int => unit) => t->onEventListener(#block, callback)
-
-  let removeOnBlockEventListener = t => t->offAllEventListeners(#block)
-
   @send
   external getBlockNumber: t => promise<int> = "getBlockNumber"
 

package/src/bindings/Postgres.res

@@ -52,7 +52,7 @@ type sslOptions =
   | @as("prefer") Prefer
   | @as("verify-full") VerifyFull
 
-let sslOptionsSchema: S.schema<sslOptions> = Utils.Schema.enum([
+let sslOptionsSchema: S.schema<sslOptions> = S.enum([
   Bool(true),
   Bool(false),
   Require,
@@ -96,3 +96,6 @@ external makeSql: (~config: poolConfig) => sql = "postgres"
 // external sql:  array<string>  => (sql, array<string>) => int = "sql"
 
 @send external unsafe: (sql, string) => promise<'a> = "unsafe"
+@send
+external preparedUnsafe: (sql, string, unknown, @as(json`{prepare: true}`) _) => promise<'a> =
+  "unsafe"

package/src/db/EntityHistory.res

@@ -144,7 +144,7 @@ let insertRow = (
   ~historyRow: historyRow<'entity>,
   ~shouldCopyCurrentEntity,
 ) => {
-  let row = historyRow->S.serializeOrRaiseWith(self.schema)
+  let row = historyRow->S.reverseConvertToJsonOrThrow(self.schema)
   self.insertFn(sql, row, ~shouldCopyCurrentEntity)
 }
 
@@ -155,7 +155,9 @@ let batchInsertRows = (
   ~shouldCopyCurrentEntity,
 ) => {
   let rows =
-    rows->S.serializeOrRaiseWith(self.schemaRows)->(Utils.magic: Js.Json.t => array<Js.Json.t>)
+    rows
+    ->S.reverseConvertToJsonOrThrow(self.schemaRows)
+    ->(Utils.magic: Js.Json.t => array<Js.Json.t>)
   rows
   ->Belt.Array.map(row => {
     self.insertFn(sql, row, ~shouldCopyCurrentEntity)

package/src/db/Table.res

@@ -80,6 +80,10 @@ let getFieldName = fieldOrDerived =>
   | DerivedFrom({fieldName}) => fieldName
   }
 
+let getFieldType = (field: field) => {
+  (field.fieldType :> string) ++ (field.isArray ? "[]" : "")
+}
+
 type table = {
   tableName: string,
   fields: array<fieldOrDerived>,
@@ -112,6 +116,10 @@ let getFields = table =>
     }
   )
 
+let getFieldNames = table => {
+  table->getFields->Array.map(getDbFieldName)
+}
+
 let getNonDefaultFields = table =>
   table.fields->Array.keepMap(field =>
     switch field {
@@ -138,16 +146,20 @@ let getDerivedFromFields = table =>
     }
   )
 
-let getFieldNames = table => {
-  table->getFields->Array.map(getDbFieldName)
-}
-
 let getNonDefaultFieldNames = table => {
   table->getNonDefaultFields->Array.map(getDbFieldName)
 }
 
-let getFieldByName = (table, fieldNameSearch) =>
-  table.fields->Js.Array2.find(field => field->getUserDefinedFieldName == fieldNameSearch)
+let getFieldByName = (table, fieldName) =>
+  table.fields->Js.Array2.find(field => field->getUserDefinedFieldName === fieldName)
+
+let getFieldByDbName = (table, dbFieldName) =>
+  table.fields->Js.Array2.find(field =>
+    switch field {
+    | Field(f) => f->getDbFieldName
+    | DerivedFrom({fieldName}) => fieldName
+    } === dbFieldName
+  )
 
 exception NonExistingTableField(string)
 
@@ -166,6 +178,92 @@ let getUnfilteredCompositeIndicesUnsafe = (table): array<array<string>> => {
   )
 }
 
+type sqlParams<'entity> = {
+  dbSchema: S.t<'entity>,
+  quotedFieldNames: array<string>,
+  quotedNonPrimaryFieldNames: array<string>,
+  arrayFieldTypes: array<string>,
+  hasArrayField: bool,
+}
+
+let toSqlParams = (table: table, ~schema) => {
+  let quotedFieldNames = []
+  let quotedNonPrimaryFieldNames = []
+  let arrayFieldTypes = []
+  let hasArrayField = ref(false)
+
+  let dbSchema: S.t<Js.Dict.t<unknown>> = S.schema(s =>
+    switch schema->S.classify {
+    | Object({items}) =>
+      let dict = Js.Dict.empty()
+      items->Belt.Array.forEach(({location, inlinedLocation, schema}) => {
+        let rec coerceSchema = schema =>
+          switch schema->S.classify {
+          | BigInt => BigInt.schema->S.toUnknown
+          | Option(child)
+          | Null(child) =>
+            S.null(child->coerceSchema)->S.toUnknown
+          | Array(child) => {
+              hasArrayField := true
+              S.array(child->coerceSchema)->S.toUnknown
+            }
+          | JSON(_) => {
+              hasArrayField := true
+              schema
+            }
+          | Bool =>
+            // Workaround for https://github.com/porsager/postgres/issues/471
+            S.union([
+              S.literal("t")->S.to(_ => true),
+              S.literal("f")->S.to(_ => false),
+            ])->S.toUnknown
+          | _ => schema
+          }
+
+        let field = switch table->getFieldByDbName(location) {
+        | Some(field) => field
+        | None => raise(NonExistingTableField(location))
+        }
+
+        quotedFieldNames
+        ->Js.Array2.push(inlinedLocation)
+        ->ignore
+        switch field {
+        | Field({isPrimaryKey: false}) =>
+          quotedNonPrimaryFieldNames
+          ->Js.Array2.push(inlinedLocation)
+          ->ignore
+        | _ => ()
+        }
+
+        arrayFieldTypes
+        ->Js.Array2.push(
+          switch field {
+          | Field(f) =>
+            switch f.fieldType {
+            | Custom(fieldType) => `${(Text :> string)}[]::${(fieldType :> string)}`
+            | fieldType => (fieldType :> string)
+            }
+          | DerivedFrom(_) => (Text :> string)
+          } ++ "[]",
+        )
+        ->ignore
+        dict->Js.Dict.set(location, s.matches(schema->coerceSchema))
+      })
+      dict
+    | _ => Js.Exn.raiseError("Failed creating db schema. Expected an object schema for table")
+    }
+  )
+
+  {
+    dbSchema: dbSchema->(Utils.magic: S.t<dict<unknown>> => S.t<'entity>),
+    quotedFieldNames,
+    quotedNonPrimaryFieldNames,
+    arrayFieldTypes,
+    hasArrayField: hasArrayField.contents,
+  }
+}
+
 /*
 Gets all single indicies
 And maps the fields defined to their actual db name (some have _id suffix)
@@ -240,11 +338,17 @@ module PostgresInterop = {
     Promise.all(responses)
   }
 
-  let makeBatchSetFn = (~table, ~rowsSchema: S.t<array<'a>>): batchSetFn<'a> => {
+  let makeBatchSetFn = (~table, ~schema: S.t<'a>): batchSetFn<'a> => {
     let batchSetFn: pgFn<array<Js.Json.t>, unit> = table->makeBatchSetFnString->eval
+    let parseOrThrow = S.compile(
+      S.array(schema),
+      ~input=Value,
+      ~output=Json,
+      ~mode=Sync,
+      ~typeValidation=true,
+    )
     async (sql, rows) => {
-      let rowsJson =
-        rows->S.serializeOrRaiseWith(rowsSchema)->(Utils.magic: Js.Json.t => array<Js.Json.t>)
+      let rowsJson = rows->parseOrThrow->(Utils.magic: Js.Json.t => array<Js.Json.t>)
       let _res = await chunkBatchQuery(sql, rowsJson, batchSetFn)
     }
   }

package/src/sources/HyperSyncJsonApi.res

@@ -22,7 +22,7 @@ module QueryTypes = {
     | @as("uncles") Uncles
     | @as("base_fee_per_gas") BaseFeePerGas
 
-  let blockFieldOptionsSchema = Utils.Schema.enum([
+  let blockFieldOptionsSchema = S.enum([
     Number,
     Hash,
     ParentHash,
@@ -76,7 +76,7 @@ module QueryTypes = {
     | @as("status") Status
     | @as("sighash") Sighash
 
-  let transactionFieldOptionsSchema = Utils.Schema.enum([
+  let transactionFieldOptionsSchema = S.enum([
     BlockHash,
     BlockNumber,
     From,
@@ -123,7 +123,7 @@ module QueryTypes = {
     | @as("topic2") Topic2
     | @as("topic3") Topic3
 
-  let logFieldOptionsSchema = Utils.Schema.enum([
+  let logFieldOptionsSchema = S.enum([
     Removed,
     LogIndex,
     TransactionIndex,

package/src/vendored/Rest.res

@@ -34,6 +34,7 @@ module Promise = {
 
 module Option = {
   let unsafeSome: 'a => option<'a> = Obj.magic
+  let unsafeUnwrap: option<'a> => 'a = Obj.magic
 }
 
 module Dict = {
@@ -328,6 +329,28 @@ let coerceSchema = schema => {
   })
 }
 
+let stripInPlace = schema => (schema->S.classify->Obj.magic)["unknownKeys"] = S.Strip
+let getSchemaField = (schema, fieldName): option<S.item> =>
+  (schema->S.classify->Obj.magic)["fields"]->Js.Dict.unsafeGet(fieldName)
+
+type typeValidation = (unknown, ~inputVar: string) => string
+let removeTypeValidationInPlace = schema => (schema->Obj.magic)["f"] = ()
+let setTypeValidationInPlace = (schema, typeValidation: typeValidation) =>
+  (schema->Obj.magic)["f"] = typeValidation
+let unsafeGetTypeValidationInPlace = (schema): typeValidation => (schema->Obj.magic)["f"]
+
+let isNestedFlattenSupported = schema =>
+  switch schema->S.classify {
+  | Object({advanced: false}) =>
+    switch schema
+    ->S.reverse
+    ->S.classify {
+    | Object({advanced: false}) => true
+    | _ => false
+    }
+  | _ => false
+  }
+
 let bearerAuthSchema = S.string->S.transform(s => {
   serializer: token => {
     `Bearer ${token}`
@@ -372,10 +395,14 @@ let params = route => {
       let variablesSchema = S.object(s => {
         routeDefinition.variables({
           field: (fieldName, schema) => {
-            s.nestedField("body", fieldName, schema)
+            s.nested("body").field(fieldName, schema)
           },
           body: schema => {
-            s.field("body", schema)
+            if schema->isNestedFlattenSupported {
+              s.nested("body").flatten(schema)
+            } else {
+              s.field("body", schema)
+            }
           },
           rawBody: schema => {
             let isNonStringBased = switch schema->S.classify {
@@ -390,20 +417,19 @@ let params = route => {
             s.field("body", schema)
           },
           header: (fieldName, schema) => {
-            s.nestedField("headers", fieldName->Js.String2.toLowerCase, coerceSchema(schema))
+            s.nested("headers").field(fieldName->Js.String2.toLowerCase, coerceSchema(schema))
           },
           query: (fieldName, schema) => {
-            s.nestedField("query", fieldName, coerceSchema(schema))
+            s.nested("query").field(fieldName, coerceSchema(schema))
           },
           param: (fieldName, schema) => {
             if !Dict.has(pathParams, fieldName) {
               panic(`Path parameter "${fieldName}" is not defined in the path`)
             }
-            s.nestedField("params", fieldName, coerceSchema(schema))
+            s.nested("params").field(fieldName, coerceSchema(schema))
           },
           auth: auth => {
-            s.nestedField(
-              "headers",
+            s.nested("headers").field(
               "authorization",
               switch auth {
               | Bearer => bearerAuthSchema
@@ -416,14 +442,22 @@ let params = route => {
 
       {
         // The variables input is guaranteed to be an object, so we reset the rescript-schema type filter here
-        (variablesSchema->Obj.magic)["f"] = ()
-        (variablesSchema->S.classify->Obj.magic)["unknownKeys"] = S.Strip
-        let items: array<S.item> = (variablesSchema->S.classify->Obj.magic)["items"]
-        items->Js.Array2.forEach(item => {
-          let schema = item.schema
-          // Remove ${inputVar}.constructor!==Object check
-          (schema->Obj.magic)["f"] = (_b, ~inputVar) => `!${inputVar}`
-        })
+        variablesSchema->stripInPlace
+        variablesSchema->removeTypeValidationInPlace
+        switch variablesSchema->getSchemaField("headers") {
+        | Some({schema}) =>
+          schema->stripInPlace
+          schema->removeTypeValidationInPlace
+        | None => ()
+        }
+        switch variablesSchema->getSchemaField("params") {
+        | Some({schema}) => schema->removeTypeValidationInPlace
+        | None => ()
+        }
+        switch variablesSchema->getSchemaField("query") {
+        | Some({schema}) => schema->removeTypeValidationInPlace
+        | None => ()
+        }
       }
 
       let responsesMap = Js.Dict.empty()
@@ -443,14 +477,18 @@ let params = route => {
             description: d => builder.description = Some(d),
             field: (fieldName, schema) => {
               builder.emptyData = false
-              s.nestedField("data", fieldName, schema)
+              s.nested("data").field(fieldName, schema)
             },
             data: schema => {
               builder.emptyData = false
-              s.field("data", schema)
+              if schema->isNestedFlattenSupported {
+                s.nested("data").flatten(schema)
+              } else {
+                s.field("data", schema)
+              }
             },
             header: (fieldName, schema) => {
-              s.nestedField("headers", fieldName->Js.String2.toLowerCase, coerceSchema(schema))
+              s.nested("headers").field(fieldName->Js.String2.toLowerCase, coerceSchema(schema))
             },
           })
           if builder.emptyData {
@@ -461,8 +499,25 @@ let params = route => {
         if builder.status === None {
           responsesMap->Response.register(#default, builder)
         }
-        (schema->S.classify->Obj.magic)["unknownKeys"] = S.Strip
-        builder.dataSchema = (schema->S.classify->Obj.magic)["fields"]["data"]["t"]
+        schema->stripInPlace
+        schema->removeTypeValidationInPlace
+        let dataSchema = (schema->getSchemaField("data")->Option.unsafeUnwrap).schema
+        builder.dataSchema = dataSchema->Option.unsafeSome
+        switch dataSchema->S.classify {
+        | Literal(_) => {
+            let dataTypeValidation = dataSchema->unsafeGetTypeValidationInPlace
+            schema->setTypeValidationInPlace((b, ~inputVar) =>
+              dataTypeValidation(b, ~inputVar=`${inputVar}.data`)
+            )
+          }
+        | _ => ()
+        }
+        switch schema->getSchemaField("headers") {
+        | Some({schema}) =>
+          schema->stripInPlace
+          schema->removeTypeValidationInPlace
+        | None => ()
+        }
         builder.schema = Option.unsafeSome(schema)
         responses
         ->Js.Array2.push(builder->(Obj.magic: Response.builder<unknown> => Response.t<unknown>))
@@ -606,7 +661,7 @@ let fetch = (
 
   let {definition, variablesSchema, responsesMap, pathItems, isRawBody} = route->params
 
-  let data = variables->S.serializeToUnknownOrRaiseWith(variablesSchema)->Obj.magic
+  let data = variables->S.reverseConvertOrThrow(variablesSchema)->Obj.magic
 
   if data["body"] !== %raw(`void 0`) {
     if !isRawBody {
@@ -643,9 +698,20 @@ let fetch = (
 
       panic(error.contents)
     | Some(response) =>
-      fetcherResponse
-      ->S.parseAnyOrRaiseWith(response.schema)
-      ->(Obj.magic: unknown => response)
+      try fetcherResponse
+      ->S.parseOrThrow(response.schema)
+      ->(Obj.magic: unknown => response) catch {
+      | S.Raised({path, code: InvalidType({expected, received})}) if path === S.Path.empty =>
+        panic(
+          `Failed parsing response data. Reason: Expected ${(
+              expected->getSchemaField("data")->Option.unsafeUnwrap
+            ).schema->S.name}, received ${(received->Obj.magic)["data"]->Obj.magic}`,
+        )
+      | S.Raised(error) =>
+        panic(
+          `Failed parsing response at ${error.path->S.Path.toString}. Reason: ${error->S.Error.reason}`,
+        )
+      }
     }
   })
 }