envio 3.3.1 → 3.4.0

package/src/bindings/ClickHouse.res

@@ -353,6 +353,72 @@ let makeCreateHistoryTableQuery = (
     }
   })
 
+  let (partitionBy, orderBy, ttl) = switch entityConfig.storage.clickhouseOptions {
+  | Some(options) => (options.partitionBy, options.orderBy, options.ttl)
+  | None => (None, None, None)
+  }
+
+  // Schema field name -> ClickHouse column name, so @storage(clickhouse: {...})
+  // options can reference fields the way they're written in the schema and get
+  // renames (`column_name_format: snake_case`) and linked-entity `_id` suffixes
+  // resolved here.
+  let columnByFieldName = Dict.make()
+  entityConfig.table.fields->Array.forEach(field =>
+    switch field {
+    | Field(f) => columnByFieldName->Dict.set(f.fieldName, f->Table.getClickHouseDbFieldName)
+    | DerivedFrom(_) => ()
+    }
+  )
+
+  let orderByColumns = switch orderBy {
+  | Some(fieldNames) =>
+    // envio_checkpoint_id stays appended so the sorting key keeps a
+    // deterministic tie-break and the view's checkpoint dedup gets a clean
+    // ascending run per prefix. id is dropped: ClickHouse entities are
+    // read-only, so nothing looks history rows up by id.
+    let userColumns =
+      fieldNames
+      ->Array.map(fieldName =>
+        switch columnByFieldName->Dict.get(fieldName) {
+        | Some(column) => `\`${column}\``
+        | None =>
+          // Validated at codegen, so a miss means the schema and the
+          // persisted config diverged.
+          JsError.throwWithMessage(
+            `ClickHouse orderBy field "${fieldName}" is not defined on entity "${entityConfig.name}"`,
+          )
+        }
+      )
+      ->Array.joinUnsafe(", ")
+    `${userColumns}, ${EntityHistory.checkpointIdFieldName}`
+  | None => `${Table.idFieldName}, ${EntityHistory.checkpointIdFieldName}`
+  }
+
+  // partitionBy/ttl are raw ClickHouse expressions. Rewrite any bare identifier
+  // that names an entity field to that field's ClickHouse column, leaving
+  // functions, keywords, numbers, string literals and already-backticked
+  // identifiers untouched (a quoted token never matches a bare field name).
+  let resolveExpressionColumns = expression =>
+    expression->String.replaceRegExpBy0Unsafe(/'[^']*'|`[^`]*`|[A-Za-z_][A-Za-z0-9_]*/g, (
+      ~match,
+      ~offset as _,
+      ~input as _,
+    ) =>
+      switch columnByFieldName->Dict.get(match) {
+      | Some(column) => `\`${column}\``
+      | None => match
+      }
+    )
+
+  let partitionByClause = switch partitionBy {
+  | Some(expression) => `\nPARTITION BY ${expression->resolveExpressionColumns}`
+  | None => ""
+  }
+  let ttlClause = switch ttl {
+  | Some(expression) => `\nTTL ${expression->resolveExpressionColumns}`
+  | None => ""
+  }
+
   `CREATE TABLE IF NOT EXISTS ${database}.\`${EntityHistory.historyTableName(
       ~entityName=entityConfig.name,
       ~entityIndex=entityConfig.index,
@@ -369,8 +435,8 @@ let makeCreateHistoryTableQuery = (
       ~isArray=false,
     )}
 )
-ENGINE = ${tableEngine}
-ORDER BY (${Table.idFieldName}, ${EntityHistory.checkpointIdFieldName})`
+ENGINE = ${tableEngine}${partitionByClause}
+ORDER BY (${orderByColumns})${ttlClause}`
 }
 
 // Generate CREATE TABLE query for checkpoints

package/src/bindings/ClickHouse.res.mjs

@@ -260,6 +260,51 @@ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onCl
     let clickHouseType = getClickHouseFieldType(field$1.fieldType, field$1.isNullable, field$1.isArray);
     return `\`` + fieldName + `\` ` + clickHouseType;
   });
+  let options = entityConfig.storage.clickhouseOptions;
+  let match = options !== undefined ? [
+      options.partitionBy,
+      options.orderBy,
+      options.ttl
+    ] : [
+      undefined,
+      undefined,
+      undefined
+    ];
+  let ttl = match[2];
+  let orderBy = match[1];
+  let partitionBy = match[0];
+  let columnByFieldName = {};
+  entityConfig.table.fields.forEach(field => {
+    if (field.TAG !== "Field") {
+      return;
+    }
+    let f = field._0;
+    columnByFieldName[f.fieldName] = Table.getClickHouseDbFieldName(f);
+  });
+  let orderByColumns;
+  if (orderBy !== undefined) {
+    let userColumns = orderBy.map(fieldName => {
+      let column = columnByFieldName[fieldName];
+      if (column !== undefined) {
+        return `\`` + column + `\``;
+      } else {
+        return Stdlib_JsError.throwWithMessage(`ClickHouse orderBy field "` + fieldName + `" is not defined on entity "` + entityConfig.name + `"`);
+      }
+    }).join(", ");
+    orderByColumns = userColumns + `, ` + EntityHistory.checkpointIdFieldName;
+  } else {
+    orderByColumns = Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName;
+  }
+  let resolveExpressionColumns = expression => expression.replace(/'[^']*'|`[^`]*`|[A-Za-z_][A-Za-z0-9_]*/g, (match, param, param$1) => {
+    let column = columnByFieldName[match];
+    if (column !== undefined) {
+      return `\`` + column + `\``;
+    } else {
+      return match;
+    }
+  });
+  let partitionByClause = partitionBy !== undefined ? `\nPARTITION BY ` + resolveExpressionColumns(partitionBy) : "";
+  let ttlClause = ttl !== undefined ? `\nTTL ` + resolveExpressionColumns(ttl) : "";
   return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\`` + (
     onCluster ? ` ON CLUSTER '{cluster}'` : ""
   ) + ` (
@@ -270,8 +315,8 @@ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onCl
     config: EntityHistory.RowAction.config
   }, false, false) + `
 )
-ENGINE = ` + tableEngine + `
-ORDER BY (` + Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName + `)`;
+ENGINE = ` + tableEngine + partitionByClause + `
+ORDER BY (` + orderByColumns + `)` + ttlClause;
 }
 
 function makeCreateCheckpointsTableQuery(database, replicatedOpt, onClusterOpt) {

package/src/sources/EvmChain.res

@@ -48,7 +48,7 @@ let makeSources = (
 ) => {
   let sources = switch hyperSync {
   | Some(endpointUrl) => [
-      HyperSyncSource.make({
+      EvmHyperSyncSource.make({
         chain,
         endpointUrl,
         onEventRegistrations,

package/src/sources/EvmChain.res.mjs

@@ -3,7 +3,7 @@
 import * as Env from "../Env.res.mjs";
 import * as RpcSource from "./RpcSource.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
-import * as HyperSyncSource from "./HyperSyncSource.res.mjs";
+import * as EvmHyperSyncSource from "./EvmHyperSyncSource.res.mjs";
 
 function getSyncConfig(param) {
   let queryTimeoutMillis = Stdlib_Option.getOr(param.queryTimeoutMillis, 20000);
@@ -20,7 +20,7 @@ function getSyncConfig(param) {
 }
 
 function makeSources(chain, onEventRegistrations, hyperSync, rpcs, lowercaseAddresses) {
-  let sources = hyperSync !== undefined ? [HyperSyncSource.make({
+  let sources = hyperSync !== undefined ? [EvmHyperSyncSource.make({
         chain: chain,
         endpointUrl: hyperSync,
         onEventRegistrations: onEventRegistrations,

package/src/sources/FuelHyperSync.res

@@ -0,0 +1,74 @@
+type logsQueryPage = {
+  items: array<FuelHyperSyncClient.EventItems.item>,
+  // Blocks referenced by `items`, one per height.
+  blocks: array<FuelHyperSyncClient.EventItems.block>,
+  nextBlock: int,
+  archiveHeight: int,
+}
+
+module GetLogs = {
+  type error =
+    | UnexpectedMissingParams({missingParams: array<string>})
+    | WrongInstance
+
+  exception Error(error)
+
+  // Rust encodes structured failures as a JSON payload in the napi error's
+  // message: `{"kind":"MissingFields","fields":["receipt.txId", ...]}`.
+  // JSON.parse + shape check is the recovery protocol — no string-grepping
+  // on anyhow's Debug format.
+  let extractMissingParams = (exn: exn): option<array<string>> => {
+    let message = switch exn {
+    | JsExn(jsExn) => jsExn->JsExn.message
+    | _ => None
+    }
+    switch message {
+    | None => None
+    | Some(msg) =>
+      switch msg->JSON.parseOrThrow->JSON.Decode.object {
+      | exception _ => None
+      | None => None
+      | Some(obj) =>
+        switch (obj->Dict.get("kind"), obj->Dict.get("fields")) {
+        | (Some(String("MissingFields")), Some(Array(fields))) =>
+          Some(fields->Array.filterMap(JSON.Decode.string))
+        | _ => None
+        }
+      }
+    }
+  }
+
+  let query = async (
+    ~client: FuelHyperSyncClient.t,
+    ~fromBlock,
+    ~toBlock,
+    ~registrationIndexes,
+    ~addressesByContractName,
+  ): logsQueryPage => {
+    let query: FuelHyperSyncClient.EventItems.query = {
+      fromBlock,
+      toBlock,
+      registrationIndexes,
+      addressesByContractName,
+    }
+
+    let res = switch await client->FuelHyperSyncClient.getEventItems(query) {
+    | res => res
+    | exception exn =>
+      switch exn->extractMissingParams {
+      | Some(missingParams) => throw(Error(UnexpectedMissingParams({missingParams: missingParams})))
+      | None => throw(exn)
+      }
+    }
+    if res.nextBlock <= fromBlock {
+      // Might happen when /height response was from another instance of HyperFuel
+      throw(Error(WrongInstance))
+    }
+    {
+      items: res.items,
+      blocks: res.blocks,
+      nextBlock: res.nextBlock,
+      archiveHeight: res.archiveHeight->Option.getOr(0), // TODO: FIXME: Shouldn't have a default here
+    }
+  }
+}

package/src/sources/FuelHyperSync.res.mjs

@@ -0,0 +1,80 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+let $$Error = /* @__PURE__ */Primitive_exceptions.create("FuelHyperSync.GetLogs.Error");
+
+function extractMissingParams(exn) {
+  let message = exn.RE_EXN_ID === "JsExn" ? Stdlib_JsExn.message(exn._1) : undefined;
+  if (message === undefined) {
+    return;
+  }
+  let obj;
+  try {
+    obj = Stdlib_JSON.Decode.object(JSON.parse(message));
+  } catch (exn$1) {
+    return;
+  }
+  if (obj === undefined) {
+    return;
+  }
+  let match = obj["kind"];
+  let match$1 = obj["fields"];
+  if (match === "MissingFields" && Array.isArray(match$1)) {
+    return Stdlib_Array.filterMap(match$1, Stdlib_JSON.Decode.string);
+  }
+}
+
+async function query(client, fromBlock, toBlock, registrationIndexes, addressesByContractName) {
+  let query$1 = {
+    fromBlock: fromBlock,
+    toBlock: toBlock,
+    registrationIndexes: registrationIndexes,
+    addressesByContractName: addressesByContractName
+  };
+  let res;
+  try {
+    res = await client.getEventItems(query$1);
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    let missingParams = extractMissingParams(exn);
+    if (missingParams !== undefined) {
+      throw {
+        RE_EXN_ID: $$Error,
+        _1: {
+          TAG: "UnexpectedMissingParams",
+          missingParams: missingParams
+        },
+        Error: new Error()
+      };
+    }
+    throw exn;
+  }
+  if (res.nextBlock <= fromBlock) {
+    throw {
+      RE_EXN_ID: $$Error,
+      _1: "WrongInstance",
+      Error: new Error()
+    };
+  }
+  return {
+    items: res.items,
+    blocks: res.blocks,
+    nextBlock: res.nextBlock,
+    archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0)
+  };
+}
+
+let GetLogs = {
+  $$Error: $$Error,
+  query: query
+};
+
+export {
+  GetLogs,
+}
+/* Stdlib_JsExn Not a pure module */

package/src/sources/FuelHyperSync.resi

@@ -0,0 +1,22 @@
+type logsQueryPage = {
+  items: array<FuelHyperSyncClient.EventItems.item>,
+  blocks: array<FuelHyperSyncClient.EventItems.block>,
+  nextBlock: int,
+  archiveHeight: int,
+}
+
+module GetLogs: {
+  type error =
+    | UnexpectedMissingParams({missingParams: array<string>})
+    | WrongInstance
+
+  exception Error(error)
+
+  let query: (
+    ~client: FuelHyperSyncClient.t,
+    ~fromBlock: int,
+    ~toBlock: option<int>,
+    ~registrationIndexes: array<int>,
+    ~addressesByContractName: dict<array<Address.t>>,
+  ) => promise<logsQueryPage>
+}

package/src/sources/FuelHyperSyncClient.res

@@ -0,0 +1,120 @@
+type t
+
+type cfg = {
+  url: string,
+  apiToken: string,
+}
+
+module Registration = {
+  type kind =
+    | @as("LogData") LogData
+    | @as("Mint") Mint
+    | @as("Burn") Burn
+    | @as("Transfer") Transfer
+    | @as("Call") Call
+
+  // The full per-(event, chain) registration passed to the Rust client at
+  // construction: routing identity plus the receipt-selection state queries
+  // are built from.
+  type input = {
+    // Chain-scoped sequential registration index, echoed back on routed items.
+    index: int,
+    eventName: string,
+    contractName: string,
+    isWildcard: bool,
+    kind: kind,
+    // The LogData `rb` value as a decimal string; absent for other kinds.
+    logId?: string,
+  }
+
+  let fromOnEventRegistrations = (
+    onEventRegistrations: array<Internal.fuelOnEventRegistration>,
+  ): array<input> =>
+    onEventRegistrations->Array.map(reg => {
+      let eventConfig =
+        reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.fuelEventConfig)
+      let (kind, logId) = switch eventConfig.kind {
+      | LogData({logId}) => (LogData, Some(logId))
+      | Mint => (Mint, None)
+      | Burn => (Burn, None)
+      | Transfer => (Transfer, None)
+      | Call => (Call, None)
+      }
+      {
+        index: reg.index,
+        eventName: eventConfig.name,
+        contractName: eventConfig.contractName,
+        isWildcard: reg.isWildcard,
+        kind,
+        ?logId,
+      }
+    })
+}
+
+module EventItems = {
+  // The whole per-query input: block range, the partition's registration
+  // selection (by index), and its current addresses. Receipt selections,
+  // field selection, and routing are derived on the Rust side.
+  type query = {
+    fromBlock: int,
+    // Inclusive; None queries to the end of available data.
+    toBlock: option<int>,
+    registrationIndexes: array<int>,
+    addressesByContractName: dict<array<Address.t>>,
+  }
+
+  // One routed receipt with its kind-specific columns flattened: LogData
+  // carries `data` (decoded here in JS), Mint/Burn carry `val`/`subId`,
+  // Transfer/TransferOut/Call carry `amount`/`assetId`/`to` (TransferOut's
+  // wallet recipient normalised into `to`).
+  type item = {
+    onEventRegistrationIndex: int,
+    receiptIndex: int,
+    txId: string,
+    blockHeight: int,
+    srcAddress: Address.t,
+    data?: string,
+    subId?: string,
+    val?: bigint,
+    amount?: bigint,
+    assetId?: string,
+    to?: string,
+  }
+
+  type block = {
+    id: string,
+    height: int,
+    time: int,
+  }
+
+  type response = {
+    archiveHeight: option<int>,
+    nextBlock: int,
+    // One block per height; items reference them by `blockHeight`.
+    blocks: array<block>,
+    items: array<item>,
+  }
+}
+
+@send
+external classNew: (
+  Core.fuelHyperSyncClientCtor,
+  cfg,
+  ~userAgent: string,
+  array<Registration.input>,
+) => t = "new"
+
+let make = (cfg: cfg, ~eventRegistrations) => {
+  let envioVersion = Utils.EnvioPackage.value.version
+  Core.getAddon().fuelHyperSyncClient->classNew(
+    cfg,
+    ~userAgent=`hyperindex/${envioVersion}`,
+    eventRegistrations,
+  )
+}
+
+@send
+external getEventItems: (t, EventItems.query) => promise<EventItems.response> = "getEventItems"
+
+@send
+external getHeight: t => promise<int> = "getHeight"

package/src/sources/FuelHyperSyncClient.res.mjs

@@ -0,0 +1,71 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Core from "../Core.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+
+function fromOnEventRegistrations(onEventRegistrations) {
+  return onEventRegistrations.map(reg => {
+    let eventConfig = reg.eventConfig;
+    let match = eventConfig.kind;
+    let match$1;
+    if (typeof match !== "object") {
+      switch (match) {
+        case "Mint" :
+          match$1 = [
+            "Mint",
+            undefined
+          ];
+          break;
+        case "Burn" :
+          match$1 = [
+            "Burn",
+            undefined
+          ];
+          break;
+        case "Transfer" :
+          match$1 = [
+            "Transfer",
+            undefined
+          ];
+          break;
+        case "Call" :
+          match$1 = [
+            "Call",
+            undefined
+          ];
+          break;
+      }
+    } else {
+      match$1 = [
+        "LogData",
+        match.logId
+      ];
+    }
+    return {
+      index: reg.index,
+      eventName: eventConfig.name,
+      contractName: eventConfig.contractName,
+      isWildcard: reg.isWildcard,
+      kind: match$1[0],
+      logId: match$1[1]
+    };
+  });
+}
+
+let Registration = {
+  fromOnEventRegistrations: fromOnEventRegistrations
+};
+
+let EventItems = {};
+
+function make(cfg, eventRegistrations) {
+  let envioVersion = Utils.EnvioPackage.value.version;
+  return Core.getAddon().FuelHyperSyncClient.new(cfg, `hyperindex/` + envioVersion, eventRegistrations);
+}
+
+export {
+  Registration,
+  EventItems,
+  make,
+}
+/* Core Not a pure module */