envio 3.9.0 → 3.10.0

package/src/bindings/ClickHouseSink.res

@@ -0,0 +1,337 @@
+type t
+
+type historySchema = {
+  idColumn: string,
+  checkpointIdColumn: string,
+  changeColumn: string,
+  changeVariants: array<string>,
+  setVariant: string,
+  checkpointsTable: string,
+  checkpointChainIdColumn: string,
+  checkpointBlockNumberColumn: string,
+}
+
+type chainProgressInput = {chainId: string, progressBlockNumber: int}
+
+type options = {
+  url: string,
+  username: string,
+  password: string,
+  database: string,
+  chainIdMode: string,
+  history: historySchema,
+}
+
+@send external classNew: (Core.clickHouseSinkCtor, options, string => unit) => t = "new"
+
+type columnSpec = {
+  name: string,
+  // Omitted when it matches `name`. Set when a column rename is configured, so
+  // `@storage(clickhouse: {...})` expressions can still name the schema field.
+  fieldName?: string,
+  fieldType: string,
+  isNullable?: bool,
+  isArray?: bool,
+  precision?: int,
+  scale?: int,
+  enumVariants?: array<string>,
+}
+
+type skippingIndexSpec = {
+  name: string,
+  expr: string,
+  indexType: string,
+  granularity?: int,
+}
+
+type entitySpec = {
+  name: string,
+  historyTable: string,
+  columns: array<columnSpec>,
+  chainIdColumn?: string,
+  partitionBy?: string,
+  orderBy?: array<string>,
+  ttl?: string,
+  skippingIndexes?: array<skippingIndexSpec>,
+}
+
+type initializeInput = {
+  entities: array<entitySpec>,
+  checkpointColumns: array<columnSpec>,
+  replicated: bool,
+  databaseEngine?: string,
+}
+
+type resumeInput = {
+  checkpointId: string,
+  chainProgress: array<chainProgressInput>,
+  historyTables: array<string>,
+  replicated: bool,
+  databaseEngine?: string,
+}
+
+type registeredTable = {
+  handle: int,
+  names: array<string>,
+  kinds: array<int>,
+  nullable: array<bool>,
+}
+
+type columnValuesInput = {
+  numbers?: Float64Array.t,
+  unsigned64?: BigUint64Array.t,
+  signed64?: BigInt64Array.t,
+  texts?: array<string>,
+  bytes?: array<Uint8Array.t>,
+  nulls?: Uint8Array.t,
+}
+
+@send
+external registerEntityTable: (t, entitySpec) => registeredTable = "registerEntityTable"
+
+@send
+external registerCheckpointsTable: (t, array<columnSpec>) => registeredTable =
+  "registerCheckpointsTable"
+
+@send external initialize: (t, initializeInput) => promise<unit> = "initialize"
+
+@send
+external resume: (t, resumeInput) => promise<unit> = "resume"
+
+@send
+external stage: (t, ~table: int, ~rows: int, ~columns: array<columnValuesInput>) => int = "stage"
+
+@send
+external writeBatch: (t, ~entities: array<int>, ~checkpoints: Null.t<int>) => promise<unit> =
+  "writeBatch"
+
+@send external discard: (t, array<int>) => unit = "discard"
+
+let historySchema = (): historySchema => {
+  idColumn: Table.idFieldName,
+  checkpointIdColumn: EntityHistory.checkpointIdFieldName,
+  changeColumn: EntityHistory.changeFieldName,
+  changeVariants: EntityHistory.RowAction.variants->Array.map(variant => (variant :> string)),
+  setVariant: (EntityHistory.RowAction.SET :> string),
+  checkpointsTable: InternalTable.Checkpoints.table.tableName,
+  checkpointChainIdColumn: (#chain_id: InternalTable.Checkpoints.field :> string),
+  checkpointBlockNumberColumn: (#block_number: InternalTable.Checkpoints.field :> string),
+}
+
+let make = (~url, ~username, ~password, ~database, ~chainIdMode: ChainId.mode, ~onWarning) =>
+  Core.getAddon().clickHouseSink->classNew(
+    {
+      url,
+      username,
+      password,
+      database,
+      chainIdMode: (chainIdMode :> string),
+      history: historySchema(),
+    },
+    onWarning,
+  )
+
+type kind =
+  | @as(0) F64
+  | @as(1) U64
+  | @as(2) I64
+  | @as(3) Text
+  | @as(4) Bytes
+
+let kindOfOrdinal = ordinal =>
+  switch ordinal {
+  | 0 => F64
+  | 1 => U64
+  | 2 => I64
+  | 3 => Text
+  | 4 => Bytes
+  | unknown => JsError.throwWithMessage(`Unknown ClickHouse column kind ${unknown->Int.toString}`)
+  }
+
+external asString: unknown => string = "%identity"
+@val external toNumber: unknown => float = "Number"
+@val external toBigInt: unknown => bigint = "BigInt"
+@val external stringOf: unknown => string = "String"
+
+// No column holds NaN or Infinity, but nothing downstream refuses them well: a
+// list renders one as `null` on the way into JSON text, which the column then
+// refuses for a reason naming `null` rather than what the handler wrote, and a
+// scalar Float64 takes the raw bytes and stores a value no reader can render.
+// Refused here instead, where the value is still itself, with one message for
+// both shapes.
+%%private(
+  let finiteOrThrow = (number: float, ~column) =>
+    if number->Float.isFinite {
+      number
+    } else {
+      JsError.throwWithMessage(
+        `${number->Float.toString} is not a finite number, so it cannot be stored in the \`${column}\` column. Store a finite number, or keep it out of the entity.`,
+      )
+    }
+)
+
+// Visits every node on the way into JSON text. A bigint would make
+// `JSON.stringify` throw and fail the whole batch, so it travels as its digits.
+%%private(
+  let jsonSafe = (~column) =>
+    (_, value: JSON.t) =>
+      switch value->typeof {
+      | #bigint =>
+        value->(Utils.magic: JSON.t => unknown)->stringOf->(Utils.magic: string => JSON.t)
+      // A Uint8Array inside a list column travels as its byte values, which the
+      // sink reads back into raw bytes.
+      | #object =>
+        switch value->(Utils.magic: JSON.t => unknown)->Utils.Bytes.asUint8Array {
+        | Some(bytes) =>
+          bytes
+          ->(Utils.magic: Uint8Array.t => Array.arrayLike<int>)
+          ->Array.fromArrayLike
+          ->(Utils.magic: array<int> => JSON.t)
+        | None => value
+        }
+      | #number =>
+        let _ = value->(Utils.magic: JSON.t => float)->finiteOrThrow(~column)
+        value
+      | _ => value
+      }
+)
+
+// The replacer closes over nothing but the column's name, so it is built once
+// with the column rather than once per column per batch.
+type column = {name: string, kind: kind, isNullable: bool, replacer: JSON.replacer}
+type table = {handle: int, name: string, columns: array<column>}
+
+let makeTable = (~name, {handle, names, kinds, nullable}: registeredTable) => {
+  handle,
+  name,
+  columns: names->Array.mapWithIndex((name, index) => {
+    name,
+    kind: kinds->Array.getUnsafe(index)->kindOfOrdinal,
+    isNullable: nullable->Array.getUnsafe(index),
+    replacer: Replacer(jsonSafe(~column=name)),
+  }),
+}
+
+let toText = (value: unknown, ~replacer) =>
+  switch value->typeof {
+  | #string => value->asString
+  | #bigint => value->stringOf
+  | _ => value->(Utils.magic: unknown => JSON.t)->JSON.stringify(~replacer)
+  }
+
+type builder = {
+  name: string,
+  kind: kind,
+  isNullable: bool,
+  floats: Float64Array.t,
+  unsigned: BigUint64Array.t,
+  signed: BigInt64Array.t,
+  texts: array<string>,
+  bytes: array<Uint8Array.t>,
+  replacer: JSON.replacer,
+  mutable nulls: option<Uint8Array.t>,
+  rows: int,
+}
+
+%%private(let noFloats = Float64Array.fromLength(0))
+%%private(let noUnsigned = BigUint64Array.fromLength(0))
+%%private(let noSigned = BigInt64Array.fromLength(0))
+%%private(let noBytes = Uint8Array.fromLength(0))
+
+let makeBuilder = ({name, kind, isNullable, replacer}: column, ~rows) => {
+  name,
+  kind,
+  isNullable,
+  floats: kind === F64 ? Float64Array.fromLength(rows) : noFloats,
+  unsigned: kind === U64 ? BigUint64Array.fromLength(rows) : noUnsigned,
+  signed: kind === I64 ? BigInt64Array.fromLength(rows) : noSigned,
+  texts: kind === Text ? Array.make(~length=rows, "") : [],
+  bytes: kind === Bytes ? Array.make(~length=rows, noBytes) : [],
+  replacer,
+  nulls: None,
+  rows,
+}
+
+let markNull = (builder, ~row) => {
+  let nulls = switch builder.nulls {
+  | Some(nulls) => nulls
+  | None =>
+    let nulls = Uint8Array.fromLength(builder.rows)
+    builder.nulls = Some(nulls)
+    nulls
+  }
+  nulls->TypedArray.set(row, 1)
+}
+
+// RowBinary carries the raw integer, so a value the column cannot hold is not
+// rejected anywhere downstream — and a typed array reduces it modulo 2^64 on
+// the way in rather than refusing it, which would leave no trace at all.
+%%private(
+  let checkedBigInt = (value: unknown, ~builder, ~min, ~max) => {
+    let value = value->toBigInt
+    if value < min || value > max {
+      JsError.throwWithMessage(
+        `${value->BigInt.toString} is out of range for the \`${builder.name}\` column`,
+      )
+    }
+    value
+  }
+)
+
+%%private(
+  let writePresent = (builder, ~row, value: unknown) =>
+    switch builder.kind {
+    | F64 =>
+      builder.floats->TypedArray.set(row, value->toNumber->finiteOrThrow(~column=builder.name))
+    | U64 =>
+      builder.unsigned->TypedArray.set(
+        row,
+        value->checkedBigInt(~builder, ~min=0n, ~max=18446744073709551615n),
+      )
+    | I64 =>
+      builder.signed->TypedArray.set(
+        row,
+        value->checkedBigInt(~builder, ~min=-9223372036854775808n, ~max=9223372036854775807n),
+      )
+    | Text => builder.texts->Array.setUnsafe(row, value->toText(~replacer=builder.replacer))
+    | Bytes => builder.bytes->Array.setUnsafe(row, value->(Utils.magic: unknown => Uint8Array.t))
+    }
+)
+
+%%private(let isAbsent = (value: unknown) => value === %raw(`undefined`) || value === %raw(`null`))
+
+// Writes one value of a row the handler set. `undefined`/`null` marks the row's
+// null bit, which a column that accepts NULL stores as such — and which one that
+// does not has no way to store: RowBinary carries no "absent", so the row would
+// land holding the type's zero, a value the handler never chose and that nothing
+// downstream could tell from one it did.
+let writeValue = (builder, ~row, value: unknown) =>
+  if value->isAbsent {
+    if builder.isNullable {
+      builder->markNull(~row)
+    } else {
+      JsError.throwWithMessage(
+        `No value for the \`${builder.name}\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`,
+      )
+    }
+  } else {
+    builder->writePresent(~row, value)
+  }
+
+let writeDeletedValue = (builder, ~row, value: unknown) =>
+  if value->isAbsent {
+    builder->markNull(~row)
+  } else {
+    builder->writePresent(~row, value)
+  }
+
+let builderPayload = (builder): columnValuesInput => {
+  let base = {nulls: ?builder.nulls}
+  switch builder.kind {
+  | F64 => {...base, numbers: builder.floats}
+  | U64 => {...base, unsigned64: builder.unsigned}
+  | I64 => {...base, signed64: builder.signed}
+  | Text => {...base, texts: builder.texts}
+  | Bytes => {...base, bytes: builder.bytes}
+  }
+}

package/src/bindings/ClickHouseSink.res.mjs

@@ -0,0 +1,246 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Core from "../Core.res.mjs";
+import * as Table from "../db/Table.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as EntityHistory from "../db/EntityHistory.res.mjs";
+import * as InternalTable from "../db/InternalTable.res.mjs";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+
+function historySchema() {
+  return {
+    idColumn: Table.idFieldName,
+    checkpointIdColumn: EntityHistory.checkpointIdFieldName,
+    changeColumn: EntityHistory.changeFieldName,
+    changeVariants: EntityHistory.RowAction.variants.map(variant => variant),
+    setVariant: "SET",
+    checkpointsTable: InternalTable.Checkpoints.table.tableName,
+    checkpointChainIdColumn: "chain_id",
+    checkpointBlockNumberColumn: "block_number"
+  };
+}
+
+function make(url, username, password, database, chainIdMode, onWarning) {
+  return Core.getAddon().ClickHouseSink.new({
+    url: url,
+    username: username,
+    password: password,
+    database: database,
+    chainIdMode: chainIdMode,
+    history: historySchema()
+  }, onWarning);
+}
+
+function kindOfOrdinal(ordinal) {
+  switch (ordinal) {
+    case 0 :
+      return 0;
+    case 1 :
+      return 1;
+    case 2 :
+      return 2;
+    case 3 :
+      return 3;
+    case 4 :
+      return 4;
+    default:
+      return Stdlib_JsError.throwWithMessage(`Unknown ClickHouse column kind ` + ordinal.toString());
+  }
+}
+
+function finiteOrThrow(number, column) {
+  if (isFinite(number)) {
+    return number;
+  } else {
+    return Stdlib_JsError.throwWithMessage(number.toString() + ` is not a finite number, so it cannot be stored in the \`` + column + `\` column. Store a finite number, or keep it out of the entity.`);
+  }
+}
+
+function jsonSafe(column) {
+  return (param, value) => {
+    let match = typeof value;
+    if (match === "bigint") {
+      return String(value);
+    }
+    if (match !== "object") {
+      if (match === "number") {
+        finiteOrThrow(value, column);
+        return value;
+      } else {
+        return value;
+      }
+    }
+    let bytes = Utils.Bytes.asUint8Array(value);
+    if (bytes !== undefined) {
+      return Array.from(bytes);
+    } else {
+      return value;
+    }
+  };
+}
+
+function makeTable(name, param) {
+  let nullable = param.nullable;
+  let kinds = param.kinds;
+  return {
+    handle: param.handle,
+    name: name,
+    columns: param.names.map((name, index) => ({
+      name: name,
+      kind: kindOfOrdinal(kinds[index]),
+      isNullable: nullable[index],
+      replacer: jsonSafe(name)
+    }))
+  };
+}
+
+function toText(value, replacer) {
+  let match = typeof value;
+  if (match === "bigint") {
+    return String(value);
+  } else if (match === "string") {
+    return value;
+  } else {
+    return JSON.stringify(value, replacer);
+  }
+}
+
+let noFloats = new Float64Array(0);
+
+let noUnsigned = new BigUint64Array(0);
+
+let noSigned = new BigInt64Array(0);
+
+let noBytes = new Uint8Array(0);
+
+function makeBuilder(param, rows) {
+  let kind = param.kind;
+  return {
+    name: param.name,
+    kind: kind,
+    isNullable: param.isNullable,
+    floats: kind === 0 ? new Float64Array(rows) : noFloats,
+    unsigned: kind === 1 ? new BigUint64Array(rows) : noUnsigned,
+    signed: kind === 2 ? new BigInt64Array(rows) : noSigned,
+    texts: kind === 3 ? Stdlib_Array.make(rows, "") : [],
+    bytes: kind === 4 ? Stdlib_Array.make(rows, noBytes) : [],
+    replacer: param.replacer,
+    nulls: undefined,
+    rows: rows
+  };
+}
+
+function markNull(builder, row) {
+  let nulls = builder.nulls;
+  let nulls$1;
+  if (nulls !== undefined) {
+    nulls$1 = nulls;
+  } else {
+    let nulls$2 = new Uint8Array(builder.rows);
+    builder.nulls = nulls$2;
+    nulls$1 = nulls$2;
+  }
+  nulls$1[row] = 1;
+}
+
+function checkedBigInt(value, builder, min, max) {
+  let value$1 = BigInt(value);
+  if (value$1 < min || value$1 > max) {
+    Stdlib_JsError.throwWithMessage(value$1.toString() + ` is out of range for the \`` + builder.name + `\` column`);
+  }
+  return value$1;
+}
+
+function writePresent(builder, row, value) {
+  let match = builder.kind;
+  switch (match) {
+    case 0 :
+      builder.floats[row] = finiteOrThrow(Number(value), builder.name);
+      return;
+    case 1 :
+      builder.unsigned[row] = checkedBigInt(value, builder, 0n, 18446744073709551615n);
+      return;
+    case 2 :
+      builder.signed[row] = checkedBigInt(value, builder, -9223372036854775808n, 9223372036854775807n);
+      return;
+    case 3 :
+      builder.texts[row] = toText(value, builder.replacer);
+      return;
+    case 4 :
+      builder.bytes[row] = value;
+      return;
+  }
+}
+
+function isAbsent(value) {
+  if (value === undefined) {
+    return true;
+  } else {
+    return value === null;
+  }
+}
+
+function writeValue(builder, row, value) {
+  if (isAbsent(value)) {
+    if (builder.isNullable) {
+      return markNull(builder, row);
+    } else {
+      return Stdlib_JsError.throwWithMessage(`No value for the \`` + builder.name + `\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`);
+    }
+  } else {
+    return writePresent(builder, row, value);
+  }
+}
+
+function writeDeletedValue(builder, row, value) {
+  if (isAbsent(value)) {
+    return markNull(builder, row);
+  } else {
+    return writePresent(builder, row, value);
+  }
+}
+
+function builderPayload(builder) {
+  let base_nulls = builder.nulls;
+  let base = {
+    nulls: base_nulls
+  };
+  let match = builder.kind;
+  switch (match) {
+    case 0 :
+      let newrecord = {...base};
+      newrecord.numbers = builder.floats;
+      return newrecord;
+    case 1 :
+      let newrecord$1 = {...base};
+      newrecord$1.unsigned64 = builder.unsigned;
+      return newrecord$1;
+    case 2 :
+      let newrecord$2 = {...base};
+      newrecord$2.signed64 = builder.signed;
+      return newrecord$2;
+    case 3 :
+      let newrecord$3 = {...base};
+      newrecord$3.texts = builder.texts;
+      return newrecord$3;
+    case 4 :
+      let newrecord$4 = {...base};
+      newrecord$4.bytes = builder.bytes;
+      return newrecord$4;
+  }
+}
+
+export {
+  historySchema,
+  make,
+  kindOfOrdinal,
+  makeTable,
+  toText,
+  makeBuilder,
+  markNull,
+  writeValue,
+  writeDeletedValue,
+  builderPayload,
+}
+/* noFloats Not a pure module */

package/src/bindings/EventSource.res

@@ -1,7 +1,13 @@
 type t
 
 module Fetch = {
-  type args = {body?: unknown, headers?: dict<string>, method?: string, path?: string}
+  type args = {
+    body?: unknown,
+    headers?: dict<string>,
+    method?: string,
+    path?: string,
+    signal?: unknown,
+  }
   type t = (string, ~args: args) => promise<unknown>
   // NOTE: don't try make the type t. Rescript 11 will curry the args which breaks
   // keet the type inline. This is a workaround for now.
@@ -14,7 +20,7 @@ external create: (~url: string, ~options: options=?) => t = "EventSource"
 
 @set external onopen: (t, unit => unit) => unit = "onopen"
 
-type errorEvent = {status: option<int>, message: option<string>}
+type errorEvent = {code: option<int>, message: option<string>}
 @set external onerror: (t, errorEvent => unit) => unit = "onerror"
 
 type event = {data: string}

package/src/bindings/NodeJs.res

@@ -72,6 +72,12 @@ module Buffer = {
   @val @scope("Buffer") external alloc: int => t = "alloc"
   @get external length: t => int = "length"
   @send external toBase64: (t, @as("base64") _) => string = "toString"
+  @val @scope("Buffer") external fromUint8Array: Uint8Array.t => t = "from"
+  // A view over the bytes rather than a copy.
+  @val @scope("Buffer")
+  external fromArrayBuffer: (ArrayBuffer.t, ~byteOffset: int, ~length: int) => t = "from"
+  @val @scope("Buffer") external fromHex: (string, @as("hex") _) => t = "from"
+  @send external toHex: (t, @as("hex") _) => string = "toString"
   let empty = alloc(0)
 }
 

package/src/bindings/Vitest.res

@@ -169,3 +169,26 @@ let messageOfThrown: (unit => 'a) => option<string> = %raw(`function (fn) {
 // happens to equal the placeholder.
 let toThrowErrorEqual = (t: testContext, fn: unit => 'a, ~message=?, expected: string) =>
   t.expect(fn->messageOfThrown, ~message?).toEqual(Some(expected))
+
+// ============================================================================
+// Fake timers
+// ============================================================================
+
+module Vi = {
+  @module("vitest") @scope("vi")
+  external useFakeTimers: unit => unit = "useFakeTimers"
+
+  @module("vitest") @scope("vi")
+  external useRealTimers: unit => unit = "useRealTimers"
+
+  // Runs every timer that comes due within the window, awaiting the
+  // microtasks each one queues before moving on.
+  @module("vitest") @scope("vi")
+  external advanceTimersByTimeAsync: int => promise<unit> = "advanceTimersByTimeAsync"
+
+  @module("vitest") @scope("vi")
+  external getTimerCount: unit => int = "getTimerCount"
+
+  @module("vitest") @scope("vi")
+  external clearAllTimers: unit => unit = "clearAllTimers"
+}

package/src/bindings/Vitest.res.mjs

@@ -16,9 +16,12 @@ function toThrowErrorEqual(t, fn, message, expected) {
   t.expect(messageOfThrown(fn), message).toEqual(expected);
 }
 
+let Vi = {};
+
 export {
   Async,
   messageOfThrown,
   toThrowErrorEqual,
+  Vi,
 }
 /* No side effect */

package/src/bindings/WebSocket.res

@@ -7,21 +7,20 @@ type t
 
 @new external create: string => t = "WebSocket"
 
-@unboxed
-type readyState =
-  | @as(0) Connecting
-  | @as(1) Open
-  | @as(2) Closing
-  | @as(3) Closed
-
-@get external readyState: t => readyState = "readyState"
-
 @set external onopen: (t, unit => unit) => unit = "onopen"
 @set external onerror: (t, JsExn.t => unit) => unit = "onerror"
 @set external onclose: (t, unit => unit) => unit = "onclose"
 
-type messageEvent = {data: string}
+// Node delivers a string for a text frame and a Blob for a binary one, so what
+// arrives is only text when the peer sent text. Typed as a string it would reach
+// a JSON parse — and, once that threw, a log line — as an object, past every
+// length cap written for a string.
+type messageEvent = {data: unknown}
 @set external onmessage: (t, messageEvent => unit) => unit = "onmessage"
 
+// Nothing in here may be a runtime value. `create` compiles to `new WebSocket`
+// against the global, and a value would make consumers import this module under
+// that same name, shadowing the global they are trying to construct.
+
 @send external send: (t, string) => unit = "send"
 @send external close: t => unit = "close"