envio 3.11.0 → 3.12.1

package/src/bindings/ClickHouseSink.res

@@ -9,9 +9,16 @@ type historySchema = {
   checkpointsTable: string,
   checkpointChainIdColumn: string,
   checkpointBlockNumberColumn: string,
+  chainsTable: string,
+  chainsCheckpointIdColumn: string,
 }
 
-type chainProgressInput = {chainId: string, progressBlockNumber: int}
+type chainProgressInput = {
+  chainId: string,
+  progressBlockNumber: int,
+  // Where the chain's own sequence stands, as Postgres has it committed.
+  committedCheckpointId: string,
+}
 
 type options = {
   url: string,
@@ -62,12 +69,14 @@ type initializeInput = {
   databaseEngine?: string,
 }
 
+// A history table and the column naming the chain its rows belong to. Absent
+// only for a cross-chain entity, which no per-chain sequence can produce.
+type historyTableInput = {name: string, chainIdColumn?: string}
+
 type resumeInput = {
-  checkpointId: string,
+  perChain: bool,
   chainProgress: array<chainProgressInput>,
-  historyTables: array<string>,
-  replicated: bool,
-  databaseEngine?: string,
+  historyTables: array<historyTableInput>,
 }
 
 type registeredTable = {
@@ -77,15 +86,6 @@ type registeredTable = {
   nullable: array<bool>,
 }
 
-type columnValuesInput = {
-  numbers?: Float64Array.t,
-  unsigned64?: BigUint64Array.t,
-  signed64?: BigInt64Array.t,
-  texts?: array<string>,
-  bytes?: array<Uint8Array.t>,
-  nulls?: Uint8Array.t,
-}
-
 @send
 external registerEntityTable: (t, entitySpec) => registeredTable = "registerEntityTable"
 
@@ -99,7 +99,30 @@ external registerCheckpointsTable: (t, array<columnSpec>) => registeredTable =
 external resume: (t, resumeInput) => promise<unit> = "resume"
 
 @send
-external stage: (t, ~table: int, ~rows: int, ~columns: array<columnValuesInput>) => int = "stage"
+external beginStage: (t, ~table: int, ~rows: int) => Staging.begun = "beginStage"
+
+@send
+external growStage: (
+  t,
+  ~handle: int,
+  ~column: int,
+  ~needed: int,
+  ~stale: ArrayBuffer.t,
+) => ArrayBuffer.t = "growStage"
+
+@send
+external commitStage: (t, ~handle: int, ~buffers: array<ArrayBuffer.t>) => unit = "commitStage"
+
+@send
+external abortStage: (t, ~handle: int, ~buffers: array<ArrayBuffer.t>) => unit = "abortStage"
+
+let arena = (sink): Staging.arena => {
+  beginStage: (~table, ~rows) => sink->beginStage(~table, ~rows),
+  growStage: (~handle, ~column, ~needed, ~stale) =>
+    sink->growStage(~handle, ~column, ~needed, ~stale),
+  commitStage: (~handle, ~buffers) => sink->commitStage(~handle, ~buffers),
+  abortStage: (~handle, ~buffers) => sink->abortStage(~handle, ~buffers),
+}
 
 @send
 external writeBatch: (t, ~entities: array<int>, ~checkpoints: Null.t<int>) => promise<unit> =
@@ -116,6 +139,8 @@ let historySchema = (): historySchema => {
   checkpointsTable: InternalTable.Checkpoints.table.tableName,
   checkpointChainIdColumn: (#chain_id: InternalTable.Checkpoints.field :> string),
   checkpointBlockNumberColumn: (#block_number: InternalTable.Checkpoints.field :> string),
+  chainsTable: InternalTable.Chains.table.tableName,
+  chainsCheckpointIdColumn: (#checkpoint_id: InternalTable.Chains.field :> string),
 }
 
 let make = (~url, ~username, ~password, ~database, ~chainIdMode: ChainId.mode, ~onWarning) =>
@@ -131,45 +156,6 @@ let make = (~url, ~username, ~password, ~database, ~chainIdMode: ChainId.mode, ~
     onWarning,
   )
 
-type kind =
-  | @as(0) F64
-  | @as(1) U64
-  | @as(2) I64
-  | @as(3) Text
-  | @as(4) Bytes
-
-let kindOfOrdinal = ordinal =>
-  switch ordinal {
-  | 0 => F64
-  | 1 => U64
-  | 2 => I64
-  | 3 => Text
-  | 4 => Bytes
-  | unknown => JsError.throwWithMessage(`Unknown ClickHouse column kind ${unknown->Int.toString}`)
-  }
-
-external asString: unknown => string = "%identity"
-@val external toNumber: unknown => float = "Number"
-@val external toBigInt: unknown => bigint = "BigInt"
-@val external stringOf: unknown => string = "String"
-
-// No column holds NaN or Infinity, but nothing downstream refuses them well: a
-// list renders one as `null` on the way into JSON text, which the column then
-// refuses for a reason naming `null` rather than what the handler wrote, and a
-// scalar Float64 takes the raw bytes and stores a value no reader can render.
-// Refused here instead, where the value is still itself, with one message for
-// both shapes.
-%%private(
-  let finiteOrThrow = (number: float, ~column) =>
-    if number->Float.isFinite {
-      number
-    } else {
-      JsError.throwWithMessage(
-        `${number->Float.toString} is not a finite number, so it cannot be stored in the \`${column}\` column. Store a finite number, or keep it out of the entity.`,
-      )
-    }
-)
-
 // Visits every node on the way into JSON text. A bigint would make
 // `JSON.stringify` throw and fail the whole batch, so it travels as its digits.
 %%private(
@@ -177,7 +163,7 @@ external asString: unknown => string = "%identity"
     (_, value: JSON.t) =>
       switch value->typeof {
       | #bigint =>
-        value->(Utils.magic: JSON.t => unknown)->stringOf->(Utils.magic: string => JSON.t)
+        value->(Utils.magic: JSON.t => unknown)->Staging.stringOf->(Utils.magic: string => JSON.t)
       // A Uint8Array inside a list column travels as its byte values, which the
       // sink reads back into raw bytes.
       | #object =>
@@ -190,148 +176,21 @@ external asString: unknown => string = "%identity"
         | None => value
         }
       | #number =>
-        let _ = value->(Utils.magic: JSON.t => float)->finiteOrThrow(~column)
+        let _ = value->(Utils.magic: JSON.t => float)->Staging.finiteOrThrow(~column)
         value
       | _ => value
       }
 )
 
-// The replacer closes over nothing but the column's name, so it is built once
-// with the column rather than once per column per batch.
-type column = {name: string, kind: kind, isNullable: bool, replacer: JSON.replacer}
-type table = {handle: int, name: string, columns: array<column>}
+type table = {handle: int, name: string, columns: array<Staging.column>}
 
 let makeTable = (~name, {handle, names, kinds, nullable}: registeredTable) => {
   handle,
   name,
-  columns: names->Array.mapWithIndex((name, index) => {
+  columns: names->Array.mapWithIndex((name, index): Staging.column => {
     name,
-    kind: kinds->Array.getUnsafe(index)->kindOfOrdinal,
+    kind: kinds->Array.getUnsafe(index)->Staging.kindOfOrdinal,
     isNullable: nullable->Array.getUnsafe(index),
     replacer: Replacer(jsonSafe(~column=name)),
   }),
 }
-
-let toText = (value: unknown, ~replacer) =>
-  switch value->typeof {
-  | #string => value->asString
-  | #bigint => value->stringOf
-  | _ => value->(Utils.magic: unknown => JSON.t)->JSON.stringify(~replacer)
-  }
-
-type builder = {
-  name: string,
-  kind: kind,
-  isNullable: bool,
-  floats: Float64Array.t,
-  unsigned: BigUint64Array.t,
-  signed: BigInt64Array.t,
-  texts: array<string>,
-  bytes: array<Uint8Array.t>,
-  replacer: JSON.replacer,
-  mutable nulls: option<Uint8Array.t>,
-  rows: int,
-}
-
-%%private(let noFloats = Float64Array.fromLength(0))
-%%private(let noUnsigned = BigUint64Array.fromLength(0))
-%%private(let noSigned = BigInt64Array.fromLength(0))
-%%private(let noBytes = Uint8Array.fromLength(0))
-
-let makeBuilder = ({name, kind, isNullable, replacer}: column, ~rows) => {
-  name,
-  kind,
-  isNullable,
-  floats: kind === F64 ? Float64Array.fromLength(rows) : noFloats,
-  unsigned: kind === U64 ? BigUint64Array.fromLength(rows) : noUnsigned,
-  signed: kind === I64 ? BigInt64Array.fromLength(rows) : noSigned,
-  texts: kind === Text ? Array.make(~length=rows, "") : [],
-  bytes: kind === Bytes ? Array.make(~length=rows, noBytes) : [],
-  replacer,
-  nulls: None,
-  rows,
-}
-
-let markNull = (builder, ~row) => {
-  let nulls = switch builder.nulls {
-  | Some(nulls) => nulls
-  | None =>
-    let nulls = Uint8Array.fromLength(builder.rows)
-    builder.nulls = Some(nulls)
-    nulls
-  }
-  nulls->TypedArray.set(row, 1)
-}
-
-// RowBinary carries the raw integer, so a value the column cannot hold is not
-// rejected anywhere downstream — and a typed array reduces it modulo 2^64 on
-// the way in rather than refusing it, which would leave no trace at all.
-%%private(
-  let checkedBigInt = (value: unknown, ~builder, ~min, ~max) => {
-    let value = value->toBigInt
-    if value < min || value > max {
-      JsError.throwWithMessage(
-        `${value->BigInt.toString} is out of range for the \`${builder.name}\` column`,
-      )
-    }
-    value
-  }
-)
-
-%%private(
-  let writePresent = (builder, ~row, value: unknown) =>
-    switch builder.kind {
-    | F64 =>
-      builder.floats->TypedArray.set(row, value->toNumber->finiteOrThrow(~column=builder.name))
-    | U64 =>
-      builder.unsigned->TypedArray.set(
-        row,
-        value->checkedBigInt(~builder, ~min=0n, ~max=18446744073709551615n),
-      )
-    | I64 =>
-      builder.signed->TypedArray.set(
-        row,
-        value->checkedBigInt(~builder, ~min=-9223372036854775808n, ~max=9223372036854775807n),
-      )
-    | Text => builder.texts->Array.setUnsafe(row, value->toText(~replacer=builder.replacer))
-    | Bytes => builder.bytes->Array.setUnsafe(row, value->(Utils.magic: unknown => Uint8Array.t))
-    }
-)
-
-%%private(let isAbsent = (value: unknown) => value === %raw(`undefined`) || value === %raw(`null`))
-
-// Writes one value of a row the handler set. `undefined`/`null` marks the row's
-// null bit, which a column that accepts NULL stores as such — and which one that
-// does not has no way to store: RowBinary carries no "absent", so the row would
-// land holding the type's zero, a value the handler never chose and that nothing
-// downstream could tell from one it did.
-let writeValue = (builder, ~row, value: unknown) =>
-  if value->isAbsent {
-    if builder.isNullable {
-      builder->markNull(~row)
-    } else {
-      JsError.throwWithMessage(
-        `No value for the \`${builder.name}\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`,
-      )
-    }
-  } else {
-    builder->writePresent(~row, value)
-  }
-
-let writeDeletedValue = (builder, ~row, value: unknown) =>
-  if value->isAbsent {
-    builder->markNull(~row)
-  } else {
-    builder->writePresent(~row, value)
-  }
-
-let builderPayload = (builder): columnValuesInput => {
-  let base = {nulls: ?builder.nulls}
-  switch builder.kind {
-  | F64 => {...base, numbers: builder.floats}
-  | U64 => {...base, unsigned64: builder.unsigned}
-  | I64 => {...base, signed64: builder.signed}
-  | Text => {...base, texts: builder.texts}
-  | Bytes => {...base, bytes: builder.bytes}
-  }
-}

package/src/bindings/ClickHouseSink.res.mjs

@@ -3,10 +3,22 @@
 import * as Core from "../Core.res.mjs";
 import * as Table from "../db/Table.res.mjs";
 import * as Utils from "../Utils.res.mjs";
-import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Staging from "../Staging.res.mjs";
 import * as EntityHistory from "../db/EntityHistory.res.mjs";
 import * as InternalTable from "../db/InternalTable.res.mjs";
-import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+
+function arena(sink) {
+  return {
+    beginStage: (table, rows) => sink.beginStage(table, rows),
+    growStage: (handle, column, needed, stale) => sink.growStage(handle, column, needed, stale),
+    commitStage: (handle, buffers) => {
+      sink.commitStage(handle, buffers);
+    },
+    abortStage: (handle, buffers) => {
+      sink.abortStage(handle, buffers);
+    }
+  };
+}
 
 function historySchema() {
   return {
@@ -17,7 +29,9 @@ function historySchema() {
     setVariant: "SET",
     checkpointsTable: InternalTable.Checkpoints.table.tableName,
     checkpointChainIdColumn: "chain_id",
-    checkpointBlockNumberColumn: "block_number"
+    checkpointBlockNumberColumn: "block_number",
+    chainsTable: InternalTable.Chains.table.tableName,
+    chainsCheckpointIdColumn: "checkpoint_id"
   };
 }
 
@@ -32,31 +46,6 @@ function make(url, username, password, database, chainIdMode, onWarning) {
   }, onWarning);
 }
 
-function kindOfOrdinal(ordinal) {
-  switch (ordinal) {
-    case 0 :
-      return 0;
-    case 1 :
-      return 1;
-    case 2 :
-      return 2;
-    case 3 :
-      return 3;
-    case 4 :
-      return 4;
-    default:
-      return Stdlib_JsError.throwWithMessage(`Unknown ClickHouse column kind ` + ordinal.toString());
-  }
-}
-
-function finiteOrThrow(number, column) {
-  if (isFinite(number)) {
-    return number;
-  } else {
-    return Stdlib_JsError.throwWithMessage(number.toString() + ` is not a finite number, so it cannot be stored in the \`` + column + `\` column. Store a finite number, or keep it out of the entity.`);
-  }
-}
-
 function jsonSafe(column) {
   return (param, value) => {
     let match = typeof value;
@@ -65,7 +54,7 @@ function jsonSafe(column) {
     }
     if (match !== "object") {
       if (match === "number") {
-        finiteOrThrow(value, column);
+        Staging.finiteOrThrow(value, column);
         return value;
       } else {
         return value;
@@ -88,159 +77,17 @@ function makeTable(name, param) {
     name: name,
     columns: param.names.map((name, index) => ({
       name: name,
-      kind: kindOfOrdinal(kinds[index]),
+      kind: Staging.kindOfOrdinal(kinds[index]),
       isNullable: nullable[index],
       replacer: jsonSafe(name)
     }))
   };
 }
 
-function toText(value, replacer) {
-  let match = typeof value;
-  if (match === "bigint") {
-    return String(value);
-  } else if (match === "string") {
-    return value;
-  } else {
-    return JSON.stringify(value, replacer);
-  }
-}
-
-let noFloats = new Float64Array(0);
-
-let noUnsigned = new BigUint64Array(0);
-
-let noSigned = new BigInt64Array(0);
-
-let noBytes = new Uint8Array(0);
-
-function makeBuilder(param, rows) {
-  let kind = param.kind;
-  return {
-    name: param.name,
-    kind: kind,
-    isNullable: param.isNullable,
-    floats: kind === 0 ? new Float64Array(rows) : noFloats,
-    unsigned: kind === 1 ? new BigUint64Array(rows) : noUnsigned,
-    signed: kind === 2 ? new BigInt64Array(rows) : noSigned,
-    texts: kind === 3 ? Stdlib_Array.make(rows, "") : [],
-    bytes: kind === 4 ? Stdlib_Array.make(rows, noBytes) : [],
-    replacer: param.replacer,
-    nulls: undefined,
-    rows: rows
-  };
-}
-
-function markNull(builder, row) {
-  let nulls = builder.nulls;
-  let nulls$1;
-  if (nulls !== undefined) {
-    nulls$1 = nulls;
-  } else {
-    let nulls$2 = new Uint8Array(builder.rows);
-    builder.nulls = nulls$2;
-    nulls$1 = nulls$2;
-  }
-  nulls$1[row] = 1;
-}
-
-function checkedBigInt(value, builder, min, max) {
-  let value$1 = BigInt(value);
-  if (value$1 < min || value$1 > max) {
-    Stdlib_JsError.throwWithMessage(value$1.toString() + ` is out of range for the \`` + builder.name + `\` column`);
-  }
-  return value$1;
-}
-
-function writePresent(builder, row, value) {
-  let match = builder.kind;
-  switch (match) {
-    case 0 :
-      builder.floats[row] = finiteOrThrow(Number(value), builder.name);
-      return;
-    case 1 :
-      builder.unsigned[row] = checkedBigInt(value, builder, 0n, 18446744073709551615n);
-      return;
-    case 2 :
-      builder.signed[row] = checkedBigInt(value, builder, -9223372036854775808n, 9223372036854775807n);
-      return;
-    case 3 :
-      builder.texts[row] = toText(value, builder.replacer);
-      return;
-    case 4 :
-      builder.bytes[row] = value;
-      return;
-  }
-}
-
-function isAbsent(value) {
-  if (value === undefined) {
-    return true;
-  } else {
-    return value === null;
-  }
-}
-
-function writeValue(builder, row, value) {
-  if (isAbsent(value)) {
-    if (builder.isNullable) {
-      return markNull(builder, row);
-    } else {
-      return Stdlib_JsError.throwWithMessage(`No value for the \`` + builder.name + `\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`);
-    }
-  } else {
-    return writePresent(builder, row, value);
-  }
-}
-
-function writeDeletedValue(builder, row, value) {
-  if (isAbsent(value)) {
-    return markNull(builder, row);
-  } else {
-    return writePresent(builder, row, value);
-  }
-}
-
-function builderPayload(builder) {
-  let base_nulls = builder.nulls;
-  let base = {
-    nulls: base_nulls
-  };
-  let match = builder.kind;
-  switch (match) {
-    case 0 :
-      let newrecord = {...base};
-      newrecord.numbers = builder.floats;
-      return newrecord;
-    case 1 :
-      let newrecord$1 = {...base};
-      newrecord$1.unsigned64 = builder.unsigned;
-      return newrecord$1;
-    case 2 :
-      let newrecord$2 = {...base};
-      newrecord$2.signed64 = builder.signed;
-      return newrecord$2;
-    case 3 :
-      let newrecord$3 = {...base};
-      newrecord$3.texts = builder.texts;
-      return newrecord$3;
-    case 4 :
-      let newrecord$4 = {...base};
-      newrecord$4.bytes = builder.bytes;
-      return newrecord$4;
-  }
-}
-
 export {
+  arena,
   historySchema,
   make,
-  kindOfOrdinal,
   makeTable,
-  toText,
-  makeBuilder,
-  markNull,
-  writeValue,
-  writeDeletedValue,
-  builderPayload,
 }
-/* noFloats Not a pure module */
+/* Core Not a pure module */

package/src/db/CheckpointSequence.res

@@ -0,0 +1,109 @@
+// How checkpoint ids are handed out. A schema with a cross-chain entity has
+// rows any chain's reorg can reach, so its checkpoints have to be comparable
+// across chains and come from one shared counter. Without one, no chain can
+// touch another's rows, and each gets a counter of its own — which is what lets
+// a rollback, a prune or a resume name one chain without saying anything about
+// the others.
+type t =
+  | SharedAcrossChains
+  | PerChain
+
+let fromEntities = (entities: array<Internal.entityConfig>) =>
+  entities->Array.some(entityConfig => entityConfig.crossChain) ? SharedAcrossChains : PerChain
+
+// Where a chain stands: under one shared counter that is wherever the counter
+// got to, whichever chain moved it last.
+let position = (sequence: t, frontier: Frontier.t, ~chainId) =>
+  switch sequence {
+  | SharedAcrossChains => frontier->Frontier.max
+  | PerChain => frontier->Frontier.get(chainId)
+  }
+
+// The committed (or processed) id a scope's rows compare against. A chain scope
+// reads its own. A cross-chain scope spans every chain: under one shared
+// sequence the highest id is exactly how far the scope has got, while per-chain
+// ids aren't comparable across chains, so only the lowest is an id every chain
+// has passed.
+let forScope = (sequence: t, frontier: Frontier.t, ~scope: Internal.chainScope) =>
+  switch scope {
+  | Chain(chainId) => frontier->Frontier.get(chainId)
+  | CrossChain =>
+    switch sequence {
+    | SharedAcrossChains => frontier->Frontier.max
+    | PerChain => frontier->Frontier.min
+    }
+  }
+
+// The diff id a scope's rows carry after a rollback, taken from the frontier the
+// rollback stamped them with. A per-chain rollback names only the chains it
+// moved: a sibling it left alone has no diff row, and so no id to compare
+// against.
+let findForScope = (sequence: t, frontier: Frontier.t, ~scope: Internal.chainScope): option<
+  Internal.checkpointId,
+> =>
+  switch scope {
+  | Chain(chainId) => frontier->Frontier.find(chainId)
+  | CrossChain => Some(sequence->forScope(frontier, ~scope))
+  }
+
+// Hands out the ids of one batch, starting from where the frontier left each
+// chain. Under `SharedAcrossChains` the ids come from a single run of the counter, so they
+// interleave across chains in allocation order; under `PerChain` each chain
+// continues its own.
+type cursor = {sequence: t, frontier: Frontier.t}
+
+let cursor = (sequence: t, ~frontier: Frontier.t) => {sequence, frontier: frontier->Frontier.copy}
+
+let next = (cursor, ~chainId): Internal.checkpointId => {
+  let checkpointId = cursor.sequence->position(cursor.frontier, ~chainId)->BigInt.add(1n)
+  cursor.frontier->Frontier.set(chainId, checkpointId)
+  checkpointId
+}
+
+// The checkpoint id each chain's rows are compared against in a query,
+// together with the sequence that decides how the comparison is rendered.
+type checkpointBoundsByChain = {sequence: t, byChain: Frontier.t}
+
+// The pieces a query splices in to read its bound as `checkpointId`.
+type sql = {
+  join: string,
+  using: string,
+  usingMatch: string,
+  checkpointId: string,
+}
+
+// The chain ids are cast to the widest integer type rather than the column's
+// own: nothing indexes the join, so the values only have to compare.
+let relation = `unnest($1::${(Postgres.BigInt :> string)}[],$2::${(Postgres.BigInt :> string)}[]) AS envio_bounds(chain_id, checkpoint_id)`
+
+let sql = (
+  bounds: checkpointBoundsByChain,
+  ~chainIdColumn: option<string>,
+  ~tableRef: string,
+): sql =>
+  switch (bounds.sequence, chainIdColumn) {
+  | (SharedAcrossChains, _) => {join: "", using: "", usingMatch: "", checkpointId: "$1"}
+  | (PerChain, Some(column)) =>
+    let chainMatch = `envio_bounds.chain_id = ${tableRef}."${column}"`
+    {
+      join: ` JOIN ${relation} ON ${chainMatch}`,
+      using: ` USING ${relation}`,
+      usingMatch: ` AND ${chainMatch}`,
+      checkpointId: "envio_bounds.checkpoint_id",
+    }
+  | (PerChain, None) =>
+    JsError.throwWithMessage(
+      "Internal error: per-chain checkpoint bounds can't bound a table with no chain-id column. Only a schema whose entities are all per-chain has them.",
+    )
+  }
+
+// Bound in the order `sql` reads them: under one shared sequence the lowest of
+// the ids as `$1` (every chain is held to it), otherwise the chains and their
+// own ids as the two parallel arrays the relation unnests.
+let params = (bounds: checkpointBoundsByChain): unknown =>
+  switch bounds.sequence {
+  | SharedAcrossChains =>
+    [bounds.byChain->Frontier.min->BigInt.toString]->(Utils.magic: array<string> => unknown)
+  | PerChain =>
+    bounds.byChain->Frontier.unnestParams->(Utils.magic: Frontier.unnestParams => unknown)
+  }