envio 3.15.0 → 3.16.0

package/src/bindings/StorageClient.res

@@ -1,8 +1,8 @@
 // Rows cross in arenas both ways. A write stages its rows into memory the addon
-// lends (`beginStage` to `commitStage`); a read lands its rows in a Rust-owned
-// arena that `lendResult` hands over and `releaseResult` takes back. Reading or
-// writing through a buffer after it has been handed back is what detaching
-// prevents — the rules are the ones at the top of
+// lends (`beginChanges`/`beginRows` to `commitStage`); a read lands its rows in
+// a Rust-owned arena that `lendResult` hands over and `releaseResult` takes
+// back. Reading or writing through a buffer after it has been handed back is
+// what detaching prevents — the rules are the ones at the top of
 // `packages/cli/src/columnar/mod.rs`.
 
 type t
@@ -19,6 +19,9 @@ type postgresOptions = {
   applicationName: string,
   pgSchema: string,
   isHasuraEnabled: bool,
+  // An `envio start --chain` process: builds and looks for a per-chain
+  // entity's indexes on its own chains' partitions only.
+  isolated: bool,
 }
 
 type clickHouseOptions = {
@@ -31,7 +34,6 @@ type clickHouseOptions = {
 }
 
 type options = {
-  chainIdMode: string,
   postgres: postgresOptions,
   clickhouse?: clickHouseOptions,
 }
@@ -45,71 +47,31 @@ type queryResult = {
   rows: int,
 }
 
-// A ClickHouse column. The rows read back are keyed by `rowName`, the name the
-// read schemas parse every store's rows by.
-type clickHouseColumn = {
-  name: string,
-  rowName?: string,
-  // The schema field, which `@storage(clickhouse: {...})` expressions name a
-  // column by even when a rename is configured.
-  fieldName?: string,
-  fieldType: string,
-  isNullable: bool,
-  isArray: bool,
-  precision?: int,
-  scale?: int,
-  enumVariants?: array<string>,
-}
-
-type skippingIndex = {
-  name: string,
-  expr: string,
-  indexType: string,
-  granularity?: int,
-}
+type store = Postgres | ClickHouse
 
-type clickHouseTable = {
-  name: string,
-  historyTable: string,
-  columns: array<clickHouseColumn>,
-  chainIdColumn?: string,
-  partitionBy?: string,
-  orderBy?: array<string>,
-  ttl?: string,
-  skippingIndexes?: array<skippingIndex>,
-}
+type stagedField = Staging.stagedField
 
-// A table described for every store that keeps it: an entity for each store its
-// schema names, the indexer's own tables for Postgres alone.
-type tableInput = {
+type entityLayout = {
   name: string,
-  postgres?: Core.pgTableInput,
-  clickhouse?: clickHouseTable,
+  // The store the entity is read from; none when no configured store keeps it.
+  home?: store,
+  fields: array<stagedField>,
 }
 
-type postgresRegistered = {kinds?: array<int>}
-
-type clickHouseRegistered = {
-  names: array<string>,
-  kinds: array<int>,
-  nullable: array<bool>,
+// How every batch is laid out: an entity's change set is its fields, then each
+// change's checkpoint and whether it deletes.
+type layouts = {
+  // By entity index.
+  entities: array<entityLayout>,
+  rawEvents: array<stagedField>,
+  effectCache: array<stagedField>,
 }
 
-type store = Postgres | ClickHouse
-
-// What each store made of a table: present for the stores that keep it.
-type registeredTable = {
-  handle: int,
-  // The store the table is read from.
-  home: store,
-  postgres?: postgresRegistered,
-  clickhouse?: clickHouseRegistered,
-}
+type stagedTable = RawEvents | EffectCache
 
 type bounds = {
-  sequence: CheckpointSequence.t,
   chainIds: array<ChainId.t>,
-  checkpointIds: array<string>,
+  checkpointIds: array<bigint>,
 }
 
 type progress = {
@@ -146,15 +108,7 @@ type chainConfig = {
   maxReorgDepth: int,
 }
 
-type partition = {table: int, chainId: ChainId.t, name: string}
-
-type enum = {name: string, variants: array<string>}
-
 type initialize = {
-  sequence: CheckpointSequence.t,
-  tables: array<int>,
-  partitions: array<partition>,
-  enums: array<enum>,
   chains: array<chainConfig>,
   envioInfo: string,
   contractNames: array<string>,
@@ -163,18 +117,9 @@ type initialize = {
 
 type addChain = {
   chain: chainConfig,
-  partitions: array<partition>,
   addresses: addresses,
 }
 
-type resume = {
-  sequence: CheckpointSequence.t,
-  // Every entity table.
-  tables: array<int>,
-  // The chains this run drives, which are the only ones trimmed.
-  chainIds: array<ChainId.t>,
-}
-
 type storedChain = {
   id: float,
   ecosystem: string,
@@ -203,11 +148,11 @@ type resumedChain = {
   // Unix seconds.
   progressBlockTime?: float,
   sourceBlock: int,
-  checkpointId: string,
+  checkpointId: bigint,
 }
 
 type reorgCheckpoint = {
-  id: string,
+  id: bigint,
   chainId: float,
   blockNumber: int,
   blockHash: string,
@@ -227,57 +172,40 @@ type progressDiff = {
 
 type rollbackData = {removed: queryResult, restored: queryResult}
 
-type cacheTable = {tableName: string, rows: int}
-
-// An optional field is left out rather than null, both ways: the addon refuses
-// a null for one, and leaves out one it has no value for.
-
-// Rows of a batch: staged into an arena, or one rendered parameter per cell,
-// every row's first column first.
-type rows = {
-  staged?: int,
-  cells?: array<Null.t<string>>,
+// `chainId` is left out for a cross-chain cache.
+type storedEffectCache = {
+  effectName: string,
+  chainId?: ChainId.t,
+  tableName: string,
   rows: int,
 }
 
-type historyWrite = {
-  backfill: array<string>,
-  sets?: rows,
-  setCheckpointIds: array<string>,
-  deleteIds: array<string>,
-  deleteCheckpointIds: array<string>,
-}
-
-type postgresEntityWrite = {
-  sets?: rows,
-  deletes: array<string>,
-  history?: historyWrite,
-}
+// An optional field is left out rather than null, both ways: the addon refuses
+// a null for one, and leaves out one it has no value for.
 
-type entityWrite = {
-  table: int,
+// One entity's changes within one chain scope, staged.
+type changes = {
+  entity: int,
   chainId?: ChainId.t,
-  postgres?: postgresEntityWrite,
-  // Every change as a history row, staged.
-  clickhouse?: int,
+  keepsHistory: bool,
+  changes: int,
 }
 
 type diffCheckpoint = {
   chainId: ChainId.t,
-  checkpointId: string,
+  checkpointId: bigint,
   blockNumber: int,
 }
 
 type rollbackWrite = {
   bounds: bounds,
-  tables: array<int>,
   progress: array<progress>,
   removedAddresses: addresses,
   diffCheckpoints: array<diffCheckpoint>,
 }
 
 type checkpoints = {
-  ids: array<string>,
+  ids: array<bigint>,
   chainIds: array<ChainId.t>,
   blockNumbers: array<int>,
   blockHashes: array<Null.t<string>>,
@@ -285,17 +213,15 @@ type checkpoints = {
   keepsHistory: array<bool>,
 }
 
-type frontier = {chainIds: array<ChainId.t>, checkpointIds: array<string>}
-
-type tableWrite = {table: int, rows: rows}
+type frontier = {chainIds: array<ChainId.t>, checkpointIds: array<bigint>}
 
-type effectCacheWrite = {table: int, create: bool, rows: rows}
+type effectCacheWrite = {table: string, create: bool, rows: int}
 
 type batch = {
   rollback?: rollbackWrite,
   progress: array<progress>,
-  rawEvents?: tableWrite,
-  entities: array<entityWrite>,
+  rawEvents?: int,
+  changes: array<changes>,
   chainMeta: array<chainMeta>,
   addresses: addresses,
   frontier: frontier,
@@ -305,37 +231,31 @@ type batch = {
 
 type writeTiming = {store: store, seconds: float}
 
-// What a build was for, which decides how much an existing index has to match.
-type indexPurpose = | @as("Query") Query | @as("Schema") Schema
-
-@tag("kind")
-type indexEvent =
-  | Invalid({names: array<string>})
-  | Planned({declared: int, missing: array<string>, rebuilt: array<string>})
-  | Building({purpose: indexPurpose, name: string, tableName: string, isRebuild: bool})
-  | Built({purpose: indexPurpose, name: string, seconds: float})
-  | Failed({
-      purpose: indexPurpose,
-      name: string,
-      tableName: string,
-      columns: array<string>,
-      error: string,
-      code?: string,
-    })
-  | ResyncFailed({name: string, error: string, code?: string})
-  | Committed({count: int, seconds: float})
-
-@send
-external classCreate: (Core.storageCtor, options, indexEvent => unit, string => unit) => t =
-  "create"
-
-let make = (options, ~onIndexEvent, ~onWarning) =>
-  Core.getAddon().storage->classCreate(options, onIndexEvent, onWarning)
-
-@send external registerTable: (t, tableInput) => registeredTable = "registerTable"
-
 @send
-external beginStage: (t, ~table: int, ~store: store, ~rows: int) => Staging.begun = "beginStage"
+external classCreate: (
+  Core.storageCtor,
+  options,
+  Null.t<string>,
+  Logger.native,
+  array<string>,
+) => t = "create"
+
+// The storage of the entities `configJson` declares — the config the CLI
+// wrote — or of none. The addon logs through `logger` itself, with the fields
+// bound to it.
+let make = (options, ~configJson=?, ~logger: Logger.t) =>
+  Core.getAddon().storage->classCreate(
+    options,
+    configJson->Null.fromOption,
+    logger.output.native,
+    logger.boundFieldsJson,
+  )
+
+@send external layouts: t => layouts = "layouts"
+
+@send external beginChanges: (t, ~entity: int, ~rows: int) => Staging.begun = "beginChanges"
+
+@send external beginRows: (t, ~table: stagedTable, ~rows: int) => Staging.begun = "beginRows"
 
 @send
 external growStage: (
@@ -353,8 +273,8 @@ external commitStage: (t, ~handle: int, ~buffers: array<ArrayBuffer.t>) => unit
 
 @send external discardStaged: (t, array<int>) => unit = "discardStaged"
 
-let arena = (client, ~store): Staging.arena => {
-  beginStage: (~table, ~rows) => client->beginStage(~table, ~store, ~rows),
+let arena = (client, begin): Staging.arena => {
+  beginStage: (~rows) => begin(~rows),
   growStage: (~handle, ~column, ~needed, ~stale) =>
     client->growStage(~handle, ~column, ~needed, ~stale),
   commitStage: (~handle, ~buffers) => client->commitStage(~handle, ~buffers),
@@ -391,13 +311,26 @@ let queryForTests = async (client, sql, ~params=[]) =>
 @send
 external loadWhereRaw: (t, int, array<LoadCondition.t>) => promise<queryResult> = "loadWhere"
 
-let loadWhere = async (client, ~table, ~conditions) =>
-  client->read(await client->loadWhereRaw(table, conditions))
+let loadWhere = async (client, ~entity, ~conditions) =>
+  client->read(await client->loadWhereRaw(entity, conditions))
+
+@send
+external loadEffectCacheRaw: (t, string, array<LoadCondition.t>) => promise<queryResult> =
+  "loadEffectCache"
+
+let loadEffectCache = async (client, ~table, ~conditions) =>
+  client->read(await client->loadEffectCacheRaw(table, conditions))
+
+@send
+external clickHouseRowsForTestsRaw: (t, int) => promise<queryResult> = "clickhouseRowsForTests"
+
+let clickHouseRowsForTests = async (client, ~entity) =>
+  client->read(await client->clickHouseRowsForTestsRaw(entity))
 
 @send external historyForTestsRaw: (t, int) => promise<queryResult> = "historyForTests"
 
-let historyForTests = async (client, ~table) =>
-  client->read(await client->historyForTestsRaw(table))
+let historyForTests = async (client, ~entity) =>
+  client->read(await client->historyForTestsRaw(entity))
 
 // What the storage holds before anything is initialized or resumed.
 type holding =
@@ -419,49 +352,49 @@ type holding =
 
 @send external readStoredConfig: t => promise<storedConfig> = "readStoredConfig"
 
-@send external resume: (t, resume) => promise<resumed> = "resume"
+// Resumes the chains this run drives, the only ones ClickHouse trims.
+@send external resume: (t, array<ChainId.t>) => promise<resumed> = "resume"
 
 @send external setChainMeta: (t, array<chainMeta>) => promise<unit> = "setChainMeta"
 
+// Builds every index the schema promises for the chains, then stamps them
+// ready at the given unix milliseconds.
 @send
-external finalizeBackfill: (t, array<Core.pgIndexInput>, array<ChainId.t>, float) => promise<unit> =
-  "finalizeBackfill"
+external finalizeBackfill: (t, array<ChainId.t>, float) => promise<unit> = "finalizeBackfill"
 
-@send
-external ensureSchemaIndexes: (t, array<Core.pgIndexInput>) => promise<unit> = "ensureSchemaIndexes"
+@send external ensureSchemaIndexes: (t, array<ChainId.t>) => promise<unit> = "ensureSchemaIndexes"
 
 // Null when the indexes are already there, which is every call but the first,
-// and every call for a table no store builds query indexes for.
+// and every call for an entity Postgres doesn't keep.
 @send
-external ensureQueryIndexes: (t, int, string, array<string>) => Null.t<promise<unit>> =
-  "ensureQueryIndexes"
+external ensureQueryIndexes: (
+  t,
+  ~entity: int,
+  ~chainId: Null.t<ChainId.t>,
+  ~columns: array<string>,
+) => Null.t<promise<unit>> = "ensureQueryIndexes"
 
 @send external pruneCheckpoints: (t, bounds) => promise<unit> = "pruneCheckpoints"
 
 @send external pruneHistory: (t, int, bounds) => promise<unit> = "pruneHistory"
 
 @send
-external rollbackTargetCheckpoint: (
-  t,
-  ChainId.t,
-  int,
-  CheckpointSequence.t,
-) => promise<Nullable.t<string>> = "rollbackTargetCheckpoint"
+external rollbackTargetCheckpoint: (t, ChainId.t, int) => promise<Nullable.t<bigint>> =
+  "rollbackTargetCheckpoint"
 
 @send
 external rollbackProgressDiff: (t, bounds) => promise<array<progressDiff>> = "rollbackProgressDiff"
 
 @send external rollbackData: (t, int, bounds) => promise<rollbackData> = "rollbackData"
 
-@send external effectCacheTables: t => promise<array<cacheTable>> = "effectCacheTables"
-
-type cacheUpload = {table: int, path: string}
-
-@send external uploadEffectCache: (t, array<cacheUpload>) => promise<unit> = "uploadEffectCache"
-
-type cacheDump = {tableName: string, path: string}
+// Uploads the caches saved under the directory first when `upload` says to,
+// logging how that went.
+@send
+external restoreEffectCache: (t, string, bool) => promise<array<storedEffectCache>> =
+  "restoreEffectCache"
 
-@send external dumpEffectCache: (t, array<cacheDump>) => promise<unit> = "dumpEffectCache"
+// Never rejects: a dump that doesn't finish is logged.
+@send external dumpEffectCache: (t, string) => promise<unit> = "dumpEffectCache"
 
 @send external reset: t => promise<unit> = "reset"
 

package/src/bindings/StorageClient.res.mjs

@@ -2,14 +2,15 @@
 
 import * as Core from "../Core.res.mjs";
 import * as Reading from "../Reading.res.mjs";
+import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 
-function make(options, onIndexEvent, onWarning) {
-  return Core.getAddon().Storage.create(options, onIndexEvent, onWarning);
+function make(options, configJson, logger) {
+  return Core.getAddon().Storage.create(options, Stdlib_Null.fromOption(configJson), logger.output.native, logger.boundFieldsJson);
 }
 
-function arena(client, store) {
+function arena(client, begin) {
   return {
-    beginStage: (table, rows) => client.beginStage(table, store, rows),
+    beginStage: rows => begin(rows),
     growStage: (handle, column, needed, stale) => client.growStage(handle, column, needed, stale),
     commitStage: (handle, buffers) => {
       client.commitStage(handle, buffers);
@@ -39,12 +40,20 @@ async function queryForTests(client, sql, paramsOpt) {
   return read(client, await client.queryForTests(sql, params));
 }
 
-async function loadWhere(client, table, conditions) {
-  return read(client, await client.loadWhere(table, conditions));
+async function loadWhere(client, entity, conditions) {
+  return read(client, await client.loadWhere(entity, conditions));
 }
 
-async function historyForTests(client, table) {
-  return read(client, await client.historyForTests(table));
+async function loadEffectCache(client, table, conditions) {
+  return read(client, await client.loadEffectCache(table, conditions));
+}
+
+async function clickHouseRowsForTests(client, entity) {
+  return read(client, await client.clickhouseRowsForTests(entity));
+}
+
+async function historyForTests(client, entity) {
+  return read(client, await client.historyForTests(entity));
 }
 
 export {
@@ -53,6 +62,8 @@ export {
   read,
   queryForTests,
   loadWhere,
+  loadEffectCache,
+  clickHouseRowsForTests,
   historyForTests,
 }
 /* Core Not a pure module */

package/src/bindings/Vitest.res

@@ -169,29 +169,3 @@ let messageOfThrown: (unit => 'a) => option<string> = %raw(`function (fn) {
 // happens to equal the placeholder.
 let toThrowErrorEqual = (t: testContext, fn: unit => 'a, ~message=?, expected: string) =>
   t.expect(fn->messageOfThrown, ~message?).toEqual(Some(expected))
-
-// ============================================================================
-// Fake timers
-// ============================================================================
-
-module Vi = {
-  @module("vitest") @scope("vi")
-  external useFakeTimers: unit => unit = "useFakeTimers"
-
-  @module("vitest") @scope("vi")
-  external useRealTimers: unit => unit = "useRealTimers"
-
-  // Runs every timer that comes due within the window, awaiting the
-  // microtasks each one queues before moving on.
-  @module("vitest") @scope("vi")
-  external advanceTimersByTimeAsync: int => promise<unit> = "advanceTimersByTimeAsync"
-
-  @module("vitest") @scope("vi")
-  external advanceTimersToNextTimerAsync: unit => promise<unit> = "advanceTimersToNextTimerAsync"
-
-  @module("vitest") @scope("vi")
-  external getTimerCount: unit => int = "getTimerCount"
-
-  @module("vitest") @scope("vi")
-  external clearAllTimers: unit => unit = "clearAllTimers"
-}

package/src/bindings/Vitest.res.mjs

@@ -16,12 +16,9 @@ function toThrowErrorEqual(t, fn, message, expected) {
   t.expect(messageOfThrown(fn), message).toEqual(expected);
 }
 
-let Vi = {};
-
 export {
   Async,
   messageOfThrown,
   toThrowErrorEqual,
-  Vi,
 }
 /* No side effect */