envio 3.13.0 → 3.14.0-subgraph

package/src/Persistence.res

@@ -39,9 +39,6 @@ type initialState = {
   // On a resume this is what the database holds, not what the config would
   // derive — the ids must never reshuffle under stored rows.
   contractMapping: ContractMapping.t,
-  // Public config snapshot, restored with the address rows. None when
-  // envio_info or envio_contracts is missing.
-  envioInfo: option<JSON.t>,
   cache: dict<effectCacheRecord>,
   chains: array<initialChainState>,
   // Where each chain's checkpoint sequence stands in the database.
@@ -123,20 +120,25 @@ type storage = {
     ~contractMapping: ContractMapping.t,
     ~envioInfo: JSON.t,
   ) => promise<initialState>,
-  // `throwIfIncompatible` gets what the storage holds before any sink is
-  // resumed, so a config the stored one rules out is reported as such rather
-  // than as the sink tripping over tables it never created.
-  //
+  // Read before anything is resumed, so a config the stored one rules out is
+  // reported as such rather than as a sink tripping over tables it never
+  // created.
+  readStoredConfig: unit => promise<ResumePlan.stored>,
+  // Brings an initialized storage a chain it doesn't have yet: its row, its
+  // partitions and its config addresses, as `initialize` would have created
+  // them. The chains already there are left untouched.
+  addChain: (
+    ~chainConfig: Config.chain,
+    ~entities: array<Internal.entityConfig>,
+    ~contractMapping: ContractMapping.t,
+  ) => promise<unit>,
   // `chainIds` is what this run drives. An isolated run resumes a subset of
   // the stored chains, and the sink's resume trims past each resumed chain's
   // checkpoint, so chains a sibling process is still writing must stay out.
   resumeInitialState: (
     ~entities: array<Internal.entityConfig>,
     ~chainIds: array<ChainId.t>,
-    ~throwIfIncompatible: (
-      ~storedEnvioInfo: option<JSON.t>,
-      ~storedContractMapping: ContractMapping.t,
-    ) => unit,
+    ~contractMapping: ContractMapping.t,
   ) => promise<initialState>,
   // Returns rows matching the filter.
   // Field values are serialized and rows parsed with the table's field schemas.
@@ -243,6 +245,10 @@ type t = {
 
 exception StorageError({message: string, reason: exn})
 
+// A start the operator has to act on, not a failure: what it says is all they
+// need, so it's printed once, without a stack trace.
+exception Refused(string)
+
 let make = (
   ~userEntities,
   // TODO: Should only pass userEnums and create internal config in runtime
@@ -271,10 +277,11 @@ let init = {
     ~runCommand,
     ~reset=false,
     ~lowercaseAddresses=false,
-    // An isolated run needs the schema to exist already: initializing under it
-    // would create rows for this process's chains only, leaving the ones it
-    // skipped with no state for their own processes to resume.
-    ~requireInitialized=false,
+    // An `envio start --chain` run. It needs the schema to exist already:
+    // initializing under it would create rows for this process's chains only,
+    // leaving the ones it skipped with no state for their own processes to
+    // resume. It is also the one run that may add its chain to the schema.
+    ~isolated=false,
     // Whether this process is the one that tells the operator the run resumed.
     // A supervisor says it once for the whole run, so the workers it forked
     // keep it to their own log files.
@@ -297,9 +304,11 @@ let init = {
         })
         persistence.storageStatus = Initializing(promise)
         if reset || !(await persistence.storage.isInitialized()) {
-          if requireInitialized {
-            JsError.throwWithMessage(
-              "`envio start --chain` needs a database that already holds every chain. Run `envio local db-migrate up` once with the full config, then start a process per chain.",
+          if isolated {
+            throw(
+              Refused(
+                "`envio start --chain` needs a database that is already set up. Run `envio local db-migrate up` once with the full config, then start a process per chain.",
+              ),
             )
           }
           Logging.info(`Initializing the indexer storage...`)
@@ -330,18 +339,51 @@ let init = {
         ) {
           let logResume = announceResume ? Logging.info : Logging.trace
           logResume(`Found existing indexer storage. Resuming indexing state...`)
+          let stored = await persistence.storage.readStoredConfig()
+          switch ResumePlan.make(
+            ~stored,
+            ~envioInfo,
+            ~chainConfigs,
+            ~contractMapping,
+            ~lowercaseAddresses,
+            ~isolated,
+          ) {
+          | Resume => ()
+          | Incompatible(changedPaths) =>
+            throw(
+              Refused(
+                ResumePlan.incompatibleMessage(
+                  changedPaths,
+                  ~envioInfo,
+                  ~resetCommand,
+                  ~runCommand,
+                ),
+              ),
+            )
+          | AddChain(chainConfig) =>
+            Logging.info({
+              "msg": `Adding the chain to the existing indexer storage...`,
+              "chainId": chainConfig.id,
+            })
+            // The same resolution `initialize` runs, and for the same reason:
+            // this is the one time the chain's row is written.
+            let chainConfig =
+              (
+                await [chainConfig]->StartBlockResolver.resolveAllOrThrow(
+                  ~lowercaseAddresses,
+                  ~retry=startBlockRetry,
+                )
+              )->Array.getUnsafe(0)
+            await persistence.storage.addChain(
+              ~chainConfig,
+              ~entities=persistence.allEntities,
+              ~contractMapping=stored.contractMapping,
+            )
+          }
           let initialState = await persistence.storage.resumeInitialState(
             ~entities=persistence.allEntities,
             ~chainIds=chainConfigs->Array.map(chain => chain.id),
-            ~throwIfIncompatible=(~storedEnvioInfo, ~storedContractMapping) =>
-              Config.throwIfResumeIncompatible(
-                ~storedEnvioInfo,
-                ~storedContractMapping,
-                ~envioInfo,
-                ~contractMapping,
-                ~resetCommand,
-                ~runCommand,
-              ),
+            ~contractMapping=stored.contractMapping,
           )
           persistence.storageStatus = Ready(initialState)
           let progress = Dict.make()
@@ -356,6 +398,7 @@ let init = {
         resolveRef.contents()
       }
     } catch {
+    | Refused(message) => JsError.throwWithMessage(message)
     | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`Failed to initialize the indexer storage.`)
     }
   }
@@ -365,23 +408,21 @@ let init = {
 // a migration command: what a config change prints names the command the
 // operator ran, and an unreachable chain is waited on rather than reported,
 // since somebody is watching the run come up.
-let initForRun = (
-  persistence,
-  ~config: Config.t,
-  ~reset,
-  ~isDevelopmentMode,
-  ~requireInitialized,
-) =>
+let initForRun = (persistence, ~config: Config.t, ~reset, ~isDevelopmentMode) =>
   persistence->init(
     ~announceResume=!Worker.isEnabled,
     ~reset,
     ~chainConfigs=config.chainMap->ChainMap.values,
     ~contractMapping=config.contractMapping,
-    ~envioInfo=Config.envioInfo(),
-    ~resetCommand=isDevelopmentMode ? "envio dev -r" : "envio start -r",
-    ~runCommand=Some(isDevelopmentMode ? "envio dev" : "envio start"),
+    ~envioInfo=config.envioInfo,
+    ~resetCommand=(isDevelopmentMode ? "envio dev -r" : "envio start -r")->Config.withProjectFlags(
+      ~config,
+    ),
+    ~runCommand=Some(
+      (isDevelopmentMode ? "envio dev" : "envio start")->Config.withProjectFlags(~config),
+    ),
     ~lowercaseAddresses=config.lowercaseAddresses,
-    ~requireInitialized,
+    ~isolated=config.isolated,
   )
 
 let getInitializedStorageOrThrow = persistence => {

package/src/Persistence.res.mjs

@@ -6,6 +6,7 @@ import * as Worker from "./Worker.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as Frontier from "./db/Frontier.res.mjs";
+import * as ResumePlan from "./ResumePlan.res.mjs";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
@@ -22,6 +23,8 @@ function writtenFrontier(batch, rollback) {
 
 let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
 
+let Refused = /* @__PURE__ */Primitive_exceptions.create("Persistence.Refused");
+
 function make(userEntities, allEnums, storage) {
   let allEnums$1 = allEnums.concat([EntityHistory.RowAction.config]);
   return {
@@ -33,10 +36,10 @@ function make(userEntities, allEnums, storage) {
   };
 }
 
-async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt, lowercaseAddressesOpt, requireInitializedOpt, announceResumeOpt, startBlockRetryOpt) {
+async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt, lowercaseAddressesOpt, isolatedOpt, announceResumeOpt, startBlockRetryOpt) {
   let reset = resetOpt !== undefined ? resetOpt : false;
   let lowercaseAddresses = lowercaseAddressesOpt !== undefined ? lowercaseAddressesOpt : false;
-  let requireInitialized = requireInitializedOpt !== undefined ? requireInitializedOpt : false;
+  let isolated = isolatedOpt !== undefined ? isolatedOpt : false;
   let announceResume = announceResumeOpt !== undefined ? announceResumeOpt : true;
   let startBlockRetry = startBlockRetryOpt !== undefined ? startBlockRetryOpt : "UntilItAnswers";
   try {
@@ -64,8 +67,12 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
       _0: promise$1
     };
     if (reset || !await persistence.storage.isInitialized()) {
-      if (requireInitialized) {
-        Stdlib_JsError.throwWithMessage("`envio start --chain` needs a database that already holds every chain. Run `envio local db-migrate up` once with the full config, then start a process per chain.");
+      if (isolated) {
+        throw {
+          RE_EXN_ID: Refused,
+          _1: "`envio start --chain` needs a database that is already set up. Run `envio local db-migrate up` once with the full config, then start a process per chain.",
+          Error: new Error()
+        };
       }
       Logging.info(`Initializing the indexer storage...`);
       let chainConfigs$1 = await StartBlockResolver.resolveAllOrThrow(chainConfigs, lowercaseAddresses, startBlockRetry, undefined, undefined);
@@ -82,7 +89,26 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
       if (tmp) {
         let logResume = announceResume ? Logging.info : Logging.trace;
         logResume(`Found existing indexer storage. Resuming indexing state...`);
-        let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, chainConfigs.map(chain => chain.id), (storedEnvioInfo, storedContractMapping) => Config.throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand));
+        let stored = await persistence.storage.readStoredConfig();
+        let changedPaths = ResumePlan.make(stored, envioInfo, chainConfigs, contractMapping, lowercaseAddresses, isolated);
+        if (typeof changedPaths === "object") {
+          if (changedPaths.TAG === "AddChain") {
+            let chainConfig = changedPaths._0;
+            Logging.info({
+              msg: `Adding the chain to the existing indexer storage...`,
+              chainId: chainConfig.id
+            });
+            let chainConfig$1 = (await StartBlockResolver.resolveAllOrThrow([chainConfig], lowercaseAddresses, startBlockRetry, undefined, undefined))[0];
+            await persistence.storage.addChain(chainConfig$1, persistence.allEntities, stored.contractMapping);
+          } else {
+            throw {
+              RE_EXN_ID: Refused,
+              _1: ResumePlan.incompatibleMessage(changedPaths._0, envioInfo, resetCommand, runCommand),
+              Error: new Error()
+            };
+          }
+        }
+        let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, chainConfigs.map(chain => chain.id), stored.contractMapping);
         persistence.storageStatus = {
           TAG: "Ready",
           _0: initialState$1
@@ -98,14 +124,18 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
       }
     }
     return resolveRef.contents();
-  } catch (raw_exn) {
-    let exn = Primitive_exceptions.internalToException(raw_exn);
-    return ErrorHandling.mkLogAndRaise(undefined, `Failed to initialize the indexer storage.`, exn);
+  } catch (raw_message) {
+    let message = Primitive_exceptions.internalToException(raw_message);
+    if (message.RE_EXN_ID === Refused) {
+      return Stdlib_JsError.throwWithMessage(message._1);
+    } else {
+      return ErrorHandling.mkLogAndRaise(undefined, `Failed to initialize the indexer storage.`, message);
+    }
   }
 }
 
-function initForRun(persistence, config, reset, isDevelopmentMode, requireInitialized) {
-  return init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.envioInfo(), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset, config.lowercaseAddresses, requireInitialized, !Worker.isEnabled, undefined);
+function initForRun(persistence, config, reset, isDevelopmentMode) {
+  return init(persistence, ChainMap.values(config.chainMap), config.contractMapping, config.envioInfo, Config.withProjectFlags(isDevelopmentMode ? "envio dev -r" : "envio start -r", config), Config.withProjectFlags(isDevelopmentMode ? "envio dev" : "envio start", config), reset, config.lowercaseAddresses, config.isolated, !Worker.isEnabled, undefined);
 }
 
 function getInitializedStorageOrThrow(persistence) {
@@ -129,6 +159,7 @@ function getInitializedState(persistence) {
 export {
   writtenFrontier,
   StorageError,
+  Refused,
   make,
   init,
   initForRun,

package/src/PgStorage.res

@@ -270,11 +270,16 @@ let getEntityHistory = (~entityConfig: Internal.entityConfig): EntityHistory.pgE
   }
 }
 
+let makeCreatePartitionQuery = (entityConfig: Internal.entityConfig, ~pgSchema, ~chainId) =>
+  `CREATE TABLE IF NOT EXISTS "${pgSchema}"."${partitionTableName(
+      ~entityConfig,
+      ~chainId,
+    )}" PARTITION OF "${pgSchema}"."${entityConfig.table.tableName}" FOR VALUES IN (${chainId->ChainId.toString});`
+
 // Every table an entity needs: its own, one partition per chain when it's
-// per-chain, and its history table. The chain set is fixed for the life of a
-// schema — changing it fails the resume compat check against `envio_info` and
-// forces a resync — so every partition the entity will ever need is created
-// here, at init.
+// per-chain, and its history table. A chain the schema gains later gets its
+// partitions from `addChain`, which the resume compat check allows only for an
+// `envio start --chain` process naming it.
 //
 // History stays unpartitioned: it is only ever read by checkpoint, never by
 // chain, so partitioning it would route every write and prune nothing.
@@ -294,12 +299,7 @@ let makeCreateEntityTableQueries = (
     [
       entityConfig.table->createTable(~partitionByColumn=chainIdField->Table.getPgDbFieldName),
     ]->Array.concat(
-      chainIds->Array.map(chainId =>
-        `CREATE TABLE IF NOT EXISTS "${pgSchema}"."${partitionTableName(
-            ~entityConfig,
-            ~chainId,
-          )}" PARTITION OF "${pgSchema}"."${entityConfig.table.tableName}" FOR VALUES IN (${chainId->ChainId.toString});`
-      ),
+      chainIds->Array.map(chainId => entityConfig->makeCreatePartitionQuery(~pgSchema, ~chainId)),
     )
   }->Array.concat([getEntityHistory(~entityConfig).table->createTable])
 }
@@ -602,7 +602,7 @@ let makeInsertValuesSetQuery = (
   ~itemsCount,
   ~chainIdMode: ChainId.mode=Int32,
 ) => {
-  let {quotedFieldNames, quotedNonPrimaryFieldNames} =
+  let {quotedFieldNames, quotedNonPrimaryFieldNames, booleanArrayColumnIndexes} =
     table->Table.toSqlParams(~schema=itemSchema, ~pgSchema, ~chainIdMode)
 
   let primaryKeyFieldNames = Table.getPgPrimaryKeyFieldNames(table)
@@ -620,6 +620,9 @@ let makeInsertValuesSetQuery = (
         placeholders := placeholders.contents ++ ","
       }
       placeholders := placeholders.contents ++ `$${(fieldIdx * itemsCount + idx)->Int.toString}`
+      if booleanArrayColumnIndexes->Array.includes(fieldIdx) {
+        placeholders := placeholders.contents ++ "::integer[]::boolean[]"
+      }
     }
     placeholders := placeholders.contents ++ ")"
   }
@@ -652,7 +655,7 @@ let makeTableBatchSetQuery = (
   ~itemSchema: S.t<'item>,
   ~chainIdMode: ChainId.mode=Int32,
 ) => {
-  let {dbSchema, hasArrayField, byteaColumnIndexes} =
+  let {dbSchema, hasArrayField, byteaColumnIndexes, booleanArrayColumnIndexes} =
     table->Table.toSqlParams(~schema=itemSchema, ~pgSchema, ~chainIdMode)
 
   // Should move this to a better place
@@ -713,9 +716,25 @@ let makeTableBatchSetQuery = (
       ),
       "convertOrThrow": S.compile(
         S.unnest(itemSchema)->S.preprocess(_ => {
-          serializer: Utils.Array.flatten->(
-            Utils.magic: (array<array<'a>> => array<'a>) => unknown => unknown
-          ),
+          serializer: columns => {
+            let columns = columns->(Utils.magic: unknown => array<array<Nullable.t<array<bool>>>>)
+            booleanArrayColumnIndexes->Array.forEach(index =>
+              columns->Array.setUnsafe(
+                index,
+                columns
+                ->Array.getUnsafe(index)
+                ->Array.map(
+                  value =>
+                    value
+                    ->Nullable.map(flags => flags->Array.map(flag => flag ? 1 : 0))
+                    ->(Utils.magic: Nullable.t<array<int>> => Nullable.t<array<bool>>),
+                ),
+              )
+            )
+            columns
+            ->Utils.Array.flatten
+            ->(Utils.magic: array<Nullable.t<array<bool>>> => unknown)
+          },
         }),
         ~input=Value,
         ~output=Unknown,
@@ -2009,7 +2028,6 @@ let make = (
       cache,
       reorgCheckpoints: [],
       contractMapping,
-      envioInfo: Some(envioInfo),
       chains: chainConfigs->Array.mapWithIndex((
         chainConfig,
         idx,
@@ -2437,17 +2455,59 @@ let make = (
     }
   }
 
+  let readStoredConfig = async (): ResumePlan.stored => {
+    let (envioInfo, contractNames) = await Promise.all2((
+      InternalTable.EnvioInfo.read(sql, ~pgSchema),
+      InternalTable.EnvioContracts.read(sql, ~pgSchema),
+    ))
+    // Both tables join the schema in one transaction. A missing mapping means
+    // an older envio wrote this schema, so treat the record as unreadable
+    // rather than decoding address rows against ids nothing assigned.
+    switch (envioInfo, contractNames) {
+    | (Some(envioInfo), Some(contractNames)) => {
+        envioInfo: Some(envioInfo),
+        chains: await InternalTable.Chains.readStoredChains(sql, ~pgSchema),
+        contractMapping: ContractMapping.fromStoredNames(contractNames),
+      }
+    | _ => {envioInfo: None, chains: [], contractMapping: ContractMapping.empty}
+    }
+  }
+
+  let addChain = async (~chainConfig: Config.chain, ~entities, ~contractMapping) => {
+    let partitionQueries =
+      entities
+      ->Array.filter((entityConfig: Internal.entityConfig) =>
+        entityConfig.storage.postgres && entityConfig.table->Table.getChainIdField->Option.isSome
+      )
+      ->Array.map(entityConfig =>
+        entityConfig->makeCreatePartitionQuery(~pgSchema, ~chainId=chainConfig.id)
+      )
+    let addressRows = chainConfig->ChainState.configStorageRows(~ecosystem, ~contractMapping)
+    // No conflict clauses: two processes adding the same chain are two
+    // processes driving it, and the second one fails on a primary key instead
+    // of indexing alongside the first.
+    let _ = await sql->Postgres.beginSql(async sql => {
+      for idx in 0 to partitionQueries->Array.length - 1 {
+        await sql
+        ->Postgres.unsafe(partitionQueries->Array.getUnsafe(idx))
+        ->Utils.Promise.ignoreValue
+      }
+      if addressRows->Utils.Array.notEmpty {
+        await InternalTable.EnvioAddresses.insert(sql, ~pgSchema, ~rows=addressRows, ~chainIdMode)
+      }
+      switch InternalTable.Chains.makeInitialValuesQuery(~pgSchema, ~chainConfigs=[chainConfig]) {
+      | Some(query) => await sql->Postgres.unsafe(query)->Utils.Promise.ignoreValue
+      | None => ()
+      }
+    })
+  }
+
   let resumeInitialState = async (
     ~entities,
     ~chainIds,
-    ~throwIfIncompatible,
+    ~contractMapping,
   ): Persistence.initialState => {
-    let (
-      cache,
-      (chains, checkpointFrontier),
-      reorgCheckpoints,
-      (storedEnvioInfo, storedContractMapping),
-    ) = await Promise.all4((
+    let (cache, (chains, checkpointFrontier), reorgCheckpoints) = await Promise.all3((
       restoreEffectCache(~withUpload=false),
       InternalTable.Chains.getInitialState(
         sql,
@@ -2491,22 +2551,8 @@ let make = (
           }>,
         >
       ),
-      Promise.all2((
-        InternalTable.EnvioInfo.read(sql, ~pgSchema),
-        InternalTable.EnvioContracts.read(sql, ~pgSchema),
-      ))->Promise.thenResolve(((info, names)) =>
-        // Both tables join the schema in one transaction. A missing mapping
-        // means an older envio wrote this schema, so treat the snapshot as
-        // unreadable rather than decoding address rows against ids nothing assigned.
-        switch (info, names) {
-        | (Some(info), Some(names)) => (Some(info), ContractMapping.fromStoredNames(names))
-        | _ => (None, ContractMapping.empty)
-        }
-      ),
     ))
 
-    throwIfIncompatible(~storedEnvioInfo, ~storedContractMapping)
-
     await reloadIndexCatalog()
 
     // Convert string checkpoint IDs from DB to bigint
@@ -2529,8 +2575,7 @@ let make = (
       cache,
       chains,
       checkpointFrontier,
-      contractMapping: storedContractMapping,
-      envioInfo: storedEnvioInfo,
+      contractMapping,
     }
   }
 
@@ -2687,6 +2732,8 @@ let make = (
     name: storageName,
     isInitialized,
     initialize,
+    readStoredConfig,
+    addChain,
     resumeInitialState,
     loadOrThrow,
     ensureQueryIndexes,