envio 3.5.0 → 3.5.1-svm-alpha.2

package/src/db/Table.res.mjs

@@ -16,11 +16,12 @@ function makeEnumConfig(name, variants) {
   };
 }
 
-function mkField(fieldName, fieldType, fieldSchema, $$default, isArrayOpt, isNullableOpt, isPrimaryKeyOpt, isIndexOpt, linkedEntity, description, postgresDbName, clickhouseDbName) {
+function mkField(fieldName, fieldType, fieldSchema, $$default, isArrayOpt, isNullableOpt, isPrimaryKeyOpt, isIndexOpt, isChainIdOpt, linkedEntity, description, postgresDbName, clickhouseDbName) {
   let isArray = isArrayOpt !== undefined ? isArrayOpt : false;
   let isNullable = isNullableOpt !== undefined ? isNullableOpt : false;
   let isPrimaryKey = isPrimaryKeyOpt !== undefined ? isPrimaryKeyOpt : false;
   let isIndex = isIndexOpt !== undefined ? isIndexOpt : false;
+  let isChainId = isChainIdOpt !== undefined ? isChainIdOpt : false;
   return {
     TAG: "Field",
     _0: {
@@ -31,6 +32,7 @@ function mkField(fieldName, fieldType, fieldSchema, $$default, isArrayOpt, isNul
       isNullable: isNullable,
       isPrimaryKey: isPrimaryKey,
       isIndex: isIndex,
+      isChainId: isChainId,
       linkedEntity: linkedEntity,
       defaultValue: $$default,
       description: description,
@@ -225,6 +227,18 @@ function getDerivedFromFields(table) {
   });
 }
 
+function getChainIdField(table) {
+  return Stdlib_Array.findMap(table.fields, field => {
+    if (field.TAG !== "Field") {
+      return;
+    }
+    let field$1 = field._0;
+    if (field$1.isChainId) {
+      return field$1;
+    }
+  });
+}
+
 function getFieldByName(table, fieldName) {
   return table.fields.find(field => getUserDefinedFieldName(field) === fieldName);
 }
@@ -288,7 +302,8 @@ let queryFields = Utils.$$WeakMap.memoize(table => {
   return dict;
 });
 
-function makeRowsSchema(table, rowFieldName) {
+function makeRowsSchema(table, rowFieldName, skipChainIdOpt) {
+  let skipChainId = skipChainIdOpt !== undefined ? skipChainIdOpt : false;
   return S$RescriptSchema.array(S$RescriptSchema.object(s => {
     let dict = {};
     table.fields.forEach(field => {
@@ -296,15 +311,20 @@ function makeRowsSchema(table, rowFieldName) {
         return;
       }
       let field$1 = field._0;
+      if (field$1.isChainId && skipChainId) {
+        return;
+      }
       dict[getApiFieldName(field$1)] = s.f(rowFieldName(field$1), field$1.fieldSchema);
     });
     return dict;
   }));
 }
 
-let rowsSchema = Utils.$$WeakMap.memoize(table => makeRowsSchema(table, getApiFieldName));
+let rowsSchema = Utils.$$WeakMap.memoize(table => makeRowsSchema(table, getApiFieldName, undefined));
+
+let pgRowsSchema = Utils.$$WeakMap.memoize(table => makeRowsSchema(table, getPgDbFieldName, undefined));
 
-let pgRowsSchema = Utils.$$WeakMap.memoize(table => makeRowsSchema(table, getPgDbFieldName));
+let pgEntityRowsSchema = Utils.$$WeakMap.memoize(table => makeRowsSchema(table, getPgDbFieldName, true));
 
 let NonExistingTableField = /* @__PURE__ */Primitive_exceptions.create("Table.NonExistingTableField");
 
@@ -465,6 +485,7 @@ export {
   getNonDefaultFields,
   getLinkedEntityFields,
   getDerivedFromFields,
+  getChainIdField,
   getFieldByName,
   NoIdField,
   getIdFieldOrThrow,
@@ -477,6 +498,7 @@ export {
   makeRowsSchema,
   rowsSchema,
   pgRowsSchema,
+  pgEntityRowsSchema,
   NonExistingTableField,
   getUnfilteredCompositeIndexesUnsafe,
   toSqlParams,

package/src/sources/AddressStore.res

@@ -7,10 +7,12 @@
 // (or address-valued param) isn't registered at the item's block.
 type t
 
-// A contract the chain has events for. An address registered for anything else
-// gets the `NoEvents` verdict: persisted so a future config that adds events
-// picks it up, but never fetched.
-type contract = {name: string, startBlock: option<int>}
+// A contract an address may be registered for. Every config contract of every
+// chain, since `context.chain.<Contract>.add` validates against that whole set
+// — a name outside it makes the store throw. `dependsOnAddresses` is whether
+// this chain fetches for the contract by address, which is what makes the store
+// able to answer `fetchable` on a verdict.
+type contract = {name: string, startBlock: option<int>, dependsOnAddresses: bool}
 
 type registration = {
   address: Address.t,
@@ -20,9 +22,10 @@ type registration = {
 }
 
 type verdict =
-  | Added({effectiveStartBlock: int})
-  // Contract has no events; tracked and persisted, never fetched.
-  | NoEvents({effectiveStartBlock: int})
+  // `fetchable` is false when nothing on the chain is fetched by address for the
+  // contract: the address is still stored and persisted, so a config that later
+  // adds events picks it up on restart, but no partition is built from it.
+  | Added({effectiveStartBlock: int, fetchable: bool})
   // Already registered for the same contract.
   | Duplicate({effectiveStartBlock: int, existingEffectiveStartBlock: int})
   // Already registered for a different contract.
@@ -32,11 +35,22 @@ type verdict =
 
 type rawVerdict = {
   kind: string,
+  fetchable: bool,
   effectiveStartBlock: int,
   existingContractName: Null.t<string>,
   existingEffectiveStartBlock: Null.t<int>,
 }
 
+// A registration the store handed over for persistence, paired with the
+// checkpoint that owns its row. `checkpointIdx` indexes the block numbers
+// passed to `drainForWrite` — the ids stay on the JS side.
+type drainedAddress = {
+  address: Address.t,
+  contractName: string,
+  registrationBlock: int,
+  checkpointIdx: int,
+}
+
 type makeSetOptions = {
   // Only addresses whose id is at or above this. Ids are handed out in
   // registration order, so a cursor read before a batch (`nextId`) selects
@@ -66,22 +80,37 @@ let make = (~ecosystem: Ecosystem.name, ~shouldChecksum: bool, ~contracts: array
   }
 }
 
-// The contracts of a chain that have events, with the earliest block any of
-// their events may fire at. `None` means unrestricted, so it wins over any
-// `Some`: one registration without a start block makes the whole contract's
-// address gate open from the chain's start, and the per-registration start
-// block still holds back the registrations that declared one.
-let contractsOf = (~onEventRegistrations: array<Internal.onEventRegistration>): array<contract> => {
+// Every contract an address may be registered for, with the earliest block any
+// of this chain's events for it may fire at. `None` means unrestricted, so it
+// wins over any `Some`: one registration without a start block makes the whole
+// contract's address gate open from the chain's start, and the per-registration
+// start block still holds back the registrations that declared one.
+//
+// A contract whose events are all wildcard is fetched without consulting
+// addresses, so registering one changes nothing about what's queried — it
+// carries `dependsOnAddresses: false` just like a contract named only by
+// `configContractNames`. Either way the addresses are stored and persisted,
+// never fetched.
+let contractsOf = (
+  ~onEventRegistrations: array<Internal.onEventRegistration>,
+  ~configContractNames: array<string>,
+): array<contract> => {
   // Only ever holds a declared start block, so a missing key is unambiguous —
   // storing `None` in a dict would be indistinguishable from "not seen yet".
   let startBlocks: dict<int> = Dict.make()
   let unrestricted = Utils.Set.make()
+  let addressDependent = Utils.Set.make()
   let names = []
-  onEventRegistrations->Array.forEach(reg => {
-    let name = reg.eventConfig.contractName
+  let addName = name =>
     if !(names->Array.includes(name)) {
       names->Array.push(name)->ignore
     }
+  onEventRegistrations->Array.forEach(reg => {
+    let name = reg.eventConfig.contractName
+    addName(name)
+    if reg.dependsOnAddresses {
+      addressDependent->Utils.Set.add(name)->ignore
+    }
     switch reg.startBlock {
     | None => unrestricted->Utils.Set.add(name)->ignore
     | Some(startBlock) =>
@@ -94,11 +123,13 @@ let contractsOf = (~onEventRegistrations: array<Internal.onEventRegistration>):
       )
     }
   })
+  configContractNames->Array.forEach(addName)
   names->Array.map(name => {
     name,
     startBlock: unrestricted->Utils.Set.has(name)
       ? None
       : startBlocks->Utils.Dict.dangerouslyGetNonOption(name),
+    dependsOnAddresses: addressDependent->Utils.Set.has(name),
   })
 }
 
@@ -112,10 +143,24 @@ let contractsOf = (~onEventRegistrations: array<Internal.onEventRegistration>):
 // come from that chain's one store — ids are store-scoped, so sets from
 // different stores can't be merged.
 @send external makeSetOf: (t, array<Address.t>) => AddressSet.t = "makeSetOf"
-@send external registerBatchRaw: (t, array<registration>) => array<rawVerdict> = "registerBatch"
-@send external makeSetRaw: (t, string, makeSetOptions) => AddressSet.t = "makeSet"
 @send
-external startBlockGroups: (t, string) => array<AddressSet.startBlockGroup> = "startBlockGroups"
+external registerBatchRaw: (t, array<registration>) => array<rawVerdict> = "registerBatch"
+@send external seedBatchRaw: (t, array<registration>) => array<rawVerdict> = "seedBatch"
+
+// Drains the registrations awaiting persistence at or below the given block —
+// what the batch being written covers — pairing each with the checkpoint at its
+// registration block. Later registrations stay pending. Throws, with the queue
+// untouched, when a drained registration's block has no checkpoint in the batch.
+@send
+external drainForWrite: (t, int, array<int>) => array<drainedAddress> = "drainForWrite"
+
+// How many registrations await persistence — lets a caller skip the work of
+// assembling what `drainForWrite` needs.
+@send external pendingCount: t => int = "pendingCount"
+
+// The registrations still awaiting persistence. For assertions.
+@send external pendingEntries: t => array<Internal.indexingContract> = "pendingEntries"
+@send external makeSetRaw: (t, string, makeSetOptions) => AddressSet.t = "makeSet"
 @send external contractCount: (t, string) => int = "contractCount"
 @send external size: t => int = "size"
 
@@ -132,12 +177,9 @@ external startBlockGroups: (t, string) => array<AddressSet.startBlockGroup> = "s
 
 @send external contractAddresses: (t, string) => array<Address.t> = "contractAddresses"
 
-@send external entries: t => array<Internal.indexingContract> = "entries"
-
 let toVerdict = (raw: rawVerdict): verdict =>
   switch raw.kind {
-  | "added" => Added({effectiveStartBlock: raw.effectiveStartBlock})
-  | "noEvents" => NoEvents({effectiveStartBlock: raw.effectiveStartBlock})
+  | "added" => Added({effectiveStartBlock: raw.effectiveStartBlock, fetchable: raw.fetchable})
   | "duplicate" =>
     Duplicate({
       effectiveStartBlock: raw.effectiveStartBlock,
@@ -148,13 +190,22 @@ let toVerdict = (raw: rawVerdict): verdict =>
   | kind => JsError.throwWithMessage(`Unexpected address registration verdict "${kind}"`)
   }
 
-// Registers a batch, resolving each address against both the store and the
-// batch's own earlier entries — so two contracts claiming one address inside a
-// single batch conflict just as they would across batches. Verdicts come back
-// in the batch's order.
+// Registers dynamic registrations, resolving each address against both the
+// store and the batch's own earlier entries — so two contracts claiming one
+// address inside a single batch conflict just as they would across batches.
+// Verdicts come back in the batch's order. What it adds is pending persistence
+// until a batch write drains it.
+//
+// An unknown contract name throws with none of the batch applied.
 let registerBatch = (store: t, registrations: array<registration>): array<verdict> =>
   store->registerBatchRaw(registrations)->Array.map(toVerdict)
 
+// `registerBatch` for addresses the database already holds — config addresses
+// and the dynamic ones a resume restores. Nothing is marked pending, so nothing
+// is ever written back.
+let seedBatch = (store: t, registrations: array<registration>): array<verdict> =>
+  store->seedBatchRaw(registrations)->Array.map(toVerdict)
+
 let makeSet = (store: t, ~contractName, ~options={}: makeSetOptions) =>
   store->makeSetRaw(contractName, options)
 

package/src/sources/AddressStore.res.mjs

@@ -17,14 +17,22 @@ function make(ecosystem, shouldChecksum, contracts) {
   }
 }
 
-function contractsOf(onEventRegistrations) {
+function contractsOf(onEventRegistrations, configContractNames) {
   let startBlocks = {};
   let unrestricted = new Set();
+  let addressDependent = new Set();
   let names = [];
-  onEventRegistrations.forEach(reg => {
-    let name = reg.eventConfig.contractName;
+  let addName = name => {
     if (!names.includes(name)) {
       names.push(name);
+      return;
+    }
+  };
+  onEventRegistrations.forEach(reg => {
+    let name = reg.eventConfig.contractName;
+    addName(name);
+    if (reg.dependsOnAddresses) {
+      addressDependent.add(name);
     }
     let startBlock = reg.startBlock;
     if (startBlock !== undefined) {
@@ -34,9 +42,11 @@ function contractsOf(onEventRegistrations) {
     }
     unrestricted.add(name);
   });
+  configContractNames.forEach(addName);
   return names.map(name => ({
     name: name,
-    startBlock: unrestricted.has(name) ? undefined : startBlocks[name]
+    startBlock: unrestricted.has(name) ? undefined : startBlocks[name],
+    dependsOnAddresses: addressDependent.has(name)
   }));
 }
 
@@ -46,7 +56,8 @@ function toVerdict(raw) {
     case "added" :
       return {
         TAG: "Added",
-        effectiveStartBlock: raw.effectiveStartBlock
+        effectiveStartBlock: raw.effectiveStartBlock,
+        fetchable: raw.fetchable
       };
     case "conflict" :
       return {
@@ -61,11 +72,6 @@ function toVerdict(raw) {
       };
     case "invalid" :
       return "Invalid";
-    case "noEvents" :
-      return {
-        TAG: "NoEvents",
-        effectiveStartBlock: raw.effectiveStartBlock
-      };
     default:
       return Stdlib_JsError.throwWithMessage(`Unexpected address registration verdict "` + kind + `"`);
   }
@@ -75,6 +81,10 @@ function registerBatch(store, registrations) {
   return store.registerBatch(registrations).map(toVerdict);
 }
 
+function seedBatch(store, registrations) {
+  return store.seedBatch(registrations).map(toVerdict);
+}
+
 function makeSet(store, contractName, optionsOpt) {
   let options = optionsOpt !== undefined ? optionsOpt : ({});
   return store.makeSet(contractName, options);
@@ -89,6 +99,7 @@ export {
   contractsOf,
   toVerdict,
   registerBatch,
+  seedBatch,
   makeSet,
   get,
 }

package/src/sources/SvmHyperSyncClient.res

@@ -110,13 +110,13 @@ module QueryTypes = {
 
   type fieldSelection = {block?: array<blockField>, transaction?: array<transactionField>}
 
-  /** Filter for selecting instructions. All non-empty fields are AND-ed: an
-   instruction must match at least one value in every non-empty field.
+  /** Filter for selecting instruction calls. All non-empty fields are AND-ed:
+   an instruction must match at least one value in every non-empty field.
 
    Discriminator filters (d1..d8) take hex-encoded byte prefixes ("0x" optional).
    Account filters (a0..a9) take base58 pubkey strings. */
   type instructionSelection = {
-    programId?: array<string>,
+    executingAccount?: array<string>,
     d1?: array<string>,
     d2?: array<string>,
     d4?: array<string>,
@@ -129,7 +129,7 @@ module QueryTypes = {
   type query = {
     fromSlot: int,
     toSlot?: int,
-    instructions?: array<instructionSelection>,
+    instructionCalls?: array<instructionSelection>,
     includeAllBlocks?: bool,
     fields?: fieldSelection,
     maxNumBlocks?: int,
@@ -159,15 +159,16 @@ module ResponseTypes = {
     slot: int,
     transactionIndex: int,
     instructionAddress: array<int>,
-    programId: string,
-    accounts: array<string>,
+    executingAccount: string,
+    accountArguments: array<string>,
     data: string,
     d1?: string,
     d2?: string,
     d4?: string,
     d8?: string,
     isInner: bool,
-    isCommitted: bool,
+    // True when the parent transaction succeeded.
+    txSuccess: bool,
   }
 
   type queryResponseData = {

package/src/sources/SvmHyperSyncSource.res

@@ -67,8 +67,20 @@ let toSvmInstruction = (
   ),
 }
 
+// The server responds with next_slot == from_slot when the requested range is
+// beyond its queryable tip (its /height can run ahead of query availability,
+// routinely by a slot at the head and by much more during an ingest incident).
+exception NonAdvancingResponse({fromSlot: int, nextSlot: int})
+
 let make = (
-  {chainId, endpointUrl, apiToken, onEventRegistrations, clientTimeoutMillis, addressStore}: options,
+  {
+    chainId,
+    endpointUrl,
+    apiToken,
+    onEventRegistrations,
+    clientTimeoutMillis,
+    addressStore,
+  }: options,
 ): t => {
   let name = "SvmHyperSync"
 
@@ -131,6 +143,31 @@ let make = (
     let pageFetchTime = pageFetchRef->Performance.secondsSince
     let requestStats = [{Source.method: "getInstructions", seconds: pageFetchTime}]
 
+    // Accepting a non-advancing page would re-issue the identical query in a
+    // hot loop with no backoff (and its zero-width range would poison the
+    // partition's event-density average), so retry it with a bounded backoff
+    // until the server tip moves past the requested slot.
+    if resp.nextSlot <= fromBlock {
+      throw(
+        Source.GetItemsError(
+          Source.FailedGettingItems({
+            exn: NonAdvancingResponse({fromSlot: fromBlock, nextSlot: resp.nextSlot}),
+            attemptedToBlock: toBlock->Option.getOr(knownHeight),
+            retry: WithBackoff({
+              message: `SVM HyperSync has no data for slot ${fromBlock->Int.toString} yet (server tip is behind the reported height). Retrying until the server catches up.`,
+              backoffMillis: Pervasives.min(
+                switch retry {
+                | 0 => 500
+                | _ => 1000 * retry
+                },
+                5_000,
+              ),
+            }),
+          }),
+        ),
+      )
+    }
+
     let parsingRef = Performance.now()
 
     // Per-slot blockTime lookup from the response's `blocks` table, for the

package/src/sources/SvmHyperSyncSource.res.mjs

@@ -3,6 +3,7 @@
 import * as Source from "./Source.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as SvmHyperSyncClient from "./SvmHyperSyncClient.res.mjs";
@@ -55,6 +56,8 @@ function toSvmInstruction(item, programName, instructionName) {
   };
 }
 
+let NonAdvancingResponse = /* @__PURE__ */Primitive_exceptions.create("SvmHyperSyncSource.NonAdvancingResponse");
+
 function make(param) {
   let onEventRegistrations = param.onEventRegistrations;
   let chainId = param.chainId;
@@ -97,6 +100,26 @@ function make(param) {
         method: "getInstructions",
         seconds: pageFetchTime
       }];
+    if (resp.nextSlot <= fromBlock) {
+      throw {
+        RE_EXN_ID: Source.GetItemsError,
+        _1: {
+          TAG: "FailedGettingItems",
+          exn: {
+            RE_EXN_ID: NonAdvancingResponse,
+            fromSlot: fromBlock,
+            nextSlot: resp.nextSlot
+          },
+          attemptedToBlock: Stdlib_Option.getOr(toBlock, knownHeight),
+          retry: {
+            TAG: "WithBackoff",
+            message: `SVM HyperSync has no data for slot ` + fromBlock.toString() + ` yet (server tip is behind the reported height). Retrying until the server catches up.`,
+            backoffMillis: Primitive_int.min(retry !== 0 ? 1000 * retry | 0 : 500, 5000)
+          }
+        },
+        Error: new Error()
+      };
+    }
     let parsingRef = Performance.now();
     let blockTimeBySlot = {};
     resp.blocks.forEach(b => {
@@ -267,6 +290,7 @@ export {
   synthLogIndex,
   parseDecoded,
   toSvmInstruction,
+  NonAdvancingResponse,
   make,
 }
 /* SvmHyperSyncClient Not a pure module */

package/svm.schema.json

@@ -49,6 +49,13 @@
         }
       ]
     },
+    "disable_default_cross_chain": {
+      "description": "Make entities and effect caches per-chain instead of shared across every chain (recommended). Sharing then becomes explicit — add `@crossChain` to an entity in schema.graphql or `crossChain: true` to an effect. (default: false)",
+      "type": [
+        "boolean",
+        "null"
+      ]
+    },
     "ecosystem": {
       "description": "Ecosystem of the project.",
       "$ref": "#/$defs/EcosystemTag"
@@ -264,8 +271,15 @@
       "type": "object",
       "properties": {
         "hypersync_config": {
-          "description": "HyperSync Config for fetching historical instructions on this chain.",
-          "$ref": "#/$defs/HypersyncConfig"
+          "description": "HyperSync Config for fetching historical instructions on this chain. Defaults to the public Solana mainnet HyperSync endpoint.",
+          "anyOf": [
+            {
+              "$ref": "#/$defs/HypersyncConfig"
+            },
+            {
+              "type": "null"
+            }
+          ]
         },
         "programs": {
           "description": "Solana programs to index on this chain.",
@@ -277,7 +291,6 @@
       },
       "additionalProperties": false,
       "required": [
-        "hypersync_config",
         "programs"
       ]
     },
@@ -285,7 +298,7 @@
       "type": "object",
       "properties": {
         "url": {
-          "description": "URL of the HyperSync endpoint (default: the public Solana HyperSync endpoint at https://solana.hypersync.xyz)",
+          "description": "URL of the HyperSync endpoint. Defaults to the public Solana mainnet HyperSync endpoint when the hypersync_config block is omitted.",
           "type": "string"
         }
       },