envio 3.3.0-alpha.8 → 3.3.0-rc.0

package/index.d.ts

@@ -48,6 +48,13 @@ export type EffectCaller = <I, O>(
   input: I extends undefined ? undefined : I
 ) => Promise<O>;
 
+/** The chain an Effect was called on. Available only on chain-scoped effects
+ * (`crossChain: false`). */
+export type EffectChain = {
+  /** The chain id the effect handler was called on. */
+  readonly id: number;
+};
+
 /** Context passed to an Effect's handler function. */
 export type EffectContext = {
   /** Access the logger instance with the event as context. */
@@ -57,6 +64,9 @@ export type EffectContext = {
   /** Whether to cache this call's result. Defaults to the effect's `cache`
    * option; set to `false` to skip caching for this specific invocation. */
   cache: boolean;
+  /** The chain the effect was called on. Only available on effects created with
+   * `crossChain: false`; accessing it on a cross-chain effect throws. */
+  readonly chain: EffectChain;
 };
 
 /** Rate-limit window for an {@link Effect}. Strings resolve to common
@@ -82,6 +92,11 @@ export type EffectOptions<Input, Output> = {
   readonly rateLimit: RateLimit;
   /** Whether the effect should be cached. */
   readonly cache?: boolean;
+  /** Whether the effect's cache is shared across all chains. Defaults to
+   * `true`. Set to `false` to isolate the cache and rate limiting per chain and
+   * enable `context.chain.id` inside the handler. Changing this changes the
+   * effect's cache identity. */
+  readonly crossChain?: boolean;
 };
 
 /** Arguments passed to the handler function of an {@link Effect}. */
@@ -209,6 +224,10 @@ export function createEffect<
     readonly rateLimit: RateLimit;
     /** Whether the effect should be cached. */
     readonly cache?: boolean;
+    /** Whether the effect's cache is shared across all chains. Defaults to
+     * `true`. Set to `false` to isolate the cache and rate limiting per chain
+     * and enable `context.chain.id` inside the handler. */
+    readonly crossChain?: boolean;
   },
   handler: (args: EffectArgs<I>) => Promise<R>
 ): Effect<I, O>;

package/licenses/CLA.md

@@ -0,0 +1,35 @@
+# Contributor License Agreement (CLA)
+
+## 1. Introduction
+
+Thank you for your interest in contributing to this project ("Project"). We value your time and effort, and we appreciate all contributions made by the community. By submitting any work (a "Contribution"), you agree to the terms below.
+
+## 2. Automatic Acceptance
+
+By submitting a Pull Request or making any contribution to this project, you automatically agree to and accept all terms outlined in this Contributor License Agreement. This includes all future contributions you may make to the project.
+
+## 3. Your Contributions
+
+3.1 A "Contribution" includes any original work you share with the Project, such as code, documentation, or other creative content.
+
+3.2 You confirm that each Contribution is either your own original work or that you have all necessary rights and permissions to submit it under this CLA.
+
+## 4. License Grant
+
+4.1 By submitting a Contribution, you grant Envio (the "Maintainer") an irrevocable, worldwide, non-exclusive, royalty-free, perpetual license to use, reproduce, modify, distribute, prepare derivative works of, publicly display, perform, sublicense, and otherwise exploit your Contribution as part of the Project.
+
+4.2 You agree that the Maintainer may license or re-license your Contribution under terms compatible with those of the Project.
+
+## 5. Ongoing Use
+
+5.1 Once provided, Contributions cannot be retracted; however, the Maintainer is not obligated to include or continue using your Contribution.
+
+5.2 You understand that this CLA helps ensure your work remains compatible with the Project's licensing, now and in the future.
+
+## 6. Disclaimer
+
+6.1 You provide your Contributions on an "as is" basis, without warranties of any kind. This includes, but is not limited to, warranties of merchantability, fitness for a particular purpose, and non-infringement.
+
+## 7. Governing Law
+
+7.1 This CLA is governed by and construed in accordance with the laws of England. Any disputes will be subject to the exclusive jurisdiction of the English courts.

package/licenses/EULA.md

@@ -0,0 +1,67 @@
+## HyperIndex End-User License Agreement (EULA)
+
+This agreement describes the users' rights and the conditions upon which the Software and Generated Code may be used. The user should review the entire agreement, including any supplemental license terms that accompany the Software, since all of the terms are important and together create this agreement that applies to them.
+
+### 1. Definitions
+
+**Software:** HyperIndex, a copyrightable work created by Envio and licensed under this End User License Agreement (“EULA”).
+
+**Generated Code:** In the context of this license agreement, the term "generated code" refers to computer programming code that is produced automatically by the Software based on input provided by the user.
+
+**Licensed Material:** The Software and Generated Code defined here will be collectively referred to as “Licensed Material”.
+
+### 2. Installation and User Rights
+
+**License:** The Software is provided under this EULA. By agreeing to the EULA terms, you are granted the right to install and operate one instance of the Software on your device (referred to as the licensed device), for the use of one individual at a time, on the condition that you adhere to all terms outlined in this agreement.
+The licensor provides you with a non-exclusive, royalty-free, worldwide license that is non-sublicensable and non-transferable. This license allows you to use the Software subject to the limitations and conditions outlined in this EULA.
+With one license, the user can only use the Software on a single device.
+
+**Device:** In this agreement, "device" refers to a hardware system, whether physical or virtual, equipped with an internal storage device capable of executing the Software. This includes hardware partitions, which are considered as individual devices for the purposes of this agreement. Updates may be provided to the Software, and these updates may alter the minimum hardware requirements necessary for the Software. It is the responsibility of users to comply with any changing hardware requirements.
+
+**Updates:** The Software may be updated automatically. With each update, the EULA may be amended, and it is the users' responsibility to comply with the amendments.
+
+**Limitations:** Envio reserves all rights, including those under intellectual property laws, not expressly granted in this agreement. For instance, this license does not confer upon you the right to, and you are prohibited from:
+
+(i) Publishing, copying (other than the permitted backup copy), renting, leasing, or lending the Software;
+
+(ii) Transferring the Software (except as permitted by this agreement);
+
+(iii) Circumventing any technical restrictions or limitations in the Software;
+
+(iv) Using the Software as server Software, for commercial hosting, making the Software available for simultaneous use by multiple users over a network, installing the Software on a server and allowing users to access it remotely, or installing the Software on a device solely for remote user use;
+
+(v) Reverse engineering, decompiling, or disassembling the Software, or attempting to do so, except and only to the extent that the foregoing restriction is (a) permitted by applicable law; (b) permitted by licensing terms governing the use of open-source components that may be included with the Software and
+
+(vi) When using the Software, you may not use any features in any manner that could interfere with anyone else's use of them, or attempt to gain unauthorized access to or use of any service, data, account, or network.
+
+These limitations apply specifically to the Software and do not extend to the Generated Code. Details regarding the use of the Generated Code, including associated limitations, are provided below.
+
+### 3. Use of the Generated Code
+
+**Limitations:** Users can use, copy, distribute, make available, and create derivative works of the Generated Code freely, subject to the limitations and conditions specified below.
+
+(i) The user is prohibited from offering the Generated Code or any software that includes the Generated Code to third parties as a hosted or managed service, where the service grants users access to a significant portion of the Software's features or functionality.
+
+(ii) The user is not permitted to tamper with, alter, disable, or bypass the functionality of the license key in the Software. Additionally, the user may not eliminate or conceal any functionality within the Software that is safeguarded by the license key.
+
+(iii) Any modification, removal, or concealment of licensing, copyright, or other notices belonging to the licensor in the Software is strictly forbidden. The use of the licensor's trademarks is subject to relevant laws.
+
+**Credit:** If the user utilizes the Generated Code to develop and release a new software, product or service, the license agreement for said software, product or service must include proper credit to HyperIndex.
+
+**Liability:** Envio does not provide any assurance that the Generated Code functions correctly, nor does it assume any responsibility in this regard.
+
+Additionally, it will be the responsibility of the user to assess whether the Generated Code is suitable for the products and services provided by the user. Envio will not bear any responsibility if the Generated Code is found unsuitable for the products and services provided by the user.
+
+### 4. Additional Terms
+
+**Disclaimer of Warranties and Limitation of Liability:**
+
+(i) Unless expressly undertaken by the Licensor separately, the Licensed Material is provided on an as-is, as-available basis, and the Licensor makes no representations or warranties of any kind regarding the Licensed Material, whether express, implied, statutory, or otherwise. This encompasses, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether known or discoverable. If disclaimers of warranties are not permitted in whole or in part, this disclaimer may not apply to You.
+
+(ii) To the fullest extent permitted by law, under no circumstances shall the Licensor be liable to You under any legal theory (including, but not limited to, negligence) for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising from the use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. If limitations of liability are not permitted in whole or in part, this limitation may not apply to You.
+
+(iii) The disclaimers of warranties and limitations of liability outlined above shall be construed in a manner that most closely approximates an absolute disclaimer and waiver of all liability, to the fullest extent permitted by law.
+
+**Applicable Law and Competent Courts:** This EULA and shall be governed by and construed in accordance with the laws of England. The courts of England shall have exclusive jurisdiction to settle any dispute arising out of or in connection with this EULA.
+
+**Additional Agreements:** If the user chooses to use the Software, it may be required to agree to additional terms or agreements outside of this EULA.

package/licenses/LICENSE.md

@@ -0,0 +1,45 @@
+# Standard SaaS End User License Agreement (EULA)
+
+## End User License Agreement (EULA)
+
+This End User License Agreement ("Agreement") is between Envio ("Company") and the user ("User") who downloads, installs, or uses the Software as a Service (SaaS) provided by the Company.
+
+### 1. License Grant
+
+1.1 Company grants User a non-exclusive, non-transferable, limited license to access and use the SaaS for business or personal purposes in accordance with this Agreement.
+
+### 2. Restrictions
+
+2.1 User shall not:
+
+- Modify, copy, or create derivative works based on the SaaS.
+- Reverse engineer, decompile, or disassemble the SaaS.
+- Rent, lease, lend, sell, redistribute, or sublicense the SaaS.
+- Use the SaaS for any unlawful purpose.
+
+### 3. Ownership
+
+3.1 The Company retains all rights, title, and interest in and to the SaaS, including all intellectual property rights.
+
+### 4. Term and Termination
+
+4.1 This Agreement is effective until terminated. User’s rights under this Agreement will terminate automatically without notice if User fails to comply with any term(s) of this Agreement.
+4.2 Upon termination, User must cease all use of the SaaS and destroy all copies of any related documentation.
+
+### 5. Disclaimer of Warranties
+
+5.1 The SaaS is provided "AS IS" without warranty of any kind, either express or implied, including, but not limited to, implied warranties of merchantability and fitness for a particular purpose.
+
+### 6. Limitation of Liability
+
+6.1 In no event shall Company be liable for any special, incidental, indirect, or consequential damages whatsoever arising out of the use of or inability to use the SaaS.
+
+### 7. Governing Law
+
+7.1 This Agreement shall be governed by and construed in accordance with the laws of the United Kingdom.
+
+### 8. Entire Agreement
+
+8.1 This Agreement constitutes the entire agreement between the parties with respect to the use of the SaaS and supersedes all prior or contemporaneous understandings regarding such subject matter.
+
+**By using the SaaS, the User agrees to be bound by this Agreement.**

package/licenses/README.md

@@ -0,0 +1,35 @@
+# Licensing Information
+
+## Overview
+
+Envio's licensing reflects open source ethos but is not OSI recognized. Developers can use Envio's services without vendor lock-in, either by self-hosting or specifying an RPC URL. The generated code is open and public and so is this repository that generates the generated code.
+
+Our licenses allows self-hosting but restricts third-party hosting with Envio's hosted service. Envio may consider open-sourcing in the future but prioritizes stakeholder interests and market traction. Please reach out to us on discord if you would like to explore hosting indexers as a third-party.
+
+## TL;DR
+
+- **Envio's licensing** mirrors open source benefits but is not OSI recognized.
+- **Developers** can use Envio's services without vendor lock-in by:
+  - Self-hosting.
+  - Specifying an RPC URL.
+- **Code** is open and public.
+- **Generated license** allows self-hosting but restricts third-party competition.
+- **Code generator license** is a commercial license.
+  - **non-comercial use** is allowed.
+  - **forking and modifying code** is not allowed. However, if you have specific needs or use cases, please contact us to discuss potential exceptions.
+- **Future considerations** for open-sourcing are dependent on market traction and stakeholder interests.
+- **Contributors** must agree to our Contributor License Agreement (CLA).
+
+## Our Position
+
+We're developers who value the open source ethos, which is why our licensing mirrors many benefits of open source licensing. However, Envio and its products do not use a recognized open source license by the OSI. We are public and open, and our licensing reflects this.
+
+Our future business model lies in our hosted service and HyperSync requests, and we are protecting this. To ensure continuity and avoid vendor lock-in, developers can run and develop on their indexer without our hosted service. This can be achieved either by self-hosting, which our license permits, or by specifying an RPC URL in their indexer configuration, thus bypassing HyperSync.
+
+Envio is in its formative stages. Although we may consider open-sourcing the software in the future, we are committed to ensuring the best interests of all stakeholders. Going open source is somewhat of a one-way function; it's easier to open source than to go "closed source." Once we have gained more market traction, we will review our position on going open source.
+
+## Our Licenses
+
+- [EULA](./EULA.md) - The license in your indexer pertaining to the generated code
+- [LICENSE](./LICENSE.md) - The non-commercial use license pertaining to this code base which is used to template & generate the indexing code
+- [CLA](./CLA.md) - The Contributor License Agreement for those who wish to contribute to the project

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.0-alpha.8",
+  "version": "3.3.0-rc.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -11,7 +11,7 @@
     "url": "git+https://github.com/enviodev/hyperindex.git"
   },
   "author": "envio contributors <about@envio.dev>",
-  "license": "GPL-3.0",
+  "license": "SEE LICENSE IN licenses/README.md",
   "bugs": {
     "url": "https://github.com/enviodev/hyperindex/issues"
   },
@@ -37,7 +37,8 @@
     "rescript.json",
     "index.d.ts",
     "index.js",
-    "src"
+    "src",
+    "licenses"
   ],
   "dependencies": {
     "@clickhouse/client": "1.17.0",
@@ -69,10 +70,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.0-alpha.8",
-    "envio-linux-x64-musl": "3.3.0-alpha.8",
-    "envio-linux-arm64": "3.3.0-alpha.8",
-    "envio-darwin-x64": "3.3.0-alpha.8",
-    "envio-darwin-arm64": "3.3.0-alpha.8"
+    "envio-linux-x64": "3.3.0-rc.0",
+    "envio-linux-x64-musl": "3.3.0-rc.0",
+    "envio-linux-arm64": "3.3.0-rc.0",
+    "envio-darwin-x64": "3.3.0-rc.0",
+    "envio-darwin-arm64": "3.3.0-rc.0"
   }
 }

package/src/Batch.res.mjs

@@ -27,7 +27,7 @@ function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressB
     let progressedChain;
     if (batchSize !== undefined) {
       let leftItems = fetchState.buffer.slice(batchSize);
-      progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined), batchSize);
+      progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined, undefined), batchSize);
     } else {
       progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
     }

package/src/BatchProcessing.res

@@ -180,13 +180,6 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
           )
         } else {
           ChainMetadata.stage(state)
-          if (
-            state
-            ->IndexerState.config
-            ->Config.shouldPruneHistory(~isInReorgThreshold=state->IndexerState.isInReorgThreshold)
-          ) {
-            state->PruneStaleHistory.schedule
-          }
         }
       }
     }

package/src/BatchProcessing.res.mjs

@@ -1,7 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
@@ -11,7 +10,6 @@ import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
 import * as EventProcessing from "./EventProcessing.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
-import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
 
@@ -79,12 +77,7 @@ async function processNextBatch(state, scheduleFetch) {
   } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {
     return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));
   } else {
-    ChainMetadata.stage(state);
-    if (Config.shouldPruneHistory(IndexerState.config(state), IndexerState.isInReorgThreshold(state))) {
-      return PruneStaleHistory.schedule(state);
-    } else {
-      return;
-    }
+    return ChainMetadata.stage(state);
   }
 }
 

package/src/ChainFetching.res

@@ -49,9 +49,9 @@ let runContractRegistersOrThrow = async (
   for idx in 0 to itemsWithContractRegister->Array.length - 1 {
     let item = itemsWithContractRegister->Array.getUnsafe(idx)
     let eventItem = item->Internal.castUnsafeEventItem
-    let contractRegister = switch eventItem {
-    | {onEventRegistration: {contractRegister: Some(contractRegister)}} => contractRegister
-    | {onEventRegistration: {contractRegister: None, eventConfig: {name: eventName}}} =>
+    let contractRegister = switch eventItem.onEventRegistration {
+    | {contractRegister: Some(contractRegister)} => contractRegister
+    | {contractRegister: None, eventConfig: {name: eventName}} =>
       // Unexpected case, since we should pass only events with contract register to this function
       JsError.throwWithMessage("Contract register is not set for event " ++ eventName)
     }
@@ -151,15 +151,26 @@ let rec onQueryResponse = async (
       let eventItem = item->Internal.castUnsafeEventItem
       eventItem.onEventRegistration.contractRegister !== None ? count + 1 : count
     })
-    Logging.trace({
-      "msg": "Finished querying",
-      "chainId": chain->ChainMap.Chain.toChainId,
-      "partitionId": query.partitionId,
-      "fromBlock": fromBlockQueried,
-      "toBlock": latestFetchedBlockNumber,
-      "numEvents": parsedQueueItems->Array.length,
-      "numContractRegisterEvents": numContractRegisterEvents,
-    })
+    if numContractRegisterEvents === 0 {
+      Logging.trace({
+        "msg": "Finished querying",
+        "chainId": chain->ChainMap.Chain.toChainId,
+        "partitionId": query.partitionId,
+        "fromBlock": fromBlockQueried,
+        "toBlock": latestFetchedBlockNumber,
+        "numEvents": parsedQueueItems->Array.length,
+      })
+    } else {
+      Logging.trace({
+        "msg": "Finished querying",
+        "chainId": chain->ChainMap.Chain.toChainId,
+        "partitionId": query.partitionId,
+        "fromBlock": fromBlockQueried,
+        "toBlock": latestFetchedBlockNumber,
+        "numEvents": parsedQueueItems->Array.length,
+        "numContractRegisterEvents": numContractRegisterEvents,
+      })
+    }
 
     let reorgResult = chainState->ChainState.registerReorgGuard(~blockHashes, ~knownHeight)
 
@@ -212,17 +223,18 @@ let rec onQueryResponse = async (
       // kick (eg from the processing loop quiescing) collapses into this one.
       scheduleRollback()
     | None =>
+      // Drop over-fetched events (a merged partition returning an address before
+      // its effectiveStartBlock, or a wildcard param referencing an address
+      // registered after the log's block) before contract registration, so they
+      // neither spawn dynamic contracts nor enter the buffer.
+      let newItems = chainState->ChainState.filterByClientAddress(parsedQueueItems)
       let itemsWithContractRegister = []
-      let newItems = []
-      for idx in 0 to parsedQueueItems->Array.length - 1 {
-        let item = parsedQueueItems->Array.getUnsafe(idx)
+      for idx in 0 to newItems->Array.length - 1 {
+        let item = newItems->Array.getUnsafe(idx)
         let eventItem = item->Internal.castUnsafeEventItem
         if eventItem.onEventRegistration.contractRegister !== None {
           itemsWithContractRegister->Array.push(item)
         }
-        // TODO: Don't really need to keep it in the queue
-        // when there's no handler (besides raw_events, processed counter, and dcsToStore consuming)
-        newItems->Array.push(item)
       }
 
       // Re-check staleness: contract registration is async, so the chain state

package/src/ChainFetching.res.mjs

@@ -111,15 +111,26 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       return count;
     }
   });
-  Logging.trace({
-    msg: "Finished querying",
-    chainId: chain,
-    partitionId: query.partitionId,
-    fromBlock: fromBlockQueried,
-    toBlock: latestFetchedBlockNumber,
-    numEvents: parsedQueueItems.length,
-    numContractRegisterEvents: numContractRegisterEvents
-  });
+  if (numContractRegisterEvents === 0) {
+    Logging.trace({
+      msg: "Finished querying",
+      chainId: chain,
+      partitionId: query.partitionId,
+      fromBlock: fromBlockQueried,
+      toBlock: latestFetchedBlockNumber,
+      numEvents: parsedQueueItems.length
+    });
+  } else {
+    Logging.trace({
+      msg: "Finished querying",
+      chainId: chain,
+      partitionId: query.partitionId,
+      fromBlock: fromBlockQueried,
+      toBlock: latestFetchedBlockNumber,
+      numEvents: parsedQueueItems.length,
+      numContractRegisterEvents: numContractRegisterEvents
+    });
+  }
   let reorgResult = ChainState.registerReorgGuard(chainState, response.blockHashes, knownHeight);
   let rollbackWithReorgDetectedBlockNumber;
   if (typeof reorgResult !== "object") {
@@ -139,14 +150,13 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
     IndexerState.beginReorg(state, chain, rollbackWithReorgDetectedBlockNumber);
     return scheduleRollback();
   }
+  let newItems = ChainState.filterByClientAddress(chainState, parsedQueueItems);
   let itemsWithContractRegister = [];
-  let newItems = [];
-  for (let idx = 0, idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx) {
-    let item = parsedQueueItems[idx];
+  for (let idx = 0, idx_finish = newItems.length; idx < idx_finish; ++idx) {
+    let item = newItems[idx];
     if (item.onEventRegistration.contractRegister !== undefined) {
       itemsWithContractRegister.push(item);
     }
-    newItems.push(item);
   }
   let proceed = newItemsWithDcs => {
     if (!IndexerState.isStale(state, stateId)) {