@dedot/api 0.9.2 → 0.9.4

package/cjs/json-rpc/group/ChainHead/ChainHead.js

@@ -410,7 +410,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
         }
         catch (e) {
             if (e instanceof error_js_1.ChainHeadError && e.retryStrategy) {
-                return this.#retryOperation(e.retryStrategy, operation);
+                return await this.#retryOperation(e.retryStrategy, operation);
             }
             throw e;
         }
@@ -437,7 +437,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
         catch (e) {
             // retry again until success, TODO we might need to limit the number of retries
             if (e instanceof error_js_1.ChainHeadError && e.retryStrategy) {
-                return this.#retryOperation(e.retryStrategy, retry);
+                return await this.#retryOperation(e.retryStrategy, retry);
             }
             throw e;
         }
@@ -550,19 +550,33 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                 return this.#cache.get(cacheKey);
             }
             this.#blockUsage.use(hash);
-            const results = [];
-            let queryItems = items;
-            while (queryItems.length > 0) {
-                const [newBatch, newDiscardedItems] = await this.#getStorage(queryItems, childTrie ?? null, hash);
-                results.push(...newBatch);
-                queryItems = newDiscardedItems;
+            let results = [];
+            // @ts-ignore a trick internally to check whether a provider is using smoldot connection
+            const isSmoldot = typeof this.client.provider['chain'] === 'function';
+            if (isSmoldot) {
+                const fetchItem = async (item) => {
+                    const [batch, newDiscardedItems] = await this.#getStorage([item], childTrie ?? null, hash);
+                    if (newDiscardedItems.length > 0) {
+                        return fetchItem(item);
+                    }
+                    return batch;
+                };
+                results = (await Promise.all(items.map((one) => fetchItem(one)))).flat();
+            }
+            else {
+                let queryItems = items;
+                while (queryItems.length > 0) {
+                    const [newBatch, newDiscardedItems] = await this.#getStorage(queryItems, childTrie ?? null, hash);
+                    results.push(...newBatch);
+                    queryItems = newDiscardedItems;
+                }
             }
             this.#cache.set(cacheKey, results);
             return results;
         }
         catch (e) {
             if (e instanceof error_js_1.ChainHeadBlockPrunedError && shouldRetryOnPrunedBlock) {
-                return this.storage(items, childTrie);
+                return await this.storage(items, childTrie);
             }
             throw e;
         }

package/cjs/storage/NewStorageQuery.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.NewStorageQuery = void 0;
+const utils_1 = require("@dedot/utils");
 const BaseStorageQuery_js_1 = require("./BaseStorageQuery.js");
 /**
  * @name NewStorageQuery
@@ -51,6 +52,11 @@ class NewStorageQuery extends BaseStorageQuery_js_1.BaseStorageQuery {
         const best = await this.client.chainHead.bestBlock();
         // Track the latest changes for each key
         const latestChanges = {};
+        // Using a queue here to make sure we don't accidentally
+        // put a pressure the json-rpc server in-case new blocks stack up too fast
+        // in case we send a lot of requests at the same time
+        // or the block time is small enough with elastic scaling
+        const pullQueue = new utils_1.AsyncQueue();
         // Function to pull storage values and call the callback if there are changes
         const pull = async ({ hash }) => {
             // Query storage using ChainHead API
@@ -76,9 +82,12 @@ class NewStorageQuery extends BaseStorageQuery_js_1.BaseStorageQuery {
         // Initial pull
         await pull(best);
         // Subscribe to best block events
-        const unsub = this.client.on('bestBlock', pull);
+        const unsub = this.client.on('bestBlock', (block) => {
+            pullQueue.enqueue(() => pull(block)).catch(console.error);
+        });
         return async () => {
             unsub();
+            pullQueue.cancel();
         };
     }
 }

package/json-rpc/group/ChainHead/ChainHead.js

@@ -407,7 +407,7 @@ export class ChainHead extends JsonRpcGroup {
         }
         catch (e) {
             if (e instanceof ChainHeadError && e.retryStrategy) {
-                return this.#retryOperation(e.retryStrategy, operation);
+                return await this.#retryOperation(e.retryStrategy, operation);
             }
             throw e;
         }
@@ -434,7 +434,7 @@ export class ChainHead extends JsonRpcGroup {
         catch (e) {
             // retry again until success, TODO we might need to limit the number of retries
             if (e instanceof ChainHeadError && e.retryStrategy) {
-                return this.#retryOperation(e.retryStrategy, retry);
+                return await this.#retryOperation(e.retryStrategy, retry);
             }
             throw e;
         }
@@ -547,19 +547,33 @@ export class ChainHead extends JsonRpcGroup {
                 return this.#cache.get(cacheKey);
             }
             this.#blockUsage.use(hash);
-            const results = [];
-            let queryItems = items;
-            while (queryItems.length > 0) {
-                const [newBatch, newDiscardedItems] = await this.#getStorage(queryItems, childTrie ?? null, hash);
-                results.push(...newBatch);
-                queryItems = newDiscardedItems;
+            let results = [];
+            // @ts-ignore a trick internally to check whether a provider is using smoldot connection
+            const isSmoldot = typeof this.client.provider['chain'] === 'function';
+            if (isSmoldot) {
+                const fetchItem = async (item) => {
+                    const [batch, newDiscardedItems] = await this.#getStorage([item], childTrie ?? null, hash);
+                    if (newDiscardedItems.length > 0) {
+                        return fetchItem(item);
+                    }
+                    return batch;
+                };
+                results = (await Promise.all(items.map((one) => fetchItem(one)))).flat();
+            }
+            else {
+                let queryItems = items;
+                while (queryItems.length > 0) {
+                    const [newBatch, newDiscardedItems] = await this.#getStorage(queryItems, childTrie ?? null, hash);
+                    results.push(...newBatch);
+                    queryItems = newDiscardedItems;
+                }
             }
             this.#cache.set(cacheKey, results);
             return results;
         }
         catch (e) {
             if (e instanceof ChainHeadBlockPrunedError && shouldRetryOnPrunedBlock) {
-                return this.storage(items, childTrie);
+                return await this.storage(items, childTrie);
             }
             throw e;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dedot/api",
-  "version": "0.9.2",
+  "version": "0.9.4",
   "description": "A delightful JavaScript/TypeScript client for Polkadot & Substrate",
   "author": "Thang X. Vu <thang@dedot.dev>",
   "homepage": "https://dedot.dev",
@@ -13,13 +13,13 @@
   "type": "module",
   "sideEffects": false,
   "dependencies": {
-    "@dedot/codecs": "0.9.2",
-    "@dedot/providers": "0.9.2",
-    "@dedot/runtime-specs": "0.9.2",
-    "@dedot/shape": "0.9.2",
-    "@dedot/storage": "0.9.2",
-    "@dedot/types": "0.9.2",
-    "@dedot/utils": "0.9.2"
+    "@dedot/codecs": "0.9.4",
+    "@dedot/providers": "0.9.4",
+    "@dedot/runtime-specs": "0.9.4",
+    "@dedot/shape": "0.9.4",
+    "@dedot/storage": "0.9.4",
+    "@dedot/types": "0.9.4",
+    "@dedot/utils": "0.9.4"
   },
   "scripts": {
     "build": "tsc --project tsconfig.build.json && tsc --project tsconfig.build.cjs.json",
@@ -48,7 +48,7 @@
     "node": ">=18"
   },
   "license": "Apache-2.0",
-  "gitHead": "b28ae48371785d44bf734cf45a5b8d0716d9bd70",
+  "gitHead": "1deabbf35425a688e76f1612210bcda3f7bedba3",
   "module": "./index.js",
   "types": "./index.d.ts"
 }

package/storage/NewStorageQuery.js

@@ -1,3 +1,4 @@
+import { AsyncQueue } from '@dedot/utils';
 import { BaseStorageQuery } from './BaseStorageQuery.js';
 /**
  * @name NewStorageQuery
@@ -48,6 +49,11 @@ export class NewStorageQuery extends BaseStorageQuery {
         const best = await this.client.chainHead.bestBlock();
         // Track the latest changes for each key
         const latestChanges = {};
+        // Using a queue here to make sure we don't accidentally
+        // put a pressure the json-rpc server in-case new blocks stack up too fast
+        // in case we send a lot of requests at the same time
+        // or the block time is small enough with elastic scaling
+        const pullQueue = new AsyncQueue();
         // Function to pull storage values and call the callback if there are changes
         const pull = async ({ hash }) => {
             // Query storage using ChainHead API
@@ -73,9 +79,12 @@ export class NewStorageQuery extends BaseStorageQuery {
         // Initial pull
         await pull(best);
         // Subscribe to best block events
-        const unsub = this.client.on('bestBlock', pull);
+        const unsub = this.client.on('bestBlock', (block) => {
+            pullQueue.enqueue(() => pull(block)).catch(console.error);
+        });
         return async () => {
             unsub();
+            pullQueue.cancel();
         };
     }
 }