@dedot/api 0.9.5-next.11b0ca8e.3 → 0.9.5-next.889fe7cd.3

package/cjs/json-rpc/group/ChainHead/ChainHead.js

@@ -22,6 +22,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
     #recovering;
     #blockUsage;
     #cache;
+    #operationQueue;
     constructor(client, options) {
         super(client, { prefix: 'chainHead', supportedVersions: ['unstable', 'v1'], ...options });
         this.#handlers = {};
@@ -32,6 +33,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
         this.#retryQueue = new utils_1.AsyncQueue();
         this.#blockUsage = new BlockUsage_js_1.BlockUsage();
         this.#cache = new Map();
+        this.#operationQueue = new utils_1.ThrottleQueue(this.#__unsafe__isSmoldot() ? 30 : 250);
     }
     async runtimeVersion() {
         await this.#ensureFollowed();
@@ -236,7 +238,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                     .then(() => {
                     // 3. Resolve the recovering promise
                     // This means to continue all pending requests while the chainHead started recovering mode at step 1.
-                    this.#recovering.resolve();
+                    this.#recovering?.resolve();
                     // 4. Recover stale operations
                     // 4.1. Operations that's going on & waiting to receiving its operationId
                     //     will eventually get an `limitedReached` error for using a stale followSubscriptionId
@@ -252,7 +254,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                     .catch((e) => {
                     console.error(e);
                     // TODO we should retry a few attempts
-                    this.#recovering.reject(new error_js_1.ChainHeadError('Cannot recover from stop event!'));
+                    this.#recovering?.reject(new error_js_1.ChainHeadError('Cannot recover from stop event!'));
                     Object.values(this.#handlers).forEach(({ defer, operationId }) => {
                         defer.reject(new error_js_1.ChainHeadError('Cannot recover from stop event!'));
                         delete this.#handlers[operationId];
@@ -400,6 +402,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
         this.#retryQueue.clear();
         this.#blockUsage.clear();
         this.#cache.clear();
+        this.#operationQueue.cancel();
     }
     async #ensureFollowed() {
         if (this.#recovering) {
@@ -485,7 +488,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                 const resp = await this.send('body', this.#subscriptionId, hash);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -514,7 +517,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                 const resp = await this.send('call', this.#subscriptionId, hash, func, params);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -557,9 +560,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
             }
             this.#blockUsage.use(hash);
             let results = [];
-            // @ts-ignore a trick internally to check whether a provider is using smoldot connection
-            const isSmoldot = typeof this.client.provider['chain'] === 'function';
-            if (isSmoldot) {
+            if (this.#__unsafe__isSmoldot()) {
                 const fetchItem = async (item) => {
                     const [batch, newDiscardedItems] = await this.#getStorage([item], childTrie ?? null, hash);
                     if (newDiscardedItems.length > 0) {
@@ -592,7 +593,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
     }
     async #getStorage(items, childTrie, at) {
         const operation = () => this.#getStorageOperation(items, childTrie, this.#ensurePinnedHash(at));
-        return this.#performOperationWithRetry(operation, at);
+        return this.#operationQueue.add(() => this.#performOperationWithRetry(operation, at));
     }
     async #getStorageOperation(items, childTrie, at) {
         await this.#ensureFollowed();
@@ -629,5 +630,9 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
             return;
         await this.send('unpin', this.#subscriptionId, hashes);
     }
+    #__unsafe__isSmoldot() {
+        // @ts-ignore  a trick internally to check whether a provider is using smoldot connection
+        return typeof this.client.provider['chain'] === 'function';
+    }
 }
 exports.ChainHead = ChainHead;

package/cjs/storage/NewStorageQuery.js

@@ -83,7 +83,13 @@ class NewStorageQuery extends BaseStorageQuery_js_1.BaseStorageQuery {
         await pull(best);
         // Subscribe to best block events
         const unsub = this.client.on('bestBlock', (block) => {
-            pullQueue.enqueue(() => pull(block)).catch(console.error);
+            // Here we're handling each pull one by one,
+            // If the queue get too long, it might take a long time for us to get the fresh & latest data
+            // This is a precaution in such case, if the queue size >= 3 we skip all the pending pull and jump to the latest pull
+            if (pullQueue.size >= 3)
+                pullQueue.clear();
+            // TODO timing out for a pull to prevent it took too long to fetch
+            pullQueue.enqueue(() => pull(block)).catch(utils_1.noop);
         });
         return async () => {
             unsub();

package/json-rpc/group/ChainHead/ChainHead.js

@@ -1,5 +1,5 @@
 import { $Header } from '@dedot/codecs';
-import { assert, AsyncQueue, deferred, ensurePresence, noop, waitFor } from '@dedot/utils';
+import { assert, AsyncQueue, deferred, ensurePresence, noop, ThrottleQueue, waitFor, } from '@dedot/utils';
 import { JsonRpcGroup } from '../JsonRpcGroup.js';
 import { BlockUsage } from './BlockUsage.js';
 import { ChainHeadBlockNotPinnedError, ChainHeadBlockPrunedError, ChainHeadError, ChainHeadLimitReachedError, ChainHeadOperationError, ChainHeadOperationInaccessibleError, ChainHeadStopError, RetryStrategy, } from './error.js';
@@ -19,6 +19,7 @@ export class ChainHead extends JsonRpcGroup {
     #recovering;
     #blockUsage;
     #cache;
+    #operationQueue;
     constructor(client, options) {
         super(client, { prefix: 'chainHead', supportedVersions: ['unstable', 'v1'], ...options });
         this.#handlers = {};
@@ -29,6 +30,7 @@ export class ChainHead extends JsonRpcGroup {
         this.#retryQueue = new AsyncQueue();
         this.#blockUsage = new BlockUsage();
         this.#cache = new Map();
+        this.#operationQueue = new ThrottleQueue(this.#__unsafe__isSmoldot() ? 30 : 250);
     }
     async runtimeVersion() {
         await this.#ensureFollowed();
@@ -233,7 +235,7 @@ export class ChainHead extends JsonRpcGroup {
                     .then(() => {
                     // 3. Resolve the recovering promise
                     // This means to continue all pending requests while the chainHead started recovering mode at step 1.
-                    this.#recovering.resolve();
+                    this.#recovering?.resolve();
                     // 4. Recover stale operations
                     // 4.1. Operations that's going on & waiting to receiving its operationId
                     //     will eventually get an `limitedReached` error for using a stale followSubscriptionId
@@ -249,7 +251,7 @@ export class ChainHead extends JsonRpcGroup {
                     .catch((e) => {
                     console.error(e);
                     // TODO we should retry a few attempts
-                    this.#recovering.reject(new ChainHeadError('Cannot recover from stop event!'));
+                    this.#recovering?.reject(new ChainHeadError('Cannot recover from stop event!'));
                     Object.values(this.#handlers).forEach(({ defer, operationId }) => {
                         defer.reject(new ChainHeadError('Cannot recover from stop event!'));
                         delete this.#handlers[operationId];
@@ -397,6 +399,7 @@ export class ChainHead extends JsonRpcGroup {
         this.#retryQueue.clear();
         this.#blockUsage.clear();
         this.#cache.clear();
+        this.#operationQueue.cancel();
     }
     async #ensureFollowed() {
         if (this.#recovering) {
@@ -482,7 +485,7 @@ export class ChainHead extends JsonRpcGroup {
                 const resp = await this.send('body', this.#subscriptionId, hash);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -511,7 +514,7 @@ export class ChainHead extends JsonRpcGroup {
                 const resp = await this.send('call', this.#subscriptionId, hash, func, params);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -554,9 +557,7 @@ export class ChainHead extends JsonRpcGroup {
             }
             this.#blockUsage.use(hash);
             let results = [];
-            // @ts-ignore a trick internally to check whether a provider is using smoldot connection
-            const isSmoldot = typeof this.client.provider['chain'] === 'function';
-            if (isSmoldot) {
+            if (this.#__unsafe__isSmoldot()) {
                 const fetchItem = async (item) => {
                     const [batch, newDiscardedItems] = await this.#getStorage([item], childTrie ?? null, hash);
                     if (newDiscardedItems.length > 0) {
@@ -589,7 +590,7 @@ export class ChainHead extends JsonRpcGroup {
     }
     async #getStorage(items, childTrie, at) {
         const operation = () => this.#getStorageOperation(items, childTrie, this.#ensurePinnedHash(at));
-        return this.#performOperationWithRetry(operation, at);
+        return this.#operationQueue.add(() => this.#performOperationWithRetry(operation, at));
     }
     async #getStorageOperation(items, childTrie, at) {
         await this.#ensureFollowed();
@@ -626,4 +627,8 @@ export class ChainHead extends JsonRpcGroup {
             return;
         await this.send('unpin', this.#subscriptionId, hashes);
     }
+    #__unsafe__isSmoldot() {
+        // @ts-ignore  a trick internally to check whether a provider is using smoldot connection
+        return typeof this.client.provider['chain'] === 'function';
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dedot/api",
-  "version": "0.9.5-next.11b0ca8e.3+11b0ca8",
+  "version": "0.9.5-next.889fe7cd.3+889fe7c",
   "description": "A delightful JavaScript/TypeScript client for Polkadot & Substrate",
   "author": "Thang X. Vu <thang@dedot.dev>",
   "homepage": "https://dedot.dev",
@@ -13,13 +13,13 @@
   "type": "module",
   "sideEffects": false,
   "dependencies": {
-    "@dedot/codecs": "0.9.5-next.11b0ca8e.3+11b0ca8",
-    "@dedot/providers": "0.9.5-next.11b0ca8e.3+11b0ca8",
-    "@dedot/runtime-specs": "0.9.5-next.11b0ca8e.3+11b0ca8",
-    "@dedot/shape": "0.9.5-next.11b0ca8e.3+11b0ca8",
-    "@dedot/storage": "0.9.5-next.11b0ca8e.3+11b0ca8",
-    "@dedot/types": "0.9.5-next.11b0ca8e.3+11b0ca8",
-    "@dedot/utils": "0.9.5-next.11b0ca8e.3+11b0ca8"
+    "@dedot/codecs": "0.9.5-next.889fe7cd.3+889fe7c",
+    "@dedot/providers": "0.9.5-next.889fe7cd.3+889fe7c",
+    "@dedot/runtime-specs": "0.9.5-next.889fe7cd.3+889fe7c",
+    "@dedot/shape": "0.9.5-next.889fe7cd.3+889fe7c",
+    "@dedot/storage": "0.9.5-next.889fe7cd.3+889fe7c",
+    "@dedot/types": "0.9.5-next.889fe7cd.3+889fe7c",
+    "@dedot/utils": "0.9.5-next.889fe7cd.3+889fe7c"
   },
   "scripts": {
     "build": "tsc --project tsconfig.build.json && tsc --project tsconfig.build.cjs.json",
@@ -48,7 +48,7 @@
     "node": ">=18"
   },
   "license": "Apache-2.0",
-  "gitHead": "11b0ca8e2df2c59e12259dc2ba0c61ec91e181cf",
+  "gitHead": "889fe7cd88f69202d4634cfe9fd25900b41a8935",
   "module": "./index.js",
   "types": "./index.d.ts"
 }

package/storage/NewStorageQuery.js

@@ -1,4 +1,4 @@
-import { AsyncQueue } from '@dedot/utils';
+import { AsyncQueue, noop } from '@dedot/utils';
 import { BaseStorageQuery } from './BaseStorageQuery.js';
 /**
  * @name NewStorageQuery
@@ -80,7 +80,13 @@ export class NewStorageQuery extends BaseStorageQuery {
         await pull(best);
         // Subscribe to best block events
         const unsub = this.client.on('bestBlock', (block) => {
-            pullQueue.enqueue(() => pull(block)).catch(console.error);
+            // Here we're handling each pull one by one,
+            // If the queue get too long, it might take a long time for us to get the fresh & latest data
+            // This is a precaution in such case, if the queue size >= 3 we skip all the pending pull and jump to the latest pull
+            if (pullQueue.size >= 3)
+                pullQueue.clear();
+            // TODO timing out for a pull to prevent it took too long to fetch
+            pullQueue.enqueue(() => pull(block)).catch(noop);
         });
         return async () => {
             unsub();