@dedot/api 0.9.5-next.20891389.0 → 0.9.5

package/cjs/json-rpc/group/ChainHead/ChainHead.js

@@ -22,6 +22,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
     #recovering;
     #blockUsage;
     #cache;
+    #operationQueue;
     constructor(client, options) {
         super(client, { prefix: 'chainHead', supportedVersions: ['unstable', 'v1'], ...options });
         this.#handlers = {};
@@ -32,6 +33,8 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
         this.#retryQueue = new utils_1.AsyncQueue();
         this.#blockUsage = new BlockUsage_js_1.BlockUsage();
         this.#cache = new Map();
+        // This helps us to not accidentally putting too much stress on the JSON-RPC server, especially smoldot/light-client
+        this.#operationQueue = new utils_1.ThrottleQueue(this.#__unsafe__isSmoldot() ? 25 : 250);
     }
     async runtimeVersion() {
         await this.#ensureFollowed();
@@ -80,11 +83,18 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
         const defer = (0, utils_1.deferred)();
         try {
             this.#unsub && this.#unsub().catch(utils_1.noop); // ensure unfollowed
+            let signals = 0;
             this.#unsub = await this.send('follow', true, (event, subscription) => {
                 this.#followResponseQueue
                     .enqueue(async () => {
                     await this.#onFollowEvent(event, subscription);
-                    if (event.event == 'initialized') {
+                    if (signals >= 2)
+                        return;
+                    signals += 1;
+                    // Sometime smoldot send a `stop` event right after `initialized`
+                    // So requests sending between `initialized` and `stop` will be on pruned hashes -> throwing out errors
+                    // Here we make sure to receive at least the first 2 signals to resolve
+                    if (signals >= 2) {
                         defer.resolve();
                     }
                 })
@@ -229,7 +239,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                     .then(() => {
                     // 3. Resolve the recovering promise
                     // This means to continue all pending requests while the chainHead started recovering mode at step 1.
-                    this.#recovering.resolve();
+                    this.#recovering?.resolve();
                     // 4. Recover stale operations
                     // 4.1. Operations that's going on & waiting to receiving its operationId
                     //     will eventually get an `limitedReached` error for using a stale followSubscriptionId
@@ -245,7 +255,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                     .catch((e) => {
                     console.error(e);
                     // TODO we should retry a few attempts
-                    this.#recovering.reject(new error_js_1.ChainHeadError('Cannot recover from stop event!'));
+                    this.#recovering?.reject(new error_js_1.ChainHeadError('Cannot recover from stop event!'));
                     Object.values(this.#handlers).forEach(({ defer, operationId }) => {
                         defer.reject(new error_js_1.ChainHeadError('Cannot recover from stop event!'));
                         delete this.#handlers[operationId];
@@ -393,6 +403,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
         this.#retryQueue.clear();
         this.#blockUsage.clear();
         this.#cache.clear();
+        this.#operationQueue.cancel();
     }
     async #ensureFollowed() {
         if (this.#recovering) {
@@ -422,17 +433,15 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
     async #retryOperation(strategy, retry) {
         try {
             return await new Promise((resolve, reject) => {
-                setTimeout(() => {
-                    if (strategy === error_js_1.RetryStrategy.NOW) {
-                        retry().then(resolve).catch(reject);
-                    }
-                    else if (strategy === error_js_1.RetryStrategy.QUEUED) {
-                        this.#retryQueue.enqueue(retry).then(resolve).catch(reject);
-                    }
-                    else {
-                        throw new Error('Invalid retry strategy');
-                    }
-                }); // retry again in the next tick
+                if (strategy === error_js_1.RetryStrategy.NOW) {
+                    retry().then(resolve).catch(reject);
+                }
+                else if (strategy === error_js_1.RetryStrategy.QUEUED) {
+                    this.#retryQueue.enqueue(retry).then(resolve).catch(reject);
+                }
+                else {
+                    throw new Error('Invalid retry strategy');
+                }
             });
         }
         catch (e) {
@@ -480,7 +489,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                 const resp = await this.send('body', this.#subscriptionId, hash);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -509,7 +518,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
                 const resp = await this.send('call', this.#subscriptionId, hash, func, params);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -552,9 +561,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
             }
             this.#blockUsage.use(hash);
             let results = [];
-            // @ts-ignore a trick internally to check whether a provider is using smoldot connection
-            const isSmoldot = typeof this.client.provider['chain'] === 'function';
-            if (isSmoldot) {
+            if (this.#__unsafe__isSmoldot()) {
                 const fetchItem = async (item) => {
                     const [batch, newDiscardedItems] = await this.#getStorage([item], childTrie ?? null, hash);
                     if (newDiscardedItems.length > 0) {
@@ -587,7 +594,7 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
     }
     async #getStorage(items, childTrie, at) {
         const operation = () => this.#getStorageOperation(items, childTrie, this.#ensurePinnedHash(at));
-        return this.#performOperationWithRetry(operation, at);
+        return this.#operationQueue.add(() => this.#performOperationWithRetry(operation, at));
     }
     async #getStorageOperation(items, childTrie, at) {
         await this.#ensureFollowed();
@@ -624,5 +631,9 @@ class ChainHead extends JsonRpcGroup_js_1.JsonRpcGroup {
             return;
         await this.send('unpin', this.#subscriptionId, hashes);
     }
+    #__unsafe__isSmoldot() {
+        // @ts-ignore  a trick internally to check whether a provider is using smoldot connection
+        return typeof this.client.provider['chain'] === 'function';
+    }
 }
 exports.ChainHead = ChainHead;

package/cjs/storage/NewStorageQuery.js

@@ -83,7 +83,13 @@ class NewStorageQuery extends BaseStorageQuery_js_1.BaseStorageQuery {
         await pull(best);
         // Subscribe to best block events
         const unsub = this.client.on('bestBlock', (block) => {
-            pullQueue.enqueue(() => pull(block)).catch(console.error);
+            // Here we're handling each pull one by one,
+            // If the queue get too long, it might take a long time for us to get the fresh & latest data
+            // This is a precaution in such case, if the queue size >= 3 we skip all the pending pull and jump to the latest pull
+            if (pullQueue.size >= 3)
+                pullQueue.clear();
+            // TODO timing out for a pull to prevent it took too long to fetch
+            pullQueue.enqueue(() => pull(block)).catch(utils_1.noop);
         });
         return async () => {
             unsub();

package/json-rpc/group/ChainHead/ChainHead.js

@@ -1,5 +1,5 @@
 import { $Header } from '@dedot/codecs';
-import { assert, AsyncQueue, deferred, ensurePresence, noop, waitFor } from '@dedot/utils';
+import { assert, AsyncQueue, deferred, ensurePresence, noop, ThrottleQueue, waitFor, } from '@dedot/utils';
 import { JsonRpcGroup } from '../JsonRpcGroup.js';
 import { BlockUsage } from './BlockUsage.js';
 import { ChainHeadBlockNotPinnedError, ChainHeadBlockPrunedError, ChainHeadError, ChainHeadLimitReachedError, ChainHeadOperationError, ChainHeadOperationInaccessibleError, ChainHeadStopError, RetryStrategy, } from './error.js';
@@ -19,6 +19,7 @@ export class ChainHead extends JsonRpcGroup {
     #recovering;
     #blockUsage;
     #cache;
+    #operationQueue;
     constructor(client, options) {
         super(client, { prefix: 'chainHead', supportedVersions: ['unstable', 'v1'], ...options });
         this.#handlers = {};
@@ -29,6 +30,8 @@ export class ChainHead extends JsonRpcGroup {
         this.#retryQueue = new AsyncQueue();
         this.#blockUsage = new BlockUsage();
         this.#cache = new Map();
+        // This helps us to not accidentally putting too much stress on the JSON-RPC server, especially smoldot/light-client
+        this.#operationQueue = new ThrottleQueue(this.#__unsafe__isSmoldot() ? 25 : 250);
     }
     async runtimeVersion() {
         await this.#ensureFollowed();
@@ -77,11 +80,18 @@ export class ChainHead extends JsonRpcGroup {
         const defer = deferred();
         try {
             this.#unsub && this.#unsub().catch(noop); // ensure unfollowed
+            let signals = 0;
             this.#unsub = await this.send('follow', true, (event, subscription) => {
                 this.#followResponseQueue
                     .enqueue(async () => {
                     await this.#onFollowEvent(event, subscription);
-                    if (event.event == 'initialized') {
+                    if (signals >= 2)
+                        return;
+                    signals += 1;
+                    // Sometime smoldot send a `stop` event right after `initialized`
+                    // So requests sending between `initialized` and `stop` will be on pruned hashes -> throwing out errors
+                    // Here we make sure to receive at least the first 2 signals to resolve
+                    if (signals >= 2) {
                         defer.resolve();
                     }
                 })
@@ -226,7 +236,7 @@ export class ChainHead extends JsonRpcGroup {
                     .then(() => {
                     // 3. Resolve the recovering promise
                     // This means to continue all pending requests while the chainHead started recovering mode at step 1.
-                    this.#recovering.resolve();
+                    this.#recovering?.resolve();
                     // 4. Recover stale operations
                     // 4.1. Operations that's going on & waiting to receiving its operationId
                     //     will eventually get an `limitedReached` error for using a stale followSubscriptionId
@@ -242,7 +252,7 @@ export class ChainHead extends JsonRpcGroup {
                     .catch((e) => {
                     console.error(e);
                     // TODO we should retry a few attempts
-                    this.#recovering.reject(new ChainHeadError('Cannot recover from stop event!'));
+                    this.#recovering?.reject(new ChainHeadError('Cannot recover from stop event!'));
                     Object.values(this.#handlers).forEach(({ defer, operationId }) => {
                         defer.reject(new ChainHeadError('Cannot recover from stop event!'));
                         delete this.#handlers[operationId];
@@ -390,6 +400,7 @@ export class ChainHead extends JsonRpcGroup {
         this.#retryQueue.clear();
         this.#blockUsage.clear();
         this.#cache.clear();
+        this.#operationQueue.cancel();
     }
     async #ensureFollowed() {
         if (this.#recovering) {
@@ -419,17 +430,15 @@ export class ChainHead extends JsonRpcGroup {
     async #retryOperation(strategy, retry) {
         try {
             return await new Promise((resolve, reject) => {
-                setTimeout(() => {
-                    if (strategy === RetryStrategy.NOW) {
-                        retry().then(resolve).catch(reject);
-                    }
-                    else if (strategy === RetryStrategy.QUEUED) {
-                        this.#retryQueue.enqueue(retry).then(resolve).catch(reject);
-                    }
-                    else {
-                        throw new Error('Invalid retry strategy');
-                    }
-                }); // retry again in the next tick
+                if (strategy === RetryStrategy.NOW) {
+                    retry().then(resolve).catch(reject);
+                }
+                else if (strategy === RetryStrategy.QUEUED) {
+                    this.#retryQueue.enqueue(retry).then(resolve).catch(reject);
+                }
+                else {
+                    throw new Error('Invalid retry strategy');
+                }
             });
         }
         catch (e) {
@@ -477,7 +486,7 @@ export class ChainHead extends JsonRpcGroup {
                 const resp = await this.send('body', this.#subscriptionId, hash);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -506,7 +515,7 @@ export class ChainHead extends JsonRpcGroup {
                 const resp = await this.send('call', this.#subscriptionId, hash, func, params);
                 return this.#awaitOperation(resp, hash);
             };
-            const resp = await this.#performOperationWithRetry(operation, atHash);
+            const resp = await this.#operationQueue.add(() => this.#performOperationWithRetry(operation, atHash));
             this.#cache.set(cacheKey, resp);
             return resp;
         }
@@ -549,9 +558,7 @@ export class ChainHead extends JsonRpcGroup {
             }
             this.#blockUsage.use(hash);
             let results = [];
-            // @ts-ignore a trick internally to check whether a provider is using smoldot connection
-            const isSmoldot = typeof this.client.provider['chain'] === 'function';
-            if (isSmoldot) {
+            if (this.#__unsafe__isSmoldot()) {
                 const fetchItem = async (item) => {
                     const [batch, newDiscardedItems] = await this.#getStorage([item], childTrie ?? null, hash);
                     if (newDiscardedItems.length > 0) {
@@ -584,7 +591,7 @@ export class ChainHead extends JsonRpcGroup {
     }
     async #getStorage(items, childTrie, at) {
         const operation = () => this.#getStorageOperation(items, childTrie, this.#ensurePinnedHash(at));
-        return this.#performOperationWithRetry(operation, at);
+        return this.#operationQueue.add(() => this.#performOperationWithRetry(operation, at));
     }
     async #getStorageOperation(items, childTrie, at) {
         await this.#ensureFollowed();
@@ -621,4 +628,8 @@ export class ChainHead extends JsonRpcGroup {
             return;
         await this.send('unpin', this.#subscriptionId, hashes);
     }
+    #__unsafe__isSmoldot() {
+        // @ts-ignore  a trick internally to check whether a provider is using smoldot connection
+        return typeof this.client.provider['chain'] === 'function';
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dedot/api",
-  "version": "0.9.5-next.20891389.0+2089138",
+  "version": "0.9.5",
   "description": "A delightful JavaScript/TypeScript client for Polkadot & Substrate",
   "author": "Thang X. Vu <thang@dedot.dev>",
   "homepage": "https://dedot.dev",
@@ -13,13 +13,13 @@
   "type": "module",
   "sideEffects": false,
   "dependencies": {
-    "@dedot/codecs": "0.9.5-next.20891389.0+2089138",
-    "@dedot/providers": "0.9.5-next.20891389.0+2089138",
-    "@dedot/runtime-specs": "0.9.5-next.20891389.0+2089138",
-    "@dedot/shape": "0.9.5-next.20891389.0+2089138",
-    "@dedot/storage": "0.9.5-next.20891389.0+2089138",
-    "@dedot/types": "0.9.5-next.20891389.0+2089138",
-    "@dedot/utils": "0.9.5-next.20891389.0+2089138"
+    "@dedot/codecs": "0.9.5",
+    "@dedot/providers": "0.9.5",
+    "@dedot/runtime-specs": "0.9.5",
+    "@dedot/shape": "0.9.5",
+    "@dedot/storage": "0.9.5",
+    "@dedot/types": "0.9.5",
+    "@dedot/utils": "0.9.5"
   },
   "scripts": {
     "build": "tsc --project tsconfig.build.json && tsc --project tsconfig.build.cjs.json",
@@ -48,7 +48,7 @@
     "node": ">=18"
   },
   "license": "Apache-2.0",
-  "gitHead": "208913895242cc9c16cb8bb9bcb880393cb61e6b",
+  "gitHead": "18f4c23ad45330f9de1b3be70d4ce7581eaccb20",
   "module": "./index.js",
   "types": "./index.d.ts"
 }

package/storage/NewStorageQuery.js

@@ -1,4 +1,4 @@
-import { AsyncQueue } from '@dedot/utils';
+import { AsyncQueue, noop } from '@dedot/utils';
 import { BaseStorageQuery } from './BaseStorageQuery.js';
 /**
  * @name NewStorageQuery
@@ -80,7 +80,13 @@ export class NewStorageQuery extends BaseStorageQuery {
         await pull(best);
         // Subscribe to best block events
         const unsub = this.client.on('bestBlock', (block) => {
-            pullQueue.enqueue(() => pull(block)).catch(console.error);
+            // Here we're handling each pull one by one,
+            // If the queue get too long, it might take a long time for us to get the fresh & latest data
+            // This is a precaution in such case, if the queue size >= 3 we skip all the pending pull and jump to the latest pull
+            if (pullQueue.size >= 3)
+                pullQueue.clear();
+            // TODO timing out for a pull to prevent it took too long to fetch
+            pullQueue.enqueue(() => pull(block)).catch(noop);
         });
         return async () => {
             unsub();