@dedot/api 0.7.1 → 0.8.1-next.4880b708.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/cjs/client/BaseSubstrateClient.js +32 -1
  2. package/cjs/executor/StorageQueryExecutor.js +14 -6
  3. package/cjs/executor/v2/StorageQueryExecutorV2.js +6 -2
  4. package/cjs/storage/QueryableStorage.js +19 -9
  5. package/client/BaseSubstrateClient.d.ts +4 -0
  6. package/client/BaseSubstrateClient.js +32 -1
  7. package/executor/StorageQueryExecutor.js +15 -7
  8. package/executor/v2/StorageQueryExecutorV2.js +6 -2
  9. package/package.json +9 -9
  10. package/storage/QueryableStorage.d.ts +2 -1
  11. package/storage/QueryableStorage.js +19 -9
package/cjs/client/BaseSubstrateClient.js CHANGED
@@ -87,13 +87,44 @@ class BaseSubstrateClient extends index_js_2.JsonRpcClient {
87
87
  }
88
88
  }
89
89
  if (shouldUpdateCache && this._localCache) {
90
- await this._localCache.set(metadataKey, (0, utils_1.u8aToHex)(codecs_1.$Metadata.tryEncode(metadata)));
90
+ const encodedMetadata = (0, utils_1.u8aToHex)(codecs_1.$Metadata.tryEncode(metadata));
91
+ await this.safeSetMetadataToCache(metadataKey, encodedMetadata);
91
92
  }
92
93
  if (!metadata) {
93
94
  throw new Error('Cannot load metadata');
94
95
  }
95
96
  this.setMetadata(metadata);
96
97
  }
98
+ /**
99
+ * Safely set metadata to cache with fallback cleanup if storage limit is exceeded
100
+ */
101
+ async safeSetMetadataToCache(key, value) {
102
+ if (!this._localCache)
103
+ return;
104
+ try {
105
+ // First attempt to set the metadata
106
+ await this._localCache.set(key, value);
107
+ }
108
+ catch (error) {
109
+ console.warn('Failed to store metadata in cache, attempting to clean up old entries:', error);
110
+ try {
111
+ // Get all keys that start with RAW_META/
112
+ const allKeys = await this._localCache.keys();
113
+ const metadataKeys = allKeys.filter(k => k.startsWith('RAW_META/') && k !== key);
114
+ // Remove all other metadata entries
115
+ for (const metaKey of metadataKeys) {
116
+ await this._localCache.remove(metaKey);
117
+ }
118
+ console.info(`Cleaned up ${metadataKeys.length} old metadata entries, trying again`);
119
+ // Try again after cleanup
120
+ await this._localCache.set(key, value);
121
+ }
122
+ catch (cleanupError) {
123
+ // If it still fails after cleanup, log the error but continue
124
+ console.error('Failed to store metadata even after cleanup:', cleanupError);
125
+ }
126
+ }
127
+ }
97
128
  setMetadata(metadata) {
98
129
  this._metadata = metadata;
99
130
  this._registry = new codecs_1.PortableRegistry(metadata.latest, this.options.hasher);
package/cjs/executor/StorageQueryExecutor.js CHANGED
@@ -77,17 +77,25 @@ class StorageQueryExecutor extends Executor_js_1.Executor {
77
77
  return queryFn;
78
78
  }
79
79
  exposeStorageMapMethods(entry) {
80
- const rawKeys = async (pagination) => {
80
+ const rawKeys = async (partialInput, pagination) => {
81
81
  const pageSize = pagination?.pageSize || DEFAULT_KEYS_PAGE_SIZE;
82
82
  const startKey = pagination?.startKey || entry.prefixKey;
83
- return await this.client.rpc.state_getKeysPaged(entry.prefixKey, pageSize, startKey, this.atBlockHash);
83
+ return await this.client.rpc.state_getKeysPaged(entry.encodeKey(partialInput, true), pageSize, startKey, this.atBlockHash);
84
84
  };
85
- const pagedKeys = async (pagination) => {
86
- const storageKeys = await rawKeys({ pageSize: DEFAULT_KEYS_PAGE_SIZE, ...pagination });
85
+ const extractArgs = (args) => {
86
+ const inArgs = args.slice();
87
+ const lastArg = args.at(-1);
88
+ const pagination = (0, utils_1.isObject)(lastArg) && ('pageSize' in lastArg || 'startKey' in lastArg) ? inArgs.pop() : undefined;
89
+ return [inArgs, pagination];
90
+ };
91
+ const pagedKeys = async (...args) => {
92
+ const [inArgs, pagination] = extractArgs(args);
93
+ const storageKeys = await rawKeys(inArgs, { pageSize: DEFAULT_KEYS_PAGE_SIZE, ...pagination });
87
94
  return storageKeys.map((key) => entry.decodeKey(key));
88
95
  };
89
- const pagedEntries = async (pagination) => {
90
- const storageKeys = await rawKeys({ pageSize: DEFAULT_ENTRIES_PAGE_SIZE, ...pagination });
96
+ const pagedEntries = async (...args) => {
97
+ const [inArgs, pagination] = extractArgs(args);
98
+ const storageKeys = await rawKeys(inArgs, { pageSize: DEFAULT_ENTRIES_PAGE_SIZE, ...pagination });
91
99
  const storageMap = await this.queryStorage(storageKeys, this.atBlockHash);
92
100
  return storageKeys.map((key) => [entry.decodeKey(key), entry.decodeValue(storageMap[key])]);
93
101
  };
package/cjs/executor/v2/StorageQueryExecutorV2.js CHANGED
@@ -18,8 +18,12 @@ class StorageQueryExecutorV2 extends StorageQueryExecutor_js_1.StorageQueryExecu
18
18
  // so for now we're trying to pull all entries from storage
19
19
  // this might take a while for large storage
20
20
  // TODO improve this, fallback to use `archive`-prefixed if available?
21
- const entries = async () => {
22
- const results = await this.chainHead.storage([{ type: 'descendantsValues', key: entry.prefixKey }]);
21
+ const entries = async (...args) => {
22
+ const withArgs = !!args && args.length > 0;
23
+ const key = withArgs ? entry.encodeKey(args, true) : entry.prefixKey;
24
+ const results = await this.chainHead.storage([
25
+ { type: 'descendantsValues', key },
26
+ ]);
23
27
  return results.map(({ key, value }) => [
24
28
  entry.decodeKey(key),
25
29
  entry.decodeValue(value),
package/cjs/storage/QueryableStorage.js CHANGED
@@ -54,17 +54,18 @@ class QueryableStorage {
54
54
  * Encode plain key input to raw/bytes storage key
55
55
  *
56
56
  * @param keyInput
57
+ * @param allowPartialKeys - allow partial keys, default is false
57
58
  */
58
- encodeKey(keyInput) {
59
+ encodeKey(keyInput, allowPartialKeys = false) {
59
60
  const { storageType } = this.storageEntry;
60
61
  if (storageType.type === 'Plain') {
61
62
  return this.prefixKey;
62
63
  }
63
64
  else if (storageType.type === 'Map') {
64
65
  const { hashers, keyTypeIds } = this.#getStorageMapInfo(storageType);
65
- const extractedInputs = this.#extractRequiredKeyInputs(keyInput, hashers.length);
66
- const keyParts = keyTypeIds.map((keyId, index) => {
67
- const input = extractedInputs[index];
66
+ const extractedInputs = this.#extractRequiredKeyInputs(keyInput, hashers.length, allowPartialKeys);
67
+ const keyParts = extractedInputs.map((input, index) => {
68
+ const keyId = keyTypeIds[index];
68
69
  const hasher = utils_1.HASHERS[hashers[index]];
69
70
  const $keyCodec = this.registry.findCodec(keyId);
70
71
  return hasher($keyCodec.tryEncode(input));
@@ -125,7 +126,7 @@ class QueryableStorage {
125
126
  return this.registry.findCodec(valueTypeId).tryDecode(codecs_1.$StorageData.tryEncode(raw));
126
127
  }
127
128
  }
128
- #extractRequiredKeyInputs(keyInput, numberOfValue) {
129
+ #extractRequiredKeyInputs(keyInput, numberOfValue, allowPartialKeys = false) {
129
130
  if (numberOfValue === 0) {
130
131
  return [];
131
132
  }
@@ -134,16 +135,25 @@ class QueryableStorage {
134
135
  throw new Error(`Invalid key inputs, required ${numberOfValue} input(s)`);
135
136
  }
136
137
  if (numberOfValue === 1) {
137
- return [keyInput];
138
+ return allowPartialKeys && Array.isArray(keyInput) ? keyInput : [keyInput];
138
139
  }
139
140
  else {
140
141
  if (!Array.isArray(keyInput)) {
141
142
  throw new Error(`Input should be an array with ${numberOfValue} value(s)`);
142
143
  }
143
- if (keyInput.length !== numberOfValue) {
144
- throw new Error(`Mismatch key inputs length, required an array of ${numberOfValue} value(s)`);
144
+ if (allowPartialKeys) {
145
+ if (keyInput.length >= numberOfValue) {
146
+ throw new Error(`Invalid key inputs, partial key inputs should be less than the required key inputs, (max: ${numberOfValue - 1})`);
147
+ }
148
+ // TODO we need to make sure only filter out values in the last positions
149
+ return keyInput.slice(0, numberOfValue).filter((one) => one !== undefined && one !== null);
150
+ }
151
+ else {
152
+ if (keyInput.length !== numberOfValue) {
153
+ throw new Error(`Mismatch key inputs length, required an array of ${numberOfValue} value(s)`);
154
+ }
155
+ return keyInput.slice(0, numberOfValue);
145
156
  }
146
- return keyInput.slice(0, numberOfValue);
147
157
  }
148
158
  }
149
159
  }
package/client/BaseSubstrateClient.d.ts CHANGED
@@ -24,6 +24,10 @@ export declare abstract class BaseSubstrateClient<Rv extends RpcVersion, ChainAp
24
24
  protected normalizeOptions(options: ApiOptions | JsonRpcProvider): ApiOptions;
25
25
  protected initializeLocalCache(): Promise<void>;
26
26
  protected setupMetadata(preloadMetadata: Metadata | undefined): Promise<void>;
27
+ /**
28
+ * Safely set metadata to cache with fallback cleanup if storage limit is exceeded
29
+ */
30
+ protected safeSetMetadataToCache(key: string, value: string): Promise<void>;
27
31
  protected setMetadata(metadata: Metadata): void;
28
32
  protected getMetadataKey(runtime?: SubstrateRuntimeVersion): MetadataKey;
29
33
  get currentMetadataKey(): string;
package/client/BaseSubstrateClient.js CHANGED
@@ -83,13 +83,44 @@ export class BaseSubstrateClient extends JsonRpcClient {
83
83
  }
84
84
  }
85
85
  if (shouldUpdateCache && this._localCache) {
86
- await this._localCache.set(metadataKey, u8aToHex($Metadata.tryEncode(metadata)));
86
+ const encodedMetadata = u8aToHex($Metadata.tryEncode(metadata));
87
+ await this.safeSetMetadataToCache(metadataKey, encodedMetadata);
87
88
  }
88
89
  if (!metadata) {
89
90
  throw new Error('Cannot load metadata');
90
91
  }
91
92
  this.setMetadata(metadata);
92
93
  }
94
+ /**
95
+ * Safely set metadata to cache with fallback cleanup if storage limit is exceeded
96
+ */
97
+ async safeSetMetadataToCache(key, value) {
98
+ if (!this._localCache)
99
+ return;
100
+ try {
101
+ // First attempt to set the metadata
102
+ await this._localCache.set(key, value);
103
+ }
104
+ catch (error) {
105
+ console.warn('Failed to store metadata in cache, attempting to clean up old entries:', error);
106
+ try {
107
+ // Get all keys that start with RAW_META/
108
+ const allKeys = await this._localCache.keys();
109
+ const metadataKeys = allKeys.filter(k => k.startsWith('RAW_META/') && k !== key);
110
+ // Remove all other metadata entries
111
+ for (const metaKey of metadataKeys) {
112
+ await this._localCache.remove(metaKey);
113
+ }
114
+ console.info(`Cleaned up ${metadataKeys.length} old metadata entries, trying again`);
115
+ // Try again after cleanup
116
+ await this._localCache.set(key, value);
117
+ }
118
+ catch (cleanupError) {
119
+ // If it still fails after cleanup, log the error but continue
120
+ console.error('Failed to store metadata even after cleanup:', cleanupError);
121
+ }
122
+ }
123
+ }
93
124
  setMetadata(metadata) {
94
125
  this._metadata = metadata;
95
126
  this._registry = new PortableRegistry(metadata.latest, this.options.hasher);
package/executor/StorageQueryExecutor.js CHANGED
@@ -1,4 +1,4 @@
1
- import { assert, isFunction } from '@dedot/utils';
1
+ import { assert, isFunction, isObject } from '@dedot/utils';
2
2
  import { QueryableStorage } from '../storage/QueryableStorage.js';
3
3
  import { Executor } from './Executor.js';
4
4
  const DEFAULT_KEYS_PAGE_SIZE = 1000;
@@ -74,17 +74,25 @@ export class StorageQueryExecutor extends Executor {
74
74
  return queryFn;
75
75
  }
76
76
  exposeStorageMapMethods(entry) {
77
- const rawKeys = async (pagination) => {
77
+ const rawKeys = async (partialInput, pagination) => {
78
78
  const pageSize = pagination?.pageSize || DEFAULT_KEYS_PAGE_SIZE;
79
79
  const startKey = pagination?.startKey || entry.prefixKey;
80
- return await this.client.rpc.state_getKeysPaged(entry.prefixKey, pageSize, startKey, this.atBlockHash);
80
+ return await this.client.rpc.state_getKeysPaged(entry.encodeKey(partialInput, true), pageSize, startKey, this.atBlockHash);
81
81
  };
82
- const pagedKeys = async (pagination) => {
83
- const storageKeys = await rawKeys({ pageSize: DEFAULT_KEYS_PAGE_SIZE, ...pagination });
82
+ const extractArgs = (args) => {
83
+ const inArgs = args.slice();
84
+ const lastArg = args.at(-1);
85
+ const pagination = isObject(lastArg) && ('pageSize' in lastArg || 'startKey' in lastArg) ? inArgs.pop() : undefined;
86
+ return [inArgs, pagination];
87
+ };
88
+ const pagedKeys = async (...args) => {
89
+ const [inArgs, pagination] = extractArgs(args);
90
+ const storageKeys = await rawKeys(inArgs, { pageSize: DEFAULT_KEYS_PAGE_SIZE, ...pagination });
84
91
  return storageKeys.map((key) => entry.decodeKey(key));
85
92
  };
86
- const pagedEntries = async (pagination) => {
87
- const storageKeys = await rawKeys({ pageSize: DEFAULT_ENTRIES_PAGE_SIZE, ...pagination });
93
+ const pagedEntries = async (...args) => {
94
+ const [inArgs, pagination] = extractArgs(args);
95
+ const storageKeys = await rawKeys(inArgs, { pageSize: DEFAULT_ENTRIES_PAGE_SIZE, ...pagination });
88
96
  const storageMap = await this.queryStorage(storageKeys, this.atBlockHash);
89
97
  return storageKeys.map((key) => [entry.decodeKey(key), entry.decodeValue(storageMap[key])]);
90
98
  };
package/executor/v2/StorageQueryExecutorV2.js CHANGED
@@ -15,8 +15,12 @@ export class StorageQueryExecutorV2 extends StorageQueryExecutor {
15
15
  // so for now we're trying to pull all entries from storage
16
16
  // this might take a while for large storage
17
17
  // TODO improve this, fallback to use `archive`-prefixed if available?
18
- const entries = async () => {
19
- const results = await this.chainHead.storage([{ type: 'descendantsValues', key: entry.prefixKey }]);
18
+ const entries = async (...args) => {
19
+ const withArgs = !!args && args.length > 0;
20
+ const key = withArgs ? entry.encodeKey(args, true) : entry.prefixKey;
21
+ const results = await this.chainHead.storage([
22
+ { type: 'descendantsValues', key },
23
+ ]);
20
24
  return results.map(({ key, value }) => [
21
25
  entry.decodeKey(key),
22
26
  entry.decodeValue(value),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dedot/api",
3
- "version": "0.7.1",
3
+ "version": "0.8.1-next.4880b708.2+4880b70",
4
4
  "description": "A delightful JavaScript/TypeScript client for Polkadot & Substrate",
5
5
  "author": "Thang X. Vu <thang@coongcrafts.io>",
6
6
  "homepage": "https://github.com/dedotdev/dedot/tree/main/packages/api",
@@ -13,13 +13,13 @@
13
13
  "type": "module",
14
14
  "sideEffects": false,
15
15
  "dependencies": {
16
- "@dedot/codecs": "0.7.1",
17
- "@dedot/providers": "0.7.1",
18
- "@dedot/runtime-specs": "0.7.1",
19
- "@dedot/shape": "0.7.1",
20
- "@dedot/storage": "0.7.1",
21
- "@dedot/types": "0.7.1",
22
- "@dedot/utils": "0.7.1"
16
+ "@dedot/codecs": "0.8.1-next.4880b708.2+4880b70",
17
+ "@dedot/providers": "0.8.1-next.4880b708.2+4880b70",
18
+ "@dedot/runtime-specs": "0.8.1-next.4880b708.2+4880b70",
19
+ "@dedot/shape": "0.8.1-next.4880b708.2+4880b70",
20
+ "@dedot/storage": "0.8.1-next.4880b708.2+4880b70",
21
+ "@dedot/types": "0.8.1-next.4880b708.2+4880b70",
22
+ "@dedot/utils": "0.8.1-next.4880b708.2+4880b70"
23
23
  },
24
24
  "scripts": {
25
25
  "build": "tsc --project tsconfig.build.json && tsc --project tsconfig.build.cjs.json",
@@ -48,7 +48,7 @@
48
48
  "node": ">=18"
49
49
  },
50
50
  "license": "Apache-2.0",
51
- "gitHead": "5743229117be17b74d16c35a2862da4c36a24045",
51
+ "gitHead": "4880b708ff98e75c257bf82ec80b3d63b0fb1458",
52
52
  "module": "./index.js",
53
53
  "types": "./index.d.ts"
54
54
  }
package/storage/QueryableStorage.d.ts CHANGED
@@ -17,8 +17,9 @@ export declare class QueryableStorage {
17
17
  * Encode plain key input to raw/bytes storage key
18
18
  *
19
19
  * @param keyInput
20
+ * @param allowPartialKeys - allow partial keys, default is false
20
21
  */
21
- encodeKey(keyInput?: any): StorageKey;
22
+ encodeKey(keyInput?: any, allowPartialKeys?: boolean): StorageKey;
22
23
  /**
23
24
  * Decode storage key to plain key input
24
25
  * Only storage keys that hashed by `twox64Concat`, `blake2_128Concat`, or `identity` can be decoded
package/storage/QueryableStorage.js CHANGED
@@ -51,17 +51,18 @@ export class QueryableStorage {
51
51
  * Encode plain key input to raw/bytes storage key
52
52
  *
53
53
  * @param keyInput
54
+ * @param allowPartialKeys - allow partial keys, default is false
54
55
  */
55
- encodeKey(keyInput) {
56
+ encodeKey(keyInput, allowPartialKeys = false) {
56
57
  const { storageType } = this.storageEntry;
57
58
  if (storageType.type === 'Plain') {
58
59
  return this.prefixKey;
59
60
  }
60
61
  else if (storageType.type === 'Map') {
61
62
  const { hashers, keyTypeIds } = this.#getStorageMapInfo(storageType);
62
- const extractedInputs = this.#extractRequiredKeyInputs(keyInput, hashers.length);
63
- const keyParts = keyTypeIds.map((keyId, index) => {
64
- const input = extractedInputs[index];
63
+ const extractedInputs = this.#extractRequiredKeyInputs(keyInput, hashers.length, allowPartialKeys);
64
+ const keyParts = extractedInputs.map((input, index) => {
65
+ const keyId = keyTypeIds[index];
65
66
  const hasher = HASHERS[hashers[index]];
66
67
  const $keyCodec = this.registry.findCodec(keyId);
67
68
  return hasher($keyCodec.tryEncode(input));
@@ -122,7 +123,7 @@ export class QueryableStorage {
122
123
  return this.registry.findCodec(valueTypeId).tryDecode($StorageData.tryEncode(raw));
123
124
  }
124
125
  }
125
- #extractRequiredKeyInputs(keyInput, numberOfValue) {
126
+ #extractRequiredKeyInputs(keyInput, numberOfValue, allowPartialKeys = false) {
126
127
  if (numberOfValue === 0) {
127
128
  return [];
128
129
  }
@@ -131,16 +132,25 @@ export class QueryableStorage {
131
132
  throw new Error(`Invalid key inputs, required ${numberOfValue} input(s)`);
132
133
  }
133
134
  if (numberOfValue === 1) {
134
- return [keyInput];
135
+ return allowPartialKeys && Array.isArray(keyInput) ? keyInput : [keyInput];
135
136
  }
136
137
  else {
137
138
  if (!Array.isArray(keyInput)) {
138
139
  throw new Error(`Input should be an array with ${numberOfValue} value(s)`);
139
140
  }
140
- if (keyInput.length !== numberOfValue) {
141
- throw new Error(`Mismatch key inputs length, required an array of ${numberOfValue} value(s)`);
141
+ if (allowPartialKeys) {
142
+ if (keyInput.length >= numberOfValue) {
143
+ throw new Error(`Invalid key inputs, partial key inputs should be less than the required key inputs, (max: ${numberOfValue - 1})`);
144
+ }
145
+ // TODO we need to make sure only filter out values in the last positions
146
+ return keyInput.slice(0, numberOfValue).filter((one) => one !== undefined && one !== null);
147
+ }
148
+ else {
149
+ if (keyInput.length !== numberOfValue) {
150
+ throw new Error(`Mismatch key inputs length, required an array of ${numberOfValue} value(s)`);
151
+ }
152
+ return keyInput.slice(0, numberOfValue);
142
153
  }
143
- return keyInput.slice(0, numberOfValue);
144
154
  }
145
155
  }
146
156
  }