@drift-labs/sdk 2.83.0-beta.9 → 2.84.0-beta.1

package/src/tx/txHandler.ts

@@ -25,6 +25,14 @@ import {
 } from '../types';
 import { containsComputeUnitIxs } from '../util/computeUnits';
 
+/**
+ * Explanation for SIGNATURE_BLOCK_AND_EXPIRY:
+ *
+ * When the whileValidTxSender waits for confirmation of a given transaction, it needs the last available blockheight and blockhash used in the signature to do so. For pre-signed transactions, these values aren't attached to the transaction object by default. For a "scrappy" workaround which doesn't break backwards compatibility, the SIGNATURE_BLOCK_AND_EXPIRY property is simply attached to the transaction objects as they are created or signed in this handler despite a mismatch in the typescript types. If the values are attached to the transaction when they reach the whileValidTxSender, it can opt-in to use these values.
+ */
+
+const DEV_TRY_FORCE_TX_TIMEOUTS = false;
+
 export const COMPUTE_UNITS_DEFAULT = 200_000;
 
 export type TxBuildingProps = {
@@ -36,7 +44,7 @@ export type TxBuildingProps = {
 	lookupTables?: AddressLookupTableAccount[];
 	forceVersionedTransaction?: boolean;
 	txParams?: TxParams;
-	recentBlockHash?: BlockhashWithExpiryBlockHeight;
+	recentBlockhash?: BlockhashWithExpiryBlockHeight;
 	wallet?: IWallet;
 };
 
@@ -139,6 +147,9 @@ export class TxHandler {
 
 		const signedTx = await this.signTx(tx, additionalSigners);
 
+		// @ts-ignore
+		signedTx.SIGNATURE_BLOCK_AND_EXPIRY = recentBlockhash;
+
 		return signedTx;
 	}
 
@@ -202,15 +213,18 @@ export class TxHandler {
 	public async signVersionedTx(
 		tx: VersionedTransaction,
 		additionalSigners: Array<Signer>,
-		recentBlockHash?: BlockhashWithExpiryBlockHeight,
+		recentBlockhash?: BlockhashWithExpiryBlockHeight,
 		wallet?: IWallet
 	): Promise<VersionedTransaction> {
 		[wallet] = this.getProps(wallet);
 
-		if (recentBlockHash) {
-			tx.message.recentBlockhash = recentBlockHash.blockhash;
+		if (recentBlockhash) {
+			tx.message.recentBlockhash = recentBlockhash.blockhash;
 
-			this.addHashAndExpiryToLookup(recentBlockHash);
+			this.addHashAndExpiryToLookup(recentBlockhash);
+
+			// @ts-ignore
+			tx.SIGNATURE_BLOCK_AND_EXPIRY = recentBlockhash;
 		}
 
 		additionalSigners
@@ -305,10 +319,10 @@ export class TxHandler {
 	}
 
 	private _generateVersionedTransaction(
-		recentBlockHash: BlockhashWithExpiryBlockHeight,
+		recentBlockhash: BlockhashWithExpiryBlockHeight,
 		message: Message | MessageV0
 	) {
-		this.addHashAndExpiryToLookup(recentBlockHash);
+		this.addHashAndExpiryToLookup(recentBlockhash);
 
 		return new VersionedTransaction(message);
 	}
@@ -326,7 +340,12 @@ export class TxHandler {
 			instructions: ixs,
 		}).compileToLegacyMessage();
 
-		return this._generateVersionedTransaction(recentBlockhash, message);
+		const tx = this._generateVersionedTransaction(recentBlockhash, message);
+
+		// @ts-ignore
+		tx.SIGNATURE_BLOCK_AND_EXPIRY = recentBlockhash;
+
+		return tx;
 	}
 
 	public generateVersionedTransaction(
@@ -343,7 +362,12 @@ export class TxHandler {
 			instructions: ixs,
 		}).compileToV0Message(lookupTableAccounts);
 
-		return this._generateVersionedTransaction(recentBlockhash, message);
+		const tx = this._generateVersionedTransaction(recentBlockhash, message);
+
+		// @ts-ignore
+		tx.SIGNATURE_BLOCK_AND_EXPIRY = recentBlockhash;
+
+		return tx;
 	}
 
 	public generateLegacyTransaction(ixs: TransactionInstruction[]) {
@@ -360,8 +384,8 @@ export class TxHandler {
 			instructions: (TransactionInstruction | TransactionInstruction[])[];
 		}
 	) {
-		const recentBlockHash =
-			props?.recentBlockHash ?? (await this.getLatestBlockhashForTransaction());
+		const recentBlockhash =
+			props?.recentBlockhash ?? (await this.getLatestBlockhashForTransaction());
 
 		return await Promise.all(
 			props.instructions.map((ix) => {
@@ -369,7 +393,7 @@ export class TxHandler {
 				return this.buildTransaction({
 					...props,
 					instructions: ix,
-					recentBlockHash,
+					recentBlockhash,
 				});
 			})
 		);
@@ -433,7 +457,13 @@ export class TxHandler {
 
 		const computeUnitsPrice = baseTxParams?.computeUnitsPrice;
 
-		if (computeUnitsPrice > 0 && !hasSetComputeUnitPriceIx) {
+		if (process.env.DEV_TRY_FORCE_TX_TIMEOUTS || DEV_TRY_FORCE_TX_TIMEOUTS) {
+			allIx.push(
+				ComputeBudgetProgram.setComputeUnitPrice({
+					microLamports: 0,
+				})
+			);
+		} else if (computeUnitsPrice > 0 && !hasSetComputeUnitPriceIx) {
 			allIx.push(
 				ComputeBudgetProgram.setComputeUnitPrice({
 					microLamports: computeUnitsPrice,
@@ -443,13 +473,13 @@ export class TxHandler {
 
 		allIx.push(...instructionsArray);
 
-		const recentBlockHash =
-			props?.recentBlockHash ?? (await this.getLatestBlockhashForTransaction());
+		const recentBlockhash =
+			props?.recentBlockhash ?? (await this.getLatestBlockhashForTransaction());
 
 		// # Create and return Transaction
 		if (txVersion === 'legacy') {
 			if (forceVersionedTransaction) {
-				return this.generateLegacyVersionedTransaction(recentBlockHash, allIx);
+				return this.generateLegacyVersionedTransaction(recentBlockhash, allIx);
 			} else {
 				return this.generateLegacyTransaction(allIx);
 			}
@@ -461,7 +491,7 @@ export class TxHandler {
 				: [marketLookupTable];
 
 			return this.generateVersionedTransaction(
-				recentBlockHash,
+				recentBlockhash,
 				allIx,
 				lookupTables
 			);
@@ -482,7 +512,13 @@ export class TxHandler {
 			);
 		}
 
-		if (computeUnitsPrice != 0) {
+		if (process.env.DEV_TRY_FORCE_TX_TIMEOUTS || DEV_TRY_FORCE_TX_TIMEOUTS) {
+			tx.add(
+				ComputeBudgetProgram.setComputeUnitPrice({
+					microLamports: 0,
+				})
+			);
+		} else if (computeUnitsPrice != 0) {
 			tx.add(
 				ComputeBudgetProgram.setComputeUnitPrice({
 					microLamports: computeUnitsPrice,
@@ -506,18 +542,21 @@ export class TxHandler {
 		keys: string[],
 		wallet?: IWallet,
 		commitment?: Commitment,
-		recentBlockHash?: BlockhashWithExpiryBlockHeight
+		recentBlockhash?: BlockhashWithExpiryBlockHeight
 	) {
-		recentBlockHash = recentBlockHash
-			? recentBlockHash
+		recentBlockhash = recentBlockhash
+			? recentBlockhash
 			: await this.getLatestBlockhashForTransaction();
 
-		this.addHashAndExpiryToLookup(recentBlockHash);
+		this.addHashAndExpiryToLookup(recentBlockhash);
 
 		for (const tx of txsToSign) {
 			if (!tx) continue;
-			tx.recentBlockhash = recentBlockHash.blockhash;
+			tx.recentBlockhash = recentBlockhash.blockhash;
 			tx.feePayer = wallet?.publicKey ?? this.wallet?.publicKey;
+
+			// @ts-ignore
+			tx.SIGNATURE_BLOCK_AND_EXPIRY = recentBlockhash;
 		}
 
 		return this.getSignedTransactionMap(txsToSign, keys, wallet);
@@ -536,18 +575,21 @@ export class TxHandler {
 		keys: string[],
 		wallet?: IWallet,
 		commitment?: Commitment,
-		recentBlockHash?: BlockhashWithExpiryBlockHeight
+		recentBlockhash?: BlockhashWithExpiryBlockHeight
 	) {
-		recentBlockHash = recentBlockHash
-			? recentBlockHash
+		recentBlockhash = recentBlockhash
+			? recentBlockhash
 			: await this.getLatestBlockhashForTransaction();
 
-		this.addHashAndExpiryToLookup(recentBlockHash);
+		this.addHashAndExpiryToLookup(recentBlockhash);
 
 		for (const tx of txsToSign) {
 			if (!tx) continue;
-			tx.recentBlockhash = recentBlockHash.blockhash;
+			tx.recentBlockhash = recentBlockhash.blockhash;
 			tx.feePayer = wallet?.publicKey ?? this.wallet?.publicKey;
+
+			// @ts-ignore
+			tx.SIGNATURE_BLOCK_AND_EXPIRY = recentBlockhash;
 		}
 
 		return this.getSignedTransactionMap(txsToSign, keys, wallet);
@@ -584,13 +626,20 @@ export class TxHandler {
 
 		this.preSignedCb?.();
 
-		const signedTxs = await wallet.signAllTransactions(
-			txsToSign
-				.map((tx) => {
-					return tx as Transaction;
-				})
-				.filter((tx) => tx !== undefined)
-		);
+		const filteredTxs = txsToSign
+			.map((tx) => {
+				return tx as Transaction;
+			})
+			.filter((tx) => tx !== undefined);
+
+		const signedTxs = await wallet.signAllTransactions(filteredTxs);
+
+		signedTxs.forEach((signedTx, index) => {
+			// @ts-ignore
+			signedTx.SIGNATURE_BLOCK_AND_EXPIRY =
+				// @ts-ignore
+				txsToSign[index]?.SIGNATURE_BLOCK_AND_EXPIRY;
+		});
 
 		this.handleSignedTxData(
 			signedTxs.map((signedTx) => {
@@ -622,15 +671,14 @@ export class TxHandler {
 	) {
 		const transactions = await this.buildBulkTransactions(props);
 
-		const preppedTransactions =
-			props.txVersion === 'legacy'
-				? this.getPreparedAndSignedLegacyTransactionMap(
-						transactions as Transaction[],
-						props.keys,
-						props.wallet,
-						props.preFlightCommitment
-				  )
-				: this.getSignedTransactionMap(transactions, props.keys, props.wallet);
+		const preppedTransactions = await (props.txVersion === 'legacy'
+			? this.getPreparedAndSignedLegacyTransactionMap(
+					transactions as Transaction[],
+					props.keys,
+					props.wallet,
+					props.preFlightCommitment
+			  )
+			: this.getSignedTransactionMap(transactions, props.keys, props.wallet));
 
 		return preppedTransactions;
 	}

package/src/tx/whileValidTxSender.ts

@@ -15,6 +15,8 @@ import { IWallet } from '../types';
 
 const DEFAULT_RETRY = 2000;
 
+const VALID_BLOCK_HEIGHT_OFFSET = -150; // This is a bit of weirdness but the lastValidBlockHeight value returned from connection.getLatestBlockhash is always 300 blocks ahead of the current block, even though the transaction actually expires after 150 blocks. This accounts for that so that we can at least accuractely estimate the transaction expiry.
+
 type ResolveReference = {
 	resolve?: () => void;
 };
@@ -93,6 +95,13 @@ export class WhileValidTxSender extends BaseTxSender {
 			);
 		}
 
+		// See SIGNATURE_BLOCK_AND_EXPIRY explanation in txHandler.ts if this is confusing
+		// @ts-ignore
+		if (preSigned && tx.SIGNATURE_BLOCK_AND_EXPIRY) {
+			// @ts-ignore
+			latestBlockhash = tx.SIGNATURE_BLOCK_AND_EXPIRY;
+		}
+
 		// handle subclass-specific side effects
 		const txSig = bs58.encode(
 			signedTx.signatures[0]?.signature || signedTx.signatures[0]
@@ -108,12 +117,20 @@ export class WhileValidTxSender extends BaseTxSender {
 		opts?: ConfirmOptions,
 		preSigned?: boolean
 	): Promise<TxSigAndSlot> {
-		const latestBlockhash =
+		let latestBlockhash =
 			await this.txHandler.getLatestBlockhashForTransaction();
 
 		let signedTx;
 		if (preSigned) {
 			signedTx = tx;
+
+			// See SIGNATURE_BLOCK_AND_EXPIRY explanation in txHandler.ts if this is confusing
+			// @ts-ignore
+			if (tx.SIGNATURE_BLOCK_AND_EXPIRY) {
+				// @ts-ignore
+				latestBlockhash = tx.SIGNATURE_BLOCK_AND_EXPIRY;
+			}
+
 			// @ts-ignore
 		} else if (this.wallet.payer) {
 			tx.message.recentBlockhash = latestBlockhash.blockhash;
@@ -185,7 +202,8 @@ export class WhileValidTxSender extends BaseTxSender {
 			const result = await this.connection.confirmTransaction(
 				{
 					signature: txid,
-					lastValidBlockHeight,
+					lastValidBlockHeight:
+						lastValidBlockHeight + VALID_BLOCK_HEIGHT_OFFSET,
 					blockhash,
 				},
 				opts.commitment

package/src/userMap/userMap.ts

@@ -29,6 +29,7 @@ import { Buffer } from 'buffer';
 import { ZSTDDecoder } from 'zstddec';
 import { getNonIdleUserFilter, getUserFilter } from '../memcmp';
 import {
+	SyncConfig,
 	UserAccountFilterCriteria as UserFilterCriteria,
 	UserMapConfig,
 } from './userMapConfig';
@@ -83,6 +84,7 @@ export class UserMap implements UserMapInterface {
 	};
 	private decode;
 	private mostRecentSlot = 0;
+	private syncConfig: SyncConfig;
 
 	private syncPromise?: Promise<void>;
 	private syncPromiseResolver: () => void;
@@ -132,6 +134,10 @@ export class UserMap implements UserMapInterface {
 				decodeFn,
 			});
 		}
+
+		this.syncConfig = config.syncConfig ?? {
+			type: 'default',
+		};
 	}
 
 	public async subscribe() {
@@ -345,6 +351,14 @@ export class UserMap implements UserMapInterface {
 	}
 
 	public async sync() {
+		if (this.syncConfig.type === 'default') {
+			return this.defaultSync();
+		} else {
+			return this.paginatedSync();
+		}
+	}
+
+	private async defaultSync() {
 		if (this.syncPromise) {
 			return this.syncPromise;
 		}
@@ -438,6 +452,123 @@ export class UserMap implements UserMapInterface {
 		}
 	}
 
+	private async paginatedSync() {
+		if (this.syncPromise) {
+			return this.syncPromise;
+		}
+
+		this.syncPromise = new Promise<void>((resolve) => {
+			this.syncPromiseResolver = resolve;
+		});
+
+		try {
+			const accountsPrefetch = await this.connection.getProgramAccounts(
+				this.driftClient.program.programId,
+				{
+					dataSlice: { offset: 0, length: 0 },
+					filters: [
+						getUserFilter(),
+						...(!this.includeIdle ? [getNonIdleUserFilter()] : []),
+					],
+				}
+			);
+			const accountPublicKeys = accountsPrefetch.map(
+				(account) => account.pubkey
+			);
+
+			const limitConcurrency = async (tasks, limit) => {
+				const executing = [];
+				const results = [];
+
+				for (let i = 0; i < tasks.length; i++) {
+					const executor = Promise.resolve().then(tasks[i]);
+					results.push(executor);
+
+					if (executing.length < limit) {
+						executing.push(executor);
+						executor.finally(() => {
+							const index = executing.indexOf(executor);
+							if (index > -1) {
+								executing.splice(index, 1);
+							}
+						});
+					} else {
+						await Promise.race(executing);
+					}
+				}
+
+				return Promise.all(results);
+			};
+
+			const programAccountBufferMap = new Map<string, Buffer>();
+
+			// @ts-ignore
+			const chunkSize = this.syncConfig.chunkSize ?? 100;
+			const tasks = [];
+			for (let i = 0; i < accountPublicKeys.length; i += chunkSize) {
+				const chunk = accountPublicKeys.slice(i, i + chunkSize);
+				tasks.push(async () => {
+					const accountInfos =
+						await this.connection.getMultipleAccountsInfoAndContext(chunk, {
+							commitment: this.commitment,
+						});
+
+					const accountInfosSlot = accountInfos.context.slot;
+
+					for (let j = 0; j < accountInfos.value.length; j += 1) {
+						const accountInfo = accountInfos.value[j];
+						if (accountInfo === null) continue;
+
+						const publicKeyString = chunk[j].toString();
+						const buffer = Buffer.from(accountInfo.data);
+						programAccountBufferMap.set(publicKeyString, buffer);
+
+						const decodedUser = this.decode('User', buffer);
+
+						const currAccountWithSlot = this.getWithSlot(publicKeyString);
+						if (
+							currAccountWithSlot &&
+							currAccountWithSlot.slot <= accountInfosSlot
+						) {
+							this.updateUserAccount(
+								publicKeyString,
+								decodedUser,
+								accountInfosSlot
+							);
+						} else {
+							await this.addPubkey(
+								new PublicKey(publicKeyString),
+								decodedUser,
+								accountInfosSlot
+							);
+						}
+					}
+				});
+			}
+
+			// @ts-ignore
+			const concurrencyLimit = this.syncConfig.concurrencyLimit ?? 10;
+			await limitConcurrency(tasks, concurrencyLimit);
+
+			for (const [key] of this.entries()) {
+				if (!programAccountBufferMap.has(key)) {
+					const user = this.get(key);
+					if (user) {
+						await user.unsubscribe();
+						this.userMap.delete(key);
+					}
+				}
+			}
+		} catch (err) {
+			console.error(`Error in UserMap.sync():`, err);
+		} finally {
+			if (this.syncPromiseResolver) {
+				this.syncPromiseResolver();
+			}
+			this.syncPromise = undefined;
+		}
+	}
+
 	public async unsubscribe() {
 		await this.subscription.unsubscribe();
 

package/src/userMap/userMapConfig.ts

@@ -7,6 +7,16 @@ export type UserAccountFilterCriteria = {
 	hasOpenOrders: boolean;
 };
 
+export type SyncConfig =
+	| {
+			type: 'default';
+	  }
+	| {
+			type: 'paginated';
+			chunkSize?: number;
+			concurrencyLimit?: number;
+	  };
+
 export type UserMapConfig = {
 	driftClient: DriftClient;
 	// connection object to use specifically for the UserMap. If undefined, will use the driftClient's connection
@@ -36,4 +46,6 @@ export type UserMapConfig = {
 	// If true, will not do a full sync whenever StateAccount.numberOfSubAccounts changes.
 	// default behavior is to do a full sync on changes.
 	disableSyncOnTotalAccountsChange?: boolean;
+
+	syncConfig?: SyncConfig;
 };