@evaafi/sdk 0.6.1 → 0.6.2-a

package/src/prices/PricesCollector.ts

@@ -0,0 +1,139 @@
+import { Cell, Dictionary } from "@ton/core";
+import { MAINNET_POOL_CONFIG } from "../constants/pools";
+import { PoolAssetConfig, PoolAssetsConfig, PoolConfig } from "../types/Master";
+import { PriceSource } from "./sources";
+import { DefaultPriceSourcesConfig, PriceSourcesConfig, RawPriceData } from "./Types";
+import { collectAndFilterPrices, generatePriceSources, getMedianPrice, packAssetsData, packOraclesData, packPrices, verifyPricesSign, verifyPricesTimestamp } from "./utils";
+import { delay } from "../utils/utils";
+import { Prices } from "./Prices";
+import { checkNotInDebtAtAll } from "../api/math";
+
+
+export class PricesCollector {
+    #prices: RawPriceData[];
+    #poolConfig: PoolConfig;
+    #sourcesConfig: PriceSourcesConfig;
+    #priceSources: PriceSource[];
+
+    constructor(poolConfig: PoolConfig = MAINNET_POOL_CONFIG, sourcesConfig: PriceSourcesConfig = DefaultPriceSourcesConfig, additionalPriceSources?: PriceSource[]) {
+        this.#poolConfig = poolConfig;
+        this.#sourcesConfig = sourcesConfig;
+        this.#priceSources = generatePriceSources(this.#sourcesConfig, this.#poolConfig.oracles);
+
+        if (additionalPriceSources) {
+            this.#priceSources.push(...additionalPriceSources);
+        }
+
+        this.#prices = [];
+    }
+
+    // TODO Make UserData class and incapsulate raw bigintegers
+
+    async getPricesForLiquidate(realPrincipals: Dictionary<bigint, bigint>, retries: number = 1, timeout: number = 3000): Promise<Prices>  {
+        const assets = this.#filterEmptyPrincipalsAndAssets(realPrincipals);
+        if (assets.includes(undefined)) {
+            throw new Error("User from another pool");
+        }
+        return await this.getPrices(assets.map(x => x!), retries, timeout);
+    }
+
+
+    async getPricesForWithdraw(realPrincipals: Dictionary<bigint, bigint>, withdrawAsset: PoolAssetConfig, collateralToDebt = false, retries: number = 1, timeout: number = 3000): Promise<Prices>  {
+        let assets = this.#filterEmptyPrincipalsAndAssets(realPrincipals);
+        if (checkNotInDebtAtAll(realPrincipals) && (realPrincipals.get(withdrawAsset.assetId) ?? 0n) > 0n && !collateralToDebt) {
+            return new Prices(Dictionary.empty<bigint, bigint>(), Cell.EMPTY);
+        }
+
+        if (assets.includes(undefined)) {
+            throw new Error("User from another pool");
+        }
+
+        if (!assets.includes(withdrawAsset)) {
+            assets.push(withdrawAsset);
+        }
+
+        if (collateralToDebt && assets.length == 1) {
+            throw new Error("Cannot debt only one supplied asset");
+        }
+
+        return await this.getPrices(assets.map(x => x!), retries, timeout);
+    }
+
+    async getPrices(assets: PoolAssetsConfig = this.#poolConfig.poolAssetsConfig, retries: number = 1, timeout: number = 3000): Promise<Prices> {
+        console.debug('[getPrices] Assets length', assets.length);
+
+        if (assets.length == 0) {
+            return new Prices(Dictionary.empty<bigint, bigint>(), Cell.EMPTY);
+        }
+
+        for (let i = 0; i <= retries; i++) {  // attemts = retries + 1
+            if (!this.#prices || this.#filterPrices() < this.#poolConfig.minimalOracles) {
+                //console.debug('[getPrices] Load prices attemp', i + 1)
+                if (i > 0) {
+                    await delay(timeout);
+                }
+                await this.#collectPrices();
+            } else {
+                break;
+            }
+        }
+
+        if (this.#prices.length < this.#poolConfig.minimalOracles) {
+            throw new Error(`Error per updating prices, valid ${this.#prices.length} of ${this.#poolConfig.minimalOracles}`);  // if still not enough data after retries
+        }
+        const prices = this.#getPricesByAssetList(assets);
+        return new Prices(prices.dict, prices.dataCell);
+    }
+
+    #getPricesByAssetList(assets: PoolAssetsConfig) {
+        //console.debug('[getPricesByAssetList] start')
+        let pricesFiltered = this.#prices;  // for strict check this.#prices.filter(x => assets.every(asset => x.dict.has(asset.assetId)));
+
+        if (pricesFiltered.length < this.#poolConfig.minimalOracles) {
+            throw new Error("Not enough price data");
+        }
+
+        if (pricesFiltered.length > this.#poolConfig.minimalOracles) {
+            const sortedByTimestamp = pricesFiltered.slice().sort((a, b) => b.timestamp - a.timestamp);
+            const newerPrices = sortedByTimestamp.slice(0, this.#poolConfig.minimalOracles);
+            pricesFiltered = newerPrices.sort((a, b) => a.oracleId - b.oracleId);
+        }
+
+        const medianData = assets.map(asset => ({ assetId: asset.assetId, medianPrice: getMedianPrice(this.#prices, asset.assetId)}));
+
+        const nonEmptymedianData = medianData.filter(x => x.medianPrice != null) as { assetId: bigint, medianPrice: bigint }[];
+
+        const packedMedianData = packAssetsData(nonEmptymedianData);
+
+        const oraclesData = this.#prices.map(x => ({oracle: {id: x.oracleId, pubkey: x.pubkey}, data: {timestamp: x.timestamp, prices: x.dict}, signature: x.signature}));
+        const packedOracleData = packOraclesData(oraclesData, nonEmptymedianData.map(x => x.assetId));
+
+        const dict = Dictionary.empty<bigint, bigint>();
+        for (const medianDataAsset of nonEmptymedianData) {
+            dict.set(medianDataAsset.assetId, medianDataAsset.medianPrice);
+        }
+
+        return {
+            dict: dict,
+            dataCell: packPrices(packedMedianData, packedOracleData)
+        }
+    }
+
+    async #collectPrices(): Promise<boolean> {
+        try {
+            this.#prices = await Promise.any(this.#priceSources.map(x => collectAndFilterPrices(x, this.#poolConfig)));
+            return true;
+        }
+        catch { }
+        return false;
+    }
+
+    #filterPrices(): number {  // filter again for expire check
+        this.#prices = this.#prices.filter(verifyPricesTimestamp());
+        return this.#prices.length;
+    }
+
+    #filterEmptyPrincipalsAndAssets(principals: Dictionary<bigint, bigint>) {
+        return principals.keys().filter(x => principals.get(x)! > 0n).map(x => this.#poolConfig.poolAssetsConfig.find(asset => asset.assetId == x));
+    }
+}

package/src/prices/Types.ts

@@ -0,0 +1,44 @@
+import { Cell, Dictionary } from '@ton/core';
+
+/**
+ * Configuration for price source endpoints.
+ */
+export type PriceSourcesConfig = {
+    /** Endpoints for backend price data */
+    backendEndpoints: string[];
+    
+    /** Endpoints for IOTA price data */
+    iotaEndpoints: string[];
+    
+    /** Endpoints for ICP price data */
+    icpEndpoints: string[];
+};
+
+/**
+ * Default configuration for price source endpoints.
+ */
+export const DefaultPriceSourcesConfig: PriceSourcesConfig = {
+    backendEndpoints: ['evaa.space'],
+    iotaEndpoints: ['api.stardust-mainnet.iotaledger.net'],
+    icpEndpoints: ['6khmc-aiaaa-aaaap-ansfq-cai.raw.icp0.io'],
+}
+
+export type RawPriceData = {
+    dict: Dictionary<bigint, bigint>;
+    dataCell: Cell;
+    oracleId: number;
+    signature: Buffer;
+    pubkey: Buffer;
+    timestamp: number;
+};
+
+
+export type PriceData = {
+    dict: Dictionary<bigint, bigint>;
+    dataCell: Cell;
+};
+
+export type OraclePricesData = {
+    timestamp: number, 
+    prices: Dictionary<bigint, bigint>
+}

package/src/prices/constants.ts

@@ -0,0 +1 @@
+export const TTL_ORACLE_DATA_SEC = 120;

package/src/prices/index.ts

@@ -0,0 +1,6 @@
+export * from './sources';
+export * from './Types';
+export * from './constants';
+export * from './utils';
+export * from './Prices';
+export * from './PricesCollector';

package/src/prices/sources/Backend.ts

@@ -0,0 +1,62 @@
+import { beginCell, Cell, Dictionary } from "@ton/core";
+import { RawPriceData } from "..";
+import { PriceSource } from "./PriceSource";
+
+export class BackendPriceSource extends PriceSource {
+    protected priceSourceName: string = 'BackendPriceSource';
+
+    async getPrices(): Promise<RawPriceData[]> {
+        const data = await this.loadOracleData();
+        return data.map(outputData => this.parsePrices(outputData));
+    }
+
+    async loadOracleData(): Promise<OutputData[]> {
+        let response = await fetch(`https://${this._endpoint}/api/prices`, {
+            headers: { accept: 'application/json' },
+            signal: AbortSignal.timeout(5000)
+        });
+
+        const resp = (await response.json());
+        const data = resp as Record<string, string>;
+        let outputData: OutputData[] = [];
+
+        for (const nft of this._nfts) {
+            outputData.push({ oracleId: nft.id, data: data[nft.address] })
+        }
+    
+        return outputData;
+    }
+
+
+    parsePrices(outputData: OutputData): RawPriceData {
+        try {
+            //console.debug('outputData', outputData);
+            //console.debug(outputData.data.replace('0x', '').replace(/[0-9a-f]{2}/g, '%$&'))
+            const data = JSON.parse(
+                decodeURIComponent(outputData.data.replace('0x', '').replace(/[0-9a-f]{2}/g, '%$&')),
+            );
+            const pricesCell = Cell.fromBoc(Buffer.from(data['packedPrices'], 'hex'))[0];
+            const signature = Buffer.from(data['signature'], 'hex');
+            const publicKey = Buffer.from(data['publicKey'], 'hex');
+            const timestamp = Number(data['timestamp']);
+
+            return {
+                dict: pricesCell.beginParse().loadRef().beginParse().loadDictDirect(Dictionary.Keys.BigUint(256), Dictionary.Values.BigVarUint(4)),
+                dataCell: beginCell().storeRef(pricesCell).storeBuffer(signature).endCell(),
+                oracleId: outputData.oracleId,
+                signature: signature,
+                pubkey: publicKey,
+                timestamp: timestamp,
+            };
+        }
+        catch (error) {
+            //console.debug(`Price source error ${this.priceSourceName} ${outputData.oracleId} ${outputData.data} ${error}`);
+            throw error;
+        }
+    }
+}
+
+type OutputData = {
+    oracleId: number,
+    data: string
+};

package/src/prices/sources/Icp.ts

@@ -0,0 +1,27 @@
+import { BackendPriceSource } from ".";
+
+export class IcpPriceSource extends BackendPriceSource {
+    protected priceSourceName: string = 'IcpPriceSource';
+
+    async loadOracleData(): Promise<OutputData[]> {
+        let response = await fetch(`https://${this._endpoint}/prices`, {
+            headers: { accept: 'application/json' },
+            signal: AbortSignal.timeout(5000)
+        });
+
+        const data = (await response.json()) as Record<string, string>;
+
+        let outputData: OutputData[] = [];
+
+        for (const nft of this._nfts) {
+            outputData.push({oracleId: nft.id, data: data[nft.address] })
+        }
+
+        return outputData;
+    }
+}
+
+type OutputData = {
+    oracleId: number,
+    data: string
+};

package/src/prices/sources/Iota.ts

@@ -0,0 +1,90 @@
+import { beginCell, Cell, Dictionary } from "@ton/core";
+import { PriceSource } from "./PriceSource";
+import { RawPriceData } from "..";
+
+export class IotaPriceSource extends PriceSource {
+    protected priceSourceName: string = 'IotaPriceSource';
+
+    async getPrices(): Promise<RawPriceData[]> {
+        return await Promise.all(this.nfts.map(nft => this.loadOracleData(nft.address)
+                .then(x => this.parsePrices(x, nft.id))));
+    }
+
+    async loadOracleData(oracleNftId: String): Promise<OutputData> {
+        let result = await fetch(`https://${this._endpoint}/api/indexer/v1/outputs/nft/${oracleNftId}`, {
+            headers: { accept: 'application/json' },
+            signal: AbortSignal.timeout(5000)
+        });
+        let outputId = (await result.json()) as NftData;
+
+        result = await fetch(`https://${this._endpoint}/api/core/v2/outputs/${outputId.items[0]}`, {
+            headers: { accept: 'application/json' },
+            signal: AbortSignal.timeout(5000)
+        });
+        return (await result.json() as OutputData);
+    }
+
+
+    parsePrices(outputData: OutputData, oracleId: number): RawPriceData {
+        const data = JSON.parse(
+            decodeURIComponent(outputData.output.features[0].data.replace('0x', '').replace(/[0-9a-f]{2}/g, '%$&')),
+        );
+    
+        try {
+            const pricesCell = Cell.fromBoc(Buffer.from(data['packedPrices'], 'hex'))[0];
+            const signature = Buffer.from(data['signature'], 'hex');
+            const publicKey = Buffer.from(data['publicKey'], 'hex');
+            const timestamp = Number(data['timestamp']);
+
+            return {
+                dict: pricesCell.beginParse().loadRef().beginParse().loadDictDirect(Dictionary.Keys.BigUint(256), Dictionary.Values.BigVarUint(4)),
+                dataCell: beginCell().storeRef(pricesCell).storeBuffer(signature).endCell(),
+                oracleId: oracleId,
+                signature: signature,
+                pubkey: publicKey,
+                timestamp: timestamp,
+            };
+        }
+        catch (error) {
+            //console.debug(`Price source error ${this.priceSourceName} ${outputData} ${error}`);
+            throw error;
+        }
+    }
+}
+
+type NftData = {
+    ledgerIndex: number;
+    pageSize: number;
+    items: string[];
+};
+
+type OutputData = {
+    metadata: {
+        blockId: string;
+        transactionId: string;
+        outputIndex: number;
+        isSpent: boolean;
+        milestoneIndexSpent: number;
+        milestoneTimestampSpent: number;
+        transactionIdSpent: string;
+        milestoneIndexBooked: number;
+        milestoneTimestampBooked: number;
+        ledgerIndex: number;
+    };
+    output: {
+        type: number;
+        amount: string;
+        nftId: string;
+        unlockConditions: {
+            type: number;
+            address: {
+                type: number;
+                pubKeyHash: string;
+            };
+        }[];
+        features: {
+            type: number;
+            data: string;
+        }[];
+    };
+};

package/src/prices/sources/PriceSource.ts

@@ -0,0 +1,35 @@
+import { RawPriceData } from "..";
+import { OracleNFT } from "../../types/Master";
+
+export abstract class PriceSource {
+    protected priceSourceName: string = 'BackendPriceSource';
+    protected _endpoint: string;
+    protected _nfts: OracleNFT[];
+
+    constructor(endpoint: string, nfts: OracleNFT[]) {
+        this._endpoint = endpoint;
+        this._nfts = nfts;
+    }
+
+    get sourceName() {
+        return this.priceSourceName;
+    }
+
+    get endpoint() {
+        return this._endpoint;
+    }
+
+    get nfts() {
+        return this._nfts;
+    }
+
+    set endpoint(endpoint: string) {
+        this._endpoint = endpoint; 
+    }
+
+    set nfts(nfts: OracleNFT[]) {
+        this._nfts = nfts; 
+    }
+
+    abstract getPrices(): Promise<RawPriceData[]>;
+}

package/src/prices/sources/index.ts

@@ -0,0 +1,4 @@
+export * from './Backend';
+export * from './Iota';
+export * from './Icp';
+export * from './PriceSource';

package/src/prices/utils.ts

@@ -0,0 +1,170 @@
+import { beginCell, Cell, Dictionary, Slice } from "@ton/core";
+import { BackendPriceSource, DefaultPriceSourcesConfig, IcpPriceSource, IotaPriceSource, MAINNET_POOL_CONFIG, OraclePricesData, PriceData, PriceSource, PriceSourcesConfig, RawPriceData, TTL_ORACLE_DATA_SEC } from "..";
+import { Oracle, OracleNFT, PoolConfig } from "../types/Master";
+import { convertToMerkleProof, generateMerkleProofDirect } from "../utils/merkleProof";
+import { signVerify } from "@ton/crypto";
+
+export function verifyPricesTimestamp() {
+    return function(priceData: RawPriceData): boolean {
+        const timestamp = Date.now() / 1000;
+        const pricesTime = priceData.timestamp;
+
+        //console.debug('timestamp - pricesTime, pricesTime', timestamp - pricesTime, pricesTime);
+        return timestamp - pricesTime < TTL_ORACLE_DATA_SEC;
+    }
+}
+
+export function verifyPricesSign(nfts: OracleNFT[]) {
+    return function(priceData: RawPriceData): boolean {
+        if (nfts.findIndex(x => x.pubkey.equals(priceData.pubkey)) == -1) {
+            //console.debug('[verifyPricesSign] nft not found');
+            return false;
+        }
+
+        return verifyRawPriceDataSign(priceData)
+    }
+}
+
+/* export function verifyPricesAssets(assets: PoolAssetsConfig) {
+    return function(priceData: RawPriceData): boolean {
+        for (const asset of assets) {
+            if(!priceData.dict.has(asset.assetId)) {
+                return false;
+            }
+        }
+        return true;
+    }
+} */
+
+export function getMedianPrice(pricesData: PriceData[], asset: bigint): bigint | null {
+    try {
+        const usingPrices = pricesData.filter(x => x.dict.has(asset));
+        const sorted = usingPrices.map(x => x.dict.get(asset)!).sort((a, b) => Number(a) - Number(b));
+        
+        if (sorted.length == 0) {
+            return null;
+        }
+
+        const mid = Math.floor(sorted.length / 2);
+        if (sorted.length % 2 === 0) {
+            return (sorted[mid - 1] + sorted[mid]) / 2n;
+        } else {
+            return sorted[mid];
+        }
+    }
+    catch {
+        return null;
+    }
+}
+
+export function packAssetsData(assetsData: {assetId: bigint, medianPrice: bigint}[]): Cell {
+    if (assetsData.length == 0) {
+        throw new Error("No assets data to pack");
+    }
+    return assetsData.reduceRight(
+        (acc: Cell | null, {assetId, medianPrice}) => beginCell()
+                                                          .storeUint(assetId, 256)
+                                                          .storeCoins(medianPrice)
+                                                          .storeMaybeRef(acc)
+                                                        .endCell(), 
+        null
+    )!;
+}
+
+export function packPrices(assetsDataCell: Cell, oraclesDataCell: Cell): Cell {
+    let pricesCell = beginCell()
+      .storeRef(assetsDataCell)
+      .storeRef(oraclesDataCell)
+    .endCell();
+    return pricesCell;
+}
+
+export function createOracleDataProof(oracle: Oracle, 
+    data: OraclePricesData, 
+    signature: Buffer,
+    assets: Array<bigint>): Slice {
+    let prunedDict = generateMerkleProofDirect(data.prices, assets, Dictionary.Keys.BigUint(256));
+    let prunedData = beginCell().storeUint(data.timestamp, 32).storeMaybeRef(prunedDict).endCell();
+    let merkleProof = convertToMerkleProof(prunedData);
+    let oracleDataProof = beginCell().storeUint(oracle.id, 32).storeRef(merkleProof).storeBuffer(signature).asSlice();
+    return oracleDataProof;
+}
+
+export function packOraclesData(oraclesData: {oracle: Oracle, data: OraclePricesData, signature: Buffer}[], 
+    assets: Array<bigint>): Cell {
+    if (oraclesData.length == 0) {
+        throw new Error("no oracles data to pack");
+    }
+    let proofs = oraclesData.sort((d1, d2) => d1.oracle.id - d2.oracle.id).map(
+        ({oracle, data, signature}) => createOracleDataProof(oracle, data, signature, assets)
+    );
+    return proofs.reduceRight((acc: Cell | null, val) => beginCell().storeSlice(val).storeMaybeRef(acc).endCell(), null)!;
+}
+
+export function sumDicts(result: Dictionary<bigint, bigint>, addendum: Dictionary<bigint, bigint>) {
+    for (const key of addendum.keys()) {
+        const current = result.get(key)!;
+        const value = addendum.get(key)!;
+
+        if (current === undefined) {
+            result.set(key, value);
+            continue;
+        }
+
+        result.set(key, current + value);
+    }
+}
+
+export function generatePriceSources(config: PriceSourcesConfig = DefaultPriceSourcesConfig, nfts: OracleNFT[] = MAINNET_POOL_CONFIG.oracles) {
+    let result: PriceSource[] = config.backendEndpoints.map(x => new BackendPriceSource(x, nfts));
+
+    result.push(...config.icpEndpoints.map(x => new IcpPriceSource(x, nfts)));
+    result.push(...config.iotaEndpoints.map(x => new IotaPriceSource(x, nfts)));
+
+    return result;
+}
+
+export async function collectAndFilterPrices(priceSource: PriceSource, poolConfig: PoolConfig): Promise<RawPriceData[]> {
+    const prices = await priceSource.getPrices();
+         
+    //console.debug('[FILTERING] before filtering prices len ', priceSource.sourceName, prices.length);
+    return await (async () => {
+            const acceptedPrices: RawPriceData[] = prices.filter(
+            price => verifyPricesTimestamp()(price) && verifyPricesSign(poolConfig.oracles)(price)
+        );
+
+        //console.debug('[FILTERING] after filtering prices len ', priceSource.sourceName, acceptedPrices.length);
+
+        if (acceptedPrices.length < poolConfig.minimalOracles) {
+            throw new Error("Prices are outdated");
+        }
+
+        return acceptedPrices;
+    })();
+}
+
+export function unpackMedianPrices(pricesCell: Cell): Dictionary<bigint, bigint> | undefined {
+    if (!pricesCell) return undefined;
+    const slice = pricesCell.beginParse();
+    let assetCell: Cell | null = slice.loadRef();
+    const res = Dictionary.empty<bigint, bigint>();
+    while (assetCell != Cell.EMPTY && assetCell !== null) {
+        const slice = assetCell.beginParse();
+        const assetId = slice.loadUintBig(256);
+        const medianPrice = slice.loadCoins();
+        res.set(assetId, medianPrice);
+        assetCell = slice.loadMaybeRef();
+    }
+    return res;
+}
+
+export function verifyRawPriceDataSign(priceData: RawPriceData): boolean {
+    const message = priceData.dataCell.refs[0].hash()
+    const signature = priceData.signature;
+    const publicKey = priceData.pubkey;
+
+    const valid: boolean = signVerify(message, signature, publicKey);
+    //console.debug('[verifyRawPriceDataSign] sign is valid:', valid);
+
+    return valid;
+}

package/src/types/Master.ts

@@ -89,7 +89,7 @@ export type AssetData = {
     balance: bigint;
     trackingSupplyIndex: bigint;
     trackingBorrowIndex: bigint;
-    awaitedSupply?: bigint;
+    awaitedSupply: bigint;
 };
 
 export type AssetInterest = {
@@ -126,7 +126,8 @@ export type AgregatedBalances = {
 
 export type OracleNFT = {
     id: number,
-    address: string
+    address: string,
+    pubkey: Buffer
 }
 
 export type Oracle = {

package/src/types/User.ts

@@ -1,4 +1,4 @@
-import { Address, Cell, Dictionary } from '@ton/core';
+import { Address, BitBuilder, Cell, Dictionary } from '@ton/core';
 import { AssetConfig, ExtendedAssetData, ExtendedAssetsConfig, ExtendedAssetsData, MasterConfig, MasterConstants, PoolAssetConfig, PoolConfig } from './Master';
 
 export enum BalanceType {
@@ -38,6 +38,7 @@ export type UserLiteData = {
     masterAddress: Address;
     ownerAddress: Address;
     principals: Dictionary<bigint, bigint>;
+    realPrincipals: Dictionary<bigint, bigint>;  // principals before applying dusts
     state: number;
     balances: Dictionary<bigint, UserBalance>;
     trackingSupplyIndex: bigint;
@@ -59,7 +60,6 @@ export type UserDataActive = UserLiteData & {
     limitUsedPercent: number;
     limitUsed: bigint;
     healthFactor: number;
-
     liquidationData: LiquidationData;
 };