@steerprotocol/data-connector-loader 2.1.1 → 3.0.1

package/lib/esm/execution/executionDataConnector.js

@@ -0,0 +1,90 @@
+import { ExecutionBase } from './executionBase';
+import { PluginManager } from '../utils/pluginManager';
+import { extendOHLCV, fetchData } from '../utils/utils';
+export class DataConnectorExecution extends ExecutionBase {
+    constructor(dataConnectors, maxRetries = 3) {
+        super(maxRetries);
+        this.dataConnectors = dataConnectors;
+    }
+    async execute(_vaultConfig) {
+        const vaultConfig = JSON.parse(_vaultConfig);
+        const transformationData = [];
+        const pluginManager = new PluginManager();
+        await pluginManager.registerFromPublicConfig();
+        for (const dataConnector of this.dataConnectors) {
+            if (!dataConnector.module) {
+                await dataConnector.load();
+            }
+            const executionContext = await this.getExecutionContext(dataConnector, vaultConfig);
+            dataConnector.setExecutionContextDataConnector(executionContext);
+            const version = await this.getDataConnectorVersion(dataConnector);
+            if (version === 1) {
+                const initialRequest = await dataConnector.execute('');
+                const finalResponse = await this.handleRequestResponse(dataConnector, pluginManager, initialRequest);
+                const transformedData = await this.transformData(dataConnector, finalResponse);
+                transformationData.push(transformedData);
+            }
+            else {
+                const finalResponse = await dataConnector.execute('');
+                const transformedData = await this.transformData(dataConnector, finalResponse);
+                transformationData.push(transformedData);
+            }
+        }
+        return transformationData;
+    }
+    async handleRequestResponse(dataConnector, pluginManager, initialRequest) {
+        let request = initialRequest;
+        let response;
+        try {
+            while (request !== 'true') {
+                if (dataConnector.configObj.isChainRead == true) {
+                    response = await dataConnector.view(JSON.parse(request));
+                }
+                else {
+                    const parsedRequest = await pluginManager.process(typeof request == 'string' ? JSON.parse(request) : request);
+                    await this.delay(1000);
+                    response = await fetchData(parsedRequest, 0);
+                }
+                request = await dataConnector.execute(JSON.stringify(response));
+            }
+        }
+        catch (e) {
+            // process.env.SILENT_ERRORS
+            if (process.env.DATA_LOADER_SILENT_ERRORS !== 'true') {
+                console.error('Data connector failed to process.', e);
+            }
+            throw e;
+        }
+        return dataConnector.getResults();
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    async transformData(dataConnector, rawData) {
+        let transformedData = typeof rawData === 'string' ? JSON.parse(rawData) : rawData;
+        if (transformedData == null || transformedData.length == 0) {
+            const results = dataConnector.getResults();
+            const parsedResults = JSON.parse(results);
+            if (parsedResults.length == null) {
+                transformedData = results;
+            }
+            else {
+                transformedData = parsedResults;
+            }
+        }
+        const version = await this.getDataConnectorVersion(dataConnector);
+        if (version == 1 && transformedData.length > 0) {
+            const keys = Object.keys(transformedData[0]);
+            const requiredKeys = ['timestamp', 'open', 'high', 'low', 'close'];
+            const hasAllKeys = requiredKeys.every((key) => keys.includes(key));
+            // has the 'lookback' key
+            const lookback = dataConnector?.configObj?.lookback;
+            if (hasAllKeys && !!lookback) {
+                // Now that we have identified a v1 data connector, we need to transform the data so that the data is up to the current execution epoch timestamp
+                if (dataConnector?.executionContext?.epochTimestamp) {
+                    const extended = extendOHLCV(transformedData, lookback, dataConnector?.executionContext?.epochTimestamp * 1000);
+                    transformedData = extended;
+                }
+            }
+        }
+        return transformedData;
+    }
+}

package/lib/esm/index.d.ts

@@ -0,0 +1,5 @@
+import { DataConnector } from './bundle/bundleDataConnector';
+import { DataConnectorExecution } from './execution/executionDataConnector';
+import { ExecutionApp } from './execution/executionApp';
+export * from './types';
+export { DataConnector, DataConnectorExecution, ExecutionApp };

package/lib/esm/index.js

@@ -0,0 +1,5 @@
+import { DataConnector } from './bundle/bundleDataConnector';
+import { DataConnectorExecution } from './execution/executionDataConnector';
+import { ExecutionApp } from './execution/executionApp';
+export * from './types';
+export { DataConnector, DataConnectorExecution, ExecutionApp };

package/lib/esm/package.json

@@ -0,0 +1 @@
+{"type": "module"}

package/lib/esm/types.d.ts

@@ -0,0 +1,104 @@
+import { ContractInterface } from 'ethers';
+export type VaultInput = {
+    _id?: string;
+    tokenId: string;
+    vaultAddress: string;
+    state: string;
+    token0: string;
+    token1: string;
+    name: string;
+    symbol: string;
+    beaconName: string;
+    token0Name: string;
+    token1Name: string;
+    token0Symbol: string;
+    token1Symbol: string;
+    decimals: string;
+    token0Decimals: string;
+    token1Decimals: string;
+    payloadIpfs: string;
+    timestamp: number;
+    vaultConfigPayload: VaultConfigPayload;
+};
+export interface VaultConfigPayload {
+    strategyConfigData: StrategyConfigData;
+    vaultPayload: VaultPayload;
+    dataConnectorsData: DataConnectorsDaum[];
+    vaultAddress: string;
+    epochTimestamp: number;
+    anchorBlockNumber?: number;
+}
+export interface StrategyConfigData {
+    [key: string]: string | number;
+    epochStart: number;
+    epochLength: string;
+}
+export interface ConcentratedLiquidityManagerPayload {
+    maxTotalSupply: string;
+    ratioErrorTolerance: number;
+    maxTickChange: number;
+    twapInterval: number;
+    fee: number;
+    slippage: number;
+}
+export type VaultPayload = ConcentratedLiquidityManagerPayload;
+export interface DataConnectorsDaum {
+    bundleHash: string;
+    configData: ConfigData;
+}
+export interface ConfigData {
+    [key: string]: string | number;
+}
+export interface ExecutionContext {
+    executionTimestamp: number;
+    epochLength: number;
+    vaultAddress: string;
+    blockTime: number;
+    blockNumber: number;
+    epochTimestamp: number;
+}
+export interface Position {
+    startTick: number;
+    endTick: number;
+    weight: number;
+}
+export interface SmartPoolExecutionContext extends ExecutionContext {
+    positions: Position[];
+    currentTick: number;
+    lastExecutionTime: number;
+}
+export type RpcConfig = {
+    url: string;
+    network?: string;
+};
+export type EtherRpcConfig = {
+    [key: string]: RpcConfig;
+};
+export interface ViewConfig {
+    address: string;
+    abi: ContractInterface;
+    method: string;
+    arguments: unknown[];
+}
+export interface Slot0 {
+    sqrtPriceX96?: bigint;
+    tick: number;
+    observationIndex?: number;
+    observationCardinality?: number;
+    observationCardinalityNext?: number;
+    feeProtocol?: number;
+    unlocked: boolean;
+    price?: bigint;
+    prevInitializedTick?: number;
+    fee?: number;
+    timepointIndex?: number;
+    communityFee?: number;
+}
+export type ViewContractConfig = {
+    epochTimestamp?: number;
+    address: string;
+    abi: any;
+    arguments: Array<any>;
+    method: string;
+    rpcUrl?: string;
+};

package/lib/esm/types.js

@@ -0,0 +1 @@
+export {};

package/lib/esm/utils/blockFinder.js

@@ -0,0 +1,20 @@
+// import { providers,  } from "ethers";
+/**
+ * Iterates down from current block to find the block previous or equal to the given timestamp
+ * Not the most efficient for far past timestamps, a binary search could be used for better performance
+ * @param timestamp
+ * @param Idenity
+ */
+export async function getBlockFromTimestamp(timestamp, provider) {
+    let currentBlock = (await provider.getBlockNumber()) - 1;
+    let runningTimestamp = (await provider.getBlock(currentBlock))?.timestamp;
+    if (!runningTimestamp)
+        return currentBlock;
+    while (runningTimestamp > timestamp) {
+        currentBlock -= 1;
+        runningTimestamp = (await provider.getBlock(currentBlock))?.timestamp;
+        if (!runningTimestamp)
+            return currentBlock;
+    }
+    return currentBlock;
+}

package/lib/esm/utils/consts.d.ts

@@ -0,0 +1,4 @@
+export declare const API_BASE_URL = "https://cvrlulno2m.execute-api.ap-southeast-1.amazonaws.com/dev/contract-view-call";
+export declare const API_MAX_RETRIES = 3;
+export declare const API_INITIAL_DELAY = 1000;
+export declare const CENTRALIZE_CONTRACT_INTERACTION_CHAINS: string[];

package/lib/esm/utils/consts.js

@@ -0,0 +1,4 @@
+export const API_BASE_URL = 'https://cvrlulno2m.execute-api.ap-southeast-1.amazonaws.com/dev/contract-view-call';
+export const API_MAX_RETRIES = 3;
+export const API_INITIAL_DELAY = 1000;
+export const CENTRALIZE_CONTRACT_INTERACTION_CHAINS = ['2222', '42161', '9001', '137', '56'];

package/lib/esm/utils/pluginManager.js

@@ -0,0 +1,67 @@
+import axios from 'axios';
+export class PluginManager {
+    constructor() {
+        this.registeredPluginConfigs = [];
+    }
+    async registerFromPublicConfig() {
+        const { data: dataConnectorPlugins } = await axios.get('https://ro81h8hq6b.execute-api.us-east-1.amazonaws.com/data-connector-plugins');
+        for (const plugin of dataConnectorPlugins) {
+            this.registerPluginConfig(plugin.name, plugin.domain, plugin.headers);
+        }
+    }
+    registerPluginConfig(name, domain, headers) {
+        this.registeredPluginConfigs.push({ name, domain, headers });
+    }
+    findMatchingPluginConfigs(url) {
+        const matchingHeaders = {};
+        for (const pluginConfig of this.registeredPluginConfigs) {
+            if (url.includes(pluginConfig.domain)) {
+                Object.assign(matchingHeaders, pluginConfig.headers);
+            }
+        }
+        return matchingHeaders;
+    }
+    interpolateEnvVars(str) {
+        return str.replace(/\{\{(\w+)\}\}/g, (_match, envVar) => {
+            const envValue = process.env[envVar];
+            if (envValue === undefined) {
+                console.log(`Environment variable '${envVar}' not found.`);
+                return '';
+            }
+            return envValue;
+        });
+    }
+    interpolateHeaders(headers) {
+        const interpolatedHeaders = {};
+        for (const [key, value] of Object.entries(headers)) {
+            interpolatedHeaders[key] = typeof value === 'string' ? this.interpolateEnvVars(value) : value;
+        }
+        return interpolatedHeaders;
+    }
+    async process(config) {
+        if (!config.url) {
+            throw new Error('Request URL is not provided');
+        }
+        const matchingHeaders = this.findMatchingPluginConfigs(config.url);
+        const interpolatedHeaders = this.interpolateHeaders(matchingHeaders);
+        const updatedHeaders = { ...config.headers, ...interpolatedHeaders };
+        const updatedConfig = {
+            ...config,
+            headers: updatedHeaders,
+        };
+        return updatedConfig;
+    }
+}
+// // Usage example:
+// (async () => {
+//   const headerMerger = new PluginManager();
+//   headerMerger.registerPluginConfig('Example API', 'example.com', { 'X-Custom-Header': '{{MY_ENV_VAR}}' });
+//   process.env['MY_ENV_VAR'] = 'ExampleValue';
+//   const requestConfig: AxiosRequestConfig = {
+//     url: 'https://example.com/api/some-resource',
+//     method: 'GET',
+//   };
+//   const updatedRequestConfig = await headerMerger.process(requestConfig);
+//   const response = await axios(updatedRequestConfig);
+//   console.log(response.data);
+// })();

package/lib/esm/utils/utils.d.ts

@@ -0,0 +1,16 @@
+import { Candle } from '@steerprotocol/app-loader';
+import { AxiosRequestConfig } from 'axios';
+import { ExecutionContext, ViewContractConfig } from '../types';
+import { providers } from 'ethers';
+export declare const extendOHLCV: (rawData: Candle[], period: number, endTimestamp: number) => Candle[];
+export declare function fetchData(request: AxiosRequestConfig, attempts: number): Promise<any>;
+export declare function readSlot0Contract({ address, abi, currentExecutionContext, functionName, provider, rpc, chainId, }: {
+    address: string;
+    abi: any;
+    currentExecutionContext: ExecutionContext;
+    functionName: string;
+    provider: providers.Provider;
+    rpc: string;
+    chainId: number;
+}): Promise<any>;
+export declare function callWithRetry(config: ViewContractConfig): Promise<any>;

package/lib/esm/utils/utils.js

@@ -0,0 +1,119 @@
+import axios from 'axios';
+import { Contract } from 'ethers';
+import { API_BASE_URL, API_INITIAL_DELAY, API_MAX_RETRIES, CENTRALIZE_CONTRACT_INTERACTION_CHAINS } from './consts';
+// The purpose of this function is to look at the end timestamp and extend the current candle array so that it backfills any missing candles up to the end timestamp
+export const extendOHLCV = (rawData, period, endTimestamp) => {
+    // Sort rawData by timestamp
+    const sortedData = rawData.sort((a, b) => a.timestamp - b.timestamp);
+    const lastCandle = sortedData[sortedData.length - 1];
+    const startTimestamp = lastCandle ? lastCandle.timestamp : endTimestamp;
+    // Calculate the number of periods to backfill
+    const periodsToBackfill = Math.floor((endTimestamp - startTimestamp) / 1000 / period);
+    // Backfill the candles
+    for (let i = 0; i < periodsToBackfill; i++) {
+        const candleWidth = period * (i + 1);
+        const newCandle = {
+            timestamp: startTimestamp + candleWidth,
+            open: lastCandle.open,
+            high: lastCandle.open,
+            low: lastCandle.open,
+            close: lastCandle.open,
+            volume: 0, // You may need to replace these with appropriate logic
+        };
+        sortedData.push(newCandle);
+    }
+    // Return the converted data
+    return sortedData;
+};
+async function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function exponentialBackoffDelay(attempts) {
+    const baseDelay = 1000; // 1 second
+    const maxDelay = 30000; // 30 seconds
+    const delay = Math.min(baseDelay * Math.pow(2, attempts - 1), maxDelay);
+    return delay;
+}
+export async function fetchData(request, attempts) {
+    log('Request payload: =============', request);
+    attempts = attempts + 1;
+    try {
+        const response = (await axios(request)).data;
+        // this.log('API Response: =============', response);
+        if ('errors' in response && response['errors'].length > 0) {
+            if (attempts < 3) {
+                await delay(exponentialBackoffDelay(attempts));
+                return await fetchData(request, attempts);
+            }
+            else {
+                throw new Error('Maximum retry attempts reached');
+            }
+        }
+        return response;
+    }
+    catch (error) {
+        if (attempts < 3) {
+            await delay(exponentialBackoffDelay(attempts));
+            return await fetchData(request, attempts);
+        }
+        else {
+            throw error;
+        }
+    }
+}
+export async function readSlot0Contract({ address, abi, currentExecutionContext, functionName, provider, rpc, chainId, }) {
+    // Create a new contract instance
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    if (!provider)
+        throw new Error('Provider not initialized');
+    const contract = new Contract(address, abi, provider);
+    // For formatting the parameters if needed
+    const paramsArray = [];
+    // Calculate block number for determinism
+    // Best way to handle contextually non-null values?
+    // const syncedBlock = await getBlockFromTimestamp(this.executionContext!.epochTimestamp, this.providerIdentity!) //@note uncomment for deterministic blocks
+    // Call the desired method with the specified arguments at synced block
+    // , {blockTag: syncedBlock}
+    let result;
+    if (CENTRALIZE_CONTRACT_INTERACTION_CHAINS.indexOf(chainId.toString()) === -1) {
+        result = await contract[functionName](...paramsArray, {
+            blockTag: currentExecutionContext?.blockNumber,
+        }); //@note uncomment for deterministic blocks
+    }
+    else {
+        const viewContractConfig = {
+            epochTimestamp: currentExecutionContext.epochTimestamp,
+            address: address,
+            abi: abi,
+            arguments: paramsArray,
+            method: functionName,
+            rpcUrl: rpc,
+        };
+        result = await callWithRetry(viewContractConfig);
+    }
+    // This gets stringified before calling main
+    return result;
+}
+export async function callWithRetry(config) {
+    let retries = 0;
+    let delay = API_INITIAL_DELAY;
+    while (retries < API_MAX_RETRIES) {
+        try {
+            const response = await axios.post(API_BASE_URL, config);
+            return JSON.parse(response.data); // Return the data if the call was successful
+        }
+        catch (error) {
+            console.error(`Attempt ${retries + 1} failed. Retrying in to fetch the contract method ${config.method} ${delay / 1000} seconds...`);
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            delay *= 2; // Double the delay for the next retry
+            retries++;
+        }
+    }
+    throw new Error(`Failed after ${API_MAX_RETRIES} attempts. to call method ${config.method}`);
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function log(...args) {
+    if (process.env.DEBUG === 'true') {
+        console.log(...args);
+    }
+}

package/lib/execution/executionApp.d.ts

@@ -1,6 +1,6 @@
-import { ethers } from "ethers";
-import { EtherRpcConfig } from "../types";
-import { ExecutionBase } from "./executionBase";
+import { ethers } from 'ethers';
+import { EtherRpcConfig } from '../types';
+import { ExecutionBase } from './executionBase';
 export declare class ExecutionApp extends ExecutionBase {
     steerSubgraph: string;
     vault: string;