@sentio/sdk 1.7.14 → 1.7.15

package/src/generic-processor.ts

@@ -0,0 +1,38 @@
+import { BaseContract } from '@ethersproject/contracts'
+import { BoundContractView, ContractView } from './context'
+import { BindOptions } from './bind-options'
+import { BaseProcessor } from './base-processor'
+import { getProvider } from './provider'
+import { addProcessor, getContractName } from './binds'
+
+export class GenericProcessor extends BaseProcessor<
+  BaseContract,
+  BoundContractView<BaseContract, ContractView<BaseContract>>
+> {
+  eventAbi: string[]
+  constructor(eventAbi: string[], options: BindOptions) {
+    super(options)
+    this.eventAbi = eventAbi
+  }
+
+  protected CreateBoundContractView(): BoundContractView<BaseContract, ContractView<BaseContract>> {
+    const contract = new BaseContract(this.config.address, this.eventAbi, getProvider(this.config.network))
+    return new BoundContractView(new ContractView<BaseContract>(contract))
+  }
+
+  public static bind(eventAbi: string[] | string, options: BindOptions): GenericProcessor {
+    const AbiName = 'Generic'
+    if (!Array.isArray(eventAbi)) {
+      eventAbi = [eventAbi]
+    }
+
+    // let processor = getProcessor(AbiName, options) as GenericProcessor;
+    // if (!processor) {
+    const finalOptions = Object.assign({}, options)
+    finalOptions.name = getContractName(AbiName, options.name, options.address, options.network)
+    const processor = new GenericProcessor(eventAbi, finalOptions)
+    addProcessor(AbiName, options, processor)
+    // }
+    return processor
+  }
+}

package/src/index.ts

@@ -0,0 +1,19 @@
+export { BaseProcessor } from './base-processor'
+export { GenericProcessor } from './generic-processor'
+export { BaseProcessorTemplate } from './base-processor-template'
+export { Context, ContractView, BoundContractView, SolanaContext } from './context'
+export { ProcessorServiceImpl } from './service'
+export { Counter, Meter, Gauge } from './meter'
+export { getProvider, setProvider, DummyProvider } from './provider'
+export { SolanaBaseProcessor } from './solana-processor'
+export { ContractNamer } from './contract-namer'
+export { BindOptions, SolanaBindOptions } from './bind-options'
+export { transformEtherError } from './error'
+export { ProcessorState } from './processor-state'
+export { BigNumber as BigDecimal } from 'bignumber.js'
+
+export { getProcessor, addProcessor, getContractByABI, addContractByABI, getContractName } from './binds'
+
+export * from './gen/processor/protos/processor'
+
+export { SPLTokenProcessor } from './solana/builtin'

package/src/meter.ts

@@ -0,0 +1,122 @@
+import { RecordMetaData } from './gen/processor/protos/processor'
+import { Context, EthContext, SolanaContext } from './context'
+import { toMetricValue, Numberish } from './numberish'
+import Long from 'long'
+
+function GetRecordMetaData(ctx: EthContext | SolanaContext, name: string, labels: Labels): RecordMetaData {
+  if (ctx instanceof Context) {
+    if (ctx.log) {
+      return {
+        contractAddress: ctx.contract.rawContract.address,
+        blockNumber: Long.fromNumber(ctx.log.blockNumber, true), // TODO need number type to be long
+        transactionIndex: ctx.log.transactionIndex,
+        logIndex: ctx.log.logIndex,
+        chainId: ctx.chainId.toString(),
+        name: name,
+        labels: labels,
+      }
+    }
+    if (ctx.block) {
+      return {
+        contractAddress: ctx.contract.rawContract.address,
+        blockNumber: Long.fromNumber(ctx.block.number, true),
+        transactionIndex: -1,
+        logIndex: -1,
+        chainId: ctx.chainId.toString(),
+        name: name,
+        labels: labels,
+      }
+    }
+  } else if (ctx instanceof SolanaContext) {
+    return {
+      contractAddress: ctx.address,
+      blockNumber: Long.ZERO, // TODO need number type to be long
+      transactionIndex: 0,
+      logIndex: 0,
+      chainId: 'SOL:mainnet', // TODO set in context
+      name: name,
+      labels: labels,
+    }
+  }
+  throw new Error("This can't happen")
+}
+
+export type Labels = { [key: string]: string }
+
+export class Counter {
+  private readonly ctx: EthContext | SolanaContext
+  private readonly name: string
+
+  constructor(name: string, ctx: EthContext | SolanaContext) {
+    this.name = name
+    this.ctx = ctx
+  }
+
+  add(value: Numberish, labels: Labels = {}) {
+    this.record(value, labels, true)
+  }
+
+  sub(value: Numberish, labels: Labels = {}) {
+    this.record(value, labels, false)
+  }
+
+  private record(value: Numberish, labels: Labels, add: boolean) {
+    this.ctx.counters.push({
+      metadata: GetRecordMetaData(this.ctx, this.name, labels),
+      metricValue: toMetricValue(value),
+      add: add,
+      runtimeInfo: undefined,
+    })
+  }
+}
+
+export class Gauge {
+  private readonly name: string
+  private readonly ctx: EthContext | SolanaContext
+
+  constructor(name: string, ctx: EthContext | SolanaContext) {
+    this.name = name
+    this.ctx = ctx
+  }
+
+  record(value: Numberish, labels: Labels = {}) {
+    this.ctx.gauges.push({
+      metadata: GetRecordMetaData(this.ctx, this.name, labels),
+      metricValue: toMetricValue(value),
+      runtimeInfo: undefined,
+    })
+  }
+}
+
+export class Meter {
+  private readonly ctx: EthContext | SolanaContext
+
+  // TODO is map necessary since we are sending request remotely?
+  // counterMap = new Map<string, Counter>()
+  // gaugeMap = new Map<string, Gauge>()
+
+  constructor(ctx: EthContext | SolanaContext) {
+    this.ctx = ctx
+  }
+
+  Counter(name: string): Counter {
+    // let counter = this.counterMap.get(name)
+
+    // if (!counter) {
+    //     counter =  new Counter(name, this.ctx)
+    // }
+    // return counter
+
+    return new Counter(name, this.ctx)
+  }
+
+  Gauge(name: string): Gauge {
+    // let gauge = this.gaugeMap.get(name)
+    //
+    // if (!gauge) {
+    //     gauge = new Gauge(name, this.ctx)
+    // }
+    // return gauge
+    return new Gauge(name, this.ctx)
+  }
+}

package/src/numberish.test.ts

@@ -0,0 +1,82 @@
+import { expect } from 'chai'
+import { toBigInteger, toMetricValue } from './numberish'
+import { webcrypto } from 'crypto'
+import { BigNumber } from 'ethers'
+import { performance } from 'perf_hooks'
+import { BigInteger } from './gen/processor/protos/processor'
+import { BigDecimal } from './index'
+
+// TODO add test for type conversion
+describe('Numberish tests', () => {
+  const values: bigint[] = [0n, -0n, 3815372408723498172304781320847103784n, 2132n, -18708707n, 123n << 100n]
+
+  test('big integer conversion correctness ', async () => {
+    for (const v of values) {
+      const b = toBigInteger(v)
+      const hex1 = BigIntegerToHex(b)
+      const hex2 = v.toString(16)
+      expect(hex1).equals(hex2)
+    }
+  })
+
+  test('random big integer conversion correctness ', async () => {
+    for (let i = 0; i < 1000; i++) {
+      const random = webcrypto.getRandomValues(new Uint8Array(256))
+      const v = BigNumber.from(random).toBigInt()
+      const b = toBigInteger(v)
+
+      expect(BigNumber.from(b.data).eq(v)).to.equals(true)
+    }
+  })
+
+  test.skip('random big integer performance', async () => {
+    jest.setTimeout(100000)
+    jest.retryTimes(3)
+
+    let timer1 = 0
+    let timer2 = 0
+    for (let i = 0; i < 1000; i++) {
+      // Use higher value for local debugging
+      const random = webcrypto.getRandomValues(new Uint8Array(256))
+      const v = BigNumber.from(random).toBigInt()
+
+      let start = performance.now()
+      toBigInteger(v)
+      timer1 += performance.now() - start
+
+      start = performance.now()
+      v.toString(16)
+      timer2 += performance.now() - start
+    }
+
+    console.log(timer1, timer2)
+    expect(timer1).to.lessThan(timer2 * 3)
+  })
+
+  test('metric values', async () => {
+    const longDec = '12.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002'
+    expect(toMetricValue(new BigDecimal(longDec)).bigDecimal == longDec)
+
+    const complexDec = '-7.350918e-428'
+    expect(toMetricValue(new BigDecimal(complexDec)).bigDecimal == complexDec)
+
+    expect(BigIntegerToBigInt(toMetricValue(new BigDecimal('100000')).bigInteger!) === 100000n)
+  })
+})
+
+// Performance very bad
+function BigIntegerToHex(b: BigInteger): string {
+  let res = BigNumber.from(b.data).toBigInt().toString(16)
+  if (b.negative) {
+    res = '-' + res
+  }
+  return res
+}
+
+function BigIntegerToBigInt(b: BigInteger): bigint {
+  let res = BigNumber.from(b.data).toBigInt()
+  if (b.negative) {
+    res = -res
+  }
+  return res
+}

package/src/numberish.ts

@@ -0,0 +1,99 @@
+import { BigNumber } from 'ethers'
+import { BigInteger, MetricValue } from './gen/processor/protos/processor'
+import { BigDecimal } from '.'
+import { BN } from '@project-serum/anchor'
+
+export type Numberish = number | BigNumber | bigint | BigDecimal //BigNumberish
+
+export function toMetricValue(value: Numberish): MetricValue {
+  if (value instanceof BigNumber) {
+    return MetricValue.fromPartial({
+      bigInteger: toBigInteger(value.toBigInt()),
+    })
+  }
+  if (value instanceof BigDecimal) {
+    // Carefully consider the use case here
+    if (value.isInteger()) {
+      return MetricValue.fromPartial({
+        bigInteger: bigDecimalToBigInteger(value),
+      })
+    } else {
+      return MetricValue.fromPartial({
+        bigDecimal: value.toString(), // e.g. -7.350918e-428
+      })
+    }
+  }
+  if (BN.isBN(value)) {
+    return MetricValue.fromPartial({
+      bigInteger: bnToBigInteger(value),
+    })
+  }
+  if (typeof value === 'bigint' || Number.isInteger(value)) {
+    return MetricValue.fromPartial({
+      bigInteger: toBigInteger(value),
+    })
+  }
+
+  return MetricValue.fromPartial({
+    doubleValue: Number(value),
+  })
+}
+function bigDecimalToBigInteger(a: BigDecimal): BigInteger {
+  const negative = a.isNegative()
+  if (negative) {
+    a = a.abs()
+  }
+  return hexToBigInteger(a.toString(16), negative)
+}
+
+function bnToBigInteger(a: BN): BigInteger {
+  const negative = a.isNeg()
+  if (negative) {
+    a = a.abs()
+  }
+  return hexToBigInteger(a.toString(16), negative)
+}
+
+function intToBigInteger(a: bigint | number): BigInteger {
+  const negative = a < 0
+  if (negative) {
+    a = -a
+  }
+  return hexToBigInteger(a.toString(16), negative)
+}
+
+export function toBigInteger(a: Numberish): BigInteger {
+  if (a instanceof BigDecimal) {
+    return bigDecimalToBigInteger(a)
+  }
+  if (a instanceof BN) {
+    return bnToBigInteger(a)
+  }
+  if (a instanceof BigNumber) {
+    return intToBigInteger(a.toBigInt())
+  }
+  return intToBigInteger(a)
+
+  // Following code is actually very slow
+  // while (a > 0) {
+  //   const d = a & 0xffn
+  //   a >>= 8n
+  //   value.push(Number(d))
+  // }
+  //
+  // return {
+  //   negative, value: new Uint8Array(value.reverse()),
+  // }
+}
+
+function hexToBigInteger(hex: string, negative: boolean): BigInteger {
+  if (hex.length % 2 === 1) {
+    hex = '0' + hex
+  }
+  const buffer = Buffer.from(hex, 'hex')
+
+  return {
+    negative: negative,
+    data: new Uint8Array(buffer),
+  }
+}

package/src/processor-runner.ts

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+
+import commandLineArgs from 'command-line-args'
+import { createServer } from 'nice-grpc'
+import { ProcessorDefinition } from './gen/processor/protos/processor'
+import { ProcessorServiceImpl } from './service'
+import { setProvider } from './provider'
+
+import path from 'path'
+import fs from 'fs-extra'
+import { ProcessorState } from './processor-state'
+
+global.sentio_sdk = require('.')
+global.PROCESSOR_STATE = new ProcessorState()
+
+function tryRequire(name: string): { module: any; name: string; path: string } | undefined {
+  const req = eval('require')
+
+  try {
+    let path: string
+    try {
+      path = req.resolve(name, { paths: [process.cwd()] })
+    } catch {
+      path = req.resolve(name)
+    }
+
+    const module = { module: req(path), name, path }
+    console.log('Load successfully: ', name)
+    return module
+  } catch (err) {
+    if (err instanceof Error && err.message.startsWith(`Cannot find module '${name}'`)) {
+      // this error is expected
+      console.log("Couldn't load (expected): ", name)
+      return undefined
+    } else {
+      throw err
+    }
+  }
+}
+
+const optionDefinitions = [
+  { name: 'target', type: String, defaultOption: true },
+  { name: 'port', alias: 'p', type: String, defaultValue: '4000' },
+  { name: 'concurrency', type: Number, defaultValue: 4 },
+  { name: 'use-chainserver', type: Boolean, defaultValue: false },
+  {
+    name: 'chains-config',
+    alias: 'c',
+    type: String,
+    defaultValue: 'chains-config.json',
+  },
+]
+
+const options = commandLineArgs(optionDefinitions, { partial: true })
+
+console.log('loading', options.target)
+
+const fullPath = path.resolve(options['chains-config'])
+const chainsConfig = fs.readJsonSync(fullPath)
+
+setProvider(chainsConfig, options.concurrency, options['use-chainserver'])
+
+tryRequire(options.target)
+
+console.log('Start Server', options)
+
+console.log(global.PROCESSOR_STATE.processors.length, ' processors loaded')
+console.log(global.PROCESSOR_STATE.solanaProcessors.length, ' solana processors loaded')
+
+const server = createServer()
+
+const service = new ProcessorServiceImpl(server.shutdown)
+server.add(ProcessorDefinition, service)
+
+server.listen('0.0.0.0:' + options.port)

package/src/processor-state.ts

@@ -0,0 +1,24 @@
+import { BaseProcessor } from './base-processor'
+import { BaseContract } from 'ethers'
+import { BoundContractView, ContractView } from './context'
+import { BaseProcessorTemplate } from './base-processor-template'
+import { TemplateInstance } from './gen/processor/protos/processor'
+import { Provider } from '@ethersproject/providers'
+import { SolanaBaseProcessor } from './solana-processor'
+
+export class ProcessorState {
+  // from abiName_address_chainId => contract wrapper
+  contracts = new Map<string, ContractView<BaseContract>>()
+  // all evm processors
+  processors: BaseProcessor<BaseContract, BoundContractView<BaseContract, any>>[] = []
+  // from abiName_options to contracts
+  processorMap = new Map<string, BaseProcessor<any, any>>()
+  // evm providers
+  providers = new Map<number, Provider>()
+  // evm processor templates
+  templates: BaseProcessorTemplate<BaseContract, BoundContractView<BaseContract, any>>[] = []
+  // evm processor template instances spec
+  templatesInstances: TemplateInstance[] = []
+
+  solanaProcessors: SolanaBaseProcessor[] = []
+}

package/src/promise-or-void.ts

@@ -0,0 +1 @@
+export type PromiseOrVoid = void | Promise<any>

package/src/provider.ts

@@ -0,0 +1,74 @@
+import { getNetwork, Provider, StaticJsonRpcProvider } from '@ethersproject/providers'
+import { Networkish } from '@ethersproject/networks'
+import PQueue from 'p-queue'
+import { ConnectionInfo } from '@ethersproject/web'
+import { ChainConfig } from './chain-config'
+
+export const DummyProvider = new StaticJsonRpcProvider(undefined, 1)
+
+export function getProvider(networkish?: Networkish): Provider {
+  if (!networkish) {
+    networkish = 1
+  }
+  const network = getNetwork(networkish)
+
+  if (!global.PROCESSOR_STATE.providers) {
+    throw Error('Provider not set')
+  }
+  const value = global.PROCESSOR_STATE.providers.get(network.chainId)
+  if (value === undefined) {
+    throw Error(
+      'Provider not found for chain ' +
+        network.chainId +
+        ', configured chains: ' +
+        [...global.PROCESSOR_STATE.providers.keys()].join(' ')
+    )
+  }
+  return value
+}
+
+export function setProvider(config: Record<string, ChainConfig>, concurrency = 4, useChainServer = false) {
+  global.PROCESSOR_STATE.providers = new Map<number, Provider>()
+
+  for (const chainIdStr in config) {
+    if (isNaN(Number.parseInt(chainIdStr))) {
+      continue
+    }
+
+    const chainConfig = config[chainIdStr]
+    const chainId = Number(chainIdStr)
+
+    // let providers: StaticJsonRpcProvider[] = []
+    // for (const http of chainConfig.Https) {
+    //   providers.push(new StaticJsonRpcProvider(http, chainId))
+    // }
+    // random shuffle
+    // providers = providers.sort(() => Math.random() - 0.5)
+
+    // const provider = new FallbackProvider(providers)
+
+    let rpcAddress = ''
+    if (useChainServer && chainConfig.ChainServer) {
+      rpcAddress = chainConfig.ChainServer
+    } else {
+      const idx = Math.floor(Math.random() * chainConfig.Https.length)
+      rpcAddress = chainConfig.Https[idx]
+    }
+
+    const provider = new QueuedStaticJsonRpcProvider(rpcAddress, chainId, concurrency)
+    global.PROCESSOR_STATE.providers.set(chainId, provider)
+  }
+}
+
+class QueuedStaticJsonRpcProvider extends StaticJsonRpcProvider {
+  executor: PQueue
+
+  constructor(url: ConnectionInfo | string, network: Networkish, concurrency: number) {
+    super(url, network)
+    this.executor = new PQueue({ concurrency: concurrency })
+  }
+
+  send(method: string, params: Array<any>): Promise<any> {
+    return this.executor.add(() => super.send(method, params))
+  }
+}