@sentio/sdk 1.0.0-development → 1.0.0-rc.3

package/src/service.ts

@@ -0,0 +1,255 @@
+import { CallContext, ServerError, Status } from 'nice-grpc'
+
+import {
+  DataBinding,
+  HandlerType,
+  ProcessBindingResponse,
+  ProcessBindingsRequest,
+  ProcessConfigRequest,
+  ProcessConfigResponse,
+  ProcessorServiceImplementation,
+  ProcessResult,
+  StartRequest,
+} from './gen'
+
+import { Empty } from './gen/google/protobuf/empty'
+import Long from 'long'
+import { MetricState } from './core/meter'
+import { ExporterState } from './core/exporter'
+import { EventTrackerState } from './core/event-tracker'
+import { ProcessorTemplateProcessorState, TemplateInstanceState } from './core/base-processor-template'
+
+// (Long.prototype as any).toBigInt = function() {
+//   return BigInt(this.toString())
+// };
+import { PluginManager } from './plugin'
+;(BigInt.prototype as any).toJSON = function () {
+  return this.toString()
+}
+
+export const DEFAULT_MAX_BLOCK = Long.ZERO
+
+export const USER_PROCESSOR = 'user_processor'
+
+export class ProcessorServiceImpl implements ProcessorServiceImplementation {
+  private started = false
+  private processorConfig: ProcessConfigResponse
+
+  private readonly loader: () => void
+
+  private readonly shutdownHandler?: () => void
+
+  constructor(loader: () => void, shutdownHandler?: () => void) {
+    this.loader = loader
+    this.shutdownHandler = shutdownHandler
+  }
+
+  async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+    if (!this.processorConfig) {
+      throw new ServerError(Status.INTERNAL, 'Process config empty.')
+    }
+    return this.processorConfig
+  }
+
+  async configure() {
+    this.processorConfig = ProcessConfigResponse.fromPartial({})
+    // This syntax is to copy values instead of using references
+    this.processorConfig.templateInstances = [...TemplateInstanceState.INSTANCE.getValues()]
+
+    // part 0, prepare metrics and event tracking configs
+    for (const metric of MetricState.INSTANCE.getValues()) {
+      this.processorConfig.metricConfigs.push({
+        ...metric.config,
+      })
+    }
+
+    for (const eventTracker of EventTrackerState.INSTANCE.getValues()) {
+      this.processorConfig.eventTrackingConfigs.push({
+        distinctAggregationByDays: eventTracker.options.distinctByDays || [],
+        eventName: eventTracker.name,
+        retentionConfig: undefined,
+        totalByDay: eventTracker.options.totalByDay || false,
+        totalPerEntity: undefined,
+        unique: eventTracker.options.unique || false,
+      })
+    }
+
+    for (const exporter of ExporterState.INSTANCE.getValues()) {
+      this.processorConfig.exportConfigs.push({
+        name: exporter.name,
+        channel: exporter.channel,
+      })
+    }
+
+    PluginManager.INSTANCE.configure(this.processorConfig)
+  }
+
+  async start(request: StartRequest, context: CallContext): Promise<Empty> {
+    if (this.started) {
+      return {}
+    }
+
+    try {
+      try {
+        require('./core/eth-plugin')
+        require('./core/sui-plugin')
+        require('./aptos/aptos-plugin')
+        require('./core/solana-plugin')
+      } catch (e) {
+        require('@sentio/sdk/lib/core/eth-plugin')
+        require('@sentio/sdk/lib/core/sui-plugin')
+        require('@sentio/sdk/lib/aptos/aptos-plugin')
+        require('@sentio/sdk/lib/core/solana-plugin')
+      }
+
+      this.loader()
+    } catch (e) {
+      throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
+    }
+
+    for (const instance of request.templateInstances) {
+      const template = ProcessorTemplateProcessorState.INSTANCE.getValues()[instance.templateId]
+      if (!template) {
+        throw new ServerError(Status.INVALID_ARGUMENT, 'Invalid template contract:' + instance)
+      }
+      if (!instance.contract) {
+        throw new ServerError(Status.INVALID_ARGUMENT, 'Contract Empty from:' + instance)
+      }
+      template.bind({
+        name: instance.contract.name,
+        address: instance.contract.address,
+        network: Number(instance.contract.chainId),
+        startBlock: instance.startBlock,
+        endBlock: instance.endBlock,
+      })
+    }
+    try {
+      await this.configure()
+    } catch (e) {
+      throw new ServerError(Status.INTERNAL, 'Failed to start processor : ' + errorString(e))
+    }
+    this.started = true
+    return {}
+  }
+
+  async stop(request: Empty, context: CallContext): Promise<Empty> {
+    console.log('Server Shutting down in 5 seconds')
+    if (this.shutdownHandler) {
+      setTimeout(this.shutdownHandler, 5000)
+    }
+    return {}
+  }
+
+  async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+
+    const promises = request.bindings.map((binding) => this.processBinding(binding))
+    const result = mergeProcessResults(await Promise.all(promises))
+
+    let updated = false
+    const t = TemplateInstanceState.INSTANCE.getValues()
+    if (TemplateInstanceState.INSTANCE.getValues().length !== this.processorConfig.templateInstances.length) {
+      await this.configure()
+      updated = true
+    }
+
+    return {
+      result,
+      configUpdated: updated,
+    }
+  }
+
+  async processBinding(request: DataBinding, options?: CallContext): Promise<ProcessResult> {
+    const result = await PluginManager.INSTANCE.processBinding(request)
+    recordRuntimeInfo(result, request.handlerType)
+    return result
+  }
+
+  async *processBindingsStream(requests: AsyncIterable<DataBinding>, context: CallContext) {
+    for await (const request of requests) {
+      const result = await this.processBinding(request)
+      let updated = false
+      if (TemplateInstanceState.INSTANCE.getValues().length !== this.processorConfig.templateInstances.length) {
+        await this.configure()
+        updated = true
+      }
+      yield {
+        result,
+        configUpdated: updated,
+      }
+    }
+  }
+}
+
+// https://ourcodeworld.com/articles/read/164/how-to-convert-an-uint8array-to-string-in-javascript
+/* eslint-disable */
+export function Utf8ArrayToStr(array: Uint8Array) {
+  let out, i, len, c
+  let char2, char3
+
+  out = ''
+  len = array.length
+  i = 0
+  while (i < len) {
+    c = array[i++]
+    switch (c >> 4) {
+      case 0:
+      case 1:
+      case 2:
+      case 3:
+      case 4:
+      case 5:
+      case 6:
+      case 7:
+        // 0xxxxxxx
+        out += String.fromCharCode(c)
+        break
+      case 12:
+      case 13:
+        // 110x xxxx   10xx xxxx
+        char2 = array[i++]
+        out += String.fromCharCode(((c & 0x1f) << 6) | (char2 & 0x3f))
+        break
+      case 14:
+        // 1110 xxxx  10xx xxxx  10xx xxxx
+        char2 = array[i++]
+        char3 = array[i++]
+        out += String.fromCharCode(((c & 0x0f) << 12) | ((char2 & 0x3f) << 6) | ((char3 & 0x3f) << 0))
+        break
+    }
+  }
+
+  return out
+}
+
+export function mergeProcessResults(results: ProcessResult[]): ProcessResult {
+  const res = ProcessResult.fromPartial({})
+
+  for (const r of results) {
+    res.counters = res.counters.concat(r.counters)
+    res.gauges = res.gauges.concat(r.gauges)
+    res.logs = res.logs.concat(r.logs)
+    res.events = res.events.concat(r.events)
+    res.exports = res.exports.concat(r.exports)
+  }
+  return res
+}
+
+function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
+  for (const list of [results.gauges, results.counters, results.logs, results.events, results.exports]) {
+    list.forEach((e) => {
+      e.runtimeInfo = {
+        from: handlerType,
+      }
+    })
+  }
+}
+
+export function errorString(e: Error): string {
+  return e.stack || e.message
+}

package/src/solana-codegen/codegen.ts

@@ -0,0 +1,116 @@
+import path from 'path'
+import fs from 'fs'
+import chalk from 'chalk'
+
+export function codeGenSolanaProcessor(abisDir: string, targetPath = path.join('src', 'types', 'solana')) {
+  if (!fs.existsSync(abisDir)) {
+    return
+  }
+
+  const abisFiles = fs.readdirSync(abisDir)
+
+  if (abisFiles.length > 0) {
+    console.log(chalk.green('Generated Types for Solana'))
+  }
+
+  for (const file of abisFiles) {
+    if (path.extname(file) === '.json') {
+      if (!fs.existsSync(targetPath)) {
+        fs.mkdirSync(targetPath, { recursive: true })
+      }
+      const idlContent = fs.readFileSync(path.join(abisDir, file), 'utf-8')
+      const idlObj = JSON.parse(idlContent)
+      const idlName = idlObj.name
+      const idlFile = path.join(targetPath, idlName + '.ts')
+      fs.writeFileSync(idlFile, `export const ${idlName}_idl = ${idlContent}`)
+      fs.writeFileSync(path.join(targetPath, `${idlName}_processor.ts`), codeGenSolanaIdlProcessor(idlObj))
+    }
+  }
+}
+
+function codeGenSolanaIdlProcessor(idlObj: any): string {
+  const idlName = idlObj.name
+  const idlNamePascalCase = toPascalCase(idlName)
+  const instructions: any[] = idlObj.instructions
+  return `import { BorshInstructionCoder, Instruction, Idl, BN } from '@project-serum/anchor'
+import { SolanaBaseProcessor, SolanaContext, SolanaBindOptions } from "@sentio/sdk"
+import { ${idlName}_idl } from "./${idlName}"
+import bs58 from 'bs58'
+import { PublicKey } from '@solana/web3.js'
+
+export class ${idlNamePascalCase}Processor extends SolanaBaseProcessor {
+  static bind(options: SolanaBindOptions): ${idlNamePascalCase}Processor {
+    if (options && !options.name) {
+      options.name = '${idlNamePascalCase}'
+    }
+    return new ${idlNamePascalCase}Processor(options)
+  }
+
+  decodeInstruction: (rawInstruction: string) => Instruction | null = (rawInstruction) => {
+    const instructionCoder = new BorshInstructionCoder(${idlName}_idl as Idl)
+    const decodedIns = instructionCoder.decode(Buffer.from(bs58.decode(rawInstruction)))
+    return decodedIns
+  }
+
+  ${instructions.map((ins) => codeGenSolanaInstruction(idlNamePascalCase, ins)).join('')}
+}
+  `
+}
+
+function codeGenSolanaInstruction(idlName: string, ins: any): string {
+  const instructionName = ins.name
+  const argsTypeString = codeGenInstructionArgs(ins.args)
+  return `
+  on${
+    instructionName.charAt(0).toUpperCase() + instructionName.slice(1)
+  }(handler: (args: ${argsTypeString}, accounts: string[], ctx: SolanaContext) => void): ${idlName}Processor {
+    this.onInstruction('${instructionName}', (ins: Instruction, ctx, accounts: string[]) => {
+      if (ins) {
+        handler(ins.data as ${argsTypeString}, accounts, ctx)
+      }
+    })
+    return this
+  }
+  `
+}
+
+function codeGenInstructionArgs(args: { name: string; type: string }[]): string {
+  return `{ ${args.map((arg) => arg.name + ': ' + mapType(arg.type)).join(', ')} }`
+}
+
+// Reference: https://github.com/coral-xyz/anchor/blob/93332766f13e86efbe77c9986722731742317ede/ts/src/program/namespace/types.ts#L104
+function mapType(tpe: string): string {
+  switch (tpe) {
+    case 'publicKey':
+      return 'PublicKey'
+    case 'bool':
+      return 'boolean'
+    case 'string':
+      return 'string'
+    case 'u8':
+    case 'i8':
+    case 'u16':
+    case 'i16':
+    case 'u32':
+    case 'i32':
+    case 'f32':
+    case 'f64':
+      return 'number'
+    case 'u64':
+    case 'i64':
+    case 'u128':
+    case 'i128':
+      return 'BN'
+    default:
+      return 'any'
+  }
+}
+
+function toPascalCase(str: string) {
+  return `${str}`
+    .toLowerCase()
+    .replace(new RegExp(/[-_]+/, 'g'), ' ')
+    .replace(new RegExp(/[^\w\s]/, 'g'), '')
+    .replace(new RegExp(/\s+(.)(\w*)/, 'g'), ($1, $2, $3) => `${$2.toUpperCase() + $3}`)
+    .replace(new RegExp(/\w/), (s) => s.toUpperCase())
+}

package/src/state/processor-state.ts

@@ -0,0 +1,3 @@
+export class ProcessorState {
+  stateMap = new Map<string, any>()
+}

package/src/state/state-storage.ts

@@ -0,0 +1,65 @@
+export abstract class StateStorage<T> {
+  // TODO learn how to define single instance for all subclasses
+
+  protected constructor() {
+    //
+  }
+
+  abstract initValue(): T
+
+  key(): string {
+    return this.constructor.name
+  }
+
+  getOrRegister(): T {
+    let metricState: T = global.PROCESSOR_STATE.stateMap.get(this.key())
+    if (!metricState) {
+      metricState = this.initValue()
+      global.PROCESSOR_STATE.stateMap.set(this.key(), metricState)
+    }
+    return metricState
+  }
+}
+
+export abstract class MapStateStorage<T> extends StateStorage<Map<string, T>> {
+  initValue() {
+    return new Map<string, T>()
+  }
+
+  getValue(key: string): T | undefined {
+    const m = this.getOrRegister()
+    return m.get(key)
+  }
+
+  getValues(): T[] {
+    const m = this.getOrRegister()
+    return Array.from(m.values())
+  }
+
+  getOrSetValue(key: string, value: T): T {
+    const m = this.getOrRegister()
+    const oldValue = m.get(key)
+    if (oldValue) {
+      console.warn(key, 'has been registered twice, use the previous one')
+      return oldValue
+    }
+    m.set(key, value)
+    return value
+  }
+}
+
+export abstract class ListStateStorage<T> extends StateStorage<T[]> {
+  initValue() {
+    return []
+  }
+
+  getValues(): T[] {
+    return this.getOrRegister()
+  }
+
+  addValue(value: T): T {
+    const m = this.getOrRegister()
+    m.push(value)
+    return value
+  }
+}

package/src/target-ethers-sentio/event-handler.ts

@@ -0,0 +1,31 @@
+import { EventDeclaration, getFullSignatureAsSymbolForEvent } from 'typechain'
+import { getFullSignatureForEventPatched } from './types'
+
+export function generateEventHandler(event: EventDeclaration, contractName: string, includeArgTypes: boolean): string {
+  const eventName = includeArgTypes ? getFullSignatureAsSymbolForEvent(event) : event.name
+  const eventNamePrefix = includeArgTypes ? eventName + '_' : eventName
+
+  const filterName = getFullSignatureForEventPatched(event)
+  return `
+  onEvent${eventName}(
+    handler: (event: ${eventNamePrefix}Event, ctx: ${contractName}Context) => void,
+    filter?: ${eventNamePrefix}EventFilter | ${eventNamePrefix}EventFilter[]
+  ) {
+    if (!filter) {
+      // @ts-ignore
+      filter = ${contractName}Processor.filters[
+        // @ts-ignore
+        '${filterName}'](${event.inputs.map(() => 'null').join(',')})
+    }
+    return super.onEvent(handler, filter!)
+  }
+  `
+}
+
+export function generateEventHandlers(events: EventDeclaration[], contractName: string): string {
+  if (events.length === 1) {
+    return generateEventHandler(events[0], contractName, false)
+  } else {
+    return events.map((e) => generateEventHandler(e, contractName, true)).join('\n')
+  }
+}