@sentio/sdk 3.0.0-rc.1 → 3.0.0-rc.10

package/src/fuel/fuel-processor-template.ts

@@ -1,8 +1,7 @@
 import { FuelContext, FuelContractContext } from './context.js'
 import { HandleInterval, TemplateInstance } from '@sentio/protos'
 import { PromiseOrVoid } from '../core/promises.js'
-import { ListStateStorage } from '@sentio/runtime'
-import { TemplateInstanceState } from '../core/template.js'
+import { ListStateStorage, processMetrics } from '@sentio/runtime'
 import { Contract } from 'fuels'
 import { FuelBlock, FuelLog, FuelTransaction } from './types.js'
 import { DEFAULT_FUEL_FETCH_CONFIG, FuelFetchConfig } from './transaction.js'
@@ -16,7 +15,7 @@ export class FuelProcessorTemplateProcessorState extends ListStateStorage<FuelBa
 
 export abstract class FuelBaseProcessorTemplate<TContract extends Contract> {
   id: number
-  binds = new Set<string>()
+  instances = new Set<string>()
   blockHandlers: {
     handlerName: string
     handler: (block: FuelBlock, ctx: FuelContractContext<TContract>) => PromiseOrVoid
@@ -52,28 +51,6 @@ export abstract class FuelBaseProcessorTemplate<TContract extends Contract> {
    * @param ctx
    */
   public bind(options: Omit<Omit<FuelProcessorConfig, 'chainId'>, 'abi'>, ctx: FuelContext): void {
-    const sig = getOptionsSignature({
-      address: options.address,
-      chainId: ctx.chainId
-    })
-    if (this.binds.has(sig)) {
-      console.log(`Same address can be bind to one template only once, ignore duplicate bind: ${sig}`)
-      return
-    }
-    this.binds.add(sig)
-
-    const processor = this.bindInternal({ ...options, chainId: ctx.chainId })
-
-    for (const eh of this.logHandlers) {
-      processor.onLog(eh.logIdFilter, eh.handler, eh.handlerOptions, eh.handlerName)
-    }
-    for (const bh of this.blockHandlers) {
-      processor.onInterval(bh.handler, bh.timeIntervalInMinutes, bh.blockInterval, bh.handlerOptions, bh.handlerName)
-    }
-    for (const th of this.transactionHandlers) {
-      processor.onTransaction(th.handler, th.handlerOptions, th.handlerName)
-    }
-
     const instance: TemplateInstance = {
       templateId: this.id,
       contract: {
@@ -87,12 +64,43 @@ export abstract class FuelBaseProcessorTemplate<TContract extends Contract> {
       baseLabels: {}
       // baseLabels: options.baseLabels
     }
-    TemplateInstanceState.INSTANCE.addValue(instance)
+
+    ctx.sendTemplateInstance(instance)
+
     ctx.update({
       states: {
         configUpdated: true
       }
     })
+
+    processMetrics.processor_template_instance_count.add(1, {
+      chain_id: ctx.chainId,
+      template: this.constructor.name
+    })
+  }
+
+  public startInstance(options: Omit<Omit<FuelProcessorConfig, 'chainId'>, 'abi'>, ctx: FuelContext): void {
+    const sig = getOptionsSignature({
+      address: options.address,
+      chainId: ctx.chainId
+    })
+    if (this.instances.has(sig)) {
+      console.debug(`Same address can be bind to one template only once, ignore duplicate bind: ${sig}`)
+      return
+    }
+    this.instances.add(sig)
+
+    const processor = this.bindInternal({ ...options, chainId: ctx.chainId })
+
+    for (const eh of this.logHandlers) {
+      processor.onLog(eh.logIdFilter, eh.handler, eh.handlerOptions, eh.handlerName)
+    }
+    for (const bh of this.blockHandlers) {
+      processor.onInterval(bh.handler, bh.timeIntervalInMinutes, bh.blockInterval, bh.handlerOptions, bh.handlerName)
+    }
+    for (const th of this.transactionHandlers) {
+      processor.onTransaction(th.handler, th.handlerOptions, th.handlerName)
+    }
   }
 
   protected onLog<T>(

package/src/iota/iota-object-processor-template.ts

@@ -1,5 +1,5 @@
-import { HandleInterval, MoveAccountFetchConfig, MoveFetchConfig } from '@sentio/protos'
-import { ListStateStorage } from '@sentio/runtime'
+import { HandleInterval, MoveAccountFetchConfig, MoveFetchConfig, TemplateInstance } from '@sentio/protos'
+import { ListStateStorage, processMetrics } from '@sentio/runtime'
 import { IotaAddressContext, IotaContext, IotaObjectContext } from './context.js'
 import { IotaMoveObject, IotaTransactionBlockResponse } from '@iota/iota-sdk/client'
 import { PromiseOrVoid } from '../core/index.js'
@@ -12,7 +12,6 @@ import {
   IotaObjectProcessor,
   IotaWrappedObjectProcessor
 } from './iota-object-processor.js'
-import { TemplateInstanceState } from '../core/template.js'
 import { IotaBindOptions } from './iota-processor.js'
 import { TransactionFilter, accountAddressString } from '../move/index.js'
 import { ServerError, Status } from 'nice-grpc'
@@ -40,7 +39,7 @@ export abstract class IotaObjectOrAddressProcessorTemplate<
 > {
   id: number
   objectHandlers: ObjectHandler<HandlerType>[] = []
-  binds = new Set<string>()
+  instances = new Set<string>()
 
   constructor() {
     this.id = IotaAccountProcessorTemplateState.INSTANCE.getValues().length
@@ -60,12 +59,49 @@ export abstract class IotaObjectOrAddressProcessorTemplate<
     }
     id = accountAddressString(id)
 
+    const instance: TemplateInstance = {
+      templateId: this.id,
+      contract: {
+        name: '',
+        chainId: options.network,
+        address: id,
+        abi: ''
+      },
+      startBlock: options.startCheckpoint || 0n,
+      endBlock: options.endCheckpoint || 0n,
+      baseLabels: options.baseLabels
+    }
+
+    ctx.sendTemplateInstance(instance)
+
+    ctx.update({
+      states: {
+        configUpdated: true
+      }
+    })
+
+    processMetrics.processor_template_instance_count.add(1, {
+      chain_id: options.network,
+      template: this.constructor.name
+    })
+  }
+
+  startInstance(options: OptionType, ctx: IotaContext): void {
+    options.network = options.network || ctx.network
+    options.startCheckpoint = options.startCheckpoint || ctx.checkpoint
+    let id = (options as IotaObjectBindOptions).objectId || (options as IotaBindOptions).address
+
+    if (id === '*') {
+      throw new ServerError(Status.INVALID_ARGUMENT, "can't bind template instance with *")
+    }
+    id = accountAddressString(id)
+
     const sig = [options.network, id].join('_')
-    if (this.binds.has(sig)) {
-      console.log(`Same object id can be bind to one template only once, ignore duplicate bind: ${sig}`)
+    if (this.instances.has(sig)) {
+      console.debug(`Same object id can be bind to one template only once, ignore duplicate bind: ${sig}`)
       return
     }
-    this.binds.add(sig)
+    this.instances.add(sig)
 
     const processor = this.createProcessor(options)
     for (const h of this.objectHandlers) {
@@ -78,25 +114,6 @@ export abstract class IotaObjectOrAddressProcessorTemplate<
         h.handlerName
       )
     }
-    const config = processor.config
-
-    ctx.update({
-      states: {
-        configUpdated: true
-      }
-    })
-    TemplateInstanceState.INSTANCE.addValue({
-      templateId: this.id,
-      contract: {
-        name: '',
-        chainId: config.network,
-        address: config.address,
-        abi: ''
-      },
-      startBlock: config.startCheckpoint,
-      endBlock: config.endCheckpoint || 0n,
-      baseLabels: config.baseLabels
-    })
     console.log(`successfully bind template ${sig}`)
   }
 
@@ -111,31 +128,14 @@ export abstract class IotaObjectOrAddressProcessorTemplate<
     id = accountAddressString(id)
 
     const sig = [options.network, id].join('_')
-    if (!this.binds.has(sig)) {
+    if (!this.instances.has(sig)) {
       console.log(`the template instance ${sig} not existed or already unbind`)
       return
     }
-    this.binds.delete(sig)
-
-    let deleted = 0
-    const oldTemplateInstances = TemplateInstanceState.INSTANCE.unregister()
-    for (const templateInstance of oldTemplateInstances) {
-      if (templateInstance.contract?.chainId === options.network && templateInstance.contract.address == id) {
-        deleted++
-        continue
-      }
-      TemplateInstanceState.INSTANCE.addValue(templateInstance)
-    }
-
-    if (deleted !== 1) {
-      throw new ServerError(
-        Status.INVALID_ARGUMENT,
-        `Failed to delete template instance ${sig}, deleted ${deleted} times`
-      )
-    }
+    this.instances.delete(sig)
 
     const oldProcessors = IotaAccountProcessorState.INSTANCE.unregister()
-    deleted = 0
+    let deleted = 0
     for (const processor of oldProcessors) {
       if (processor.templateId === this.id) {
         if (processor.config.network == options.network && processor.config.address === id) {
@@ -160,6 +160,21 @@ export abstract class IotaObjectOrAddressProcessorTemplate<
         configUpdated: true
       }
     })
+    ctx.sendTemplateInstance(
+      {
+        templateId: this.id,
+        contract: {
+          name: '',
+          chainId: options.network,
+          address: id,
+          abi: ''
+        },
+        startBlock: options.startCheckpoint || 0n,
+        endBlock: options.endCheckpoint || 0n,
+        baseLabels: options.baseLabels
+      },
+      true
+    )
   }
 
   protected onInterval(

package/src/iota/iota-plugin-part.ts

@@ -45,7 +45,7 @@ export class IotaPluginPart {
       const template: IotaObjectOrAddressProcessorTemplate<any, any, any> =
         IotaAccountProcessorTemplateState.INSTANCE.getValues()[instance.templateId]
 
-      template.bind(
+      template.startInstance(
         {
           address: instance.contract?.address || '',
           objectId: instance.contract?.address || '',

package/src/stark/starknet-plugin.ts

@@ -10,7 +10,6 @@ import {
 } from '@sentio/protos'
 
 import { ServerError, Status } from 'nice-grpc'
-import { TemplateInstanceState } from '../core/template.js'
 import { HandlerRegister } from '../core/handler-register.js'
 import { StarknetProcessorState } from './starknet-processor.js'
 import { hash } from 'starknet'
@@ -79,10 +78,6 @@ export class StarknetPlugin extends Plugin {
 
   async start(request: StartRequest) {}
 
-  stateDiff(config: ProcessConfigResponse): boolean {
-    return TemplateInstanceState.INSTANCE.getValues().length !== config.templateInstances.length
-  }
-
   async processEvent(binding: DataBinding): Promise<ProcessResult> {
     if (!binding.data?.starknetEvents?.result) {
       throw new ServerError(Status.INVALID_ARGUMENT, "starknetEvents can't be null")

package/src/sui/sui-object-processor-template.ts

@@ -1,5 +1,5 @@
-import { HandleInterval, MoveAccountFetchConfig, MoveFetchConfig } from '@sentio/protos'
-import { ListStateStorage } from '@sentio/runtime'
+import { HandleInterval, MoveAccountFetchConfig, MoveFetchConfig, TemplateInstance } from '@sentio/protos'
+import { ListStateStorage, processMetrics } from '@sentio/runtime'
 import { SuiAddressContext, SuiContext, SuiObjectContext } from './context.js'
 import { SuiMoveObject, SuiTransactionBlockResponse } from '@mysten/sui/client'
 import { PromiseOrVoid } from '../core/index.js'
@@ -12,9 +12,8 @@ import {
   SuiObjectProcessor,
   SuiWrappedObjectProcessor
 } from './sui-object-processor.js'
-import { TemplateInstanceState } from '../core/template.js'
 import { SuiBindOptions } from './sui-processor.js'
-import { TransactionFilter, accountAddressString } from '../move/index.js'
+import { accountAddressString, TransactionFilter } from '../move/index.js'
 import { ServerError, Status } from 'nice-grpc'
 import { getHandlerName, proxyProcessor } from '../utils/metrics.js'
 
@@ -40,7 +39,7 @@ export abstract class SuiObjectOrAddressProcessorTemplate<
 > {
   id: number
   objectHandlers: ObjectHandler<HandlerType>[] = []
-  binds = new Set<string>()
+  instances = new Set<string>()
 
   constructor() {
     this.id = SuiAccountProcessorTemplateState.INSTANCE.getValues().length
@@ -60,12 +59,49 @@ export abstract class SuiObjectOrAddressProcessorTemplate<
     }
     id = accountAddressString(id)
 
+    const instance: TemplateInstance = {
+      templateId: this.id,
+      contract: {
+        name: '',
+        chainId: options.network,
+        address: id,
+        abi: ''
+      },
+      startBlock: options.startCheckpoint || 0n,
+      endBlock: options.endCheckpoint || 0n,
+      baseLabels: options.baseLabels
+    }
+
+    ctx.sendTemplateInstance(instance)
+
+    ctx.update({
+      states: {
+        configUpdated: true
+      }
+    })
+
+    processMetrics.processor_template_instance_count.add(1, {
+      chain_id: options.network,
+      template: this.constructor.name
+    })
+  }
+
+  startInstance(options: OptionType, ctx: SuiContext): void {
+    options.network = options.network || ctx.network
+    options.startCheckpoint = options.startCheckpoint || ctx.checkpoint
+    let id = (options as SuiObjectBindOptions).objectId || (options as SuiBindOptions).address
+
+    if (id === '*') {
+      throw new ServerError(Status.INVALID_ARGUMENT, "can't bind template instance with *")
+    }
+    id = accountAddressString(id)
+
     const sig = [options.network, id].join('_')
-    if (this.binds.has(sig)) {
-      console.log(`Same object id can be bind to one template only once, ignore duplicate bind: ${sig}`)
+    if (this.instances.has(sig)) {
+      console.debug(`Same object id can be bind to one template only once, ignore duplicate bind: ${sig}`)
       return
     }
-    this.binds.add(sig)
+    this.instances.add(sig)
 
     const processor = this.createProcessor(options)
     for (const h of this.objectHandlers) {
@@ -78,25 +114,6 @@ export abstract class SuiObjectOrAddressProcessorTemplate<
         h.handlerName
       )
     }
-    const config = processor.config
-
-    ctx.update({
-      states: {
-        configUpdated: true
-      }
-    })
-    TemplateInstanceState.INSTANCE.addValue({
-      templateId: this.id,
-      contract: {
-        name: '',
-        chainId: config.network,
-        address: config.address,
-        abi: ''
-      },
-      startBlock: config.startCheckpoint,
-      endBlock: config.endCheckpoint || 0n,
-      baseLabels: config.baseLabels
-    })
     console.log(`successfully bind template ${sig}`)
   }
 
@@ -111,31 +128,14 @@ export abstract class SuiObjectOrAddressProcessorTemplate<
     id = accountAddressString(id)
 
     const sig = [options.network, id].join('_')
-    if (!this.binds.has(sig)) {
+    if (!this.instances.has(sig)) {
       console.log(`the template instance ${sig} not existed or already unbind`)
       return
     }
-    this.binds.delete(sig)
-
-    let deleted = 0
-    const oldTemplateInstances = TemplateInstanceState.INSTANCE.unregister()
-    for (const templateInstance of oldTemplateInstances) {
-      if (templateInstance.contract?.chainId === options.network && templateInstance.contract.address == id) {
-        deleted++
-        continue
-      }
-      TemplateInstanceState.INSTANCE.addValue(templateInstance)
-    }
-
-    if (deleted !== 1) {
-      throw new ServerError(
-        Status.INVALID_ARGUMENT,
-        `Failed to delete template instance ${sig}, deleted ${deleted} times`
-      )
-    }
+    this.instances.delete(sig)
 
     const oldProcessors = SuiAccountProcessorState.INSTANCE.unregister()
-    deleted = 0
+    let deleted = 0
     for (const processor of oldProcessors) {
       if (processor.templateId === this.id) {
         if (processor.config.network == options.network && processor.config.address === id) {
@@ -160,6 +160,22 @@ export abstract class SuiObjectOrAddressProcessorTemplate<
         configUpdated: true
       }
     })
+
+    ctx.sendTemplateInstance(
+      {
+        templateId: this.id,
+        contract: {
+          name: '',
+          chainId: options.network,
+          address: id,
+          abi: ''
+        },
+        startBlock: options.startCheckpoint || 0n,
+        endBlock: options.endCheckpoint || 0n,
+        baseLabels: options.baseLabels
+      },
+      true
+    )
   }
 
   protected onInterval(

package/src/sui/sui-plugin-part.ts

@@ -45,7 +45,7 @@ export class SuiPluginPart {
       const template: SuiObjectOrAddressProcessorTemplate<any, any, any> =
         SuiAccountProcessorTemplateState.INSTANCE.getValues()[instance.templateId]
 
-      template.bind(
+      template.startInstance(
         {
           address: instance.contract?.address || '',
           objectId: instance.contract?.address || '',

package/src/testing/test-processor-server.ts

@@ -14,10 +14,19 @@ import {
   ProcessStreamRequest,
   ProcessStreamResponse,
   ServerStreamingMethodResult,
-  StartRequest
+  StartRequest,
+  TemplateInstance,
+  TimeseriesResult
 } from '@sentio/protos'
 import { CallContext } from 'nice-grpc-common'
-import { Endpoints, PluginManager, ProcessorServiceImpl, State, StoreContext } from '@sentio/runtime'
+import {
+  Endpoints,
+  IDataBindingContext,
+  PluginManager,
+  ProcessorServiceImpl,
+  State,
+  StoreContext
+} from '@sentio/runtime'
 
 import { AptosFacet } from './aptos-facet.js'
 import { SolanaFacet } from './solana-facet.js'
@@ -45,7 +54,7 @@ export class TestProcessorServer implements ProcessorServiceImplementation {
   service: ProcessorServiceImpl
   contractConfigs: ContractConfig[]
   accountConfigs: AccountConfig[]
-  storeContext: StoreContext
+  storeContext: TestStoreContext
 
   aptos: AptosFacet
   eth: EthFacet
@@ -77,7 +86,7 @@ export class TestProcessorServer implements ProcessorServiceImplementation {
 
     // start a memory database for testing
     const subject = new Subject<DeepPartial<ProcessStreamResponse>>()
-    this.storeContext = new StoreContext(subject, 1)
+    this.storeContext = new TestStoreContext(subject, 1)
     this._db = new MemoryDatabase(this.storeContext)
   }
 
@@ -87,6 +96,7 @@ export class TestProcessorServer implements ProcessorServiceImplementation {
     this.contractConfigs = config.contractConfigs
     this.accountConfigs = config.accountConfigs
     this._db.start()
+    this.storeContext.templateInstances = request.templateInstances
     return res
   }
 
@@ -94,23 +104,43 @@ export class TestProcessorServer implements ProcessorServiceImplementation {
     return this.service.stop(request, context)
   }
 
-  getConfig(request: ProcessConfigRequest, context = TEST_CONTEXT): Promise<ProcessConfigResponse> {
-    return this.service.getConfig(request, context)
+  async getConfig(request: ProcessConfigRequest, context = TEST_CONTEXT): Promise<ProcessConfigResponse> {
+    const config = await this.service.getConfig(request, context)
+    return {
+      ...config,
+      templateInstances: this.storeContext.templateInstances
+    }
   }
 
   processBindings(
     request: ProcessBindingsRequest,
     context: CallContext = TEST_CONTEXT
   ): Promise<ProcessBindingResponse> {
-    return PluginManager.INSTANCE.dbContextLocalStorage.run(this.storeContext, () => {
-      return this.service.processBindings(request, context)
+    return PluginManager.INSTANCE.dbContextLocalStorage.run(this.storeContext, async () => {
+      const ret = await this.service.processBindings(request, context)
+      if (ret.result?.states?.configUpdated) {
+        // template may changed
+        await PluginManager.INSTANCE.updateTemplates({
+          chainId: request.bindings[0].chainId,
+          templateInstances: this.storeContext.templateInstances
+        })
+      }
+      return ret
     })
   }
 
-  processBinding(request: DataBinding, context: CallContext = TEST_CONTEXT): Promise<ProcessBindingResponse> {
-    return PluginManager.INSTANCE.dbContextLocalStorage.run(this.storeContext, () => {
+  async processBinding(request: DataBinding, context: CallContext = TEST_CONTEXT): Promise<ProcessBindingResponse> {
+    const ret = await PluginManager.INSTANCE.dbContextLocalStorage.run(this.storeContext, () => {
       return this.service.processBindings({ bindings: [request] }, context)
     })
+    if (ret.result?.states?.configUpdated) {
+      // template may changed
+      await PluginManager.INSTANCE.updateTemplates({
+        chainId: request.chainId,
+        templateInstances: this.storeContext.templateInstances
+      })
+    }
+    return ret
   }
 
   processBindingsStream(
@@ -138,3 +168,27 @@ export class TestProcessorServer implements ProcessorServiceImplementation {
     return this._db.store
   }
 }
+
+class TestStoreContext extends StoreContext implements IDataBindingContext {
+  constructor(
+    readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
+    processId: number
+  ) {
+    super(subject, processId)
+  }
+
+  templateInstances: TemplateInstance[] = []
+
+  sendTemplateRequest(templates: Array<TemplateInstance>, remove: boolean): void {
+    if (remove) {
+      this.templateInstances = this.templateInstances.filter(
+        (i) => !templates.find((t) => t.templateId === i.templateId && t.contract?.address === i.contract?.address)
+      )
+    } else {
+      this.templateInstances.push(...templates)
+    }
+  }
+  sendTimeseriesRequest(timeseries: Array<TimeseriesResult>): void {
+    throw new Error('Method not implemented.')
+  }
+}

package/lib/core/template.d.ts

@@ -1,2 +0,0 @@
-export { TemplateInstanceState } from '@sentio/runtime';
-//# sourceMappingURL=template.d.ts.map

package/lib/core/template.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"template.d.ts","sourceRoot":"","sources":["../../src/core/template.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAA"}

package/lib/core/template.js

@@ -1,2 +0,0 @@
-export { TemplateInstanceState } from '@sentio/runtime';
-//# sourceMappingURL=template.js.map

package/lib/core/template.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"template.js","sourceRoot":"","sources":["../../src/core/template.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAA"}

package/src/core/template.ts

@@ -1 +0,0 @@
-export { TemplateInstanceState } from '@sentio/runtime'