@sentio/runtime 2.59.4 → 2.60.0-rc.1

package/src/gen/service/common/protos/common.ts

@@ -313,6 +313,17 @@ export interface UsageTracker {
   versionField: string;
 }
 
+export interface Auth {
+  permission: string[];
+  metadata: { [key: string]: string };
+  allowAnonymous: boolean;
+}
+
+export interface Auth_MetadataEntry {
+  key: string;
+  value: string;
+}
+
 export interface AccessMeta {
   projectIdField: string;
   projectSlugField: string;
@@ -2183,6 +2194,190 @@ export const UsageTracker = {
   },
 };
 
+function createBaseAuth(): Auth {
+  return { permission: [], metadata: {}, allowAnonymous: false };
+}
+
+export const Auth = {
+  encode(message: Auth, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    for (const v of message.permission) {
+      writer.uint32(10).string(v!);
+    }
+    Object.entries(message.metadata).forEach(([key, value]) => {
+      Auth_MetadataEntry.encode({ key: key as any, value }, writer.uint32(18).fork()).ldelim();
+    });
+    if (message.allowAnonymous !== false) {
+      writer.uint32(24).bool(message.allowAnonymous);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): Auth {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseAuth();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.permission.push(reader.string());
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          const entry2 = Auth_MetadataEntry.decode(reader, reader.uint32());
+          if (entry2.value !== undefined) {
+            message.metadata[entry2.key] = entry2.value;
+          }
+          continue;
+        case 3:
+          if (tag !== 24) {
+            break;
+          }
+
+          message.allowAnonymous = reader.bool();
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): Auth {
+    return {
+      permission: globalThis.Array.isArray(object?.permission)
+        ? object.permission.map((e: any) => globalThis.String(e))
+        : [],
+      metadata: isObject(object.metadata)
+        ? Object.entries(object.metadata).reduce<{ [key: string]: string }>((acc, [key, value]) => {
+          acc[key] = String(value);
+          return acc;
+        }, {})
+        : {},
+      allowAnonymous: isSet(object.allowAnonymous) ? globalThis.Boolean(object.allowAnonymous) : false,
+    };
+  },
+
+  toJSON(message: Auth): unknown {
+    const obj: any = {};
+    if (message.permission?.length) {
+      obj.permission = message.permission;
+    }
+    if (message.metadata) {
+      const entries = Object.entries(message.metadata);
+      if (entries.length > 0) {
+        obj.metadata = {};
+        entries.forEach(([k, v]) => {
+          obj.metadata[k] = v;
+        });
+      }
+    }
+    if (message.allowAnonymous !== false) {
+      obj.allowAnonymous = message.allowAnonymous;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<Auth>): Auth {
+    return Auth.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<Auth>): Auth {
+    const message = createBaseAuth();
+    message.permission = object.permission?.map((e) => e) || [];
+    message.metadata = Object.entries(object.metadata ?? {}).reduce<{ [key: string]: string }>((acc, [key, value]) => {
+      if (value !== undefined) {
+        acc[key] = globalThis.String(value);
+      }
+      return acc;
+    }, {});
+    message.allowAnonymous = object.allowAnonymous ?? false;
+    return message;
+  },
+};
+
+function createBaseAuth_MetadataEntry(): Auth_MetadataEntry {
+  return { key: "", value: "" };
+}
+
+export const Auth_MetadataEntry = {
+  encode(message: Auth_MetadataEntry, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.key !== "") {
+      writer.uint32(10).string(message.key);
+    }
+    if (message.value !== "") {
+      writer.uint32(18).string(message.value);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): Auth_MetadataEntry {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseAuth_MetadataEntry();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.key = reader.string();
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          message.value = reader.string();
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): Auth_MetadataEntry {
+    return {
+      key: isSet(object.key) ? globalThis.String(object.key) : "",
+      value: isSet(object.value) ? globalThis.String(object.value) : "",
+    };
+  },
+
+  toJSON(message: Auth_MetadataEntry): unknown {
+    const obj: any = {};
+    if (message.key !== "") {
+      obj.key = message.key;
+    }
+    if (message.value !== "") {
+      obj.value = message.value;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<Auth_MetadataEntry>): Auth_MetadataEntry {
+    return Auth_MetadataEntry.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<Auth_MetadataEntry>): Auth_MetadataEntry {
+    const message = createBaseAuth_MetadataEntry();
+    message.key = object.key ?? "";
+    message.value = object.value ?? "";
+    return message;
+  },
+};
+
 function createBaseAccessMeta(): AccessMeta {
   return {
     projectIdField: "",

package/src/plugin.ts

@@ -1,6 +1,7 @@
 import {
   DataBinding,
   HandlerType,
+  InitResponse,
   PreparedData,
   PreprocessResult,
   ProcessConfigResponse,
@@ -16,7 +17,7 @@ export abstract class Plugin {
   name: string
   supportedHandlers: HandlerType[] = []
 
-  async configure(config: ProcessConfigResponse): Promise<void> {}
+  async configure(config: ProcessConfigResponse, forChainId?: string): Promise<void> {}
 
   async start(start: StartRequest): Promise<void> {}
 
@@ -59,6 +60,12 @@ export abstract class Plugin {
    * method used by action server only
    */
   shutdownServer() {}
+
+  /**
+   * Initialize the plugin, for service v2.
+   * @param config
+   */
+  async init(config: InitResponse): Promise<void> {}
 }
 
 export class PluginManager {
@@ -83,8 +90,8 @@ export class PluginManager {
     }
   }
 
-  configure(config: ProcessConfigResponse) {
-    return Promise.all(this.plugins.map((plugin) => plugin.configure(config)))
+  async configure(config: ProcessConfigResponse, forChainId?: string): Promise<void> {
+    await Promise.all(this.plugins.map((plugin) => plugin.configure(config, forChainId)))
   }
 
   start(start: StartRequest, actionServerPort?: number) {
@@ -141,4 +148,8 @@ export class PluginManager {
       return plugin.preprocessBinding(request, preprocessStore)
     })
   }
+
+  async init(resp: InitResponse) {
+    return Promise.all(this.plugins.map((plugin) => plugin.init(resp)))
+  }
 }

package/src/service-v2.ts

@@ -0,0 +1,207 @@
+import {
+  ConfigureHandlersRequest,
+  DataBinding,
+  DeepPartial,
+  Empty,
+  HandlerType,
+  InitResponse,
+  ProcessConfigResponse,
+  ProcessorV2ServiceImplementation,
+  ProcessResult,
+  ProcessStreamRequest,
+  ProcessStreamResponseV2,
+  StartRequest
+} from '@sentio/protos'
+import { CallContext } from 'nice-grpc'
+import { AsyncIterable } from 'ix'
+import { PluginManager } from './plugin.js'
+import { Subject } from 'rxjs'
+import { from } from 'ix/asynciterable'
+import { withAbort } from 'ix/asynciterable/operators'
+import { errorString } from './utils.js'
+
+import { processMetrics } from './metrics.js'
+import { recordRuntimeInfo } from './service.js'
+import { DataBindingContext } from './db-context.js'
+import { TemplateInstanceState } from './state.js'
+
+const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
+
+export class ProcessorServiceImplV2 implements ProcessorV2ServiceImplementation {
+  readonly enablePartition: boolean
+  private readonly loader: () => Promise<any>
+  private readonly shutdownHandler?: () => void
+
+  constructor(loader: () => Promise<any>, options?: any, shutdownHandler?: () => void) {
+    this.loader = loader
+    this.shutdownHandler = shutdownHandler
+
+    this.enablePartition = options?.['enable-partition'] == true
+  }
+
+  async init(request: Empty, context: CallContext): Promise<DeepPartial<InitResponse>> {
+    const resp = InitResponse.fromPartial({
+      chainIds: []
+    })
+    await PluginManager.INSTANCE.init(resp)
+    resp.chainIds = Array.from(new Set(resp.chainIds))
+    return resp
+  }
+
+  async *processBindingsStream(requests: AsyncIterable<ProcessStreamRequest>, context: CallContext) {
+    const subject = new Subject<DeepPartial<ProcessStreamResponseV2>>()
+    let lastBinding: DataBinding | undefined = undefined
+    for await (const request of requests) {
+      try {
+        // console.log('received request:', request, 'lastBinding:', lastBinding)
+        if (request.binding) {
+          lastBinding = request.binding
+        }
+        this.handleRequest(request, lastBinding, subject)
+      } catch (e) {
+        // should not happen
+        console.error('unexpect error during handle loop', e)
+      }
+    }
+    yield* from(subject).pipe(withAbort(context.signal))
+  }
+  private contexts = new Contexts()
+
+  async handleRequest(
+    request: ProcessStreamRequest,
+    lastBinding: DataBinding | undefined,
+    subject: Subject<DeepPartial<ProcessStreamResponseV2>>
+  ) {
+    if (request.binding) {
+      process_binding_count.add(1)
+
+      if (request.binding.handlerType === HandlerType.UNKNOWN) {
+        subject.next({
+          processId: request.processId,
+          result: ProcessResult.create()
+        })
+        return
+      }
+
+      if (this.enablePartition) {
+        try {
+          const partitions = await PluginManager.INSTANCE.partition(request.binding)
+          subject.next({
+            processId: request.processId,
+            partitions
+          })
+        } catch (e) {
+          console.error('Partition error:', e)
+          subject.error(new Error('Partition error: ' + errorString(e)))
+          return
+        }
+      } else {
+        this.startProcess(request.processId, request.binding, subject)
+      }
+    }
+
+    if (request.start) {
+      if (!lastBinding) {
+        console.error('start request received without binding')
+        subject.error(new Error('start request received without binding'))
+        return
+      }
+      this.startProcess(request.processId, lastBinding, subject)
+    }
+
+    if (request.dbResult) {
+      const context = this.contexts.get(request.processId)
+      try {
+        context?.result(request.dbResult)
+      } catch (e) {
+        subject.error(new Error('db result error, process should stop'))
+      }
+    }
+  }
+
+  private startProcess(
+    processId: number,
+    binding: DataBinding,
+    subject: Subject<DeepPartial<ProcessStreamResponseV2>>
+  ) {
+    const context = this.contexts.new(processId, subject)
+    const start = Date.now()
+    PluginManager.INSTANCE.processBinding(binding, undefined, context)
+      .then(async (result) => {
+        // await all pending db requests
+        await context.awaitPendings()
+
+        for (const ts of result.timeseriesResult) {
+          subject.next({
+            processId,
+            tsRequest: {
+              data: [ts]
+            }
+          })
+        }
+
+        if (result.states?.configUpdated) {
+          subject.next({
+            processId,
+            tplRequest: {
+              templates: TemplateInstanceState.INSTANCE.getValues()
+            }
+          })
+        }
+
+        subject.next({
+          result: {
+            states: result.states,
+            exports: result.exports
+          },
+          processId: processId
+        })
+        recordRuntimeInfo(result, binding.handlerType)
+      })
+      .catch((e) => {
+        console.error(e, e.stack)
+        context.error(processId, e)
+        process_binding_error.add(1)
+      })
+      .finally(() => {
+        const cost = Date.now() - start
+        process_binding_time.add(cost)
+        this.contexts.delete(processId)
+      })
+  }
+
+  async configureHandlers(
+    request: ConfigureHandlersRequest,
+    context: CallContext
+  ): Promise<DeepPartial<ProcessConfigResponse>> {
+    await PluginManager.INSTANCE.start(
+      StartRequest.fromPartial({
+        templateInstances: request.templateInstances
+      })
+    )
+
+    const newConfig = ProcessConfigResponse.fromPartial({})
+    await PluginManager.INSTANCE.configure(newConfig, request.chainId)
+    return newConfig
+  }
+}
+
+class Contexts {
+  private contexts: Map<number, DataBindingContext> = new Map()
+
+  get(processId: number) {
+    return this.contexts.get(processId)
+  }
+
+  new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponseV2>>) {
+    const context = new DataBindingContext(processId, subject)
+    this.contexts.set(processId, context)
+    return context
+  }
+
+  delete(processId: number) {
+    const context = this.get(processId)
+    context?.close()
+    this.contexts.delete(processId)
+  }
+}