@sentio/runtime 2.62.0-rc.8 → 2.62.0

package/src/processor-runner-program.ts

@@ -0,0 +1,57 @@
+import { Command, InvalidArgumentError } from '@commander-js/extra-typings'
+
+let workerNum = 1
+try {
+  workerNum = parseInt(process.env['PROCESSOR_WORKER']?.trim() ?? '1')
+} catch (e) {
+  console.error('Failed to parse worker number', e)
+}
+
+function myParseInt(value: string, dummyPrevious: number): number {
+  // parseInt takes a string and a radix
+  const parsedValue = parseInt(value, 10)
+  if (isNaN(parsedValue)) {
+    throw new InvalidArgumentError('Not a number.')
+  }
+  return parsedValue
+}
+
+export const program = new Command('processor-runner')
+  .allowUnknownOption()
+  .allowExcessArguments()
+  .name('processor-runner')
+  .description('Sentio Processor Runtime')
+  .argument('<target>', 'Path to the processor module to load')
+  .option('-p, --port <port>', 'Port to listen on', '4000')
+  .option('--concurrency <number>', 'Number of concurrent workers', myParseInt, 4)
+  .option('--batch-count <number>', 'Batch count for processing', myParseInt, 1)
+  .option('-c, --chains-config <path>', 'Path to chains configuration file', 'chains-config.json')
+  .option('--chainquery-server <url>', 'Chain query server URL')
+  .option('--pricefeed-server <url>', 'Price feed server URL')
+  .option('--log-format <format>', 'Log format (console|json)', 'console')
+  .option('--debug', 'Enable debug mode')
+  .option('--otlp-debug', 'Enable OTLP debug mode')
+  .option('--start-action-server', 'Start action server instead of processor server')
+  .option('--worker <number>', 'Number of worker threads', myParseInt, workerNum)
+  .option('--process-timeout <seconds>', 'Process timeout in seconds', myParseInt, 60)
+  .option(
+    '--worker-timeout <seconds>',
+    'Worker timeout in seconds',
+    myParseInt,
+    parseInt(process.env['WORKER_TIMEOUT_SECONDS'] || '60')
+  )
+  .option(
+    '--enable-partition',
+    'Enable binding data partition',
+    process.env['SENTIO_ENABLE_BINDING_DATA_PARTITION'] === 'true'
+  )
+
+export type ProcessorRuntimeOptions = ReturnType<typeof program.opts> & { target: string }
+
+export function getTestConfig(config?: Partial<ProcessorRuntimeOptions>): ProcessorRuntimeOptions {
+  return {
+    ...program.opts(),
+    target: './test-processor.test.js',
+    ...config
+  }
+}

package/src/processor-runner.ts

@@ -2,8 +2,6 @@
 
 import fs from 'fs-extra'
 
-import { compressionAlgorithms } from '@grpc/grpc-js'
-import { Command, InvalidArgumentError } from 'commander'
 import { createServer } from 'nice-grpc'
 import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
 // import { registry as niceGrpcRegistry } from 'nice-grpc-prometheus'
@@ -21,76 +19,21 @@ import { setupLogger } from './logger.js'
 import { setupOTLP } from './otlp.js'
 import { ActionServer } from './action-server.js'
 import { ServiceManager } from './service-manager.js'
-import path from 'path'
 import { ProcessorV3Definition } from '@sentio/protos'
 import { ProcessorServiceImplV3 } from './service-v3.js'
-import { readFileSync } from 'fs'
-import { fileURLToPath } from 'url'
 import { dirname, join } from 'path'
+import { program, ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
-const __filename = fileURLToPath(import.meta.url)
-const __dirname = dirname(__filename)
-const packageJson = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf8'))
+program.parse()
 
-// const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
-
-let workerNum = 1
-try {
-  workerNum = parseInt(process.env['PROCESSOR_WORKER']?.trim() ?? '1')
-} catch (e) {
-  console.error('Failed to parse worker number', e)
+const options: ProcessorRuntimeOptions = {
+  ...program.opts(),
+  target: program.args[program.args.length - 1]
 }
 
-function myParseInt(value: string, dummyPrevious: unknown): number {
-  // parseInt takes a string and a radix
-  const parsedValue = parseInt(value, 10)
-  if (isNaN(parsedValue)) {
-    throw new InvalidArgumentError('Not a number.')
-  }
-  return parsedValue
-}
-
-// Create Commander.js program
-const program = new Command()
-
-program
-  .allowUnknownOption()
-  // .allowExcessArguments()
-  .name('processor-runner')
-  .description('Sentio Processor Runtime')
-  .version(packageJson.version)
-  .argument('<target>', 'Path to the processor module to load')
-  .option('-p, --port <port>', 'Port to listen on', '4000')
-  .option('--concurrency <number>', 'Number of concurrent workers', myParseInt, 4)
-  .option('--batch-count <number>', 'Batch count for processing', myParseInt, 1)
-  .option('-c, --chains-config <path>', 'Path to chains configuration file', 'chains-config.json')
-  .option('--chainquery-server <url>', 'Chain query server URL', '')
-  .option('--pricefeed-server <url>', 'Price feed server URL', '')
-  .option('--log-format <format>', 'Log format (console|json)', 'console')
-  .option('--debug', 'Enable debug mode', false)
-  .option('--otlp-debug', 'Enable OTLP debug mode', false)
-  .option('--start-action-server', 'Start action server instead of processor server', false)
-  .option('--worker <number>', 'Number of worker threads', myParseInt, workerNum)
-  .option('--process-timeout <seconds>', 'Process timeout in seconds', myParseInt, 60)
-  .option(
-    '--worker-timeout <seconds>',
-    'Worker timeout in seconds',
-    myParseInt,
-    parseInt(process.env['WORKER_TIMEOUT_SECONDS'] || '60')
-  )
-  .option(
-    '--enable-partition',
-    'Enable binding data partition',
-    process.env['SENTIO_ENABLE_BINDING_DATA_PARTITION'] === 'true'
-  )
-  .parse()
-
-const options = program.opts()
-options.target = program.processedArgs[0]
-
 const logLevel = process.env['LOG_LEVEL']?.toLowerCase()
 
-setupLogger(options.logFormat === 'json', logLevel === 'debug' ? true : options.debug)
+setupLogger(options.logFormat === 'json', logLevel === 'debug' ? true : options.debug!)
 console.debug('Starting with', options.target)
 
 await setupOTLP(options.otlpDebug)
@@ -114,8 +57,8 @@ if (options.startActionServer) {
 } else {
   server = createServer({
     'grpc.max_send_message_length': 768 * 1024 * 1024,
-    'grpc.max_receive_message_length': 768 * 1024 * 1024,
-    'grpc.default_compression_algorithm': compressionAlgorithms.gzip
+    'grpc.max_receive_message_length': 768 * 1024 * 1024
+    // 'grpc.default_compression_algorithm': compressionAlgorithms.gzip
   })
     // .use(prometheusServerMiddleware())
     .use(openTelemetryServerMiddleware())
@@ -230,7 +173,7 @@ if (process.env['OOM_DUMP_MEMORY_SIZE_GB']) {
     console.log('Current Memory Usage', mem)
     // if memory usage is greater this size, dump heap and exit
     if (mem.heapTotal > memorySize * 1024 * 1024 * 1024 && !dumping) {
-      const file = path.join(dir, `${Date.now()}.heapsnapshot`)
+      const file = join(dir, `${Date.now()}.heapsnapshot`)
       dumping = true
       await dumpHeap(file)
       // force exit and keep pod running
@@ -243,11 +186,11 @@ if (process.env['OOM_DUMP_MEMORY_SIZE_GB']) {
 async function dumpHeap(file: string): Promise<void> {
   console.log('Heap dumping to', file)
   const session = new Session()
-  fs.mkdirSync(path.dirname(file), { recursive: true })
+  fs.mkdirSync(dirname(file), { recursive: true })
   const fd = fs.openSync(file, 'w')
   try {
     session.connect()
-    session.on('HeapProfiler.addHeapSnapshotChunk', (m) => {
+    session.on('HeapProfiler.addHeapSnapshotChunk', (m: any) => {
       fs.writeSync(fd, m.params.chunk)
     })
 

package/src/service-manager.ts

@@ -16,6 +16,7 @@ import { Subject } from 'rxjs'
 import { MessageChannel } from 'node:worker_threads'
 import { ProcessorServiceImpl } from './service.js'
 import { TemplateInstanceState } from './state.js'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 ;(BigInt.prototype as any).toJSON = function () {
   return this.toString()
 }
@@ -26,7 +27,7 @@ export class ServiceManager extends ProcessorServiceImpl {
 
   constructor(
     loader: () => Promise<any>,
-    readonly options: any,
+    readonly options: ProcessorRuntimeOptions,
     shutdownHandler?: () => void
   ) {
     super(loader, options, shutdownHandler)
@@ -123,7 +124,7 @@ export class ServiceManager extends ProcessorServiceImpl {
 
     if (this.enablePartition) {
       const concurrent = parseInt(process.env['PROCESS_CONCURRENCY'] || '0')
-      if (this.options.worker < concurrent) {
+      if (this.options.worker! < concurrent) {
         console.warn(
           `When partition is enabled, the worker count must >= 'PROCESS_CONCURRENCY', will set worker count to ${concurrent})`
         )

package/src/service-v3.ts

@@ -26,6 +26,7 @@ import { recordRuntimeInfo } from './service.js'
 import { DataBindingContext } from './db-context.js'
 import { TemplateInstanceState } from './state.js'
 import { freezeGlobalConfig } from './global-config.js'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
@@ -35,11 +36,11 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
   private readonly shutdownHandler?: () => void
   private started = false
 
-  constructor(loader: () => Promise<any>, options?: any, shutdownHandler?: () => void) {
+  constructor(loader: () => Promise<any>, options?: ProcessorRuntimeOptions, shutdownHandler?: () => void) {
     this.loader = loader
     this.shutdownHandler = shutdownHandler
 
-    this.enablePartition = options?.['enable-partition'] == true
+    this.enablePartition = options?.enablePartition == true
   }
 
   async start(request: StartRequest, context: CallContext): Promise<Empty> {
@@ -170,9 +171,9 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
       .then(async (result) => {
         console.debug(`process binding ${processId} done`)
         await context.awaitPendings()
-
+        const { timeseriesResult, ...otherResults } = result
         console.debug('sending ts data length:', result.timeseriesResult.length)
-        for (const ts of result.timeseriesResult) {
+        for (const ts of timeseriesResult) {
           subject.next({
             processId,
             tsRequest: {
@@ -193,10 +194,7 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
 
         console.debug('sending binding result', processId)
         subject.next({
-          result: {
-            states: result.states,
-            exports: result.exports
-          },
+          result: otherResults,
           processId: processId
         })
         recordRuntimeInfo(result, binding.handlerType)

package/src/service-worker.ts

@@ -9,6 +9,7 @@ import { Piscina } from 'piscina'
 import { configureEndpoints } from './endpoints.js'
 import { setupLogger } from './logger.js'
 import { Subject } from 'rxjs'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
 let started = false
 
@@ -34,7 +35,7 @@ process
 
 let service: ProcessorServiceImpl | undefined
 
-const loader = async (options: any) => {
+const loader = async (options: ProcessorRuntimeOptions) => {
   if (options.target) {
     const m = await import(options.target)
     console.debug('Module loaded, path:', options.target, 'module:', m)
@@ -44,7 +45,7 @@ const loader = async (options: any) => {
 
 const emptyCallContext = <CallContext>{}
 
-async function start(request: StartRequest, options: any): Promise<Empty> {
+async function start(request: StartRequest, options: ProcessorRuntimeOptions): Promise<Empty> {
   if (started) {
     return {}
   }

package/src/service.ts

@@ -36,6 +36,7 @@ import { Provider } from 'ethers'
 import { decodeMulticallResult, encodeMulticallData, getMulticallAddress, Multicall3Call } from './multicall.js'
 
 import { processMetrics } from './metrics.js'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
@@ -58,7 +59,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   private preparedData: PreparedData | undefined
   readonly enablePartition: boolean
 
-  constructor(loader: () => Promise<any>, options?: any, shutdownHandler?: () => void) {
+  constructor(loader: () => Promise<any>, options?: ProcessorRuntimeOptions, shutdownHandler?: () => void) {
     this.loader = loader
     this.shutdownHandler = shutdownHandler
 
@@ -66,7 +67,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
       ? process.env['ENABLE_PREPROCESS'].toLowerCase() == 'true'
       : false
 
-    this.enablePartition = options?.['enable-partition'] == true
+    this.enablePartition = options?.enablePartition == true
   }
 
   async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {

package/lib/chunk-6XHWJ2VS.js

@@ -1,92 +0,0 @@
-import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);
-
-// src/state.ts
-import { isMainThread, parentPort, threadId } from "node:worker_threads";
-var State = class _State {
-  stateMap = /* @__PURE__ */ new Map();
-  static INSTANCE = new _State();
-  static reset() {
-    _State.INSTANCE = new _State();
-  }
-};
-var StateStorage = class {
-  // TODO learn how to define single instance for all subclasses
-  constructor() {
-  }
-  key() {
-    return this.constructor.name;
-  }
-  getOrRegister() {
-    let metricState = State.INSTANCE.stateMap.get(this.key());
-    if (!metricState) {
-      metricState = this.initValue();
-      State.INSTANCE.stateMap.set(this.key(), metricState);
-    }
-    return metricState;
-  }
-  unregister() {
-    const value = State.INSTANCE.stateMap.get(this.key());
-    State.INSTANCE.stateMap.delete(this.key());
-    return value;
-  }
-};
-var MapStateStorage = class extends StateStorage {
-  initValue() {
-    return /* @__PURE__ */ new Map();
-  }
-  getValue(key) {
-    const m = this.getOrRegister();
-    return m.get(key);
-  }
-  getValues() {
-    const m = this.getOrRegister();
-    return Array.from(m.values());
-  }
-  getOrSetValue(key, value) {
-    const m = this.getOrRegister();
-    const oldValue = m.get(key);
-    if (oldValue) {
-      if (oldValue !== value) {
-        console.warn(key, "has been registered twice, use the previous one");
-      }
-      return oldValue;
-    }
-    m.set(key, value);
-    return value;
-  }
-};
-var ListStateStorage = class extends StateStorage {
-  initValue() {
-    return [];
-  }
-  getValues() {
-    return this.getOrRegister();
-  }
-  addValue(value) {
-    const m = this.getOrRegister();
-    m.push(value);
-    return value;
-  }
-};
-var TemplateInstanceState = class _TemplateInstanceState extends ListStateStorage {
-  static INSTANCE = new _TemplateInstanceState();
-  constructor() {
-    super();
-  }
-  addValue(value) {
-    if (!isMainThread) {
-      parentPort?.postMessage({ event: "add_template_instance", value, from: threadId });
-    }
-    return super.addValue(value);
-  }
-};
-import("node:process").then((p) => p.stdout.write(""));
-
-export {
-  State,
-  StateStorage,
-  MapStateStorage,
-  ListStateStorage,
-  TemplateInstanceState
-};
-//# sourceMappingURL=chunk-6XHWJ2VS.js.map