@sentio/runtime 3.0.0-rc.8 → 3.0.0

package/src/gen/service/common/protos/common.ts

@@ -1,7 +1,7 @@
 /* eslint-disable */
 import Long from "long";
 import _m0 from "protobufjs/minimal.js";
-import { Struct } from "../../../google/protobuf/struct.js";
+import { ListValue, Struct } from "../../../google/protobuf/struct.js";
 import { Timestamp } from "../../../google/protobuf/timestamp.js";
 import { Money } from "../../../google/type/money.js";
 
@@ -311,6 +311,45 @@ export function notificationTypeToJSON(object: NotificationType): string {
   }
 }
 
+export enum ChatType {
+  CHAT_TYPE_UNSPECIFIED = 0,
+  CHAT_TYPE_CHAT = 1,
+  CHAT_TYPE_ACTION = 2,
+  UNRECOGNIZED = -1,
+}
+
+export function chatTypeFromJSON(object: any): ChatType {
+  switch (object) {
+    case 0:
+    case "CHAT_TYPE_UNSPECIFIED":
+      return ChatType.CHAT_TYPE_UNSPECIFIED;
+    case 1:
+    case "CHAT_TYPE_CHAT":
+      return ChatType.CHAT_TYPE_CHAT;
+    case 2:
+    case "CHAT_TYPE_ACTION":
+      return ChatType.CHAT_TYPE_ACTION;
+    case -1:
+    case "UNRECOGNIZED":
+    default:
+      return ChatType.UNRECOGNIZED;
+  }
+}
+
+export function chatTypeToJSON(object: ChatType): string {
+  switch (object) {
+    case ChatType.CHAT_TYPE_UNSPECIFIED:
+      return "CHAT_TYPE_UNSPECIFIED";
+    case ChatType.CHAT_TYPE_CHAT:
+      return "CHAT_TYPE_CHAT";
+    case ChatType.CHAT_TYPE_ACTION:
+      return "CHAT_TYPE_ACTION";
+    case ChatType.UNRECOGNIZED:
+    default:
+      return "UNRECOGNIZED";
+  }
+}
+
 export interface UsageTracker {
   apiSku: string;
   webuiSku: string;
@@ -412,6 +451,8 @@ export interface User {
   accountStatus: User_AccountStatus;
   tier: Tier;
   isOrganization: boolean;
+  walletAddress: string;
+  identities: string[];
 }
 
 export enum User_AccountStatus {
@@ -2050,6 +2091,23 @@ export interface RequestLog {
   chainId: string;
 }
 
+export interface HistoryChat {
+  id: string;
+  title: string;
+  messages: Array<any> | undefined;
+  type: ChatType;
+  meta: { [key: string]: any } | undefined;
+  createdAt: Date | undefined;
+  updatedAt: Date | undefined;
+  projectOwner?: string | undefined;
+  projectSlug?: string | undefined;
+}
+
+export interface ProjectOwnerAndSlug {
+  ownerName: string;
+  slug: string;
+}
+
 function createBaseUsageTracker(): UsageTracker {
   return {
     apiSku: "",
@@ -2796,6 +2854,8 @@ function createBaseUser(): User {
     accountStatus: 0,
     tier: 0,
     isOrganization: false,
+    walletAddress: "",
+    identities: [],
   };
 }
 
@@ -2852,6 +2912,12 @@ export const User = {
     if (message.isOrganization !== false) {
       writer.uint32(128).bool(message.isOrganization);
     }
+    if (message.walletAddress !== "") {
+      writer.uint32(138).string(message.walletAddress);
+    }
+    for (const v of message.identities) {
+      writer.uint32(146).string(v!);
+    }
     return writer;
   },
 
@@ -2967,6 +3033,20 @@ export const User = {
 
           message.isOrganization = reader.bool();
           continue;
+        case 17:
+          if (tag !== 138) {
+            break;
+          }
+
+          message.walletAddress = reader.string();
+          continue;
+        case 18:
+          if (tag !== 146) {
+            break;
+          }
+
+          message.identities.push(reader.string());
+          continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
         break;
@@ -2993,6 +3073,10 @@ export const User = {
       accountStatus: isSet(object.accountStatus) ? user_AccountStatusFromJSON(object.accountStatus) : 0,
       tier: isSet(object.tier) ? tierFromJSON(object.tier) : 0,
       isOrganization: isSet(object.isOrganization) ? globalThis.Boolean(object.isOrganization) : false,
+      walletAddress: isSet(object.walletAddress) ? globalThis.String(object.walletAddress) : "",
+      identities: globalThis.Array.isArray(object?.identities)
+        ? object.identities.map((e: any) => globalThis.String(e))
+        : [],
     };
   },
 
@@ -3043,6 +3127,12 @@ export const User = {
     if (message.isOrganization !== false) {
       obj.isOrganization = message.isOrganization;
     }
+    if (message.walletAddress !== "") {
+      obj.walletAddress = message.walletAddress;
+    }
+    if (message.identities?.length) {
+      obj.identities = message.identities;
+    }
     return obj;
   },
 
@@ -3066,6 +3156,8 @@ export const User = {
     message.accountStatus = object.accountStatus ?? 0;
     message.tier = object.tier ?? 0;
     message.isOrganization = object.isOrganization ?? false;
+    message.walletAddress = object.walletAddress ?? "";
+    message.identities = object.identities?.map((e) => e) || [];
     return message;
   },
 };
@@ -14695,6 +14787,269 @@ export const RequestLog = {
   },
 };
 
+function createBaseHistoryChat(): HistoryChat {
+  return {
+    id: "",
+    title: "",
+    messages: undefined,
+    type: 0,
+    meta: undefined,
+    createdAt: undefined,
+    updatedAt: undefined,
+    projectOwner: undefined,
+    projectSlug: undefined,
+  };
+}
+
+export const HistoryChat = {
+  encode(message: HistoryChat, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.id !== "") {
+      writer.uint32(10).string(message.id);
+    }
+    if (message.title !== "") {
+      writer.uint32(18).string(message.title);
+    }
+    if (message.messages !== undefined) {
+      ListValue.encode(ListValue.wrap(message.messages), writer.uint32(34).fork()).ldelim();
+    }
+    if (message.type !== 0) {
+      writer.uint32(40).int32(message.type);
+    }
+    if (message.meta !== undefined) {
+      Struct.encode(Struct.wrap(message.meta), writer.uint32(50).fork()).ldelim();
+    }
+    if (message.createdAt !== undefined) {
+      Timestamp.encode(toTimestamp(message.createdAt), writer.uint32(58).fork()).ldelim();
+    }
+    if (message.updatedAt !== undefined) {
+      Timestamp.encode(toTimestamp(message.updatedAt), writer.uint32(66).fork()).ldelim();
+    }
+    if (message.projectOwner !== undefined) {
+      writer.uint32(74).string(message.projectOwner);
+    }
+    if (message.projectSlug !== undefined) {
+      writer.uint32(82).string(message.projectSlug);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): HistoryChat {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseHistoryChat();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.id = reader.string();
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          message.title = reader.string();
+          continue;
+        case 4:
+          if (tag !== 34) {
+            break;
+          }
+
+          message.messages = ListValue.unwrap(ListValue.decode(reader, reader.uint32()));
+          continue;
+        case 5:
+          if (tag !== 40) {
+            break;
+          }
+
+          message.type = reader.int32() as any;
+          continue;
+        case 6:
+          if (tag !== 50) {
+            break;
+          }
+
+          message.meta = Struct.unwrap(Struct.decode(reader, reader.uint32()));
+          continue;
+        case 7:
+          if (tag !== 58) {
+            break;
+          }
+
+          message.createdAt = fromTimestamp(Timestamp.decode(reader, reader.uint32()));
+          continue;
+        case 8:
+          if (tag !== 66) {
+            break;
+          }
+
+          message.updatedAt = fromTimestamp(Timestamp.decode(reader, reader.uint32()));
+          continue;
+        case 9:
+          if (tag !== 74) {
+            break;
+          }
+
+          message.projectOwner = reader.string();
+          continue;
+        case 10:
+          if (tag !== 82) {
+            break;
+          }
+
+          message.projectSlug = reader.string();
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): HistoryChat {
+    return {
+      id: isSet(object.id) ? globalThis.String(object.id) : "",
+      title: isSet(object.title) ? globalThis.String(object.title) : "",
+      messages: globalThis.Array.isArray(object.messages) ? [...object.messages] : undefined,
+      type: isSet(object.type) ? chatTypeFromJSON(object.type) : 0,
+      meta: isObject(object.meta) ? object.meta : undefined,
+      createdAt: isSet(object.createdAt) ? fromJsonTimestamp(object.createdAt) : undefined,
+      updatedAt: isSet(object.updatedAt) ? fromJsonTimestamp(object.updatedAt) : undefined,
+      projectOwner: isSet(object.projectOwner) ? globalThis.String(object.projectOwner) : undefined,
+      projectSlug: isSet(object.projectSlug) ? globalThis.String(object.projectSlug) : undefined,
+    };
+  },
+
+  toJSON(message: HistoryChat): unknown {
+    const obj: any = {};
+    if (message.id !== "") {
+      obj.id = message.id;
+    }
+    if (message.title !== "") {
+      obj.title = message.title;
+    }
+    if (message.messages !== undefined) {
+      obj.messages = message.messages;
+    }
+    if (message.type !== 0) {
+      obj.type = chatTypeToJSON(message.type);
+    }
+    if (message.meta !== undefined) {
+      obj.meta = message.meta;
+    }
+    if (message.createdAt !== undefined) {
+      obj.createdAt = message.createdAt.toISOString();
+    }
+    if (message.updatedAt !== undefined) {
+      obj.updatedAt = message.updatedAt.toISOString();
+    }
+    if (message.projectOwner !== undefined) {
+      obj.projectOwner = message.projectOwner;
+    }
+    if (message.projectSlug !== undefined) {
+      obj.projectSlug = message.projectSlug;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<HistoryChat>): HistoryChat {
+    return HistoryChat.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<HistoryChat>): HistoryChat {
+    const message = createBaseHistoryChat();
+    message.id = object.id ?? "";
+    message.title = object.title ?? "";
+    message.messages = object.messages ?? undefined;
+    message.type = object.type ?? 0;
+    message.meta = object.meta ?? undefined;
+    message.createdAt = object.createdAt ?? undefined;
+    message.updatedAt = object.updatedAt ?? undefined;
+    message.projectOwner = object.projectOwner ?? undefined;
+    message.projectSlug = object.projectSlug ?? undefined;
+    return message;
+  },
+};
+
+function createBaseProjectOwnerAndSlug(): ProjectOwnerAndSlug {
+  return { ownerName: "", slug: "" };
+}
+
+export const ProjectOwnerAndSlug = {
+  encode(message: ProjectOwnerAndSlug, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.ownerName !== "") {
+      writer.uint32(10).string(message.ownerName);
+    }
+    if (message.slug !== "") {
+      writer.uint32(18).string(message.slug);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): ProjectOwnerAndSlug {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseProjectOwnerAndSlug();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.ownerName = reader.string();
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          message.slug = reader.string();
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): ProjectOwnerAndSlug {
+    return {
+      ownerName: isSet(object.ownerName) ? globalThis.String(object.ownerName) : "",
+      slug: isSet(object.slug) ? globalThis.String(object.slug) : "",
+    };
+  },
+
+  toJSON(message: ProjectOwnerAndSlug): unknown {
+    const obj: any = {};
+    if (message.ownerName !== "") {
+      obj.ownerName = message.ownerName;
+    }
+    if (message.slug !== "") {
+      obj.slug = message.slug;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<ProjectOwnerAndSlug>): ProjectOwnerAndSlug {
+    return ProjectOwnerAndSlug.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<ProjectOwnerAndSlug>): ProjectOwnerAndSlug {
+    const message = createBaseProjectOwnerAndSlug();
+    message.ownerName = object.ownerName ?? "";
+    message.slug = object.slug ?? "";
+    return message;
+  },
+};
+
 function bytesFromBase64(b64: string): Uint8Array {
   if ((globalThis as any).Buffer) {
     return Uint8Array.from(globalThis.Buffer.from(b64, "base64"));

package/src/otlp.ts

@@ -1,32 +1,40 @@
-import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { IMetricReader, MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
 import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
-import { NodeSDK } from '@opentelemetry/sdk-node'
-import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
 import { diag, DiagConsoleLogger, DiagLogLevel, metrics } from '@opentelemetry/api'
 
 export async function setupOTLP(debug?: boolean) {
+  console.log('Setting up OTLP metrics, debug=', !!debug)
   if (debug) {
     diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG)
   }
 
-  const sdk = new NodeSDK({
-    autoDetectResources: true,
-    traceExporter: new OTLPTraceExporter(),
-    metricReaders: [
-      new PeriodicExportingMetricReader({
-        exporter: new OTLPMetricExporter()
-      }),
+  const readers: IMetricReader[] = [
+    new PeriodicExportingMetricReader({
+      exporter: new OTLPMetricExporter(),
+      exportIntervalMillis: 60000
+    })
+  ]
+  if (debug) {
+    readers.push(
       new PrometheusExporter({
         host: '0.0.0.0',
         port: 4041
       })
+    )
+  }
+  const meterProvider = new MeterProvider({
+    readers: [
+      new PeriodicExportingMetricReader({
+        exporter: new OTLPMetricExporter(),
+        exportIntervalMillis: 60000
+      })
     ]
   })
 
-  sdk.start()
+  metrics.setGlobalMeterProvider(meterProvider)
   ;['SIGINT', 'SIGTERM'].forEach((signal) => {
-    process.on(signal as any, () => sdk.shutdown().catch(console.error))
+    process.on(signal as any, () => meterProvider.forceFlush().catch(console.error))
   })
 
   metrics.getMeter('processor').createGauge('up').record(1)

package/src/plugin.ts

@@ -8,7 +8,6 @@ import {
   ProcessStreamResponse_Partitions,
   ProcessStreamResponse_Partitions_Partition_SysValue,
   StartRequest,
-  TemplateInstance,
   UpdateTemplatesRequest
 } from '@sentio/protos'
 import { IDataBindingContext, IStoreContext } from './db-context.js'
@@ -153,14 +152,4 @@ export class PluginManager {
       })
     }
   }
-
-  sendTemplateInstance(instance: TemplateInstance) {
-    // send template instance within the current db context
-    // this is only work in user handlers, when dbContextLocalStorage is set
-    // plugin.Configure does not have db context
-    const store = this.dbContextLocalStorage.getStore()
-    if (store && 'sendTemplateRequest' in store) {
-      store?.sendTemplateRequest([instance])
-    }
-  }
 }

package/src/processor-runner.ts

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
 
 import fs from 'fs-extra'
-
+// import { compressionAlgorithms } from '@grpc/grpc-js'
 import { createServer } from 'nice-grpc'
 import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
 // import { registry as niceGrpcRegistry } from 'nice-grpc-prometheus'
-import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
+// import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
 import http from 'http'
 // @ts-ignore inspector promises is not included in @type/node
 import { Session } from 'node:inspector/promises'
@@ -61,7 +61,7 @@ if (options.startActionServer) {
     // 'grpc.default_compression_algorithm': compressionAlgorithms.gzip
   })
     // .use(prometheusServerMiddleware())
-    .use(openTelemetryServerMiddleware())
+    // .use(openTelemetryServerMiddleware())
     .use(errorDetailsServerMiddleware)
 
   if (options.worker > 1) {

package/src/service-manager.ts

@@ -15,7 +15,6 @@ import { Subject } from 'rxjs'
 
 import { MessageChannel } from 'node:worker_threads'
 import { ProcessorServiceImpl } from './service.js'
-import { TemplateInstanceState } from './state.js'
 import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 ;(BigInt.prototype as any).toJSON = function () {
   return this.toString()
@@ -140,13 +139,6 @@ export class ServiceManager extends ProcessorServiceImpl {
       argv: process.argv,
       workerData: this.workerData
     })
-    // @ts-ignore - Piscina message handling for template instance sync
-    this.pool.on('message', (msg: any) => {
-      if (msg.event == 'add_template_instance') {
-        // sync the template state from worker to the main thread
-        TemplateInstanceState.INSTANCE.addValue(msg.value)
-      }
-    })
   }
 }
 

package/src/service-v3.ts

@@ -29,6 +29,9 @@ import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
+const WRITE_V2_EVENT_LOGS = process.env.WRITE_V2_EVENT_LOGS !== 'false'
+const TIME_SERIES_RESULT_BATCH_SIZE = 1000
+
 export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation {
   readonly enablePartition: boolean
   private readonly loader: () => Promise<any>
@@ -165,35 +168,26 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
   ) {
     const context = this.contexts.new(processId, subject)
     const start = Date.now()
-    console.debug('process binding', processId)
     PluginManager.INSTANCE.processBinding(binding, undefined, context)
       .then(async (result) => {
-        console.debug(`process binding ${processId} done`)
         await context.awaitPendings()
         const { timeseriesResult, ...otherResults } = result
-        console.debug('sending ts data length:', result.timeseriesResult.length)
-        for (const ts of timeseriesResult) {
+        for (let i = 0; i < timeseriesResult.length; i += TIME_SERIES_RESULT_BATCH_SIZE) {
+          const batch = timeseriesResult.slice(i, i + TIME_SERIES_RESULT_BATCH_SIZE)
           subject.next({
             processId,
             tsRequest: {
-              data: [ts]
+              data: batch
             }
           })
         }
 
-        /* if (result.states?.configUpdated) {
-          console.debug('sending tpl updates:')
-          subject.next({
-            processId,
-            tplRequest: {
-              templates: TemplateInstanceState.INSTANCE.getValues()
-            }
-          })
-        }*/
-
-        console.debug('sending binding result', processId)
         subject.next({
-          result: otherResults,
+          result: WRITE_V2_EVENT_LOGS
+            ? otherResults
+            : {
+                states: otherResults.states
+              },
           processId: processId
         })
         recordRuntimeInfo(result, binding.handlerType)
@@ -207,7 +201,6 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
         const cost = Date.now() - start
         process_binding_time.add(cost)
         this.contexts.delete(processId)
-        console.debug('process binding done', processId)
       })
   }
 

package/src/state.ts

@@ -1,7 +1,3 @@
-import { TemplateInstance } from '@sentio/protos'
-import { isMainThread, parentPort, threadId } from 'node:worker_threads'
-import { PluginManager } from './plugin.js'
-
 export class State {
   stateMap = new Map<string, any>()
 
@@ -85,21 +81,3 @@ export abstract class ListStateStorage<T> extends StateStorage<T[]> {
     return value
   }
 }
-
-export class TemplateInstanceState extends ListStateStorage<TemplateInstance> {
-  static INSTANCE = new TemplateInstanceState()
-
-  constructor() {
-    super()
-  }
-
-  override addValue(value: TemplateInstance): TemplateInstance {
-    if (!isMainThread) {
-      // I'm worker thread, should notice the main thread
-      parentPort?.postMessage({ event: 'add_template_instance', value, from: threadId })
-    }
-    const ret = super.addValue(value)
-    PluginManager.INSTANCE.sendTemplateInstance(ret)
-    return ret
-  }
-}