@sentio/runtime 2.57.12-rc.f → 2.57.12-rc.h
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/processor-runner.js +15 -15
- package/lib/processor-runner.js.map +1 -1
- package/lib/service-worker.js +15 -5
- package/lib/service-worker.js.map +1 -1
- package/lib/test-processor.test.js.map +1 -1
- package/package.json +1 -1
- package/src/service-manager.ts +19 -17
- package/src/service-worker.ts +16 -5
package/lib/service-worker.js
CHANGED
|
@@ -64,15 +64,25 @@ async function start(request, options) {
|
|
|
64
64
|
}
|
|
65
65
|
async function service_worker_default({ request }) {
|
|
66
66
|
try {
|
|
67
|
+
const { startRequest, configRequest, options } = Piscina.workerData;
|
|
67
68
|
if (!started) {
|
|
68
|
-
const { startRequest, configRequest, options } = Piscina.workerData;
|
|
69
69
|
if (startRequest) {
|
|
70
|
-
|
|
70
|
+
try {
|
|
71
|
+
await start(startRequest, options);
|
|
72
|
+
console.info("worker started");
|
|
73
|
+
} catch (e) {
|
|
74
|
+
console.log("worker start failed", e);
|
|
75
|
+
}
|
|
71
76
|
}
|
|
77
|
+
console.info("new processor worker started");
|
|
72
78
|
if (configRequest) {
|
|
73
|
-
|
|
79
|
+
try {
|
|
80
|
+
await getConfig(configRequest);
|
|
81
|
+
console.info("worker configured");
|
|
82
|
+
} catch (e) {
|
|
83
|
+
console.log("worker configure failed", e);
|
|
84
|
+
}
|
|
74
85
|
}
|
|
75
|
-
console.info("new processor worker started");
|
|
76
86
|
}
|
|
77
87
|
if (unhandled) {
|
|
78
88
|
throw new import_nice_grpc_error_details.RichServerError(
|
|
@@ -94,7 +104,7 @@ async function service_worker_default({ request }) {
|
|
|
94
104
|
recordRuntimeInfo(result, request.handlerType);
|
|
95
105
|
return result;
|
|
96
106
|
} catch (e) {
|
|
97
|
-
console.
|
|
107
|
+
console.log("worker process binding failed", e);
|
|
98
108
|
throw e;
|
|
99
109
|
}
|
|
100
110
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/service-worker.ts"],"sourcesContent":["import {\n DataBinding,\n Empty,\n HandlerType,\n PreparedData,\n ProcessConfigRequest,\n ProcessConfigResponse,\n ProcessResult,\n StartRequest\n} from '@sentio/protos'\nimport { CallContext, ServerError, Status } from 'nice-grpc'\nimport { PluginManager } from './plugin.js'\nimport commandLineArgs from 'command-line-args'\nimport { optionDefinitions } from './processor-runner.js'\nimport { errorString } from './utils.js'\nimport { freezeGlobalConfig } from './global-config.js'\nimport { DebugInfo, RichServerError } from 'nice-grpc-error-details'\nimport { recordRuntimeInfo } from './service.js'\nimport { BroadcastChannel } from 'worker_threads'\nimport { Piscina } from 'piscina'\n\nlet started = false\n\nlet unhandled: Error | undefined\n\nprocess\n .on('uncaughtException', (err) => {\n console.error('Uncaught Exception, please checking if await is properly used', err)\n unhandled = err\n })\n .on('unhandledRejection', (reason, p) => {\n // @ts-ignore ignore invalid ens error\n if (reason?.message.startsWith('invalid ENS name (disallowed character: \"*\"')) {\n return\n }\n console.error('Unhandled Rejection, please checking if await is properly', reason)\n unhandled = reason as Error\n // shutdownServers(1)\n })\n\nasync function getConfig(request: ProcessConfigRequest, context?: CallContext): Promise<ProcessConfigResponse> {\n if (!started) {\n throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')\n }\n\n const newConfig = ProcessConfigResponse.fromPartial({})\n await PluginManager.INSTANCE.configure(newConfig)\n return newConfig\n}\n\nconst loader = async (options: any) => {\n if (options.target) {\n const m = await import(options.target)\n console.debug('Module loaded, path:', options.target, 'module:', m)\n return m\n }\n}\n\nconst configureChannel = new BroadcastChannel('configure_channel')\nconfigureChannel.onmessage = (request: ProcessConfigRequest) => {\n getConfig(request)\n}\n\nasync function start(request: StartRequest, options: any): Promise<Empty> {\n if (started) {\n return {}\n }\n freezeGlobalConfig()\n\n try {\n await loader(options)\n } catch (e) {\n throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))\n }\n\n await PluginManager.INSTANCE.start(request)\n started = true\n return {}\n}\n\nexport default async function ({ request }: { request: DataBinding }) {\n try {\n
|
|
1
|
+
{"version":3,"sources":["../src/service-worker.ts"],"sourcesContent":["import {\n DataBinding,\n Empty,\n HandlerType,\n PreparedData,\n ProcessConfigRequest,\n ProcessConfigResponse,\n ProcessResult,\n StartRequest\n} from '@sentio/protos'\nimport { CallContext, ServerError, Status } from 'nice-grpc'\nimport { PluginManager } from './plugin.js'\nimport commandLineArgs from 'command-line-args'\nimport { optionDefinitions } from './processor-runner.js'\nimport { errorString } from './utils.js'\nimport { freezeGlobalConfig } from './global-config.js'\nimport { DebugInfo, RichServerError } from 'nice-grpc-error-details'\nimport { recordRuntimeInfo } from './service.js'\nimport { BroadcastChannel } from 'worker_threads'\nimport { Piscina } from 'piscina'\n\nlet started = false\n\nlet unhandled: Error | undefined\n\nprocess\n .on('uncaughtException', (err) => {\n console.error('Uncaught Exception, please checking if await is properly used', err)\n unhandled = err\n })\n .on('unhandledRejection', (reason, p) => {\n // @ts-ignore ignore invalid ens error\n if (reason?.message.startsWith('invalid ENS name (disallowed character: \"*\"')) {\n return\n }\n console.error('Unhandled Rejection, please checking if await is properly', reason)\n unhandled = reason as Error\n // shutdownServers(1)\n })\n\nasync function getConfig(request: ProcessConfigRequest, context?: CallContext): Promise<ProcessConfigResponse> {\n if (!started) {\n throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')\n }\n\n const newConfig = ProcessConfigResponse.fromPartial({})\n await PluginManager.INSTANCE.configure(newConfig)\n return newConfig\n}\n\nconst loader = async (options: any) => {\n if (options.target) {\n const m = await import(options.target)\n console.debug('Module loaded, path:', options.target, 'module:', m)\n return m\n }\n}\n\nconst configureChannel = new BroadcastChannel('configure_channel')\nconfigureChannel.onmessage = (request: ProcessConfigRequest) => {\n getConfig(request)\n}\n\nasync function start(request: StartRequest, options: any): Promise<Empty> {\n if (started) {\n return {}\n }\n freezeGlobalConfig()\n\n try {\n await loader(options)\n } catch (e) {\n throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))\n }\n\n await PluginManager.INSTANCE.start(request)\n started = true\n return {}\n}\n\nexport default async function ({ request }: { request: DataBinding }) {\n try {\n const { startRequest, configRequest, options } = Piscina.workerData\n if (!started) {\n if (startRequest) {\n try {\n await start(startRequest, options)\n console.info('worker started')\n } catch (e) {\n console.log('worker start failed', e)\n }\n }\n console.info('new processor worker started')\n\n if (configRequest) {\n try {\n await getConfig(configRequest)\n console.info('worker configured')\n } catch (e) {\n console.log('worker configure failed', e)\n }\n }\n }\n\n if (unhandled) {\n throw new RichServerError(\n Status.UNAVAILABLE,\n 'Unhandled exception/rejection in previous request: ' + errorString(unhandled),\n [\n DebugInfo.fromPartial({\n detail: unhandled.message,\n stackEntries: unhandled.stack?.split('\\n')\n })\n ]\n )\n }\n\n const result = await PluginManager.INSTANCE.processBinding(\n request,\n undefined\n // PluginManager.INSTANCE.dbContextLocalStorage.getStore()\n )\n recordRuntimeInfo(result, request.handlerType)\n return result\n } catch (e) {\n console.log('worker process binding failed', e)\n throw e\n }\n}\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));"],"mappings":";;;;;;;;;;;;;;;AAUA,uBAAiD;AAMjD,qCAA2C;AAE3C,SAAS,wBAAwB;AACjC,SAAS,eAAe;AAExB,IAAI,UAAU;AAEd,IAAI;AAEJ,QACG,GAAG,qBAAqB,CAAC,QAAQ;AAChC,UAAQ,MAAM,iEAAiE,GAAG;AAClF,cAAY;AACd,CAAC,EACA,GAAG,sBAAsB,CAAC,QAAQ,MAAM;AAEvC,MAAI,QAAQ,QAAQ,WAAW,6CAA6C,GAAG;AAC7E;AAAA,EACF;AACA,UAAQ,MAAM,6DAA6D,MAAM;AACjF,cAAY;AAEd,CAAC;AAEH,eAAe,UAAU,SAA+B,SAAuD;AAC7G,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,6BAAY,wBAAO,aAAa,sBAAsB;AAAA,EAClE;AAEA,QAAM,YAAY,sBAAsB,YAAY,CAAC,CAAC;AACtD,QAAM,cAAc,SAAS,UAAU,SAAS;AAChD,SAAO;AACT;AAEA,IAAM,SAAS,OAAO,YAAiB;AACrC,MAAI,QAAQ,QAAQ;AAClB,UAAM,IAAI,MAAM,OAAO,QAAQ;AAC/B,YAAQ,MAAM,wBAAwB,QAAQ,QAAQ,WAAW,CAAC;AAClE,WAAO;AAAA,EACT;AACF;AAEA,IAAM,mBAAmB,IAAI,iBAAiB,mBAAmB;AACjE,iBAAiB,YAAY,CAAC,YAAkC;AAC9D,YAAU,OAAO;AACnB;AAEA,eAAe,MAAM,SAAuB,SAA8B;AACxE,MAAI,SAAS;AACX,WAAO,CAAC;AAAA,EACV;AACA,qBAAmB;AAEnB,MAAI;AACF,UAAM,OAAO,OAAO;AAAA,EACtB,SAAS,GAAG;AACV,UAAM,IAAI,6BAAY,wBAAO,kBAAkB,+BAA+B,YAAY,CAAC,CAAC;AAAA,EAC9F;AAEA,QAAM,cAAc,SAAS,MAAM,OAAO;AAC1C,YAAU;AACV,SAAO,CAAC;AACV;AAEA,eAAO,uBAAwB,EAAE,QAAQ,GAA6B;AACpE,MAAI;AACF,UAAM,EAAE,cAAc,eAAe,QAAQ,IAAI,QAAQ;AACzD,QAAI,CAAC,SAAS;AACZ,UAAI,cAAc;AAChB,YAAI;AACF,gBAAM,MAAM,cAAc,OAAO;AACjC,kBAAQ,KAAK,gBAAgB;AAAA,QAC/B,SAAS,GAAG;AACV,kBAAQ,IAAI,uBAAuB,CAAC;AAAA,QACtC;AAAA,MACF;AACA,cAAQ,KAAK,8BAA8B;AAE3C,UAAI,eAAe;AACjB,YAAI;AACF,gBAAM,UAAU,aAAa;AAC7B,kBAAQ,KAAK,mBAAmB;AAAA,QAClC,SAAS,GAAG;AACV,kBAAQ,IAAI,2BAA2B,CAAC;AAAA,QAC1C;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW;AACb,YAAM,IAAI;AAAA,QACR,wBAAO;AAAA,QACP,wDAAwD,YAAY,SAAS;AAAA,QAC7E;AAAA,UACE,yCAAU,YAAY;AAAA,YACpB,QAAQ,UAAU;AAAA,YAClB,cAAc,UAAU,OAAO,MAAM,IAAI;AAAA,UAC3C,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,UAAM,SAAS,MAAM,cAAc,SAAS;AAAA,MAC1C;AAAA,MACA;AAAA;AAAA,IAEF;AACA,sBAAkB,QAAQ,QAAQ,WAAW;AAC7C,WAAO;AAAA,EACT,SAAS,GAAG;AACV,YAAQ,IAAI,iCAAiC,CAAC;AAC9C,UAAM;AAAA,EACR;AACF;AACC,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;","names":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/test-processor.test.ts"],"sourcesContent":["import { Plugin, PluginManager } from './plugin.js'\nimport { DataBinding, HandlerType, ProcessResult } from './gen/processor/protos/processor.js'\n\nclass TestPlugin extends Plugin {\n async processBinding(request: DataBinding): Promise<ProcessResult> {\n return ProcessResult.fromPartial({\n states: {\n configUpdated: true\n }\n })\n }\n supportedHandlers = [HandlerType.UNKNOWN
|
|
1
|
+
{"version":3,"sources":["../src/test-processor.test.ts"],"sourcesContent":["import { Plugin, PluginManager } from './plugin.js'\nimport { DataBinding, HandlerType, ProcessResult } from './gen/processor/protos/processor.js'\n\nclass TestPlugin extends Plugin {\n async processBinding(request: DataBinding): Promise<ProcessResult> {\n return ProcessResult.fromPartial({\n states: {\n configUpdated: true\n }\n })\n }\n supportedHandlers = [HandlerType.UNKNOWN]\n}\n\nPluginManager.INSTANCE.plugins = []\nPluginManager.INSTANCE.register(new TestPlugin())\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));"],"mappings":";;;;;;;;AAGA,IAAM,aAAN,cAAyB,OAAO;AAAA,EAC9B,MAAM,eAAe,SAA8C;AACjE,WAAO,cAAc,YAAY;AAAA,MAC/B,QAAQ;AAAA,QACN,eAAe;AAAA,MACjB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EACA,oBAAoB,gBAAoB;AAC1C;AAEA,cAAc,SAAS,UAAU,CAAC;AAClC,cAAc,SAAS,SAAS,IAAI,WAAW,CAAC;AAC/C,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;","names":[]}
|
package/package.json
CHANGED
package/src/service-manager.ts
CHANGED
|
@@ -44,11 +44,15 @@ export class ServiceManager implements ProcessorServiceImplementation {
|
|
|
44
44
|
// private processorConfig: ProcessConfigResponse
|
|
45
45
|
private pool: Piscina<any, any>
|
|
46
46
|
|
|
47
|
+
private workerData: any = {}
|
|
48
|
+
|
|
47
49
|
constructor(
|
|
48
50
|
readonly options: any,
|
|
49
51
|
readonly loader: () => Promise<any>,
|
|
50
52
|
readonly shutdownHandler?: () => void
|
|
51
|
-
) {
|
|
53
|
+
) {
|
|
54
|
+
this.workerData.options = options
|
|
55
|
+
}
|
|
52
56
|
|
|
53
57
|
async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
|
|
54
58
|
throw new Error('not supported')
|
|
@@ -66,10 +70,8 @@ export class ServiceManager implements ProcessorServiceImplementation {
|
|
|
66
70
|
// also configure the main thread
|
|
67
71
|
const newConfig = ProcessConfigResponse.fromPartial({})
|
|
68
72
|
await PluginManager.INSTANCE.configure(newConfig)
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
...this.pool.options.workerData
|
|
72
|
-
}
|
|
73
|
+
|
|
74
|
+
this.workerData.configRequest = request
|
|
73
75
|
return newConfig
|
|
74
76
|
}
|
|
75
77
|
|
|
@@ -84,26 +86,17 @@ export class ServiceManager implements ProcessorServiceImplementation {
|
|
|
84
86
|
throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
|
|
85
87
|
}
|
|
86
88
|
|
|
87
|
-
this.pool = new Piscina({
|
|
88
|
-
maxThreads: this.options.worker,
|
|
89
|
-
minThreads: this.options.worker,
|
|
90
|
-
filename: new URL('./service-worker.js', import.meta.url).href.replaceAll('runtime/src', 'runtime/lib'),
|
|
91
|
-
argv: process.argv,
|
|
92
|
-
workerData: {
|
|
93
|
-
startRequest: request,
|
|
94
|
-
options: this.options
|
|
95
|
-
}
|
|
96
|
-
})
|
|
97
|
-
|
|
98
89
|
// also start the processor in main thread
|
|
99
90
|
await PluginManager.INSTANCE.start(request)
|
|
100
91
|
this.started = true
|
|
92
|
+
this.workerData.startRequest = request
|
|
101
93
|
return {}
|
|
102
94
|
}
|
|
103
95
|
|
|
104
96
|
async stop(request: Empty, context: CallContext): Promise<Empty> {
|
|
105
|
-
|
|
97
|
+
await this.pool.destroy()
|
|
106
98
|
if (this.shutdownHandler) {
|
|
99
|
+
console.log('Server Shutting down in 5 seconds')
|
|
107
100
|
setTimeout(this.shutdownHandler, 5000)
|
|
108
101
|
}
|
|
109
102
|
return {}
|
|
@@ -143,6 +136,15 @@ export class ServiceManager implements ProcessorServiceImplementation {
|
|
|
143
136
|
if (!this.started) {
|
|
144
137
|
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
|
145
138
|
}
|
|
139
|
+
if (!this.pool) {
|
|
140
|
+
this.pool = new Piscina({
|
|
141
|
+
maxThreads: this.options.worker,
|
|
142
|
+
minThreads: this.options.worker,
|
|
143
|
+
filename: new URL('./service-worker.js', import.meta.url).href.replaceAll('runtime/src', 'runtime/lib'),
|
|
144
|
+
argv: process.argv,
|
|
145
|
+
workerData: this.workerData
|
|
146
|
+
})
|
|
147
|
+
}
|
|
146
148
|
if (this.unhandled) {
|
|
147
149
|
throw new RichServerError(
|
|
148
150
|
Status.UNAVAILABLE,
|
package/src/service-worker.ts
CHANGED
|
@@ -80,15 +80,26 @@ async function start(request: StartRequest, options: any): Promise<Empty> {
|
|
|
80
80
|
|
|
81
81
|
export default async function ({ request }: { request: DataBinding }) {
|
|
82
82
|
try {
|
|
83
|
+
const { startRequest, configRequest, options } = Piscina.workerData
|
|
83
84
|
if (!started) {
|
|
84
|
-
const { startRequest, configRequest, options } = Piscina.workerData
|
|
85
85
|
if (startRequest) {
|
|
86
|
-
|
|
86
|
+
try {
|
|
87
|
+
await start(startRequest, options)
|
|
88
|
+
console.info('worker started')
|
|
89
|
+
} catch (e) {
|
|
90
|
+
console.log('worker start failed', e)
|
|
91
|
+
}
|
|
87
92
|
}
|
|
93
|
+
console.info('new processor worker started')
|
|
94
|
+
|
|
88
95
|
if (configRequest) {
|
|
89
|
-
|
|
96
|
+
try {
|
|
97
|
+
await getConfig(configRequest)
|
|
98
|
+
console.info('worker configured')
|
|
99
|
+
} catch (e) {
|
|
100
|
+
console.log('worker configure failed', e)
|
|
101
|
+
}
|
|
90
102
|
}
|
|
91
|
-
console.info('new processor worker started')
|
|
92
103
|
}
|
|
93
104
|
|
|
94
105
|
if (unhandled) {
|
|
@@ -112,7 +123,7 @@ export default async function ({ request }: { request: DataBinding }) {
|
|
|
112
123
|
recordRuntimeInfo(result, request.handlerType)
|
|
113
124
|
return result
|
|
114
125
|
} catch (e) {
|
|
115
|
-
console.
|
|
126
|
+
console.log('worker process binding failed', e)
|
|
116
127
|
throw e
|
|
117
128
|
}
|
|
118
129
|
}
|