@sentio/runtime 2.59.0-rc.9 → 2.59.1-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/{chunk-J23VJQSB.js → chunk-4UYFH4YB.js} +4387 -4321
- package/lib/{chunk-J23VJQSB.js.map → chunk-4UYFH4YB.js.map} +1 -1
- package/lib/{chunk-3T4PWGQU.js → chunk-LCS6SRJY.js} +1036 -122
- package/lib/chunk-LCS6SRJY.js.map +1 -0
- package/lib/{chunk-GD7EL7GH.js → chunk-U5UZYRQ4.js} +2 -2
- package/lib/index.d.ts +193 -3
- package/lib/index.js +4 -2
- package/lib/index.js.map +1 -1
- package/lib/processor-runner.js +116 -94
- package/lib/processor-runner.js.map +1 -1
- package/lib/service-worker.d.ts +5 -6
- package/lib/service-worker.js +49 -47
- package/lib/service-worker.js.map +1 -1
- package/lib/test-processor.test.js.map +1 -1
- package/package.json +1 -1
- package/src/db-context.ts +3 -5
- package/src/full-service.ts +59 -13
- package/src/gen/processor/protos/processor.ts +1077 -159
- package/src/gen/service/common/protos/common.ts +17 -0
- package/src/metrics.ts +8 -4
- package/src/plugin.ts +24 -0
- package/src/processor-runner.ts +10 -3
- package/src/service-manager.ts +72 -106
- package/src/service-worker.ts +49 -60
- package/src/service.ts +89 -46
- package/src/utils.ts +20 -4
- package/lib/chunk-3T4PWGQU.js.map +0 -1
- /package/lib/{chunk-GD7EL7GH.js.map → chunk-U5UZYRQ4.js.map} +0 -0
package/lib/processor-runner.js
CHANGED
@@ -5,7 +5,7 @@ import {
|
|
5
5
|
} from "./chunk-6XHWJ2VS.js";
|
6
6
|
import {
|
7
7
|
setupLogger
|
8
|
-
} from "./chunk-
|
8
|
+
} from "./chunk-U5UZYRQ4.js";
|
9
9
|
import {
|
10
10
|
DiagConsoleLogger,
|
11
11
|
DiagLogLevel,
|
@@ -35,7 +35,6 @@ import {
|
|
35
35
|
isValidTraceId,
|
36
36
|
metrics,
|
37
37
|
parseSemver,
|
38
|
-
processMetrics,
|
39
38
|
propagation,
|
40
39
|
require_lib,
|
41
40
|
require_lib2,
|
@@ -44,13 +43,11 @@ import {
|
|
44
43
|
require_lodash,
|
45
44
|
require_src,
|
46
45
|
trace
|
47
|
-
} from "./chunk-
|
46
|
+
} from "./chunk-4UYFH4YB.js";
|
48
47
|
import {
|
49
48
|
ExecutionConfig,
|
50
|
-
HandlerType,
|
51
49
|
PluginManager,
|
52
50
|
ProcessConfigResponse,
|
53
|
-
ProcessResult,
|
54
51
|
ProcessorDefinition,
|
55
52
|
StartRequest,
|
56
53
|
__commonJS,
|
@@ -60,7 +57,7 @@ import {
|
|
60
57
|
__toCommonJS,
|
61
58
|
__toESM,
|
62
59
|
require_minimal2 as require_minimal
|
63
|
-
} from "./chunk-
|
60
|
+
} from "./chunk-LCS6SRJY.js";
|
64
61
|
|
65
62
|
// ../../node_modules/.pnpm/@opentelemetry+semantic-conventions@1.25.1/node_modules/@opentelemetry/semantic-conventions/build/esm/internal/utils.js
|
66
63
|
// @__NO_SIDE_EFFECTS__
|
@@ -25956,6 +25953,7 @@ import { createRequire } from "module";
|
|
25956
25953
|
var import_fs_extra = __toESM(require_lib(), 1);
|
25957
25954
|
import path from "path";
|
25958
25955
|
import os from "os";
|
25956
|
+
import { createHash } from "crypto";
|
25959
25957
|
var require2 = createRequire(import.meta.url);
|
25960
25958
|
var FUEL_PROTO_UPDATE_VERSION = parseSemver("2.54.0-rc.7");
|
25961
25959
|
var FUEL_PROTO_NO_FUEL_TRANSACTION_AS_CALL_VERSION = parseSemver("2.55.0-rc.1");
|
@@ -25964,6 +25962,22 @@ var ETH_USE_RAW_VERSION = parseSemver("2.57.9-rc.12");
|
|
25964
25962
|
var PROCESSED_MOVE_EVENT_TX_HANDLER = new LRUCache({
|
25965
25963
|
max: 1e4
|
25966
25964
|
});
|
25965
|
+
var enableTxCache = process.env.ENABLE_PARSE_CACHE === "true";
|
25966
|
+
var PARSED_DATA_CACHE = new LRUCache({
|
25967
|
+
max: enableTxCache ? 5e3 : 1
|
25968
|
+
});
|
25969
|
+
function getParsedData(rawData) {
|
25970
|
+
if (!enableTxCache) {
|
25971
|
+
return JSON.parse(rawData);
|
25972
|
+
}
|
25973
|
+
const digest = createHash("md5").update(rawData).digest("hex");
|
25974
|
+
let parsedData = PARSED_DATA_CACHE.get(digest);
|
25975
|
+
if (!parsedData) {
|
25976
|
+
parsedData = JSON.parse(rawData);
|
25977
|
+
PARSED_DATA_CACHE.set(digest, parsedData);
|
25978
|
+
}
|
25979
|
+
return parsedData;
|
25980
|
+
}
|
25967
25981
|
function locatePackageJson(pkgId) {
|
25968
25982
|
const m = require2.resolve(pkgId);
|
25969
25983
|
let dir = path.dirname(m);
|
@@ -26064,17 +26078,31 @@ var FullProcessorServiceImpl = class {
|
|
26064
26078
|
const ethLog = dataBinding.data?.ethLog;
|
26065
26079
|
if (ethLog?.log == null && ethLog?.rawLog) {
|
26066
26080
|
ethLog.log = JSON.parse(ethLog.rawLog);
|
26067
|
-
|
26068
|
-
|
26069
|
-
|
26081
|
+
if (ethLog.rawTransaction) {
|
26082
|
+
ethLog.transaction = getParsedData(ethLog.rawTransaction);
|
26083
|
+
}
|
26084
|
+
if (ethLog.rawBlock) {
|
26085
|
+
ethLog.block = getParsedData(ethLog.rawBlock);
|
26086
|
+
}
|
26087
|
+
if (ethLog.rawTransactionReceipt) {
|
26088
|
+
ethLog.transactionReceipt = getParsedData(ethLog.rawTransactionReceipt);
|
26089
|
+
}
|
26070
26090
|
}
|
26071
26091
|
break;
|
26072
26092
|
case 11 /* ETH_TRANSACTION */:
|
26073
26093
|
const ethTx = dataBinding.data?.ethTransaction;
|
26074
26094
|
if (ethTx?.transaction == null && ethTx?.rawTransaction) {
|
26075
|
-
ethTx.transaction =
|
26076
|
-
|
26077
|
-
|
26095
|
+
ethTx.transaction = getParsedData(ethTx.rawTransaction);
|
26096
|
+
if (ethTx.rawBlock) {
|
26097
|
+
ethTx.block = getParsedData(ethTx.rawBlock);
|
26098
|
+
} else {
|
26099
|
+
ethTx.block = void 0;
|
26100
|
+
}
|
26101
|
+
if (ethTx.rawTransactionReceipt) {
|
26102
|
+
ethTx.transactionReceipt = getParsedData(ethTx.rawTransactionReceipt);
|
26103
|
+
} else {
|
26104
|
+
ethTx.transactionReceipt = void 0;
|
26105
|
+
}
|
26078
26106
|
}
|
26079
26107
|
break;
|
26080
26108
|
case 20 /* FUEL_TRANSACTION */:
|
@@ -26097,7 +26125,7 @@ var FullProcessorServiceImpl = class {
|
|
26097
26125
|
const aptEvent = dataBinding.data?.aptEvent;
|
26098
26126
|
if (aptEvent) {
|
26099
26127
|
if (isBeforeMoveUseRawVersion && aptEvent.rawTransaction) {
|
26100
|
-
const transaction =
|
26128
|
+
const transaction = getParsedData(aptEvent.rawTransaction);
|
26101
26129
|
const key = `${transaction.hash}-${dataBinding.handlerIds[0]}`;
|
26102
26130
|
if (PROCESSED_MOVE_EVENT_TX_HANDLER.has(key)) {
|
26103
26131
|
console.debug("skip binding", key);
|
@@ -26116,7 +26144,7 @@ var FullProcessorServiceImpl = class {
|
|
26116
26144
|
const aptCall = dataBinding.data?.aptCall;
|
26117
26145
|
if (aptCall) {
|
26118
26146
|
if (isBeforeMoveUseRawVersion && aptCall.rawTransaction) {
|
26119
|
-
aptCall.transaction =
|
26147
|
+
aptCall.transaction = getParsedData(aptCall.rawTransaction);
|
26120
26148
|
}
|
26121
26149
|
}
|
26122
26150
|
break;
|
@@ -26132,7 +26160,7 @@ var FullProcessorServiceImpl = class {
|
|
26132
26160
|
const suiEvent = dataBinding.data?.suiEvent;
|
26133
26161
|
if (suiEvent) {
|
26134
26162
|
if (isBeforeMoveUseRawVersion && suiEvent.rawTransaction) {
|
26135
|
-
const transaction =
|
26163
|
+
const transaction = getParsedData(suiEvent.rawTransaction);
|
26136
26164
|
const key = `${transaction.digest}-${dataBinding.handlerIds[0]}`;
|
26137
26165
|
if (PROCESSED_MOVE_EVENT_TX_HANDLER.has(key)) {
|
26138
26166
|
console.debug("skip binding", key);
|
@@ -26151,7 +26179,7 @@ var FullProcessorServiceImpl = class {
|
|
26151
26179
|
const suiCall = dataBinding.data?.suiCall;
|
26152
26180
|
if (suiCall) {
|
26153
26181
|
if (isBeforeMoveUseRawVersion && suiCall.rawTransaction) {
|
26154
|
-
suiCall.transaction =
|
26182
|
+
suiCall.transaction = getParsedData(suiCall.rawTransaction);
|
26155
26183
|
}
|
26156
26184
|
}
|
26157
26185
|
break;
|
@@ -26258,13 +26286,12 @@ import("node:process").then((p) => p.stdout.write(""));
|
|
26258
26286
|
// src/service-manager.ts
|
26259
26287
|
import { Piscina } from "piscina";
|
26260
26288
|
import { MessageChannel } from "node:worker_threads";
|
26261
|
-
var { process_binding_count, process_binding_time, process_binding_error } = processMetrics;
|
26262
26289
|
BigInt.prototype.toJSON = function() {
|
26263
26290
|
return this.toString();
|
26264
26291
|
};
|
26265
26292
|
var ServiceManager = class extends ProcessorServiceImpl {
|
26266
|
-
constructor(
|
26267
|
-
super(loader2, shutdownHandler);
|
26293
|
+
constructor(loader2, options2, shutdownHandler) {
|
26294
|
+
super(loader2, options2, shutdownHandler);
|
26268
26295
|
this.options = options2;
|
26269
26296
|
this.workerData.options = options2;
|
26270
26297
|
}
|
@@ -26290,62 +26317,65 @@ var ServiceManager = class extends ProcessorServiceImpl {
|
|
26290
26317
|
await this.pool?.destroy();
|
26291
26318
|
return await super.stop(request3, context2);
|
26292
26319
|
}
|
26293
|
-
|
26320
|
+
contexts = new Contexts();
|
26321
|
+
async handleRequests(requests, subject) {
|
26294
26322
|
if (!this.pool) {
|
26295
26323
|
await this.initPool();
|
26296
26324
|
}
|
26297
|
-
return this.pool.run(
|
26298
|
-
{ request: request3, workerPort: dbContext?.workerPort },
|
26299
|
-
{ transferList: dbContext?.workerPort ? [dbContext?.workerPort] : [] }
|
26300
|
-
);
|
26301
|
-
}
|
26302
|
-
contexts = new Contexts();
|
26303
|
-
async handleRequests(requests, subject) {
|
26304
26325
|
for await (const request3 of requests) {
|
26305
|
-
|
26306
|
-
|
26307
|
-
|
26308
|
-
|
26309
|
-
|
26310
|
-
|
26311
|
-
|
26312
|
-
|
26313
|
-
|
26314
|
-
|
26315
|
-
|
26316
|
-
const dbContext = this.contexts.new(request3.processId, subject);
|
26317
|
-
const start = Date.now();
|
26318
|
-
this.process(binding, dbContext).then(async (result) => {
|
26319
|
-
subject.next({
|
26320
|
-
result,
|
26321
|
-
processId: request3.processId
|
26322
|
-
});
|
26323
|
-
}).catch((e) => {
|
26324
|
-
dbContext.error(request3.processId, e);
|
26325
|
-
process_binding_error.add(1);
|
26326
|
-
}).finally(() => {
|
26327
|
-
const cost = Date.now() - start;
|
26328
|
-
process_binding_time.add(cost);
|
26329
|
-
this.contexts.delete(request3.processId);
|
26330
|
-
});
|
26331
|
-
}
|
26332
|
-
if (request3.dbResult) {
|
26333
|
-
const dbContext = this.contexts.get(request3.processId);
|
26334
|
-
try {
|
26335
|
-
dbContext?.result(request3.dbResult);
|
26336
|
-
} catch (e) {
|
26337
|
-
subject.error(new Error("db result error, process should stop"));
|
26338
|
-
}
|
26326
|
+
this.handleSingleRequest(request3, subject);
|
26327
|
+
}
|
26328
|
+
}
|
26329
|
+
async handleSingleRequest(request3, subject) {
|
26330
|
+
const processId = request3.processId;
|
26331
|
+
if (request3.binding) {
|
26332
|
+
const context2 = this.contexts.new(processId);
|
26333
|
+
context2.mainPort.on("message", (resp) => {
|
26334
|
+
subject.next(resp);
|
26335
|
+
if (resp.result) {
|
26336
|
+
this.contexts.delete(processId);
|
26339
26337
|
}
|
26340
|
-
}
|
26341
|
-
|
26338
|
+
});
|
26339
|
+
try {
|
26340
|
+
await this.pool.run(
|
26341
|
+
{ request: request3, workerPort: context2.workerPort, processId },
|
26342
|
+
{ transferList: [context2.workerPort] }
|
26343
|
+
);
|
26344
|
+
} catch (err) {
|
26345
|
+
console.error("Error processing request:", err);
|
26346
|
+
subject.error(err);
|
26347
|
+
}
|
26348
|
+
} else {
|
26349
|
+
const context2 = this.contexts.get(processId);
|
26350
|
+
if (!context2) {
|
26351
|
+
console.error("No context found for processId:", processId);
|
26352
|
+
throw new Error(`No context found for processId: ${processId}`);
|
26342
26353
|
}
|
26354
|
+
context2.sendRequest(request3);
|
26343
26355
|
}
|
26344
26356
|
}
|
26357
|
+
async process(processId, context2) {
|
26358
|
+
if (!this.pool) {
|
26359
|
+
await this.initPool();
|
26360
|
+
}
|
26361
|
+
return this.pool.run(
|
26362
|
+
{ workerPort: context2?.workerPort, processId },
|
26363
|
+
{ transferList: context2?.workerPort ? [context2?.workerPort] : [] }
|
26364
|
+
);
|
26365
|
+
}
|
26345
26366
|
async initPool() {
|
26346
26367
|
if (this.pool) {
|
26347
26368
|
await this.pool.close();
|
26348
26369
|
}
|
26370
|
+
if (this.enablePartition) {
|
26371
|
+
const concurrent = parseInt(process.env["PROCESS_CONCURRENCY"] || "0");
|
26372
|
+
if (this.options.worker < concurrent) {
|
26373
|
+
console.warn(
|
26374
|
+
`When partition is enabled, the worker count must >= 'PROCESS_CONCURRENCY', will set worker count to ${concurrent})`
|
26375
|
+
);
|
26376
|
+
this.options.worker = concurrent;
|
26377
|
+
}
|
26378
|
+
}
|
26349
26379
|
console.info("Initializing worker pool with worker count:", this.options.worker);
|
26350
26380
|
this.pool = new Piscina({
|
26351
26381
|
maxThreads: this.options.worker,
|
@@ -26366,8 +26396,12 @@ var Contexts = class {
|
|
26366
26396
|
get(processId) {
|
26367
26397
|
return this.contexts.get(processId);
|
26368
26398
|
}
|
26369
|
-
new(processId
|
26370
|
-
|
26399
|
+
new(processId) {
|
26400
|
+
let context2 = this.get(processId);
|
26401
|
+
if (context2) {
|
26402
|
+
return context2;
|
26403
|
+
}
|
26404
|
+
context2 = new ChannelContext(processId);
|
26371
26405
|
this.contexts.set(processId, context2);
|
26372
26406
|
return context2;
|
26373
26407
|
}
|
@@ -26376,21 +26410,17 @@ var Contexts = class {
|
|
26376
26410
|
context2?.close();
|
26377
26411
|
this.contexts.delete(processId);
|
26378
26412
|
}
|
26413
|
+
has(processId) {
|
26414
|
+
return this.contexts.has(processId);
|
26415
|
+
}
|
26379
26416
|
};
|
26380
|
-
var
|
26381
|
-
constructor(
|
26382
|
-
this.subject = subject;
|
26417
|
+
var ChannelContext = class {
|
26418
|
+
constructor(processId) {
|
26383
26419
|
this.processId = processId;
|
26384
|
-
this.mainPort.on("message", (req) => {
|
26385
|
-
subject.next({
|
26386
|
-
...req,
|
26387
|
-
processId
|
26388
|
-
});
|
26389
|
-
});
|
26390
26420
|
}
|
26391
26421
|
channel = new MessageChannel();
|
26392
|
-
sendRequest(request3
|
26393
|
-
|
26422
|
+
sendRequest(request3) {
|
26423
|
+
this.mainPort.postMessage(request3);
|
26394
26424
|
}
|
26395
26425
|
get workerPort() {
|
26396
26426
|
return this.channel.port2;
|
@@ -26398,24 +26428,9 @@ var ChannelStoreContext = class {
|
|
26398
26428
|
get mainPort() {
|
26399
26429
|
return this.channel.port1;
|
26400
26430
|
}
|
26401
|
-
result(dbResult) {
|
26402
|
-
this.mainPort.postMessage(dbResult);
|
26403
|
-
}
|
26404
26431
|
close() {
|
26405
26432
|
this.mainPort.close();
|
26406
26433
|
}
|
26407
|
-
error(processId, e) {
|
26408
|
-
console.error("process error", processId, e);
|
26409
|
-
const errorResult = ProcessResult.create({
|
26410
|
-
states: {
|
26411
|
-
error: e?.toString()
|
26412
|
-
}
|
26413
|
-
});
|
26414
|
-
this.subject.next({
|
26415
|
-
result: errorResult,
|
26416
|
-
processId
|
26417
|
-
});
|
26418
|
-
}
|
26419
26434
|
};
|
26420
26435
|
import("node:process").then((p) => p.stdout.write(""));
|
26421
26436
|
|
@@ -26445,7 +26460,14 @@ var optionDefinitions = [
|
|
26445
26460
|
{ name: "debug", type: Boolean, defaultValue: false },
|
26446
26461
|
{ name: "otlp-debug", type: Boolean, defaultValue: false },
|
26447
26462
|
{ name: "start-action-server", type: Boolean, defaultValue: false },
|
26448
|
-
{ name: "worker", type: Number, defaultValue: workerNum }
|
26463
|
+
{ name: "worker", type: Number, defaultValue: workerNum },
|
26464
|
+
{ name: "process-timeout", type: Number, defaultValue: 60 },
|
26465
|
+
{ name: "worker-timeout", type: Number, defaultValue: 60 },
|
26466
|
+
{
|
26467
|
+
name: "enable-partition",
|
26468
|
+
type: Boolean,
|
26469
|
+
defaultValue: process.env["SENTIO_ENABLE_BINDING_DATA_PARTITION"] === "true"
|
26470
|
+
}
|
26449
26471
|
];
|
26450
26472
|
var options = command_line_args_default(optionDefinitions, { partial: true });
|
26451
26473
|
var logLevel = process.env["LOG_LEVEL"]?.toLowerCase();
|
@@ -26472,9 +26494,9 @@ if (options["start-action-server"]) {
|
|
26472
26494
|
"grpc.default_compression_algorithm": import_grpc_js.compressionAlgorithms.gzip
|
26473
26495
|
}).use((0, import_nice_grpc_opentelemetry.openTelemetryServerMiddleware)()).use(import_nice_grpc_error_details.errorDetailsServerMiddleware);
|
26474
26496
|
if (options.worker > 1) {
|
26475
|
-
baseService = new ServiceManager(
|
26497
|
+
baseService = new ServiceManager(loader, options, server.shutdown);
|
26476
26498
|
} else {
|
26477
|
-
baseService = new ProcessorServiceImpl(loader, server.shutdown);
|
26499
|
+
baseService = new ProcessorServiceImpl(loader, options, server.shutdown);
|
26478
26500
|
}
|
26479
26501
|
const service = new FullProcessorServiceImpl(baseService);
|
26480
26502
|
server.add(ProcessorDefinition, service);
|