@coherentglobal/wasm-runner 0.1.19 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CancellationToken.js +2 -0
- package/dist/CancellationToken.js.map +1 -1
- package/dist/browser/logger.js +6 -2
- package/dist/browser/logger.js.map +1 -1
- package/dist/browser/template/worker.template.js +111 -105
- package/dist/browser/template/worker.template.js.map +1 -1
- package/dist/browser/template.js +1 -1
- package/dist/browser.d.ts +3 -3
- package/dist/browser.js +242 -257
- package/dist/browser.js.map +1 -1
- package/dist/constants.d.ts +1 -0
- package/dist/constants.js +2 -0
- package/dist/constants.js.map +1 -1
- package/dist/defaultExternalResolver.js +6 -15
- package/dist/defaultExternalResolver.js.map +1 -1
- package/dist/error.js +10 -2
- package/dist/error.js.map +1 -1
- package/dist/node/logger.d.ts +2 -1
- package/dist/node/logger.js +1 -1
- package/dist/node/logger.js.map +1 -1
- package/dist/node/logger.ts +1 -1
- package/dist/node/mockLogger.d.ts +2 -1
- package/dist/node/template/main.template.ejs +54 -28
- package/dist/node/threads/mockWorkerThread.d.ts +1 -0
- package/dist/node/threads/mockWorkerThread.js +10 -3
- package/dist/node/threads/mockWorkerThread.js.map +1 -1
- package/dist/node/threads/workerPool.d.ts +2 -1
- package/dist/node/threads/workerPool.js +7 -6
- package/dist/node/threads/workerPool.js.map +1 -1
- package/dist/node/threads/workerPool.ts +10 -7
- package/dist/node/threads/workerThread.d.ts +1 -2
- package/dist/node/threads/workerThread.js +35 -42
- package/dist/node/threads/workerThread.js.map +1 -1
- package/dist/node/threads/workerThread.ts +8 -12
- package/dist/node.d.ts +13 -3
- package/dist/node.js +523 -459
- package/dist/node.js.map +1 -1
- package/dist/responseTimeMetric.d.ts +7 -3
- package/dist/responseTimeMetric.js +131 -28
- package/dist/responseTimeMetric.js.map +1 -1
- package/dist/serializer/columnarSerializer.d.ts +11 -3
- package/dist/serializer/columnarSerializer.js +49 -29
- package/dist/serializer/columnarSerializer.js.map +1 -1
- package/dist/types.d.ts +3 -1
- package/dist/types.js.map +1 -1
- package/dist/utils.d.ts +3 -2
- package/dist/utils.js +38 -48
- package/dist/utils.js.map +1 -1
- package/package.json +71 -60
package/dist/node.js
CHANGED
|
@@ -15,35 +15,30 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
15
15
|
}) : function(o, v) {
|
|
16
16
|
o["default"] = v;
|
|
17
17
|
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
return
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
};
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
34
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
35
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
36
37
|
};
|
|
37
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
38
39
|
exports.WasmRunner = exports.ModelExecuteCancelled = exports.CancellationToken = exports.ColumnarSerializer = void 0;
|
|
39
|
-
/* eslint-disable consistent-return */
|
|
40
|
-
/* eslint-disable no-shadow */
|
|
41
|
-
/* eslint-disable camelcase */
|
|
42
|
-
/* eslint-disable no-param-reassign */
|
|
43
|
-
/* eslint-disable no-underscore-dangle */
|
|
44
40
|
const tmp_1 = __importDefault(require("tmp"));
|
|
45
41
|
const cuid2_1 = require("@paralleldrive/cuid2");
|
|
46
|
-
const logger_1 = __importDefault(require("./node/logger"));
|
|
47
42
|
const unzipper_1 = require("unzipper");
|
|
48
43
|
const ejs_1 = __importDefault(require("ejs"));
|
|
49
44
|
const stream_1 = __importStar(require("stream"));
|
|
@@ -52,13 +47,15 @@ const path_1 = __importDefault(require("path"));
|
|
|
52
47
|
const util_1 = require("util");
|
|
53
48
|
const got_1 = __importDefault(require("got"));
|
|
54
49
|
const semver_1 = __importDefault(require("semver"));
|
|
50
|
+
const events_1 = __importDefault(require("events"));
|
|
51
|
+
const node_os_1 = require("node:os");
|
|
55
52
|
const workerPool_1 = __importDefault(require("./node/threads/workerPool"));
|
|
56
53
|
const utils = __importStar(require("./utils"));
|
|
57
54
|
const types_1 = require("./types");
|
|
58
55
|
const error_1 = __importDefault(require("./error"));
|
|
59
|
-
const node_os_1 = require("node:os");
|
|
60
56
|
const constants_1 = require("./constants");
|
|
61
57
|
const responseTimeMetric_1 = __importDefault(require("./responseTimeMetric"));
|
|
58
|
+
const logger_1 = __importDefault(require("./node/logger"));
|
|
62
59
|
/* istanbul ignore next */
|
|
63
60
|
var columnarSerializer_1 = require("./serializer/columnarSerializer");
|
|
64
61
|
Object.defineProperty(exports, "ColumnarSerializer", { enumerable: true, get: function () { return columnarSerializer_1.ColumnarSerializer; } });
|
|
@@ -66,17 +63,16 @@ var CancellationToken_1 = require("./CancellationToken");
|
|
|
66
63
|
Object.defineProperty(exports, "CancellationToken", { enumerable: true, get: function () { return CancellationToken_1.CancellationToken; } });
|
|
67
64
|
var error_2 = require("./error");
|
|
68
65
|
Object.defineProperty(exports, "ModelExecuteCancelled", { enumerable: true, get: function () { return error_2.ModelExecuteCancelled; } });
|
|
66
|
+
tmp_1.default.setGracefulCleanup();
|
|
69
67
|
const platformUsed = (0, node_os_1.platform)();
|
|
70
68
|
const pipeline = (0, util_1.promisify)(stream_1.default.pipeline);
|
|
71
|
-
function replaceRegexInFile(file, search, replace) {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return true;
|
|
79
|
-
});
|
|
69
|
+
async function replaceRegexInFile(file, search, replace) {
|
|
70
|
+
const contents = await fs_1.default.promises.readFile(file, "utf8");
|
|
71
|
+
const replaced_contents = contents.replace(search, replace);
|
|
72
|
+
const tmpfile = `${file}-${(0, cuid2_1.createId)()}.jstmpreplace`;
|
|
73
|
+
await fs_1.default.promises.writeFile(tmpfile, replaced_contents, "utf8");
|
|
74
|
+
await fs_1.default.promises.rename(tmpfile, file);
|
|
75
|
+
return true;
|
|
80
76
|
}
|
|
81
77
|
/* istanbul ignore next */
|
|
82
78
|
const toWindowsPath = (fileLocation) => fileLocation.split(path_1.default.sep).join("//");
|
|
@@ -97,7 +93,6 @@ const unzipFiles = (id, url, workerFolder) => {
|
|
|
97
93
|
let metadata;
|
|
98
94
|
let formspec;
|
|
99
95
|
let compilerLog;
|
|
100
|
-
// @ts-ignore
|
|
101
96
|
const stream = fs_1.default.createReadStream(url).pipe((0, unzipper_1.Parse)());
|
|
102
97
|
return new Promise((resolve, reject) => {
|
|
103
98
|
stream.on("entry", (entry) => {
|
|
@@ -149,48 +144,59 @@ const template = ejs_1.default.compile(fs_1.default.readFileSync(path_1.default.
|
|
|
149
144
|
encoding: "utf8",
|
|
150
145
|
flag: "r",
|
|
151
146
|
}), { async: false });
|
|
152
|
-
const delay = (time) => {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
});
|
|
156
|
-
};
|
|
147
|
+
const delay = (time) => new Promise((res) => {
|
|
148
|
+
setTimeout(res, time);
|
|
149
|
+
});
|
|
157
150
|
/**
|
|
158
151
|
* WasmRunner class, responsible for managing model and it's lifecycle
|
|
159
152
|
*
|
|
160
153
|
* @class
|
|
161
154
|
* @classdesc WasmRunner class, responsible for managing model and it's lifecycle
|
|
162
155
|
*/
|
|
163
|
-
|
|
164
|
-
|
|
156
|
+
class WasmRunner extends events_1.default {
|
|
157
|
+
_tempWorkerFolder;
|
|
158
|
+
_tempModelFolder;
|
|
159
|
+
license;
|
|
160
|
+
models;
|
|
161
|
+
externalResolverModule;
|
|
162
|
+
initializingModels;
|
|
163
|
+
options;
|
|
164
|
+
safeCompiler;
|
|
165
|
+
logger;
|
|
165
166
|
/**
|
|
166
167
|
* Create new Wasm Runner instance
|
|
167
168
|
*
|
|
168
169
|
* @param {RunnerConfig} wasmRunnerConfig
|
|
169
170
|
*/
|
|
170
171
|
constructor(wasmRunnerConfig, externalResolverModule = "", options = {}, license = "") {
|
|
172
|
+
super();
|
|
171
173
|
let baseTmpDir = null;
|
|
172
174
|
if (process.env.DEV_DEBUG_TMP_PATH) {
|
|
173
175
|
baseTmpDir = path_1.default.join(process.cwd(), process.env.DEV_DEBUG_TMP_PATH);
|
|
174
176
|
}
|
|
177
|
+
if (options.tmpDir) {
|
|
178
|
+
baseTmpDir = options.tmpDir;
|
|
179
|
+
}
|
|
175
180
|
this._tempWorkerFolder = tmp_1.default.dirSync({
|
|
176
181
|
prefix: "wasmserver_worker",
|
|
177
|
-
// @ts-ignore
|
|
178
182
|
mode: 0o750,
|
|
179
183
|
tmpdir: baseTmpDir,
|
|
180
184
|
});
|
|
181
185
|
this._tempModelFolder = tmp_1.default.dirSync({
|
|
182
186
|
prefix: "wasmserver_model",
|
|
183
|
-
// @ts-ignore
|
|
184
187
|
mode: 0o750,
|
|
185
188
|
tmpdir: baseTmpDir,
|
|
186
189
|
});
|
|
187
190
|
this.license = license;
|
|
188
191
|
this.initializingModels = [];
|
|
192
|
+
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
189
193
|
this.externalResolverModule = externalResolverModule
|
|
190
194
|
? require(useCorrectPath(require.resolve(externalResolverModule)))
|
|
191
195
|
: require(require.resolve("./defaultExternalResolver"));
|
|
196
|
+
/* eslint-enable @typescript-eslint/no-require-imports */
|
|
192
197
|
this.models = [];
|
|
193
198
|
this.options = options;
|
|
199
|
+
this.logger = this.options.logger || logger_1.default;
|
|
194
200
|
if (!utils.isEmpty(wasmRunnerConfig)) {
|
|
195
201
|
if (Array.isArray(wasmRunnerConfig)) {
|
|
196
202
|
this.models = wasmRunnerConfig.map((wsconfig) => ({
|
|
@@ -215,247 +221,292 @@ class WasmRunner {
|
|
|
215
221
|
* @function initialize
|
|
216
222
|
*/
|
|
217
223
|
/* istanbul ignore next */
|
|
218
|
-
initialize() {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
yield Promise.all(this.models.map((m) => __awaiter(this, void 0, void 0, function* () {
|
|
224
|
-
return this._initializeModelInstance(m);
|
|
225
|
-
})));
|
|
226
|
-
});
|
|
224
|
+
async initialize(license = "") {
|
|
225
|
+
if (this.options.compatibilityCheckEnabled) {
|
|
226
|
+
this.safeCompiler = await this.getSafeCompilerVersion(this.options.runnerVersion);
|
|
227
|
+
}
|
|
228
|
+
await Promise.all(this.models.map(async (m) => this._initializeModelInstance(m)));
|
|
227
229
|
}
|
|
228
230
|
/* istanbul ignore next */
|
|
229
|
-
_createModelInstance(
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
231
|
+
async _createModelInstance(id, url, workerFolder, size = 1) {
|
|
232
|
+
const workerSize = size || 1;
|
|
233
|
+
this.logger.debug({
|
|
234
|
+
TimeStamp: Date.now(),
|
|
235
|
+
EventType: "Runner._createModelInstance",
|
|
236
|
+
size: workerSize,
|
|
237
|
+
ModelId: id,
|
|
238
|
+
});
|
|
239
|
+
const tempFolder = path_1.default.join(workerFolder, `${id}-${Date.now()}`);
|
|
240
|
+
try {
|
|
241
|
+
await fs_1.default.promises.mkdir(tempFolder, { mode: 0o750 });
|
|
242
|
+
}
|
|
243
|
+
catch (_err) {
|
|
244
|
+
this.logger.info(`${tempFolder} existed`);
|
|
245
|
+
}
|
|
246
|
+
const modelPack = await unzipFiles(id, url, tempFolder);
|
|
247
|
+
this.logger.debug({
|
|
248
|
+
TimeStamp: Date.now(),
|
|
249
|
+
EventType: "Runner.UnZipFiles",
|
|
250
|
+
ModelId: id,
|
|
251
|
+
});
|
|
252
|
+
const modelName = id;
|
|
253
|
+
await replaceRegexInFile(modelPack.js, `wasmBinaryFile="${path_1.default.basename(modelPack.wasm)}"`, `scriptDirectory="";wasmBinaryFile="${modelPack.wasm}"`);
|
|
254
|
+
const replaceIsFileURLFunc = "function isFileURI(filename) { if(typeof process !== undefined && process.versions && process.versions.node) { return true; }";
|
|
255
|
+
await replaceRegexInFile(modelPack.js, "function isFileURI(filename) {", replaceIsFileURLFunc);
|
|
256
|
+
await replaceRegexInFile(modelPack.js, "function isFileURI(filename){", replaceIsFileURLFunc);
|
|
257
|
+
// Remove `new URL()` since `isFileURI` will always be true for Node
|
|
258
|
+
await replaceRegexInFile(modelPack.js, /\s*isFileURI\s*\(\s*filename\s*\)\s*\?\s*new\s*URL\s*\(\s*filename\s*\)\s*:\s*/g, "");
|
|
259
|
+
if (modelPack.data) {
|
|
260
|
+
await replaceRegexInFile(modelPack.js, `"${path_1.default.basename(modelPack.data)}"`, `"${modelPack.data}"`);
|
|
261
|
+
await replaceRegexInFile(modelPack.js, `"${path_1.default.basename(modelPack.data)}"`, `"${modelPack.data}"`);
|
|
262
|
+
await replaceRegexInFile(modelPack.js, `"${path_1.default.basename(modelPack.data)}"`, `"${modelPack.data}"`);
|
|
263
|
+
await replaceRegexInFile(modelPack.js, `"${path_1.default.basename(modelPack.data)}"`, `"${modelPack.data}"`);
|
|
264
|
+
}
|
|
265
|
+
// Prepare File Paths
|
|
266
|
+
// const externalResolverPath = useCorrectPath(this.externalResolverModule);
|
|
267
|
+
const loggerPath = useCorrectPath(process.env.NODE_ENV === "test"
|
|
268
|
+
? require.resolve("./node/mockLogger.js")
|
|
269
|
+
: require.resolve("./node/logger"));
|
|
270
|
+
const workerThread = useCorrectPath(process.env.NODE_ENV === "test"
|
|
271
|
+
? require.resolve("./node/threads/mockWorkerThread.js")
|
|
272
|
+
: require.resolve("./node/threads/workerThread"));
|
|
273
|
+
let metadata = "undefined";
|
|
274
|
+
if (!utils.isEmpty(modelPack.metadata)) {
|
|
275
|
+
const metadataPath = useCorrectPath(require.resolve(modelPack.metadata));
|
|
276
|
+
metadata = `require("${metadataPath}")`;
|
|
277
|
+
}
|
|
278
|
+
let formspec = "undefined";
|
|
279
|
+
if (!utils.isEmpty(modelPack.formspec)) {
|
|
280
|
+
const formSpecPath = useCorrectPath(require.resolve(modelPack.formspec));
|
|
281
|
+
formspec = `require("${formSpecPath}")`;
|
|
282
|
+
}
|
|
283
|
+
let compilerVersion = "undefined";
|
|
284
|
+
let compatibilityStatus = "undefined";
|
|
285
|
+
compilerVersion = !utils.isEmpty(modelPack.compilerLog)
|
|
286
|
+
? await this.getNeuronCompilerVersion(`"${modelPack.compilerLog}"`)
|
|
287
|
+
: undefined;
|
|
288
|
+
if (this.options.compatibilityCheckEnabled) {
|
|
289
|
+
compatibilityStatus =
|
|
290
|
+
await this.checkCompilerVersionCompatibility(compilerVersion);
|
|
291
|
+
}
|
|
292
|
+
const fileContent = template({
|
|
293
|
+
runtime: modelPack.js.replace(".js", ""),
|
|
294
|
+
model: modelPack.wasm,
|
|
295
|
+
metadata,
|
|
296
|
+
formspec,
|
|
297
|
+
modelName,
|
|
298
|
+
modules: {
|
|
299
|
+
logger: loggerPath,
|
|
300
|
+
workerThread: workerThread,
|
|
301
|
+
},
|
|
302
|
+
});
|
|
303
|
+
const workerPath = path_1.default.join(tempFolder, `worker-${id}.js`);
|
|
304
|
+
await fs_1.default.promises.writeFile(workerPath, fileContent);
|
|
305
|
+
this.logger.debug({
|
|
306
|
+
TimeStamp: Date.now(),
|
|
307
|
+
EventType: "Runner.CreateTemplates",
|
|
308
|
+
ModelId: id,
|
|
309
|
+
});
|
|
310
|
+
let workerPool;
|
|
311
|
+
let reportedStats = 0;
|
|
312
|
+
/**
|
|
313
|
+
* Processes the requests from Worker thread. Interprets the request
|
|
314
|
+
* `type` and implement necessary logic and returns the result.
|
|
315
|
+
*
|
|
316
|
+
* @param {Object} requestData
|
|
317
|
+
* Event details from Worker thread.
|
|
318
|
+
*/
|
|
319
|
+
const processWorkerThreadRequest = async (requestData) => {
|
|
320
|
+
const { type, payload, context, threadId } = requestData;
|
|
321
|
+
// Worker request of model execution on main thread.
|
|
322
|
+
if (type === constants_1.THREAD_EVENT_TYPES.REQUEST_EXECUTE) {
|
|
323
|
+
try {
|
|
324
|
+
// Callback the resolver function
|
|
325
|
+
const result = await this.externalResolverModule.sparkService(payload, context);
|
|
326
|
+
// Send back the response to worker thread
|
|
327
|
+
workerPool.postMessage({
|
|
328
|
+
payload: result,
|
|
329
|
+
type: constants_1.THREAD_EVENT_TYPES.RESPONSE,
|
|
330
|
+
threadId,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
catch (e) {
|
|
334
|
+
// Send back the error to worker thread
|
|
335
|
+
workerPool.postMessage({
|
|
336
|
+
payload: e,
|
|
337
|
+
type: constants_1.THREAD_EVENT_TYPES.ERROR,
|
|
338
|
+
threadId,
|
|
339
|
+
});
|
|
340
|
+
}
|
|
289
341
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
const processWorkerThreadRequest = (requestData) => __awaiter(this, void 0, void 0, function* () {
|
|
317
|
-
const { type, payload, context, threadId } = requestData;
|
|
318
|
-
// Worker request of model execution on main thread.
|
|
319
|
-
if (type === constants_1.THREAD_EVENT_TYPES.REQUEST_EXECUTE) {
|
|
320
|
-
try {
|
|
321
|
-
// Callback the resolver function
|
|
322
|
-
const result = yield this.externalResolverModule.sparkService(payload, context);
|
|
323
|
-
// Send back the response to worker thread
|
|
324
|
-
workerPool.postMessage({
|
|
325
|
-
payload: result,
|
|
326
|
-
type: constants_1.THREAD_EVENT_TYPES.RESPONSE,
|
|
327
|
-
threadId,
|
|
328
|
-
});
|
|
342
|
+
if (type === constants_1.THREAD_EVENT_TYPES.STATS_REPORT) {
|
|
343
|
+
const { payload, threadId } = requestData;
|
|
344
|
+
const worker = this.models.find((m) => m.id === workerPool.id);
|
|
345
|
+
if (!worker) {
|
|
346
|
+
// Worker has been removed and this is a stale report
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
// Initialize stats object if it doesn't exist
|
|
350
|
+
if (!worker.stats) {
|
|
351
|
+
worker.stats = {};
|
|
352
|
+
}
|
|
353
|
+
// Initialize stats for threadId if it doesn't exist
|
|
354
|
+
if (!worker.stats[threadId]) {
|
|
355
|
+
worker.stats[threadId] = {
|
|
356
|
+
init_memory: 0,
|
|
357
|
+
init_time_ms: 0,
|
|
358
|
+
ready_ts: Date.now(),
|
|
359
|
+
peak_memory: 0,
|
|
360
|
+
current_memory: 0,
|
|
361
|
+
last_execute_consume_memory: 0,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
if (worker && worker.stats && worker.stats[threadId]) {
|
|
365
|
+
worker.stats[threadId].current_memory = payload.memoryUsage;
|
|
366
|
+
if (payload.memoryUsage > worker.stats[threadId].peak_memory) {
|
|
367
|
+
worker.stats[threadId].peak_memory = payload.memoryUsage;
|
|
329
368
|
}
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
369
|
+
reportedStats += 1;
|
|
370
|
+
if (reportedStats >= workerSize) {
|
|
371
|
+
reportedStats = 0;
|
|
372
|
+
setTimeout(() => {
|
|
373
|
+
if (!this.isExist(workerPool.id)) {
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
this.emit("worker_report_stats", {
|
|
377
|
+
id: workerPool.id,
|
|
378
|
+
thread_stats: worker.stats,
|
|
379
|
+
current_memory: Object.values(worker.stats)
|
|
380
|
+
.map((thread) => {
|
|
381
|
+
if (utils.isNumber(thread.current_memory)) {
|
|
382
|
+
return thread.current_memory;
|
|
383
|
+
}
|
|
384
|
+
return 0;
|
|
385
|
+
})
|
|
386
|
+
.reduce((sum, memory) => sum + memory, 0),
|
|
387
|
+
});
|
|
388
|
+
}, 10);
|
|
337
389
|
}
|
|
338
390
|
}
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
391
|
+
}
|
|
392
|
+
};
|
|
393
|
+
const workerName = path_1.default.basename(modelPack.js, path_1.default.extname(modelPack.js)) || "";
|
|
394
|
+
workerPool = new workerPool_1.default(id, workerSize, workerPath, {
|
|
395
|
+
errorHandler: (e) => logger_1.default &&
|
|
396
|
+
logger_1.default.error({
|
|
397
|
+
EventType: "Worker.Error",
|
|
398
|
+
Error: e,
|
|
399
|
+
ModelId: id,
|
|
400
|
+
TextMessage: e.message,
|
|
401
|
+
}, "Worker Error"),
|
|
402
|
+
onlineHandler: () => logger_1.default &&
|
|
403
|
+
logger_1.default.info({ EventType: "Worker.Online", ModelId: id }, `Worker ${id} online`),
|
|
404
|
+
// Receive messages from worker here
|
|
405
|
+
messageHandler: processWorkerThreadRequest,
|
|
406
|
+
workerOptions: {
|
|
407
|
+
name: workerName,
|
|
408
|
+
},
|
|
409
|
+
});
|
|
410
|
+
this.logger.debug({
|
|
411
|
+
TimeStamp: Date.now(),
|
|
412
|
+
EventType: "Runner.ModelInitialize",
|
|
413
|
+
ModelId: id,
|
|
414
|
+
});
|
|
415
|
+
return new Promise((resolve, reject) => {
|
|
416
|
+
let limit = 0;
|
|
417
|
+
let readyCount = workerSize;
|
|
418
|
+
const stats = {};
|
|
419
|
+
const checkModelStatus = async () => {
|
|
420
|
+
const readyRes = await workerPool.execute("isReady");
|
|
421
|
+
if (readyRes && !stats[readyRes.threadId]) {
|
|
422
|
+
stats[readyRes.threadId] = {
|
|
423
|
+
init_memory: readyRes.memoryUsage,
|
|
424
|
+
init_time_ms: readyRes.time,
|
|
425
|
+
ready_ts: readyRes.payload?.readyTs || readyRes.readyTs,
|
|
426
|
+
peak_memory: readyRes.memoryUsage,
|
|
427
|
+
last_execute_consume_memory: readyRes.memoryUsage,
|
|
428
|
+
};
|
|
429
|
+
readyCount -= 1;
|
|
430
|
+
this.logger.debug({
|
|
431
|
+
TimeStamp: Date.now(),
|
|
432
|
+
EventType: "Runner.ModelInitialize.Completed",
|
|
433
|
+
ModelId: id,
|
|
434
|
+
});
|
|
435
|
+
if (readyCount === 0) {
|
|
436
|
+
resolve({
|
|
437
|
+
instance: workerPool,
|
|
438
|
+
compilerVersion,
|
|
439
|
+
compatibilityStatus,
|
|
440
|
+
size: workerSize,
|
|
441
|
+
stats: stats,
|
|
377
442
|
});
|
|
378
|
-
if (readyCount === 0) {
|
|
379
|
-
resolve({
|
|
380
|
-
instance: workerPool,
|
|
381
|
-
compilerVersion,
|
|
382
|
-
compatibilityStatus,
|
|
383
|
-
size: workerSize,
|
|
384
|
-
stats: stats
|
|
385
|
-
});
|
|
386
|
-
}
|
|
387
|
-
else {
|
|
388
|
-
setTimeout(checkModelStatus, 1);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
else if (limit >= 2000) {
|
|
392
|
-
// 1 second
|
|
393
|
-
reject(new Error("Model initialize issue"));
|
|
394
443
|
}
|
|
395
444
|
else {
|
|
396
|
-
setTimeout(checkModelStatus,
|
|
397
|
-
// eslint-disable-next-line no-plusplus
|
|
398
|
-
limit++;
|
|
445
|
+
setTimeout(checkModelStatus, 1);
|
|
399
446
|
}
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
|
|
447
|
+
}
|
|
448
|
+
else if (limit >= 2000) {
|
|
449
|
+
// 1 second
|
|
450
|
+
reject(new Error("Model initialize issue"));
|
|
451
|
+
}
|
|
452
|
+
else {
|
|
453
|
+
setTimeout(checkModelStatus, 15);
|
|
454
|
+
limit++;
|
|
455
|
+
}
|
|
456
|
+
};
|
|
457
|
+
setTimeout(checkModelStatus, 15);
|
|
403
458
|
});
|
|
404
459
|
}
|
|
405
460
|
/* istanbul ignore next */
|
|
406
|
-
_initializeModelInstance(m) {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
}
|
|
442
|
-
});
|
|
461
|
+
async _initializeModelInstance(m) {
|
|
462
|
+
if (utils.isHttpURL(m.url)) {
|
|
463
|
+
const { instance, compilerVersion, compatibilityStatus, stats } = await this._createModelInstanceFromURL(m.id, m.url, this._tempWorkerFolder.name, this._tempModelFolder.name, m.size);
|
|
464
|
+
m.ready = true;
|
|
465
|
+
m.worker = instance;
|
|
466
|
+
m.compatibility_status = compatibilityStatus;
|
|
467
|
+
m.compilerVersion = compilerVersion;
|
|
468
|
+
m.stats = stats;
|
|
469
|
+
m.responseTimeMetrics = new responseTimeMetric_1.default();
|
|
470
|
+
return Promise.resolve();
|
|
471
|
+
}
|
|
472
|
+
if (await utils.isPath(m.url)) {
|
|
473
|
+
const { instance, compilerVersion, compatibilityStatus, stats } = await this._createModelInstance(m.id, m.url, this._tempWorkerFolder.name, m.size);
|
|
474
|
+
m.ready = true;
|
|
475
|
+
m.worker = instance;
|
|
476
|
+
m.compatibility_status = compatibilityStatus;
|
|
477
|
+
m.compilerVersion = compilerVersion;
|
|
478
|
+
m.stats = stats;
|
|
479
|
+
m.responseTimeMetrics = new responseTimeMetric_1.default();
|
|
480
|
+
return Promise.resolve();
|
|
481
|
+
}
|
|
482
|
+
if (Buffer.isBuffer(m.url)) {
|
|
483
|
+
const model = stream_1.Readable.from(m.url);
|
|
484
|
+
const modelPath = path_1.default.join(this._tempWorkerFolder.name, `${m.id}.zip`);
|
|
485
|
+
const fileWriterStream = (0, fs_1.createWriteStream)(modelPath);
|
|
486
|
+
await pipeline(model, fileWriterStream);
|
|
487
|
+
const { instance, compilerVersion, compatibilityStatus, stats } = await this._createModelInstance(m.id, modelPath, this._tempWorkerFolder.name, m.size);
|
|
488
|
+
m.ready = true;
|
|
489
|
+
m.worker = instance;
|
|
490
|
+
m.compatibility_status = compatibilityStatus;
|
|
491
|
+
m.compilerVersion = compilerVersion;
|
|
492
|
+
m.stats = stats;
|
|
493
|
+
m.responseTimeMetrics = new responseTimeMetric_1.default();
|
|
494
|
+
return Promise.resolve();
|
|
495
|
+
}
|
|
443
496
|
}
|
|
444
497
|
/* istanbul ignore next */
|
|
445
|
-
_createModelInstanceFromURL(id, url, workerFolder, modelFolder, size) {
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
}
|
|
458
|
-
});
|
|
498
|
+
async _createModelInstanceFromURL(id, url, workerFolder, modelFolder, size) {
|
|
499
|
+
try {
|
|
500
|
+
const modelPath = path_1.default.join(modelFolder, `${id}.zip`);
|
|
501
|
+
const downloadStream = got_1.default.stream(url);
|
|
502
|
+
const fileWriterStream = (0, fs_1.createWriteStream)(modelPath);
|
|
503
|
+
await pipeline(downloadStream, fileWriterStream);
|
|
504
|
+
const instance = await this._createModelInstance(id, modelPath, workerFolder, size);
|
|
505
|
+
return instance;
|
|
506
|
+
}
|
|
507
|
+
catch (err) {
|
|
508
|
+
throw new error_1.default.ModelError(`Error createModelInstanceFromURL: id:${id}, url:${url}, error:${err.message}`);
|
|
509
|
+
}
|
|
459
510
|
}
|
|
460
511
|
/**
|
|
461
512
|
* Append and initialize model in preparation for the `execute()`.
|
|
@@ -465,49 +516,44 @@ class WasmRunner {
|
|
|
465
516
|
*
|
|
466
517
|
* @returns {Promise<void>}
|
|
467
518
|
*/
|
|
468
|
-
append(modelConfig) {
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
519
|
+
async append(modelConfig) {
|
|
520
|
+
if (this.options.compatibilityCheckEnabled) {
|
|
521
|
+
this.safeCompiler = await this.getSafeCompilerVersion(this.options.runnerVersion);
|
|
522
|
+
}
|
|
523
|
+
if (!this.isExist(modelConfig.id)) {
|
|
524
|
+
const model = {
|
|
525
|
+
id: modelConfig.id,
|
|
526
|
+
url: modelConfig.url,
|
|
527
|
+
size: modelConfig.size,
|
|
528
|
+
busy: 0,
|
|
529
|
+
};
|
|
473
530
|
if (!this.isExist(modelConfig.id)) {
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
if (!this.isExist(modelConfig.id)) {
|
|
481
|
-
if (this.initializingModels.findIndex((im) => im === modelConfig.id) ===
|
|
482
|
-
-1) {
|
|
483
|
-
try {
|
|
484
|
-
this.initializingModels.push(modelConfig.id);
|
|
485
|
-
yield this._initializeModelInstance(model);
|
|
486
|
-
this.models.push(model);
|
|
487
|
-
}
|
|
488
|
-
catch (e) {
|
|
489
|
-
throw e;
|
|
490
|
-
}
|
|
491
|
-
finally {
|
|
492
|
-
this.initializingModels = this.initializingModels.filter((im) => im !== modelConfig.id);
|
|
493
|
-
}
|
|
531
|
+
if (this.initializingModels.findIndex((im) => im === modelConfig.id) ===
|
|
532
|
+
-1) {
|
|
533
|
+
try {
|
|
534
|
+
this.initializingModels.push(modelConfig.id);
|
|
535
|
+
await this._initializeModelInstance(model);
|
|
536
|
+
this.models.push(model);
|
|
494
537
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
538
|
+
finally {
|
|
539
|
+
this.initializingModels = this.initializingModels.filter((im) => im !== modelConfig.id);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
else {
|
|
543
|
+
const maxRetry = 600;
|
|
544
|
+
let retry = 0;
|
|
545
|
+
while (this.initializingModels.findIndex((im) => im === modelConfig.id) >
|
|
546
|
+
-1 &&
|
|
547
|
+
retry < maxRetry) {
|
|
548
|
+
await delay(100);
|
|
549
|
+
retry++;
|
|
550
|
+
}
|
|
551
|
+
if (retry >= maxRetry || !this.isExist(modelConfig.id)) {
|
|
552
|
+
throw new error_1.default.ModelInitializationError(modelConfig.id);
|
|
507
553
|
}
|
|
508
554
|
}
|
|
509
555
|
}
|
|
510
|
-
}
|
|
556
|
+
}
|
|
511
557
|
}
|
|
512
558
|
/**
|
|
513
559
|
* Remove model from the list of initialized model useful for GC.
|
|
@@ -517,21 +563,20 @@ class WasmRunner {
|
|
|
517
563
|
*
|
|
518
564
|
* @returns {Promise<void>}
|
|
519
565
|
*/
|
|
520
|
-
remove(id) {
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
566
|
+
async remove(id) {
|
|
567
|
+
if (this.isExist(id)) {
|
|
568
|
+
const model = this.models.find((m) => m.id === id);
|
|
569
|
+
this.models = this.models.filter((m) => m.id !== id);
|
|
570
|
+
setImmediate(async () => {
|
|
571
|
+
await model.worker.execute("destroy");
|
|
525
572
|
model.worker.destroy();
|
|
526
573
|
delete model.worker;
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
});
|
|
574
|
+
});
|
|
575
|
+
}
|
|
530
576
|
}
|
|
531
577
|
/* istanbul ignore next */
|
|
532
578
|
_parseError(rawResponse) {
|
|
533
|
-
|
|
534
|
-
let errors = (_a = rawResponse.response_data) === null || _a === void 0 ? void 0 : _a.errors.reduce((prev, curr) => {
|
|
579
|
+
let errors = rawResponse.response_data?.errors.reduce((prev, curr) => {
|
|
535
580
|
const { source_path } = curr;
|
|
536
581
|
if (!source_path) {
|
|
537
582
|
return prev;
|
|
@@ -588,7 +633,7 @@ class WasmRunner {
|
|
|
588
633
|
}
|
|
589
634
|
getModelCompilerVersion(id) {
|
|
590
635
|
const model = this._getModel(id);
|
|
591
|
-
if (model.compilerVersion && model.compilerVersion !==
|
|
636
|
+
if (model.compilerVersion && model.compilerVersion !== "undefined") {
|
|
592
637
|
return model.compilerVersion;
|
|
593
638
|
}
|
|
594
639
|
return undefined;
|
|
@@ -602,7 +647,7 @@ class WasmRunner {
|
|
|
602
647
|
try {
|
|
603
648
|
return !!this._getModel(id);
|
|
604
649
|
}
|
|
605
|
-
catch (
|
|
650
|
+
catch (_err) {
|
|
606
651
|
return false;
|
|
607
652
|
}
|
|
608
653
|
}
|
|
@@ -614,7 +659,7 @@ class WasmRunner {
|
|
|
614
659
|
case types_1.Compatibility.INCOMPATIBLE:
|
|
615
660
|
throw new error_1.default.ModelExecuteError(this.getIncompatibleMessage(model), model.id, input);
|
|
616
661
|
case types_1.Compatibility.PARTIAL:
|
|
617
|
-
|
|
662
|
+
this.logger.warn({
|
|
618
663
|
TimeStamp: Date.now(),
|
|
619
664
|
EventType: "Runner.Execute.Compatibility.Warning",
|
|
620
665
|
TextMessage: this.getPartiallyCompatibleMessage(model),
|
|
@@ -623,30 +668,42 @@ class WasmRunner {
|
|
|
623
668
|
return true;
|
|
624
669
|
case types_1.Compatibility.COMPATIBLE:
|
|
625
670
|
return true;
|
|
671
|
+
default:
|
|
672
|
+
return true;
|
|
626
673
|
}
|
|
627
|
-
return true;
|
|
628
674
|
}
|
|
629
675
|
getModelsStats() {
|
|
630
|
-
return this.models.map(model => {
|
|
631
|
-
const stats =
|
|
632
|
-
Object.keys(stats).forEach(threadId => {
|
|
633
|
-
stats[threadId] =
|
|
634
|
-
Object.keys(stats[threadId]).forEach(k => {
|
|
635
|
-
if (k.indexOf(
|
|
676
|
+
return this.models.map((model) => {
|
|
677
|
+
const stats = { ...model.stats };
|
|
678
|
+
Object.keys(stats).forEach((threadId) => {
|
|
679
|
+
stats[threadId] = { ...stats[threadId] };
|
|
680
|
+
Object.keys(stats[threadId]).forEach((k) => {
|
|
681
|
+
if (k.indexOf("_memory") > -1) {
|
|
636
682
|
stats[threadId][`${k}_mb`] = utils.formatMemory(stats[threadId][k]);
|
|
637
683
|
delete stats[threadId][k];
|
|
638
684
|
}
|
|
639
|
-
if (k.indexOf(
|
|
640
|
-
stats[threadId].uptime_ms = Date.now() - stats[threadId]
|
|
685
|
+
if (k.indexOf("ready_ts") > -1) {
|
|
686
|
+
stats[threadId].uptime_ms = Date.now() - stats[threadId].ready_ts;
|
|
641
687
|
delete stats[threadId][k];
|
|
642
688
|
}
|
|
643
689
|
});
|
|
644
690
|
});
|
|
645
691
|
return {
|
|
646
|
-
|
|
692
|
+
thread_stats: {
|
|
693
|
+
...stats,
|
|
694
|
+
},
|
|
695
|
+
memory_usage_mb: Object.values(stats)
|
|
696
|
+
.map((thread) => thread.current_memory_mb || 0)
|
|
697
|
+
.reduce((sum, memory) => sum + memory, 0),
|
|
698
|
+
uptime_ms: Math.min(...Object.keys(stats).map((k) => stats[k].uptime_ms)),
|
|
699
|
+
min_time_ms: model.responseTimeMetrics.min,
|
|
700
|
+
mean_time_ms: model.responseTimeMetrics.mean,
|
|
701
|
+
p95_time_ms: model.responseTimeMetrics.p95,
|
|
702
|
+
p99_time_ms: model.responseTimeMetrics.p99,
|
|
703
|
+
max_time_ms: model.responseTimeMetrics.max,
|
|
647
704
|
busy: model.busy,
|
|
648
705
|
size: model.size,
|
|
649
|
-
id: model.id
|
|
706
|
+
id: model.id,
|
|
650
707
|
};
|
|
651
708
|
});
|
|
652
709
|
}
|
|
@@ -662,159 +719,166 @@ class WasmRunner {
|
|
|
662
719
|
* @param {InputObject} input - Input data for calculation
|
|
663
720
|
* @returns {Promise<ResponseObject>} - Response model data
|
|
664
721
|
*/
|
|
665
|
-
execute(
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
722
|
+
async execute(input, id = undefined, raw = false, cancelToken) {
|
|
723
|
+
let logInput;
|
|
724
|
+
const token = cancelToken ? cancelToken.getToken() : undefined;
|
|
725
|
+
if (utils.isV3Input(input)) {
|
|
726
|
+
logInput = {
|
|
727
|
+
request_data: {
|
|
728
|
+
inputs: input?.request_data?.inputs,
|
|
729
|
+
},
|
|
730
|
+
request_meta: {
|
|
731
|
+
...input.request_meta,
|
|
732
|
+
...(input.request_meta._ctx ? { _ctx: "[REDACTED]" } : {}),
|
|
733
|
+
},
|
|
734
|
+
};
|
|
735
|
+
this.logger.trace({
|
|
736
|
+
TimeStamp: Date.now(),
|
|
737
|
+
EventType: "Runner.Execute.V3",
|
|
738
|
+
TextMessage: `EXECUTING`,
|
|
739
|
+
JSONPayload: { input: logInput, modelId: id },
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
else {
|
|
743
|
+
this.logger.trace({
|
|
685
744
|
TimeStamp: Date.now(),
|
|
686
745
|
EventType: "Runner.Execute",
|
|
687
|
-
|
|
688
|
-
|
|
746
|
+
TextMessage: `EXECUTING`,
|
|
747
|
+
JSONPayload: { input: input, modelId: id },
|
|
689
748
|
});
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
input.request_data
|
|
749
|
+
}
|
|
750
|
+
const callid = input.callid || (0, cuid2_1.createId)();
|
|
751
|
+
const version_uuid = id ||
|
|
752
|
+
input?.request_meta?.version_uuid ||
|
|
753
|
+
input?.request_meta?.version_id ||
|
|
754
|
+
input?.version_id;
|
|
755
|
+
if (input) {
|
|
756
|
+
if (raw) {
|
|
757
|
+
if (!input.request_data) {
|
|
758
|
+
input.request_data = {};
|
|
700
759
|
}
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
}
|
|
722
|
-
model.busy = model.busy + 1;
|
|
760
|
+
input.request_data._raw = true;
|
|
761
|
+
}
|
|
762
|
+
const model = this._getModel(version_uuid);
|
|
763
|
+
try {
|
|
764
|
+
if (!input.callid) {
|
|
765
|
+
input.callid = callid;
|
|
766
|
+
}
|
|
767
|
+
this.logger.debug({
|
|
768
|
+
TimeStamp: Date.now(),
|
|
769
|
+
EventType: "Runner.Execute",
|
|
770
|
+
TextMessage: `PREEXECUTING ${id}`,
|
|
771
|
+
JSONPayload: {
|
|
772
|
+
runnerCallId: callid,
|
|
773
|
+
ModelId: id,
|
|
774
|
+
},
|
|
775
|
+
});
|
|
776
|
+
if (this.isModelCompatible(model, input)) {
|
|
777
|
+
const start = Date.now();
|
|
778
|
+
if (token) {
|
|
779
|
+
token.throwIfCancelled(version_uuid, input);
|
|
723
780
|
}
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
if (res.memoryUsage > model.stats[res.threadId].peak_memory) {
|
|
730
|
-
model.stats[res.threadId].peak_memory = res.memoryUsage;
|
|
731
|
-
}
|
|
732
|
-
model.stats[res.threadId].last_execute_consume_memory = res.memoryUsage;
|
|
733
|
-
model.responseTimeMetrics.record(execTime);
|
|
734
|
-
if (res.error) {
|
|
735
|
-
throw res.error;
|
|
736
|
-
}
|
|
737
|
-
parsedResult = res.payload;
|
|
738
|
-
logger_1.default.debug({
|
|
739
|
-
TimeStamp: Date.now(),
|
|
740
|
-
EventType: "Runner.Execute.Completed",
|
|
741
|
-
executeDuration: Date.now() - start,
|
|
781
|
+
this.logger.debug({
|
|
782
|
+
TimeStamp: Date.now(),
|
|
783
|
+
EventType: "Runner.Execute",
|
|
784
|
+
TextMessage: `START EXECUTING ${id}`,
|
|
785
|
+
JSONPayload: {
|
|
742
786
|
runnerCallId: callid,
|
|
743
787
|
ModelId: id,
|
|
744
|
-
}
|
|
745
|
-
|
|
788
|
+
},
|
|
789
|
+
});
|
|
790
|
+
const res = await model.worker.execute(input);
|
|
791
|
+
const execTime = Date.now() - start;
|
|
792
|
+
this.logger.debug({
|
|
793
|
+
TimeStamp: Date.now(),
|
|
794
|
+
EventType: "Runner.Execute.Completed",
|
|
795
|
+
TextMessage: `EXECUTE COMPLETED ${id}`,
|
|
796
|
+
JSONPayload: {
|
|
797
|
+
executeDuration: execTime,
|
|
798
|
+
runnerCallId: callid,
|
|
799
|
+
ModelId: id,
|
|
800
|
+
},
|
|
801
|
+
});
|
|
802
|
+
if (res.memoryUsage > model.stats[res.threadId].peak_memory) {
|
|
803
|
+
model.stats[res.threadId].peak_memory = res.memoryUsage;
|
|
746
804
|
}
|
|
747
|
-
|
|
748
|
-
|
|
805
|
+
model.stats[res.threadId].last_execute_consume_memory =
|
|
806
|
+
res.memoryUsage;
|
|
807
|
+
model.responseTimeMetrics.record(execTime);
|
|
808
|
+
if (res.error) {
|
|
809
|
+
throw res.error;
|
|
749
810
|
}
|
|
811
|
+
return res.payload;
|
|
750
812
|
}
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
throw err;
|
|
758
|
-
}
|
|
759
|
-
throw new error_1.default.ModelExecuteError(err.message, version_uuid, input);
|
|
813
|
+
throw new error_1.default.ModelExecuteError(this.getIncompatibleMessage(model), version_uuid, input);
|
|
814
|
+
}
|
|
815
|
+
catch (err) {
|
|
816
|
+
this.logger.error(err);
|
|
817
|
+
if (err.name === "validation_error") {
|
|
818
|
+
throw err;
|
|
760
819
|
}
|
|
761
|
-
|
|
762
|
-
|
|
820
|
+
if (err.name === "execution_cancelled") {
|
|
821
|
+
throw err;
|
|
763
822
|
}
|
|
823
|
+
throw new error_1.default.ModelExecuteError(err.message, version_uuid, input);
|
|
764
824
|
}
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
}
|
|
825
|
+
}
|
|
826
|
+
else {
|
|
827
|
+
throw new error_1.default.ParameterRequiredError("request_meta.version_uuid", input);
|
|
828
|
+
}
|
|
769
829
|
}
|
|
770
|
-
getNeuronCompilerVersion(logFile) {
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
830
|
+
async getNeuronCompilerVersion(logFile) {
|
|
831
|
+
let data;
|
|
832
|
+
let modelJsonInfo;
|
|
833
|
+
try {
|
|
834
|
+
data = fs_1.default.readFileSync(logFile.replace(/"/g, ""), "utf8");
|
|
835
|
+
modelJsonInfo = JSON.parse(data);
|
|
836
|
+
}
|
|
837
|
+
catch (err) {
|
|
838
|
+
this.logger.warn({
|
|
839
|
+
TimeStamp: Date.now(),
|
|
840
|
+
EventType: "Runner.GetNeuronCompilerVersion",
|
|
841
|
+
TextMessage: `Failed to get neuron compiler version from ${logFile}`,
|
|
842
|
+
JSONPayload: {
|
|
843
|
+
error: err.message,
|
|
844
|
+
logFile,
|
|
845
|
+
},
|
|
846
|
+
});
|
|
847
|
+
return undefined;
|
|
848
|
+
}
|
|
849
|
+
if (!semver_1.default.valid(modelJsonInfo.CompilerVersion)) {
|
|
850
|
+
throw new error_1.default.CompatibilityJsonError(`The Wasm was compiled using ${modelJsonInfo.CompilerVersion} which is not supported by the compatibility check.`);
|
|
851
|
+
}
|
|
852
|
+
const compilerVersion = semver_1.default.coerce(modelJsonInfo.CompilerVersion).version;
|
|
853
|
+
return compilerVersion;
|
|
781
854
|
}
|
|
782
|
-
checkCompilerVersionCompatibility(modelCompilerVersion) {
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
if (modelCompilerVersion) {
|
|
787
|
-
if (semver_1.default.lte(modelCompilerVersion, safeCompilerVersion)) {
|
|
788
|
-
return types_1.Compatibility.COMPATIBLE;
|
|
789
|
-
}
|
|
790
|
-
else {
|
|
791
|
-
return types_1.Compatibility.PARTIAL;
|
|
792
|
-
}
|
|
793
|
-
}
|
|
794
|
-
else {
|
|
855
|
+
async checkCompilerVersionCompatibility(modelCompilerVersion) {
|
|
856
|
+
try {
|
|
857
|
+
const safeCompilerVersion = this.safeCompiler;
|
|
858
|
+
if (modelCompilerVersion) {
|
|
859
|
+
if (semver_1.default.lte(modelCompilerVersion, safeCompilerVersion)) {
|
|
795
860
|
return types_1.Compatibility.COMPATIBLE;
|
|
796
861
|
}
|
|
862
|
+
return types_1.Compatibility.PARTIAL;
|
|
797
863
|
}
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
return types_1.Compatibility.INCOMPATIBLE;
|
|
804
|
-
}
|
|
805
|
-
else {
|
|
806
|
-
throw err;
|
|
807
|
-
}
|
|
864
|
+
return types_1.Compatibility.COMPATIBLE;
|
|
865
|
+
}
|
|
866
|
+
catch (err) {
|
|
867
|
+
if (err.code === "ENOENT" || err.message === "File not found") {
|
|
868
|
+
return types_1.Compatibility.COMPATIBLE;
|
|
808
869
|
}
|
|
809
|
-
|
|
870
|
+
if (err instanceof SyntaxError) {
|
|
871
|
+
return types_1.Compatibility.INCOMPATIBLE;
|
|
872
|
+
}
|
|
873
|
+
throw err;
|
|
874
|
+
}
|
|
810
875
|
}
|
|
811
876
|
getSafeCompilerVersion(runnerVersion) {
|
|
812
|
-
|
|
813
|
-
const runnerVersions = (_a = this.options.compatibilityList) === null || _a === void 0 ? void 0 : _a.runner_versions;
|
|
877
|
+
const runnerVersions = this.options.compatibilityList?.runner_versions;
|
|
814
878
|
if (!this.options.compatibilityList) {
|
|
815
879
|
throw new error_1.default.CompatibilityJsonError("No runner compiler compatibility list provided");
|
|
816
880
|
}
|
|
817
|
-
if (!
|
|
881
|
+
if (!Object.prototype.hasOwnProperty.call(runnerVersions, runnerVersion)) {
|
|
818
882
|
throw new error_1.default.CompatibilityJsonError(`No compiler compatibility found for the runner version ${runnerVersion}`);
|
|
819
883
|
}
|
|
820
884
|
return runnerVersions[runnerVersion].safe_compiler;
|