@quatrain/worker 1.1.45 → 1.1.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/FileSystem.d.ts +20 -0
- package/dist/FileSystem.js +182 -0
- package/dist/Handler.d.ts +1 -0
- package/dist/Handler.js +39 -0
- package/dist/Helpers.d.ts +11 -0
- package/dist/Helpers.js +34 -0
- package/dist/Worker.d.ts +37 -0
- package/dist/Worker.js +151 -0
- package/dist/Worker.test.d.ts +1 -0
- package/dist/Worker.test.js +219 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +11 -0
- package/dist/types/HandlerParameters.d.ts +9 -0
- package/dist/types/HandlerParameters.js +2 -0
- package/dist/types/MessagehandlerParameters.d.ts +4 -0
- package/dist/types/MessagehandlerParameters.js +2 -0
- package/dist/types/ModeEnum.d.ts +5 -0
- package/dist/types/ModeEnum.js +9 -0
- package/package.json +44 -44
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { FileType } from '@quatrain/storage';
|
|
2
|
+
export declare class FileSystem {
|
|
3
|
+
static prepare(folder: string): void;
|
|
4
|
+
static makeFolder(folder: string): void;
|
|
5
|
+
static removeFolder(folder: string, recursively?: boolean): void;
|
|
6
|
+
static downloadFile(url: string, filepath: string): Promise<unknown>;
|
|
7
|
+
static safeString(name: string): string;
|
|
8
|
+
/**
|
|
9
|
+
* Upload file to public URL and return meta data
|
|
10
|
+
* @param filename string
|
|
11
|
+
* @param meta
|
|
12
|
+
* @param mime
|
|
13
|
+
* @returns Promise
|
|
14
|
+
*/
|
|
15
|
+
static uploadFile(filename: string, meta: FileType, mime?: string): Promise<FileType>;
|
|
16
|
+
/**
|
|
17
|
+
* Return meta data on given file
|
|
18
|
+
*/
|
|
19
|
+
static getInfo: (file: string) => Promise<any>;
|
|
20
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.FileSystem = void 0;
|
|
30
|
+
const node_fs_1 = __importStar(require("node:fs"));
|
|
31
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
32
|
+
const axios_1 = __importDefault(require("axios"));
|
|
33
|
+
const Worker_1 = require("./Worker");
|
|
34
|
+
const node_fetch_native_1 = __importDefault(require("node-fetch-native"));
|
|
35
|
+
const ffmpeg = __importStar(require("fluent-ffmpeg"));
|
|
36
|
+
class FileSystem {
|
|
37
|
+
static prepare(folder) {
|
|
38
|
+
Worker_1.Worker.debug(`Setting up process folder ${folder}`);
|
|
39
|
+
this.removeFolder(folder);
|
|
40
|
+
node_fs_1.default.mkdirSync(folder);
|
|
41
|
+
node_fs_1.default.mkdirSync(folder + node_path_1.default.sep + 'images');
|
|
42
|
+
node_fs_1.default.mkdirSync(folder + node_path_1.default.sep + 'vecto');
|
|
43
|
+
}
|
|
44
|
+
static makeFolder(folder) {
|
|
45
|
+
node_fs_1.default.mkdirSync(folder);
|
|
46
|
+
}
|
|
47
|
+
static removeFolder(folder, recursively = true) {
|
|
48
|
+
if (node_fs_1.default.existsSync(folder)) {
|
|
49
|
+
node_fs_1.default.readdirSync(folder).forEach((element) => {
|
|
50
|
+
const item = node_path_1.default.join(folder, element);
|
|
51
|
+
if (node_fs_1.default.lstatSync(item).isDirectory()) {
|
|
52
|
+
if (recursively !== true) {
|
|
53
|
+
throw new Error(`Folder contains folder`);
|
|
54
|
+
}
|
|
55
|
+
this.removeFolder(item);
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
node_fs_1.default.unlinkSync(item);
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
node_fs_1.default.rmdirSync(folder);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
static async downloadFile(url, filepath) {
|
|
65
|
+
try {
|
|
66
|
+
Worker_1.Worker.debug(`Downloading file at ${url} to ${filepath}`);
|
|
67
|
+
const writer = node_fs_1.default.createWriteStream(filepath);
|
|
68
|
+
const response = await axios_1.default.get(url, {
|
|
69
|
+
responseType: 'stream',
|
|
70
|
+
});
|
|
71
|
+
response.data.pipe(writer);
|
|
72
|
+
return new Promise((resolve, reject) => {
|
|
73
|
+
writer.on('finish', resolve);
|
|
74
|
+
writer.on('error', reject);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
catch (err) {
|
|
78
|
+
Worker_1.Worker.error(`Download error: ${err.message}`);
|
|
79
|
+
throw err;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
static safeString(name) {
|
|
83
|
+
return name.replace(/\s+/g, '_');
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Upload file to public URL and return meta data
|
|
87
|
+
* @param filename string
|
|
88
|
+
* @param meta
|
|
89
|
+
* @param mime
|
|
90
|
+
* @returns Promise
|
|
91
|
+
*/
|
|
92
|
+
static uploadFile(filename, meta, mime = 'video/mp4') {
|
|
93
|
+
// try to get more file metadata
|
|
94
|
+
meta = { ...meta, ...FileSystem.getInfo(filename) };
|
|
95
|
+
Worker_1.Worker.info(`Uploading file ${filename} to ${meta.uploadUrl}`);
|
|
96
|
+
const { size } = node_fs_1.default.statSync(filename);
|
|
97
|
+
if (size < 32 * 1024) {
|
|
98
|
+
return new Promise((resolve, reject) => {
|
|
99
|
+
(0, node_fetch_native_1.default)(meta.uploadUrl, {
|
|
100
|
+
method: 'PUT',
|
|
101
|
+
mode: 'cors',
|
|
102
|
+
duplex: 'half',
|
|
103
|
+
body: (0, node_fs_1.readFileSync)(filename),
|
|
104
|
+
headers: {
|
|
105
|
+
'Content-Type': meta.contentType || mime,
|
|
106
|
+
'Content-length': String(size),
|
|
107
|
+
},
|
|
108
|
+
})
|
|
109
|
+
.then(() => resolve({ ...meta, size, uploadUrl: undefined }))
|
|
110
|
+
.catch((err) => {
|
|
111
|
+
Worker_1.Worker.error(`An error occured while uploading ${filename}: ${err.message}`);
|
|
112
|
+
reject(err);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return new Promise((resolve, reject) => {
|
|
117
|
+
try {
|
|
118
|
+
let done = 0; // total bytes uploaded
|
|
119
|
+
let prev = 0; // latest value of done stored when % done was displayed
|
|
120
|
+
const { size } = node_fs_1.default.statSync(filename);
|
|
121
|
+
const readStream = node_fs_1.default.createReadStream(filename);
|
|
122
|
+
const sizeMB = (size / (1024 * 1024)).toFixed(2);
|
|
123
|
+
readStream.on('data', (data) => {
|
|
124
|
+
done += data.length;
|
|
125
|
+
if (done - prev >= size / 20) {
|
|
126
|
+
// only display message when 5% more has been uploaded
|
|
127
|
+
Worker_1.Worker.info(`${((done / size) * 100).toFixed(2)}% - ${(done /
|
|
128
|
+
(1024 * 1024)).toFixed(2)}MB / ${sizeMB}MB`);
|
|
129
|
+
prev = done;
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
Worker_1.Worker.info(`Uploading file ${filename} with size ${size} bytes`);
|
|
133
|
+
(0, node_fetch_native_1.default)(meta.uploadUrl, {
|
|
134
|
+
method: 'PUT',
|
|
135
|
+
mode: 'cors',
|
|
136
|
+
duplex: 'half',
|
|
137
|
+
body: readStream,
|
|
138
|
+
headers: {
|
|
139
|
+
'Content-Type': meta.contentType || mime,
|
|
140
|
+
'Content-length': String(size),
|
|
141
|
+
},
|
|
142
|
+
})
|
|
143
|
+
.then(() => resolve({ ...meta, size, uploadUrl: undefined }))
|
|
144
|
+
.catch((err) => {
|
|
145
|
+
Worker_1.Worker.error(`An error occured while uploading ${filename}: ${err.message}`);
|
|
146
|
+
reject(err);
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
Worker_1.Worker.error(err);
|
|
151
|
+
reject(err);
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Return meta data on given file
|
|
157
|
+
*/
|
|
158
|
+
static getInfo = (file) => {
|
|
159
|
+
if (file.endsWith('.mp4') || file.endsWith('.insv')) {
|
|
160
|
+
return new Promise((resolve, reject) => ffmpeg.ffprobe(file, (err, metadata) => {
|
|
161
|
+
if (err) {
|
|
162
|
+
Worker_1.Worker.error(err);
|
|
163
|
+
reject(err);
|
|
164
|
+
}
|
|
165
|
+
// Worker.debug(`ffprobe getInfo: ${JSON.stringify(metadata)}`)
|
|
166
|
+
const { width, height, duration, bit_rate: bitrate, nb_frames: nbFramees, } = metadata.streams[0];
|
|
167
|
+
const nb_frames = parseFloat(nbFramees);
|
|
168
|
+
const framerate = nb_frames / parseFloat(duration);
|
|
169
|
+
resolve({
|
|
170
|
+
width,
|
|
171
|
+
height,
|
|
172
|
+
framerate: parseInt(framerate.toFixed(0)),
|
|
173
|
+
duration: parseInt(duration),
|
|
174
|
+
bitrate,
|
|
175
|
+
});
|
|
176
|
+
}));
|
|
177
|
+
}
|
|
178
|
+
// Handle other cases silently
|
|
179
|
+
return new Promise((resolve) => resolve({}));
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
exports.FileSystem = FileSystem;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const handler: (messageHandler: Function, config: any, adapters: any) => Promise<void>;
|
package/dist/Handler.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handler = void 0;
|
|
4
|
+
const Worker_1 = require("./Worker");
|
|
5
|
+
const handler = async (messageHandler, config, adapters) => {
|
|
6
|
+
try {
|
|
7
|
+
Worker_1.Worker.info(`Worker version ${require('../package.json').version} started in ${config.MQ_MODE} mode`);
|
|
8
|
+
switch (config.MQ_MODE) {
|
|
9
|
+
case 'queue':
|
|
10
|
+
const { Queue } = require('@quatrain/queue');
|
|
11
|
+
Queue.addQueue(adapters.queue, 'default', true);
|
|
12
|
+
Queue.info(`Connecting to amqp://${config.MQ_HOST}:${config.MQ_PORT}`);
|
|
13
|
+
Queue.getQueue().listen(config.MQ_TOPIC, messageHandler, {
|
|
14
|
+
concurrency: config.MQ_CONCURRENCY,
|
|
15
|
+
gpu: config.MQ_GPU,
|
|
16
|
+
});
|
|
17
|
+
break;
|
|
18
|
+
case 'cli':
|
|
19
|
+
case 'test':
|
|
20
|
+
Worker_1.Worker.warn(`Message received from CLI.`);
|
|
21
|
+
const json = config.MQ_MODE === 'test'
|
|
22
|
+
? require('../test.json')
|
|
23
|
+
: process.env.JSON;
|
|
24
|
+
if (!json) {
|
|
25
|
+
throw new Error(`CLI call with missing environment variables`);
|
|
26
|
+
}
|
|
27
|
+
await messageHandler(json);
|
|
28
|
+
break;
|
|
29
|
+
default:
|
|
30
|
+
Worker_1.Worker.error(`Unknown mode option: '${config.MQ_MODE}'`);
|
|
31
|
+
process.exit(1);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
Worker_1.Worker.error(error.message);
|
|
36
|
+
process.exit(1);
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
exports.handler = handler;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export declare class Helpers {
|
|
2
|
+
static FFMPEG: string;
|
|
3
|
+
/**
|
|
4
|
+
* Generate a thubnail from a video file at given frame position
|
|
5
|
+
* @param videoPath path to video file
|
|
6
|
+
* @param frame frame to extract thumbnail from
|
|
7
|
+
* @param width width of thumbnail (4/3 ratio)
|
|
8
|
+
* @returns
|
|
9
|
+
*/
|
|
10
|
+
static generateVideoThumbnail: (videoPath: string, outputPath: string, frame?: number, width?: number) => Promise<any>;
|
|
11
|
+
}
|
package/dist/Helpers.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Helpers = void 0;
|
|
4
|
+
const Worker_1 = require("./Worker");
|
|
5
|
+
class Helpers {
|
|
6
|
+
static FFMPEG = '/usr/bin/ffmpeg';
|
|
7
|
+
/**
|
|
8
|
+
* Generate a thubnail from a video file at given frame position
|
|
9
|
+
* @param videoPath path to video file
|
|
10
|
+
* @param frame frame to extract thumbnail from
|
|
11
|
+
* @param width width of thumbnail (4/3 ratio)
|
|
12
|
+
* @returns
|
|
13
|
+
*/
|
|
14
|
+
static generateVideoThumbnail = async (videoPath, outputPath, frame = 0, width = 320) => {
|
|
15
|
+
const resolution = `${width}x${Math.ceil(width * 0.75)}`;
|
|
16
|
+
const ffmpegParams = [
|
|
17
|
+
'-i',
|
|
18
|
+
videoPath,
|
|
19
|
+
'-vframes',
|
|
20
|
+
'1',
|
|
21
|
+
'-vf',
|
|
22
|
+
String.raw `select=gte(n\,${frame})`,
|
|
23
|
+
'-s',
|
|
24
|
+
resolution,
|
|
25
|
+
'-ss',
|
|
26
|
+
'1',
|
|
27
|
+
outputPath,
|
|
28
|
+
'-y',
|
|
29
|
+
];
|
|
30
|
+
Worker_1.Worker.info(`Generating Thumbnail : ${outputPath}`);
|
|
31
|
+
return await Worker_1.Worker.execPromise(Helpers.FFMPEG, ffmpegParams);
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
exports.Helpers = Helpers;
|
package/dist/Worker.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { Core } from '@quatrain/core';
|
|
2
|
+
import { HandlerParameters } from './types/HandlerParameters';
|
|
3
|
+
export declare class Worker extends Core {
|
|
4
|
+
static endpoint: string;
|
|
5
|
+
static readonly logger: any;
|
|
6
|
+
/**
|
|
7
|
+
* Execute an external command in a promise
|
|
8
|
+
* @see https://stackoverflow.com/questions/46289682/how-to-wait-for-child-process-spawn-execution-with-async-await
|
|
9
|
+
* @see https://dzone.com/articles/understanding-execfile-spawn-exec-and-fork-in-node
|
|
10
|
+
* @param string command
|
|
11
|
+
* @param array args
|
|
12
|
+
* @return Promise
|
|
13
|
+
*/
|
|
14
|
+
static readonly execPromise: (command: string, args?: any[], cwd?: string) => Promise<any>;
|
|
15
|
+
/**
|
|
16
|
+
* Push an event to the backend endpoint, if available
|
|
17
|
+
* @param event string
|
|
18
|
+
* @param data
|
|
19
|
+
* @param ts timestamp
|
|
20
|
+
* @returns boolean
|
|
21
|
+
*/
|
|
22
|
+
static pushEvent(event: string, data?: {}, ts?: number): false | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* Async Push an event to the backend endpoint, if available
|
|
25
|
+
* @param event string
|
|
26
|
+
* @param data
|
|
27
|
+
* @param ts timestamp
|
|
28
|
+
* @returns boolean
|
|
29
|
+
*/
|
|
30
|
+
static pushEventAsync(event: string, data?: {}, ts?: number): Promise<boolean | undefined>;
|
|
31
|
+
/**
|
|
32
|
+
* Global handling function to process received messages
|
|
33
|
+
* @param messageHandler function
|
|
34
|
+
* @param config object
|
|
35
|
+
*/
|
|
36
|
+
static readonly handler: (messageHandler: Function, config: HandlerParameters) => Promise<void>;
|
|
37
|
+
}
|
package/dist/Worker.js
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Worker = void 0;
|
|
7
|
+
const core_1 = require("@quatrain/core");
|
|
8
|
+
const os_1 = __importDefault(require("os"));
|
|
9
|
+
const axios_1 = __importDefault(require("axios"));
|
|
10
|
+
const child_process_1 = require("child_process");
|
|
11
|
+
class Worker extends core_1.Core {
|
|
12
|
+
static endpoint = '';
|
|
13
|
+
static logger = this.addLogger('Worker');
|
|
14
|
+
/**
|
|
15
|
+
* Execute an external command in a promise
|
|
16
|
+
* @see https://stackoverflow.com/questions/46289682/how-to-wait-for-child-process-spawn-execution-with-async-await
|
|
17
|
+
* @see https://dzone.com/articles/understanding-execfile-spawn-exec-and-fork-in-node
|
|
18
|
+
* @param string command
|
|
19
|
+
* @param array args
|
|
20
|
+
* @return Promise
|
|
21
|
+
*/
|
|
22
|
+
static execPromise = (command, args = [], cwd = process.cwd()) => {
|
|
23
|
+
try {
|
|
24
|
+
Worker.info(`Executing command ${command} in ${cwd} with arguments:`);
|
|
25
|
+
args.forEach((arg) => console.log(`\t${arg}`));
|
|
26
|
+
return new Promise((resolve, reject) => {
|
|
27
|
+
const child = (0, child_process_1.spawn)(command, args, { cwd });
|
|
28
|
+
child.stdout.on('data', (data) => Worker.debug(data.toString()));
|
|
29
|
+
child.stderr.on('data', (data) => Worker.debug(data.toString()));
|
|
30
|
+
child.on('close', (code) => {
|
|
31
|
+
if (code !== 0) {
|
|
32
|
+
Worker.error(`Command execution failed with code: ${code}`);
|
|
33
|
+
reject(new Error(`Process failed and returned code: ${code}`));
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
Worker.info(`Command execution completed with code: ${code}`);
|
|
37
|
+
resolve(undefined);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
Worker.error(err.message);
|
|
44
|
+
throw err;
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Push an event to the backend endpoint, if available
|
|
49
|
+
* @param event string
|
|
50
|
+
* @param data
|
|
51
|
+
* @param ts timestamp
|
|
52
|
+
* @returns boolean
|
|
53
|
+
*/
|
|
54
|
+
static pushEvent(event, data = {}, ts = 0) {
|
|
55
|
+
if (!this.endpoint) {
|
|
56
|
+
Worker.warn(`Events endpoint is not set, can't send update!`);
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
ts = ts === Date.now() ? Date.now() + 1 : Date.now();
|
|
60
|
+
const payload = {
|
|
61
|
+
event,
|
|
62
|
+
worker: `Container ${os_1.default.hostname}`,
|
|
63
|
+
os: `${os_1.default.type} ${os_1.default.release} (${os_1.default.platform} ${os_1.default.arch})`,
|
|
64
|
+
...data,
|
|
65
|
+
ts,
|
|
66
|
+
};
|
|
67
|
+
Worker.debug('event payload', payload);
|
|
68
|
+
axios_1.default
|
|
69
|
+
.patch(Worker.endpoint, payload)
|
|
70
|
+
.then((res) => {
|
|
71
|
+
Worker.info(`Event pushed to backend: ${res.statusText}`);
|
|
72
|
+
Worker.info(res.data);
|
|
73
|
+
return true;
|
|
74
|
+
})
|
|
75
|
+
.catch((err) => {
|
|
76
|
+
Worker.error(`Failed to push event to backend: ${err}`);
|
|
77
|
+
return false;
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Async Push an event to the backend endpoint, if available
|
|
82
|
+
* @param event string
|
|
83
|
+
* @param data
|
|
84
|
+
* @param ts timestamp
|
|
85
|
+
* @returns boolean
|
|
86
|
+
*/
|
|
87
|
+
static async pushEventAsync(event, data = {}, ts = 0) {
|
|
88
|
+
if (!this.endpoint) {
|
|
89
|
+
Worker.warn(`Events endpoint is not set, can't send update!`);
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
ts = ts === Date.now() ? Date.now() + 1 : Date.now();
|
|
94
|
+
const payload = {
|
|
95
|
+
event,
|
|
96
|
+
worker: `Container ${os_1.default.hostname}`,
|
|
97
|
+
os: `${os_1.default.type} ${os_1.default.release} (${os_1.default.platform} ${os_1.default.arch})`,
|
|
98
|
+
...data,
|
|
99
|
+
ts,
|
|
100
|
+
};
|
|
101
|
+
const res = await axios_1.default.patch(Worker.endpoint, payload);
|
|
102
|
+
if (res.statusText === 'OK') {
|
|
103
|
+
Worker.info(`Event pushed to backend: ${res.statusText}`);
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
Worker.error(`Failed to push event to backend: ${err}`);
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Global handling function to process received messages
|
|
114
|
+
* @param messageHandler function
|
|
115
|
+
* @param config object
|
|
116
|
+
*/
|
|
117
|
+
static handler = async (messageHandler, config) => {
|
|
118
|
+
try {
|
|
119
|
+
switch (config.mode) {
|
|
120
|
+
case 'queue':
|
|
121
|
+
const { Queue } = require('@quatrain/queue');
|
|
122
|
+
Queue.addQueue(config.queueAdapter, 'default', true);
|
|
123
|
+
Queue.getQueue().listen(config.topic, messageHandler, {
|
|
124
|
+
concurrency: config.concurrency,
|
|
125
|
+
gpu: config.gpu,
|
|
126
|
+
});
|
|
127
|
+
Queue.info(`Connected and listening to ${config.topic}, ready to receive messages.`);
|
|
128
|
+
break;
|
|
129
|
+
case 'cli':
|
|
130
|
+
case 'test':
|
|
131
|
+
Worker.warn(`Message received from CLI.`);
|
|
132
|
+
const json = config.mode === 'test'
|
|
133
|
+
? require('../test.json')
|
|
134
|
+
: process.env.JSON;
|
|
135
|
+
if (!json) {
|
|
136
|
+
throw new Error(`CLI call with missing environment variables`);
|
|
137
|
+
}
|
|
138
|
+
await messageHandler(json);
|
|
139
|
+
break;
|
|
140
|
+
default:
|
|
141
|
+
Worker.error(`Unknown mode option: '${config.mode}'`);
|
|
142
|
+
process.exit(1);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
Worker.error(error.message);
|
|
147
|
+
process.exit(1);
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
exports.Worker = Worker;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const Worker_1 = require("./Worker");
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const child_process_1 = require("child_process");
|
|
9
|
+
// Mock dependencies
|
|
10
|
+
jest.mock('axios');
|
|
11
|
+
jest.mock('child_process');
|
|
12
|
+
const mockedAxios = axios_1.default;
|
|
13
|
+
const mockedSpawn = child_process_1.spawn;
|
|
14
|
+
describe('Worker', () => {
|
|
15
|
+
beforeEach(() => {
|
|
16
|
+
// Clear all mocks before each test
|
|
17
|
+
jest.clearAllMocks();
|
|
18
|
+
// Reset Worker.endpoint
|
|
19
|
+
Worker_1.Worker.endpoint = '';
|
|
20
|
+
});
|
|
21
|
+
describe('endpoint property', () => {
|
|
22
|
+
it('should have an empty endpoint by default', () => {
|
|
23
|
+
expect(Worker_1.Worker.endpoint).toBe('');
|
|
24
|
+
});
|
|
25
|
+
it('should allow setting the endpoint', () => {
|
|
26
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
27
|
+
expect(Worker_1.Worker.endpoint).toBe('https://api.example.com/events');
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
describe('pushEvent', () => {
|
|
31
|
+
it('should return false when endpoint is not set', () => {
|
|
32
|
+
const result = Worker_1.Worker.pushEvent('test-event');
|
|
33
|
+
expect(result).toBe(false);
|
|
34
|
+
expect(mockedAxios.patch).not.toHaveBeenCalled();
|
|
35
|
+
});
|
|
36
|
+
it('should send event when endpoint is set', () => {
|
|
37
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
38
|
+
mockedAxios.patch.mockResolvedValue({
|
|
39
|
+
statusText: 'OK',
|
|
40
|
+
data: { success: true },
|
|
41
|
+
});
|
|
42
|
+
Worker_1.Worker.pushEvent('test-event', { foo: 'bar' });
|
|
43
|
+
expect(mockedAxios.patch).toHaveBeenCalledWith('https://api.example.com/events', expect.objectContaining({
|
|
44
|
+
event: 'test-event',
|
|
45
|
+
foo: 'bar',
|
|
46
|
+
}));
|
|
47
|
+
});
|
|
48
|
+
it('should include custom timestamp in payload', () => {
|
|
49
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
50
|
+
const customTs = 1234567890;
|
|
51
|
+
mockedAxios.patch.mockResolvedValue({
|
|
52
|
+
statusText: 'OK',
|
|
53
|
+
data: { success: true },
|
|
54
|
+
});
|
|
55
|
+
Worker_1.Worker.pushEvent('test-event', {}, customTs);
|
|
56
|
+
expect(mockedAxios.patch).toHaveBeenCalledWith('https://api.example.com/events', expect.objectContaining({
|
|
57
|
+
event: 'test-event',
|
|
58
|
+
ts: expect.any(Number),
|
|
59
|
+
}));
|
|
60
|
+
});
|
|
61
|
+
it('should include additional data in payload', () => {
|
|
62
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
63
|
+
mockedAxios.patch.mockResolvedValue({
|
|
64
|
+
statusText: 'OK',
|
|
65
|
+
data: { success: true },
|
|
66
|
+
});
|
|
67
|
+
const additionalData = {
|
|
68
|
+
userId: '123',
|
|
69
|
+
action: 'upload',
|
|
70
|
+
metadata: { size: 1024 },
|
|
71
|
+
};
|
|
72
|
+
Worker_1.Worker.pushEvent('custom-event', additionalData);
|
|
73
|
+
expect(mockedAxios.patch).toHaveBeenCalledWith('https://api.example.com/events', expect.objectContaining({
|
|
74
|
+
event: 'custom-event',
|
|
75
|
+
userId: '123',
|
|
76
|
+
action: 'upload',
|
|
77
|
+
metadata: { size: 1024 },
|
|
78
|
+
}));
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
describe('pushEventAsync', () => {
|
|
82
|
+
it('should return false when endpoint is not set', async () => {
|
|
83
|
+
const result = await Worker_1.Worker.pushEventAsync('test-event');
|
|
84
|
+
expect(result).toBe(false);
|
|
85
|
+
expect(mockedAxios.patch).not.toHaveBeenCalled();
|
|
86
|
+
});
|
|
87
|
+
it('should send event and return true on success', async () => {
|
|
88
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
89
|
+
mockedAxios.patch.mockResolvedValue({
|
|
90
|
+
statusText: 'OK',
|
|
91
|
+
data: { success: true },
|
|
92
|
+
});
|
|
93
|
+
const result = await Worker_1.Worker.pushEventAsync('test-event', {
|
|
94
|
+
foo: 'bar',
|
|
95
|
+
});
|
|
96
|
+
expect(result).toBe(true);
|
|
97
|
+
expect(mockedAxios.patch).toHaveBeenCalledWith('https://api.example.com/events', expect.objectContaining({
|
|
98
|
+
event: 'test-event',
|
|
99
|
+
foo: 'bar',
|
|
100
|
+
}));
|
|
101
|
+
});
|
|
102
|
+
it('should return false on axios error', async () => {
|
|
103
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
104
|
+
mockedAxios.patch.mockRejectedValue(new Error('Network error'));
|
|
105
|
+
const result = await Worker_1.Worker.pushEventAsync('test-event');
|
|
106
|
+
expect(result).toBe(false);
|
|
107
|
+
});
|
|
108
|
+
it('should handle non-OK status response', async () => {
|
|
109
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
110
|
+
mockedAxios.patch.mockResolvedValue({
|
|
111
|
+
statusText: 'Bad Request',
|
|
112
|
+
data: { error: 'Invalid payload' },
|
|
113
|
+
});
|
|
114
|
+
const result = await Worker_1.Worker.pushEventAsync('test-event');
|
|
115
|
+
expect(result).toBe(undefined);
|
|
116
|
+
});
|
|
117
|
+
it('should include custom timestamp', async () => {
|
|
118
|
+
Worker_1.Worker.endpoint = 'https://api.example.com/events';
|
|
119
|
+
const customTs = 9876543210;
|
|
120
|
+
mockedAxios.patch.mockResolvedValue({
|
|
121
|
+
statusText: 'OK',
|
|
122
|
+
data: { success: true },
|
|
123
|
+
});
|
|
124
|
+
await Worker_1.Worker.pushEventAsync('test-event', {}, customTs);
|
|
125
|
+
expect(mockedAxios.patch).toHaveBeenCalledWith('https://api.example.com/events', expect.objectContaining({
|
|
126
|
+
event: 'test-event',
|
|
127
|
+
ts: expect.any(Number),
|
|
128
|
+
}));
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
describe('execPromise', () => {
|
|
132
|
+
it('should execute command successfully', async () => {
|
|
133
|
+
const mockChild = {
|
|
134
|
+
stdout: {
|
|
135
|
+
on: jest.fn((event, callback) => {
|
|
136
|
+
if (event === 'data') {
|
|
137
|
+
// Simulate stdout data
|
|
138
|
+
setTimeout(() => callback(Buffer.from('Command output')), 10);
|
|
139
|
+
}
|
|
140
|
+
}),
|
|
141
|
+
},
|
|
142
|
+
stderr: {
|
|
143
|
+
on: jest.fn(),
|
|
144
|
+
},
|
|
145
|
+
on: jest.fn((event, callback) => {
|
|
146
|
+
if (event === 'close') {
|
|
147
|
+
// Simulate successful completion
|
|
148
|
+
setTimeout(() => callback(0), 20);
|
|
149
|
+
}
|
|
150
|
+
}),
|
|
151
|
+
};
|
|
152
|
+
mockedSpawn.mockReturnValue(mockChild);
|
|
153
|
+
await expect(Worker_1.Worker.execPromise('ls', ['-la'], '/tmp')).resolves.toBeUndefined();
|
|
154
|
+
expect(mockedSpawn).toHaveBeenCalledWith('ls', ['-la'], {
|
|
155
|
+
cwd: '/tmp',
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
it('should reject on command failure', async () => {
|
|
159
|
+
const mockChild = {
|
|
160
|
+
stdout: {
|
|
161
|
+
on: jest.fn(),
|
|
162
|
+
},
|
|
163
|
+
stderr: {
|
|
164
|
+
on: jest.fn((event, callback) => {
|
|
165
|
+
if (event === 'data') {
|
|
166
|
+
setTimeout(() => callback(Buffer.from('Error output')), 10);
|
|
167
|
+
}
|
|
168
|
+
}),
|
|
169
|
+
},
|
|
170
|
+
on: jest.fn((event, callback) => {
|
|
171
|
+
if (event === 'close') {
|
|
172
|
+
// Simulate failure with exit code 1
|
|
173
|
+
setTimeout(() => callback(1), 20);
|
|
174
|
+
}
|
|
175
|
+
}),
|
|
176
|
+
};
|
|
177
|
+
mockedSpawn.mockReturnValue(mockChild);
|
|
178
|
+
await expect(Worker_1.Worker.execPromise('invalid-command', [])).rejects.toThrow('Process failed and returned code: 1');
|
|
179
|
+
});
|
|
180
|
+
it('should use current working directory by default', async () => {
|
|
181
|
+
const mockChild = {
|
|
182
|
+
stdout: { on: jest.fn() },
|
|
183
|
+
stderr: { on: jest.fn() },
|
|
184
|
+
on: jest.fn((event, callback) => {
|
|
185
|
+
if (event === 'close') {
|
|
186
|
+
setTimeout(() => callback(0), 10);
|
|
187
|
+
}
|
|
188
|
+
}),
|
|
189
|
+
};
|
|
190
|
+
mockedSpawn.mockReturnValue(mockChild);
|
|
191
|
+
await Worker_1.Worker.execPromise('echo', ['test']);
|
|
192
|
+
expect(mockedSpawn).toHaveBeenCalledWith('echo', ['test'], {
|
|
193
|
+
cwd: process.cwd(),
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
it('should pass arguments to command', async () => {
|
|
197
|
+
const mockChild = {
|
|
198
|
+
stdout: { on: jest.fn() },
|
|
199
|
+
stderr: { on: jest.fn() },
|
|
200
|
+
on: jest.fn((event, callback) => {
|
|
201
|
+
if (event === 'close') {
|
|
202
|
+
setTimeout(() => callback(0), 10);
|
|
203
|
+
}
|
|
204
|
+
}),
|
|
205
|
+
};
|
|
206
|
+
mockedSpawn.mockReturnValue(mockChild);
|
|
207
|
+
const args = ['arg1', 'arg2', '--flag'];
|
|
208
|
+
await Worker_1.Worker.execPromise('command', args, '/custom/path');
|
|
209
|
+
expect(mockedSpawn).toHaveBeenCalledWith('command', args, {
|
|
210
|
+
cwd: '/custom/path',
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
describe('logger', () => {
|
|
215
|
+
it('should have a logger instance', () => {
|
|
216
|
+
expect(Worker_1.Worker.logger).toBeDefined();
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
});
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Worker } from './Worker';
|
|
2
|
+
import { FileSystem } from './FileSystem';
|
|
3
|
+
import { Helpers } from './Helpers';
|
|
4
|
+
import { HandlerParameters } from './types/HandlerParameters';
|
|
5
|
+
import { MessagehandlerParameters } from './types/MessagehandlerParameters';
|
|
6
|
+
import { ModeEnum } from './types/ModeEnum';
|
|
7
|
+
export { Worker, FileSystem, Helpers, ModeEnum, HandlerParameters, MessagehandlerParameters, };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ModeEnum = exports.Helpers = exports.FileSystem = exports.Worker = void 0;
|
|
4
|
+
const Worker_1 = require("./Worker");
|
|
5
|
+
Object.defineProperty(exports, "Worker", { enumerable: true, get: function () { return Worker_1.Worker; } });
|
|
6
|
+
const FileSystem_1 = require("./FileSystem");
|
|
7
|
+
Object.defineProperty(exports, "FileSystem", { enumerable: true, get: function () { return FileSystem_1.FileSystem; } });
|
|
8
|
+
const Helpers_1 = require("./Helpers");
|
|
9
|
+
Object.defineProperty(exports, "Helpers", { enumerable: true, get: function () { return Helpers_1.Helpers; } });
|
|
10
|
+
const ModeEnum_1 = require("./types/ModeEnum");
|
|
11
|
+
Object.defineProperty(exports, "ModeEnum", { enumerable: true, get: function () { return ModeEnum_1.ModeEnum; } });
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { ModeEnum } from './ModeEnum';
|
|
2
|
+
import { AbstractQueueAdapter } from '@quatrain/queue';
|
|
3
|
+
export interface HandlerParameters {
|
|
4
|
+
mode: string | typeof ModeEnum;
|
|
5
|
+
topic?: string;
|
|
6
|
+
concurrency?: number;
|
|
7
|
+
queueAdapter?: AbstractQueueAdapter;
|
|
8
|
+
gpu?: boolean;
|
|
9
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ModeEnum = void 0;
|
|
4
|
+
var ModeEnum;
|
|
5
|
+
(function (ModeEnum) {
|
|
6
|
+
ModeEnum["MODE_CLI"] = "cli";
|
|
7
|
+
ModeEnum["MODE_QUEUE"] = "queue";
|
|
8
|
+
ModeEnum["MODE_TEST"] = "test";
|
|
9
|
+
})(ModeEnum || (exports.ModeEnum = ModeEnum = {}));
|
package/package.json
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
2
|
+
"name": "@quatrain/worker",
|
|
3
|
+
"version": "1.1.47",
|
|
4
|
+
"description": "Container Worker helpers",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"types": "dist/index.d.ts",
|
|
7
|
+
"bun": "src/index.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"LICENSE.md",
|
|
10
|
+
"src/",
|
|
11
|
+
"dist/",
|
|
12
|
+
"README.md"
|
|
13
|
+
],
|
|
14
|
+
"author": "Quatrain Développement SAS <developers@quatrain.com>",
|
|
15
|
+
"license": "AGPL-3.0-only",
|
|
16
|
+
"devDependencies": {
|
|
17
|
+
"@tsconfig/recommended": "^1.0.1",
|
|
18
|
+
"@types/fluent-ffmpeg": "^2",
|
|
19
|
+
"@types/fs-extra": "^11.0.4",
|
|
20
|
+
"@types/jest": "^27.0.3",
|
|
21
|
+
"@types/node": "^22.9.0",
|
|
22
|
+
"@types/object-hash": "^3.0.6",
|
|
23
|
+
"jest": "^30.2.0",
|
|
24
|
+
"jest-node-exports-resolver": "^1.1.6",
|
|
25
|
+
"trace-unhandled": "^2.0.1",
|
|
26
|
+
"ts-jest": "^27.1.2",
|
|
27
|
+
"ts-node": "^10.4.0",
|
|
28
|
+
"typescript": "^5.1.5"
|
|
29
|
+
},
|
|
30
|
+
"dependencies": {
|
|
31
|
+
"@quatrain/core": "^1.1.46",
|
|
32
|
+
"@quatrain/queue": "^1.1.24",
|
|
33
|
+
"@quatrain/storage": "^1.1.26",
|
|
34
|
+
"axios": "^1.7.7",
|
|
35
|
+
"fluent-ffmpeg": "^2.1.2",
|
|
36
|
+
"fs-extra": "^11.2.0",
|
|
37
|
+
"node-fetch-native": "^1.6.4"
|
|
38
|
+
},
|
|
39
|
+
"scripts": {
|
|
40
|
+
"test-ci": "jest --runInBand",
|
|
41
|
+
"build": "tsc",
|
|
42
|
+
"wbuild": "tsc --watch",
|
|
43
|
+
"bump-to": "yarn version"
|
|
44
|
+
}
|
|
45
|
+
}
|