mb-rrvideo-server 1.0.14 → 1.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docker-compose.yml +1 -1
- package/ecosystem.docker.config.js +2 -1
- package/package.json +1 -1
- package/src/logger.js +8 -0
- package/src/routes/convert.js +48 -130
- package/src/routes/merge.js +6 -1
- package/src/utils/download.js +192 -0
- package/src/utils/event-files.js +157 -0
- package/src/utils/pull-task-data.js +117 -0
- package/src/utils/video.js +5 -0
package/docker-compose.yml
CHANGED
|
@@ -11,7 +11,8 @@ module.exports = {
|
|
|
11
11
|
max_memory_restart: '2G',
|
|
12
12
|
env: {
|
|
13
13
|
NODE_ENV: 'production',
|
|
14
|
-
CONFIG_PATH: '/app/config/config.json'
|
|
14
|
+
CONFIG_PATH: '/app/config/config.json',
|
|
15
|
+
HOSTNAME: process.env.HOSTNAME // 显式传递 Docker Hostname
|
|
15
16
|
},
|
|
16
17
|
error_file: '/app/logs/pm2-error.log',
|
|
17
18
|
out_file: '/app/logs/pm2-out.log',
|
package/package.json
CHANGED
package/src/logger.js
CHANGED
|
@@ -58,6 +58,14 @@ class Logger {
|
|
|
58
58
|
const dateStr = this.getCurrentDate();
|
|
59
59
|
const logDir = path.join(this.baseDir, logType, dateStr);
|
|
60
60
|
this.ensureDir(logDir);
|
|
61
|
+
|
|
62
|
+
// 在多实例/集群环境下,避免 system.log 冲突
|
|
63
|
+
// 优先使用 HOSTNAME (Docker环境),其次使用 PID (普通Node环境)
|
|
64
|
+
if (fileName === 'system') {
|
|
65
|
+
const instanceId = process.env.HOSTNAME || process.pid;
|
|
66
|
+
return path.join(logDir, `${fileName}_${instanceId}.log`);
|
|
67
|
+
}
|
|
68
|
+
|
|
61
69
|
return path.join(logDir, `${fileName}.log`);
|
|
62
70
|
}
|
|
63
71
|
|
package/src/routes/convert.js
CHANGED
|
@@ -5,118 +5,15 @@
|
|
|
5
5
|
|
|
6
6
|
const fs = require('fs');
|
|
7
7
|
const path = require('path');
|
|
8
|
-
const crypto = require('crypto');
|
|
9
|
-
const http = require('http');
|
|
10
|
-
const https = require('https');
|
|
11
|
-
const url = require('url');
|
|
12
8
|
const config = require('../config');
|
|
13
9
|
const logger = require('../logger');
|
|
14
10
|
const taskManager = require('../task');
|
|
15
11
|
const storageManager = require('../storage');
|
|
16
12
|
const { sendCallback } = require('../utils/callback');
|
|
13
|
+
const { downloadAndMergeManifestEventFiles, normalizeManifestEventFiles } = require('../utils/event-files');
|
|
14
|
+
const { pullTaskManifest } = require('../utils/pull-task-data');
|
|
17
15
|
const { runRrvideosCommand, buildConvertCommand, setPermissions, cleanupTempConfigFile } = require('../utils/video');
|
|
18
16
|
|
|
19
|
-
/**
|
|
20
|
-
* 从pull_data_url拉取任务数据(带签名验证)
|
|
21
|
-
* @param {string} pullDataUrl - 拉取数据的URL
|
|
22
|
-
* @param {string} recordId - 记录ID
|
|
23
|
-
* @param {string} logFileName - 日志文件名
|
|
24
|
-
* @param {Array} tasks - 任务列表(V2使用,可选)
|
|
25
|
-
* @returns {Promise<Object>} - 返回拉取到的数据
|
|
26
|
-
*/
|
|
27
|
-
async function pullTaskData(pullDataUrl, recordId, logFileName, tasks = null) {
|
|
28
|
-
return new Promise((resolve, reject) => {
|
|
29
|
-
try {
|
|
30
|
-
// 构造签名参数
|
|
31
|
-
const source = 'converter_internal';
|
|
32
|
-
const timestamp = Math.floor(Date.now() / 1000);
|
|
33
|
-
const secret = crypto.createHash('md5').update(source + '@BBX').digest('hex');
|
|
34
|
-
|
|
35
|
-
// 签名数组(用于计算签名)
|
|
36
|
-
const signArray = {
|
|
37
|
-
record_id: recordId,
|
|
38
|
-
source: source,
|
|
39
|
-
timestamp: timestamp,
|
|
40
|
-
secret: secret
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
// 排序并生成签名字符串
|
|
44
|
-
const sortedKeys = Object.keys(signArray).sort();
|
|
45
|
-
const signParts = sortedKeys.map(key => `${key}=${signArray[key]}`);
|
|
46
|
-
const signStr = signParts.join('&');
|
|
47
|
-
const sign = crypto.createHash('md5').update(signStr).digest('hex');
|
|
48
|
-
|
|
49
|
-
logger.debug(`[pullTaskData] 签名计算: sign_str=${signStr}, sign=${sign}`, logFileName, 'convert');
|
|
50
|
-
|
|
51
|
-
// ✅ 判断是V1还是V2:V2会传入tasks参数,V1不会
|
|
52
|
-
const isV2 = tasks && Array.isArray(tasks) && tasks.length > 0;
|
|
53
|
-
|
|
54
|
-
// ✅ 构造请求参数(V1使用record_id,V2使用record_no)
|
|
55
|
-
const postData = {
|
|
56
|
-
[isV2 ? 'record_no' : 'record_id']: recordId,
|
|
57
|
-
source: source,
|
|
58
|
-
timestamp: timestamp,
|
|
59
|
-
sign: sign
|
|
60
|
-
};
|
|
61
|
-
|
|
62
|
-
// V2需要传递tasks
|
|
63
|
-
if (isV2) {
|
|
64
|
-
postData.tasks = JSON.stringify(tasks);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
const postDataStr = Object.keys(postData).map(key =>
|
|
68
|
-
`${encodeURIComponent(key)}=${encodeURIComponent(postData[key])}`
|
|
69
|
-
).join('&');
|
|
70
|
-
|
|
71
|
-
logger.debug(`[pullTaskData] 请求参数 (${isV2 ? 'V2' : 'V1'}): ${postDataStr}`, logFileName, 'convert');
|
|
72
|
-
|
|
73
|
-
const pullUrl = new URL(pullDataUrl);
|
|
74
|
-
const httpModule = pullUrl.protocol === 'https:' ? https : http;
|
|
75
|
-
|
|
76
|
-
const options = {
|
|
77
|
-
hostname: pullUrl.hostname,
|
|
78
|
-
port: pullUrl.port || (pullUrl.protocol === 'https:' ? 443 : 80),
|
|
79
|
-
path: pullUrl.pathname + (pullUrl.search || ''),
|
|
80
|
-
method: 'POST',
|
|
81
|
-
headers: {
|
|
82
|
-
'Content-Type': 'application/x-www-form-urlencoded',
|
|
83
|
-
'Content-Length': Buffer.byteLength(postDataStr)
|
|
84
|
-
},
|
|
85
|
-
rejectUnauthorized: false
|
|
86
|
-
};
|
|
87
|
-
|
|
88
|
-
const req = httpModule.request(options, (resp) => {
|
|
89
|
-
let body = '';
|
|
90
|
-
resp.on('data', (chunk) => (body += chunk));
|
|
91
|
-
resp.on('end', () => {
|
|
92
|
-
try {
|
|
93
|
-
const data = JSON.parse(body);
|
|
94
|
-
logger.debug(`[pullTaskData] 响应数据: ${body}`, logFileName, 'convert');
|
|
95
|
-
|
|
96
|
-
if (data.result_code !== 0) {
|
|
97
|
-
reject(new Error(data.message || '拉取数据失败'));
|
|
98
|
-
} else {
|
|
99
|
-
resolve(data);
|
|
100
|
-
}
|
|
101
|
-
} catch (parseError) {
|
|
102
|
-
reject(new Error(`解析响应失败: ${parseError.message}`));
|
|
103
|
-
}
|
|
104
|
-
});
|
|
105
|
-
});
|
|
106
|
-
|
|
107
|
-
req.on('error', (err) => {
|
|
108
|
-
reject(new Error(`请求失败: ${err.message}`));
|
|
109
|
-
});
|
|
110
|
-
|
|
111
|
-
req.write(postDataStr);
|
|
112
|
-
req.end();
|
|
113
|
-
|
|
114
|
-
} catch (error) {
|
|
115
|
-
reject(error);
|
|
116
|
-
}
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
|
-
|
|
120
17
|
/**
|
|
121
18
|
* 生成任务ID
|
|
122
19
|
*/
|
|
@@ -183,18 +80,21 @@ async function handleConvertV1(inputData, res, logFileName) {
|
|
|
183
80
|
logger.info(`[${actionName}] 开始拉取任务详细数据,URL: ${pullDataUrl}`, logFileName, 'convert');
|
|
184
81
|
|
|
185
82
|
try {
|
|
186
|
-
const pullData = await
|
|
83
|
+
const pullData = await pullTaskManifest(pullDataUrl, recordId, logFileName);
|
|
84
|
+
logger.info(`[${actionName}] manifest 接口返回: ${JSON.stringify(pullData)}`, logFileName, 'convert');
|
|
187
85
|
|
|
188
|
-
|
|
86
|
+
const eventFiles = pullData.data ? normalizeManifestEventFiles(pullData.data) : [];
|
|
87
|
+
if (!eventFiles.length) {
|
|
189
88
|
logger.error(`[${actionName}] 拉取数据格式错误,完整数据: ${JSON.stringify(pullData)}`, logFileName, 'convert');
|
|
190
89
|
throw new Error('拉取的数据格式不正确');
|
|
191
90
|
}
|
|
192
|
-
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
91
|
+
|
|
92
|
+
const eventsSize = eventFiles.reduce((sum, item) => sum + Number(item.size || 0), 0);
|
|
93
|
+
logger.info(
|
|
94
|
+
`[${actionName}] manifest 拉取完成,事件分片数=${eventFiles.length}, events_size=${eventsSize}`,
|
|
95
|
+
logFileName,
|
|
96
|
+
'convert'
|
|
97
|
+
);
|
|
198
98
|
|
|
199
99
|
// ✅ 创建临时目录和文件
|
|
200
100
|
const basePath = config.get('storage.local.temp_dir') || config.get('storage.local.base_path') || './';
|
|
@@ -204,10 +104,17 @@ async function handleConvertV1(inputData, res, logFileName) {
|
|
|
204
104
|
|
|
205
105
|
const tempFile = 'record.data';
|
|
206
106
|
const tempFilePath = path.join(tempDir, tempFile);
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
107
|
+
|
|
108
|
+
await downloadAndMergeManifestEventFiles(pullData.data, {
|
|
109
|
+
targetPath: tempFilePath,
|
|
110
|
+
tempDir,
|
|
111
|
+
logger,
|
|
112
|
+
logFileName,
|
|
113
|
+
logType: 'convert',
|
|
114
|
+
retries: 2,
|
|
115
|
+
retryDelayMs: 1000
|
|
116
|
+
});
|
|
117
|
+
logger.debug(`[${actionName}] 事件JSON分片已下载并拼接到临时文件: ${tempFilePath}`, logFileName, 'convert');
|
|
211
118
|
|
|
212
119
|
// ✅ 构造输出路径
|
|
213
120
|
const now = new Date();
|
|
@@ -488,13 +395,14 @@ async function handleConvertV2(inputData, res, logFileName) {
|
|
|
488
395
|
return;
|
|
489
396
|
}
|
|
490
397
|
|
|
491
|
-
|
|
398
|
+
logger.info(`[${actionName}] 开始拉取任务详细数据,URL: ${pullDataUrl}`, logFileName, 'convert');
|
|
492
399
|
|
|
493
400
|
try {
|
|
494
|
-
const pullData = await
|
|
401
|
+
const pullData = await pullTaskManifest(pullDataUrl, recordId, logFileName, tasks.map(t => ({
|
|
495
402
|
record_page_id: t.record_page_id,
|
|
496
403
|
record_page_no: t.record_page_no
|
|
497
404
|
})));
|
|
405
|
+
logger.info(`[${actionName}] manifest 接口返回: ${JSON.stringify(pullData)}`, logFileName, 'convert');
|
|
498
406
|
|
|
499
407
|
if (!pullData.data || !pullData.data.tasks) {
|
|
500
408
|
logger.error(`[${actionName}] 拉取数据格式错误,完整数据: ${JSON.stringify(pullData)}`, logFileName, 'convert');
|
|
@@ -521,15 +429,18 @@ async function handleConvertV2(inputData, res, logFileName) {
|
|
|
521
429
|
pt.record_page_no == recordPageNo
|
|
522
430
|
);
|
|
523
431
|
|
|
524
|
-
|
|
525
|
-
|
|
432
|
+
const eventFiles = pulledTask ? normalizeManifestEventFiles(pulledTask) : [];
|
|
433
|
+
if (!eventFiles.length) {
|
|
434
|
+
logger.warn(`[${actionName}] 第${i}个任务(page_id=${recordPageId})缺少事件文件清单,跳过`, logFileName, 'convert');
|
|
526
435
|
continue; // 跳过,不添加到validTasksTemp
|
|
527
436
|
}
|
|
528
|
-
|
|
529
|
-
const
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
437
|
+
|
|
438
|
+
const eventsSize = eventFiles.reduce((sum, item) => sum + Number(item.size || 0), 0);
|
|
439
|
+
logger.info(
|
|
440
|
+
`[${actionName}] 第${i}个任务 manifest: page_id=${recordPageId}, 分片数=${eventFiles.length}, events_size=${eventsSize}`,
|
|
441
|
+
logFileName,
|
|
442
|
+
'convert'
|
|
443
|
+
);
|
|
533
444
|
|
|
534
445
|
// ✅ 创建临时目录和文件(照着PHP的逻辑)
|
|
535
446
|
const basePath = config.get('storage.local.temp_dir') || config.get('storage.local.base_path') || './';
|
|
@@ -539,10 +450,17 @@ async function handleConvertV2(inputData, res, logFileName) {
|
|
|
539
450
|
|
|
540
451
|
const tempFile = `record_${recordPageId}.data`;
|
|
541
452
|
const tempFilePath = path.join(tempDir, tempFile);
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
453
|
+
|
|
454
|
+
await downloadAndMergeManifestEventFiles(pulledTask, {
|
|
455
|
+
targetPath: tempFilePath,
|
|
456
|
+
tempDir,
|
|
457
|
+
logger,
|
|
458
|
+
logFileName,
|
|
459
|
+
logType: 'convert',
|
|
460
|
+
retries: 2,
|
|
461
|
+
retryDelayMs: 1000
|
|
462
|
+
});
|
|
463
|
+
logger.debug(`[${actionName}] 第${i}个任务事件JSON分片已下载并拼接: ${tempFilePath}`, logFileName, 'convert');
|
|
546
464
|
|
|
547
465
|
// ✅ 构造输出路径(照着PHP的逻辑)
|
|
548
466
|
const now = new Date();
|
package/src/routes/merge.js
CHANGED
|
@@ -83,13 +83,18 @@ async function handleMerge(inputData, res, logFileName) {
|
|
|
83
83
|
logger.debug(`[${actionName}] 视频列表(${videos.length}个): ${JSON.stringify(videos)}`, taskId, 'merge');
|
|
84
84
|
logger.info(`[${actionName}] 输出路径: ${output}`, taskId, 'merge');
|
|
85
85
|
logger.debug(`[${actionName}] 回调数据: ${JSON.stringify(callbackData)}`, taskId, 'merge');
|
|
86
|
+
|
|
87
|
+
const mergeDownloadDir = path.join(process.cwd(), '__rrvideos__temp__', 'merge_downloads', String(taskId));
|
|
88
|
+
ensureDir(mergeDownloadDir);
|
|
89
|
+
logger.info(`[${actionName}] 合并临时下载目录: ${mergeDownloadDir}`, taskId, 'merge');
|
|
86
90
|
|
|
87
91
|
// 生成合并命令
|
|
88
92
|
const command = buildMergeCommand(videos, output, {
|
|
89
93
|
format,
|
|
90
94
|
method,
|
|
91
95
|
auto_mode: autoMode,
|
|
92
|
-
headers
|
|
96
|
+
headers,
|
|
97
|
+
download_dir: mergeDownloadDir
|
|
93
98
|
});
|
|
94
99
|
|
|
95
100
|
logger.info(`[${actionName}] 合并指令:\n${command}`, taskId, 'merge');
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const http = require('http');
|
|
4
|
+
const https = require('https');
|
|
5
|
+
|
|
6
|
+
function ensureDir(dirPath) {
|
|
7
|
+
if (!fs.existsSync(dirPath)) {
|
|
8
|
+
fs.mkdirSync(dirPath, { recursive: true });
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function formatBytes(bytes) {
|
|
13
|
+
if (!bytes || bytes < 0) {
|
|
14
|
+
return '0 B';
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const units = ['B', 'KB', 'MB', 'GB'];
|
|
18
|
+
let size = bytes;
|
|
19
|
+
let unitIndex = 0;
|
|
20
|
+
|
|
21
|
+
while (size >= 1024 && unitIndex < units.length - 1) {
|
|
22
|
+
size /= 1024;
|
|
23
|
+
unitIndex++;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
return `${size.toFixed(unitIndex === 0 ? 0 : 2)} ${units[unitIndex]}`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function unlinkQuietly(filePath) {
|
|
30
|
+
try {
|
|
31
|
+
if (filePath && fs.existsSync(filePath)) {
|
|
32
|
+
fs.unlinkSync(filePath);
|
|
33
|
+
}
|
|
34
|
+
} catch (error) {
|
|
35
|
+
// Ignore cleanup failures.
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function sleep(ms) {
|
|
40
|
+
return new Promise((resolve) => {
|
|
41
|
+
setTimeout(resolve, ms);
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function downloadUrlToFile(downloadUrl, targetPath, options = {}) {
|
|
46
|
+
const logger = options.logger;
|
|
47
|
+
const logFileName = options.logFileName || 'download';
|
|
48
|
+
const logType = options.logType || 'convert';
|
|
49
|
+
const maxRedirects = typeof options.maxRedirects === 'number' ? options.maxRedirects : 5;
|
|
50
|
+
|
|
51
|
+
return new Promise((resolve, reject) => {
|
|
52
|
+
const startedAt = Date.now();
|
|
53
|
+
ensureDir(path.dirname(targetPath));
|
|
54
|
+
|
|
55
|
+
const requestOnce = (requestUrl, redirectCount) => {
|
|
56
|
+
let request;
|
|
57
|
+
let fileStream;
|
|
58
|
+
let bytesWritten = 0;
|
|
59
|
+
let contentLength = 0;
|
|
60
|
+
|
|
61
|
+
try {
|
|
62
|
+
const parsed = new URL(requestUrl);
|
|
63
|
+
const client = parsed.protocol === 'https:' ? https : http;
|
|
64
|
+
|
|
65
|
+
logger && logger.info(`[downloadUrlToFile] 开始下载事件文件: ${requestUrl}`, logFileName, logType);
|
|
66
|
+
|
|
67
|
+
request = client.get({
|
|
68
|
+
hostname: parsed.hostname,
|
|
69
|
+
port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
|
|
70
|
+
path: parsed.pathname + (parsed.search || ''),
|
|
71
|
+
headers: options.headers || {},
|
|
72
|
+
rejectUnauthorized: false
|
|
73
|
+
}, (response) => {
|
|
74
|
+
if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
|
|
75
|
+
if (redirectCount >= maxRedirects) {
|
|
76
|
+
response.resume();
|
|
77
|
+
reject(new Error(`下载重定向次数过多: ${requestUrl}`));
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const nextUrl = new URL(response.headers.location, requestUrl).toString();
|
|
82
|
+
logger && logger.info(`[downloadUrlToFile] 跟随重定向: ${nextUrl}`, logFileName, logType);
|
|
83
|
+
response.resume();
|
|
84
|
+
requestOnce(nextUrl, redirectCount + 1);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (response.statusCode !== 200) {
|
|
89
|
+
response.resume();
|
|
90
|
+
reject(new Error(`下载失败,HTTP状态码: ${response.statusCode}`));
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
contentLength = Number(response.headers['content-length'] || 0);
|
|
95
|
+
fileStream = fs.createWriteStream(targetPath);
|
|
96
|
+
|
|
97
|
+
response.on('data', (chunk) => {
|
|
98
|
+
bytesWritten += chunk.length;
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
response.on('error', (error) => {
|
|
102
|
+
if (fileStream) {
|
|
103
|
+
fileStream.destroy();
|
|
104
|
+
}
|
|
105
|
+
unlinkQuietly(targetPath);
|
|
106
|
+
reject(error);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
fileStream.on('error', (error) => {
|
|
110
|
+
response.destroy(error);
|
|
111
|
+
unlinkQuietly(targetPath);
|
|
112
|
+
reject(error);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
fileStream.on('finish', () => {
|
|
116
|
+
const durationMs = Date.now() - startedAt;
|
|
117
|
+
logger && logger.info(
|
|
118
|
+
`[downloadUrlToFile] 下载完成: ${targetPath}, 大小=${formatBytes(bytesWritten)}${contentLength ? `/${formatBytes(contentLength)}` : ''}, 耗时=${durationMs}ms`,
|
|
119
|
+
logFileName,
|
|
120
|
+
logType
|
|
121
|
+
);
|
|
122
|
+
resolve({
|
|
123
|
+
bytesWritten,
|
|
124
|
+
contentLength,
|
|
125
|
+
durationMs,
|
|
126
|
+
targetPath
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
response.pipe(fileStream);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
request.on('error', (error) => {
|
|
134
|
+
unlinkQuietly(targetPath);
|
|
135
|
+
reject(error);
|
|
136
|
+
});
|
|
137
|
+
} catch (error) {
|
|
138
|
+
unlinkQuietly(targetPath);
|
|
139
|
+
reject(error);
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
requestOnce(downloadUrl, 0);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async function downloadUrlToFileWithRetry(downloadUrl, targetPath, options = {}) {
|
|
148
|
+
const retries = typeof options.retries === 'number' ? options.retries : 2;
|
|
149
|
+
const retryDelayMs = typeof options.retryDelayMs === 'number' ? options.retryDelayMs : 1000;
|
|
150
|
+
const logger = options.logger;
|
|
151
|
+
const logFileName = options.logFileName || 'download';
|
|
152
|
+
const logType = options.logType || 'convert';
|
|
153
|
+
|
|
154
|
+
let lastError = null;
|
|
155
|
+
|
|
156
|
+
for (let attempt = 0; attempt <= retries; attempt += 1) {
|
|
157
|
+
try {
|
|
158
|
+
if (attempt > 0) {
|
|
159
|
+
logger && logger.info(
|
|
160
|
+
`[downloadUrlToFileWithRetry] 第${attempt + 1}次下载重试: ${downloadUrl}`,
|
|
161
|
+
logFileName,
|
|
162
|
+
logType
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return await downloadUrlToFile(downloadUrl, targetPath, options);
|
|
167
|
+
} catch (error) {
|
|
168
|
+
lastError = error;
|
|
169
|
+
unlinkQuietly(targetPath);
|
|
170
|
+
|
|
171
|
+
if (attempt >= retries) {
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
logger && logger.warn && logger.warn(
|
|
176
|
+
`[downloadUrlToFileWithRetry] 下载失败,${retryDelayMs}ms后重试: ${error.message}`,
|
|
177
|
+
logFileName,
|
|
178
|
+
logType
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
await sleep(retryDelayMs);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
throw lastError;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
module.exports = {
|
|
189
|
+
downloadUrlToFile,
|
|
190
|
+
downloadUrlToFileWithRetry,
|
|
191
|
+
formatBytes
|
|
192
|
+
};
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const { downloadUrlToFileWithRetry } = require('./download');
|
|
4
|
+
|
|
5
|
+
function ensureDir(dirPath) {
|
|
6
|
+
if (!fs.existsSync(dirPath)) {
|
|
7
|
+
fs.mkdirSync(dirPath, { recursive: true });
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function normalizeManifestEventFiles(payload) {
|
|
12
|
+
if (!payload || typeof payload !== 'object') {
|
|
13
|
+
return [];
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
if (Array.isArray(payload.events_files) && payload.events_files.length) {
|
|
17
|
+
return payload.events_files.map((item) => ({
|
|
18
|
+
url: item.url || item.events_url || '',
|
|
19
|
+
size: Number(item.size || item.events_size || 0)
|
|
20
|
+
})).filter((item) => item.url);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (Array.isArray(payload.events_urls) && payload.events_urls.length) {
|
|
24
|
+
return payload.events_urls.map((item) => {
|
|
25
|
+
if (item && typeof item === 'object') {
|
|
26
|
+
return {
|
|
27
|
+
url: item.url || item.events_url || '',
|
|
28
|
+
size: Number(item.size || item.events_size || 0)
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return {
|
|
33
|
+
url: item,
|
|
34
|
+
size: 0
|
|
35
|
+
};
|
|
36
|
+
}).filter((item) => item.url);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (payload.events_url) {
|
|
40
|
+
return [{
|
|
41
|
+
url: payload.events_url,
|
|
42
|
+
size: Number(payload.events_size || 0)
|
|
43
|
+
}];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return [];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function extractEventArrayContent(rawContent) {
|
|
50
|
+
const trimmed = String(rawContent || '').trim();
|
|
51
|
+
if (!trimmed) {
|
|
52
|
+
return '';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
|
|
56
|
+
return trimmed.slice(1, -1).trim();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const parsed = JSON.parse(trimmed);
|
|
60
|
+
const events = Array.isArray(parsed) ? parsed : (parsed && Array.isArray(parsed.events) ? parsed.events : null);
|
|
61
|
+
if (!events) {
|
|
62
|
+
throw new Error('事件文件不是数组结构');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return JSON.stringify(events).slice(1, -1).trim();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function mergeEventJsonFilesToFile(sourcePaths, targetPath) {
|
|
69
|
+
ensureDir(path.dirname(targetPath));
|
|
70
|
+
|
|
71
|
+
const chunks = [];
|
|
72
|
+
for (const sourcePath of sourcePaths) {
|
|
73
|
+
const content = fs.readFileSync(sourcePath, 'utf8');
|
|
74
|
+
const innerContent = extractEventArrayContent(content);
|
|
75
|
+
if (innerContent) {
|
|
76
|
+
chunks.push(innerContent);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const mergedContent = `[${chunks.join(',')}]`;
|
|
81
|
+
fs.writeFileSync(targetPath, mergedContent, 'utf8');
|
|
82
|
+
|
|
83
|
+
return {
|
|
84
|
+
targetPath,
|
|
85
|
+
fileCount: sourcePaths.length
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async function downloadAndMergeManifestEventFiles(payload, options = {}) {
|
|
90
|
+
const eventFiles = normalizeManifestEventFiles(payload);
|
|
91
|
+
if (!eventFiles.length) {
|
|
92
|
+
throw new Error('manifest 中缺少事件文件地址');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const logger = options.logger;
|
|
96
|
+
const logFileName = options.logFileName || 'event-files';
|
|
97
|
+
const logType = options.logType || 'convert';
|
|
98
|
+
const tempDir = options.tempDir;
|
|
99
|
+
const targetPath = options.targetPath;
|
|
100
|
+
|
|
101
|
+
if (!tempDir || !targetPath) {
|
|
102
|
+
throw new Error('tempDir 和 targetPath 不能为空');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
ensureDir(tempDir);
|
|
106
|
+
|
|
107
|
+
const downloadedPaths = [];
|
|
108
|
+
try {
|
|
109
|
+
for (let index = 0; index < eventFiles.length; index += 1) {
|
|
110
|
+
const file = eventFiles[index];
|
|
111
|
+
const partPath = path.join(tempDir, `part_${index + 1}.json`);
|
|
112
|
+
logger && logger.info(
|
|
113
|
+
`[downloadAndMergeManifestEventFiles] 下载事件分片 ${index + 1}/${eventFiles.length}: ${file.url}`,
|
|
114
|
+
logFileName,
|
|
115
|
+
logType
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
await downloadUrlToFileWithRetry(file.url, partPath, {
|
|
119
|
+
logger,
|
|
120
|
+
logFileName,
|
|
121
|
+
logType,
|
|
122
|
+
retries: typeof options.retries === 'number' ? options.retries : 2,
|
|
123
|
+
retryDelayMs: typeof options.retryDelayMs === 'number' ? options.retryDelayMs : 1000
|
|
124
|
+
});
|
|
125
|
+
downloadedPaths.push(partPath);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
await mergeEventJsonFilesToFile(downloadedPaths, targetPath);
|
|
129
|
+
logger && logger.info(
|
|
130
|
+
`[downloadAndMergeManifestEventFiles] 事件分片拼接完成: ${targetPath}, 分片数=${eventFiles.length}`,
|
|
131
|
+
logFileName,
|
|
132
|
+
logType
|
|
133
|
+
);
|
|
134
|
+
|
|
135
|
+
return {
|
|
136
|
+
targetPath,
|
|
137
|
+
fileCount: eventFiles.length,
|
|
138
|
+
totalSize: eventFiles.reduce((sum, item) => sum + Number(item.size || 0), 0)
|
|
139
|
+
};
|
|
140
|
+
} finally {
|
|
141
|
+
for (const downloadedPath of downloadedPaths) {
|
|
142
|
+
try {
|
|
143
|
+
if (fs.existsSync(downloadedPath)) {
|
|
144
|
+
fs.unlinkSync(downloadedPath);
|
|
145
|
+
}
|
|
146
|
+
} catch (error) {
|
|
147
|
+
// Ignore cleanup failures.
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
module.exports = {
|
|
154
|
+
normalizeManifestEventFiles,
|
|
155
|
+
mergeEventJsonFilesToFile,
|
|
156
|
+
downloadAndMergeManifestEventFiles
|
|
157
|
+
};
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
const crypto = require('crypto');
|
|
2
|
+
const http = require('http');
|
|
3
|
+
const https = require('https');
|
|
4
|
+
const logger = require('../logger');
|
|
5
|
+
|
|
6
|
+
function buildSignedPayload(recordId, tasks = null) {
|
|
7
|
+
const source = 'converter_internal';
|
|
8
|
+
const timestamp = Math.floor(Date.now() / 1000);
|
|
9
|
+
const secret = crypto.createHash('md5').update(source + '@BBX').digest('hex');
|
|
10
|
+
const signArray = {
|
|
11
|
+
record_id: recordId,
|
|
12
|
+
source,
|
|
13
|
+
timestamp,
|
|
14
|
+
secret
|
|
15
|
+
};
|
|
16
|
+
const sortedKeys = Object.keys(signArray).sort();
|
|
17
|
+
const signStr = sortedKeys.map((key) => `${key}=${signArray[key]}`).join('&');
|
|
18
|
+
const sign = crypto.createHash('md5').update(signStr).digest('hex');
|
|
19
|
+
|
|
20
|
+
const isV2 = Array.isArray(tasks) && tasks.length > 0;
|
|
21
|
+
const payload = {
|
|
22
|
+
[isV2 ? 'record_no' : 'record_id']: recordId,
|
|
23
|
+
source,
|
|
24
|
+
timestamp,
|
|
25
|
+
sign,
|
|
26
|
+
data_mode: 'url'
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
if (isV2) {
|
|
30
|
+
const pageIds = tasks
|
|
31
|
+
.map((task) => task.record_page_id || task.recordPageId || '')
|
|
32
|
+
.filter((value) => value !== '' && value !== null && value !== undefined)
|
|
33
|
+
.join(',');
|
|
34
|
+
|
|
35
|
+
if (pageIds) {
|
|
36
|
+
payload.page_ids = pageIds;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
payload,
|
|
42
|
+
signStr,
|
|
43
|
+
sign,
|
|
44
|
+
isV2
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function pullTaskManifest(pullDataUrl, recordId, logFileName, tasks = null) {
|
|
49
|
+
return new Promise((resolve, reject) => {
|
|
50
|
+
try {
|
|
51
|
+
const { payload, signStr, sign, isV2 } = buildSignedPayload(recordId, tasks);
|
|
52
|
+
const postDataStr = Object.keys(payload).map((key) =>
|
|
53
|
+
`${encodeURIComponent(key)}=${encodeURIComponent(payload[key])}`
|
|
54
|
+
).join('&');
|
|
55
|
+
|
|
56
|
+
logger.debug(`[pullTaskManifest] 签名计算: sign_str=${signStr}, sign=${sign}`, logFileName, 'convert');
|
|
57
|
+
logger.debug(`[pullTaskManifest] 请求参数 (${isV2 ? 'V2' : 'V1'}): ${postDataStr}`, logFileName, 'convert');
|
|
58
|
+
|
|
59
|
+
const pullUrl = new URL(pullDataUrl);
|
|
60
|
+
const httpModule = pullUrl.protocol === 'https:' ? https : http;
|
|
61
|
+
const options = {
|
|
62
|
+
hostname: pullUrl.hostname,
|
|
63
|
+
port: pullUrl.port || (pullUrl.protocol === 'https:' ? 443 : 80),
|
|
64
|
+
path: pullUrl.pathname + (pullUrl.search || ''),
|
|
65
|
+
method: 'POST',
|
|
66
|
+
headers: {
|
|
67
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
68
|
+
'Content-Length': Buffer.byteLength(postDataStr)
|
|
69
|
+
},
|
|
70
|
+
rejectUnauthorized: false
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const req = httpModule.request(options, (resp) => {
|
|
74
|
+
let body = '';
|
|
75
|
+
|
|
76
|
+
resp.on('data', (chunk) => {
|
|
77
|
+
body += chunk;
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
resp.on('end', () => {
|
|
81
|
+
if (resp.statusCode !== 200) {
|
|
82
|
+
reject(new Error(`拉取 manifest 失败,HTTP状态码: ${resp.statusCode}`));
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
try {
|
|
87
|
+
const data = JSON.parse(body);
|
|
88
|
+
logger.debug(`[pullTaskManifest] 响应数据: ${body}`, logFileName, 'convert');
|
|
89
|
+
|
|
90
|
+
if (data.result_code !== 0) {
|
|
91
|
+
reject(new Error(data.message || '拉取 manifest 失败'));
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
resolve(data);
|
|
96
|
+
} catch (parseError) {
|
|
97
|
+
reject(new Error(`解析 manifest 响应失败: ${parseError.message}`));
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
req.on('error', (err) => {
|
|
103
|
+
reject(new Error(`请求 manifest 失败: ${err.message}`));
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
req.write(postDataStr);
|
|
107
|
+
req.end();
|
|
108
|
+
} catch (error) {
|
|
109
|
+
reject(error);
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
module.exports = {
|
|
115
|
+
buildSignedPayload,
|
|
116
|
+
pullTaskManifest
|
|
117
|
+
};
|
package/src/utils/video.js
CHANGED
|
@@ -183,12 +183,17 @@ function buildMergeCommand(videos, output, options = {}) {
|
|
|
183
183
|
const method = options.method || 'concat';
|
|
184
184
|
const autoMode = options.auto_mode || 'smart';
|
|
185
185
|
const headers = options.headers || {};
|
|
186
|
+
const downloadDir = options.download_dir;
|
|
186
187
|
|
|
187
188
|
let command = `${nodePath} --expose-gc ${rrvideosPath} merge --output ${escapeArg(output)} --format ${escapeArg(format)} --method ${escapeArg(method)} --auto-mode ${escapeArg(autoMode)}`;
|
|
188
189
|
|
|
189
190
|
videos.forEach((v) => {
|
|
190
191
|
command += ` --videos ${escapeArg(v)}`;
|
|
191
192
|
});
|
|
193
|
+
|
|
194
|
+
if (downloadDir) {
|
|
195
|
+
command += ` --download-dir ${escapeArg(downloadDir)}`;
|
|
196
|
+
}
|
|
192
197
|
|
|
193
198
|
Object.keys(headers).forEach((k) => {
|
|
194
199
|
const pair = `${k}:${headers[k]}`;
|