@mkterswingman/5mghost-yonder 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -3
- package/dist/auth/sharedAuth.d.ts +10 -0
- package/dist/auth/sharedAuth.js +24 -0
- package/dist/auth/tokenManager.d.ts +10 -1
- package/dist/auth/tokenManager.js +14 -22
- package/dist/cli/check.js +6 -3
- package/dist/cli/index.d.ts +15 -1
- package/dist/cli/index.js +74 -31
- package/dist/cli/runtime.d.ts +9 -0
- package/dist/cli/runtime.js +35 -0
- package/dist/cli/serve.js +3 -1
- package/dist/cli/setup.d.ts +3 -0
- package/dist/cli/setup.js +84 -68
- package/dist/cli/setupCookies.js +2 -2
- package/dist/cli/smoke.d.ts +27 -0
- package/dist/cli/smoke.js +108 -0
- package/dist/cli/uninstall.d.ts +1 -0
- package/dist/cli/uninstall.js +67 -0
- package/dist/download/downloader.d.ts +64 -0
- package/dist/download/downloader.js +264 -0
- package/dist/download/jobManager.d.ts +21 -0
- package/dist/download/jobManager.js +198 -0
- package/dist/download/types.d.ts +43 -0
- package/dist/download/types.js +1 -0
- package/dist/runtime/ffmpegRuntime.d.ts +13 -0
- package/dist/runtime/ffmpegRuntime.js +51 -0
- package/dist/runtime/installers.d.ts +12 -0
- package/dist/runtime/installers.js +45 -0
- package/dist/runtime/manifest.d.ts +18 -0
- package/dist/runtime/manifest.js +43 -0
- package/dist/runtime/playwrightRuntime.d.ts +13 -0
- package/dist/runtime/playwrightRuntime.js +37 -0
- package/dist/runtime/systemDeps.d.ts +3 -0
- package/dist/runtime/systemDeps.js +30 -0
- package/dist/runtime/ytdlpRuntime.d.ts +14 -0
- package/dist/runtime/ytdlpRuntime.js +58 -0
- package/dist/server.d.ts +3 -1
- package/dist/server.js +4 -1
- package/dist/tools/downloads.d.ts +11 -0
- package/dist/tools/downloads.js +220 -0
- package/dist/tools/subtitles.d.ts +25 -0
- package/dist/tools/subtitles.js +135 -47
- package/dist/utils/config.d.ts +28 -0
- package/dist/utils/config.js +40 -11
- package/dist/utils/ffmpeg.d.ts +5 -0
- package/dist/utils/ffmpeg.js +16 -0
- package/dist/utils/ffmpegPath.d.ts +8 -0
- package/dist/utils/ffmpegPath.js +21 -0
- package/dist/utils/formatters.d.ts +4 -0
- package/dist/utils/formatters.js +42 -0
- package/dist/utils/mediaPaths.d.ts +7 -0
- package/dist/utils/mediaPaths.js +10 -0
- package/dist/utils/openClaw.d.ts +17 -0
- package/dist/utils/openClaw.js +79 -0
- package/dist/utils/videoInput.js +3 -0
- package/dist/utils/videoMetadata.d.ts +11 -0
- package/dist/utils/videoMetadata.js +1 -0
- package/dist/utils/ytdlp.d.ts +17 -1
- package/dist/utils/ytdlp.js +89 -2
- package/dist/utils/ytdlpPath.d.ts +9 -2
- package/dist/utils/ytdlpPath.js +19 -20
- package/dist/utils/ytdlpProgress.d.ts +13 -0
- package/dist/utils/ytdlpProgress.js +77 -0
- package/package.json +5 -3
- package/scripts/download-ytdlp.mjs +1 -1
- package/scripts/install.ps1 +9 -0
- package/scripts/install.sh +15 -0
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import { mkdir, rm, stat, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { downloadSubtitlesForLanguages, toReadableSubtitleJobError } from "../tools/subtitles.js";
|
|
4
|
+
import { loadConfig } from "../utils/config.js";
|
|
5
|
+
import { hasFfmpeg, isRuntimeMissingMessage } from "../utils/ffmpeg.js";
|
|
6
|
+
import { formatBytes } from "../utils/formatters.js";
|
|
7
|
+
import { hasYtDlp, runYtDlp, runYtDlpJson } from "../utils/ytdlp.js";
|
|
8
|
+
import { parseYtDlpProgressLine } from "../utils/ytdlpProgress.js";
|
|
9
|
+
export async function downloadOneItem(input) {
|
|
10
|
+
const deps = resolveDeps(input.deps);
|
|
11
|
+
const manager = input.jobManager;
|
|
12
|
+
const jobId = input.jobId;
|
|
13
|
+
const videoQuality = normalizeVideoQuality(input.videoQuality);
|
|
14
|
+
const subtitleFormats = [...input.subtitleFormats];
|
|
15
|
+
const usingDefaultSubtitleLanguages = input.subtitleLanguages == null;
|
|
16
|
+
const subtitleLanguages = [...(input.subtitleLanguages ?? input.defaultSubtitleLanguages ?? loadConfig().default_languages)];
|
|
17
|
+
const metadataFile = join(input.item.output_dir, "metadata.json");
|
|
18
|
+
const subtitlesDir = join(input.item.output_dir, "subtitles");
|
|
19
|
+
const videoFile = join(input.item.output_dir, "video.mp4");
|
|
20
|
+
let currentStep = "metadata";
|
|
21
|
+
try {
|
|
22
|
+
assertManagerInput(manager, jobId);
|
|
23
|
+
manager?.startItem(jobId, input.item.video_id, currentStep);
|
|
24
|
+
if (!deps.hasYtDlp()) {
|
|
25
|
+
throw new Error(isRuntimeMissingMessage("yt-dlp"));
|
|
26
|
+
}
|
|
27
|
+
if (needsVideo(input.mode) && !deps.hasFfmpeg()) {
|
|
28
|
+
throw new Error(isRuntimeMissingMessage("ffmpeg"));
|
|
29
|
+
}
|
|
30
|
+
// Why: reruns intentionally reuse the same YYYY-MM-DD_<video_id> directory, so stale artifacts must be cleared first.
|
|
31
|
+
await deps.rm(input.item.output_dir, { recursive: true, force: true });
|
|
32
|
+
await deps.mkdir(input.item.output_dir, { recursive: true });
|
|
33
|
+
await deps.mkdir(subtitlesDir, { recursive: true });
|
|
34
|
+
const metadata = await deps.fetchMetadata({
|
|
35
|
+
sourceUrl: input.item.source_url,
|
|
36
|
+
videoId: input.item.video_id,
|
|
37
|
+
title: input.item.title,
|
|
38
|
+
});
|
|
39
|
+
const result = {
|
|
40
|
+
video_file: null,
|
|
41
|
+
metadata_file: metadataFile,
|
|
42
|
+
final_file_size: null,
|
|
43
|
+
subtitle_files: {},
|
|
44
|
+
metadata,
|
|
45
|
+
};
|
|
46
|
+
const onProgress = (update) => {
|
|
47
|
+
currentStep = update.step;
|
|
48
|
+
manager?.updateItemProgress(jobId, input.item.video_id, update.step, update.progressPercent);
|
|
49
|
+
};
|
|
50
|
+
if (needsVideo(input.mode)) {
|
|
51
|
+
currentStep = "download_video";
|
|
52
|
+
manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 0);
|
|
53
|
+
result.video_file = await deps.downloadVideo({
|
|
54
|
+
sourceUrl: input.item.source_url,
|
|
55
|
+
outputFile: videoFile,
|
|
56
|
+
videoQuality,
|
|
57
|
+
onProgress,
|
|
58
|
+
});
|
|
59
|
+
result.final_file_size = formatBytes((await deps.stat(result.video_file)).size);
|
|
60
|
+
}
|
|
61
|
+
if (needsSubtitles(input.mode)) {
|
|
62
|
+
currentStep = "download_subtitles";
|
|
63
|
+
manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 0);
|
|
64
|
+
result.subtitle_files = await deps.downloadSubtitles({
|
|
65
|
+
videoId: input.item.video_id,
|
|
66
|
+
sourceUrl: input.item.source_url,
|
|
67
|
+
subtitlesDir,
|
|
68
|
+
formats: subtitleFormats,
|
|
69
|
+
languages: subtitleLanguages,
|
|
70
|
+
skipMissingLanguages: usingDefaultSubtitleLanguages,
|
|
71
|
+
onProgress,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
currentStep = "write_metadata";
|
|
75
|
+
manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 100);
|
|
76
|
+
await deps.writeMetadata(metadataFile, JSON.stringify(buildMetadataDocument({
|
|
77
|
+
item: input.item,
|
|
78
|
+
mode: input.mode,
|
|
79
|
+
videoQuality,
|
|
80
|
+
subtitleFormats,
|
|
81
|
+
metadata,
|
|
82
|
+
result,
|
|
83
|
+
}), null, 2));
|
|
84
|
+
manager?.completeItem(jobId, input.item.video_id, {
|
|
85
|
+
video_file: result.video_file,
|
|
86
|
+
metadata_file: result.metadata_file,
|
|
87
|
+
final_file_size: result.final_file_size,
|
|
88
|
+
subtitle_files: result.subtitle_files,
|
|
89
|
+
});
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
const readableError = toErrorMessage(error);
|
|
94
|
+
if (manager && jobId) {
|
|
95
|
+
manager.failItem(jobId, input.item.video_id, readableError, currentStep);
|
|
96
|
+
}
|
|
97
|
+
throw new Error(readableError);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function resolveDeps(overrides) {
|
|
101
|
+
return {
|
|
102
|
+
hasYtDlp,
|
|
103
|
+
hasFfmpeg,
|
|
104
|
+
fetchMetadata: fetchMetadataWithYtDlp,
|
|
105
|
+
downloadVideo: downloadVideoWithYtDlp,
|
|
106
|
+
downloadSubtitles: downloadSubtitlesWithYtDlp,
|
|
107
|
+
mkdir: async (path, options) => {
|
|
108
|
+
await mkdir(path, options);
|
|
109
|
+
},
|
|
110
|
+
rm: async (path, options) => {
|
|
111
|
+
await rm(path, options);
|
|
112
|
+
},
|
|
113
|
+
stat,
|
|
114
|
+
writeMetadata: async (path, contents) => {
|
|
115
|
+
await writeFile(path, contents, "utf8");
|
|
116
|
+
},
|
|
117
|
+
...overrides,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
async function fetchMetadataWithYtDlp(input) {
|
|
121
|
+
const result = await runYtDlpJson([
|
|
122
|
+
"--dump-single-json",
|
|
123
|
+
"--skip-download",
|
|
124
|
+
"--no-playlist",
|
|
125
|
+
input.sourceUrl,
|
|
126
|
+
]);
|
|
127
|
+
if (!result.ok) {
|
|
128
|
+
throw new Error(result.error);
|
|
129
|
+
}
|
|
130
|
+
return {
|
|
131
|
+
video_id: result.value.id ?? input.videoId,
|
|
132
|
+
source_url: result.value.webpage_url ?? input.sourceUrl,
|
|
133
|
+
title: result.value.title ?? input.title,
|
|
134
|
+
description: result.value.description,
|
|
135
|
+
channel_id: result.value.channel_id ?? result.value.uploader_id,
|
|
136
|
+
channel_title: result.value.channel ?? result.value.uploader,
|
|
137
|
+
published_at: normalizeUploadDate(result.value.upload_date),
|
|
138
|
+
duration: result.value.duration_string ?? formatDuration(result.value.duration),
|
|
139
|
+
thumbnail_url: result.value.thumbnail,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
async function downloadVideoWithYtDlp(input) {
|
|
143
|
+
let lastProgress = 0;
|
|
144
|
+
const result = await runYtDlp([
|
|
145
|
+
"--no-playlist",
|
|
146
|
+
"--newline",
|
|
147
|
+
"--progress",
|
|
148
|
+
"--format",
|
|
149
|
+
resolveVideoFormat(input.videoQuality),
|
|
150
|
+
"--merge-output-format",
|
|
151
|
+
"mp4",
|
|
152
|
+
"--output",
|
|
153
|
+
input.outputFile.replace(/\.mp4$/u, ".%(ext)s"),
|
|
154
|
+
"--print",
|
|
155
|
+
"after_move:filepath",
|
|
156
|
+
input.sourceUrl,
|
|
157
|
+
], 600_000, (line) => {
|
|
158
|
+
const parsed = parseYtDlpProgressLine(line);
|
|
159
|
+
if (parsed) {
|
|
160
|
+
lastProgress = Number(parsed.progress.replace(/%$/u, ""));
|
|
161
|
+
input.onProgress({ step: "download_video", progressPercent: lastProgress });
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (line.includes("[Merger]")) {
|
|
165
|
+
input.onProgress({ step: "merge", progressPercent: Math.max(lastProgress, 99) });
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
if (result.exitCode !== 0) {
|
|
169
|
+
throw new Error(result.stderr.slice(0, 500) || `yt-dlp exited with ${result.exitCode}`);
|
|
170
|
+
}
|
|
171
|
+
return resolveYtDlpOutputPath(input.outputFile, result.stdout);
|
|
172
|
+
}
|
|
173
|
+
async function downloadSubtitlesWithYtDlp(input) {
|
|
174
|
+
input.onProgress({ step: "download_subtitles", progressPercent: 0 });
|
|
175
|
+
return downloadSubtitlesForLanguages({
|
|
176
|
+
videoId: input.videoId,
|
|
177
|
+
sourceUrl: input.sourceUrl,
|
|
178
|
+
formats: input.formats,
|
|
179
|
+
languages: input.languages,
|
|
180
|
+
subtitlesDir: input.subtitlesDir,
|
|
181
|
+
skipMissingLanguages: input.skipMissingLanguages,
|
|
182
|
+
onProgress: (completed, total) => {
|
|
183
|
+
input.onProgress({
|
|
184
|
+
step: "download_subtitles",
|
|
185
|
+
progressPercent: total === 0 ? 100 : Math.round((completed / total) * 100),
|
|
186
|
+
});
|
|
187
|
+
},
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
function buildMetadataDocument(input) {
|
|
191
|
+
return {
|
|
192
|
+
video_id: input.metadata.video_id,
|
|
193
|
+
source_url: input.metadata.source_url,
|
|
194
|
+
title: input.metadata.title,
|
|
195
|
+
description: input.metadata.description ?? null,
|
|
196
|
+
channel_id: input.metadata.channel_id ?? null,
|
|
197
|
+
channel_title: input.metadata.channel_title ?? null,
|
|
198
|
+
published_at: input.metadata.published_at ?? null,
|
|
199
|
+
duration: input.metadata.duration ?? null,
|
|
200
|
+
thumbnail_url: input.metadata.thumbnail_url ?? null,
|
|
201
|
+
mode: input.mode,
|
|
202
|
+
video_quality: needsVideo(input.mode) ? input.videoQuality : null,
|
|
203
|
+
subtitle_formats: [...input.subtitleFormats],
|
|
204
|
+
output_dir: input.item.output_dir,
|
|
205
|
+
video_file: input.result.video_file,
|
|
206
|
+
metadata_file: input.result.metadata_file,
|
|
207
|
+
final_file_size: input.result.final_file_size,
|
|
208
|
+
subtitle_files: sortSubtitleFiles(input.result.subtitle_files),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
export function resolveYtDlpOutputPath(expectedPath, stdout) {
|
|
212
|
+
const lines = stdout
|
|
213
|
+
.split(/\r?\n/u)
|
|
214
|
+
.map((line) => line.trim())
|
|
215
|
+
.filter(Boolean);
|
|
216
|
+
return lines.at(-1) ?? expectedPath;
|
|
217
|
+
}
|
|
218
|
+
function assertManagerInput(manager, jobId) {
|
|
219
|
+
if ((manager == null) !== (jobId == null)) {
|
|
220
|
+
throw new Error("jobManager and jobId must be provided together.");
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
function normalizeVideoQuality(videoQuality) {
|
|
224
|
+
return videoQuality === "max" ? "max" : "1080p";
|
|
225
|
+
}
|
|
226
|
+
function resolveVideoFormat(videoQuality) {
|
|
227
|
+
if (videoQuality === "max") {
|
|
228
|
+
return "bv*+ba/b";
|
|
229
|
+
}
|
|
230
|
+
return "bv*[height<=1080]+ba/b[height<=1080]/b";
|
|
231
|
+
}
|
|
232
|
+
function needsVideo(mode) {
|
|
233
|
+
return mode === "video" || mode === "both";
|
|
234
|
+
}
|
|
235
|
+
function needsSubtitles(mode) {
|
|
236
|
+
return mode === "subtitles" || mode === "both";
|
|
237
|
+
}
|
|
238
|
+
function normalizeUploadDate(uploadDate) {
|
|
239
|
+
if (!uploadDate || !/^\d{8}$/u.test(uploadDate)) {
|
|
240
|
+
return undefined;
|
|
241
|
+
}
|
|
242
|
+
return `${uploadDate.slice(0, 4)}-${uploadDate.slice(4, 6)}-${uploadDate.slice(6, 8)}`;
|
|
243
|
+
}
|
|
244
|
+
function formatDuration(durationSeconds) {
|
|
245
|
+
if (!Number.isFinite(durationSeconds) || durationSeconds == null || durationSeconds < 0) {
|
|
246
|
+
return undefined;
|
|
247
|
+
}
|
|
248
|
+
const wholeSeconds = Math.floor(durationSeconds);
|
|
249
|
+
const hours = Math.floor(wholeSeconds / 3600);
|
|
250
|
+
const minutes = Math.floor((wholeSeconds % 3600) / 60);
|
|
251
|
+
const seconds = wholeSeconds % 60;
|
|
252
|
+
if (hours > 0) {
|
|
253
|
+
return `${hours}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}`;
|
|
254
|
+
}
|
|
255
|
+
return `${minutes}:${String(seconds).padStart(2, "0")}`;
|
|
256
|
+
}
|
|
257
|
+
function sortSubtitleFiles(files) {
|
|
258
|
+
return Object.fromEntries(Object.entries(files)
|
|
259
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
260
|
+
.map(([format, paths]) => [format, [...paths].sort()]));
|
|
261
|
+
}
|
|
262
|
+
function toErrorMessage(error) {
|
|
263
|
+
return toReadableSubtitleJobError(error);
|
|
264
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { CreateDownloadJobInput, DownloadJobItemSnapshot, DownloadJobSnapshot, DownloadStep } from "./types.js";
|
|
2
|
+
interface DownloadJobManagerOptions {
|
|
3
|
+
now?: () => string;
|
|
4
|
+
}
|
|
5
|
+
export declare class DownloadJobManager {
|
|
6
|
+
private readonly jobs;
|
|
7
|
+
private readonly now;
|
|
8
|
+
constructor(options?: DownloadJobManagerOptions);
|
|
9
|
+
createJob(input: CreateDownloadJobInput): DownloadJobSnapshot;
|
|
10
|
+
pollJob(jobId: string): DownloadJobSnapshot;
|
|
11
|
+
startItem(jobId: string, videoId: string, step?: DownloadStep): DownloadJobSnapshot;
|
|
12
|
+
updateItemProgress(jobId: string, videoId: string, step: DownloadStep, progressPercent: number | null): DownloadJobSnapshot;
|
|
13
|
+
completeItem(jobId: string, videoId: string, outputs?: Pick<DownloadJobItemSnapshot, "video_file" | "metadata_file" | "final_file_size" | "subtitle_files">): DownloadJobSnapshot;
|
|
14
|
+
failItem(jobId: string, videoId: string, error: string, step?: DownloadStep): DownloadJobSnapshot;
|
|
15
|
+
private applyItemMutation;
|
|
16
|
+
private refreshJobStatus;
|
|
17
|
+
private getJob;
|
|
18
|
+
private getItem;
|
|
19
|
+
private toSnapshot;
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
export class DownloadJobManager {
|
|
3
|
+
jobs = new Map();
|
|
4
|
+
now;
|
|
5
|
+
constructor(options = {}) {
|
|
6
|
+
this.now = options.now ?? (() => new Date().toISOString());
|
|
7
|
+
}
|
|
8
|
+
createJob(input) {
|
|
9
|
+
assertNonEmptyVideos(input);
|
|
10
|
+
const createdAt = this.now();
|
|
11
|
+
const jobId = randomUUID();
|
|
12
|
+
assertUniqueVideoIds(input);
|
|
13
|
+
const job = {
|
|
14
|
+
job_id: jobId,
|
|
15
|
+
status: "queued",
|
|
16
|
+
mode: input.mode,
|
|
17
|
+
video_quality: input.video_quality ?? null,
|
|
18
|
+
subtitle_formats: [...input.subtitle_formats],
|
|
19
|
+
created_at: createdAt,
|
|
20
|
+
started_at: null,
|
|
21
|
+
completed_at: null,
|
|
22
|
+
total: input.videos.length,
|
|
23
|
+
items: input.videos.map((video) => ({
|
|
24
|
+
video_id: video.video_id,
|
|
25
|
+
source_url: video.source_url,
|
|
26
|
+
title: video.title,
|
|
27
|
+
output_dir: video.output_dir,
|
|
28
|
+
status: "queued",
|
|
29
|
+
step: null,
|
|
30
|
+
progress_percent: null,
|
|
31
|
+
error: null,
|
|
32
|
+
started_at: null,
|
|
33
|
+
completed_at: null,
|
|
34
|
+
video_file: null,
|
|
35
|
+
metadata_file: null,
|
|
36
|
+
final_file_size: null,
|
|
37
|
+
subtitle_files: {},
|
|
38
|
+
})),
|
|
39
|
+
};
|
|
40
|
+
this.jobs.set(jobId, job);
|
|
41
|
+
return this.toSnapshot(job);
|
|
42
|
+
}
|
|
43
|
+
pollJob(jobId) {
|
|
44
|
+
return this.toSnapshot(this.getJob(jobId));
|
|
45
|
+
}
|
|
46
|
+
startItem(jobId, videoId, step = "metadata") {
|
|
47
|
+
const job = this.getJob(jobId);
|
|
48
|
+
const item = this.getItem(job, videoId);
|
|
49
|
+
this.applyItemMutation(job, item, {
|
|
50
|
+
status: "running",
|
|
51
|
+
step,
|
|
52
|
+
progress_percent: 0,
|
|
53
|
+
error: null,
|
|
54
|
+
});
|
|
55
|
+
return this.toSnapshot(job);
|
|
56
|
+
}
|
|
57
|
+
updateItemProgress(jobId, videoId, step, progressPercent) {
|
|
58
|
+
const job = this.getJob(jobId);
|
|
59
|
+
const item = this.getItem(job, videoId);
|
|
60
|
+
this.applyItemMutation(job, item, {
|
|
61
|
+
status: "running",
|
|
62
|
+
step,
|
|
63
|
+
progress_percent: progressPercent == null ? null : clampProgress(progressPercent),
|
|
64
|
+
error: null,
|
|
65
|
+
});
|
|
66
|
+
return this.toSnapshot(job);
|
|
67
|
+
}
|
|
68
|
+
completeItem(jobId, videoId, outputs) {
|
|
69
|
+
const job = this.getJob(jobId);
|
|
70
|
+
const item = this.getItem(job, videoId);
|
|
71
|
+
this.applyItemMutation(job, item, {
|
|
72
|
+
status: "completed",
|
|
73
|
+
step: "done",
|
|
74
|
+
progress_percent: 100,
|
|
75
|
+
error: null,
|
|
76
|
+
video_file: outputs?.video_file,
|
|
77
|
+
metadata_file: outputs?.metadata_file,
|
|
78
|
+
final_file_size: outputs?.final_file_size,
|
|
79
|
+
subtitle_files: outputs?.subtitle_files,
|
|
80
|
+
});
|
|
81
|
+
return this.toSnapshot(job);
|
|
82
|
+
}
|
|
83
|
+
failItem(jobId, videoId, error, step) {
|
|
84
|
+
const job = this.getJob(jobId);
|
|
85
|
+
const item = this.getItem(job, videoId);
|
|
86
|
+
this.applyItemMutation(job, item, {
|
|
87
|
+
status: "failed",
|
|
88
|
+
step: step ?? item.step,
|
|
89
|
+
error,
|
|
90
|
+
});
|
|
91
|
+
return this.toSnapshot(job);
|
|
92
|
+
}
|
|
93
|
+
applyItemMutation(job, item, mutation) {
|
|
94
|
+
const now = this.now();
|
|
95
|
+
const nextStatus = mutation.status;
|
|
96
|
+
const nextStep = mutation.step === undefined ? item.step : mutation.step;
|
|
97
|
+
const previousStatus = item.status;
|
|
98
|
+
item.status = nextStatus;
|
|
99
|
+
item.step = nextStep;
|
|
100
|
+
item.video_file = mutation.video_file === undefined ? item.video_file : mutation.video_file;
|
|
101
|
+
item.metadata_file = mutation.metadata_file === undefined ? item.metadata_file : mutation.metadata_file;
|
|
102
|
+
item.final_file_size =
|
|
103
|
+
mutation.final_file_size === undefined ? item.final_file_size : mutation.final_file_size;
|
|
104
|
+
item.subtitle_files = mutation.subtitle_files === undefined ? item.subtitle_files : cloneSubtitleFiles(mutation.subtitle_files);
|
|
105
|
+
if (nextStatus === "running") {
|
|
106
|
+
item.started_at = item.started_at ?? now;
|
|
107
|
+
job.started_at = job.started_at ?? now;
|
|
108
|
+
item.completed_at = null;
|
|
109
|
+
item.error = null;
|
|
110
|
+
item.progress_percent = resolveRunningProgress(item.progress_percent, mutation.progress_percent, previousStatus);
|
|
111
|
+
}
|
|
112
|
+
if (nextStatus === "completed" || nextStatus === "failed") {
|
|
113
|
+
item.started_at = item.started_at ?? now;
|
|
114
|
+
job.started_at = job.started_at ?? now;
|
|
115
|
+
item.completed_at = now;
|
|
116
|
+
item.error = mutation.error === undefined ? item.error : mutation.error;
|
|
117
|
+
item.progress_percent =
|
|
118
|
+
mutation.progress_percent === undefined ? item.progress_percent : clampProgressOrNull(mutation.progress_percent);
|
|
119
|
+
}
|
|
120
|
+
this.refreshJobStatus(job, now);
|
|
121
|
+
}
|
|
122
|
+
refreshJobStatus(job, now = this.now()) {
|
|
123
|
+
const statuses = job.items.map((item) => item.status);
|
|
124
|
+
if (statuses.every((status) => status === "completed")) {
|
|
125
|
+
job.status = "completed";
|
|
126
|
+
job.completed_at = job.completed_at ?? now;
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (statuses.every((status) => status === "completed" || status === "failed")) {
|
|
130
|
+
job.status = "failed";
|
|
131
|
+
job.completed_at = job.completed_at ?? now;
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
if (job.started_at !== null || statuses.some((status) => status !== "queued")) {
|
|
135
|
+
job.status = "running";
|
|
136
|
+
job.completed_at = null;
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
job.status = "queued";
|
|
140
|
+
job.completed_at = null;
|
|
141
|
+
}
|
|
142
|
+
getJob(jobId) {
|
|
143
|
+
const job = this.jobs.get(jobId);
|
|
144
|
+
if (!job) {
|
|
145
|
+
throw new Error(`Unknown download job: ${jobId}`);
|
|
146
|
+
}
|
|
147
|
+
return job;
|
|
148
|
+
}
|
|
149
|
+
getItem(job, videoId) {
|
|
150
|
+
const item = job.items.find((entry) => entry.video_id === videoId);
|
|
151
|
+
if (!item) {
|
|
152
|
+
throw new Error(`Unknown download item: ${videoId}`);
|
|
153
|
+
}
|
|
154
|
+
return item;
|
|
155
|
+
}
|
|
156
|
+
toSnapshot(job) {
|
|
157
|
+
return {
|
|
158
|
+
...job,
|
|
159
|
+
subtitle_formats: [...job.subtitle_formats],
|
|
160
|
+
items: job.items.map((item) => ({
|
|
161
|
+
...item,
|
|
162
|
+
subtitle_files: cloneSubtitleFiles(item.subtitle_files),
|
|
163
|
+
})),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
function clampProgress(value) {
|
|
168
|
+
return Math.max(0, Math.min(100, value));
|
|
169
|
+
}
|
|
170
|
+
function clampProgressOrNull(value) {
|
|
171
|
+
return value == null ? null : clampProgress(value);
|
|
172
|
+
}
|
|
173
|
+
function resolveRunningProgress(currentProgress, nextProgress, previousStatus) {
|
|
174
|
+
if (nextProgress !== undefined) {
|
|
175
|
+
return clampProgressOrNull(nextProgress);
|
|
176
|
+
}
|
|
177
|
+
if (previousStatus === "completed" || previousStatus === "failed") {
|
|
178
|
+
return 0;
|
|
179
|
+
}
|
|
180
|
+
return currentProgress ?? 0;
|
|
181
|
+
}
|
|
182
|
+
function assertUniqueVideoIds(input) {
|
|
183
|
+
const seen = new Set();
|
|
184
|
+
for (const video of input.videos) {
|
|
185
|
+
if (seen.has(video.video_id)) {
|
|
186
|
+
throw new Error(`Duplicate video_id in download job: ${video.video_id}`);
|
|
187
|
+
}
|
|
188
|
+
seen.add(video.video_id);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
function assertNonEmptyVideos(input) {
|
|
192
|
+
if (input.videos.length === 0) {
|
|
193
|
+
throw new Error("Download job must include at least one video");
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
function cloneSubtitleFiles(files) {
|
|
197
|
+
return Object.fromEntries(Object.entries(files).map(([format, paths]) => [format, [...paths]]));
|
|
198
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export type DownloadMode = "video" | "subtitles" | "both";
|
|
2
|
+
export type DownloadItemStatus = "queued" | "running" | "completed" | "failed";
|
|
3
|
+
export type DownloadStep = "metadata" | "download_video" | "download_subtitles" | "merge" | "write_metadata" | "done";
|
|
4
|
+
export interface DownloadJobVideoInput {
|
|
5
|
+
video_id: string;
|
|
6
|
+
source_url: string;
|
|
7
|
+
title: string;
|
|
8
|
+
output_dir: string;
|
|
9
|
+
}
|
|
10
|
+
export interface CreateDownloadJobInput {
|
|
11
|
+
videos: DownloadJobVideoInput[];
|
|
12
|
+
mode: DownloadMode;
|
|
13
|
+
video_quality?: string;
|
|
14
|
+
subtitle_formats: string[];
|
|
15
|
+
}
|
|
16
|
+
export interface DownloadJobItemSnapshot {
|
|
17
|
+
video_id: string;
|
|
18
|
+
source_url: string;
|
|
19
|
+
title: string;
|
|
20
|
+
output_dir: string;
|
|
21
|
+
status: DownloadItemStatus;
|
|
22
|
+
step: DownloadStep | null;
|
|
23
|
+
progress_percent: number | null;
|
|
24
|
+
error: string | null;
|
|
25
|
+
started_at: string | null;
|
|
26
|
+
completed_at: string | null;
|
|
27
|
+
video_file: string | null;
|
|
28
|
+
metadata_file: string | null;
|
|
29
|
+
final_file_size: string | null;
|
|
30
|
+
subtitle_files: Record<string, string[]>;
|
|
31
|
+
}
|
|
32
|
+
export interface DownloadJobSnapshot {
|
|
33
|
+
job_id: string;
|
|
34
|
+
status: DownloadItemStatus;
|
|
35
|
+
mode: DownloadMode;
|
|
36
|
+
video_quality: string | null;
|
|
37
|
+
subtitle_formats: string[];
|
|
38
|
+
created_at: string;
|
|
39
|
+
started_at: string | null;
|
|
40
|
+
completed_at: string | null;
|
|
41
|
+
total: number;
|
|
42
|
+
items: DownloadJobItemSnapshot[];
|
|
43
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RuntimeComponentState } from "./manifest.js";
|
|
2
|
+
export declare function checkFfmpegRuntime(): RuntimeComponentState & {
|
|
3
|
+
name: "ffmpeg";
|
|
4
|
+
message?: string;
|
|
5
|
+
};
|
|
6
|
+
export declare function installFfmpegRuntime(): Promise<RuntimeComponentState & {
|
|
7
|
+
name: "ffmpeg";
|
|
8
|
+
message?: string;
|
|
9
|
+
}>;
|
|
10
|
+
export declare function updateFfmpegRuntime(): Promise<RuntimeComponentState & {
|
|
11
|
+
name: "ffmpeg";
|
|
12
|
+
message?: string;
|
|
13
|
+
}>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { execFileSync, execSync } from "node:child_process";
|
|
2
|
+
import { getFfmpegPath, getRuntimeFfmpegPath } from "../utils/ffmpegPath.js";
|
|
3
|
+
import { getFfmpegInstallHint } from "./systemDeps.js";
|
|
4
|
+
function getFfmpegVersion(binPath) {
|
|
5
|
+
try {
|
|
6
|
+
const version = execFileSync(binPath, ["-version"], {
|
|
7
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
8
|
+
timeout: 30_000,
|
|
9
|
+
}).toString("utf8");
|
|
10
|
+
const firstLine = version.split("\n")[0]?.trim();
|
|
11
|
+
return firstLine?.replace(/^ffmpeg version\s+/, "") ?? null;
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export function checkFfmpegRuntime() {
|
|
18
|
+
const resolved = getFfmpegPath();
|
|
19
|
+
const version = getFfmpegVersion(resolved);
|
|
20
|
+
if (!version) {
|
|
21
|
+
return {
|
|
22
|
+
name: "ffmpeg",
|
|
23
|
+
status: "missing",
|
|
24
|
+
version: null,
|
|
25
|
+
source: "system",
|
|
26
|
+
installed_at: null,
|
|
27
|
+
binary_path: null,
|
|
28
|
+
message: getFfmpegInstallHint(),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
const source = process.env.FFMPEG_PATH
|
|
32
|
+
? "env"
|
|
33
|
+
: resolved === getRuntimeFfmpegPath()
|
|
34
|
+
? "runtime"
|
|
35
|
+
: "system";
|
|
36
|
+
return {
|
|
37
|
+
name: "ffmpeg",
|
|
38
|
+
status: "installed",
|
|
39
|
+
version,
|
|
40
|
+
source,
|
|
41
|
+
installed_at: null,
|
|
42
|
+
binary_path: resolved,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export async function installFfmpegRuntime() {
|
|
46
|
+
execSync(getFfmpegInstallHint(), { stdio: "inherit" });
|
|
47
|
+
return checkFfmpegRuntime();
|
|
48
|
+
}
|
|
49
|
+
export async function updateFfmpegRuntime() {
|
|
50
|
+
return installFfmpegRuntime();
|
|
51
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type RuntimeComponentName, type RuntimeComponentState } from "./manifest.js";
|
|
2
|
+
export interface RuntimeComponentSummary extends RuntimeComponentState {
|
|
3
|
+
name: RuntimeComponentName;
|
|
4
|
+
message?: string;
|
|
5
|
+
}
|
|
6
|
+
export interface RuntimeSummary {
|
|
7
|
+
action: "check" | "install" | "update";
|
|
8
|
+
components: RuntimeComponentSummary[];
|
|
9
|
+
}
|
|
10
|
+
export declare function checkAll(): Promise<RuntimeSummary>;
|
|
11
|
+
export declare function installAll(): Promise<RuntimeSummary>;
|
|
12
|
+
export declare function updateAll(): Promise<RuntimeSummary>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { checkPlaywrightRuntime, installPlaywrightRuntime, updatePlaywrightRuntime, } from "./playwrightRuntime.js";
|
|
2
|
+
import { checkYtDlpRuntime, installYtDlpRuntime, updateYtDlpRuntime } from "./ytdlpRuntime.js";
|
|
3
|
+
import { checkFfmpegRuntime, installFfmpegRuntime, updateFfmpegRuntime } from "./ffmpegRuntime.js";
|
|
4
|
+
import { createEmptyRuntimeManifest, loadRuntimeManifest, saveRuntimeManifest, updateRuntimeComponent, } from "./manifest.js";
|
|
5
|
+
import { PATHS } from "../utils/config.js";
|
|
6
|
+
function persistComponents(components) {
|
|
7
|
+
let manifest = loadRuntimeManifest(PATHS.runtimeManifestJson) ?? createEmptyRuntimeManifest();
|
|
8
|
+
for (const component of components) {
|
|
9
|
+
manifest = updateRuntimeComponent(manifest, component.name, {
|
|
10
|
+
status: component.status,
|
|
11
|
+
version: component.version,
|
|
12
|
+
source: component.source,
|
|
13
|
+
installed_at: component.installed_at,
|
|
14
|
+
binary_path: component.binary_path,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
saveRuntimeManifest(PATHS.runtimeManifestJson, manifest);
|
|
18
|
+
}
|
|
19
|
+
export async function checkAll() {
|
|
20
|
+
const components = [
|
|
21
|
+
await checkPlaywrightRuntime(),
|
|
22
|
+
checkYtDlpRuntime(),
|
|
23
|
+
checkFfmpegRuntime(),
|
|
24
|
+
];
|
|
25
|
+
persistComponents(components);
|
|
26
|
+
return { action: "check", components };
|
|
27
|
+
}
|
|
28
|
+
export async function installAll() {
|
|
29
|
+
const components = [
|
|
30
|
+
await installPlaywrightRuntime(),
|
|
31
|
+
await installYtDlpRuntime(),
|
|
32
|
+
await installFfmpegRuntime(),
|
|
33
|
+
];
|
|
34
|
+
persistComponents(components);
|
|
35
|
+
return { action: "install", components };
|
|
36
|
+
}
|
|
37
|
+
export async function updateAll() {
|
|
38
|
+
const components = [
|
|
39
|
+
await updatePlaywrightRuntime(),
|
|
40
|
+
await updateYtDlpRuntime(),
|
|
41
|
+
await updateFfmpegRuntime(),
|
|
42
|
+
];
|
|
43
|
+
persistComponents(components);
|
|
44
|
+
return { action: "update", components };
|
|
45
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export type RuntimeComponentName = "playwright" | "yt-dlp" | "ffmpeg";
|
|
2
|
+
export type RuntimeComponentStatus = "missing" | "installed";
|
|
3
|
+
export type RuntimeComponentSource = "runtime" | "system" | "env";
|
|
4
|
+
export interface RuntimeComponentState {
|
|
5
|
+
status: RuntimeComponentStatus;
|
|
6
|
+
version: string | null;
|
|
7
|
+
source: RuntimeComponentSource;
|
|
8
|
+
installed_at: string | null;
|
|
9
|
+
binary_path: string | null;
|
|
10
|
+
}
|
|
11
|
+
export interface RuntimeManifest {
|
|
12
|
+
schema_version: 1;
|
|
13
|
+
components: Partial<Record<RuntimeComponentName, RuntimeComponentState>>;
|
|
14
|
+
}
|
|
15
|
+
export declare function createEmptyRuntimeManifest(): RuntimeManifest;
|
|
16
|
+
export declare function loadRuntimeManifest(manifestPath?: string): RuntimeManifest;
|
|
17
|
+
export declare function saveRuntimeManifest(manifestPath: string | undefined, manifest: RuntimeManifest): void;
|
|
18
|
+
export declare function updateRuntimeComponent(manifest: RuntimeManifest, name: RuntimeComponentName, state: RuntimeComponentState): RuntimeManifest;
|