getraw 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitattributes +4 -0
- package/CLAUDE.md +57 -0
- package/README.md +166 -0
- package/RESEARCH.md +109 -0
- package/STATUS.md +23 -0
- package/bun.lock +50 -0
- package/bunfig.toml +3 -0
- package/docs/plugin-guide.md +166 -0
- package/docs/supported-sites.md +41 -0
- package/package.json +30 -0
- package/src/cli/index.ts +52 -0
- package/src/cli/options.ts +97 -0
- package/src/core/format-sorter.ts +208 -0
- package/src/core/logger.ts +101 -0
- package/src/core/orchestrator.ts +140 -0
- package/src/core/output-template.ts +58 -0
- package/src/core/types.ts +237 -0
- package/src/downloaders/base.ts +25 -0
- package/src/downloaders/dash.ts +287 -0
- package/src/downloaders/fragment.ts +226 -0
- package/src/downloaders/hls.ts +170 -0
- package/src/downloaders/http.ts +260 -0
- package/src/extractors/archive-org.ts +126 -0
- package/src/extractors/bandcamp.ts +130 -0
- package/src/extractors/base.ts +29 -0
- package/src/extractors/bilibili/bangumi.ts +205 -0
- package/src/extractors/bilibili/index.ts +233 -0
- package/src/extractors/bilibili/wbi.ts +60 -0
- package/src/extractors/coub.ts +137 -0
- package/src/extractors/dailymotion.ts +99 -0
- package/src/extractors/dropbox.ts +52 -0
- package/src/extractors/generic.ts +118 -0
- package/src/extractors/google-drive.ts +106 -0
- package/src/extractors/imgur.ts +156 -0
- package/src/extractors/instagram/index.ts +263 -0
- package/src/extractors/instagram/reels.ts +166 -0
- package/src/extractors/kick/clips.ts +91 -0
- package/src/extractors/kick/index.ts +118 -0
- package/src/extractors/kick/live.ts +89 -0
- package/src/extractors/niconico/index.ts +209 -0
- package/src/extractors/odysee.ts +126 -0
- package/src/extractors/peertube.ts +143 -0
- package/src/extractors/reddit/gallery.ts +124 -0
- package/src/extractors/reddit/index.ts +203 -0
- package/src/extractors/rumble.ts +127 -0
- package/src/extractors/soundcloud/index.ts +161 -0
- package/src/extractors/soundcloud/playlist.ts +129 -0
- package/src/extractors/spotify.ts +97 -0
- package/src/extractors/streamable.ts +121 -0
- package/src/extractors/ted.ts +151 -0
- package/src/extractors/tiktok/index.ts +207 -0
- package/src/extractors/tiktok/user.ts +176 -0
- package/src/extractors/twitch/clips.ts +125 -0
- package/src/extractors/twitch/index.ts +136 -0
- package/src/extractors/twitch/live.ts +132 -0
- package/src/extractors/twitter/index.ts +140 -0
- package/src/extractors/twitter/spaces.ts +200 -0
- package/src/extractors/vimeo/index.ts +187 -0
- package/src/extractors/youtube/captions.ts +111 -0
- package/src/extractors/youtube/index.ts +252 -0
- package/src/extractors/youtube/innertube.ts +364 -0
- package/src/extractors/youtube/nsig.ts +105 -0
- package/src/extractors/youtube/playlist.ts +227 -0
- package/src/extractors/youtube/signature.ts +163 -0
- package/src/networking/client.ts +311 -0
- package/src/networking/cookies.ts +138 -0
- package/src/networking/proxy.ts +132 -0
- package/src/networking/tls.ts +67 -0
- package/src/networking/user-agents.ts +88 -0
- package/src/postprocessors/base.ts +44 -0
- package/src/postprocessors/extract-audio.ts +98 -0
- package/src/postprocessors/ffmpeg.ts +146 -0
- package/src/postprocessors/merge.ts +102 -0
- package/src/postprocessors/metadata.ts +73 -0
- package/src/postprocessors/sponsorblock.ts +162 -0
- package/src/postprocessors/subtitles.ts +285 -0
- package/src/postprocessors/thumbnails.ts +194 -0
- package/src/utils/sanitize.ts +36 -0
- package/src/utils/traverse.ts +68 -0
- package/tests/core/format-sorter.test.ts +96 -0
- package/tests/core/output-template.test.ts +56 -0
- package/tests/core/types.test.ts +79 -0
- package/tests/unit/downloaders/dash.test.ts +57 -0
- package/tests/unit/downloaders/hls.test.ts +120 -0
- package/tests/unit/downloaders/http.test.ts +114 -0
- package/tests/unit/extractors/bilibili.test.ts +83 -0
- package/tests/unit/extractors/instagram.test.ts +273 -0
- package/tests/unit/extractors/kick.test.ts +85 -0
- package/tests/unit/extractors/misc.test.ts +942 -0
- package/tests/unit/extractors/niconico.test.ts +61 -0
- package/tests/unit/extractors/reddit.test.ts +222 -0
- package/tests/unit/extractors/soundcloud.test.ts +299 -0
- package/tests/unit/extractors/tiktok.test.ts +260 -0
- package/tests/unit/extractors/twitch.test.ts +250 -0
- package/tests/unit/extractors/twitter.test.ts +181 -0
- package/tests/unit/extractors/vimeo.test.ts +253 -0
- package/tests/unit/extractors/youtube.test.ts +259 -0
- package/tests/unit/networking/client.test.ts +272 -0
- package/tests/unit/networking/cookies.test.ts +256 -0
- package/tests/unit/networking/proxy.test.ts +137 -0
- package/tests/unit/postprocessors/extract-audio.test.ts +63 -0
- package/tests/unit/postprocessors/merge.test.ts +61 -0
- package/tests/unit/postprocessors/subtitles.test.ts +89 -0
- package/tools/dashboard.ts +112 -0
- package/tsconfig.json +17 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
export type ProxyProtocol = "http" | "https" | "socks5";
|
|
2
|
+
|
|
3
|
+
export interface ParsedProxy {
|
|
4
|
+
protocol: ProxyProtocol;
|
|
5
|
+
host: string;
|
|
6
|
+
port: number;
|
|
7
|
+
username?: string;
|
|
8
|
+
password?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export class ProxyParseError extends Error {
|
|
12
|
+
constructor(message: string) {
|
|
13
|
+
super(message);
|
|
14
|
+
this.name = "ProxyParseError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function parseProxyUrl(proxyUrl: string): ParsedProxy {
|
|
19
|
+
let url: URL;
|
|
20
|
+
try {
|
|
21
|
+
url = new URL(proxyUrl);
|
|
22
|
+
} catch {
|
|
23
|
+
throw new ProxyParseError(`Invalid proxy URL: ${proxyUrl}`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const protocol = url.protocol.replace(":", "") as ProxyProtocol;
|
|
27
|
+
if (!["http", "https", "socks5"].includes(protocol)) {
|
|
28
|
+
throw new ProxyParseError(
|
|
29
|
+
`Unsupported proxy protocol: ${protocol}. Supported: http, https, socks5`,
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const host = url.hostname;
|
|
34
|
+
if (!host) {
|
|
35
|
+
throw new ProxyParseError(`Proxy URL missing host: ${proxyUrl}`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const port = url.port ? parseInt(url.port, 10) : protocol === "https" ? 443 : 1080;
|
|
39
|
+
if (isNaN(port) || port < 1 || port > 65535) {
|
|
40
|
+
throw new ProxyParseError(`Invalid proxy port: ${url.port}`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const result: ParsedProxy = { protocol, host, port };
|
|
44
|
+
if (url.username) result.username = decodeURIComponent(url.username);
|
|
45
|
+
if (url.password) result.password = decodeURIComponent(url.password);
|
|
46
|
+
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function buildProxyAuthHeader(proxy: ParsedProxy): string | undefined {
|
|
51
|
+
if (!proxy.username) return undefined;
|
|
52
|
+
const creds = `${proxy.username}:${proxy.password ?? ""}`;
|
|
53
|
+
return `Basic ${btoa(creds)}`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface ProxyAgent {
|
|
57
|
+
proxy: ParsedProxy;
|
|
58
|
+
getProxyUrl(): string;
|
|
59
|
+
getAuthHeader(): string | undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function createProxyAgent(proxyUrl: string): ProxyAgent {
|
|
63
|
+
const proxy = parseProxyUrl(proxyUrl);
|
|
64
|
+
|
|
65
|
+
return {
|
|
66
|
+
proxy,
|
|
67
|
+
getProxyUrl(): string {
|
|
68
|
+
const auth =
|
|
69
|
+
proxy.username
|
|
70
|
+
? `${encodeURIComponent(proxy.username)}:${encodeURIComponent(proxy.password ?? "")}@`
|
|
71
|
+
: "";
|
|
72
|
+
return `${proxy.protocol}://${auth}${proxy.host}:${proxy.port}`;
|
|
73
|
+
},
|
|
74
|
+
getAuthHeader(): string | undefined {
|
|
75
|
+
return buildProxyAuthHeader(proxy);
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface HttpConnectOptions {
|
|
81
|
+
targetHost: string;
|
|
82
|
+
targetPort: number;
|
|
83
|
+
proxy: ParsedProxy;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function buildConnectRequest(opts: HttpConnectOptions): string {
|
|
87
|
+
const target = `${opts.targetHost}:${opts.targetPort}`;
|
|
88
|
+
const authHeader = buildProxyAuthHeader(opts.proxy);
|
|
89
|
+
const lines = [
|
|
90
|
+
`CONNECT ${target} HTTP/1.1`,
|
|
91
|
+
`Host: ${target}`,
|
|
92
|
+
"Proxy-Connection: keep-alive",
|
|
93
|
+
];
|
|
94
|
+
if (authHeader) {
|
|
95
|
+
lines.push(`Proxy-Authorization: ${authHeader}`);
|
|
96
|
+
}
|
|
97
|
+
lines.push("", "");
|
|
98
|
+
return lines.join("\r\n");
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface Socks5ConnectOptions {
|
|
102
|
+
targetHost: string;
|
|
103
|
+
targetPort: number;
|
|
104
|
+
proxy: ParsedProxy;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function buildSocks5Greeting(proxy: ParsedProxy): Uint8Array {
|
|
108
|
+
if (proxy.username) {
|
|
109
|
+
return new Uint8Array([0x05, 0x02, 0x00, 0x02]);
|
|
110
|
+
}
|
|
111
|
+
return new Uint8Array([0x05, 0x01, 0x00]);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function buildSocks5ConnectRequest(
|
|
115
|
+
opts: Socks5ConnectOptions,
|
|
116
|
+
): Uint8Array {
|
|
117
|
+
const encoder = new TextEncoder();
|
|
118
|
+
const hostBytes = encoder.encode(opts.targetHost);
|
|
119
|
+
const buf = new Uint8Array(4 + 1 + hostBytes.length + 2);
|
|
120
|
+
let i = 0;
|
|
121
|
+
buf[i++] = 0x05;
|
|
122
|
+
buf[i++] = 0x01;
|
|
123
|
+
buf[i++] = 0x00;
|
|
124
|
+
buf[i++] = 0x03;
|
|
125
|
+
buf[i++] = hostBytes.length;
|
|
126
|
+
buf.set(hostBytes, i);
|
|
127
|
+
i += hostBytes.length;
|
|
128
|
+
const port = opts.targetPort;
|
|
129
|
+
buf[i++] = (port >> 8) & 0xff;
|
|
130
|
+
buf[i++] = port & 0xff;
|
|
131
|
+
return buf;
|
|
132
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
export interface TlsFingerprint {
|
|
2
|
+
ja3?: string;
|
|
3
|
+
ja4?: string;
|
|
4
|
+
userAgent: string;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export interface TlsProfile {
|
|
8
|
+
name: string;
|
|
9
|
+
userAgent: string;
|
|
10
|
+
fingerprint?: TlsFingerprint;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export const TLS_PROFILES: Record<string, TlsProfile> = {
|
|
14
|
+
chrome_131: {
|
|
15
|
+
name: "Chrome 131",
|
|
16
|
+
userAgent:
|
|
17
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
18
|
+
fingerprint: {
|
|
19
|
+
ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513-21,29-23-24,0",
|
|
20
|
+
userAgent:
|
|
21
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
firefox_133: {
|
|
25
|
+
name: "Firefox 133",
|
|
26
|
+
userAgent:
|
|
27
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
28
|
+
fingerprint: {
|
|
29
|
+
ja3: "771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28-65037,29-23-24-25-256-257,0",
|
|
30
|
+
userAgent:
|
|
31
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
safari_18: {
|
|
35
|
+
name: "Safari 18",
|
|
36
|
+
userAgent:
|
|
37
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
|
|
38
|
+
fingerprint: {
|
|
39
|
+
userAgent:
|
|
40
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export function getTlsProfileForUserAgent(
|
|
46
|
+
userAgent: string,
|
|
47
|
+
): TlsProfile | undefined {
|
|
48
|
+
for (const profile of Object.values(TLS_PROFILES)) {
|
|
49
|
+
if (profile.userAgent === userAgent) return profile;
|
|
50
|
+
}
|
|
51
|
+
if (userAgent.includes("Chrome")) return TLS_PROFILES["chrome_131"];
|
|
52
|
+
if (userAgent.includes("Firefox")) return TLS_PROFILES["firefox_133"];
|
|
53
|
+
if (userAgent.includes("Safari")) return TLS_PROFILES["safari_18"];
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const TLS_IMPLEMENTATION_NOTE = `
|
|
58
|
+
TLS fingerprinting (JA3/JA4) requires low-level TLS control not available through
|
|
59
|
+
standard fetch()/WebSocket APIs. Full impersonation requires:
|
|
60
|
+
1. A native Bun binding or FFI to BoringSSL/uTLS
|
|
61
|
+
2. Custom cipher suite ordering matching the target browser
|
|
62
|
+
3. Custom TLS extensions ordering
|
|
63
|
+
|
|
64
|
+
For now, dlpx sets a matching User-Agent header which satisfies most sites.
|
|
65
|
+
Sites that perform JA3 fingerprint checking (e.g., some CDN bot detection) may
|
|
66
|
+
require a future implementation using Bun FFI + uTLS or a headless browser approach.
|
|
67
|
+
`;
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export interface UserAgentEntry {
|
|
2
|
+
ua: string;
|
|
3
|
+
platform: "macos" | "windows" | "linux";
|
|
4
|
+
browser: "chrome" | "firefox" | "safari" | "edge";
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export const USER_AGENTS: UserAgentEntry[] = [
|
|
8
|
+
{
|
|
9
|
+
ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
10
|
+
platform: "macos",
|
|
11
|
+
browser: "chrome",
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
15
|
+
platform: "windows",
|
|
16
|
+
browser: "chrome",
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
ua: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
20
|
+
platform: "linux",
|
|
21
|
+
browser: "chrome",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
25
|
+
platform: "macos",
|
|
26
|
+
browser: "firefox",
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
30
|
+
platform: "windows",
|
|
31
|
+
browser: "firefox",
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
ua: "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
35
|
+
platform: "linux",
|
|
36
|
+
browser: "firefox",
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
|
|
40
|
+
platform: "macos",
|
|
41
|
+
browser: "safari",
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
|
45
|
+
platform: "windows",
|
|
46
|
+
browser: "edge",
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
|
50
|
+
platform: "macos",
|
|
51
|
+
browser: "edge",
|
|
52
|
+
},
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
let roundRobinIndex = 0;
|
|
56
|
+
|
|
57
|
+
export function getRandomUserAgent(): string {
|
|
58
|
+
return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)].ua;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function getRoundRobinUserAgent(): string {
|
|
62
|
+
const ua = USER_AGENTS[roundRobinIndex % USER_AGENTS.length].ua;
|
|
63
|
+
roundRobinIndex++;
|
|
64
|
+
return ua;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function getUserAgentByBrowser(
|
|
68
|
+
browser: UserAgentEntry["browser"],
|
|
69
|
+
platform?: UserAgentEntry["platform"],
|
|
70
|
+
): string {
|
|
71
|
+
const filtered = USER_AGENTS.filter(
|
|
72
|
+
(e) => e.browser === browser && (!platform || e.platform === platform),
|
|
73
|
+
);
|
|
74
|
+
if (filtered.length === 0) {
|
|
75
|
+
return getRandomUserAgent();
|
|
76
|
+
}
|
|
77
|
+
return filtered[Math.floor(Math.random() * filtered.length)].ua;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function getUserAgentByPlatform(
|
|
81
|
+
platform: UserAgentEntry["platform"],
|
|
82
|
+
): string {
|
|
83
|
+
const filtered = USER_AGENTS.filter((e) => e.platform === platform);
|
|
84
|
+
if (filtered.length === 0) {
|
|
85
|
+
return getRandomUserAgent();
|
|
86
|
+
}
|
|
87
|
+
return filtered[Math.floor(Math.random() * filtered.length)].ua;
|
|
88
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { PostProcessor } from "../core/types";
|
|
2
|
+
import type { InfoDict, Options, PostProcessResult } from "../core/types";
|
|
3
|
+
import { logger } from "../core/logger";
|
|
4
|
+
|
|
5
|
+
export { PostProcessor };
|
|
6
|
+
|
|
7
|
+
const postProcessors: PostProcessor[] = [];
|
|
8
|
+
|
|
9
|
+
export function registerPostProcessor(pp: PostProcessor): void {
|
|
10
|
+
postProcessors.push(pp);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export async function runPostProcessors(
|
|
14
|
+
info: InfoDict,
|
|
15
|
+
filepath: string,
|
|
16
|
+
_options: Options,
|
|
17
|
+
): Promise<string> {
|
|
18
|
+
let currentPath = filepath;
|
|
19
|
+
const filesToDelete: string[] = [];
|
|
20
|
+
|
|
21
|
+
for (const pp of postProcessors) {
|
|
22
|
+
logger.debug(`Running post-processor: ${pp._NAME}`);
|
|
23
|
+
try {
|
|
24
|
+
const result: PostProcessResult = await pp.run(info, currentPath);
|
|
25
|
+
currentPath = result.filepath;
|
|
26
|
+
filesToDelete.push(...result.files_to_delete);
|
|
27
|
+
} catch (err) {
|
|
28
|
+
logger.warn(
|
|
29
|
+
`Post-processor ${pp._NAME} failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
for (const file of filesToDelete) {
|
|
35
|
+
try {
|
|
36
|
+
const { unlink } = await import("node:fs/promises");
|
|
37
|
+
await unlink(file);
|
|
38
|
+
} catch {
|
|
39
|
+
logger.debug(`Could not delete temp file: ${file}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return currentPath;
|
|
44
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { PostProcessor, PostProcessError } from "../core/types";
|
|
2
|
+
import type { InfoDict, PostProcessResult } from "../core/types";
|
|
3
|
+
import { FFmpegRunner } from "./ffmpeg";
|
|
4
|
+
import { dirname, basename, extname, join } from "node:path";
|
|
5
|
+
|
|
6
|
+
export type AudioFormat = "mp3" | "flac" | "wav" | "aac" | "opus" | "vorbis" | "m4a";
|
|
7
|
+
|
|
8
|
+
const FORMAT_CODEC: Record<AudioFormat, string> = {
|
|
9
|
+
mp3: "libmp3lame",
|
|
10
|
+
flac: "flac",
|
|
11
|
+
wav: "pcm_s16le",
|
|
12
|
+
aac: "aac",
|
|
13
|
+
opus: "libopus",
|
|
14
|
+
vorbis: "libvorbis",
|
|
15
|
+
m4a: "aac",
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const FORMAT_EXT: Record<AudioFormat, string> = {
|
|
19
|
+
mp3: "mp3",
|
|
20
|
+
flac: "flac",
|
|
21
|
+
wav: "wav",
|
|
22
|
+
aac: "aac",
|
|
23
|
+
opus: "opus",
|
|
24
|
+
vorbis: "ogg",
|
|
25
|
+
m4a: "m4a",
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export interface ExtractAudioOptions {
|
|
29
|
+
format?: AudioFormat;
|
|
30
|
+
quality?: number;
|
|
31
|
+
preserveMetadata?: boolean;
|
|
32
|
+
ffmpegLocation?: string | null;
|
|
33
|
+
onProgress?: (percent: number) => void;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export class ExtractAudioPostProcessor extends PostProcessor {
|
|
37
|
+
readonly _NAME = "ExtractAudio";
|
|
38
|
+
|
|
39
|
+
private opts: ExtractAudioOptions;
|
|
40
|
+
|
|
41
|
+
constructor(opts: ExtractAudioOptions = {}) {
|
|
42
|
+
super();
|
|
43
|
+
this.opts = opts;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async run(info: InfoDict, filepath: string): Promise<PostProcessResult> {
|
|
47
|
+
const format: AudioFormat = this.opts.format ?? "mp3";
|
|
48
|
+
const codec = FORMAT_CODEC[format];
|
|
49
|
+
const ext = FORMAT_EXT[format];
|
|
50
|
+
|
|
51
|
+
const dir = dirname(filepath);
|
|
52
|
+
const stem = basename(filepath, extname(filepath));
|
|
53
|
+
const outputPath = join(dir, `${stem}.${ext}`);
|
|
54
|
+
|
|
55
|
+
const runner = await FFmpegRunner.detect(this.opts.ffmpegLocation);
|
|
56
|
+
|
|
57
|
+
const args: string[] = ["-i", filepath, "-vn", "-acodec", codec];
|
|
58
|
+
|
|
59
|
+
const quality = this.opts.quality ?? 5;
|
|
60
|
+
|
|
61
|
+
if (format === "mp3") {
|
|
62
|
+
args.push("-q:a", String(quality));
|
|
63
|
+
} else if (format === "opus") {
|
|
64
|
+
const bitrate = qualityToBitrate(quality, 32, 320);
|
|
65
|
+
args.push("-b:a", `${bitrate}k`);
|
|
66
|
+
} else if (format === "vorbis") {
|
|
67
|
+
args.push("-q:a", String(quality));
|
|
68
|
+
} else if (format === "aac" || format === "m4a") {
|
|
69
|
+
const bitrate = qualityToBitrate(quality, 32, 320);
|
|
70
|
+
args.push("-b:a", `${bitrate}k`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (this.opts.preserveMetadata !== false) {
|
|
74
|
+
args.push("-map_metadata", "0");
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
args.push(outputPath);
|
|
78
|
+
|
|
79
|
+
await runner.run(
|
|
80
|
+
args,
|
|
81
|
+
this.opts.onProgress
|
|
82
|
+
? (p) => { if (p.percent !== undefined) this.opts.onProgress!(p.percent); }
|
|
83
|
+
: undefined,
|
|
84
|
+
info.duration,
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
filepath: outputPath,
|
|
89
|
+
info,
|
|
90
|
+
files_to_delete: outputPath !== filepath ? [filepath] : [],
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function qualityToBitrate(quality: number, min: number, max: number): number {
|
|
96
|
+
const clamped = Math.max(0, Math.min(10, quality));
|
|
97
|
+
return Math.round(min + ((10 - clamped) / 10) * (max - min));
|
|
98
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { PostProcessError } from "../core/types";
|
|
2
|
+
|
|
3
|
+
export interface FFmpegProgress {
|
|
4
|
+
frame?: number;
|
|
5
|
+
fps?: number;
|
|
6
|
+
time?: string;
|
|
7
|
+
speed?: string;
|
|
8
|
+
percent?: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type ProgressCallback = (progress: FFmpegProgress) => void;
|
|
12
|
+
|
|
13
|
+
export class FFmpegRunner {
|
|
14
|
+
private binary: string;
|
|
15
|
+
|
|
16
|
+
constructor(binary: string = "ffmpeg") {
|
|
17
|
+
this.binary = binary;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
static async detect(ffmpegLocation?: string | null): Promise<FFmpegRunner> {
|
|
21
|
+
const candidates = ffmpegLocation
|
|
22
|
+
? [ffmpegLocation, "ffmpeg"]
|
|
23
|
+
: ["ffmpeg", "/usr/local/bin/ffmpeg", "/opt/homebrew/bin/ffmpeg", "/usr/bin/ffmpeg"];
|
|
24
|
+
|
|
25
|
+
for (const candidate of candidates) {
|
|
26
|
+
try {
|
|
27
|
+
const result = await Bun.$`${candidate} -version`.quiet();
|
|
28
|
+
if (result.exitCode === 0) {
|
|
29
|
+
return new FFmpegRunner(candidate);
|
|
30
|
+
}
|
|
31
|
+
} catch {
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
throw new PostProcessError(
|
|
37
|
+
"FFmpeg not found. Install FFmpeg or pass --ffmpeg-location.",
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async checkCapabilities(): Promise<{ codecs: string[]; formats: string[] }> {
|
|
42
|
+
const [codecResult, formatResult] = await Promise.all([
|
|
43
|
+
Bun.$`${this.binary} -codecs -v quiet`.quiet(),
|
|
44
|
+
Bun.$`${this.binary} -formats -v quiet`.quiet(),
|
|
45
|
+
]);
|
|
46
|
+
|
|
47
|
+
const codecs = codecResult.stdout
|
|
48
|
+
.toString()
|
|
49
|
+
.split("\n")
|
|
50
|
+
.filter((l) => /^\s*[D.][E.][VASD.][I.][L.][S.]/.test(l))
|
|
51
|
+
.map((l) => l.trim().split(/\s+/)[1])
|
|
52
|
+
.filter(Boolean) as string[];
|
|
53
|
+
|
|
54
|
+
const formats = formatResult.stdout
|
|
55
|
+
.toString()
|
|
56
|
+
.split("\n")
|
|
57
|
+
.filter((l) => /^\s*[D.][E.]/.test(l))
|
|
58
|
+
.map((l) => l.trim().split(/\s+/)[1])
|
|
59
|
+
.filter(Boolean) as string[];
|
|
60
|
+
|
|
61
|
+
return { codecs, formats };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async run(
|
|
65
|
+
args: string[],
|
|
66
|
+
onProgress?: ProgressCallback,
|
|
67
|
+
durationSeconds?: number,
|
|
68
|
+
): Promise<void> {
|
|
69
|
+
const proc = Bun.spawn([this.binary, "-y", ...args], {
|
|
70
|
+
stdout: "pipe",
|
|
71
|
+
stderr: "pipe",
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const stderrChunks: Uint8Array[] = [];
|
|
75
|
+
const decoder = new TextDecoder();
|
|
76
|
+
let partialLine = "";
|
|
77
|
+
|
|
78
|
+
if (proc.stderr) {
|
|
79
|
+
const reader = proc.stderr.getReader();
|
|
80
|
+
while (true) {
|
|
81
|
+
const { done, value } = await reader.read();
|
|
82
|
+
if (done) break;
|
|
83
|
+
stderrChunks.push(value);
|
|
84
|
+
|
|
85
|
+
if (onProgress) {
|
|
86
|
+
partialLine += decoder.decode(value, { stream: true });
|
|
87
|
+
const lines = partialLine.split("\r");
|
|
88
|
+
partialLine = lines[lines.length - 1];
|
|
89
|
+
for (const line of lines.slice(0, -1)) {
|
|
90
|
+
const progress = parseFFmpegProgress(line, durationSeconds);
|
|
91
|
+
if (progress) onProgress(progress);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
await proc.exited;
|
|
98
|
+
|
|
99
|
+
if (proc.exitCode !== 0) {
|
|
100
|
+
const stderr = stderrChunks.map((c) => decoder.decode(c)).join("");
|
|
101
|
+
const lastLines = stderr.split("\n").slice(-5).join("\n");
|
|
102
|
+
throw new PostProcessError(`FFmpeg failed (exit ${proc.exitCode}):\n${lastLines}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
getBinary(): string {
|
|
107
|
+
return this.binary;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function parseFFmpegProgress(line: string, durationSeconds?: number): FFmpegProgress | null {
|
|
112
|
+
if (!line.includes("time=") && !line.includes("frame=")) return null;
|
|
113
|
+
|
|
114
|
+
const progress: FFmpegProgress = {};
|
|
115
|
+
|
|
116
|
+
const frameMatch = line.match(/frame=\s*(\d+)/);
|
|
117
|
+
if (frameMatch) progress.frame = parseInt(frameMatch[1], 10);
|
|
118
|
+
|
|
119
|
+
const fpsMatch = line.match(/fps=\s*([\d.]+)/);
|
|
120
|
+
if (fpsMatch) progress.fps = parseFloat(fpsMatch[1]);
|
|
121
|
+
|
|
122
|
+
const timeMatch = line.match(/time=\s*([\d:]+\.?\d*)/);
|
|
123
|
+
if (timeMatch) {
|
|
124
|
+
progress.time = timeMatch[1];
|
|
125
|
+
if (durationSeconds && durationSeconds > 0) {
|
|
126
|
+
const elapsed = parseTimeToSeconds(timeMatch[1]);
|
|
127
|
+
progress.percent = Math.min(100, (elapsed / durationSeconds) * 100);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const speedMatch = line.match(/speed=\s*([\d.]+x)/);
|
|
132
|
+
if (speedMatch) progress.speed = speedMatch[1];
|
|
133
|
+
|
|
134
|
+
return Object.keys(progress).length > 0 ? progress : null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function parseTimeToSeconds(time: string): number {
|
|
138
|
+
const parts = time.split(":").map(parseFloat);
|
|
139
|
+
if (parts.length === 3) {
|
|
140
|
+
return (parts[0] ?? 0) * 3600 + (parts[1] ?? 0) * 60 + (parts[2] ?? 0);
|
|
141
|
+
}
|
|
142
|
+
if (parts.length === 2) {
|
|
143
|
+
return (parts[0] ?? 0) * 60 + (parts[1] ?? 0);
|
|
144
|
+
}
|
|
145
|
+
return parts[0] ?? 0;
|
|
146
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { PostProcessor, PostProcessError } from "../core/types";
|
|
2
|
+
import type { InfoDict, PostProcessResult } from "../core/types";
|
|
3
|
+
import { FFmpegRunner } from "./ffmpeg";
|
|
4
|
+
import { join, dirname, extname, basename } from "node:path";
|
|
5
|
+
|
|
6
|
+
type MergeContainer = "mkv" | "mp4" | "webm";
|
|
7
|
+
|
|
8
|
+
const COPY_COMPATIBLE: Record<MergeContainer, { video: string[]; audio: string[] }> = {
|
|
9
|
+
mkv: {
|
|
10
|
+
video: ["h264", "hevc", "vp8", "vp9", "av1", "mpeg4", "theora"],
|
|
11
|
+
audio: ["aac", "mp3", "opus", "vorbis", "flac", "ac3", "eac3", "truehd", "dts"],
|
|
12
|
+
},
|
|
13
|
+
mp4: {
|
|
14
|
+
video: ["h264", "hevc", "av1"],
|
|
15
|
+
audio: ["aac", "mp3", "ac3", "eac3"],
|
|
16
|
+
},
|
|
17
|
+
webm: {
|
|
18
|
+
video: ["vp8", "vp9", "av1"],
|
|
19
|
+
audio: ["opus", "vorbis"],
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export interface MergeOptions {
|
|
24
|
+
outputContainer?: MergeContainer;
|
|
25
|
+
audioFilepath?: string;
|
|
26
|
+
ffmpegLocation?: string | null;
|
|
27
|
+
onProgress?: (percent: number) => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export class MergePostProcessor extends PostProcessor {
|
|
31
|
+
readonly _NAME = "Merger";
|
|
32
|
+
|
|
33
|
+
private opts: MergeOptions;
|
|
34
|
+
|
|
35
|
+
constructor(opts: MergeOptions = {}) {
|
|
36
|
+
super();
|
|
37
|
+
this.opts = opts;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async run(info: InfoDict, filepath: string): Promise<PostProcessResult> {
|
|
41
|
+
const audioPath = this.opts.audioFilepath;
|
|
42
|
+
if (!audioPath) {
|
|
43
|
+
return { filepath, info, files_to_delete: [] };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const container = this.opts.outputContainer ?? inferContainer(filepath);
|
|
47
|
+
const dir = dirname(filepath);
|
|
48
|
+
const stem = basename(filepath, extname(filepath));
|
|
49
|
+
const outputPath = join(dir, `${stem}.${container}`);
|
|
50
|
+
|
|
51
|
+
const runner = await FFmpegRunner.detect(this.opts.ffmpegLocation);
|
|
52
|
+
|
|
53
|
+
const vcodec = info.requested_formats?.[0]?.vcodec?.split(".")[0] ?? "";
|
|
54
|
+
const acodec = info.requested_formats?.[1]?.acodec?.split(".")[0] ?? "";
|
|
55
|
+
|
|
56
|
+
const compatible = COPY_COMPATIBLE[container];
|
|
57
|
+
const canCopyVideo = compatible.video.includes(vcodec);
|
|
58
|
+
const canCopyAudio = compatible.audio.includes(acodec);
|
|
59
|
+
|
|
60
|
+
const args: string[] = [
|
|
61
|
+
"-i", filepath,
|
|
62
|
+
"-i", audioPath,
|
|
63
|
+
];
|
|
64
|
+
|
|
65
|
+
if (canCopyVideo && canCopyAudio) {
|
|
66
|
+
args.push("-c", "copy");
|
|
67
|
+
} else {
|
|
68
|
+
args.push(
|
|
69
|
+
"-c:v", canCopyVideo ? "copy" : "libx264",
|
|
70
|
+
"-c:a", canCopyAudio ? "copy" : "aac",
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (container === "mp4") {
|
|
75
|
+
args.push("-movflags", "+faststart");
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
args.push(outputPath);
|
|
79
|
+
|
|
80
|
+
await runner.run(
|
|
81
|
+
args,
|
|
82
|
+
this.opts.onProgress
|
|
83
|
+
? (p) => { if (p.percent !== undefined) this.opts.onProgress!(p.percent); }
|
|
84
|
+
: undefined,
|
|
85
|
+
info.duration,
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
return {
|
|
89
|
+
filepath: outputPath,
|
|
90
|
+
info,
|
|
91
|
+
files_to_delete: outputPath !== filepath ? [filepath, audioPath] : [],
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function inferContainer(filepath: string): MergeContainer {
|
|
97
|
+
const ext = extname(filepath).slice(1).toLowerCase();
|
|
98
|
+
if (ext === "mkv" || ext === "mp4" || ext === "webm") {
|
|
99
|
+
return ext as MergeContainer;
|
|
100
|
+
}
|
|
101
|
+
return "mkv";
|
|
102
|
+
}
|