getraw 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitattributes +4 -0
- package/CLAUDE.md +57 -0
- package/README.md +166 -0
- package/RESEARCH.md +109 -0
- package/STATUS.md +23 -0
- package/bun.lock +50 -0
- package/bunfig.toml +3 -0
- package/docs/plugin-guide.md +166 -0
- package/docs/supported-sites.md +41 -0
- package/package.json +30 -0
- package/src/cli/index.ts +52 -0
- package/src/cli/options.ts +97 -0
- package/src/core/format-sorter.ts +208 -0
- package/src/core/logger.ts +101 -0
- package/src/core/orchestrator.ts +140 -0
- package/src/core/output-template.ts +58 -0
- package/src/core/types.ts +237 -0
- package/src/downloaders/base.ts +25 -0
- package/src/downloaders/dash.ts +287 -0
- package/src/downloaders/fragment.ts +226 -0
- package/src/downloaders/hls.ts +170 -0
- package/src/downloaders/http.ts +260 -0
- package/src/extractors/archive-org.ts +126 -0
- package/src/extractors/bandcamp.ts +130 -0
- package/src/extractors/base.ts +29 -0
- package/src/extractors/bilibili/bangumi.ts +205 -0
- package/src/extractors/bilibili/index.ts +233 -0
- package/src/extractors/bilibili/wbi.ts +60 -0
- package/src/extractors/coub.ts +137 -0
- package/src/extractors/dailymotion.ts +99 -0
- package/src/extractors/dropbox.ts +52 -0
- package/src/extractors/generic.ts +118 -0
- package/src/extractors/google-drive.ts +106 -0
- package/src/extractors/imgur.ts +156 -0
- package/src/extractors/instagram/index.ts +263 -0
- package/src/extractors/instagram/reels.ts +166 -0
- package/src/extractors/kick/clips.ts +91 -0
- package/src/extractors/kick/index.ts +118 -0
- package/src/extractors/kick/live.ts +89 -0
- package/src/extractors/niconico/index.ts +209 -0
- package/src/extractors/odysee.ts +126 -0
- package/src/extractors/peertube.ts +143 -0
- package/src/extractors/reddit/gallery.ts +124 -0
- package/src/extractors/reddit/index.ts +203 -0
- package/src/extractors/rumble.ts +127 -0
- package/src/extractors/soundcloud/index.ts +161 -0
- package/src/extractors/soundcloud/playlist.ts +129 -0
- package/src/extractors/spotify.ts +97 -0
- package/src/extractors/streamable.ts +121 -0
- package/src/extractors/ted.ts +151 -0
- package/src/extractors/tiktok/index.ts +207 -0
- package/src/extractors/tiktok/user.ts +176 -0
- package/src/extractors/twitch/clips.ts +125 -0
- package/src/extractors/twitch/index.ts +136 -0
- package/src/extractors/twitch/live.ts +132 -0
- package/src/extractors/twitter/index.ts +140 -0
- package/src/extractors/twitter/spaces.ts +200 -0
- package/src/extractors/vimeo/index.ts +187 -0
- package/src/extractors/youtube/captions.ts +111 -0
- package/src/extractors/youtube/index.ts +252 -0
- package/src/extractors/youtube/innertube.ts +364 -0
- package/src/extractors/youtube/nsig.ts +105 -0
- package/src/extractors/youtube/playlist.ts +227 -0
- package/src/extractors/youtube/signature.ts +163 -0
- package/src/networking/client.ts +311 -0
- package/src/networking/cookies.ts +138 -0
- package/src/networking/proxy.ts +132 -0
- package/src/networking/tls.ts +67 -0
- package/src/networking/user-agents.ts +88 -0
- package/src/postprocessors/base.ts +44 -0
- package/src/postprocessors/extract-audio.ts +98 -0
- package/src/postprocessors/ffmpeg.ts +146 -0
- package/src/postprocessors/merge.ts +102 -0
- package/src/postprocessors/metadata.ts +73 -0
- package/src/postprocessors/sponsorblock.ts +162 -0
- package/src/postprocessors/subtitles.ts +285 -0
- package/src/postprocessors/thumbnails.ts +194 -0
- package/src/utils/sanitize.ts +36 -0
- package/src/utils/traverse.ts +68 -0
- package/tests/core/format-sorter.test.ts +96 -0
- package/tests/core/output-template.test.ts +56 -0
- package/tests/core/types.test.ts +79 -0
- package/tests/unit/downloaders/dash.test.ts +57 -0
- package/tests/unit/downloaders/hls.test.ts +120 -0
- package/tests/unit/downloaders/http.test.ts +114 -0
- package/tests/unit/extractors/bilibili.test.ts +83 -0
- package/tests/unit/extractors/instagram.test.ts +273 -0
- package/tests/unit/extractors/kick.test.ts +85 -0
- package/tests/unit/extractors/misc.test.ts +942 -0
- package/tests/unit/extractors/niconico.test.ts +61 -0
- package/tests/unit/extractors/reddit.test.ts +222 -0
- package/tests/unit/extractors/soundcloud.test.ts +299 -0
- package/tests/unit/extractors/tiktok.test.ts +260 -0
- package/tests/unit/extractors/twitch.test.ts +250 -0
- package/tests/unit/extractors/twitter.test.ts +181 -0
- package/tests/unit/extractors/vimeo.test.ts +253 -0
- package/tests/unit/extractors/youtube.test.ts +259 -0
- package/tests/unit/networking/client.test.ts +272 -0
- package/tests/unit/networking/cookies.test.ts +256 -0
- package/tests/unit/networking/proxy.test.ts +137 -0
- package/tests/unit/postprocessors/extract-audio.test.ts +63 -0
- package/tests/unit/postprocessors/merge.test.ts +61 -0
- package/tests/unit/postprocessors/subtitles.test.ts +89 -0
- package/tools/dashboard.ts +112 -0
- package/tsconfig.json +17 -0
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
import { describe, test, expect, mock } from "bun:test";
|
|
2
|
+
import { TikTokExtractor } from "../../../src/extractors/tiktok/index";
|
|
3
|
+
import { TikTokUserExtractor } from "../../../src/extractors/tiktok/user";
|
|
4
|
+
import { ExtractorError } from "../../../src/core/types";
|
|
5
|
+
|
|
6
|
+
const MOCK_ITEM_STRUCT = {
|
|
7
|
+
id: "7123456789012345678",
|
|
8
|
+
desc: "Cool video #fyp #trending",
|
|
9
|
+
createTime: 1700000000,
|
|
10
|
+
author: {
|
|
11
|
+
uniqueId: "coolcreator",
|
|
12
|
+
nickname: "Cool Creator",
|
|
13
|
+
id: "123456789",
|
|
14
|
+
avatarThumb: "https://p16-sign.tiktokcdn-us.com/avatar.jpg",
|
|
15
|
+
},
|
|
16
|
+
music: {
|
|
17
|
+
title: "Original Sound",
|
|
18
|
+
authorName: "coolcreator",
|
|
19
|
+
id: "987654321",
|
|
20
|
+
},
|
|
21
|
+
video: {
|
|
22
|
+
playAddr: "https://v19.tiktok.com/play/video.mp4",
|
|
23
|
+
downloadAddr: "https://v19.tiktok.com/download/video.mp4",
|
|
24
|
+
width: 1080,
|
|
25
|
+
height: 1920,
|
|
26
|
+
duration: 15,
|
|
27
|
+
bitrate: 2000000,
|
|
28
|
+
format: "mp4",
|
|
29
|
+
codecType: "h264",
|
|
30
|
+
cover: "https://p16.tiktok.com/cover.jpg",
|
|
31
|
+
dynamicCover: "https://p16.tiktok.com/dynamic_cover.webp",
|
|
32
|
+
originCover: "https://p16.tiktok.com/origin_cover.jpg",
|
|
33
|
+
},
|
|
34
|
+
stats: {
|
|
35
|
+
diggCount: 50000,
|
|
36
|
+
shareCount: 1000,
|
|
37
|
+
commentCount: 500,
|
|
38
|
+
playCount: 1000000,
|
|
39
|
+
},
|
|
40
|
+
textExtra: [{ hashtagName: "fyp" }, { hashtagName: "trending" }],
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const MOCK_REHYDRATION_JSON = JSON.stringify({
|
|
44
|
+
__DEFAULT_SCOPE__: {
|
|
45
|
+
"webapp.video-detail": {
|
|
46
|
+
itemInfo: {
|
|
47
|
+
itemStruct: MOCK_ITEM_STRUCT,
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
const MOCK_HTML = `<!DOCTYPE html>
|
|
54
|
+
<html>
|
|
55
|
+
<head><title>TikTok</title></head>
|
|
56
|
+
<body>
|
|
57
|
+
<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">${MOCK_REHYDRATION_JSON}</script>
|
|
58
|
+
</body>
|
|
59
|
+
</html>`;
|
|
60
|
+
|
|
61
|
+
describe("TikTokExtractor", () => {
|
|
62
|
+
const extractor = new TikTokExtractor();
|
|
63
|
+
|
|
64
|
+
test("canHandle matches tiktok.com video URLs", () => {
|
|
65
|
+
expect(extractor.canHandle("https://www.tiktok.com/@user/video/7123456789012345678")).toBe(true);
|
|
66
|
+
expect(extractor.canHandle("https://tiktok.com/@creator.name/video/1234567890123456789")).toBe(true);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("canHandle matches vm.tiktok.com short URLs", () => {
|
|
70
|
+
expect(extractor.canHandle("https://vm.tiktok.com/ZMeABCDEF/")).toBe(true);
|
|
71
|
+
expect(extractor.canHandle("https://vm.tiktok.com/ABC123/")).toBe(true);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test("canHandle rejects non-tiktok URLs", () => {
|
|
75
|
+
expect(extractor.canHandle("https://tiktok.com/@user")).toBe(false);
|
|
76
|
+
expect(extractor.canHandle("https://youtube.com/watch?v=abc")).toBe(false);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
test("_NAME is tiktok", () => {
|
|
80
|
+
expect(extractor._NAME).toBe("tiktok");
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("extract returns correct info from rehydration data", async () => {
|
|
84
|
+
const originalFetch = global.fetch;
|
|
85
|
+
global.fetch = mock(() =>
|
|
86
|
+
Promise.resolve({
|
|
87
|
+
ok: true,
|
|
88
|
+
url: "https://www.tiktok.com/@coolcreator/video/7123456789012345678",
|
|
89
|
+
text: () => Promise.resolve(MOCK_HTML),
|
|
90
|
+
} as Response),
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
try {
|
|
94
|
+
const info = await extractor.extract(
|
|
95
|
+
"https://www.tiktok.com/@coolcreator/video/7123456789012345678",
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
expect(info.id).toBe("7123456789012345678");
|
|
99
|
+
expect(info.title).toBe("Cool video #fyp #trending");
|
|
100
|
+
expect(info.description).toBe("Cool video #fyp #trending");
|
|
101
|
+
expect(info.uploader).toBe("Cool Creator");
|
|
102
|
+
expect(info.uploader_id).toBe("coolcreator");
|
|
103
|
+
expect(info.uploader_url).toBe("https://www.tiktok.com/@coolcreator");
|
|
104
|
+
expect(info.duration).toBe(15);
|
|
105
|
+
expect(info.view_count).toBe(1000000);
|
|
106
|
+
expect(info.like_count).toBe(50000);
|
|
107
|
+
expect(info.comment_count).toBe(500);
|
|
108
|
+
expect(info.tags).toEqual(["fyp", "trending"]);
|
|
109
|
+
expect(info.extractor).toBe("tiktok");
|
|
110
|
+
|
|
111
|
+
expect(info.formats).toBeDefined();
|
|
112
|
+
expect(info.formats!.length).toBeGreaterThanOrEqual(1);
|
|
113
|
+
|
|
114
|
+
const downloadFormat = info.formats!.find((f) => f.format_id === "download");
|
|
115
|
+
expect(downloadFormat).toBeDefined();
|
|
116
|
+
expect(downloadFormat!.url).toBe("https://v19.tiktok.com/download/video.mp4");
|
|
117
|
+
expect(downloadFormat!.width).toBe(1080);
|
|
118
|
+
expect(downloadFormat!.height).toBe(1920);
|
|
119
|
+
|
|
120
|
+
expect(info.thumbnails).toBeDefined();
|
|
121
|
+
expect(info.thumbnails!.length).toBeGreaterThan(0);
|
|
122
|
+
} finally {
|
|
123
|
+
global.fetch = originalFetch;
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("extract throws ExtractorError when rehydration data is missing", async () => {
|
|
128
|
+
const originalFetch = global.fetch;
|
|
129
|
+
global.fetch = mock(() =>
|
|
130
|
+
Promise.resolve({
|
|
131
|
+
ok: true,
|
|
132
|
+
url: "https://www.tiktok.com/@user/video/1234567890",
|
|
133
|
+
text: () => Promise.resolve("<html><body>No data here</body></html>"),
|
|
134
|
+
} as Response),
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
try {
|
|
138
|
+
await expect(
|
|
139
|
+
extractor.extract("https://www.tiktok.com/@user/video/1234567890"),
|
|
140
|
+
).rejects.toThrow(ExtractorError);
|
|
141
|
+
} finally {
|
|
142
|
+
global.fetch = originalFetch;
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("extract throws ExtractorError on page fetch failure", async () => {
|
|
147
|
+
const originalFetch = global.fetch;
|
|
148
|
+
global.fetch = mock(() =>
|
|
149
|
+
Promise.resolve({
|
|
150
|
+
ok: false,
|
|
151
|
+
status: 404,
|
|
152
|
+
statusText: "Not Found",
|
|
153
|
+
url: "https://www.tiktok.com/@user/video/1234567890",
|
|
154
|
+
} as Response),
|
|
155
|
+
);
|
|
156
|
+
|
|
157
|
+
try {
|
|
158
|
+
await expect(
|
|
159
|
+
extractor.extract("https://www.tiktok.com/@user/video/1234567890"),
|
|
160
|
+
).rejects.toThrow(ExtractorError);
|
|
161
|
+
} finally {
|
|
162
|
+
global.fetch = originalFetch;
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
describe("TikTokUserExtractor", () => {
|
|
168
|
+
const extractor = new TikTokUserExtractor();
|
|
169
|
+
|
|
170
|
+
test("canHandle matches tiktok.com user profile URLs", () => {
|
|
171
|
+
expect(extractor.canHandle("https://www.tiktok.com/@coolcreator")).toBe(true);
|
|
172
|
+
expect(extractor.canHandle("https://tiktok.com/@creator.name")).toBe(true);
|
|
173
|
+
expect(extractor.canHandle("https://www.tiktok.com/@user123/")).toBe(true);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test("canHandle rejects video and other URLs", () => {
|
|
177
|
+
expect(
|
|
178
|
+
extractor.canHandle("https://www.tiktok.com/@user/video/123456789"),
|
|
179
|
+
).toBe(false);
|
|
180
|
+
expect(extractor.canHandle("https://youtube.com/@channel")).toBe(false);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test("_NAME is tiktok:user", () => {
|
|
184
|
+
expect(extractor._NAME).toBe("tiktok:user");
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
test("extract returns playlist InfoDict", async () => {
|
|
188
|
+
const mockUserRehydration = JSON.stringify({
|
|
189
|
+
__DEFAULT_SCOPE__: {
|
|
190
|
+
"webapp.user-detail": {
|
|
191
|
+
userInfo: {
|
|
192
|
+
user: {
|
|
193
|
+
id: "123456789",
|
|
194
|
+
uniqueId: "coolcreator",
|
|
195
|
+
nickname: "Cool Creator",
|
|
196
|
+
signature: "Creating cool content",
|
|
197
|
+
},
|
|
198
|
+
stats: { videoCount: 2 },
|
|
199
|
+
},
|
|
200
|
+
},
|
|
201
|
+
},
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
const mockUserHtml = `<!DOCTYPE html>
|
|
205
|
+
<html>
|
|
206
|
+
<body>
|
|
207
|
+
<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">${mockUserRehydration}</script>
|
|
208
|
+
</body>
|
|
209
|
+
</html>`;
|
|
210
|
+
|
|
211
|
+
const mockVideoList = {
|
|
212
|
+
itemList: [
|
|
213
|
+
{
|
|
214
|
+
id: "1111111111",
|
|
215
|
+
desc: "First video",
|
|
216
|
+
createTime: 1700000000,
|
|
217
|
+
video: { cover: "https://p16.tiktok.com/cover1.jpg" },
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
id: "2222222222",
|
|
221
|
+
desc: "Second video",
|
|
222
|
+
createTime: 1700000100,
|
|
223
|
+
video: { cover: "https://p16.tiktok.com/cover2.jpg" },
|
|
224
|
+
},
|
|
225
|
+
],
|
|
226
|
+
hasMore: false,
|
|
227
|
+
maxCursor: 2,
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
const originalFetch = global.fetch;
|
|
231
|
+
let callCount = 0;
|
|
232
|
+
global.fetch = mock(() => {
|
|
233
|
+
callCount++;
|
|
234
|
+
if (callCount === 1) {
|
|
235
|
+
return Promise.resolve({
|
|
236
|
+
ok: true,
|
|
237
|
+
text: () => Promise.resolve(mockUserHtml),
|
|
238
|
+
} as Response);
|
|
239
|
+
}
|
|
240
|
+
return Promise.resolve({
|
|
241
|
+
ok: true,
|
|
242
|
+
json: () => Promise.resolve(mockVideoList),
|
|
243
|
+
} as Response);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
try {
|
|
247
|
+
const info = await extractor.extract("https://www.tiktok.com/@coolcreator");
|
|
248
|
+
expect(info._type).toBe("playlist");
|
|
249
|
+
expect(info.id).toBe("123456789");
|
|
250
|
+
expect(info.uploader_id).toBe("coolcreator");
|
|
251
|
+
expect(info.entries).toBeDefined();
|
|
252
|
+
expect(info.entries!.length).toBe(2);
|
|
253
|
+
expect(info.entries![0].id).toBe("1111111111");
|
|
254
|
+
expect(info.entries![1].id).toBe("2222222222");
|
|
255
|
+
expect(info.playlist_count).toBe(2);
|
|
256
|
+
} finally {
|
|
257
|
+
global.fetch = originalFetch;
|
|
258
|
+
}
|
|
259
|
+
});
|
|
260
|
+
});
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
import { describe, test, expect, mock } from "bun:test";
|
|
2
|
+
import { TwitchVODExtractor } from "../../../src/extractors/twitch/index";
|
|
3
|
+
import { TwitchClipExtractor } from "../../../src/extractors/twitch/clips";
|
|
4
|
+
import { TwitchLiveExtractor } from "../../../src/extractors/twitch/live";
|
|
5
|
+
import { ExtractorError } from "../../../src/core/types";
|
|
6
|
+
|
|
7
|
+
const vodExtractor = new TwitchVODExtractor();
|
|
8
|
+
const clipExtractor = new TwitchClipExtractor();
|
|
9
|
+
const liveExtractor = new TwitchLiveExtractor();
|
|
10
|
+
|
|
11
|
+
describe("TwitchVODExtractor", () => {
|
|
12
|
+
test("canHandle matches VOD URL", () => {
|
|
13
|
+
expect(vodExtractor.canHandle("https://www.twitch.tv/videos/1234567890")).toBe(true);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test("canHandle rejects clip URL", () => {
|
|
17
|
+
expect(vodExtractor.canHandle("https://www.twitch.tv/streamer/clip/ClipSlug")).toBe(false);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("canHandle rejects live URL", () => {
|
|
21
|
+
expect(vodExtractor.canHandle("https://www.twitch.tv/streamer")).toBe(false);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test("_NAME is twitch:vod", () => {
|
|
25
|
+
expect(vodExtractor._NAME).toBe("twitch:vod");
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("extract returns HLS format with access token", async () => {
|
|
29
|
+
const originalFetch = globalThis.fetch;
|
|
30
|
+
|
|
31
|
+
const mockTokenResponse = {
|
|
32
|
+
data: { videoPlaybackAccessToken: { value: "token123", signature: "sig456" } },
|
|
33
|
+
};
|
|
34
|
+
const mockMetaResponse = {
|
|
35
|
+
data: {
|
|
36
|
+
video: {
|
|
37
|
+
id: "1234567890",
|
|
38
|
+
title: "Epic Stream VOD",
|
|
39
|
+
description: "A great stream",
|
|
40
|
+
lengthSeconds: 7200,
|
|
41
|
+
viewCount: 50000,
|
|
42
|
+
publishedAt: "2024-01-15T20:00:00Z",
|
|
43
|
+
owner: { displayName: "Streamer", login: "streamer", id: "12345" },
|
|
44
|
+
previewThumbnailURL: "https://vod-secure.twitch.tv/thumbnail.jpg",
|
|
45
|
+
},
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
let callCount = 0;
|
|
50
|
+
globalThis.fetch = mock(() => {
|
|
51
|
+
callCount++;
|
|
52
|
+
const response = callCount === 1 ? mockTokenResponse : mockMetaResponse;
|
|
53
|
+
return Promise.resolve(
|
|
54
|
+
new Response(JSON.stringify(response), {
|
|
55
|
+
status: 200,
|
|
56
|
+
headers: { "Content-Type": "application/json" },
|
|
57
|
+
})
|
|
58
|
+
);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
const info = await vodExtractor.extract("https://www.twitch.tv/videos/1234567890");
|
|
62
|
+
|
|
63
|
+
expect(info.id).toBe("1234567890");
|
|
64
|
+
expect(info.title).toBe("Epic Stream VOD");
|
|
65
|
+
expect(info.duration).toBe(7200);
|
|
66
|
+
expect(info.uploader).toBe("Streamer");
|
|
67
|
+
expect(info.formats).toBeDefined();
|
|
68
|
+
expect(info.formats!.length).toBeGreaterThan(0);
|
|
69
|
+
|
|
70
|
+
const hlsFormat = info.formats!.find((f) => f.protocol === "m3u8");
|
|
71
|
+
expect(hlsFormat).toBeDefined();
|
|
72
|
+
expect(hlsFormat!.url).toContain("usher.twitchapps.com");
|
|
73
|
+
expect(hlsFormat!.url).toContain("sig456");
|
|
74
|
+
expect(hlsFormat!.url).toContain("token123");
|
|
75
|
+
|
|
76
|
+
globalThis.fetch = originalFetch;
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
test("extract throws ExtractorError on GQL failure", async () => {
|
|
80
|
+
const originalFetch = globalThis.fetch;
|
|
81
|
+
|
|
82
|
+
globalThis.fetch = mock(() =>
|
|
83
|
+
Promise.resolve(new Response(null, { status: 401 }))
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
await expect(
|
|
87
|
+
vodExtractor.extract("https://www.twitch.tv/videos/9999999999")
|
|
88
|
+
).rejects.toThrow(ExtractorError);
|
|
89
|
+
|
|
90
|
+
globalThis.fetch = originalFetch;
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
describe("TwitchClipExtractor", () => {
|
|
95
|
+
test("canHandle matches clip URL with channel", () => {
|
|
96
|
+
expect(
|
|
97
|
+
clipExtractor.canHandle("https://www.twitch.tv/streamer/clip/AwesomeClip")
|
|
98
|
+
).toBe(true);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("canHandle matches clips.twitch.tv URL", () => {
|
|
102
|
+
expect(clipExtractor.canHandle("https://clips.twitch.tv/AwesomeClip")).toBe(true);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("canHandle rejects VOD URL", () => {
|
|
106
|
+
expect(clipExtractor.canHandle("https://www.twitch.tv/videos/123456")).toBe(false);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("_NAME is twitch:clip", () => {
|
|
110
|
+
expect(clipExtractor._NAME).toBe("twitch:clip");
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("extract returns clip formats with quality variants", async () => {
|
|
114
|
+
const originalFetch = globalThis.fetch;
|
|
115
|
+
|
|
116
|
+
const mockResponse = {
|
|
117
|
+
data: {
|
|
118
|
+
clip: {
|
|
119
|
+
id: "clip123",
|
|
120
|
+
slug: "AwesomeClip",
|
|
121
|
+
title: "Amazing Play",
|
|
122
|
+
durationSeconds: 30,
|
|
123
|
+
viewCount: 10000,
|
|
124
|
+
createdAt: "2024-01-10T15:30:00Z",
|
|
125
|
+
broadcaster: { displayName: "Streamer", login: "streamer", id: "12345" },
|
|
126
|
+
thumbnailURL: "https://clips-media-assets2.twitch.tv/thumb.jpg",
|
|
127
|
+
playbackAccessToken: { value: "cliptoken", signature: "clipsig" },
|
|
128
|
+
videoQualities: [
|
|
129
|
+
{ quality: "1080", frameRate: 60, sourceURL: "https://clips-media.twitch.tv/clip-1080.mp4" },
|
|
130
|
+
{ quality: "720", frameRate: 30, sourceURL: "https://clips-media.twitch.tv/clip-720.mp4" },
|
|
131
|
+
{ quality: "480", frameRate: 30, sourceURL: "https://clips-media.twitch.tv/clip-480.mp4" },
|
|
132
|
+
],
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
globalThis.fetch = mock(() =>
|
|
138
|
+
Promise.resolve(
|
|
139
|
+
new Response(JSON.stringify(mockResponse), {
|
|
140
|
+
status: 200,
|
|
141
|
+
headers: { "Content-Type": "application/json" },
|
|
142
|
+
})
|
|
143
|
+
)
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
const info = await clipExtractor.extract("https://clips.twitch.tv/AwesomeClip");
|
|
147
|
+
|
|
148
|
+
expect(info.id).toBe("clip123");
|
|
149
|
+
expect(info.title).toBe("Amazing Play");
|
|
150
|
+
expect(info.duration).toBe(30);
|
|
151
|
+
expect(info.formats).toBeDefined();
|
|
152
|
+
expect(info.formats!.length).toBe(3);
|
|
153
|
+
|
|
154
|
+
const highestQuality = info.formats!.find((f) => f.format_id === "clip-1080");
|
|
155
|
+
expect(highestQuality).toBeDefined();
|
|
156
|
+
expect(highestQuality!.height).toBe(1080);
|
|
157
|
+
expect(highestQuality!.url).toContain("clipsig");
|
|
158
|
+
|
|
159
|
+
globalThis.fetch = originalFetch;
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
describe("TwitchLiveExtractor", () => {
|
|
164
|
+
test("canHandle matches live channel URL", () => {
|
|
165
|
+
expect(liveExtractor.canHandle("https://www.twitch.tv/streamer")).toBe(true);
|
|
166
|
+
expect(liveExtractor.canHandle("https://twitch.tv/streamer")).toBe(true);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
test("canHandle rejects VOD URL", () => {
|
|
170
|
+
expect(liveExtractor.canHandle("https://www.twitch.tv/videos/123456")).toBe(false);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("_NAME is twitch:live", () => {
|
|
174
|
+
expect(liveExtractor._NAME).toBe("twitch:live");
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test("extract returns HLS live stream URL", async () => {
|
|
178
|
+
const originalFetch = globalThis.fetch;
|
|
179
|
+
|
|
180
|
+
const mockTokenResponse = {
|
|
181
|
+
data: { streamPlaybackAccessToken: { value: "livetoken", signature: "livesig" } },
|
|
182
|
+
};
|
|
183
|
+
const mockStreamResponse = {
|
|
184
|
+
data: {
|
|
185
|
+
user: {
|
|
186
|
+
stream: {
|
|
187
|
+
id: "stream123",
|
|
188
|
+
title: "Playing Games Live",
|
|
189
|
+
viewersCount: 5000,
|
|
190
|
+
previewImageURL: "https://static-cdn.jtvnw.net/previews-ttv/thumb.jpg",
|
|
191
|
+
broadcaster: { displayName: "Streamer", login: "streamer", id: "12345" },
|
|
192
|
+
game: { name: "Minecraft" },
|
|
193
|
+
},
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
let callCount = 0;
|
|
199
|
+
globalThis.fetch = mock(() => {
|
|
200
|
+
callCount++;
|
|
201
|
+
const response = callCount === 1 ? mockTokenResponse : mockStreamResponse;
|
|
202
|
+
return Promise.resolve(
|
|
203
|
+
new Response(JSON.stringify(response), {
|
|
204
|
+
status: 200,
|
|
205
|
+
headers: { "Content-Type": "application/json" },
|
|
206
|
+
})
|
|
207
|
+
);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
const info = await liveExtractor.extract("https://www.twitch.tv/streamer");
|
|
211
|
+
|
|
212
|
+
expect(info.id).toBe("streamer");
|
|
213
|
+
expect(info.title).toBe("Playing Games Live");
|
|
214
|
+
expect(info.live_status).toBe("is_live");
|
|
215
|
+
expect(info.view_count).toBe(5000);
|
|
216
|
+
expect(info.formats).toBeDefined();
|
|
217
|
+
|
|
218
|
+
const liveFormat = info.formats!.find((f) => f.protocol === "m3u8");
|
|
219
|
+
expect(liveFormat).toBeDefined();
|
|
220
|
+
expect(liveFormat!.url).toContain("usher.twitchapps.com");
|
|
221
|
+
expect(liveFormat!.url).toContain("livesig");
|
|
222
|
+
|
|
223
|
+
globalThis.fetch = originalFetch;
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
test("extract throws when no access token returned", async () => {
|
|
227
|
+
const originalFetch = globalThis.fetch;
|
|
228
|
+
|
|
229
|
+
const mockTokenResponse = { data: { streamPlaybackAccessToken: null } };
|
|
230
|
+
const mockStreamResponse = { data: { user: null } };
|
|
231
|
+
|
|
232
|
+
let callCount = 0;
|
|
233
|
+
globalThis.fetch = mock(() => {
|
|
234
|
+
callCount++;
|
|
235
|
+
const response = callCount === 1 ? mockTokenResponse : mockStreamResponse;
|
|
236
|
+
return Promise.resolve(
|
|
237
|
+
new Response(JSON.stringify(response), {
|
|
238
|
+
status: 200,
|
|
239
|
+
headers: { "Content-Type": "application/json" },
|
|
240
|
+
})
|
|
241
|
+
);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
await expect(
|
|
245
|
+
liveExtractor.extract("https://www.twitch.tv/offlinechannel")
|
|
246
|
+
).rejects.toThrow(ExtractorError);
|
|
247
|
+
|
|
248
|
+
globalThis.fetch = originalFetch;
|
|
249
|
+
});
|
|
250
|
+
});
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { describe, test, expect, mock, beforeEach } from "bun:test";
|
|
2
|
+
import { TwitterExtractor } from "../../../src/extractors/twitter/index";
|
|
3
|
+
import { TwitterSpacesExtractor } from "../../../src/extractors/twitter/spaces";
|
|
4
|
+
import { ExtractorError } from "../../../src/core/types";
|
|
5
|
+
|
|
6
|
+
describe("TwitterExtractor", () => {
|
|
7
|
+
const extractor = new TwitterExtractor();
|
|
8
|
+
|
|
9
|
+
test("canHandle matches twitter.com status URLs", () => {
|
|
10
|
+
expect(extractor.canHandle("https://twitter.com/user/status/1234567890")).toBe(true);
|
|
11
|
+
expect(extractor.canHandle("https://www.twitter.com/user/status/1234567890")).toBe(true);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test("canHandle matches x.com status URLs", () => {
|
|
15
|
+
expect(extractor.canHandle("https://x.com/user/status/9876543210")).toBe(true);
|
|
16
|
+
expect(extractor.canHandle("https://www.x.com/elonmusk/status/9876543210")).toBe(true);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test("canHandle rejects non-tweet URLs", () => {
|
|
20
|
+
expect(extractor.canHandle("https://twitter.com/user")).toBe(false);
|
|
21
|
+
expect(extractor.canHandle("https://youtube.com/watch?v=abc")).toBe(false);
|
|
22
|
+
expect(extractor.canHandle("https://x.com/i/spaces/abc123")).toBe(false);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("_NAME is twitter", () => {
|
|
26
|
+
expect(extractor._NAME).toBe("twitter");
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("extract sets extractor metadata", async () => {
|
|
30
|
+
const mockTweetData = {
|
|
31
|
+
id_str: "1234567890",
|
|
32
|
+
full_text: "Test tweet with video",
|
|
33
|
+
user: { name: "Test User", screen_name: "testuser", id_str: "999" },
|
|
34
|
+
created_at: "Mon Jan 01 12:00:00 +0000 2024",
|
|
35
|
+
favorite_count: 100,
|
|
36
|
+
views: { count: "5000" },
|
|
37
|
+
mediaDetails: [
|
|
38
|
+
{
|
|
39
|
+
type: "video",
|
|
40
|
+
media_url_https: "https://pbs.twimg.com/ext_tw_video_thumb/1234567890/pu/img/thumb.jpg",
|
|
41
|
+
original_info: { width: 1280, height: 720 },
|
|
42
|
+
video_info: {
|
|
43
|
+
duration_millis: 30000,
|
|
44
|
+
variants: [
|
|
45
|
+
{
|
|
46
|
+
content_type: "video/mp4",
|
|
47
|
+
url: "https://video.twimg.com/ext_tw_video/1234567890/pu/vid/1280x720/video.mp4",
|
|
48
|
+
bitrate: 2176000,
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
content_type: "video/mp4",
|
|
52
|
+
url: "https://video.twimg.com/ext_tw_video/1234567890/pu/vid/640x360/video.mp4",
|
|
53
|
+
bitrate: 832000,
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
content_type: "application/x-mpegURL",
|
|
57
|
+
url: "https://video.twimg.com/ext_tw_video/1234567890/pu/pl/playlist.m3u8",
|
|
58
|
+
},
|
|
59
|
+
],
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const originalFetch = global.fetch;
|
|
66
|
+
global.fetch = mock(() =>
|
|
67
|
+
Promise.resolve({
|
|
68
|
+
ok: true,
|
|
69
|
+
json: () => Promise.resolve(mockTweetData),
|
|
70
|
+
} as Response),
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
try {
|
|
74
|
+
const info = await extractor.extract("https://twitter.com/testuser/status/1234567890");
|
|
75
|
+
|
|
76
|
+
expect(info.id).toBe("1234567890");
|
|
77
|
+
expect(info.title).toBe("Test tweet with video");
|
|
78
|
+
expect(info.uploader).toBe("Test User");
|
|
79
|
+
expect(info.uploader_id).toBe("testuser");
|
|
80
|
+
expect(info.view_count).toBe(5000);
|
|
81
|
+
expect(info.like_count).toBe(100);
|
|
82
|
+
expect(info.duration).toBe(30);
|
|
83
|
+
expect(info.extractor).toBe("twitter");
|
|
84
|
+
expect(info.extractor_key).toBe("TwitterExtractor");
|
|
85
|
+
expect(info.formats).toBeDefined();
|
|
86
|
+
expect(info.formats!.length).toBeGreaterThan(0);
|
|
87
|
+
|
|
88
|
+
const mp4Formats = info.formats!.filter((f) => f.ext === "mp4" && !f.protocol);
|
|
89
|
+
expect(mp4Formats.length).toBe(2);
|
|
90
|
+
|
|
91
|
+
const highestBitrate = mp4Formats[0];
|
|
92
|
+
expect(highestBitrate.tbr).toBe(2176);
|
|
93
|
+
|
|
94
|
+
const hlsFormat = info.formats!.find((f) => f.protocol === "m3u8");
|
|
95
|
+
expect(hlsFormat).toBeDefined();
|
|
96
|
+
} finally {
|
|
97
|
+
global.fetch = originalFetch;
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("extract throws ExtractorError when API returns no data", async () => {
|
|
102
|
+
const originalFetch = global.fetch;
|
|
103
|
+
global.fetch = mock(() =>
|
|
104
|
+
Promise.resolve({
|
|
105
|
+
ok: true,
|
|
106
|
+
json: () => Promise.resolve({}),
|
|
107
|
+
} as Response),
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
await expect(
|
|
112
|
+
extractor.extract("https://twitter.com/user/status/1234567890"),
|
|
113
|
+
).rejects.toThrow(ExtractorError);
|
|
114
|
+
} finally {
|
|
115
|
+
global.fetch = originalFetch;
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test("extract throws ExtractorError on API failure", async () => {
|
|
120
|
+
const originalFetch = global.fetch;
|
|
121
|
+
global.fetch = mock(() =>
|
|
122
|
+
Promise.resolve({
|
|
123
|
+
ok: false,
|
|
124
|
+
status: 403,
|
|
125
|
+
statusText: "Forbidden",
|
|
126
|
+
} as Response),
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
try {
|
|
130
|
+
await expect(
|
|
131
|
+
extractor.extract("https://twitter.com/user/status/1234567890"),
|
|
132
|
+
).rejects.toThrow(ExtractorError);
|
|
133
|
+
} finally {
|
|
134
|
+
global.fetch = originalFetch;
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("extract handles tweet without video", async () => {
|
|
139
|
+
const mockTweetData = {
|
|
140
|
+
id_str: "9999999999",
|
|
141
|
+
full_text: "Tweet without video",
|
|
142
|
+
user: { name: "User", screen_name: "user", id_str: "1" },
|
|
143
|
+
created_at: "Mon Jan 01 12:00:00 +0000 2024",
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
const originalFetch = global.fetch;
|
|
147
|
+
global.fetch = mock(() =>
|
|
148
|
+
Promise.resolve({
|
|
149
|
+
ok: true,
|
|
150
|
+
json: () => Promise.resolve(mockTweetData),
|
|
151
|
+
} as Response),
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
try {
|
|
155
|
+
const info = await extractor.extract("https://twitter.com/user/status/9999999999");
|
|
156
|
+
expect(info.id).toBe("9999999999");
|
|
157
|
+
expect(info.formats).toEqual([]);
|
|
158
|
+
} finally {
|
|
159
|
+
global.fetch = originalFetch;
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
describe("TwitterSpacesExtractor", () => {
|
|
165
|
+
const extractor = new TwitterSpacesExtractor();
|
|
166
|
+
|
|
167
|
+
test("canHandle matches Twitter Spaces URLs", () => {
|
|
168
|
+
expect(extractor.canHandle("https://twitter.com/i/spaces/1YqKDqPQjnkKV")).toBe(true);
|
|
169
|
+
expect(extractor.canHandle("https://x.com/i/spaces/1YqKDqPQjnkKV")).toBe(true);
|
|
170
|
+
expect(extractor.canHandle("https://www.twitter.com/i/spaces/abc123XYZ")).toBe(true);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("canHandle rejects non-spaces URLs", () => {
|
|
174
|
+
expect(extractor.canHandle("https://twitter.com/user/status/123")).toBe(false);
|
|
175
|
+
expect(extractor.canHandle("https://twitter.com/user")).toBe(false);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test("_NAME is twitter:spaces", () => {
|
|
179
|
+
expect(extractor._NAME).toBe("twitter:spaces");
|
|
180
|
+
});
|
|
181
|
+
});
|