@postfetch/core 0.12.5 → 0.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -37,6 +37,21 @@ function createNet(baseFetch = globalThis.fetch) {
37
37
  throw lastError instanceof Error ? lastError : new Error("request failed");
38
38
  };
39
39
  }
40
+ async function readBody(net, url, init, read) {
41
+ for (let attempt = 1;; attempt += 1) {
42
+ const response = await net(url, init);
43
+ if (!response.ok || !response.body)
44
+ return read(response);
45
+ try {
46
+ return await read(response);
47
+ } catch (error) {
48
+ if (attempt === 3)
49
+ throw error;
50
+ await response.body?.cancel().catch(() => {});
51
+ await sleep(500 * 2 ** (attempt - 1));
52
+ }
53
+ }
54
+ }
40
55
  function object(value) {
41
56
  return typeof value === "object" && value !== null && !Array.isArray(value);
42
57
  }
@@ -1256,11 +1271,13 @@ function parseMaster(text, baseUrl, options = {}) {
1256
1271
  return { variants, audio };
1257
1272
  }
1258
1273
  async function assembleHls(net, playlistUrl, headers) {
1259
- const response = await net(playlistUrl, { headers });
1260
- if (!response.ok) {
1261
- throw new Error(`HLS playlist failed: ${response.status}`);
1262
- }
1263
- const segments = mediaSegments(await response.text(), playlistUrl);
1274
+ const text = await readBody(net, playlistUrl, { headers }, async (response) => {
1275
+ if (!response.ok) {
1276
+ throw new Error(`HLS playlist failed: ${response.status}`);
1277
+ }
1278
+ return response.text();
1279
+ });
1280
+ const segments = mediaSegments(text, playlistUrl);
1264
1281
  if (segments.length === 0) {
1265
1282
  throw new Error("HLS playlist has no segments");
1266
1283
  }
@@ -1317,11 +1334,12 @@ async function fetchSegment(net, segment, headers) {
1317
1334
  if (segment.range) {
1318
1335
  requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
1319
1336
  }
1320
- const response = await net(segment.url, { headers: requestHeaders });
1321
- if (!response.ok) {
1322
- throw new Error(`HLS segment failed: ${response.status}`);
1323
- }
1324
- return new Uint8Array(await response.arrayBuffer());
1337
+ return readBody(net, segment.url, { headers: requestHeaders }, async (response) => {
1338
+ if (!response.ok) {
1339
+ throw new Error(`HLS segment failed: ${response.status}`);
1340
+ }
1341
+ return new Uint8Array(await response.arrayBuffer());
1342
+ });
1325
1343
  }
1326
1344
 
1327
1345
  // src/pinterest.ts
@@ -2015,950 +2033,1216 @@ function trackId(track) {
2015
2033
  return string(track.permalink) ?? "track";
2016
2034
  }
2017
2035
 
2018
- // src/tiktok.ts
2019
- var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
2020
- var pageAttemptCount = 12;
2021
- async function resolveTiktok(input) {
2022
- const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
2023
- const id = videoId(pageUrl);
2036
+ // src/youtube.ts
2037
+ var androidVrClient = {
2038
+ name: "ANDROID_VR",
2039
+ number: "28",
2040
+ userAgent: "com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip",
2041
+ version: "1.65.10"
2042
+ };
2043
+ var visionOsClient = {
2044
+ clientName: "VISIONOS",
2045
+ clientVersion: "1.02",
2046
+ deviceMake: "Apple",
2047
+ deviceModel: "RealityDevice17,1",
2048
+ userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
2049
+ osName: "visionOS",
2050
+ osVersion: "26.5.23O471"
2051
+ };
2052
+ var browserCookie = "PREF=hl=en&tz=UTC; SOCS=CAI";
2053
+ async function resolveYoutube(input) {
2054
+ const id = youtubeVideoId(input.url);
2024
2055
  if (!id) {
2025
- throw new Error("TikTok video id not found");
2026
- }
2027
- const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
2028
- const user = author(page.item) ?? username(pageUrl) ?? "i";
2029
- const headers = {
2030
- referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
2031
- "user-agent": page.userAgent
2032
- };
2033
- if (page.cookie) {
2034
- headers.cookie = page.cookie;
2035
- }
2036
- const items = mediaItems4(page.item, user, id, headers);
2037
- if (items.length === 0) {
2038
- throw new Error("TikTok media not found");
2056
+ throw new Error("YouTube video id not found");
2039
2057
  }
2040
- return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
2041
- }
2042
- async function videoPage(net, id, photo) {
2043
- for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
2044
- const baseHeaders2 = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
2045
- const userAgent2 = baseHeaders2["user-agent"];
2046
- const headers2 = {
2047
- ...baseHeaders2,
2048
- referer: "https://www.tiktok.com/",
2049
- "sec-fetch-site": "same-origin"
2050
- };
2058
+ let session;
2059
+ try {
2060
+ session = await youtubeSession(input.net, id);
2061
+ const payload = await youtubePlayer(input.net, playerBody(id, session), playerHeaders(session));
2062
+ const streams = selectStreams(payload, input.preferredWidth);
2063
+ if (!streams) {
2064
+ throw new Error("YouTube mp4 stream not found");
2065
+ }
2066
+ const headers = { "user-agent": androidVrClient.userAgent };
2067
+ await Promise.all([streams.video, streams.audio].filter((url) => url !== null).map((url) => probeStream(input.net, url, headers)));
2068
+ return youtubeResult(id, payload, streams, headers);
2069
+ } catch (primaryError) {
2051
2070
  try {
2052
- return { ...await fetchVideoPage(net, id, headers2), userAgent: userAgent2 };
2053
- } catch (error) {
2054
- if (!recoverablePageError(error)) {
2055
- throw error;
2056
- }
2071
+ return await resolveVisionOs(input, id, session?.visitorData);
2072
+ } catch (fallbackError) {
2073
+ const message = (error) => error instanceof Error ? error.message : String(error);
2074
+ throw new Error(`YouTube resolution failed: ${message(primaryError)}; visionOS HLS: ${message(fallbackError)}`, { cause: fallbackError });
2057
2075
  }
2058
2076
  }
2059
- if (!photo) {
2060
- throw new Error("TikTok hydration not found");
2061
- }
2062
- const baseHeaders = navigationHeaders();
2063
- const userAgent = baseHeaders["user-agent"];
2064
- const headers = {
2065
- ...baseHeaders,
2066
- referer: "https://www.tiktok.com/",
2067
- "sec-fetch-site": "same-origin"
2068
- };
2069
- return { ...await fetchEmbedPage(net, id, headers), userAgent };
2070
- }
2071
- function tiktokMetadata(item) {
2072
- const user = object(item.author) ? item.author : null;
2073
- const stats = object(item.stats) ? item.stats : null;
2074
- const music = object(item.music) ? item.music : null;
2075
- const video = object(item.video) ? item.video : null;
2076
- return {
2077
- text: string(item.desc) ?? undefined,
2078
- author: user ? {
2079
- handle: string(user.uniqueId) ?? undefined,
2080
- name: string(user.nickname) ?? undefined,
2081
- verified: bool(user.verified)
2082
- } : undefined,
2083
- createdAt: isoFromEpochSeconds(item.createTime),
2084
- likeCount: stats ? count(stats.diggCount) : undefined,
2085
- commentCount: stats ? count(stats.commentCount) : undefined,
2086
- shareCount: stats ? count(stats.shareCount) : undefined,
2087
- viewCount: stats ? count(stats.playCount) : undefined,
2088
- extra: {
2089
- saveCount: stats ? count(stats.collectCount) : undefined,
2090
- musicTitle: music ? string(music.title) ?? undefined : undefined,
2091
- musicAuthor: music ? string(music.authorName) ?? undefined : undefined,
2092
- region: string(item.locationCreated) ?? undefined,
2093
- durationSeconds: video ? count(video.duration) : undefined
2094
- }
2095
- };
2096
- }
2097
- async function fetchVideoPage(net, id, headers) {
2098
- const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
2099
- headers
2100
- });
2101
- return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
2102
2077
  }
2103
- async function fetchEmbedPage(net, id, headers) {
2104
- const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
2105
- headers
2078
+ async function youtubePlayer(net, body, headers) {
2079
+ const response = await net("https://www.youtube.com/youtubei/v1/player?prettyPrint=false", {
2080
+ body: JSON.stringify(body),
2081
+ headers,
2082
+ method: "POST"
2106
2083
  });
2107
- return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
2108
- }
2109
- function isShortlinkHost(hostname) {
2110
- return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
2111
- }
2112
- async function followShortlink(net, userAgent, input) {
2113
- const url = asUrl(input);
2114
- if (!isShortlinkHost(url.hostname)) {
2115
- return input;
2084
+ if (!response.ok) {
2085
+ throw new Error(`YouTube player failed: ${response.status}`);
2116
2086
  }
2117
- const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
2118
- const location = response.headers.get("location");
2119
- if (location) {
2120
- return new URL(location, input).href;
2087
+ const payload = await response.json();
2088
+ const status = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.status) : null;
2089
+ if (status !== "OK") {
2090
+ const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
2091
+ throw new Error(reason ?? "YouTube video unavailable");
2121
2092
  }
2122
- const html = await response.text();
2123
- const href = html.match(/<a href="(https:\/\/[^"]+)"/)?.[1];
2124
- return href ? new URL(href.split("?")[0]).href : input;
2093
+ return payload;
2125
2094
  }
2126
- function videoId(input) {
2127
- return asUrl(input).pathname.match(/(?:video|photo)\/(\d+)/)?.[1] ?? null;
2095
+ function youtubeResult(id, payload, streams, headers) {
2096
+ const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
2097
+ const media = {
2098
+ filename: filename(`youtube_${title ?? id}_${id}.mp4`),
2099
+ headers,
2100
+ id,
2101
+ kind: "video",
2102
+ mime: "video/mp4",
2103
+ platform: "youtube",
2104
+ url: streams.video,
2105
+ ...streams.hls ? { hls: true } : {},
2106
+ ...streams.audio ? { audio: { headers, url: streams.audio } } : {}
2107
+ };
2108
+ return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
2128
2109
  }
2129
- function itemStruct(html) {
2130
- if (html.includes("SlardarWAF") || html.includes("_wafchallengeid")) {
2131
- throw new Error("TikTok WAF challenge");
2110
+ async function probeStream(net, url, headers) {
2111
+ const response = await net(url, { headers }, 1);
2112
+ await response.body?.cancel();
2113
+ if (!response.ok) {
2114
+ throw new Error(`YouTube stream failed: ${response.status}`);
2132
2115
  }
2133
- const start = html.indexOf(marker);
2134
- const end = start === -1 ? -1 : html.indexOf("</script>", start + marker.length);
2135
- if (start === -1 || end === -1) {
2136
- throw new Error("TikTok hydration not found");
2116
+ }
2117
+ async function resolveVisionOs(input, id, visitorData) {
2118
+ const headers = { "user-agent": visionOsClient.userAgent };
2119
+ const payload = await youtubePlayer(input.net, {
2120
+ videoId: id,
2121
+ contentCheckOk: true,
2122
+ racyCheckOk: true,
2123
+ context: { client: { ...visionOsClient, hl: "en", gl: "US", ...visitorData ? { visitorData } : {} } }
2124
+ }, {
2125
+ ...headers,
2126
+ "content-type": "application/json",
2127
+ "x-youtube-client-name": "101",
2128
+ "x-youtube-client-version": visionOsClient.clientVersion,
2129
+ ...visitorData ? { "x-goog-visitor-id": visitorData } : {}
2130
+ });
2131
+ const streaming = object(payload) && object(payload.streamingData) ? payload.streamingData : null;
2132
+ const manifestUrl = streaming ? string(streaming.hlsManifestUrl) : null;
2133
+ if (!manifestUrl) {
2134
+ throw new Error("YouTube HLS manifest not found");
2137
2135
  }
2138
- const parsed = JSON.parse(html.slice(start + marker.length, end));
2139
- const scope = object(parsed) && object(parsed.__DEFAULT_SCOPE__) ? parsed.__DEFAULT_SCOPE__ : null;
2140
- const detail = scope && object(scope["webapp.video-detail"]) ? scope["webapp.video-detail"] : null;
2141
- const info = detail && object(detail.itemInfo) ? detail.itemInfo : null;
2142
- const item = info && object(info.itemStruct) ? info.itemStruct : null;
2143
- if (!item) {
2144
- throw new Error("TikTok itemStruct not found");
2136
+ const response = await input.net(manifestUrl, { headers });
2137
+ if (!response.ok) {
2138
+ throw new Error(`YouTube HLS manifest failed: ${response.status}`);
2145
2139
  }
2146
- return item;
2147
- }
2148
- function embedItemStruct(html, id) {
2149
- const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
2150
- if (!raw) {
2151
- throw new Error("TikTok embed state not found");
2140
+ const master = parseMaster(await response.text(), manifestUrl, { preferOriginalAudio: true });
2141
+ const variants = master.variants.filter((variant2) => /(?:^|,)(?:vp09|av01)\./i.test(variant2.codecs ?? "") && /(?:^|,)mp4a\.40\.2(?:,|$)/i.test(variant2.codecs ?? "") && variant2.audioGroup && master.audio[variant2.audioGroup]).sort((left, right) => Math.abs(left.width - input.preferredWidth) - Math.abs(right.width - input.preferredWidth) || right.bandwidth - left.bandwidth);
2142
+ const variant = variants[0];
2143
+ if (!variant?.audioGroup) {
2144
+ throw new Error("YouTube fragmented MP4 HLS video with AAC audio not found");
2152
2145
  }
2153
- const parsed = JSON.parse(raw);
2154
- const source2 = object(parsed) && object(parsed.source) ? parsed.source : null;
2155
- const data = source2 && object(source2.data) ? source2.data : null;
2156
- const rawRoute = data ? data[`/embed/v2/${id}`] : null;
2157
- const route = object(rawRoute) ? rawRoute : null;
2158
- const videoData = route && object(route.videoData) ? route.videoData : null;
2159
- const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
2160
- if (!videoData || !item) {
2161
- throw new Error("TikTok embed item not found");
2146
+ const playlist = await input.net(variant.url, { headers });
2147
+ if (!playlist.ok) {
2148
+ throw new Error(`YouTube HLS playlist failed: ${playlist.status}`);
2162
2149
  }
2163
- const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
2164
- const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
2165
- const video = object(item.video) ? item.video : {};
2166
- const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
2167
- const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
2168
- const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
2169
- if (displayImages.length === 0) {
2170
- throw new Error("TikTok video hydration not found");
2150
+ if (!(await playlist.text()).includes("#EXT-X-MAP:")) {
2151
+ throw new Error("YouTube HLS video is not fragmented MP4");
2171
2152
  }
2172
- return {
2173
- ...item,
2174
- author: {
2175
- nickname: string(authorInfo.nickName),
2176
- uniqueId: string(authorInfo.uniqueId),
2177
- verified: bool(authorInfo.verified)
2178
- },
2179
- desc: string(item.text),
2180
- imagePost: {
2181
- images: displayImages.map((image) => ({
2182
- imageURL: {
2183
- urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
2184
- }
2185
- }))
2186
- },
2187
- music: {
2188
- authorName: string(musicInfo.authorName),
2189
- playUrl: firstString(musicInfo.playUrl),
2190
- title: string(musicInfo.musicName)
2191
- },
2192
- stats: {
2193
- commentCount: item.commentCount,
2194
- diggCount: item.diggCount,
2195
- playCount: item.playCount,
2196
- shareCount: item.shareCount
2197
- },
2198
- video: {
2199
- duration: videoMeta.duration
2153
+ return youtubeResult(id, payload, { video: variant.url, audio: master.audio[variant.audioGroup], hls: true }, headers);
2154
+ }
2155
+ function youtubeMetadata(payload) {
2156
+ const details = object(payload) && object(payload.videoDetails) ? payload.videoDetails : null;
2157
+ const renderer = object(payload) && object(payload.microformat) && object(payload.microformat.playerMicroformatRenderer) ? payload.microformat.playerMicroformatRenderer : null;
2158
+ const author = details ? string(details.author) : null;
2159
+ const keywords = details && Array.isArray(details.keywords) ? details.keywords.filter((keyword) => typeof keyword === "string") : [];
2160
+ return {
2161
+ title: details ? string(details.title) ?? undefined : undefined,
2162
+ text: details ? string(details.shortDescription) ?? undefined : undefined,
2163
+ author: author ? { name: author } : undefined,
2164
+ createdAt: renderer ? isoFromDateString(renderer.publishDate) : undefined,
2165
+ viewCount: details ? count(details.viewCount) : undefined,
2166
+ extra: {
2167
+ channelId: details ? string(details.channelId) ?? undefined : undefined,
2168
+ durationSeconds: details ? count(details.lengthSeconds) : undefined,
2169
+ keywords: keywords.length > 0 ? keywords : undefined
2200
2170
  }
2201
2171
  };
2202
2172
  }
2203
- function recoverablePageError(error) {
2204
- const message = error instanceof Error ? error.message : "";
2205
- return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
2173
+ function selectStreams(payload, preferredWidth) {
2174
+ const adaptive = adaptiveFormats(payload);
2175
+ const video = bestVideo(adaptive, preferredWidth);
2176
+ const audio = bestAudio(adaptive);
2177
+ if (video && audio) {
2178
+ return { video, audio };
2179
+ }
2180
+ const progressive = selectFormat(payload);
2181
+ return progressive ? { video: progressive, audio: null } : null;
2206
2182
  }
2207
- function firstString(value) {
2208
- return Array.isArray(value) ? value.map(string).find((candidate) => candidate !== null) ?? null : string(value);
2183
+ function adaptiveFormats(payload) {
2184
+ const root = object(payload) ? payload : null;
2185
+ const streaming = root && object(root.streamingData) ? root.streamingData : null;
2186
+ return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
2209
2187
  }
2210
- function downloadUrl(item) {
2211
- const video = object(item.video) ? item.video : null;
2212
- return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
2188
+ function bestVideo(formats, preferredWidth) {
2189
+ const candidates = formats.filter((format) => {
2190
+ const mimeType2 = string(format.mimeType);
2191
+ return Boolean(string(format.url)) && mimeType2?.startsWith("video/mp4") === true && mimeType2.includes("avc1");
2192
+ });
2193
+ const best = candidates.reduce((current, format) => {
2194
+ if (!current) {
2195
+ return format;
2196
+ }
2197
+ const width = number(format.width) ?? 0;
2198
+ const currentWidth = number(current.width) ?? 0;
2199
+ return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
2200
+ }, null);
2201
+ return best ? string(best.url) : null;
2213
2202
  }
2214
- function mediaItems4(item, user, id, headers) {
2215
- const images = imageItems3(item, user, id, headers);
2216
- if (images.length > 0) {
2217
- const audio = audioItem2(item, user, id, headers);
2218
- return audio ? [...images, audio] : images;
2203
+ function bestAudio(formats) {
2204
+ const candidates = formats.filter((format) => Boolean(string(format.url)) && string(format.mimeType)?.startsWith("audio/mp4") === true);
2205
+ const best = candidates.reduce((current, format) => {
2206
+ if (!current) {
2207
+ return format;
2208
+ }
2209
+ const preference = audioPreference(format) - audioPreference(current);
2210
+ if (preference !== 0) {
2211
+ return preference > 0 ? format : current;
2212
+ }
2213
+ return (number(format.bitrate) ?? 0) > (number(current.bitrate) ?? 0) ? format : current;
2214
+ }, null);
2215
+ return best ? string(best.url) : null;
2216
+ }
2217
+ function audioPreference(format) {
2218
+ const track = object(format.audioTrack) ? format.audioTrack : null;
2219
+ if (track && /\boriginal\b/i.test(string(track.displayName) ?? "")) {
2220
+ return 2;
2219
2221
  }
2220
- const url = downloadUrl(item);
2221
- return url ? [{
2222
- filename: filename(`tiktok_${user}_${id}.mp4`),
2223
- headers,
2224
- id,
2225
- kind: "video",
2226
- mime: "video/mp4",
2227
- platform: "tiktok",
2228
- url
2229
- }] : [];
2222
+ return track?.audioIsDefault === true ? 1 : 0;
2230
2223
  }
2231
- function imageItems3(item, user, id, headers) {
2232
- const imagePost = object(item.imagePost) ? item.imagePost : null;
2233
- const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
2234
- return images.flatMap((image, index) => {
2235
- const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
2236
- const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
2237
- const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
2238
- return url ? [{
2239
- filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
2240
- headers,
2241
- id,
2242
- kind: "image",
2243
- mime: "image/jpeg",
2244
- platform: "tiktok",
2245
- url
2246
- }] : [];
2247
- });
2224
+ function playerBody(id, session) {
2225
+ return {
2226
+ contentCheckOk: true,
2227
+ context: {
2228
+ client: {
2229
+ androidSdkVersion: 32,
2230
+ clientName: androidVrClient.name,
2231
+ clientVersion: androidVrClient.version,
2232
+ deviceMake: "Oculus",
2233
+ deviceModel: "Quest 3",
2234
+ gl: "US",
2235
+ hl: "en",
2236
+ osName: "Android",
2237
+ osVersion: "12L",
2238
+ timeZone: "UTC",
2239
+ userAgent: androidVrClient.userAgent,
2240
+ utcOffsetMinutes: 0,
2241
+ visitorData: session.visitorData
2242
+ }
2243
+ },
2244
+ playbackContext: {
2245
+ contentPlaybackContext: {
2246
+ html5Preference: "HTML5_PREF_WANTS",
2247
+ signatureTimestamp: session.signatureTimestamp
2248
+ }
2249
+ },
2250
+ racyCheckOk: true,
2251
+ videoId: id
2252
+ };
2248
2253
  }
2249
- function audioItem2(item, user, id, headers) {
2250
- const video = object(item.video) ? item.video : null;
2251
- const music = object(item.music) ? item.music : null;
2252
- const url = video ? string(video.playAddr) : null;
2253
- const fallback = music ? string(music.playUrl) : null;
2254
- const selected = url ?? fallback;
2255
- if (!selected) {
2256
- return null;
2254
+ async function youtubeSession(net, id) {
2255
+ const response = await net(`https://www.youtube.com/watch?v=${id}&bpctr=9999999999&has_verified=1`, {
2256
+ headers: {
2257
+ accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
2258
+ "accept-language": "en-us,en;q=0.5",
2259
+ cookie: browserCookie,
2260
+ "sec-fetch-mode": "navigate",
2261
+ "user-agent": browserUserAgent()
2262
+ }
2263
+ });
2264
+ if (!response.ok) {
2265
+ throw new Error(`YouTube page failed: ${response.status}`);
2266
+ }
2267
+ const page = await response.text();
2268
+ const visitorData = string(page.match(/"visitorData":"([^"]+)"/)?.[1]);
2269
+ if (!visitorData) {
2270
+ throw new Error("YouTube visitor data not found");
2271
+ }
2272
+ const playerPath = string(page.match(/"jsUrl":"([^"]+)"/)?.[1]) ?? string(page.match(/"PLAYER_JS_URL":"([^"]+)"/)?.[1]);
2273
+ if (!playerPath) {
2274
+ throw new Error("YouTube player url not found");
2257
2275
  }
2258
- const extension = selected.includes("mime_type=audio_mpeg") ? "mp3" : "m4a";
2259
2276
  return {
2260
- filename: filename(`tiktok_${user}_${id}_audio.${extension}`),
2261
- headers,
2262
- id,
2263
- kind: "audio",
2264
- mime: extension === "mp3" ? "audio/mpeg" : "audio/mp4",
2265
- platform: "tiktok",
2266
- url: selected
2277
+ cookie: youtubeCookie(response.headers),
2278
+ signatureTimestamp: await signatureTimestamp(net, playerPath),
2279
+ visitorData
2267
2280
  };
2268
2281
  }
2269
- function author(item) {
2270
- const user = object(item.author) ? item.author : null;
2271
- return user ? string(user.uniqueId) : null;
2282
+ function playerHeaders(session) {
2283
+ return new Headers({
2284
+ "content-type": "application/json",
2285
+ cookie: `${browserCookie}; ${session.cookie}`,
2286
+ origin: "https://www.youtube.com",
2287
+ "user-agent": androidVrClient.userAgent,
2288
+ "x-goog-visitor-id": session.visitorData,
2289
+ "x-youtube-client-name": androidVrClient.number,
2290
+ "x-youtube-client-version": androidVrClient.version
2291
+ });
2272
2292
  }
2273
- function username(input) {
2274
- return asUrl(input).pathname.match(/@([^/]+)/)?.[1] ?? null;
2293
+ function youtubeCookie(headers) {
2294
+ const readable = headers;
2295
+ const fallback = headers.get("set-cookie");
2296
+ const cookieHeaders = readable.getSetCookie ? readable.getSetCookie() : fallback ? [fallback] : [];
2297
+ return cookieHeaders.map((header) => string(header.split(";", 1)[0])).filter(string).join("; ");
2275
2298
  }
2276
- function cookieHeader(headers) {
2277
- const getter = headers.getSetCookie;
2278
- const setCookie = typeof getter === "function" ? getter.call(headers).join(",") : headers.get("set-cookie");
2279
- const cookies = setCookie?.split(/,(?=[^;]+?=)/).map((part) => part.split(";")[0]?.trim()).filter((part) => Boolean(part));
2280
- return cookies && cookies.length > 0 ? cookies.join("; ") : null;
2299
+ async function signatureTimestamp(net, playerPath) {
2300
+ const response = await net(new URL(playerPath, "https://www.youtube.com").toString(), {
2301
+ headers: { "user-agent": browserUserAgent() }
2302
+ });
2303
+ if (!response.ok) {
2304
+ throw new Error(`YouTube player failed: ${response.status}`);
2305
+ }
2306
+ const player = await response.text();
2307
+ const timestamp = number(Number(player.match(/signatureTimestamp[:=](\d+)/)?.[1] ?? player.match(/sts[:=](\d+)/)?.[1]));
2308
+ if (!timestamp) {
2309
+ throw new Error("YouTube signature timestamp not found");
2310
+ }
2311
+ return timestamp;
2281
2312
  }
2282
-
2283
- // src/twitter.ts
2284
- async function resolveTwitter(input) {
2285
- const id = tweetId(input.url);
2286
- if (!id) {
2287
- throw new Error("Tweet id not found");
2313
+ function youtubeVideoId(input) {
2314
+ const url = asUrl(input);
2315
+ if (url.hostname === "youtu.be") {
2316
+ return cleanId(url.pathname.split("/").filter(Boolean)[0]);
2288
2317
  }
2289
- const tweet = await syndication(input.net, id);
2290
- if (string(tweet.__typename) === "TweetTombstone") {
2291
- throw new Error("Tweet is unavailable or age-restricted");
2318
+ const fromQuery = cleanId(url.searchParams.get("v"));
2319
+ if (fromQuery) {
2320
+ return fromQuery;
2292
2321
  }
2293
- await expandTwitterText(input.net, tweet, id);
2294
- const posts = twitterTweets(tweet, id);
2295
- const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
2296
- const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
2297
- return media.flatMap((entry, index) => {
2298
- const candidates = twitterCandidates(entry, postId2, index + 1);
2299
- return candidates.length > 0 ? [candidates] : [];
2300
- });
2301
- });
2302
- for (const { id: postId2, tweet: post } of posts.toReversed()) {
2303
- const card = twitterCardImage(post, postId2);
2304
- if (card)
2305
- groups.push([{ bitrate: 0, item: card }]);
2306
- }
2307
- const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
2308
- return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
2322
+ const parts = url.pathname.split("/").filter(Boolean);
2323
+ const index = parts.findIndex((part) => part === "shorts" || part === "live" || part === "embed");
2324
+ return index >= 0 ? cleanId(parts[index + 1]) : null;
2309
2325
  }
2310
- function twitterMetadata(tweet) {
2311
- const user = object(tweet.user) ? tweet.user : null;
2312
- const extra = {};
2313
- const lang = string(tweet.lang);
2314
- if (lang) {
2315
- extra.lang = lang;
2316
- }
2317
- const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
2318
- const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
2319
- if (quotedTweet && quotedId) {
2320
- extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
2321
- }
2322
- const parent = object(tweet.parent) ? tweet.parent : null;
2323
- const parentId = parent ? string(parent.id_str) : null;
2324
- const replyToId = string(tweet.in_reply_to_status_id_str) ?? parentId;
2325
- if (replyToId)
2326
- extra.replyToId = replyToId;
2327
- if (parent && parentId) {
2328
- extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
2329
- }
2330
- return {
2331
- text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
2332
- author: user ? {
2333
- handle: string(user.screen_name) ?? undefined,
2334
- name: string(user.name) ?? undefined,
2335
- verified: bool(user.verified) ?? bool(user.is_blue_verified)
2336
- } : undefined,
2337
- createdAt: isoFromDateString(tweet.created_at),
2338
- likeCount: count(tweet.favorite_count),
2339
- commentCount: count(tweet.conversation_count),
2340
- shareCount: count(tweet.retweet_count),
2341
- viewCount: count(tweet.view_count),
2342
- nsfw: bool(tweet.possibly_sensitive),
2343
- extra
2344
- };
2326
+ function cleanId(value) {
2327
+ return typeof value === "string" && /^[A-Za-z0-9_-]{11}$/.test(value) ? value : null;
2345
2328
  }
2346
- function noteText(tweet) {
2347
- const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
2348
- const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
2349
- const result = results && object(results.result) ? results.result : null;
2350
- return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
2329
+ function selectFormat(payload) {
2330
+ const root = object(payload) ? payload : null;
2331
+ const streaming = root && object(root.streamingData) ? root.streamingData : null;
2332
+ const formats = streaming && Array.isArray(streaming.formats) ? streaming.formats.filter(object) : [];
2333
+ const mp4 = formats.filter((format) => string(format.url) && string(format.mimeType)?.startsWith("video/mp4")).sort((left, right) => height(right) - height(left));
2334
+ return mp4[0] ? string(mp4[0].url) : null;
2351
2335
  }
2352
- function twitterText(tweet) {
2353
- return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
2336
+ function height(format) {
2337
+ return number(format.height) ?? Number(string(format.qualityLabel)?.match(/(\d+)p/)?.[1] ?? 0);
2354
2338
  }
2355
- function cleanTwitterText(text, tweet) {
2356
- if (text === null)
2357
- return null;
2358
- const entities = object(tweet.entities) ? tweet.entities : null;
2359
- const media = [
2360
- ...entities && Array.isArray(entities.media) ? entities.media : [],
2361
- ...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
2362
- ];
2363
- const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
2364
- while (true) {
2365
- const trailing = text.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
2366
- if (!trailing || !urls.has(trailing[1]))
2367
- break;
2368
- text = text.slice(0, trailing.index).trimEnd();
2339
+
2340
+ // src/spotify.ts
2341
+ async function resolveSpotify(input) {
2342
+ const url = asUrl(input.url);
2343
+ const id = url.pathname.match(/^\/(?:intl-[a-z-]+\/)?(?:embed\/)?track\/([A-Za-z0-9]{22})\/?$/i)?.[1];
2344
+ if (!id) {
2345
+ throw new PostfetchError(400, "Spotify requires an open.spotify.com track URL");
2369
2346
  }
2370
- return text.replace(/&(?:amp|gt|lt);/g, (entity) => entity === "&amp;" ? "&" : entity === "&gt;" ? ">" : "<");
2347
+ const track = await spotifyTrack(input, id);
2348
+ const { youtube, youtubeUrl, liveFallback } = await matchingYoutube(input, track);
2349
+ return {
2350
+ archiveFilename: filename(`spotify_${id}.zip`),
2351
+ comments: [],
2352
+ id,
2353
+ items: youtube.items,
2354
+ platform: "spotify",
2355
+ metadata: {
2356
+ title: track.title,
2357
+ author: { name: track.artists.join(", ") },
2358
+ extra: {
2359
+ artists: track.artists,
2360
+ durationSeconds: track.durationSeconds,
2361
+ youtubeUrl,
2362
+ ...liveFallback ? { youtubeLiveFallback: true } : {}
2363
+ }
2364
+ }
2365
+ };
2371
2366
  }
2372
- function needsFullText(tweet) {
2373
- if (noteText(tweet) || string(tweet.full_text)) {
2374
- return false;
2367
+ async function spotifyTrack(input, id) {
2368
+ const response = await input.net(`https://open.spotify.com/embed/track/${id}`, {
2369
+ headers: { "user-agent": browserUserAgent() }
2370
+ });
2371
+ if (!response.ok) {
2372
+ throw new PostfetchError(response.status === 404 ? 404 : 502, `Spotify track lookup failed: ${response.status}`);
2375
2373
  }
2376
- return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
2377
- }
2378
- async function expandTwitterText(net, tweet, rootId) {
2379
- const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
2380
- for (const [id, post] of posts) {
2381
- if (!needsFullText(post)) {
2374
+ const html = await response.text();
2375
+ const raw = html.match(/<script\b[^>]*\bid=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
2376
+ const payload = parseJson2(raw);
2377
+ const props = payload && object(payload.props) ? payload.props : null;
2378
+ const page = props && object(props.pageProps) ? props.pageProps : null;
2379
+ const state = page && object(page.state) ? page.state : null;
2380
+ const data = state && object(state.data) ? state.data : null;
2381
+ const entity = data && object(data.entity) ? data.entity : null;
2382
+ const title = entity && (string(entity.title) ?? string(entity.name));
2383
+ const artists = entity && Array.isArray(entity.artists) ? entity.artists.filter(object).map((artist) => string(artist.name)).filter((name) => name !== null) : [];
2384
+ if (!entity || entity.type !== "track" || entity.id !== id || !title || artists.length === 0) {
2385
+ throw new PostfetchError(502, "Spotify track metadata not found");
2386
+ }
2387
+ const duration = number(entity.duration);
2388
+ return { title, artists, durationSeconds: duration !== null && duration > 0 ? duration / 1000 : undefined };
2389
+ }
2390
+ async function matchingYoutube(input, track) {
2391
+ const quotedTitle = `"${track.title.replace(/"/g, "")}"`;
2392
+ const primaryQuery = `${track.artists.join(" ")} ${track.title} audio`;
2393
+ const queries = new Set([
2394
+ primaryQuery,
2395
+ `${track.artists[0]} ${quotedTitle}`,
2396
+ track.title,
2397
+ `${track.artists[0]} ${track.title.replace(/\s+/g, "")}`
2398
+ ]);
2399
+ const attempted = new Set;
2400
+ const live = new Map;
2401
+ let resolutionError;
2402
+ let searchError;
2403
+ for (const query of queries) {
2404
+ let candidates;
2405
+ try {
2406
+ candidates = await searchYoutube(input, query);
2407
+ } catch (error) {
2408
+ if (query === primaryQuery)
2409
+ throw error;
2410
+ searchError = error;
2382
2411
  continue;
2383
2412
  }
2384
- try {
2385
- const response = await net(`https://api.fxtwitter.com/status/${id}`, {
2386
- headers: { accept: "application/json", "user-agent": browserUserAgent() }
2387
- }, 1);
2388
- if (!response.ok) {
2413
+ const originals = candidates.map((candidate, index) => ({
2414
+ candidate,
2415
+ score: matchScore(track, candidate) - index / 1000
2416
+ })).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
2417
+ for (const candidate of candidates) {
2418
+ if (Number.isFinite(matchScore(track, candidate)))
2389
2419
  continue;
2420
+ const score = matchScore(track, candidate, true);
2421
+ if (Number.isFinite(score)) {
2422
+ const url = `https://www.youtube.com/watch?v=${candidate.videoId}`;
2423
+ live.set(url, Math.max(live.get(url) ?? -Infinity, score));
2390
2424
  }
2391
- const payload = await response.json();
2392
- if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
2425
+ }
2426
+ for (const { candidate } of originals) {
2427
+ const youtubeUrl = `https://www.youtube.com/watch?v=${candidate.videoId}`;
2428
+ if (attempted.has(youtubeUrl) || attempted.size >= 3)
2393
2429
  continue;
2430
+ attempted.add(youtubeUrl);
2431
+ try {
2432
+ const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
2433
+ return { youtube, youtubeUrl, liveFallback: false };
2434
+ } catch (error) {
2435
+ resolutionError = error;
2394
2436
  }
2395
- let current = payload.tweet;
2396
- const seen = new Set;
2397
- while (current) {
2398
- const currentId = string(current.id);
2399
- if (!currentId || seen.has(currentId)) {
2400
- break;
2401
- }
2402
- seen.add(currentId);
2403
- const target = posts.get(currentId);
2404
- const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
2405
- const text = raw ?? string(current.text);
2406
- if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
2407
- target.full_text = text;
2408
- }
2409
- current = object(current.quote) ? current.quote : null;
2410
- }
2411
- } catch {}
2437
+ }
2412
2438
  }
2439
+ const performances = [...live].sort((left, right) => right[1] - left[1]).slice(0, 2);
2440
+ for (const [youtubeUrl] of performances) {
2441
+ try {
2442
+ const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
2443
+ return { youtube, youtubeUrl, liveFallback: true };
2444
+ } catch (error) {
2445
+ resolutionError = error;
2446
+ }
2447
+ }
2448
+ if (resolutionError)
2449
+ throw resolutionError;
2450
+ if (searchError)
2451
+ throw searchError;
2452
+ throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "noMatch");
2413
2453
  }
2414
- function twitterTweets(tweet, rootId, includeParents = false) {
2415
- const tweets = [];
2416
- const seen = new Set;
2417
- const visit = (current, fallbackId) => {
2418
- const id = string(current.id_str) ?? fallbackId;
2419
- if (!id || seen.has(id))
2420
- return;
2421
- seen.add(id);
2422
- tweets.push({ id, tweet: current });
2423
- if (object(current.quoted_tweet))
2424
- visit(current.quoted_tweet);
2425
- if (includeParents && object(current.parent))
2426
- visit(current.parent);
2427
- };
2428
- visit(tweet, rootId);
2429
- return tweets;
2430
- }
2431
- async function syndication(net, id) {
2432
- const url = `https://cdn.syndication.twimg.com/tweet-result?id=${id}&lang=en&token=${syndicationToken(id)}`;
2433
- const response = await net(url, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
2454
+ async function searchYoutube(input, query) {
2455
+ const url = new URL("https://www.youtube.com/results");
2456
+ url.searchParams.set("search_query", query);
2457
+ url.searchParams.set("hl", "en");
2458
+ const response = await input.net(url.href, {
2459
+ headers: { "accept-language": "en-US,en;q=0.9", cookie: "SOCS=CAI", "user-agent": browserUserAgent() }
2460
+ });
2434
2461
  if (!response.ok) {
2435
- throw new Error(`Twitter syndication failed: ${response.status}`);
2436
- }
2437
- const payload = await response.json();
2438
- if (!object(payload)) {
2439
- throw new Error("Twitter response invalid");
2462
+ throw new PostfetchError(502, `YouTube search failed: ${response.status}`);
2440
2463
  }
2441
- return payload;
2464
+ const html = await response.text();
2465
+ const raw = html.match(/(?:\bytInitialData|window\[["']ytInitialData["']\])\s*=\s*(\{[\s\S]*?)<\/script>/)?.[1];
2466
+ const jsonScript = html.match(/<script\b[^>]*\bid=["']yt-initial-data["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
2467
+ const payload = parseJson2(jsonScript) ?? parseAssignment(raw);
2468
+ if (!payload || !object(payload.contents)) {
2469
+ throw new PostfetchError(502, "YouTube search results not found");
2470
+ }
2471
+ return videoRenderers(payload.contents);
2472
+ }
2473
+ function videoRenderers(value) {
2474
+ if (Array.isArray(value))
2475
+ return value.flatMap(videoRenderers);
2476
+ if (!object(value))
2477
+ return [];
2478
+ if (object(value.videoRenderer))
2479
+ return [value.videoRenderer];
2480
+ return Object.values(value).flatMap(videoRenderers);
2442
2481
  }
2443
- function syndicationToken(id) {
2444
- return (Number(id) / 1000000000000000 * Math.PI).toString(36).replace(/(0+|\.)/g, "");
2482
+ function text(value) {
2483
+ if (!object(value))
2484
+ return "";
2485
+ return string(value.simpleText) ?? (Array.isArray(value.runs) ? value.runs.filter(object).map((run) => typeof run.text === "string" ? run.text : "").join("") : "");
2445
2486
  }
2446
- function twitterCandidates(entry, id, index) {
2447
- const headers = { "user-agent": browserUserAgent() };
2448
- const type = string(entry.type);
2449
- if (type === "video" || type === "animated_gif") {
2450
- return videoVariants(entry).map(({ bitrate, url }) => ({
2451
- bitrate,
2452
- item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }
2453
- }));
2454
- }
2455
- const photo = string(entry.media_url_https);
2456
- return photo ? [{ bitrate: 0, item: { filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` } }] : [];
2487
+ function normalize(value) {
2488
+ return value.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
2457
2489
  }
2458
- function twitterCardImage(tweet, id) {
2459
- const card = object(tweet.card) ? tweet.card : null;
2460
- if (!card || card.name !== "summary" && card.name !== "summary_large_image")
2490
+ function contains(haystack, needle) {
2491
+ return needle.length > 0 && ` ${haystack} `.includes(` ${needle} `);
2492
+ }
2493
+ function artistMatches(value, artist) {
2494
+ if (contains(value, artist))
2495
+ return true;
2496
+ return /^[bcdfghjklmnpqrstvwxyz]{6,}$/.test(artist) && value.split(" ").some((word) => /^[a-z]+$/.test(word) && word.replace(/[aeiou]/g, "") === artist);
2497
+ }
2498
+ function durationSeconds(value) {
2499
+ if (!/^\d+(?::\d{2}){1,2}$/.test(value))
2461
2500
  return null;
2462
- const bindings = object(card.binding_values) ? card.binding_values : {};
2463
- const images = Object.entries(bindings).flatMap(([key, value]) => {
2464
- if (!/^(photo_image_full_size|summary_photo_image|thumbnail_image)(_(original|x_large|large|small))?$/.test(key) || !object(value) || !object(value.image_value))
2465
- return [];
2466
- const image2 = value.image_value;
2467
- const rawUrl = string(image2.url);
2468
- if (!rawUrl)
2469
- return [];
2470
- try {
2471
- const url = new URL(rawUrl);
2472
- if (url.protocol !== "https:" && url.protocol !== "http:")
2473
- return [];
2474
- const format = (url.searchParams.get("format") ?? url.pathname.match(/\.(jpe?g|png|webp|gif)$/i)?.[1] ?? "jpg").toLowerCase();
2475
- const extension = format === "jpeg" ? "jpg" : format;
2476
- if (!["jpg", "png", "webp", "gif"].includes(extension))
2477
- return [];
2478
- return [{
2479
- url: url.href,
2480
- extension,
2481
- original: key.endsWith("_original") ? 1 : 0,
2482
- area: Math.max(0, number(image2.width) ?? 0) * Math.max(0, number(image2.height) ?? 0)
2483
- }];
2484
- } catch {
2485
- return [];
2486
- }
2487
- }).sort((a, b) => b.original - a.original || b.area - a.area);
2488
- const image = images[0];
2489
- return image ? {
2490
- filename: filename(`twitter_${id}_preview.${image.extension}`),
2491
- headers: { "user-agent": browserUserAgent() },
2492
- id,
2493
- kind: "image",
2494
- mime: `image/${image.extension === "jpg" ? "jpeg" : image.extension}`,
2495
- platform: "twitter",
2496
- role: "linkPreview",
2497
- url: image.url
2498
- } : null;
2501
+ return value.split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0);
2499
2502
  }
2500
- function videoVariants(entry) {
2501
- const info = object(entry.video_info) ? entry.video_info : null;
2502
- const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
2503
- const seen = new Set;
2504
- return variants.flatMap((variant) => {
2505
- const url = string(variant.url);
2506
- if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
2507
- return [];
2503
+ function topicArtistMatch(author, artist) {
2504
+ if (!author.endsWith(" topic"))
2505
+ return false;
2506
+ const actual = author.slice(0, -6).trim().split(" ");
2507
+ const expected = artist.split(" ");
2508
+ if (actual.length !== expected.length)
2509
+ return false;
2510
+ let typos = 0;
2511
+ return expected.every((word, index) => {
2512
+ const candidate = actual[index];
2513
+ if (candidate === word)
2514
+ return true;
2515
+ if (word.length < 6 || candidate.length !== word.length || ++typos > 1)
2516
+ return false;
2517
+ for (let offset = 0;offset < word.length - 1; offset += 1) {
2518
+ const swapped = word.slice(0, offset) + word[offset + 1] + word[offset] + word.slice(offset + 2);
2519
+ if (candidate === swapped)
2520
+ return true;
2508
2521
  }
2509
- seen.add(url);
2510
- return [{ bitrate: number(variant.bitrate) ?? 0, url }];
2511
- }).sort((left, right) => right.bitrate - left.bitrate);
2522
+ return false;
2523
+ });
2512
2524
  }
2513
- async function sizeLimitedItems(input, groups) {
2514
- if (groups.length === 0 || input.tryMaxBytes === undefined) {
2515
- return groups.map((group) => group[0].item);
2525
+ function matchScore(track, candidate, allowLive = false) {
2526
+ if (typeof candidate.videoId !== "string" || !/^[A-Za-z0-9_-]{11}$/.test(candidate.videoId))
2527
+ return -Infinity;
2528
+ const title = normalize(text(candidate.title));
2529
+ const author = normalize(text(candidate.ownerText) || text(candidate.longBylineText) || text(candidate.shortBylineText));
2530
+ const trackTitle = track.title.replace(/\s*[([](?:feat\.?|ft\.?).*?[)\]]/gi, "");
2531
+ const expected = normalize(trackTitle);
2532
+ const artists = track.artists.map(normalize);
2533
+ const artistUpload = artistMatches(author, artists[0]) || topicArtistMatch(author, artists[0]);
2534
+ const credited = artists.map((artist, index) => artistMatches(`${title} ${author}`, artist) || index === 0 && artistUpload);
2535
+ if (!credited[0])
2536
+ return -Infinity;
2537
+ const tokens = expected.split(" ").filter(Boolean);
2538
+ if (tokens.length === 0)
2539
+ return -Infinity;
2540
+ let coverage = tokens.filter((token) => contains(title, token)).length / tokens.length;
2541
+ const joinedTitle = expected.replace(/ /g, "");
2542
+ const joinedMatch = tokens.length > 1 && joinedTitle.length >= 6 && contains(title, joinedTitle);
2543
+ if (joinedMatch)
2544
+ coverage = 1;
2545
+ if (coverage < (tokens.length <= 3 ? 1 : 0.8) || tokens.length <= 3 && !contains(title, expected) && !joinedMatch) {
2546
+ const parts = trackTitle.split(/\s+[-–—|]\s+/);
2547
+ if (parts.length < 3 || !parts[0].includes(":") || artists.length < 2 || !credited.every(Boolean) || !contains(title, normalize(parts.slice(1).join(" ")))) {
2548
+ return -Infinity;
2549
+ }
2550
+ coverage = 1;
2551
+ }
2552
+ const duration = durationSeconds(text(candidate.lengthText));
2553
+ if (duration === null || duration <= 0)
2554
+ return -Infinity;
2555
+ const tolerance = Math.min(30, Math.max(12, (track.durationSeconds ?? duration) * 0.08));
2556
+ const delta = track.durationSeconds === undefined ? 0 : Math.abs(duration - track.durationSeconds);
2557
+ if (delta > tolerance)
2558
+ return -Infinity;
2559
+ let recordingTitle = title;
2560
+ for (const artist of artists)
2561
+ recordingTitle = recordingTitle.replace(artist, "");
2562
+ const isLive = /\blive\b/.test(recordingTitle);
2563
+ const requestedLive = /\blive\b/.test(expected);
2564
+ if (isLive !== requestedLive && !(allowLive && isLive && !requestedLive))
2565
+ return -Infinity;
2566
+ for (const version of [/\bremix\b/, /\bcover\b/, /\bkaraoke\b/, /\binstrumental\b/, /\bslowed\b/, /\b(?:sped|speed) up\b/, /\bnightcore\b/, /\bacoustic\b/]) {
2567
+ if (version.test(recordingTitle) !== version.test(expected))
2568
+ return -Infinity;
2569
+ }
2570
+ const creditCoverage = credited.filter(Boolean).length / artists.length;
2571
+ return coverage * 10 + (artistUpload ? 8 : 0) + creditCoverage * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
2572
+ }
2573
+ function parseJson2(value) {
2574
+ try {
2575
+ const parsed = JSON.parse(value ?? "");
2576
+ return object(parsed) ? parsed : null;
2577
+ } catch {
2578
+ return null;
2516
2579
  }
2517
- const sizes = new Map;
2518
- const size = (candidate) => {
2519
- const existing = sizes.get(candidate.item.url);
2520
- if (existing) {
2521
- return existing;
2522
- }
2523
- const pending = contentLength(input.net, candidate.item);
2524
- sizes.set(candidate.item.url, pending);
2525
- return pending;
2526
- };
2527
- const highest = groups.map((group) => group[0]);
2528
- const highestSizes = await Promise.all(highest.map(size));
2529
- const highestTotal = knownTotal(highestSizes);
2530
- if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
2531
- return highest.map(({ item }) => item);
2580
+ }
2581
+ function parseAssignment(value) {
2582
+ if (!value)
2583
+ return null;
2584
+ let depth = 0;
2585
+ let quoted = false;
2586
+ let escaped = false;
2587
+ for (let index = 0;index < value.length; index += 1) {
2588
+ const character = value[index];
2589
+ if (quoted) {
2590
+ if (escaped)
2591
+ escaped = false;
2592
+ else if (character === "\\")
2593
+ escaped = true;
2594
+ else if (character === '"')
2595
+ quoted = false;
2596
+ } else if (character === '"')
2597
+ quoted = true;
2598
+ else if (character === "{")
2599
+ depth += 1;
2600
+ else if (character === "}" && --depth === 0)
2601
+ return parseJson2(value.slice(0, index + 1));
2532
2602
  }
2533
- const sized = await Promise.all(groups.map(async (group) => Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) })))));
2534
- const fitting = bestFitting(sized, input.tryMaxBytes);
2535
- return (fitting ?? groups.map((group) => group.at(-1))).map(({ item }) => item);
2603
+ return null;
2536
2604
  }
2537
- function knownTotal(sizes) {
2538
- return sizes.some((size) => size === null) ? null : sizes.reduce((total, size) => total + (size ?? 0), 0);
2605
+
2606
+ // src/tiktok.ts
2607
+ var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
2608
+ var pageAttemptCount = 12;
2609
+ async function resolveTiktok(input) {
2610
+ const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
2611
+ const id = videoId(pageUrl);
2612
+ if (!id) {
2613
+ throw new Error("TikTok video id not found");
2614
+ }
2615
+ const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
2616
+ const user = author(page.item) ?? username(pageUrl) ?? "i";
2617
+ const headers = {
2618
+ referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
2619
+ "user-agent": page.userAgent
2620
+ };
2621
+ if (page.cookie) {
2622
+ headers.cookie = page.cookie;
2623
+ }
2624
+ const items = mediaItems4(page.item, user, id, headers);
2625
+ if (items.length === 0) {
2626
+ throw new Error("TikTok media not found");
2627
+ }
2628
+ return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
2539
2629
  }
2540
- function bestFitting(groups, maxBytes) {
2541
- let selections = [
2542
- { bytes: 0, candidates: [], quality: 0 }
2543
- ];
2544
- for (const group of groups) {
2545
- selections = selections.flatMap((selection) => group.flatMap((candidate) => {
2546
- if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
2547
- return [];
2630
+ async function videoPage(net, id, photo) {
2631
+ for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
2632
+ const baseHeaders2 = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
2633
+ const userAgent2 = baseHeaders2["user-agent"];
2634
+ const headers2 = {
2635
+ ...baseHeaders2,
2636
+ referer: "https://www.tiktok.com/",
2637
+ "sec-fetch-site": "same-origin"
2638
+ };
2639
+ try {
2640
+ return { ...await fetchVideoPage(net, id, headers2), userAgent: userAgent2 };
2641
+ } catch (error) {
2642
+ if (!recoverablePageError(error)) {
2643
+ throw error;
2548
2644
  }
2549
- return [{
2550
- bytes: selection.bytes + candidate.bytes,
2551
- candidates: [...selection.candidates, candidate],
2552
- quality: selection.quality + candidate.bitrate
2553
- }];
2554
- }));
2555
- if (selections.length === 0) {
2556
- return null;
2557
2645
  }
2558
2646
  }
2559
- const best = selections.reduce((current, selection) => selection.quality > current.quality ? selection : current);
2560
- return best.candidates;
2647
+ if (!photo) {
2648
+ throw new Error("TikTok hydration not found");
2649
+ }
2650
+ const baseHeaders = navigationHeaders();
2651
+ const userAgent = baseHeaders["user-agent"];
2652
+ const headers = {
2653
+ ...baseHeaders,
2654
+ referer: "https://www.tiktok.com/",
2655
+ "sec-fetch-site": "same-origin"
2656
+ };
2657
+ return { ...await fetchEmbedPage(net, id, headers), userAgent };
2561
2658
  }
2562
- async function contentLength(net, item) {
2563
- try {
2564
- const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
2565
- const raw = response.ok ? response.headers.get("content-length") : null;
2566
- if (!raw || !/^\d+$/.test(raw)) {
2567
- return null;
2659
+ function tiktokMetadata(item) {
2660
+ const user = object(item.author) ? item.author : null;
2661
+ const stats = object(item.stats) ? item.stats : null;
2662
+ const music = object(item.music) ? item.music : null;
2663
+ const video = object(item.video) ? item.video : null;
2664
+ return {
2665
+ text: string(item.desc) ?? undefined,
2666
+ author: user ? {
2667
+ handle: string(user.uniqueId) ?? undefined,
2668
+ name: string(user.nickname) ?? undefined,
2669
+ verified: bool(user.verified)
2670
+ } : undefined,
2671
+ createdAt: isoFromEpochSeconds(item.createTime),
2672
+ likeCount: stats ? count(stats.diggCount) : undefined,
2673
+ commentCount: stats ? count(stats.commentCount) : undefined,
2674
+ shareCount: stats ? count(stats.shareCount) : undefined,
2675
+ viewCount: stats ? count(stats.playCount) : undefined,
2676
+ extra: {
2677
+ saveCount: stats ? count(stats.collectCount) : undefined,
2678
+ musicTitle: music ? string(music.title) ?? undefined : undefined,
2679
+ musicAuthor: music ? string(music.authorName) ?? undefined : undefined,
2680
+ region: string(item.locationCreated) ?? undefined,
2681
+ durationSeconds: video ? count(video.duration) : undefined
2568
2682
  }
2569
- const bytes = Number(raw);
2570
- return Number.isSafeInteger(bytes) ? bytes : null;
2571
- } catch {
2572
- return null;
2573
- }
2683
+ };
2574
2684
  }
2575
- function tweetId(input) {
2576
- return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
2685
+ async function fetchVideoPage(net, id, headers) {
2686
+ const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
2687
+ headers
2688
+ });
2689
+ return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
2577
2690
  }
2578
- async function twitterComments(input, id) {
2579
- const limit = input.comments;
2580
- if (!limit || !Number.isSafeInteger(limit) || limit < 0)
2581
- return [];
2582
- try {
2583
- const comments = [];
2584
- const seen = new Set;
2585
- const cursors = new Set;
2586
- let cursor = null;
2587
- do {
2588
- const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
2589
- if (cursor)
2590
- url.searchParams.set("cursor", cursor);
2591
- const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
2592
- if (!response.ok)
2593
- throw new Error("Comment lookup failed");
2594
- const payload = await response.json();
2595
- if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
2596
- throw new Error("Invalid comments");
2597
- const previousCount = comments.length;
2598
- for (const reply of payload.replies) {
2599
- if (!object(reply))
2600
- continue;
2601
- const replyId = string(reply.id);
2602
- const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
2603
- if (!replyId || parent !== id || seen.has(replyId))
2604
- continue;
2605
- seen.add(replyId);
2606
- comments.push(twitterComment(reply, replyId));
2607
- if (comments.length >= limit)
2608
- return comments;
2609
- }
2610
- cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
2611
- if (!cursor || cursors.has(cursor) || comments.length === previousCount)
2612
- break;
2613
- cursors.add(cursor);
2614
- } while (comments.length < limit);
2615
- return comments;
2616
- } catch {
2617
- return [];
2691
+ async function fetchEmbedPage(net, id, headers) {
2692
+ const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
2693
+ headers
2694
+ });
2695
+ return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
2696
+ }
2697
+ function isShortlinkHost(hostname) {
2698
+ return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
2699
+ }
2700
+ async function followShortlink(net, userAgent, input) {
2701
+ const url = asUrl(input);
2702
+ if (!isShortlinkHost(url.hostname)) {
2703
+ return input;
2618
2704
  }
2705
+ const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
2706
+ const location = response.headers.get("location");
2707
+ if (location) {
2708
+ return new URL(location, input).href;
2709
+ }
2710
+ const html = await response.text();
2711
+ const href = html.match(/<a href="(https:\/\/[^"]+)"/)?.[1];
2712
+ return href ? new URL(href.split("?")[0]).href : input;
2619
2713
  }
2620
- function twitterComment(reply, id) {
2621
- const author2 = object(reply.author) ? reply.author : {};
2622
- const verification = object(author2.verification) ? author2.verification : {};
2623
- const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
2624
- const items = media.flatMap((entry, index) => {
2625
- const url = string(entry.url);
2626
- const photo = entry.type === "photo";
2627
- if (!url || !photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")
2628
- return [];
2629
- return [{
2630
- id,
2631
- platform: "twitter",
2632
- filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
2633
- kind: photo ? "image" : "video",
2634
- mime: photo ? "image/jpeg" : "video/mp4",
2635
- url,
2636
- headers: { "user-agent": browserUserAgent() }
2637
- }];
2638
- });
2639
- return {
2640
- id,
2641
- url: `https://x.com/i/status/${id}`,
2642
- items,
2643
- metadata: {
2644
- text: cleanTwitterText(string(reply.text), reply) ?? undefined,
2645
- author: {
2646
- handle: string(author2.screen_name) ?? undefined,
2647
- name: string(author2.name) ?? undefined,
2648
- verified: bool(verification.verified)
2649
- },
2650
- createdAt: isoFromDateString(reply.created_at),
2651
- likeCount: count(reply.likes),
2652
- commentCount: count(reply.replies),
2653
- shareCount: count(reply.reposts),
2654
- viewCount: count(reply.views),
2655
- nsfw: bool(reply.possibly_sensitive)
2656
- }
2657
- };
2714
+ function videoId(input) {
2715
+ return asUrl(input).pathname.match(/(?:video|photo)\/(\d+)/)?.[1] ?? null;
2658
2716
  }
2659
-
2660
- // src/youtube.ts
2661
- var androidVrClient = {
2662
- name: "ANDROID_VR",
2663
- number: "28",
2664
- userAgent: "com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip",
2665
- version: "1.65.10"
2666
- };
2667
- var visionOsClient = {
2668
- clientName: "VISIONOS",
2669
- clientVersion: "1.02",
2670
- deviceMake: "Apple",
2671
- deviceModel: "RealityDevice17,1",
2672
- userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
2673
- osName: "visionOS",
2674
- osVersion: "26.5.23O471"
2675
- };
2676
- var browserCookie = "PREF=hl=en&tz=UTC; SOCS=CAI";
2677
- async function resolveYoutube(input) {
2678
- const id = youtubeVideoId(input.url);
2679
- if (!id) {
2680
- throw new Error("YouTube video id not found");
2717
+ function itemStruct(html) {
2718
+ if (html.includes("SlardarWAF") || html.includes("_wafchallengeid")) {
2719
+ throw new Error("TikTok WAF challenge");
2681
2720
  }
2682
- let session;
2683
- try {
2684
- session = await youtubeSession(input.net, id);
2685
- const payload = await youtubePlayer(input.net, playerBody(id, session), playerHeaders(session));
2686
- const streams = selectStreams(payload, input.preferredWidth);
2687
- if (!streams) {
2688
- throw new Error("YouTube mp4 stream not found");
2689
- }
2690
- const headers = { "user-agent": androidVrClient.userAgent };
2691
- await Promise.all([streams.video, streams.audio].filter((url) => url !== null).map((url) => probeStream(input.net, url, headers)));
2692
- return youtubeResult(id, payload, streams, headers);
2693
- } catch (primaryError) {
2694
- try {
2695
- return await resolveVisionOs(input, id, session?.visitorData);
2696
- } catch (fallbackError) {
2697
- const message = (error) => error instanceof Error ? error.message : String(error);
2698
- throw new Error(`YouTube resolution failed: ${message(primaryError)}; visionOS HLS: ${message(fallbackError)}`, { cause: fallbackError });
2699
- }
2721
+ const start = html.indexOf(marker);
2722
+ const end = start === -1 ? -1 : html.indexOf("</script>", start + marker.length);
2723
+ if (start === -1 || end === -1) {
2724
+ throw new Error("TikTok hydration not found");
2725
+ }
2726
+ const parsed = JSON.parse(html.slice(start + marker.length, end));
2727
+ const scope = object(parsed) && object(parsed.__DEFAULT_SCOPE__) ? parsed.__DEFAULT_SCOPE__ : null;
2728
+ const detail = scope && object(scope["webapp.video-detail"]) ? scope["webapp.video-detail"] : null;
2729
+ const info = detail && object(detail.itemInfo) ? detail.itemInfo : null;
2730
+ const item = info && object(info.itemStruct) ? info.itemStruct : null;
2731
+ if (!item) {
2732
+ throw new Error("TikTok itemStruct not found");
2700
2733
  }
2734
+ return item;
2701
2735
  }
2702
- async function youtubePlayer(net, body, headers) {
2703
- const response = await net("https://www.youtube.com/youtubei/v1/player?prettyPrint=false", {
2704
- body: JSON.stringify(body),
2705
- headers,
2706
- method: "POST"
2707
- });
2708
- if (!response.ok) {
2709
- throw new Error(`YouTube player failed: ${response.status}`);
2736
+ function embedItemStruct(html, id) {
2737
+ const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
2738
+ if (!raw) {
2739
+ throw new Error("TikTok embed state not found");
2710
2740
  }
2711
- const payload = await response.json();
2712
- const status = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.status) : null;
2713
- if (status !== "OK") {
2714
- const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
2715
- throw new Error(reason ?? "YouTube video unavailable");
2741
+ const parsed = JSON.parse(raw);
2742
+ const source2 = object(parsed) && object(parsed.source) ? parsed.source : null;
2743
+ const data = source2 && object(source2.data) ? source2.data : null;
2744
+ const rawRoute = data ? data[`/embed/v2/${id}`] : null;
2745
+ const route = object(rawRoute) ? rawRoute : null;
2746
+ const videoData = route && object(route.videoData) ? route.videoData : null;
2747
+ const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
2748
+ if (!videoData || !item) {
2749
+ throw new Error("TikTok embed item not found");
2716
2750
  }
2717
- return payload;
2751
+ const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
2752
+ const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
2753
+ const video = object(item.video) ? item.video : {};
2754
+ const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
2755
+ const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
2756
+ const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
2757
+ if (displayImages.length === 0) {
2758
+ throw new Error("TikTok video hydration not found");
2759
+ }
2760
+ return {
2761
+ ...item,
2762
+ author: {
2763
+ nickname: string(authorInfo.nickName),
2764
+ uniqueId: string(authorInfo.uniqueId),
2765
+ verified: bool(authorInfo.verified)
2766
+ },
2767
+ desc: string(item.text),
2768
+ imagePost: {
2769
+ images: displayImages.map((image) => ({
2770
+ imageURL: {
2771
+ urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
2772
+ }
2773
+ }))
2774
+ },
2775
+ music: {
2776
+ authorName: string(musicInfo.authorName),
2777
+ playUrl: firstString(musicInfo.playUrl),
2778
+ title: string(musicInfo.musicName)
2779
+ },
2780
+ stats: {
2781
+ commentCount: item.commentCount,
2782
+ diggCount: item.diggCount,
2783
+ playCount: item.playCount,
2784
+ shareCount: item.shareCount
2785
+ },
2786
+ video: {
2787
+ duration: videoMeta.duration
2788
+ }
2789
+ };
2718
2790
  }
2719
- function youtubeResult(id, payload, streams, headers) {
2720
- const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
2721
- const media = {
2722
- filename: filename(`youtube_${title ?? id}_${id}.mp4`),
2791
+ function recoverablePageError(error) {
2792
+ const message = error instanceof Error ? error.message : "";
2793
+ return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
2794
+ }
2795
+ function firstString(value) {
2796
+ return Array.isArray(value) ? value.map(string).find((candidate) => candidate !== null) ?? null : string(value);
2797
+ }
2798
+ function downloadUrl(item) {
2799
+ const video = object(item.video) ? item.video : null;
2800
+ return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
2801
+ }
2802
+ function mediaItems4(item, user, id, headers) {
2803
+ const images = imageItems3(item, user, id, headers);
2804
+ if (images.length > 0) {
2805
+ const audio = audioItem2(item, user, id, headers);
2806
+ return audio ? [...images, audio] : images;
2807
+ }
2808
+ const url = downloadUrl(item);
2809
+ return url ? [{
2810
+ filename: filename(`tiktok_${user}_${id}.mp4`),
2723
2811
  headers,
2724
2812
  id,
2725
2813
  kind: "video",
2726
2814
  mime: "video/mp4",
2727
- platform: "youtube",
2728
- url: streams.video,
2729
- ...streams.hls ? { hls: true } : {},
2730
- ...streams.audio ? { audio: { headers, url: streams.audio } } : {}
2731
- };
2732
- return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
2815
+ platform: "tiktok",
2816
+ url
2817
+ }] : [];
2733
2818
  }
2734
- async function probeStream(net, url, headers) {
2735
- const response = await net(url, { headers }, 1);
2736
- await response.body?.cancel();
2737
- if (!response.ok) {
2738
- throw new Error(`YouTube stream failed: ${response.status}`);
2819
+ function imageItems3(item, user, id, headers) {
2820
+ const imagePost = object(item.imagePost) ? item.imagePost : null;
2821
+ const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
2822
+ return images.flatMap((image, index) => {
2823
+ const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
2824
+ const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
2825
+ const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
2826
+ return url ? [{
2827
+ filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
2828
+ headers,
2829
+ id,
2830
+ kind: "image",
2831
+ mime: "image/jpeg",
2832
+ platform: "tiktok",
2833
+ url
2834
+ }] : [];
2835
+ });
2836
+ }
2837
+ function audioItem2(item, user, id, headers) {
2838
+ const video = object(item.video) ? item.video : null;
2839
+ const music = object(item.music) ? item.music : null;
2840
+ const url = video ? string(video.playAddr) : null;
2841
+ const fallback = music ? string(music.playUrl) : null;
2842
+ const selected = url ?? fallback;
2843
+ if (!selected) {
2844
+ return null;
2739
2845
  }
2846
+ const extension = selected.includes("mime_type=audio_mpeg") ? "mp3" : "m4a";
2847
+ return {
2848
+ filename: filename(`tiktok_${user}_${id}_audio.${extension}`),
2849
+ headers,
2850
+ id,
2851
+ kind: "audio",
2852
+ mime: extension === "mp3" ? "audio/mpeg" : "audio/mp4",
2853
+ platform: "tiktok",
2854
+ url: selected
2855
+ };
2740
2856
  }
2741
- async function resolveVisionOs(input, id, visitorData) {
2742
- const headers = { "user-agent": visionOsClient.userAgent };
2743
- const payload = await youtubePlayer(input.net, {
2744
- videoId: id,
2745
- contentCheckOk: true,
2746
- racyCheckOk: true,
2747
- context: { client: { ...visionOsClient, hl: "en", gl: "US", ...visitorData ? { visitorData } : {} } }
2748
- }, {
2749
- ...headers,
2750
- "content-type": "application/json",
2751
- "x-youtube-client-name": "101",
2752
- "x-youtube-client-version": visionOsClient.clientVersion,
2753
- ...visitorData ? { "x-goog-visitor-id": visitorData } : {}
2754
- });
2755
- const streaming = object(payload) && object(payload.streamingData) ? payload.streamingData : null;
2756
- const manifestUrl = streaming ? string(streaming.hlsManifestUrl) : null;
2757
- if (!manifestUrl) {
2758
- throw new Error("YouTube HLS manifest not found");
2857
+ function author(item) {
2858
+ const user = object(item.author) ? item.author : null;
2859
+ return user ? string(user.uniqueId) : null;
2860
+ }
2861
+ function username(input) {
2862
+ return asUrl(input).pathname.match(/@([^/]+)/)?.[1] ?? null;
2863
+ }
2864
+ function cookieHeader(headers) {
2865
+ const getter = headers.getSetCookie;
2866
+ const setCookie = typeof getter === "function" ? getter.call(headers).join(",") : headers.get("set-cookie");
2867
+ const cookies = setCookie?.split(/,(?=[^;]+?=)/).map((part) => part.split(";")[0]?.trim()).filter((part) => Boolean(part));
2868
+ return cookies && cookies.length > 0 ? cookies.join("; ") : null;
2869
+ }
2870
+
2871
+ // src/twitter.ts
2872
+ async function resolveTwitter(input) {
2873
+ const id = tweetId(input.url);
2874
+ if (!id) {
2875
+ throw new Error("Tweet id not found");
2759
2876
  }
2760
- const response = await input.net(manifestUrl, { headers });
2761
- if (!response.ok) {
2762
- throw new Error(`YouTube HLS manifest failed: ${response.status}`);
2877
+ const tweet = await syndication(input.net, id);
2878
+ if (string(tweet.__typename) === "TweetTombstone") {
2879
+ throw new Error("Tweet is unavailable or age-restricted");
2763
2880
  }
2764
- const master = parseMaster(await response.text(), manifestUrl, { preferOriginalAudio: true });
2765
- const variants = master.variants.filter((variant2) => /(?:^|,)(?:vp09|av01)\./i.test(variant2.codecs ?? "") && /(?:^|,)mp4a\.40\.2(?:,|$)/i.test(variant2.codecs ?? "") && variant2.audioGroup && master.audio[variant2.audioGroup]).sort((left, right) => Math.abs(left.width - input.preferredWidth) - Math.abs(right.width - input.preferredWidth) || right.bandwidth - left.bandwidth);
2766
- const variant = variants[0];
2767
- if (!variant?.audioGroup) {
2768
- throw new Error("YouTube fragmented MP4 HLS video with AAC audio not found");
2881
+ await expandTwitterText(input.net, tweet, id);
2882
+ const posts = twitterTweets(tweet, id);
2883
+ const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
2884
+ const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
2885
+ return media.flatMap((entry, index) => {
2886
+ const candidates = twitterCandidates(entry, postId2, index + 1);
2887
+ return candidates.length > 0 ? [candidates] : [];
2888
+ });
2889
+ });
2890
+ for (const { id: postId2, tweet: post } of posts.toReversed()) {
2891
+ const card = twitterCardImage(post, postId2);
2892
+ if (card)
2893
+ groups.push([{ bitrate: 0, item: card }]);
2769
2894
  }
2770
- const playlist = await input.net(variant.url, { headers });
2771
- if (!playlist.ok) {
2772
- throw new Error(`YouTube HLS playlist failed: ${playlist.status}`);
2895
+ const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
2896
+ return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
2897
+ }
2898
+ function twitterMetadata(tweet) {
2899
+ const user = object(tweet.user) ? tweet.user : null;
2900
+ const extra = {};
2901
+ const lang = string(tweet.lang);
2902
+ if (lang) {
2903
+ extra.lang = lang;
2773
2904
  }
2774
- if (!(await playlist.text()).includes("#EXT-X-MAP:")) {
2775
- throw new Error("YouTube HLS video is not fragmented MP4");
2905
+ const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
2906
+ const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
2907
+ if (quotedTweet && quotedId) {
2908
+ extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
2909
+ }
2910
+ const parent = object(tweet.parent) ? tweet.parent : null;
2911
+ const parentId = parent ? string(parent.id_str) : null;
2912
+ const replyToId = string(tweet.in_reply_to_status_id_str) ?? parentId;
2913
+ if (replyToId)
2914
+ extra.replyToId = replyToId;
2915
+ if (parent && parentId) {
2916
+ extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
2776
2917
  }
2777
- return youtubeResult(id, payload, { video: variant.url, audio: master.audio[variant.audioGroup], hls: true }, headers);
2778
- }
2779
- function youtubeMetadata(payload) {
2780
- const details = object(payload) && object(payload.videoDetails) ? payload.videoDetails : null;
2781
- const renderer = object(payload) && object(payload.microformat) && object(payload.microformat.playerMicroformatRenderer) ? payload.microformat.playerMicroformatRenderer : null;
2782
- const author2 = details ? string(details.author) : null;
2783
- const keywords = details && Array.isArray(details.keywords) ? details.keywords.filter((keyword) => typeof keyword === "string") : [];
2784
2918
  return {
2785
- title: details ? string(details.title) ?? undefined : undefined,
2786
- text: details ? string(details.shortDescription) ?? undefined : undefined,
2787
- author: author2 ? { name: author2 } : undefined,
2788
- createdAt: renderer ? isoFromDateString(renderer.publishDate) : undefined,
2789
- viewCount: details ? count(details.viewCount) : undefined,
2790
- extra: {
2791
- channelId: details ? string(details.channelId) ?? undefined : undefined,
2792
- durationSeconds: details ? count(details.lengthSeconds) : undefined,
2793
- keywords: keywords.length > 0 ? keywords : undefined
2794
- }
2919
+ text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
2920
+ author: user ? {
2921
+ handle: string(user.screen_name) ?? undefined,
2922
+ name: string(user.name) ?? undefined,
2923
+ verified: bool(user.verified) ?? bool(user.is_blue_verified)
2924
+ } : undefined,
2925
+ createdAt: isoFromDateString(tweet.created_at),
2926
+ likeCount: count(tweet.favorite_count),
2927
+ commentCount: count(tweet.conversation_count),
2928
+ shareCount: count(tweet.retweet_count),
2929
+ viewCount: count(tweet.view_count),
2930
+ nsfw: bool(tweet.possibly_sensitive),
2931
+ extra
2795
2932
  };
2796
2933
  }
2797
- function selectStreams(payload, preferredWidth) {
2798
- const adaptive = adaptiveFormats(payload);
2799
- const video = bestVideo(adaptive, preferredWidth);
2800
- const audio = bestAudio(adaptive);
2801
- if (video && audio) {
2802
- return { video, audio };
2803
- }
2804
- const progressive = selectFormat(payload);
2805
- return progressive ? { video: progressive, audio: null } : null;
2806
- }
2807
- function adaptiveFormats(payload) {
2808
- const root = object(payload) ? payload : null;
2809
- const streaming = root && object(root.streamingData) ? root.streamingData : null;
2810
- return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
2934
+ function noteText(tweet) {
2935
+ const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
2936
+ const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
2937
+ const result = results && object(results.result) ? results.result : null;
2938
+ return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
2811
2939
  }
2812
- function bestVideo(formats, preferredWidth) {
2813
- const candidates = formats.filter((format) => {
2814
- const mimeType2 = string(format.mimeType);
2815
- return Boolean(string(format.url)) && mimeType2?.startsWith("video/mp4") === true && mimeType2.includes("avc1");
2816
- });
2817
- const best = candidates.reduce((current, format) => {
2818
- if (!current) {
2819
- return format;
2820
- }
2821
- const width = number(format.width) ?? 0;
2822
- const currentWidth = number(current.width) ?? 0;
2823
- return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
2824
- }, null);
2825
- return best ? string(best.url) : null;
2940
+ function twitterText(tweet) {
2941
+ return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
2826
2942
  }
2827
- function bestAudio(formats) {
2828
- const candidates = formats.filter((format) => Boolean(string(format.url)) && string(format.mimeType)?.startsWith("audio/mp4") === true);
2829
- const best = candidates.reduce((current, format) => {
2830
- if (!current) {
2831
- return format;
2832
- }
2833
- const preference = audioPreference(format) - audioPreference(current);
2834
- if (preference !== 0) {
2835
- return preference > 0 ? format : current;
2836
- }
2837
- return (number(format.bitrate) ?? 0) > (number(current.bitrate) ?? 0) ? format : current;
2838
- }, null);
2839
- return best ? string(best.url) : null;
2943
+ function cleanTwitterText(text2, tweet) {
2944
+ if (text2 === null)
2945
+ return null;
2946
+ const entities = object(tweet.entities) ? tweet.entities : null;
2947
+ const media = [
2948
+ ...entities && Array.isArray(entities.media) ? entities.media : [],
2949
+ ...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
2950
+ ];
2951
+ const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
2952
+ while (true) {
2953
+ const trailing = text2.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
2954
+ if (!trailing || !urls.has(trailing[1]))
2955
+ break;
2956
+ text2 = text2.slice(0, trailing.index).trimEnd();
2957
+ }
2958
+ return text2.replace(/&(?:amp|gt|lt);/g, (entity) => entity === "&amp;" ? "&" : entity === "&gt;" ? ">" : "<");
2840
2959
  }
2841
- function audioPreference(format) {
2842
- const track = object(format.audioTrack) ? format.audioTrack : null;
2843
- if (track && /\boriginal\b/i.test(string(track.displayName) ?? "")) {
2844
- return 2;
2960
+ function needsFullText(tweet) {
2961
+ if (noteText(tweet) || string(tweet.full_text)) {
2962
+ return false;
2845
2963
  }
2846
- return track?.audioIsDefault === true ? 1 : 0;
2964
+ return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
2847
2965
  }
2848
- function playerBody(id, session) {
2849
- return {
2850
- contentCheckOk: true,
2851
- context: {
2852
- client: {
2853
- androidSdkVersion: 32,
2854
- clientName: androidVrClient.name,
2855
- clientVersion: androidVrClient.version,
2856
- deviceMake: "Oculus",
2857
- deviceModel: "Quest 3",
2858
- gl: "US",
2859
- hl: "en",
2860
- osName: "Android",
2861
- osVersion: "12L",
2862
- timeZone: "UTC",
2863
- userAgent: androidVrClient.userAgent,
2864
- utcOffsetMinutes: 0,
2865
- visitorData: session.visitorData
2966
+ async function expandTwitterText(net, tweet, rootId) {
2967
+ const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
2968
+ for (const [id, post] of posts) {
2969
+ if (!needsFullText(post)) {
2970
+ continue;
2971
+ }
2972
+ try {
2973
+ const response = await net(`https://api.fxtwitter.com/status/${id}`, {
2974
+ headers: { accept: "application/json", "user-agent": browserUserAgent() }
2975
+ }, 1);
2976
+ if (!response.ok) {
2977
+ continue;
2866
2978
  }
2867
- },
2868
- playbackContext: {
2869
- contentPlaybackContext: {
2870
- html5Preference: "HTML5_PREF_WANTS",
2871
- signatureTimestamp: session.signatureTimestamp
2979
+ const payload = await response.json();
2980
+ if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
2981
+ continue;
2872
2982
  }
2873
- },
2874
- racyCheckOk: true,
2875
- videoId: id
2983
+ let current = payload.tweet;
2984
+ const seen = new Set;
2985
+ while (current) {
2986
+ const currentId = string(current.id);
2987
+ if (!currentId || seen.has(currentId)) {
2988
+ break;
2989
+ }
2990
+ seen.add(currentId);
2991
+ const target = posts.get(currentId);
2992
+ const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
2993
+ const text2 = raw ?? string(current.text);
2994
+ if (target && needsFullText(target) && text2 && text2.length > (twitterText(target)?.length ?? 0)) {
2995
+ target.full_text = text2;
2996
+ }
2997
+ current = object(current.quote) ? current.quote : null;
2998
+ }
2999
+ } catch {}
3000
+ }
3001
+ }
3002
+ function twitterTweets(tweet, rootId, includeParents = false) {
3003
+ const tweets = [];
3004
+ const seen = new Set;
3005
+ const visit = (current, fallbackId) => {
3006
+ const id = string(current.id_str) ?? fallbackId;
3007
+ if (!id || seen.has(id))
3008
+ return;
3009
+ seen.add(id);
3010
+ tweets.push({ id, tweet: current });
3011
+ if (object(current.quoted_tweet))
3012
+ visit(current.quoted_tweet);
3013
+ if (includeParents && object(current.parent))
3014
+ visit(current.parent);
2876
3015
  };
3016
+ visit(tweet, rootId);
3017
+ return tweets;
2877
3018
  }
2878
- async function youtubeSession(net, id) {
2879
- const response = await net(`https://www.youtube.com/watch?v=${id}&bpctr=9999999999&has_verified=1`, {
2880
- headers: {
2881
- accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
2882
- "accept-language": "en-us,en;q=0.5",
2883
- cookie: browserCookie,
2884
- "sec-fetch-mode": "navigate",
2885
- "user-agent": browserUserAgent()
2886
- }
2887
- });
3019
+ async function syndication(net, id) {
3020
+ const url = `https://cdn.syndication.twimg.com/tweet-result?id=${id}&lang=en&token=${syndicationToken(id)}`;
3021
+ const response = await net(url, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
2888
3022
  if (!response.ok) {
2889
- throw new Error(`YouTube page failed: ${response.status}`);
3023
+ throw new Error(`Twitter syndication failed: ${response.status}`);
2890
3024
  }
2891
- const page = await response.text();
2892
- const visitorData = string(page.match(/"visitorData":"([^"]+)"/)?.[1]);
2893
- if (!visitorData) {
2894
- throw new Error("YouTube visitor data not found");
3025
+ const payload = await response.json();
3026
+ if (!object(payload)) {
3027
+ throw new Error("Twitter response invalid");
2895
3028
  }
2896
- const playerPath = string(page.match(/"jsUrl":"([^"]+)"/)?.[1]) ?? string(page.match(/"PLAYER_JS_URL":"([^"]+)"/)?.[1]);
2897
- if (!playerPath) {
2898
- throw new Error("YouTube player url not found");
3029
+ return payload;
3030
+ }
3031
+ function syndicationToken(id) {
3032
+ return (Number(id) / 1000000000000000 * Math.PI).toString(36).replace(/(0+|\.)/g, "");
3033
+ }
3034
+ function twitterCandidates(entry, id, index) {
3035
+ const headers = { "user-agent": browserUserAgent() };
3036
+ const type = string(entry.type);
3037
+ if (type === "video" || type === "animated_gif") {
3038
+ return videoVariants(entry).map(({ bitrate, url }) => ({
3039
+ bitrate,
3040
+ item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }
3041
+ }));
2899
3042
  }
2900
- return {
2901
- cookie: youtubeCookie(response.headers),
2902
- signatureTimestamp: await signatureTimestamp(net, playerPath),
2903
- visitorData
2904
- };
3043
+ const photo = string(entry.media_url_https);
3044
+ return photo ? [{ bitrate: 0, item: { filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` } }] : [];
2905
3045
  }
2906
- function playerHeaders(session) {
2907
- return new Headers({
2908
- "content-type": "application/json",
2909
- cookie: `${browserCookie}; ${session.cookie}`,
2910
- origin: "https://www.youtube.com",
2911
- "user-agent": androidVrClient.userAgent,
2912
- "x-goog-visitor-id": session.visitorData,
2913
- "x-youtube-client-name": androidVrClient.number,
2914
- "x-youtube-client-version": androidVrClient.version
2915
- });
3046
+ function twitterCardImage(tweet, id) {
3047
+ const card = object(tweet.card) ? tweet.card : null;
3048
+ if (!card || card.name !== "summary" && card.name !== "summary_large_image")
3049
+ return null;
3050
+ const bindings = object(card.binding_values) ? card.binding_values : {};
3051
+ const images = Object.entries(bindings).flatMap(([key, value]) => {
3052
+ if (!/^(photo_image_full_size|summary_photo_image|thumbnail_image)(_(original|x_large|large|small))?$/.test(key) || !object(value) || !object(value.image_value))
3053
+ return [];
3054
+ const image2 = value.image_value;
3055
+ const rawUrl = string(image2.url);
3056
+ if (!rawUrl)
3057
+ return [];
3058
+ try {
3059
+ const url = new URL(rawUrl);
3060
+ if (url.protocol !== "https:" && url.protocol !== "http:")
3061
+ return [];
3062
+ const format = (url.searchParams.get("format") ?? url.pathname.match(/\.(jpe?g|png|webp|gif)$/i)?.[1] ?? "jpg").toLowerCase();
3063
+ const extension = format === "jpeg" ? "jpg" : format;
3064
+ if (!["jpg", "png", "webp", "gif"].includes(extension))
3065
+ return [];
3066
+ return [{
3067
+ url: url.href,
3068
+ extension,
3069
+ original: key.endsWith("_original") ? 1 : 0,
3070
+ area: Math.max(0, number(image2.width) ?? 0) * Math.max(0, number(image2.height) ?? 0)
3071
+ }];
3072
+ } catch {
3073
+ return [];
3074
+ }
3075
+ }).sort((a, b) => b.original - a.original || b.area - a.area);
3076
+ const image = images[0];
3077
+ return image ? {
3078
+ filename: filename(`twitter_${id}_preview.${image.extension}`),
3079
+ headers: { "user-agent": browserUserAgent() },
3080
+ id,
3081
+ kind: "image",
3082
+ mime: `image/${image.extension === "jpg" ? "jpeg" : image.extension}`,
3083
+ platform: "twitter",
3084
+ role: "linkPreview",
3085
+ url: image.url
3086
+ } : null;
2916
3087
  }
2917
- function youtubeCookie(headers) {
2918
- const readable = headers;
2919
- const fallback = headers.get("set-cookie");
2920
- const cookieHeaders = readable.getSetCookie ? readable.getSetCookie() : fallback ? [fallback] : [];
2921
- return cookieHeaders.map((header) => string(header.split(";", 1)[0])).filter(string).join("; ");
3088
+ function videoVariants(entry) {
3089
+ const info = object(entry.video_info) ? entry.video_info : null;
3090
+ const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
3091
+ const seen = new Set;
3092
+ return variants.flatMap((variant) => {
3093
+ const url = string(variant.url);
3094
+ if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
3095
+ return [];
3096
+ }
3097
+ seen.add(url);
3098
+ return [{ bitrate: number(variant.bitrate) ?? 0, url }];
3099
+ }).sort((left, right) => right.bitrate - left.bitrate);
2922
3100
  }
2923
- async function signatureTimestamp(net, playerPath) {
2924
- const response = await net(new URL(playerPath, "https://www.youtube.com").toString(), {
2925
- headers: { "user-agent": browserUserAgent() }
2926
- });
2927
- if (!response.ok) {
2928
- throw new Error(`YouTube player failed: ${response.status}`);
3101
+ async function sizeLimitedItems(input, groups) {
3102
+ if (groups.length === 0 || input.tryMaxBytes === undefined) {
3103
+ return groups.map((group) => group[0].item);
2929
3104
  }
2930
- const player = await response.text();
2931
- const timestamp = number(Number(player.match(/signatureTimestamp[:=](\d+)/)?.[1] ?? player.match(/sts[:=](\d+)/)?.[1]));
2932
- if (!timestamp) {
2933
- throw new Error("YouTube signature timestamp not found");
3105
+ const sizes = new Map;
3106
+ const size = (candidate) => {
3107
+ const existing = sizes.get(candidate.item.url);
3108
+ if (existing) {
3109
+ return existing;
3110
+ }
3111
+ const pending = contentLength(input.net, candidate.item);
3112
+ sizes.set(candidate.item.url, pending);
3113
+ return pending;
3114
+ };
3115
+ const highest = groups.map((group) => group[0]);
3116
+ const highestSizes = await Promise.all(highest.map(size));
3117
+ const highestTotal = knownTotal(highestSizes);
3118
+ if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
3119
+ return highest.map(({ item }) => item);
2934
3120
  }
2935
- return timestamp;
3121
+ const sized = await Promise.all(groups.map(async (group) => Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) })))));
3122
+ const fitting = bestFitting(sized, input.tryMaxBytes);
3123
+ return (fitting ?? groups.map((group) => group.at(-1))).map(({ item }) => item);
2936
3124
  }
2937
- function youtubeVideoId(input) {
2938
- const url = asUrl(input);
2939
- if (url.hostname === "youtu.be") {
2940
- return cleanId(url.pathname.split("/").filter(Boolean)[0]);
3125
+ function knownTotal(sizes) {
3126
+ return sizes.some((size) => size === null) ? null : sizes.reduce((total, size) => total + (size ?? 0), 0);
3127
+ }
3128
+ function bestFitting(groups, maxBytes) {
3129
+ let selections = [
3130
+ { bytes: 0, candidates: [], quality: 0 }
3131
+ ];
3132
+ for (const group of groups) {
3133
+ selections = selections.flatMap((selection) => group.flatMap((candidate) => {
3134
+ if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
3135
+ return [];
3136
+ }
3137
+ return [{
3138
+ bytes: selection.bytes + candidate.bytes,
3139
+ candidates: [...selection.candidates, candidate],
3140
+ quality: selection.quality + candidate.bitrate
3141
+ }];
3142
+ }));
3143
+ if (selections.length === 0) {
3144
+ return null;
3145
+ }
2941
3146
  }
2942
- const fromQuery = cleanId(url.searchParams.get("v"));
2943
- if (fromQuery) {
2944
- return fromQuery;
3147
+ const best = selections.reduce((current, selection) => selection.quality > current.quality ? selection : current);
3148
+ return best.candidates;
3149
+ }
3150
+ async function contentLength(net, item) {
3151
+ try {
3152
+ const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
3153
+ const raw = response.ok ? response.headers.get("content-length") : null;
3154
+ if (!raw || !/^\d+$/.test(raw)) {
3155
+ return null;
3156
+ }
3157
+ const bytes = Number(raw);
3158
+ return Number.isSafeInteger(bytes) ? bytes : null;
3159
+ } catch {
3160
+ return null;
2945
3161
  }
2946
- const parts = url.pathname.split("/").filter(Boolean);
2947
- const index = parts.findIndex((part) => part === "shorts" || part === "live" || part === "embed");
2948
- return index >= 0 ? cleanId(parts[index + 1]) : null;
2949
3162
  }
2950
- function cleanId(value) {
2951
- return typeof value === "string" && /^[A-Za-z0-9_-]{11}$/.test(value) ? value : null;
3163
+ function tweetId(input) {
3164
+ return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
2952
3165
  }
2953
- function selectFormat(payload) {
2954
- const root = object(payload) ? payload : null;
2955
- const streaming = root && object(root.streamingData) ? root.streamingData : null;
2956
- const formats = streaming && Array.isArray(streaming.formats) ? streaming.formats.filter(object) : [];
2957
- const mp4 = formats.filter((format) => string(format.url) && string(format.mimeType)?.startsWith("video/mp4")).sort((left, right) => height(right) - height(left));
2958
- return mp4[0] ? string(mp4[0].url) : null;
3166
+ async function twitterComments(input, id) {
3167
+ const limit = input.comments;
3168
+ if (!limit || !Number.isSafeInteger(limit) || limit < 0)
3169
+ return [];
3170
+ try {
3171
+ const comments = [];
3172
+ const seen = new Set;
3173
+ const cursors = new Set;
3174
+ let cursor = null;
3175
+ do {
3176
+ const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
3177
+ if (cursor)
3178
+ url.searchParams.set("cursor", cursor);
3179
+ const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
3180
+ if (!response.ok)
3181
+ throw new Error("Comment lookup failed");
3182
+ const payload = await response.json();
3183
+ if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
3184
+ throw new Error("Invalid comments");
3185
+ const previousCount = comments.length;
3186
+ for (const reply of payload.replies) {
3187
+ if (!object(reply))
3188
+ continue;
3189
+ const replyId = string(reply.id);
3190
+ const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
3191
+ if (!replyId || parent !== id || seen.has(replyId))
3192
+ continue;
3193
+ seen.add(replyId);
3194
+ comments.push(twitterComment(reply, replyId));
3195
+ if (comments.length >= limit)
3196
+ return comments;
3197
+ }
3198
+ cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
3199
+ if (!cursor || cursors.has(cursor) || comments.length === previousCount)
3200
+ break;
3201
+ cursors.add(cursor);
3202
+ } while (comments.length < limit);
3203
+ return comments;
3204
+ } catch {
3205
+ return [];
3206
+ }
2959
3207
  }
2960
- function height(format) {
2961
- return number(format.height) ?? Number(string(format.qualityLabel)?.match(/(\d+)p/)?.[1] ?? 0);
3208
+ function twitterComment(reply, id) {
3209
+ const author2 = object(reply.author) ? reply.author : {};
3210
+ const verification = object(author2.verification) ? author2.verification : {};
3211
+ const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
3212
+ const items = media.flatMap((entry, index) => {
3213
+ const url = string(entry.url);
3214
+ const photo = entry.type === "photo";
3215
+ if (!url || !photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")
3216
+ return [];
3217
+ return [{
3218
+ id,
3219
+ platform: "twitter",
3220
+ filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
3221
+ kind: photo ? "image" : "video",
3222
+ mime: photo ? "image/jpeg" : "video/mp4",
3223
+ url,
3224
+ headers: { "user-agent": browserUserAgent() }
3225
+ }];
3226
+ });
3227
+ return {
3228
+ id,
3229
+ url: `https://x.com/i/status/${id}`,
3230
+ items,
3231
+ metadata: {
3232
+ text: cleanTwitterText(string(reply.text), reply) ?? undefined,
3233
+ author: {
3234
+ handle: string(author2.screen_name) ?? undefined,
3235
+ name: string(author2.name) ?? undefined,
3236
+ verified: bool(verification.verified)
3237
+ },
3238
+ createdAt: isoFromDateString(reply.created_at),
3239
+ likeCount: count(reply.likes),
3240
+ commentCount: count(reply.replies),
3241
+ shareCount: count(reply.reposts),
3242
+ viewCount: count(reply.views),
3243
+ nsfw: bool(reply.possibly_sensitive)
3244
+ }
3245
+ };
2962
3246
  }
2963
3247
 
2964
3248
  // src/postfetch.ts
@@ -2994,6 +3278,8 @@ async function resolve(context) {
2994
3278
  return resolveReddit(context);
2995
3279
  case "soundcloud":
2996
3280
  return resolveSoundcloud(context);
3281
+ case "spotify":
3282
+ return resolveSpotify(context);
2997
3283
  case "tiktok":
2998
3284
  return resolveTiktok(context);
2999
3285
  case "twitter":
@@ -3074,6 +3360,9 @@ function detect(url) {
3074
3360
  if (host.includes("soundcloud.com") || host === "snd.sc") {
3075
3361
  return "soundcloud";
3076
3362
  }
3363
+ if (host === "open.spotify.com") {
3364
+ return "spotify";
3365
+ }
3077
3366
  if (host.includes("youtube.com") || host === "youtu.be") {
3078
3367
  return "youtube";
3079
3368
  }
@@ -3083,7 +3372,7 @@ function detect(url) {
3083
3372
  if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
3084
3373
  return "twitter";
3085
3374
  }
3086
- throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
3375
+ throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube URLs are supported");
3087
3376
  }
3088
3377
  // src/mp4-remux.ts
3089
3378
  async function prepareMp4(bytes, ffmpegPath = "ffmpeg", ffprobePath = "ffprobe", runtime) {
@@ -3472,11 +3761,13 @@ async function downloadBlob(input, optionsOrHeaders = {}, legacyOptions = {}) {
3472
3761
  const options = modern ? optionsOrHeaders : legacyOptions;
3473
3762
  const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
3474
3763
  const net = createNet(options.fetch ?? globalThis.fetch);
3475
- const response = typeof input === "string" ? await net(input, { headers }) : await download({ ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers }, options);
3476
- if (!response.ok || !response.body) {
3477
- throw new PostfetchError(502, `download failed: ${response.status}`);
3478
- }
3479
- const blob = await response.blob();
3764
+ const item = typeof input === "string" ? null : { ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers };
3765
+ const blob = item && buffered(item) ? new Blob([toArrayBuffer(await itemBytes2(net, item))], { type: item.mime }) : await readBody(net, typeof input === "string" ? input : input.url, { headers: item?.headers ?? headers }, async (response) => {
3766
+ if (!response.ok || !response.body) {
3767
+ throw new PostfetchError(502, `download failed: ${response.status}`);
3768
+ }
3769
+ return response.blob();
3770
+ });
3480
3771
  if (!modern || optionsOrHeaders.remux !== true) {
3481
3772
  return blob;
3482
3773
  }
@@ -3505,10 +3796,15 @@ async function archive(result, options = {}) {
3505
3796
  return { bytes: zip(files), filename: result.archiveFilename, mime: "application/zip" };
3506
3797
  }
3507
3798
  async function toResponse(result, options = {}) {
3508
- if (result.items.length === 1) {
3509
- return singleResponse(result.items[0], options);
3799
+ const response = result.items.length === 1 ? await singleResponse(result.items[0], options) : await archiveResponse(result, options);
3800
+ if (result.platform === "spotify") {
3801
+ response.headers.set("x-media-platform", result.platform);
3802
+ response.headers.set("x-media-id", result.id);
3803
+ const youtubeUrl = result.metadata?.extra?.youtubeUrl;
3804
+ if (youtubeUrl)
3805
+ response.headers.set("x-media-youtube-url", youtubeUrl);
3510
3806
  }
3511
- return archiveResponse(result, options);
3807
+ return response;
3512
3808
  }
3513
3809
  async function singleResponse(item, options) {
3514
3810
  if (buffered(item)) {
@@ -3575,11 +3871,12 @@ async function mergedBytes(net, item, audio) {
3575
3871
  return mergeAudioVideo(video, audioBytes);
3576
3872
  }
3577
3873
  async function fetchBytes(net, url, headers) {
3578
- const response = await net(url, { headers });
3579
- if (!response.ok) {
3580
- throw new PostfetchError(502, `download failed: ${response.status}`);
3581
- }
3582
- return new Uint8Array(await response.arrayBuffer());
3874
+ return readBody(net, url, { headers }, async (response) => {
3875
+ if (!response.ok) {
3876
+ throw new PostfetchError(502, `download failed: ${response.status}`);
3877
+ }
3878
+ return new Uint8Array(await response.arrayBuffer());
3879
+ });
3583
3880
  }
3584
3881
  function toArrayBuffer(bytes) {
3585
3882
  return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);