@postfetch/core 0.12.4 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +1009 -788
- package/dist/internal.d.ts +13 -2
- package/dist/spotify.d.ts +3 -0
- package/package.json +3 -2
- package/src/download.ts +9 -3
- package/src/index.ts +4 -2
- package/src/internal.ts +14 -1
- package/src/postfetch.ts +7 -1
- package/src/spotify.ts +170 -0
- package/src/twitter.ts +43 -1
package/dist/index.js
CHANGED
|
@@ -2015,902 +2015,1113 @@ function trackId(track) {
|
|
|
2015
2015
|
return string(track.permalink) ?? "track";
|
|
2016
2016
|
}
|
|
2017
2017
|
|
|
2018
|
-
// src/
|
|
2019
|
-
var
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2018
|
+
// src/youtube.ts
|
|
2019
|
+
var androidVrClient = {
|
|
2020
|
+
name: "ANDROID_VR",
|
|
2021
|
+
number: "28",
|
|
2022
|
+
userAgent: "com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip",
|
|
2023
|
+
version: "1.65.10"
|
|
2024
|
+
};
|
|
2025
|
+
var visionOsClient = {
|
|
2026
|
+
clientName: "VISIONOS",
|
|
2027
|
+
clientVersion: "1.02",
|
|
2028
|
+
deviceMake: "Apple",
|
|
2029
|
+
deviceModel: "RealityDevice17,1",
|
|
2030
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
|
|
2031
|
+
osName: "visionOS",
|
|
2032
|
+
osVersion: "26.5.23O471"
|
|
2033
|
+
};
|
|
2034
|
+
var browserCookie = "PREF=hl=en&tz=UTC; SOCS=CAI";
|
|
2035
|
+
async function resolveYoutube(input) {
|
|
2036
|
+
const id = youtubeVideoId(input.url);
|
|
2024
2037
|
if (!id) {
|
|
2025
|
-
throw new Error("
|
|
2026
|
-
}
|
|
2027
|
-
const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
|
|
2028
|
-
const user = author(page.item) ?? username(pageUrl) ?? "i";
|
|
2029
|
-
const headers = {
|
|
2030
|
-
referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
|
|
2031
|
-
"user-agent": page.userAgent
|
|
2032
|
-
};
|
|
2033
|
-
if (page.cookie) {
|
|
2034
|
-
headers.cookie = page.cookie;
|
|
2035
|
-
}
|
|
2036
|
-
const items = mediaItems4(page.item, user, id, headers);
|
|
2037
|
-
if (items.length === 0) {
|
|
2038
|
-
throw new Error("TikTok media not found");
|
|
2038
|
+
throw new Error("YouTube video id not found");
|
|
2039
2039
|
}
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
const
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2040
|
+
let session;
|
|
2041
|
+
try {
|
|
2042
|
+
session = await youtubeSession(input.net, id);
|
|
2043
|
+
const payload = await youtubePlayer(input.net, playerBody(id, session), playerHeaders(session));
|
|
2044
|
+
const streams = selectStreams(payload, input.preferredWidth);
|
|
2045
|
+
if (!streams) {
|
|
2046
|
+
throw new Error("YouTube mp4 stream not found");
|
|
2047
|
+
}
|
|
2048
|
+
const headers = { "user-agent": androidVrClient.userAgent };
|
|
2049
|
+
await Promise.all([streams.video, streams.audio].filter((url) => url !== null).map((url) => probeStream(input.net, url, headers)));
|
|
2050
|
+
return youtubeResult(id, payload, streams, headers);
|
|
2051
|
+
} catch (primaryError) {
|
|
2051
2052
|
try {
|
|
2052
|
-
return
|
|
2053
|
-
} catch (
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
}
|
|
2053
|
+
return await resolveVisionOs(input, id, session?.visitorData);
|
|
2054
|
+
} catch (fallbackError) {
|
|
2055
|
+
const message = (error) => error instanceof Error ? error.message : String(error);
|
|
2056
|
+
throw new Error(`YouTube resolution failed: ${message(primaryError)}; visionOS HLS: ${message(fallbackError)}`, { cause: fallbackError });
|
|
2057
2057
|
}
|
|
2058
2058
|
}
|
|
2059
|
-
|
|
2060
|
-
|
|
2059
|
+
}
|
|
2060
|
+
async function youtubePlayer(net, body, headers) {
|
|
2061
|
+
const response = await net("https://www.youtube.com/youtubei/v1/player?prettyPrint=false", {
|
|
2062
|
+
body: JSON.stringify(body),
|
|
2063
|
+
headers,
|
|
2064
|
+
method: "POST"
|
|
2065
|
+
});
|
|
2066
|
+
if (!response.ok) {
|
|
2067
|
+
throw new Error(`YouTube player failed: ${response.status}`);
|
|
2061
2068
|
}
|
|
2062
|
-
const
|
|
2063
|
-
const
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2069
|
+
const payload = await response.json();
|
|
2070
|
+
const status = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.status) : null;
|
|
2071
|
+
if (status !== "OK") {
|
|
2072
|
+
const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
|
|
2073
|
+
throw new Error(reason ?? "YouTube video unavailable");
|
|
2074
|
+
}
|
|
2075
|
+
return payload;
|
|
2076
|
+
}
|
|
2077
|
+
function youtubeResult(id, payload, streams, headers) {
|
|
2078
|
+
const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
|
|
2079
|
+
const media = {
|
|
2080
|
+
filename: filename(`youtube_${title ?? id}_${id}.mp4`),
|
|
2081
|
+
headers,
|
|
2082
|
+
id,
|
|
2083
|
+
kind: "video",
|
|
2084
|
+
mime: "video/mp4",
|
|
2085
|
+
platform: "youtube",
|
|
2086
|
+
url: streams.video,
|
|
2087
|
+
...streams.hls ? { hls: true } : {},
|
|
2088
|
+
...streams.audio ? { audio: { headers, url: streams.audio } } : {}
|
|
2068
2089
|
};
|
|
2069
|
-
return {
|
|
2090
|
+
return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
2070
2091
|
}
|
|
2071
|
-
function
|
|
2072
|
-
const
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2092
|
+
async function probeStream(net, url, headers) {
|
|
2093
|
+
const response = await net(url, { headers }, 1);
|
|
2094
|
+
await response.body?.cancel();
|
|
2095
|
+
if (!response.ok) {
|
|
2096
|
+
throw new Error(`YouTube stream failed: ${response.status}`);
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
async function resolveVisionOs(input, id, visitorData) {
|
|
2100
|
+
const headers = { "user-agent": visionOsClient.userAgent };
|
|
2101
|
+
const payload = await youtubePlayer(input.net, {
|
|
2102
|
+
videoId: id,
|
|
2103
|
+
contentCheckOk: true,
|
|
2104
|
+
racyCheckOk: true,
|
|
2105
|
+
context: { client: { ...visionOsClient, hl: "en", gl: "US", ...visitorData ? { visitorData } : {} } }
|
|
2106
|
+
}, {
|
|
2107
|
+
...headers,
|
|
2108
|
+
"content-type": "application/json",
|
|
2109
|
+
"x-youtube-client-name": "101",
|
|
2110
|
+
"x-youtube-client-version": visionOsClient.clientVersion,
|
|
2111
|
+
...visitorData ? { "x-goog-visitor-id": visitorData } : {}
|
|
2112
|
+
});
|
|
2113
|
+
const streaming = object(payload) && object(payload.streamingData) ? payload.streamingData : null;
|
|
2114
|
+
const manifestUrl = streaming ? string(streaming.hlsManifestUrl) : null;
|
|
2115
|
+
if (!manifestUrl) {
|
|
2116
|
+
throw new Error("YouTube HLS manifest not found");
|
|
2117
|
+
}
|
|
2118
|
+
const response = await input.net(manifestUrl, { headers });
|
|
2119
|
+
if (!response.ok) {
|
|
2120
|
+
throw new Error(`YouTube HLS manifest failed: ${response.status}`);
|
|
2121
|
+
}
|
|
2122
|
+
const master = parseMaster(await response.text(), manifestUrl, { preferOriginalAudio: true });
|
|
2123
|
+
const variants = master.variants.filter((variant2) => /(?:^|,)(?:vp09|av01)\./i.test(variant2.codecs ?? "") && /(?:^|,)mp4a\.40\.2(?:,|$)/i.test(variant2.codecs ?? "") && variant2.audioGroup && master.audio[variant2.audioGroup]).sort((left, right) => Math.abs(left.width - input.preferredWidth) - Math.abs(right.width - input.preferredWidth) || right.bandwidth - left.bandwidth);
|
|
2124
|
+
const variant = variants[0];
|
|
2125
|
+
if (!variant?.audioGroup) {
|
|
2126
|
+
throw new Error("YouTube fragmented MP4 HLS video with AAC audio not found");
|
|
2127
|
+
}
|
|
2128
|
+
const playlist = await input.net(variant.url, { headers });
|
|
2129
|
+
if (!playlist.ok) {
|
|
2130
|
+
throw new Error(`YouTube HLS playlist failed: ${playlist.status}`);
|
|
2131
|
+
}
|
|
2132
|
+
if (!(await playlist.text()).includes("#EXT-X-MAP:")) {
|
|
2133
|
+
throw new Error("YouTube HLS video is not fragmented MP4");
|
|
2134
|
+
}
|
|
2135
|
+
return youtubeResult(id, payload, { video: variant.url, audio: master.audio[variant.audioGroup], hls: true }, headers);
|
|
2136
|
+
}
|
|
2137
|
+
function youtubeMetadata(payload) {
|
|
2138
|
+
const details = object(payload) && object(payload.videoDetails) ? payload.videoDetails : null;
|
|
2139
|
+
const renderer = object(payload) && object(payload.microformat) && object(payload.microformat.playerMicroformatRenderer) ? payload.microformat.playerMicroformatRenderer : null;
|
|
2140
|
+
const author = details ? string(details.author) : null;
|
|
2141
|
+
const keywords = details && Array.isArray(details.keywords) ? details.keywords.filter((keyword) => typeof keyword === "string") : [];
|
|
2076
2142
|
return {
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
} : undefined,
|
|
2083
|
-
createdAt: isoFromEpochSeconds(item.createTime),
|
|
2084
|
-
likeCount: stats ? count(stats.diggCount) : undefined,
|
|
2085
|
-
commentCount: stats ? count(stats.commentCount) : undefined,
|
|
2086
|
-
shareCount: stats ? count(stats.shareCount) : undefined,
|
|
2087
|
-
viewCount: stats ? count(stats.playCount) : undefined,
|
|
2143
|
+
title: details ? string(details.title) ?? undefined : undefined,
|
|
2144
|
+
text: details ? string(details.shortDescription) ?? undefined : undefined,
|
|
2145
|
+
author: author ? { name: author } : undefined,
|
|
2146
|
+
createdAt: renderer ? isoFromDateString(renderer.publishDate) : undefined,
|
|
2147
|
+
viewCount: details ? count(details.viewCount) : undefined,
|
|
2088
2148
|
extra: {
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
region: string(item.locationCreated) ?? undefined,
|
|
2093
|
-
durationSeconds: video ? count(video.duration) : undefined
|
|
2149
|
+
channelId: details ? string(details.channelId) ?? undefined : undefined,
|
|
2150
|
+
durationSeconds: details ? count(details.lengthSeconds) : undefined,
|
|
2151
|
+
keywords: keywords.length > 0 ? keywords : undefined
|
|
2094
2152
|
}
|
|
2095
2153
|
};
|
|
2096
2154
|
}
|
|
2097
|
-
|
|
2098
|
-
const
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2155
|
+
function selectStreams(payload, preferredWidth) {
|
|
2156
|
+
const adaptive = adaptiveFormats(payload);
|
|
2157
|
+
const video = bestVideo(adaptive, preferredWidth);
|
|
2158
|
+
const audio = bestAudio(adaptive);
|
|
2159
|
+
if (video && audio) {
|
|
2160
|
+
return { video, audio };
|
|
2161
|
+
}
|
|
2162
|
+
const progressive = selectFormat(payload);
|
|
2163
|
+
return progressive ? { video: progressive, audio: null } : null;
|
|
2102
2164
|
}
|
|
2103
|
-
|
|
2104
|
-
const
|
|
2105
|
-
|
|
2165
|
+
function adaptiveFormats(payload) {
|
|
2166
|
+
const root = object(payload) ? payload : null;
|
|
2167
|
+
const streaming = root && object(root.streamingData) ? root.streamingData : null;
|
|
2168
|
+
return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
|
|
2169
|
+
}
|
|
2170
|
+
function bestVideo(formats, preferredWidth) {
|
|
2171
|
+
const candidates = formats.filter((format) => {
|
|
2172
|
+
const mimeType2 = string(format.mimeType);
|
|
2173
|
+
return Boolean(string(format.url)) && mimeType2?.startsWith("video/mp4") === true && mimeType2.includes("avc1");
|
|
2106
2174
|
});
|
|
2107
|
-
|
|
2175
|
+
const best = candidates.reduce((current, format) => {
|
|
2176
|
+
if (!current) {
|
|
2177
|
+
return format;
|
|
2178
|
+
}
|
|
2179
|
+
const width = number(format.width) ?? 0;
|
|
2180
|
+
const currentWidth = number(current.width) ?? 0;
|
|
2181
|
+
return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
|
|
2182
|
+
}, null);
|
|
2183
|
+
return best ? string(best.url) : null;
|
|
2108
2184
|
}
|
|
2109
|
-
function
|
|
2110
|
-
|
|
2185
|
+
function bestAudio(formats) {
|
|
2186
|
+
const candidates = formats.filter((format) => Boolean(string(format.url)) && string(format.mimeType)?.startsWith("audio/mp4") === true);
|
|
2187
|
+
const best = candidates.reduce((current, format) => {
|
|
2188
|
+
if (!current) {
|
|
2189
|
+
return format;
|
|
2190
|
+
}
|
|
2191
|
+
const preference = audioPreference(format) - audioPreference(current);
|
|
2192
|
+
if (preference !== 0) {
|
|
2193
|
+
return preference > 0 ? format : current;
|
|
2194
|
+
}
|
|
2195
|
+
return (number(format.bitrate) ?? 0) > (number(current.bitrate) ?? 0) ? format : current;
|
|
2196
|
+
}, null);
|
|
2197
|
+
return best ? string(best.url) : null;
|
|
2111
2198
|
}
|
|
2112
|
-
|
|
2113
|
-
const
|
|
2114
|
-
if (
|
|
2115
|
-
return
|
|
2116
|
-
}
|
|
2117
|
-
const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
|
|
2118
|
-
const location = response.headers.get("location");
|
|
2119
|
-
if (location) {
|
|
2120
|
-
return new URL(location, input).href;
|
|
2199
|
+
function audioPreference(format) {
|
|
2200
|
+
const track = object(format.audioTrack) ? format.audioTrack : null;
|
|
2201
|
+
if (track && /\boriginal\b/i.test(string(track.displayName) ?? "")) {
|
|
2202
|
+
return 2;
|
|
2121
2203
|
}
|
|
2122
|
-
|
|
2123
|
-
const href = html.match(/<a href="(https:\/\/[^"]+)"/)?.[1];
|
|
2124
|
-
return href ? new URL(href.split("?")[0]).href : input;
|
|
2204
|
+
return track?.audioIsDefault === true ? 1 : 0;
|
|
2125
2205
|
}
|
|
2126
|
-
function
|
|
2127
|
-
return
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2206
|
+
function playerBody(id, session) {
|
|
2207
|
+
return {
|
|
2208
|
+
contentCheckOk: true,
|
|
2209
|
+
context: {
|
|
2210
|
+
client: {
|
|
2211
|
+
androidSdkVersion: 32,
|
|
2212
|
+
clientName: androidVrClient.name,
|
|
2213
|
+
clientVersion: androidVrClient.version,
|
|
2214
|
+
deviceMake: "Oculus",
|
|
2215
|
+
deviceModel: "Quest 3",
|
|
2216
|
+
gl: "US",
|
|
2217
|
+
hl: "en",
|
|
2218
|
+
osName: "Android",
|
|
2219
|
+
osVersion: "12L",
|
|
2220
|
+
timeZone: "UTC",
|
|
2221
|
+
userAgent: androidVrClient.userAgent,
|
|
2222
|
+
utcOffsetMinutes: 0,
|
|
2223
|
+
visitorData: session.visitorData
|
|
2224
|
+
}
|
|
2225
|
+
},
|
|
2226
|
+
playbackContext: {
|
|
2227
|
+
contentPlaybackContext: {
|
|
2228
|
+
html5Preference: "HTML5_PREF_WANTS",
|
|
2229
|
+
signatureTimestamp: session.signatureTimestamp
|
|
2230
|
+
}
|
|
2231
|
+
},
|
|
2232
|
+
racyCheckOk: true,
|
|
2233
|
+
videoId: id
|
|
2234
|
+
};
|
|
2147
2235
|
}
|
|
2148
|
-
function
|
|
2149
|
-
const
|
|
2150
|
-
|
|
2151
|
-
|
|
2236
|
+
async function youtubeSession(net, id) {
|
|
2237
|
+
const response = await net(`https://www.youtube.com/watch?v=${id}&bpctr=9999999999&has_verified=1`, {
|
|
2238
|
+
headers: {
|
|
2239
|
+
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
2240
|
+
"accept-language": "en-us,en;q=0.5",
|
|
2241
|
+
cookie: browserCookie,
|
|
2242
|
+
"sec-fetch-mode": "navigate",
|
|
2243
|
+
"user-agent": browserUserAgent()
|
|
2244
|
+
}
|
|
2245
|
+
});
|
|
2246
|
+
if (!response.ok) {
|
|
2247
|
+
throw new Error(`YouTube page failed: ${response.status}`);
|
|
2152
2248
|
}
|
|
2153
|
-
const
|
|
2154
|
-
const
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
const route = object(rawRoute) ? rawRoute : null;
|
|
2158
|
-
const videoData = route && object(route.videoData) ? route.videoData : null;
|
|
2159
|
-
const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
|
|
2160
|
-
if (!videoData || !item) {
|
|
2161
|
-
throw new Error("TikTok embed item not found");
|
|
2249
|
+
const page = await response.text();
|
|
2250
|
+
const visitorData = string(page.match(/"visitorData":"([^"]+)"/)?.[1]);
|
|
2251
|
+
if (!visitorData) {
|
|
2252
|
+
throw new Error("YouTube visitor data not found");
|
|
2162
2253
|
}
|
|
2163
|
-
const
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
|
|
2167
|
-
const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
|
|
2168
|
-
const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
|
|
2169
|
-
if (displayImages.length === 0) {
|
|
2170
|
-
throw new Error("TikTok video hydration not found");
|
|
2254
|
+
const playerPath = string(page.match(/"jsUrl":"([^"]+)"/)?.[1]) ?? string(page.match(/"PLAYER_JS_URL":"([^"]+)"/)?.[1]);
|
|
2255
|
+
if (!playerPath) {
|
|
2256
|
+
throw new Error("YouTube player url not found");
|
|
2171
2257
|
}
|
|
2172
2258
|
return {
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
uniqueId: string(authorInfo.uniqueId),
|
|
2177
|
-
verified: bool(authorInfo.verified)
|
|
2178
|
-
},
|
|
2179
|
-
desc: string(item.text),
|
|
2180
|
-
imagePost: {
|
|
2181
|
-
images: displayImages.map((image) => ({
|
|
2182
|
-
imageURL: {
|
|
2183
|
-
urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
|
|
2184
|
-
}
|
|
2185
|
-
}))
|
|
2186
|
-
},
|
|
2187
|
-
music: {
|
|
2188
|
-
authorName: string(musicInfo.authorName),
|
|
2189
|
-
playUrl: firstString(musicInfo.playUrl),
|
|
2190
|
-
title: string(musicInfo.musicName)
|
|
2191
|
-
},
|
|
2192
|
-
stats: {
|
|
2193
|
-
commentCount: item.commentCount,
|
|
2194
|
-
diggCount: item.diggCount,
|
|
2195
|
-
playCount: item.playCount,
|
|
2196
|
-
shareCount: item.shareCount
|
|
2197
|
-
},
|
|
2198
|
-
video: {
|
|
2199
|
-
duration: videoMeta.duration
|
|
2200
|
-
}
|
|
2259
|
+
cookie: youtubeCookie(response.headers),
|
|
2260
|
+
signatureTimestamp: await signatureTimestamp(net, playerPath),
|
|
2261
|
+
visitorData
|
|
2201
2262
|
};
|
|
2202
2263
|
}
|
|
2203
|
-
function
|
|
2204
|
-
|
|
2205
|
-
|
|
2264
|
+
function playerHeaders(session) {
|
|
2265
|
+
return new Headers({
|
|
2266
|
+
"content-type": "application/json",
|
|
2267
|
+
cookie: `${browserCookie}; ${session.cookie}`,
|
|
2268
|
+
origin: "https://www.youtube.com",
|
|
2269
|
+
"user-agent": androidVrClient.userAgent,
|
|
2270
|
+
"x-goog-visitor-id": session.visitorData,
|
|
2271
|
+
"x-youtube-client-name": androidVrClient.number,
|
|
2272
|
+
"x-youtube-client-version": androidVrClient.version
|
|
2273
|
+
});
|
|
2206
2274
|
}
|
|
2207
|
-
function
|
|
2208
|
-
|
|
2275
|
+
function youtubeCookie(headers) {
|
|
2276
|
+
const readable = headers;
|
|
2277
|
+
const fallback = headers.get("set-cookie");
|
|
2278
|
+
const cookieHeaders = readable.getSetCookie ? readable.getSetCookie() : fallback ? [fallback] : [];
|
|
2279
|
+
return cookieHeaders.map((header) => string(header.split(";", 1)[0])).filter(string).join("; ");
|
|
2209
2280
|
}
|
|
2210
|
-
function
|
|
2211
|
-
const
|
|
2212
|
-
|
|
2281
|
+
async function signatureTimestamp(net, playerPath) {
|
|
2282
|
+
const response = await net(new URL(playerPath, "https://www.youtube.com").toString(), {
|
|
2283
|
+
headers: { "user-agent": browserUserAgent() }
|
|
2284
|
+
});
|
|
2285
|
+
if (!response.ok) {
|
|
2286
|
+
throw new Error(`YouTube player failed: ${response.status}`);
|
|
2287
|
+
}
|
|
2288
|
+
const player = await response.text();
|
|
2289
|
+
const timestamp = number(Number(player.match(/signatureTimestamp[:=](\d+)/)?.[1] ?? player.match(/sts[:=](\d+)/)?.[1]));
|
|
2290
|
+
if (!timestamp) {
|
|
2291
|
+
throw new Error("YouTube signature timestamp not found");
|
|
2292
|
+
}
|
|
2293
|
+
return timestamp;
|
|
2213
2294
|
}
|
|
2214
|
-
function
|
|
2215
|
-
const
|
|
2216
|
-
if (
|
|
2217
|
-
|
|
2218
|
-
return audio ? [...images, audio] : images;
|
|
2295
|
+
function youtubeVideoId(input) {
|
|
2296
|
+
const url = asUrl(input);
|
|
2297
|
+
if (url.hostname === "youtu.be") {
|
|
2298
|
+
return cleanId(url.pathname.split("/").filter(Boolean)[0]);
|
|
2219
2299
|
}
|
|
2220
|
-
const
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
platform: "tiktok",
|
|
2228
|
-
url
|
|
2229
|
-
}] : [];
|
|
2300
|
+
const fromQuery = cleanId(url.searchParams.get("v"));
|
|
2301
|
+
if (fromQuery) {
|
|
2302
|
+
return fromQuery;
|
|
2303
|
+
}
|
|
2304
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
2305
|
+
const index = parts.findIndex((part) => part === "shorts" || part === "live" || part === "embed");
|
|
2306
|
+
return index >= 0 ? cleanId(parts[index + 1]) : null;
|
|
2230
2307
|
}
|
|
2231
|
-
function
|
|
2232
|
-
|
|
2233
|
-
const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
|
|
2234
|
-
return images.flatMap((image, index) => {
|
|
2235
|
-
const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
|
|
2236
|
-
const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
|
|
2237
|
-
const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
|
|
2238
|
-
return url ? [{
|
|
2239
|
-
filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
|
|
2240
|
-
headers,
|
|
2241
|
-
id,
|
|
2242
|
-
kind: "image",
|
|
2243
|
-
mime: "image/jpeg",
|
|
2244
|
-
platform: "tiktok",
|
|
2245
|
-
url
|
|
2246
|
-
}] : [];
|
|
2247
|
-
});
|
|
2308
|
+
function cleanId(value) {
|
|
2309
|
+
return typeof value === "string" && /^[A-Za-z0-9_-]{11}$/.test(value) ? value : null;
|
|
2248
2310
|
}
|
|
2249
|
-
function
|
|
2250
|
-
const
|
|
2251
|
-
const
|
|
2252
|
-
const
|
|
2253
|
-
const
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2311
|
+
function selectFormat(payload) {
|
|
2312
|
+
const root = object(payload) ? payload : null;
|
|
2313
|
+
const streaming = root && object(root.streamingData) ? root.streamingData : null;
|
|
2314
|
+
const formats = streaming && Array.isArray(streaming.formats) ? streaming.formats.filter(object) : [];
|
|
2315
|
+
const mp4 = formats.filter((format) => string(format.url) && string(format.mimeType)?.startsWith("video/mp4")).sort((left, right) => height(right) - height(left));
|
|
2316
|
+
return mp4[0] ? string(mp4[0].url) : null;
|
|
2317
|
+
}
|
|
2318
|
+
function height(format) {
|
|
2319
|
+
return number(format.height) ?? Number(string(format.qualityLabel)?.match(/(\d+)p/)?.[1] ?? 0);
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
// src/spotify.ts
|
|
2323
|
+
async function resolveSpotify(input) {
|
|
2324
|
+
const url = asUrl(input.url);
|
|
2325
|
+
const id = url.pathname.match(/^\/(?:intl-[a-z-]+\/)?(?:embed\/)?track\/([A-Za-z0-9]{22})\/?$/i)?.[1];
|
|
2326
|
+
if (!id) {
|
|
2327
|
+
throw new PostfetchError(400, "Spotify requires an open.spotify.com track URL");
|
|
2257
2328
|
}
|
|
2258
|
-
const
|
|
2329
|
+
const track = await spotifyTrack(input, id);
|
|
2330
|
+
const youtubeUrl = await matchYoutube(input, track);
|
|
2331
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2259
2332
|
return {
|
|
2260
|
-
|
|
2261
|
-
|
|
2333
|
+
archiveFilename: filename(`spotify_${id}.zip`),
|
|
2334
|
+
comments: [],
|
|
2262
2335
|
id,
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2336
|
+
items: youtube.items,
|
|
2337
|
+
platform: "spotify",
|
|
2338
|
+
metadata: {
|
|
2339
|
+
title: track.title,
|
|
2340
|
+
author: { name: track.artists.join(", ") },
|
|
2341
|
+
extra: { artists: track.artists, durationSeconds: track.durationSeconds, youtubeUrl }
|
|
2342
|
+
}
|
|
2267
2343
|
};
|
|
2268
2344
|
}
|
|
2269
|
-
function
|
|
2270
|
-
const
|
|
2271
|
-
|
|
2345
|
+
async function spotifyTrack(input, id) {
|
|
2346
|
+
const response = await input.net(`https://open.spotify.com/embed/track/${id}`, {
|
|
2347
|
+
headers: { "user-agent": browserUserAgent() }
|
|
2348
|
+
});
|
|
2349
|
+
if (!response.ok) {
|
|
2350
|
+
throw new PostfetchError(response.status === 404 ? 404 : 502, `Spotify track lookup failed: ${response.status}`);
|
|
2351
|
+
}
|
|
2352
|
+
const html = await response.text();
|
|
2353
|
+
const raw = html.match(/<script\b[^>]*\bid=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
2354
|
+
const payload = parseJson2(raw);
|
|
2355
|
+
const props = payload && object(payload.props) ? payload.props : null;
|
|
2356
|
+
const page = props && object(props.pageProps) ? props.pageProps : null;
|
|
2357
|
+
const state = page && object(page.state) ? page.state : null;
|
|
2358
|
+
const data = state && object(state.data) ? state.data : null;
|
|
2359
|
+
const entity = data && object(data.entity) ? data.entity : null;
|
|
2360
|
+
const title = entity && (string(entity.title) ?? string(entity.name));
|
|
2361
|
+
const artists = entity && Array.isArray(entity.artists) ? entity.artists.filter(object).map((artist) => string(artist.name)).filter((name) => name !== null) : [];
|
|
2362
|
+
if (!entity || entity.type !== "track" || entity.id !== id || !title || artists.length === 0) {
|
|
2363
|
+
throw new PostfetchError(502, "Spotify track metadata not found");
|
|
2364
|
+
}
|
|
2365
|
+
const duration = number(entity.duration);
|
|
2366
|
+
return { title, artists, durationSeconds: duration !== null && duration > 0 ? duration / 1000 : undefined };
|
|
2367
|
+
}
|
|
2368
|
+
async function matchYoutube(input, track) {
|
|
2369
|
+
const url = new URL("https://www.youtube.com/results");
|
|
2370
|
+
url.searchParams.set("search_query", `${track.artists.join(" ")} ${track.title} audio`);
|
|
2371
|
+
url.searchParams.set("hl", "en");
|
|
2372
|
+
const response = await input.net(url.href, {
|
|
2373
|
+
headers: { "accept-language": "en-US,en;q=0.9", cookie: "SOCS=CAI", "user-agent": browserUserAgent() }
|
|
2374
|
+
});
|
|
2375
|
+
if (!response.ok) {
|
|
2376
|
+
throw new PostfetchError(502, `YouTube search failed: ${response.status}`);
|
|
2377
|
+
}
|
|
2378
|
+
const html = await response.text();
|
|
2379
|
+
const raw = html.match(/(?:\bytInitialData|window\[["']ytInitialData["']\])\s*=\s*(\{[\s\S]*?)<\/script>/)?.[1];
|
|
2380
|
+
const payload = parseAssignment(raw);
|
|
2381
|
+
if (!payload || !object(payload.contents)) {
|
|
2382
|
+
throw new PostfetchError(502, "YouTube search results not found");
|
|
2383
|
+
}
|
|
2384
|
+
const candidates = videoRenderers(payload.contents);
|
|
2385
|
+
const ranked = candidates.map((candidate, index) => ({ candidate, score: matchScore(track, candidate) - index / 1000 })).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
2386
|
+
const id = ranked[0]?.candidate.videoId;
|
|
2387
|
+
if (typeof id !== "string") {
|
|
2388
|
+
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "notFound");
|
|
2389
|
+
}
|
|
2390
|
+
return `https://www.youtube.com/watch?v=${id}`;
|
|
2391
|
+
}
|
|
2392
|
+
function videoRenderers(value) {
|
|
2393
|
+
if (Array.isArray(value))
|
|
2394
|
+
return value.flatMap(videoRenderers);
|
|
2395
|
+
if (!object(value))
|
|
2396
|
+
return [];
|
|
2397
|
+
if (object(value.videoRenderer))
|
|
2398
|
+
return [value.videoRenderer];
|
|
2399
|
+
return Object.values(value).flatMap(videoRenderers);
|
|
2272
2400
|
}
|
|
2273
|
-
function
|
|
2274
|
-
|
|
2401
|
+
function text(value) {
|
|
2402
|
+
if (!object(value))
|
|
2403
|
+
return "";
|
|
2404
|
+
return string(value.simpleText) ?? (Array.isArray(value.runs) ? value.runs.filter(object).map((run) => typeof run.text === "string" ? run.text : "").join("") : "");
|
|
2275
2405
|
}
|
|
2276
|
-
function
|
|
2277
|
-
|
|
2278
|
-
const setCookie = typeof getter === "function" ? getter.call(headers).join(",") : headers.get("set-cookie");
|
|
2279
|
-
const cookies = setCookie?.split(/,(?=[^;]+?=)/).map((part) => part.split(";")[0]?.trim()).filter((part) => Boolean(part));
|
|
2280
|
-
return cookies && cookies.length > 0 ? cookies.join("; ") : null;
|
|
2406
|
+
function normalize(value) {
|
|
2407
|
+
return value.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
2281
2408
|
}
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
if (
|
|
2287
|
-
|
|
2409
|
+
function contains(haystack, needle) {
|
|
2410
|
+
return needle.length > 0 && ` ${haystack} `.includes(` ${needle} `);
|
|
2411
|
+
}
|
|
2412
|
+
function durationSeconds(value) {
|
|
2413
|
+
if (!/^\d+(?::\d{2}){1,2}$/.test(value))
|
|
2414
|
+
return null;
|
|
2415
|
+
return value.split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0);
|
|
2416
|
+
}
|
|
2417
|
+
function matchScore(track, candidate) {
|
|
2418
|
+
if (typeof candidate.videoId !== "string" || !/^[A-Za-z0-9_-]{11}$/.test(candidate.videoId))
|
|
2419
|
+
return -Infinity;
|
|
2420
|
+
const title = normalize(text(candidate.title));
|
|
2421
|
+
const author = normalize(text(candidate.ownerText) || text(candidate.longBylineText) || text(candidate.shortBylineText));
|
|
2422
|
+
const expected = normalize(track.title.replace(/\s*[([](?:feat\.?|ft\.?).*?[)\]]/gi, ""));
|
|
2423
|
+
const tokens = expected.split(" ").filter(Boolean);
|
|
2424
|
+
if (tokens.length === 0)
|
|
2425
|
+
return -Infinity;
|
|
2426
|
+
const coverage = tokens.filter((token) => contains(title, token)).length / tokens.length;
|
|
2427
|
+
if (coverage < (tokens.length <= 3 ? 1 : 0.8))
|
|
2428
|
+
return -Infinity;
|
|
2429
|
+
if (tokens.length <= 3 && !contains(title, expected))
|
|
2430
|
+
return -Infinity;
|
|
2431
|
+
const artists = track.artists.map(normalize);
|
|
2432
|
+
if (!contains(`${title} ${author}`, artists[0]))
|
|
2433
|
+
return -Infinity;
|
|
2434
|
+
const duration = durationSeconds(text(candidate.lengthText));
|
|
2435
|
+
if (duration === null || duration <= 0)
|
|
2436
|
+
return -Infinity;
|
|
2437
|
+
const tolerance = Math.min(30, Math.max(12, (track.durationSeconds ?? duration) * 0.08));
|
|
2438
|
+
const delta = track.durationSeconds === undefined ? 0 : Math.abs(duration - track.durationSeconds);
|
|
2439
|
+
if (delta > tolerance)
|
|
2440
|
+
return -Infinity;
|
|
2441
|
+
let recordingTitle = title;
|
|
2442
|
+
for (const artist of artists)
|
|
2443
|
+
recordingTitle = recordingTitle.replace(artist, "");
|
|
2444
|
+
for (const version of [/\bremix\b/, /\bcover\b/, /\bkaraoke\b/, /\binstrumental\b/, /\bslowed\b/, /\b(?:sped|speed) up\b/, /\bnightcore\b/, /\blive\b/, /\bacoustic\b/]) {
|
|
2445
|
+
if (version.test(recordingTitle) !== version.test(expected))
|
|
2446
|
+
return -Infinity;
|
|
2447
|
+
}
|
|
2448
|
+
const artistUpload = contains(author, artists[0]);
|
|
2449
|
+
const credited = artists.filter((artist) => contains(`${title} ${author}`, artist)).length / artists.length;
|
|
2450
|
+
return coverage * 10 + (artistUpload ? 8 : 0) + credited * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
2451
|
+
}
|
|
2452
|
+
function parseJson2(value) {
|
|
2453
|
+
try {
|
|
2454
|
+
const parsed = JSON.parse(value ?? "");
|
|
2455
|
+
return object(parsed) ? parsed : null;
|
|
2456
|
+
} catch {
|
|
2457
|
+
return null;
|
|
2288
2458
|
}
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2459
|
+
}
|
|
2460
|
+
function parseAssignment(value) {
|
|
2461
|
+
if (!value)
|
|
2462
|
+
return null;
|
|
2463
|
+
let depth = 0;
|
|
2464
|
+
let quoted = false;
|
|
2465
|
+
let escaped = false;
|
|
2466
|
+
for (let index = 0;index < value.length; index += 1) {
|
|
2467
|
+
const character = value[index];
|
|
2468
|
+
if (quoted) {
|
|
2469
|
+
if (escaped)
|
|
2470
|
+
escaped = false;
|
|
2471
|
+
else if (character === "\\")
|
|
2472
|
+
escaped = true;
|
|
2473
|
+
else if (character === '"')
|
|
2474
|
+
quoted = false;
|
|
2475
|
+
} else if (character === '"')
|
|
2476
|
+
quoted = true;
|
|
2477
|
+
else if (character === "{")
|
|
2478
|
+
depth += 1;
|
|
2479
|
+
else if (character === "}" && --depth === 0)
|
|
2480
|
+
return parseJson2(value.slice(0, index + 1));
|
|
2292
2481
|
}
|
|
2293
|
-
|
|
2294
|
-
const groups = twitterTweets(tweet, id).flatMap(({ id: postId2, tweet: post }) => {
|
|
2295
|
-
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
2296
|
-
return media.flatMap((entry, index) => {
|
|
2297
|
-
const candidates = twitterCandidates(entry, postId2, index + 1);
|
|
2298
|
-
return candidates.length > 0 ? [candidates] : [];
|
|
2299
|
-
});
|
|
2300
|
-
});
|
|
2301
|
-
const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
|
|
2302
|
-
return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
2482
|
+
return null;
|
|
2303
2483
|
}
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2484
|
+
|
|
2485
|
+
// src/tiktok.ts
|
|
2486
|
+
var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
|
|
2487
|
+
var pageAttemptCount = 12;
|
|
2488
|
+
async function resolveTiktok(input) {
|
|
2489
|
+
const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
|
|
2490
|
+
const id = videoId(pageUrl);
|
|
2491
|
+
if (!id) {
|
|
2492
|
+
throw new Error("TikTok video id not found");
|
|
2310
2493
|
}
|
|
2311
|
-
const
|
|
2312
|
-
const
|
|
2313
|
-
|
|
2314
|
-
|
|
2494
|
+
const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
|
|
2495
|
+
const user = author(page.item) ?? username(pageUrl) ?? "i";
|
|
2496
|
+
const headers = {
|
|
2497
|
+
referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
|
|
2498
|
+
"user-agent": page.userAgent
|
|
2499
|
+
};
|
|
2500
|
+
if (page.cookie) {
|
|
2501
|
+
headers.cookie = page.cookie;
|
|
2315
2502
|
}
|
|
2316
|
-
const
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2503
|
+
const items = mediaItems4(page.item, user, id, headers);
|
|
2504
|
+
if (items.length === 0) {
|
|
2505
|
+
throw new Error("TikTok media not found");
|
|
2506
|
+
}
|
|
2507
|
+
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
2508
|
+
}
|
|
2509
|
+
async function videoPage(net, id, photo) {
|
|
2510
|
+
for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
|
|
2511
|
+
const baseHeaders2 = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
|
|
2512
|
+
const userAgent2 = baseHeaders2["user-agent"];
|
|
2513
|
+
const headers2 = {
|
|
2514
|
+
...baseHeaders2,
|
|
2515
|
+
referer: "https://www.tiktok.com/",
|
|
2516
|
+
"sec-fetch-site": "same-origin"
|
|
2517
|
+
};
|
|
2518
|
+
try {
|
|
2519
|
+
return { ...await fetchVideoPage(net, id, headers2), userAgent: userAgent2 };
|
|
2520
|
+
} catch (error) {
|
|
2521
|
+
if (!recoverablePageError(error)) {
|
|
2522
|
+
throw error;
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
if (!photo) {
|
|
2527
|
+
throw new Error("TikTok hydration not found");
|
|
2323
2528
|
}
|
|
2529
|
+
const baseHeaders = navigationHeaders();
|
|
2530
|
+
const userAgent = baseHeaders["user-agent"];
|
|
2531
|
+
const headers = {
|
|
2532
|
+
...baseHeaders,
|
|
2533
|
+
referer: "https://www.tiktok.com/",
|
|
2534
|
+
"sec-fetch-site": "same-origin"
|
|
2535
|
+
};
|
|
2536
|
+
return { ...await fetchEmbedPage(net, id, headers), userAgent };
|
|
2537
|
+
}
|
|
2538
|
+
function tiktokMetadata(item) {
|
|
2539
|
+
const user = object(item.author) ? item.author : null;
|
|
2540
|
+
const stats = object(item.stats) ? item.stats : null;
|
|
2541
|
+
const music = object(item.music) ? item.music : null;
|
|
2542
|
+
const video = object(item.video) ? item.video : null;
|
|
2324
2543
|
return {
|
|
2325
|
-
text:
|
|
2544
|
+
text: string(item.desc) ?? undefined,
|
|
2326
2545
|
author: user ? {
|
|
2327
|
-
handle: string(user.
|
|
2328
|
-
name: string(user.
|
|
2329
|
-
verified: bool(user.verified)
|
|
2546
|
+
handle: string(user.uniqueId) ?? undefined,
|
|
2547
|
+
name: string(user.nickname) ?? undefined,
|
|
2548
|
+
verified: bool(user.verified)
|
|
2330
2549
|
} : undefined,
|
|
2331
|
-
createdAt:
|
|
2332
|
-
likeCount: count(
|
|
2333
|
-
commentCount: count(
|
|
2334
|
-
shareCount: count(
|
|
2335
|
-
viewCount: count(
|
|
2336
|
-
|
|
2337
|
-
|
|
2550
|
+
createdAt: isoFromEpochSeconds(item.createTime),
|
|
2551
|
+
likeCount: stats ? count(stats.diggCount) : undefined,
|
|
2552
|
+
commentCount: stats ? count(stats.commentCount) : undefined,
|
|
2553
|
+
shareCount: stats ? count(stats.shareCount) : undefined,
|
|
2554
|
+
viewCount: stats ? count(stats.playCount) : undefined,
|
|
2555
|
+
extra: {
|
|
2556
|
+
saveCount: stats ? count(stats.collectCount) : undefined,
|
|
2557
|
+
musicTitle: music ? string(music.title) ?? undefined : undefined,
|
|
2558
|
+
musicAuthor: music ? string(music.authorName) ?? undefined : undefined,
|
|
2559
|
+
region: string(item.locationCreated) ?? undefined,
|
|
2560
|
+
durationSeconds: video ? count(video.duration) : undefined
|
|
2561
|
+
}
|
|
2338
2562
|
};
|
|
2339
2563
|
}
|
|
2340
|
-
function
|
|
2341
|
-
const
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
return
|
|
2564
|
+
async function fetchVideoPage(net, id, headers) {
|
|
2565
|
+
const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
|
|
2566
|
+
headers
|
|
2567
|
+
});
|
|
2568
|
+
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
2345
2569
|
}
|
|
2346
|
-
function
|
|
2347
|
-
|
|
2570
|
+
async function fetchEmbedPage(net, id, headers) {
|
|
2571
|
+
const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
|
|
2572
|
+
headers
|
|
2573
|
+
});
|
|
2574
|
+
return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
|
|
2348
2575
|
}
|
|
2349
|
-
function
|
|
2350
|
-
|
|
2351
|
-
return null;
|
|
2352
|
-
const entities = object(tweet.entities) ? tweet.entities : null;
|
|
2353
|
-
const media = [
|
|
2354
|
-
...entities && Array.isArray(entities.media) ? entities.media : [],
|
|
2355
|
-
...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
|
|
2356
|
-
];
|
|
2357
|
-
const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
|
|
2358
|
-
while (true) {
|
|
2359
|
-
const trailing = text.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
|
|
2360
|
-
if (!trailing || !urls.has(trailing[1]))
|
|
2361
|
-
break;
|
|
2362
|
-
text = text.slice(0, trailing.index).trimEnd();
|
|
2363
|
-
}
|
|
2364
|
-
return text.replace(/&(?:amp|gt|lt);/g, (entity) => entity === "&" ? "&" : entity === ">" ? ">" : "<");
|
|
2576
|
+
function isShortlinkHost(hostname) {
|
|
2577
|
+
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
2365
2578
|
}
|
|
2366
|
-
function
|
|
2367
|
-
|
|
2368
|
-
|
|
2579
|
+
async function followShortlink(net, userAgent, input) {
|
|
2580
|
+
const url = asUrl(input);
|
|
2581
|
+
if (!isShortlinkHost(url.hostname)) {
|
|
2582
|
+
return input;
|
|
2369
2583
|
}
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
for (const [id, post] of posts) {
|
|
2375
|
-
if (!needsFullText(post)) {
|
|
2376
|
-
continue;
|
|
2377
|
-
}
|
|
2378
|
-
try {
|
|
2379
|
-
const response = await net(`https://api.fxtwitter.com/status/${id}`, {
|
|
2380
|
-
headers: { accept: "application/json", "user-agent": browserUserAgent() }
|
|
2381
|
-
}, 1);
|
|
2382
|
-
if (!response.ok) {
|
|
2383
|
-
continue;
|
|
2384
|
-
}
|
|
2385
|
-
const payload = await response.json();
|
|
2386
|
-
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
|
|
2387
|
-
continue;
|
|
2388
|
-
}
|
|
2389
|
-
let current = payload.tweet;
|
|
2390
|
-
const seen = new Set;
|
|
2391
|
-
while (current) {
|
|
2392
|
-
const currentId = string(current.id);
|
|
2393
|
-
if (!currentId || seen.has(currentId)) {
|
|
2394
|
-
break;
|
|
2395
|
-
}
|
|
2396
|
-
seen.add(currentId);
|
|
2397
|
-
const target = posts.get(currentId);
|
|
2398
|
-
const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
|
|
2399
|
-
const text = raw ?? string(current.text);
|
|
2400
|
-
if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
|
|
2401
|
-
target.full_text = text;
|
|
2402
|
-
}
|
|
2403
|
-
current = object(current.quote) ? current.quote : null;
|
|
2404
|
-
}
|
|
2405
|
-
} catch {}
|
|
2584
|
+
const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
|
|
2585
|
+
const location = response.headers.get("location");
|
|
2586
|
+
if (location) {
|
|
2587
|
+
return new URL(location, input).href;
|
|
2406
2588
|
}
|
|
2589
|
+
const html = await response.text();
|
|
2590
|
+
const href = html.match(/<a href="(https:\/\/[^"]+)"/)?.[1];
|
|
2591
|
+
return href ? new URL(href.split("?")[0]).href : input;
|
|
2407
2592
|
}
|
|
2408
|
-
function
|
|
2409
|
-
|
|
2410
|
-
const seen = new Set;
|
|
2411
|
-
const visit = (current, fallbackId) => {
|
|
2412
|
-
const id = string(current.id_str) ?? fallbackId;
|
|
2413
|
-
if (!id || seen.has(id))
|
|
2414
|
-
return;
|
|
2415
|
-
seen.add(id);
|
|
2416
|
-
tweets.push({ id, tweet: current });
|
|
2417
|
-
if (object(current.quoted_tweet))
|
|
2418
|
-
visit(current.quoted_tweet);
|
|
2419
|
-
if (includeParents && object(current.parent))
|
|
2420
|
-
visit(current.parent);
|
|
2421
|
-
};
|
|
2422
|
-
visit(tweet, rootId);
|
|
2423
|
-
return tweets;
|
|
2593
|
+
function videoId(input) {
|
|
2594
|
+
return asUrl(input).pathname.match(/(?:video|photo)\/(\d+)/)?.[1] ?? null;
|
|
2424
2595
|
}
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
if (!response.ok) {
|
|
2429
|
-
throw new Error(`Twitter syndication failed: ${response.status}`);
|
|
2596
|
+
function itemStruct(html) {
|
|
2597
|
+
if (html.includes("SlardarWAF") || html.includes("_wafchallengeid")) {
|
|
2598
|
+
throw new Error("TikTok WAF challenge");
|
|
2430
2599
|
}
|
|
2431
|
-
const
|
|
2432
|
-
|
|
2433
|
-
|
|
2600
|
+
const start = html.indexOf(marker);
|
|
2601
|
+
const end = start === -1 ? -1 : html.indexOf("</script>", start + marker.length);
|
|
2602
|
+
if (start === -1 || end === -1) {
|
|
2603
|
+
throw new Error("TikTok hydration not found");
|
|
2434
2604
|
}
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
const type = string(entry.type);
|
|
2443
|
-
if (type === "video" || type === "animated_gif") {
|
|
2444
|
-
return videoVariants(entry).map(({ bitrate, url }) => ({
|
|
2445
|
-
bitrate,
|
|
2446
|
-
item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }
|
|
2447
|
-
}));
|
|
2605
|
+
const parsed = JSON.parse(html.slice(start + marker.length, end));
|
|
2606
|
+
const scope = object(parsed) && object(parsed.__DEFAULT_SCOPE__) ? parsed.__DEFAULT_SCOPE__ : null;
|
|
2607
|
+
const detail = scope && object(scope["webapp.video-detail"]) ? scope["webapp.video-detail"] : null;
|
|
2608
|
+
const info = detail && object(detail.itemInfo) ? detail.itemInfo : null;
|
|
2609
|
+
const item = info && object(info.itemStruct) ? info.itemStruct : null;
|
|
2610
|
+
if (!item) {
|
|
2611
|
+
throw new Error("TikTok itemStruct not found");
|
|
2448
2612
|
}
|
|
2449
|
-
|
|
2450
|
-
return photo ? [{ bitrate: 0, item: { filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` } }] : [];
|
|
2613
|
+
return item;
|
|
2451
2614
|
}
|
|
2452
|
-
function
|
|
2453
|
-
const
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
return variants.flatMap((variant) => {
|
|
2457
|
-
const url = string(variant.url);
|
|
2458
|
-
if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
|
|
2459
|
-
return [];
|
|
2460
|
-
}
|
|
2461
|
-
seen.add(url);
|
|
2462
|
-
return [{ bitrate: number(variant.bitrate) ?? 0, url }];
|
|
2463
|
-
}).sort((left, right) => right.bitrate - left.bitrate);
|
|
2464
|
-
}
|
|
2465
|
-
async function sizeLimitedItems(input, groups) {
|
|
2466
|
-
if (groups.length === 0 || input.tryMaxBytes === undefined) {
|
|
2467
|
-
return groups.map((group) => group[0].item);
|
|
2615
|
+
function embedItemStruct(html, id) {
|
|
2616
|
+
const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
2617
|
+
if (!raw) {
|
|
2618
|
+
throw new Error("TikTok embed state not found");
|
|
2468
2619
|
}
|
|
2469
|
-
const
|
|
2470
|
-
const
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2620
|
+
const parsed = JSON.parse(raw);
|
|
2621
|
+
const source2 = object(parsed) && object(parsed.source) ? parsed.source : null;
|
|
2622
|
+
const data = source2 && object(source2.data) ? source2.data : null;
|
|
2623
|
+
const rawRoute = data ? data[`/embed/v2/${id}`] : null;
|
|
2624
|
+
const route = object(rawRoute) ? rawRoute : null;
|
|
2625
|
+
const videoData = route && object(route.videoData) ? route.videoData : null;
|
|
2626
|
+
const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
|
|
2627
|
+
if (!videoData || !item) {
|
|
2628
|
+
throw new Error("TikTok embed item not found");
|
|
2629
|
+
}
|
|
2630
|
+
const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
|
|
2631
|
+
const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
|
|
2632
|
+
const video = object(item.video) ? item.video : {};
|
|
2633
|
+
const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
|
|
2634
|
+
const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
|
|
2635
|
+
const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
|
|
2636
|
+
if (displayImages.length === 0) {
|
|
2637
|
+
throw new Error("TikTok video hydration not found");
|
|
2638
|
+
}
|
|
2639
|
+
return {
|
|
2640
|
+
...item,
|
|
2641
|
+
author: {
|
|
2642
|
+
nickname: string(authorInfo.nickName),
|
|
2643
|
+
uniqueId: string(authorInfo.uniqueId),
|
|
2644
|
+
verified: bool(authorInfo.verified)
|
|
2645
|
+
},
|
|
2646
|
+
desc: string(item.text),
|
|
2647
|
+
imagePost: {
|
|
2648
|
+
images: displayImages.map((image) => ({
|
|
2649
|
+
imageURL: {
|
|
2650
|
+
urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
|
|
2651
|
+
}
|
|
2652
|
+
}))
|
|
2653
|
+
},
|
|
2654
|
+
music: {
|
|
2655
|
+
authorName: string(musicInfo.authorName),
|
|
2656
|
+
playUrl: firstString(musicInfo.playUrl),
|
|
2657
|
+
title: string(musicInfo.musicName)
|
|
2658
|
+
},
|
|
2659
|
+
stats: {
|
|
2660
|
+
commentCount: item.commentCount,
|
|
2661
|
+
diggCount: item.diggCount,
|
|
2662
|
+
playCount: item.playCount,
|
|
2663
|
+
shareCount: item.shareCount
|
|
2664
|
+
},
|
|
2665
|
+
video: {
|
|
2666
|
+
duration: videoMeta.duration
|
|
2474
2667
|
}
|
|
2475
|
-
const pending = contentLength(input.net, candidate.item);
|
|
2476
|
-
sizes.set(candidate.item.url, pending);
|
|
2477
|
-
return pending;
|
|
2478
2668
|
};
|
|
2479
|
-
const highest = groups.map((group) => group[0]);
|
|
2480
|
-
const highestSizes = await Promise.all(highest.map(size));
|
|
2481
|
-
const highestTotal = knownTotal(highestSizes);
|
|
2482
|
-
if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
|
|
2483
|
-
return highest.map(({ item }) => item);
|
|
2484
|
-
}
|
|
2485
|
-
const sized = await Promise.all(groups.map(async (group) => Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) })))));
|
|
2486
|
-
const fitting = bestFitting(sized, input.tryMaxBytes);
|
|
2487
|
-
return (fitting ?? groups.map((group) => group.at(-1))).map(({ item }) => item);
|
|
2488
|
-
}
|
|
2489
|
-
function knownTotal(sizes) {
|
|
2490
|
-
return sizes.some((size) => size === null) ? null : sizes.reduce((total, size) => total + (size ?? 0), 0);
|
|
2491
2669
|
}
|
|
2492
|
-
function
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
];
|
|
2496
|
-
for (const group of groups) {
|
|
2497
|
-
selections = selections.flatMap((selection) => group.flatMap((candidate) => {
|
|
2498
|
-
if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
|
|
2499
|
-
return [];
|
|
2500
|
-
}
|
|
2501
|
-
return [{
|
|
2502
|
-
bytes: selection.bytes + candidate.bytes,
|
|
2503
|
-
candidates: [...selection.candidates, candidate],
|
|
2504
|
-
quality: selection.quality + candidate.bitrate
|
|
2505
|
-
}];
|
|
2506
|
-
}));
|
|
2507
|
-
if (selections.length === 0) {
|
|
2508
|
-
return null;
|
|
2509
|
-
}
|
|
2510
|
-
}
|
|
2511
|
-
const best = selections.reduce((current, selection) => selection.quality > current.quality ? selection : current);
|
|
2512
|
-
return best.candidates;
|
|
2670
|
+
function recoverablePageError(error) {
|
|
2671
|
+
const message = error instanceof Error ? error.message : "";
|
|
2672
|
+
return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
|
|
2513
2673
|
}
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
|
|
2517
|
-
const raw = response.ok ? response.headers.get("content-length") : null;
|
|
2518
|
-
if (!raw || !/^\d+$/.test(raw)) {
|
|
2519
|
-
return null;
|
|
2520
|
-
}
|
|
2521
|
-
const bytes = Number(raw);
|
|
2522
|
-
return Number.isSafeInteger(bytes) ? bytes : null;
|
|
2523
|
-
} catch {
|
|
2524
|
-
return null;
|
|
2525
|
-
}
|
|
2674
|
+
function firstString(value) {
|
|
2675
|
+
return Array.isArray(value) ? value.map(string).find((candidate) => candidate !== null) ?? null : string(value);
|
|
2526
2676
|
}
|
|
2527
|
-
function
|
|
2528
|
-
|
|
2677
|
+
function downloadUrl(item) {
|
|
2678
|
+
const video = object(item.video) ? item.video : null;
|
|
2679
|
+
return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
|
|
2529
2680
|
}
|
|
2530
|
-
|
|
2531
|
-
const
|
|
2532
|
-
if (
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
const comments = [];
|
|
2536
|
-
const seen = new Set;
|
|
2537
|
-
const cursors = new Set;
|
|
2538
|
-
let cursor = null;
|
|
2539
|
-
do {
|
|
2540
|
-
const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
|
|
2541
|
-
if (cursor)
|
|
2542
|
-
url.searchParams.set("cursor", cursor);
|
|
2543
|
-
const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
|
|
2544
|
-
if (!response.ok)
|
|
2545
|
-
throw new Error("Comment lookup failed");
|
|
2546
|
-
const payload = await response.json();
|
|
2547
|
-
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
|
|
2548
|
-
throw new Error("Invalid comments");
|
|
2549
|
-
const previousCount = comments.length;
|
|
2550
|
-
for (const reply of payload.replies) {
|
|
2551
|
-
if (!object(reply))
|
|
2552
|
-
continue;
|
|
2553
|
-
const replyId = string(reply.id);
|
|
2554
|
-
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
2555
|
-
if (!replyId || parent !== id || seen.has(replyId))
|
|
2556
|
-
continue;
|
|
2557
|
-
seen.add(replyId);
|
|
2558
|
-
comments.push(twitterComment(reply, replyId));
|
|
2559
|
-
if (comments.length >= limit)
|
|
2560
|
-
return comments;
|
|
2561
|
-
}
|
|
2562
|
-
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
2563
|
-
if (!cursor || cursors.has(cursor) || comments.length === previousCount)
|
|
2564
|
-
break;
|
|
2565
|
-
cursors.add(cursor);
|
|
2566
|
-
} while (comments.length < limit);
|
|
2567
|
-
return comments;
|
|
2568
|
-
} catch {
|
|
2569
|
-
return [];
|
|
2681
|
+
function mediaItems4(item, user, id, headers) {
|
|
2682
|
+
const images = imageItems3(item, user, id, headers);
|
|
2683
|
+
if (images.length > 0) {
|
|
2684
|
+
const audio = audioItem2(item, user, id, headers);
|
|
2685
|
+
return audio ? [...images, audio] : images;
|
|
2570
2686
|
}
|
|
2687
|
+
const url = downloadUrl(item);
|
|
2688
|
+
return url ? [{
|
|
2689
|
+
filename: filename(`tiktok_${user}_${id}.mp4`),
|
|
2690
|
+
headers,
|
|
2691
|
+
id,
|
|
2692
|
+
kind: "video",
|
|
2693
|
+
mime: "video/mp4",
|
|
2694
|
+
platform: "tiktok",
|
|
2695
|
+
url
|
|
2696
|
+
}] : [];
|
|
2571
2697
|
}
|
|
2572
|
-
function
|
|
2573
|
-
const
|
|
2574
|
-
const
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
const
|
|
2578
|
-
const
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2698
|
+
function imageItems3(item, user, id, headers) {
|
|
2699
|
+
const imagePost = object(item.imagePost) ? item.imagePost : null;
|
|
2700
|
+
const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
|
|
2701
|
+
return images.flatMap((image, index) => {
|
|
2702
|
+
const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
|
|
2703
|
+
const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
|
|
2704
|
+
const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
|
|
2705
|
+
return url ? [{
|
|
2706
|
+
filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
|
|
2707
|
+
headers,
|
|
2582
2708
|
id,
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
headers: { "user-agent": browserUserAgent() }
|
|
2589
|
-
}];
|
|
2709
|
+
kind: "image",
|
|
2710
|
+
mime: "image/jpeg",
|
|
2711
|
+
platform: "tiktok",
|
|
2712
|
+
url
|
|
2713
|
+
}] : [];
|
|
2590
2714
|
});
|
|
2591
|
-
return {
|
|
2592
|
-
id,
|
|
2593
|
-
url: `https://x.com/i/status/${id}`,
|
|
2594
|
-
items,
|
|
2595
|
-
metadata: {
|
|
2596
|
-
text: cleanTwitterText(string(reply.text), reply) ?? undefined,
|
|
2597
|
-
author: {
|
|
2598
|
-
handle: string(author2.screen_name) ?? undefined,
|
|
2599
|
-
name: string(author2.name) ?? undefined,
|
|
2600
|
-
verified: bool(verification.verified)
|
|
2601
|
-
},
|
|
2602
|
-
createdAt: isoFromDateString(reply.created_at),
|
|
2603
|
-
likeCount: count(reply.likes),
|
|
2604
|
-
commentCount: count(reply.replies),
|
|
2605
|
-
shareCount: count(reply.reposts),
|
|
2606
|
-
viewCount: count(reply.views),
|
|
2607
|
-
nsfw: bool(reply.possibly_sensitive)
|
|
2608
|
-
}
|
|
2609
|
-
};
|
|
2610
2715
|
}
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
var visionOsClient = {
|
|
2620
|
-
clientName: "VISIONOS",
|
|
2621
|
-
clientVersion: "1.02",
|
|
2622
|
-
deviceMake: "Apple",
|
|
2623
|
-
deviceModel: "RealityDevice17,1",
|
|
2624
|
-
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
|
|
2625
|
-
osName: "visionOS",
|
|
2626
|
-
osVersion: "26.5.23O471"
|
|
2627
|
-
};
|
|
2628
|
-
var browserCookie = "PREF=hl=en&tz=UTC; SOCS=CAI";
|
|
2629
|
-
async function resolveYoutube(input) {
|
|
2630
|
-
const id = youtubeVideoId(input.url);
|
|
2631
|
-
if (!id) {
|
|
2632
|
-
throw new Error("YouTube video id not found");
|
|
2633
|
-
}
|
|
2634
|
-
let session;
|
|
2635
|
-
try {
|
|
2636
|
-
session = await youtubeSession(input.net, id);
|
|
2637
|
-
const payload = await youtubePlayer(input.net, playerBody(id, session), playerHeaders(session));
|
|
2638
|
-
const streams = selectStreams(payload, input.preferredWidth);
|
|
2639
|
-
if (!streams) {
|
|
2640
|
-
throw new Error("YouTube mp4 stream not found");
|
|
2641
|
-
}
|
|
2642
|
-
const headers = { "user-agent": androidVrClient.userAgent };
|
|
2643
|
-
await Promise.all([streams.video, streams.audio].filter((url) => url !== null).map((url) => probeStream(input.net, url, headers)));
|
|
2644
|
-
return youtubeResult(id, payload, streams, headers);
|
|
2645
|
-
} catch (primaryError) {
|
|
2646
|
-
try {
|
|
2647
|
-
return await resolveVisionOs(input, id, session?.visitorData);
|
|
2648
|
-
} catch (fallbackError) {
|
|
2649
|
-
const message = (error) => error instanceof Error ? error.message : String(error);
|
|
2650
|
-
throw new Error(`YouTube resolution failed: ${message(primaryError)}; visionOS HLS: ${message(fallbackError)}`, { cause: fallbackError });
|
|
2651
|
-
}
|
|
2652
|
-
}
|
|
2653
|
-
}
|
|
2654
|
-
async function youtubePlayer(net, body, headers) {
|
|
2655
|
-
const response = await net("https://www.youtube.com/youtubei/v1/player?prettyPrint=false", {
|
|
2656
|
-
body: JSON.stringify(body),
|
|
2657
|
-
headers,
|
|
2658
|
-
method: "POST"
|
|
2659
|
-
});
|
|
2660
|
-
if (!response.ok) {
|
|
2661
|
-
throw new Error(`YouTube player failed: ${response.status}`);
|
|
2662
|
-
}
|
|
2663
|
-
const payload = await response.json();
|
|
2664
|
-
const status = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.status) : null;
|
|
2665
|
-
if (status !== "OK") {
|
|
2666
|
-
const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
|
|
2667
|
-
throw new Error(reason ?? "YouTube video unavailable");
|
|
2716
|
+
function audioItem2(item, user, id, headers) {
|
|
2717
|
+
const video = object(item.video) ? item.video : null;
|
|
2718
|
+
const music = object(item.music) ? item.music : null;
|
|
2719
|
+
const url = video ? string(video.playAddr) : null;
|
|
2720
|
+
const fallback = music ? string(music.playUrl) : null;
|
|
2721
|
+
const selected = url ?? fallback;
|
|
2722
|
+
if (!selected) {
|
|
2723
|
+
return null;
|
|
2668
2724
|
}
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
|
|
2673
|
-
const media = {
|
|
2674
|
-
filename: filename(`youtube_${title ?? id}_${id}.mp4`),
|
|
2725
|
+
const extension = selected.includes("mime_type=audio_mpeg") ? "mp3" : "m4a";
|
|
2726
|
+
return {
|
|
2727
|
+
filename: filename(`tiktok_${user}_${id}_audio.${extension}`),
|
|
2675
2728
|
headers,
|
|
2676
2729
|
id,
|
|
2677
|
-
kind: "
|
|
2678
|
-
mime: "
|
|
2679
|
-
platform: "
|
|
2680
|
-
url:
|
|
2681
|
-
...streams.hls ? { hls: true } : {},
|
|
2682
|
-
...streams.audio ? { audio: { headers, url: streams.audio } } : {}
|
|
2730
|
+
kind: "audio",
|
|
2731
|
+
mime: extension === "mp3" ? "audio/mpeg" : "audio/mp4",
|
|
2732
|
+
platform: "tiktok",
|
|
2733
|
+
url: selected
|
|
2683
2734
|
};
|
|
2684
|
-
return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
2685
2735
|
}
|
|
2686
|
-
|
|
2687
|
-
const
|
|
2688
|
-
|
|
2689
|
-
if (!response.ok) {
|
|
2690
|
-
throw new Error(`YouTube stream failed: ${response.status}`);
|
|
2691
|
-
}
|
|
2736
|
+
function author(item) {
|
|
2737
|
+
const user = object(item.author) ? item.author : null;
|
|
2738
|
+
return user ? string(user.uniqueId) : null;
|
|
2692
2739
|
}
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
const manifestUrl = streaming ? string(streaming.hlsManifestUrl) : null;
|
|
2709
|
-
if (!manifestUrl) {
|
|
2710
|
-
throw new Error("YouTube HLS manifest not found");
|
|
2740
|
+
function username(input) {
|
|
2741
|
+
return asUrl(input).pathname.match(/@([^/]+)/)?.[1] ?? null;
|
|
2742
|
+
}
|
|
2743
|
+
function cookieHeader(headers) {
|
|
2744
|
+
const getter = headers.getSetCookie;
|
|
2745
|
+
const setCookie = typeof getter === "function" ? getter.call(headers).join(",") : headers.get("set-cookie");
|
|
2746
|
+
const cookies = setCookie?.split(/,(?=[^;]+?=)/).map((part) => part.split(";")[0]?.trim()).filter((part) => Boolean(part));
|
|
2747
|
+
return cookies && cookies.length > 0 ? cookies.join("; ") : null;
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
// src/twitter.ts
|
|
2751
|
+
async function resolveTwitter(input) {
|
|
2752
|
+
const id = tweetId(input.url);
|
|
2753
|
+
if (!id) {
|
|
2754
|
+
throw new Error("Tweet id not found");
|
|
2711
2755
|
}
|
|
2712
|
-
const
|
|
2713
|
-
if (
|
|
2714
|
-
throw new Error(
|
|
2756
|
+
const tweet = await syndication(input.net, id);
|
|
2757
|
+
if (string(tweet.__typename) === "TweetTombstone") {
|
|
2758
|
+
throw new Error("Tweet is unavailable or age-restricted");
|
|
2715
2759
|
}
|
|
2716
|
-
|
|
2717
|
-
const
|
|
2718
|
-
const
|
|
2719
|
-
|
|
2720
|
-
|
|
2760
|
+
await expandTwitterText(input.net, tweet, id);
|
|
2761
|
+
const posts = twitterTweets(tweet, id);
|
|
2762
|
+
const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
|
|
2763
|
+
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
2764
|
+
return media.flatMap((entry, index) => {
|
|
2765
|
+
const candidates = twitterCandidates(entry, postId2, index + 1);
|
|
2766
|
+
return candidates.length > 0 ? [candidates] : [];
|
|
2767
|
+
});
|
|
2768
|
+
});
|
|
2769
|
+
for (const { id: postId2, tweet: post } of posts.toReversed()) {
|
|
2770
|
+
const card = twitterCardImage(post, postId2);
|
|
2771
|
+
if (card)
|
|
2772
|
+
groups.push([{ bitrate: 0, item: card }]);
|
|
2721
2773
|
}
|
|
2722
|
-
const
|
|
2723
|
-
|
|
2724
|
-
|
|
2774
|
+
const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
|
|
2775
|
+
return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
2776
|
+
}
|
|
2777
|
+
function twitterMetadata(tweet) {
|
|
2778
|
+
const user = object(tweet.user) ? tweet.user : null;
|
|
2779
|
+
const extra = {};
|
|
2780
|
+
const lang = string(tweet.lang);
|
|
2781
|
+
if (lang) {
|
|
2782
|
+
extra.lang = lang;
|
|
2725
2783
|
}
|
|
2726
|
-
|
|
2727
|
-
|
|
2784
|
+
const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
|
|
2785
|
+
const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
|
|
2786
|
+
if (quotedTweet && quotedId) {
|
|
2787
|
+
extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
|
|
2788
|
+
}
|
|
2789
|
+
const parent = object(tweet.parent) ? tweet.parent : null;
|
|
2790
|
+
const parentId = parent ? string(parent.id_str) : null;
|
|
2791
|
+
const replyToId = string(tweet.in_reply_to_status_id_str) ?? parentId;
|
|
2792
|
+
if (replyToId)
|
|
2793
|
+
extra.replyToId = replyToId;
|
|
2794
|
+
if (parent && parentId) {
|
|
2795
|
+
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
2728
2796
|
}
|
|
2729
|
-
return youtubeResult(id, payload, { video: variant.url, audio: master.audio[variant.audioGroup], hls: true }, headers);
|
|
2730
|
-
}
|
|
2731
|
-
function youtubeMetadata(payload) {
|
|
2732
|
-
const details = object(payload) && object(payload.videoDetails) ? payload.videoDetails : null;
|
|
2733
|
-
const renderer = object(payload) && object(payload.microformat) && object(payload.microformat.playerMicroformatRenderer) ? payload.microformat.playerMicroformatRenderer : null;
|
|
2734
|
-
const author2 = details ? string(details.author) : null;
|
|
2735
|
-
const keywords = details && Array.isArray(details.keywords) ? details.keywords.filter((keyword) => typeof keyword === "string") : [];
|
|
2736
2797
|
return {
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2798
|
+
text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
|
|
2799
|
+
author: user ? {
|
|
2800
|
+
handle: string(user.screen_name) ?? undefined,
|
|
2801
|
+
name: string(user.name) ?? undefined,
|
|
2802
|
+
verified: bool(user.verified) ?? bool(user.is_blue_verified)
|
|
2803
|
+
} : undefined,
|
|
2804
|
+
createdAt: isoFromDateString(tweet.created_at),
|
|
2805
|
+
likeCount: count(tweet.favorite_count),
|
|
2806
|
+
commentCount: count(tweet.conversation_count),
|
|
2807
|
+
shareCount: count(tweet.retweet_count),
|
|
2808
|
+
viewCount: count(tweet.view_count),
|
|
2809
|
+
nsfw: bool(tweet.possibly_sensitive),
|
|
2810
|
+
extra
|
|
2747
2811
|
};
|
|
2748
2812
|
}
|
|
2749
|
-
function
|
|
2750
|
-
const
|
|
2751
|
-
const
|
|
2752
|
-
const
|
|
2753
|
-
|
|
2754
|
-
return { video, audio };
|
|
2755
|
-
}
|
|
2756
|
-
const progressive = selectFormat(payload);
|
|
2757
|
-
return progressive ? { video: progressive, audio: null } : null;
|
|
2758
|
-
}
|
|
2759
|
-
function adaptiveFormats(payload) {
|
|
2760
|
-
const root = object(payload) ? payload : null;
|
|
2761
|
-
const streaming = root && object(root.streamingData) ? root.streamingData : null;
|
|
2762
|
-
return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
|
|
2813
|
+
function noteText(tweet) {
|
|
2814
|
+
const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
|
|
2815
|
+
const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
|
|
2816
|
+
const result = results && object(results.result) ? results.result : null;
|
|
2817
|
+
return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
|
|
2763
2818
|
}
|
|
2764
|
-
function
|
|
2765
|
-
|
|
2766
|
-
const mimeType2 = string(format.mimeType);
|
|
2767
|
-
return Boolean(string(format.url)) && mimeType2?.startsWith("video/mp4") === true && mimeType2.includes("avc1");
|
|
2768
|
-
});
|
|
2769
|
-
const best = candidates.reduce((current, format) => {
|
|
2770
|
-
if (!current) {
|
|
2771
|
-
return format;
|
|
2772
|
-
}
|
|
2773
|
-
const width = number(format.width) ?? 0;
|
|
2774
|
-
const currentWidth = number(current.width) ?? 0;
|
|
2775
|
-
return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
|
|
2776
|
-
}, null);
|
|
2777
|
-
return best ? string(best.url) : null;
|
|
2819
|
+
function twitterText(tweet) {
|
|
2820
|
+
return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
|
|
2778
2821
|
}
|
|
2779
|
-
function
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2822
|
+
function cleanTwitterText(text2, tweet) {
|
|
2823
|
+
if (text2 === null)
|
|
2824
|
+
return null;
|
|
2825
|
+
const entities = object(tweet.entities) ? tweet.entities : null;
|
|
2826
|
+
const media = [
|
|
2827
|
+
...entities && Array.isArray(entities.media) ? entities.media : [],
|
|
2828
|
+
...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
|
|
2829
|
+
];
|
|
2830
|
+
const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
|
|
2831
|
+
while (true) {
|
|
2832
|
+
const trailing = text2.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
|
|
2833
|
+
if (!trailing || !urls.has(trailing[1]))
|
|
2834
|
+
break;
|
|
2835
|
+
text2 = text2.slice(0, trailing.index).trimEnd();
|
|
2836
|
+
}
|
|
2837
|
+
return text2.replace(/&(?:amp|gt|lt);/g, (entity) => entity === "&" ? "&" : entity === ">" ? ">" : "<");
|
|
2792
2838
|
}
|
|
2793
|
-
function
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
return 2;
|
|
2839
|
+
function needsFullText(tweet) {
|
|
2840
|
+
if (noteText(tweet) || string(tweet.full_text)) {
|
|
2841
|
+
return false;
|
|
2797
2842
|
}
|
|
2798
|
-
return
|
|
2843
|
+
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
2799
2844
|
}
|
|
2800
|
-
function
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
osName: "Android",
|
|
2813
|
-
osVersion: "12L",
|
|
2814
|
-
timeZone: "UTC",
|
|
2815
|
-
userAgent: androidVrClient.userAgent,
|
|
2816
|
-
utcOffsetMinutes: 0,
|
|
2817
|
-
visitorData: session.visitorData
|
|
2845
|
+
async function expandTwitterText(net, tweet, rootId) {
|
|
2846
|
+
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
2847
|
+
for (const [id, post] of posts) {
|
|
2848
|
+
if (!needsFullText(post)) {
|
|
2849
|
+
continue;
|
|
2850
|
+
}
|
|
2851
|
+
try {
|
|
2852
|
+
const response = await net(`https://api.fxtwitter.com/status/${id}`, {
|
|
2853
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() }
|
|
2854
|
+
}, 1);
|
|
2855
|
+
if (!response.ok) {
|
|
2856
|
+
continue;
|
|
2818
2857
|
}
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
html5Preference: "HTML5_PREF_WANTS",
|
|
2823
|
-
signatureTimestamp: session.signatureTimestamp
|
|
2858
|
+
const payload = await response.json();
|
|
2859
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
|
|
2860
|
+
continue;
|
|
2824
2861
|
}
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2862
|
+
let current = payload.tweet;
|
|
2863
|
+
const seen = new Set;
|
|
2864
|
+
while (current) {
|
|
2865
|
+
const currentId = string(current.id);
|
|
2866
|
+
if (!currentId || seen.has(currentId)) {
|
|
2867
|
+
break;
|
|
2868
|
+
}
|
|
2869
|
+
seen.add(currentId);
|
|
2870
|
+
const target = posts.get(currentId);
|
|
2871
|
+
const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
|
|
2872
|
+
const text2 = raw ?? string(current.text);
|
|
2873
|
+
if (target && needsFullText(target) && text2 && text2.length > (twitterText(target)?.length ?? 0)) {
|
|
2874
|
+
target.full_text = text2;
|
|
2875
|
+
}
|
|
2876
|
+
current = object(current.quote) ? current.quote : null;
|
|
2877
|
+
}
|
|
2878
|
+
} catch {}
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
function twitterTweets(tweet, rootId, includeParents = false) {
|
|
2882
|
+
const tweets = [];
|
|
2883
|
+
const seen = new Set;
|
|
2884
|
+
const visit = (current, fallbackId) => {
|
|
2885
|
+
const id = string(current.id_str) ?? fallbackId;
|
|
2886
|
+
if (!id || seen.has(id))
|
|
2887
|
+
return;
|
|
2888
|
+
seen.add(id);
|
|
2889
|
+
tweets.push({ id, tweet: current });
|
|
2890
|
+
if (object(current.quoted_tweet))
|
|
2891
|
+
visit(current.quoted_tweet);
|
|
2892
|
+
if (includeParents && object(current.parent))
|
|
2893
|
+
visit(current.parent);
|
|
2828
2894
|
};
|
|
2895
|
+
visit(tweet, rootId);
|
|
2896
|
+
return tweets;
|
|
2829
2897
|
}
|
|
2830
|
-
async function
|
|
2831
|
-
const
|
|
2832
|
-
|
|
2833
|
-
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
2834
|
-
"accept-language": "en-us,en;q=0.5",
|
|
2835
|
-
cookie: browserCookie,
|
|
2836
|
-
"sec-fetch-mode": "navigate",
|
|
2837
|
-
"user-agent": browserUserAgent()
|
|
2838
|
-
}
|
|
2839
|
-
});
|
|
2898
|
+
async function syndication(net, id) {
|
|
2899
|
+
const url = `https://cdn.syndication.twimg.com/tweet-result?id=${id}&lang=en&token=${syndicationToken(id)}`;
|
|
2900
|
+
const response = await net(url, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
|
|
2840
2901
|
if (!response.ok) {
|
|
2841
|
-
throw new Error(`
|
|
2902
|
+
throw new Error(`Twitter syndication failed: ${response.status}`);
|
|
2842
2903
|
}
|
|
2843
|
-
const
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
throw new Error("YouTube visitor data not found");
|
|
2904
|
+
const payload = await response.json();
|
|
2905
|
+
if (!object(payload)) {
|
|
2906
|
+
throw new Error("Twitter response invalid");
|
|
2847
2907
|
}
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2908
|
+
return payload;
|
|
2909
|
+
}
|
|
2910
|
+
function syndicationToken(id) {
|
|
2911
|
+
return (Number(id) / 1000000000000000 * Math.PI).toString(36).replace(/(0+|\.)/g, "");
|
|
2912
|
+
}
|
|
2913
|
+
function twitterCandidates(entry, id, index) {
|
|
2914
|
+
const headers = { "user-agent": browserUserAgent() };
|
|
2915
|
+
const type = string(entry.type);
|
|
2916
|
+
if (type === "video" || type === "animated_gif") {
|
|
2917
|
+
return videoVariants(entry).map(({ bitrate, url }) => ({
|
|
2918
|
+
bitrate,
|
|
2919
|
+
item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }
|
|
2920
|
+
}));
|
|
2851
2921
|
}
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
signatureTimestamp: await signatureTimestamp(net, playerPath),
|
|
2855
|
-
visitorData
|
|
2856
|
-
};
|
|
2922
|
+
const photo = string(entry.media_url_https);
|
|
2923
|
+
return photo ? [{ bitrate: 0, item: { filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` } }] : [];
|
|
2857
2924
|
}
|
|
2858
|
-
function
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2925
|
+
function twitterCardImage(tweet, id) {
|
|
2926
|
+
const card = object(tweet.card) ? tweet.card : null;
|
|
2927
|
+
if (!card || card.name !== "summary" && card.name !== "summary_large_image")
|
|
2928
|
+
return null;
|
|
2929
|
+
const bindings = object(card.binding_values) ? card.binding_values : {};
|
|
2930
|
+
const images = Object.entries(bindings).flatMap(([key, value]) => {
|
|
2931
|
+
if (!/^(photo_image_full_size|summary_photo_image|thumbnail_image)(_(original|x_large|large|small))?$/.test(key) || !object(value) || !object(value.image_value))
|
|
2932
|
+
return [];
|
|
2933
|
+
const image2 = value.image_value;
|
|
2934
|
+
const rawUrl = string(image2.url);
|
|
2935
|
+
if (!rawUrl)
|
|
2936
|
+
return [];
|
|
2937
|
+
try {
|
|
2938
|
+
const url = new URL(rawUrl);
|
|
2939
|
+
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
2940
|
+
return [];
|
|
2941
|
+
const format = (url.searchParams.get("format") ?? url.pathname.match(/\.(jpe?g|png|webp|gif)$/i)?.[1] ?? "jpg").toLowerCase();
|
|
2942
|
+
const extension = format === "jpeg" ? "jpg" : format;
|
|
2943
|
+
if (!["jpg", "png", "webp", "gif"].includes(extension))
|
|
2944
|
+
return [];
|
|
2945
|
+
return [{
|
|
2946
|
+
url: url.href,
|
|
2947
|
+
extension,
|
|
2948
|
+
original: key.endsWith("_original") ? 1 : 0,
|
|
2949
|
+
area: Math.max(0, number(image2.width) ?? 0) * Math.max(0, number(image2.height) ?? 0)
|
|
2950
|
+
}];
|
|
2951
|
+
} catch {
|
|
2952
|
+
return [];
|
|
2953
|
+
}
|
|
2954
|
+
}).sort((a, b) => b.original - a.original || b.area - a.area);
|
|
2955
|
+
const image = images[0];
|
|
2956
|
+
return image ? {
|
|
2957
|
+
filename: filename(`twitter_${id}_preview.${image.extension}`),
|
|
2958
|
+
headers: { "user-agent": browserUserAgent() },
|
|
2959
|
+
id,
|
|
2960
|
+
kind: "image",
|
|
2961
|
+
mime: `image/${image.extension === "jpg" ? "jpeg" : image.extension}`,
|
|
2962
|
+
platform: "twitter",
|
|
2963
|
+
role: "linkPreview",
|
|
2964
|
+
url: image.url
|
|
2965
|
+
} : null;
|
|
2868
2966
|
}
|
|
2869
|
-
function
|
|
2870
|
-
const
|
|
2871
|
-
const
|
|
2872
|
-
const
|
|
2873
|
-
return
|
|
2967
|
+
function videoVariants(entry) {
|
|
2968
|
+
const info = object(entry.video_info) ? entry.video_info : null;
|
|
2969
|
+
const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
|
|
2970
|
+
const seen = new Set;
|
|
2971
|
+
return variants.flatMap((variant) => {
|
|
2972
|
+
const url = string(variant.url);
|
|
2973
|
+
if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
|
|
2974
|
+
return [];
|
|
2975
|
+
}
|
|
2976
|
+
seen.add(url);
|
|
2977
|
+
return [{ bitrate: number(variant.bitrate) ?? 0, url }];
|
|
2978
|
+
}).sort((left, right) => right.bitrate - left.bitrate);
|
|
2874
2979
|
}
|
|
2875
|
-
async function
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
});
|
|
2879
|
-
if (!response.ok) {
|
|
2880
|
-
throw new Error(`YouTube player failed: ${response.status}`);
|
|
2980
|
+
async function sizeLimitedItems(input, groups) {
|
|
2981
|
+
if (groups.length === 0 || input.tryMaxBytes === undefined) {
|
|
2982
|
+
return groups.map((group) => group[0].item);
|
|
2881
2983
|
}
|
|
2882
|
-
const
|
|
2883
|
-
const
|
|
2884
|
-
|
|
2885
|
-
|
|
2984
|
+
const sizes = new Map;
|
|
2985
|
+
const size = (candidate) => {
|
|
2986
|
+
const existing = sizes.get(candidate.item.url);
|
|
2987
|
+
if (existing) {
|
|
2988
|
+
return existing;
|
|
2989
|
+
}
|
|
2990
|
+
const pending = contentLength(input.net, candidate.item);
|
|
2991
|
+
sizes.set(candidate.item.url, pending);
|
|
2992
|
+
return pending;
|
|
2993
|
+
};
|
|
2994
|
+
const highest = groups.map((group) => group[0]);
|
|
2995
|
+
const highestSizes = await Promise.all(highest.map(size));
|
|
2996
|
+
const highestTotal = knownTotal(highestSizes);
|
|
2997
|
+
if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
|
|
2998
|
+
return highest.map(({ item }) => item);
|
|
2886
2999
|
}
|
|
2887
|
-
|
|
3000
|
+
const sized = await Promise.all(groups.map(async (group) => Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) })))));
|
|
3001
|
+
const fitting = bestFitting(sized, input.tryMaxBytes);
|
|
3002
|
+
return (fitting ?? groups.map((group) => group.at(-1))).map(({ item }) => item);
|
|
2888
3003
|
}
|
|
2889
|
-
function
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
3004
|
+
function knownTotal(sizes) {
|
|
3005
|
+
return sizes.some((size) => size === null) ? null : sizes.reduce((total, size) => total + (size ?? 0), 0);
|
|
3006
|
+
}
|
|
3007
|
+
function bestFitting(groups, maxBytes) {
|
|
3008
|
+
let selections = [
|
|
3009
|
+
{ bytes: 0, candidates: [], quality: 0 }
|
|
3010
|
+
];
|
|
3011
|
+
for (const group of groups) {
|
|
3012
|
+
selections = selections.flatMap((selection) => group.flatMap((candidate) => {
|
|
3013
|
+
if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
|
|
3014
|
+
return [];
|
|
3015
|
+
}
|
|
3016
|
+
return [{
|
|
3017
|
+
bytes: selection.bytes + candidate.bytes,
|
|
3018
|
+
candidates: [...selection.candidates, candidate],
|
|
3019
|
+
quality: selection.quality + candidate.bitrate
|
|
3020
|
+
}];
|
|
3021
|
+
}));
|
|
3022
|
+
if (selections.length === 0) {
|
|
3023
|
+
return null;
|
|
3024
|
+
}
|
|
2893
3025
|
}
|
|
2894
|
-
const
|
|
2895
|
-
|
|
2896
|
-
|
|
3026
|
+
const best = selections.reduce((current, selection) => selection.quality > current.quality ? selection : current);
|
|
3027
|
+
return best.candidates;
|
|
3028
|
+
}
|
|
3029
|
+
async function contentLength(net, item) {
|
|
3030
|
+
try {
|
|
3031
|
+
const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
|
|
3032
|
+
const raw = response.ok ? response.headers.get("content-length") : null;
|
|
3033
|
+
if (!raw || !/^\d+$/.test(raw)) {
|
|
3034
|
+
return null;
|
|
3035
|
+
}
|
|
3036
|
+
const bytes = Number(raw);
|
|
3037
|
+
return Number.isSafeInteger(bytes) ? bytes : null;
|
|
3038
|
+
} catch {
|
|
3039
|
+
return null;
|
|
2897
3040
|
}
|
|
2898
|
-
const parts = url.pathname.split("/").filter(Boolean);
|
|
2899
|
-
const index = parts.findIndex((part) => part === "shorts" || part === "live" || part === "embed");
|
|
2900
|
-
return index >= 0 ? cleanId(parts[index + 1]) : null;
|
|
2901
3041
|
}
|
|
2902
|
-
function
|
|
2903
|
-
return
|
|
3042
|
+
function tweetId(input) {
|
|
3043
|
+
return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
|
|
2904
3044
|
}
|
|
2905
|
-
function
|
|
2906
|
-
const
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
3045
|
+
async function twitterComments(input, id) {
|
|
3046
|
+
const limit = input.comments;
|
|
3047
|
+
if (!limit || !Number.isSafeInteger(limit) || limit < 0)
|
|
3048
|
+
return [];
|
|
3049
|
+
try {
|
|
3050
|
+
const comments = [];
|
|
3051
|
+
const seen = new Set;
|
|
3052
|
+
const cursors = new Set;
|
|
3053
|
+
let cursor = null;
|
|
3054
|
+
do {
|
|
3055
|
+
const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
|
|
3056
|
+
if (cursor)
|
|
3057
|
+
url.searchParams.set("cursor", cursor);
|
|
3058
|
+
const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
|
|
3059
|
+
if (!response.ok)
|
|
3060
|
+
throw new Error("Comment lookup failed");
|
|
3061
|
+
const payload = await response.json();
|
|
3062
|
+
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
|
|
3063
|
+
throw new Error("Invalid comments");
|
|
3064
|
+
const previousCount = comments.length;
|
|
3065
|
+
for (const reply of payload.replies) {
|
|
3066
|
+
if (!object(reply))
|
|
3067
|
+
continue;
|
|
3068
|
+
const replyId = string(reply.id);
|
|
3069
|
+
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
3070
|
+
if (!replyId || parent !== id || seen.has(replyId))
|
|
3071
|
+
continue;
|
|
3072
|
+
seen.add(replyId);
|
|
3073
|
+
comments.push(twitterComment(reply, replyId));
|
|
3074
|
+
if (comments.length >= limit)
|
|
3075
|
+
return comments;
|
|
3076
|
+
}
|
|
3077
|
+
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
3078
|
+
if (!cursor || cursors.has(cursor) || comments.length === previousCount)
|
|
3079
|
+
break;
|
|
3080
|
+
cursors.add(cursor);
|
|
3081
|
+
} while (comments.length < limit);
|
|
3082
|
+
return comments;
|
|
3083
|
+
} catch {
|
|
3084
|
+
return [];
|
|
3085
|
+
}
|
|
2911
3086
|
}
|
|
2912
|
-
function
|
|
2913
|
-
|
|
3087
|
+
function twitterComment(reply, id) {
|
|
3088
|
+
const author2 = object(reply.author) ? reply.author : {};
|
|
3089
|
+
const verification = object(author2.verification) ? author2.verification : {};
|
|
3090
|
+
const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
|
|
3091
|
+
const items = media.flatMap((entry, index) => {
|
|
3092
|
+
const url = string(entry.url);
|
|
3093
|
+
const photo = entry.type === "photo";
|
|
3094
|
+
if (!url || !photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")
|
|
3095
|
+
return [];
|
|
3096
|
+
return [{
|
|
3097
|
+
id,
|
|
3098
|
+
platform: "twitter",
|
|
3099
|
+
filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
|
|
3100
|
+
kind: photo ? "image" : "video",
|
|
3101
|
+
mime: photo ? "image/jpeg" : "video/mp4",
|
|
3102
|
+
url,
|
|
3103
|
+
headers: { "user-agent": browserUserAgent() }
|
|
3104
|
+
}];
|
|
3105
|
+
});
|
|
3106
|
+
return {
|
|
3107
|
+
id,
|
|
3108
|
+
url: `https://x.com/i/status/${id}`,
|
|
3109
|
+
items,
|
|
3110
|
+
metadata: {
|
|
3111
|
+
text: cleanTwitterText(string(reply.text), reply) ?? undefined,
|
|
3112
|
+
author: {
|
|
3113
|
+
handle: string(author2.screen_name) ?? undefined,
|
|
3114
|
+
name: string(author2.name) ?? undefined,
|
|
3115
|
+
verified: bool(verification.verified)
|
|
3116
|
+
},
|
|
3117
|
+
createdAt: isoFromDateString(reply.created_at),
|
|
3118
|
+
likeCount: count(reply.likes),
|
|
3119
|
+
commentCount: count(reply.replies),
|
|
3120
|
+
shareCount: count(reply.reposts),
|
|
3121
|
+
viewCount: count(reply.views),
|
|
3122
|
+
nsfw: bool(reply.possibly_sensitive)
|
|
3123
|
+
}
|
|
3124
|
+
};
|
|
2914
3125
|
}
|
|
2915
3126
|
|
|
2916
3127
|
// src/postfetch.ts
|
|
@@ -2946,6 +3157,8 @@ async function resolve(context) {
|
|
|
2946
3157
|
return resolveReddit(context);
|
|
2947
3158
|
case "soundcloud":
|
|
2948
3159
|
return resolveSoundcloud(context);
|
|
3160
|
+
case "spotify":
|
|
3161
|
+
return resolveSpotify(context);
|
|
2949
3162
|
case "tiktok":
|
|
2950
3163
|
return resolveTiktok(context);
|
|
2951
3164
|
case "twitter":
|
|
@@ -3026,6 +3239,9 @@ function detect(url) {
|
|
|
3026
3239
|
if (host.includes("soundcloud.com") || host === "snd.sc") {
|
|
3027
3240
|
return "soundcloud";
|
|
3028
3241
|
}
|
|
3242
|
+
if (host === "open.spotify.com") {
|
|
3243
|
+
return "spotify";
|
|
3244
|
+
}
|
|
3029
3245
|
if (host.includes("youtube.com") || host === "youtu.be") {
|
|
3030
3246
|
return "youtube";
|
|
3031
3247
|
}
|
|
@@ -3035,7 +3251,7 @@ function detect(url) {
|
|
|
3035
3251
|
if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
|
|
3036
3252
|
return "twitter";
|
|
3037
3253
|
}
|
|
3038
|
-
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
3254
|
+
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube URLs are supported");
|
|
3039
3255
|
}
|
|
3040
3256
|
// src/mp4-remux.ts
|
|
3041
3257
|
async function prepareMp4(bytes, ffmpegPath = "ffmpeg", ffprobePath = "ffprobe", runtime) {
|
|
@@ -3457,10 +3673,15 @@ async function archive(result, options = {}) {
|
|
|
3457
3673
|
return { bytes: zip(files), filename: result.archiveFilename, mime: "application/zip" };
|
|
3458
3674
|
}
|
|
3459
3675
|
async function toResponse(result, options = {}) {
|
|
3460
|
-
|
|
3461
|
-
|
|
3676
|
+
const response = result.items.length === 1 ? await singleResponse(result.items[0], options) : await archiveResponse(result, options);
|
|
3677
|
+
if (result.platform === "spotify") {
|
|
3678
|
+
response.headers.set("x-media-platform", result.platform);
|
|
3679
|
+
response.headers.set("x-media-id", result.id);
|
|
3680
|
+
const youtubeUrl = result.metadata?.extra?.youtubeUrl;
|
|
3681
|
+
if (youtubeUrl)
|
|
3682
|
+
response.headers.set("x-media-youtube-url", youtubeUrl);
|
|
3462
3683
|
}
|
|
3463
|
-
return
|
|
3684
|
+
return response;
|
|
3464
3685
|
}
|
|
3465
3686
|
async function singleResponse(item, options) {
|
|
3466
3687
|
if (buffered(item)) {
|