@postfetch/core 0.12.5 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +1009 -836
- package/dist/internal.d.ts +11 -2
- package/dist/spotify.d.ts +3 -0
- package/package.json +3 -2
- package/src/download.ts +9 -3
- package/src/index.ts +4 -2
- package/src/internal.ts +12 -1
- package/src/postfetch.ts +7 -1
- package/src/spotify.ts +170 -0
package/dist/index.js
CHANGED
|
@@ -2015,950 +2015,1113 @@ function trackId(track) {
|
|
|
2015
2015
|
return string(track.permalink) ?? "track";
|
|
2016
2016
|
}
|
|
2017
2017
|
|
|
2018
|
-
// src/
|
|
2019
|
-
var
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2018
|
+
// src/youtube.ts
|
|
2019
|
+
var androidVrClient = {
|
|
2020
|
+
name: "ANDROID_VR",
|
|
2021
|
+
number: "28",
|
|
2022
|
+
userAgent: "com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip",
|
|
2023
|
+
version: "1.65.10"
|
|
2024
|
+
};
|
|
2025
|
+
var visionOsClient = {
|
|
2026
|
+
clientName: "VISIONOS",
|
|
2027
|
+
clientVersion: "1.02",
|
|
2028
|
+
deviceMake: "Apple",
|
|
2029
|
+
deviceModel: "RealityDevice17,1",
|
|
2030
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
|
|
2031
|
+
osName: "visionOS",
|
|
2032
|
+
osVersion: "26.5.23O471"
|
|
2033
|
+
};
|
|
2034
|
+
var browserCookie = "PREF=hl=en&tz=UTC; SOCS=CAI";
|
|
2035
|
+
async function resolveYoutube(input) {
|
|
2036
|
+
const id = youtubeVideoId(input.url);
|
|
2024
2037
|
if (!id) {
|
|
2025
|
-
throw new Error("
|
|
2026
|
-
}
|
|
2027
|
-
const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
|
|
2028
|
-
const user = author(page.item) ?? username(pageUrl) ?? "i";
|
|
2029
|
-
const headers = {
|
|
2030
|
-
referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
|
|
2031
|
-
"user-agent": page.userAgent
|
|
2032
|
-
};
|
|
2033
|
-
if (page.cookie) {
|
|
2034
|
-
headers.cookie = page.cookie;
|
|
2035
|
-
}
|
|
2036
|
-
const items = mediaItems4(page.item, user, id, headers);
|
|
2037
|
-
if (items.length === 0) {
|
|
2038
|
-
throw new Error("TikTok media not found");
|
|
2038
|
+
throw new Error("YouTube video id not found");
|
|
2039
2039
|
}
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
const
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2040
|
+
let session;
|
|
2041
|
+
try {
|
|
2042
|
+
session = await youtubeSession(input.net, id);
|
|
2043
|
+
const payload = await youtubePlayer(input.net, playerBody(id, session), playerHeaders(session));
|
|
2044
|
+
const streams = selectStreams(payload, input.preferredWidth);
|
|
2045
|
+
if (!streams) {
|
|
2046
|
+
throw new Error("YouTube mp4 stream not found");
|
|
2047
|
+
}
|
|
2048
|
+
const headers = { "user-agent": androidVrClient.userAgent };
|
|
2049
|
+
await Promise.all([streams.video, streams.audio].filter((url) => url !== null).map((url) => probeStream(input.net, url, headers)));
|
|
2050
|
+
return youtubeResult(id, payload, streams, headers);
|
|
2051
|
+
} catch (primaryError) {
|
|
2051
2052
|
try {
|
|
2052
|
-
return
|
|
2053
|
-
} catch (
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
}
|
|
2053
|
+
return await resolveVisionOs(input, id, session?.visitorData);
|
|
2054
|
+
} catch (fallbackError) {
|
|
2055
|
+
const message = (error) => error instanceof Error ? error.message : String(error);
|
|
2056
|
+
throw new Error(`YouTube resolution failed: ${message(primaryError)}; visionOS HLS: ${message(fallbackError)}`, { cause: fallbackError });
|
|
2057
2057
|
}
|
|
2058
2058
|
}
|
|
2059
|
-
|
|
2060
|
-
|
|
2059
|
+
}
|
|
2060
|
+
async function youtubePlayer(net, body, headers) {
|
|
2061
|
+
const response = await net("https://www.youtube.com/youtubei/v1/player?prettyPrint=false", {
|
|
2062
|
+
body: JSON.stringify(body),
|
|
2063
|
+
headers,
|
|
2064
|
+
method: "POST"
|
|
2065
|
+
});
|
|
2066
|
+
if (!response.ok) {
|
|
2067
|
+
throw new Error(`YouTube player failed: ${response.status}`);
|
|
2061
2068
|
}
|
|
2062
|
-
const
|
|
2063
|
-
const
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2069
|
+
const payload = await response.json();
|
|
2070
|
+
const status = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.status) : null;
|
|
2071
|
+
if (status !== "OK") {
|
|
2072
|
+
const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
|
|
2073
|
+
throw new Error(reason ?? "YouTube video unavailable");
|
|
2074
|
+
}
|
|
2075
|
+
return payload;
|
|
2076
|
+
}
|
|
2077
|
+
function youtubeResult(id, payload, streams, headers) {
|
|
2078
|
+
const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
|
|
2079
|
+
const media = {
|
|
2080
|
+
filename: filename(`youtube_${title ?? id}_${id}.mp4`),
|
|
2081
|
+
headers,
|
|
2082
|
+
id,
|
|
2083
|
+
kind: "video",
|
|
2084
|
+
mime: "video/mp4",
|
|
2085
|
+
platform: "youtube",
|
|
2086
|
+
url: streams.video,
|
|
2087
|
+
...streams.hls ? { hls: true } : {},
|
|
2088
|
+
...streams.audio ? { audio: { headers, url: streams.audio } } : {}
|
|
2068
2089
|
};
|
|
2069
|
-
return {
|
|
2090
|
+
return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
2070
2091
|
}
|
|
2071
|
-
function
|
|
2072
|
-
const
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2092
|
+
async function probeStream(net, url, headers) {
|
|
2093
|
+
const response = await net(url, { headers }, 1);
|
|
2094
|
+
await response.body?.cancel();
|
|
2095
|
+
if (!response.ok) {
|
|
2096
|
+
throw new Error(`YouTube stream failed: ${response.status}`);
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
async function resolveVisionOs(input, id, visitorData) {
|
|
2100
|
+
const headers = { "user-agent": visionOsClient.userAgent };
|
|
2101
|
+
const payload = await youtubePlayer(input.net, {
|
|
2102
|
+
videoId: id,
|
|
2103
|
+
contentCheckOk: true,
|
|
2104
|
+
racyCheckOk: true,
|
|
2105
|
+
context: { client: { ...visionOsClient, hl: "en", gl: "US", ...visitorData ? { visitorData } : {} } }
|
|
2106
|
+
}, {
|
|
2107
|
+
...headers,
|
|
2108
|
+
"content-type": "application/json",
|
|
2109
|
+
"x-youtube-client-name": "101",
|
|
2110
|
+
"x-youtube-client-version": visionOsClient.clientVersion,
|
|
2111
|
+
...visitorData ? { "x-goog-visitor-id": visitorData } : {}
|
|
2112
|
+
});
|
|
2113
|
+
const streaming = object(payload) && object(payload.streamingData) ? payload.streamingData : null;
|
|
2114
|
+
const manifestUrl = streaming ? string(streaming.hlsManifestUrl) : null;
|
|
2115
|
+
if (!manifestUrl) {
|
|
2116
|
+
throw new Error("YouTube HLS manifest not found");
|
|
2117
|
+
}
|
|
2118
|
+
const response = await input.net(manifestUrl, { headers });
|
|
2119
|
+
if (!response.ok) {
|
|
2120
|
+
throw new Error(`YouTube HLS manifest failed: ${response.status}`);
|
|
2121
|
+
}
|
|
2122
|
+
const master = parseMaster(await response.text(), manifestUrl, { preferOriginalAudio: true });
|
|
2123
|
+
const variants = master.variants.filter((variant2) => /(?:^|,)(?:vp09|av01)\./i.test(variant2.codecs ?? "") && /(?:^|,)mp4a\.40\.2(?:,|$)/i.test(variant2.codecs ?? "") && variant2.audioGroup && master.audio[variant2.audioGroup]).sort((left, right) => Math.abs(left.width - input.preferredWidth) - Math.abs(right.width - input.preferredWidth) || right.bandwidth - left.bandwidth);
|
|
2124
|
+
const variant = variants[0];
|
|
2125
|
+
if (!variant?.audioGroup) {
|
|
2126
|
+
throw new Error("YouTube fragmented MP4 HLS video with AAC audio not found");
|
|
2127
|
+
}
|
|
2128
|
+
const playlist = await input.net(variant.url, { headers });
|
|
2129
|
+
if (!playlist.ok) {
|
|
2130
|
+
throw new Error(`YouTube HLS playlist failed: ${playlist.status}`);
|
|
2131
|
+
}
|
|
2132
|
+
if (!(await playlist.text()).includes("#EXT-X-MAP:")) {
|
|
2133
|
+
throw new Error("YouTube HLS video is not fragmented MP4");
|
|
2134
|
+
}
|
|
2135
|
+
return youtubeResult(id, payload, { video: variant.url, audio: master.audio[variant.audioGroup], hls: true }, headers);
|
|
2136
|
+
}
|
|
2137
|
+
function youtubeMetadata(payload) {
|
|
2138
|
+
const details = object(payload) && object(payload.videoDetails) ? payload.videoDetails : null;
|
|
2139
|
+
const renderer = object(payload) && object(payload.microformat) && object(payload.microformat.playerMicroformatRenderer) ? payload.microformat.playerMicroformatRenderer : null;
|
|
2140
|
+
const author = details ? string(details.author) : null;
|
|
2141
|
+
const keywords = details && Array.isArray(details.keywords) ? details.keywords.filter((keyword) => typeof keyword === "string") : [];
|
|
2076
2142
|
return {
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
} : undefined,
|
|
2083
|
-
createdAt: isoFromEpochSeconds(item.createTime),
|
|
2084
|
-
likeCount: stats ? count(stats.diggCount) : undefined,
|
|
2085
|
-
commentCount: stats ? count(stats.commentCount) : undefined,
|
|
2086
|
-
shareCount: stats ? count(stats.shareCount) : undefined,
|
|
2087
|
-
viewCount: stats ? count(stats.playCount) : undefined,
|
|
2143
|
+
title: details ? string(details.title) ?? undefined : undefined,
|
|
2144
|
+
text: details ? string(details.shortDescription) ?? undefined : undefined,
|
|
2145
|
+
author: author ? { name: author } : undefined,
|
|
2146
|
+
createdAt: renderer ? isoFromDateString(renderer.publishDate) : undefined,
|
|
2147
|
+
viewCount: details ? count(details.viewCount) : undefined,
|
|
2088
2148
|
extra: {
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
region: string(item.locationCreated) ?? undefined,
|
|
2093
|
-
durationSeconds: video ? count(video.duration) : undefined
|
|
2149
|
+
channelId: details ? string(details.channelId) ?? undefined : undefined,
|
|
2150
|
+
durationSeconds: details ? count(details.lengthSeconds) : undefined,
|
|
2151
|
+
keywords: keywords.length > 0 ? keywords : undefined
|
|
2094
2152
|
}
|
|
2095
2153
|
};
|
|
2096
2154
|
}
|
|
2097
|
-
|
|
2098
|
-
const
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2155
|
+
function selectStreams(payload, preferredWidth) {
|
|
2156
|
+
const adaptive = adaptiveFormats(payload);
|
|
2157
|
+
const video = bestVideo(adaptive, preferredWidth);
|
|
2158
|
+
const audio = bestAudio(adaptive);
|
|
2159
|
+
if (video && audio) {
|
|
2160
|
+
return { video, audio };
|
|
2161
|
+
}
|
|
2162
|
+
const progressive = selectFormat(payload);
|
|
2163
|
+
return progressive ? { video: progressive, audio: null } : null;
|
|
2102
2164
|
}
|
|
2103
|
-
|
|
2104
|
-
const
|
|
2105
|
-
|
|
2165
|
+
function adaptiveFormats(payload) {
|
|
2166
|
+
const root = object(payload) ? payload : null;
|
|
2167
|
+
const streaming = root && object(root.streamingData) ? root.streamingData : null;
|
|
2168
|
+
return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
|
|
2169
|
+
}
|
|
2170
|
+
function bestVideo(formats, preferredWidth) {
|
|
2171
|
+
const candidates = formats.filter((format) => {
|
|
2172
|
+
const mimeType2 = string(format.mimeType);
|
|
2173
|
+
return Boolean(string(format.url)) && mimeType2?.startsWith("video/mp4") === true && mimeType2.includes("avc1");
|
|
2106
2174
|
});
|
|
2107
|
-
|
|
2175
|
+
const best = candidates.reduce((current, format) => {
|
|
2176
|
+
if (!current) {
|
|
2177
|
+
return format;
|
|
2178
|
+
}
|
|
2179
|
+
const width = number(format.width) ?? 0;
|
|
2180
|
+
const currentWidth = number(current.width) ?? 0;
|
|
2181
|
+
return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
|
|
2182
|
+
}, null);
|
|
2183
|
+
return best ? string(best.url) : null;
|
|
2108
2184
|
}
|
|
2109
|
-
function
|
|
2110
|
-
|
|
2185
|
+
function bestAudio(formats) {
|
|
2186
|
+
const candidates = formats.filter((format) => Boolean(string(format.url)) && string(format.mimeType)?.startsWith("audio/mp4") === true);
|
|
2187
|
+
const best = candidates.reduce((current, format) => {
|
|
2188
|
+
if (!current) {
|
|
2189
|
+
return format;
|
|
2190
|
+
}
|
|
2191
|
+
const preference = audioPreference(format) - audioPreference(current);
|
|
2192
|
+
if (preference !== 0) {
|
|
2193
|
+
return preference > 0 ? format : current;
|
|
2194
|
+
}
|
|
2195
|
+
return (number(format.bitrate) ?? 0) > (number(current.bitrate) ?? 0) ? format : current;
|
|
2196
|
+
}, null);
|
|
2197
|
+
return best ? string(best.url) : null;
|
|
2111
2198
|
}
|
|
2112
|
-
|
|
2113
|
-
const
|
|
2114
|
-
if (
|
|
2115
|
-
return
|
|
2116
|
-
}
|
|
2117
|
-
const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
|
|
2118
|
-
const location = response.headers.get("location");
|
|
2119
|
-
if (location) {
|
|
2120
|
-
return new URL(location, input).href;
|
|
2199
|
+
function audioPreference(format) {
|
|
2200
|
+
const track = object(format.audioTrack) ? format.audioTrack : null;
|
|
2201
|
+
if (track && /\boriginal\b/i.test(string(track.displayName) ?? "")) {
|
|
2202
|
+
return 2;
|
|
2121
2203
|
}
|
|
2122
|
-
|
|
2123
|
-
const href = html.match(/<a href="(https:\/\/[^"]+)"/)?.[1];
|
|
2124
|
-
return href ? new URL(href.split("?")[0]).href : input;
|
|
2204
|
+
return track?.audioIsDefault === true ? 1 : 0;
|
|
2125
2205
|
}
|
|
2126
|
-
function
|
|
2127
|
-
return
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2206
|
+
function playerBody(id, session) {
|
|
2207
|
+
return {
|
|
2208
|
+
contentCheckOk: true,
|
|
2209
|
+
context: {
|
|
2210
|
+
client: {
|
|
2211
|
+
androidSdkVersion: 32,
|
|
2212
|
+
clientName: androidVrClient.name,
|
|
2213
|
+
clientVersion: androidVrClient.version,
|
|
2214
|
+
deviceMake: "Oculus",
|
|
2215
|
+
deviceModel: "Quest 3",
|
|
2216
|
+
gl: "US",
|
|
2217
|
+
hl: "en",
|
|
2218
|
+
osName: "Android",
|
|
2219
|
+
osVersion: "12L",
|
|
2220
|
+
timeZone: "UTC",
|
|
2221
|
+
userAgent: androidVrClient.userAgent,
|
|
2222
|
+
utcOffsetMinutes: 0,
|
|
2223
|
+
visitorData: session.visitorData
|
|
2224
|
+
}
|
|
2225
|
+
},
|
|
2226
|
+
playbackContext: {
|
|
2227
|
+
contentPlaybackContext: {
|
|
2228
|
+
html5Preference: "HTML5_PREF_WANTS",
|
|
2229
|
+
signatureTimestamp: session.signatureTimestamp
|
|
2230
|
+
}
|
|
2231
|
+
},
|
|
2232
|
+
racyCheckOk: true,
|
|
2233
|
+
videoId: id
|
|
2234
|
+
};
|
|
2147
2235
|
}
|
|
2148
|
-
function
|
|
2149
|
-
const
|
|
2150
|
-
|
|
2151
|
-
|
|
2236
|
+
async function youtubeSession(net, id) {
|
|
2237
|
+
const response = await net(`https://www.youtube.com/watch?v=${id}&bpctr=9999999999&has_verified=1`, {
|
|
2238
|
+
headers: {
|
|
2239
|
+
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
2240
|
+
"accept-language": "en-us,en;q=0.5",
|
|
2241
|
+
cookie: browserCookie,
|
|
2242
|
+
"sec-fetch-mode": "navigate",
|
|
2243
|
+
"user-agent": browserUserAgent()
|
|
2244
|
+
}
|
|
2245
|
+
});
|
|
2246
|
+
if (!response.ok) {
|
|
2247
|
+
throw new Error(`YouTube page failed: ${response.status}`);
|
|
2152
2248
|
}
|
|
2153
|
-
const
|
|
2154
|
-
const
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
const route = object(rawRoute) ? rawRoute : null;
|
|
2158
|
-
const videoData = route && object(route.videoData) ? route.videoData : null;
|
|
2159
|
-
const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
|
|
2160
|
-
if (!videoData || !item) {
|
|
2161
|
-
throw new Error("TikTok embed item not found");
|
|
2249
|
+
const page = await response.text();
|
|
2250
|
+
const visitorData = string(page.match(/"visitorData":"([^"]+)"/)?.[1]);
|
|
2251
|
+
if (!visitorData) {
|
|
2252
|
+
throw new Error("YouTube visitor data not found");
|
|
2162
2253
|
}
|
|
2163
|
-
const
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
|
|
2167
|
-
const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
|
|
2168
|
-
const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
|
|
2169
|
-
if (displayImages.length === 0) {
|
|
2170
|
-
throw new Error("TikTok video hydration not found");
|
|
2254
|
+
const playerPath = string(page.match(/"jsUrl":"([^"]+)"/)?.[1]) ?? string(page.match(/"PLAYER_JS_URL":"([^"]+)"/)?.[1]);
|
|
2255
|
+
if (!playerPath) {
|
|
2256
|
+
throw new Error("YouTube player url not found");
|
|
2171
2257
|
}
|
|
2172
2258
|
return {
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
uniqueId: string(authorInfo.uniqueId),
|
|
2177
|
-
verified: bool(authorInfo.verified)
|
|
2178
|
-
},
|
|
2179
|
-
desc: string(item.text),
|
|
2180
|
-
imagePost: {
|
|
2181
|
-
images: displayImages.map((image) => ({
|
|
2182
|
-
imageURL: {
|
|
2183
|
-
urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
|
|
2184
|
-
}
|
|
2185
|
-
}))
|
|
2186
|
-
},
|
|
2187
|
-
music: {
|
|
2188
|
-
authorName: string(musicInfo.authorName),
|
|
2189
|
-
playUrl: firstString(musicInfo.playUrl),
|
|
2190
|
-
title: string(musicInfo.musicName)
|
|
2191
|
-
},
|
|
2192
|
-
stats: {
|
|
2193
|
-
commentCount: item.commentCount,
|
|
2194
|
-
diggCount: item.diggCount,
|
|
2195
|
-
playCount: item.playCount,
|
|
2196
|
-
shareCount: item.shareCount
|
|
2197
|
-
},
|
|
2198
|
-
video: {
|
|
2199
|
-
duration: videoMeta.duration
|
|
2200
|
-
}
|
|
2259
|
+
cookie: youtubeCookie(response.headers),
|
|
2260
|
+
signatureTimestamp: await signatureTimestamp(net, playerPath),
|
|
2261
|
+
visitorData
|
|
2201
2262
|
};
|
|
2202
2263
|
}
|
|
2203
|
-
function
|
|
2204
|
-
|
|
2205
|
-
|
|
2264
|
+
function playerHeaders(session) {
|
|
2265
|
+
return new Headers({
|
|
2266
|
+
"content-type": "application/json",
|
|
2267
|
+
cookie: `${browserCookie}; ${session.cookie}`,
|
|
2268
|
+
origin: "https://www.youtube.com",
|
|
2269
|
+
"user-agent": androidVrClient.userAgent,
|
|
2270
|
+
"x-goog-visitor-id": session.visitorData,
|
|
2271
|
+
"x-youtube-client-name": androidVrClient.number,
|
|
2272
|
+
"x-youtube-client-version": androidVrClient.version
|
|
2273
|
+
});
|
|
2206
2274
|
}
|
|
2207
|
-
function
|
|
2208
|
-
|
|
2275
|
+
function youtubeCookie(headers) {
|
|
2276
|
+
const readable = headers;
|
|
2277
|
+
const fallback = headers.get("set-cookie");
|
|
2278
|
+
const cookieHeaders = readable.getSetCookie ? readable.getSetCookie() : fallback ? [fallback] : [];
|
|
2279
|
+
return cookieHeaders.map((header) => string(header.split(";", 1)[0])).filter(string).join("; ");
|
|
2209
2280
|
}
|
|
2210
|
-
function
|
|
2211
|
-
const
|
|
2212
|
-
|
|
2281
|
+
async function signatureTimestamp(net, playerPath) {
|
|
2282
|
+
const response = await net(new URL(playerPath, "https://www.youtube.com").toString(), {
|
|
2283
|
+
headers: { "user-agent": browserUserAgent() }
|
|
2284
|
+
});
|
|
2285
|
+
if (!response.ok) {
|
|
2286
|
+
throw new Error(`YouTube player failed: ${response.status}`);
|
|
2287
|
+
}
|
|
2288
|
+
const player = await response.text();
|
|
2289
|
+
const timestamp = number(Number(player.match(/signatureTimestamp[:=](\d+)/)?.[1] ?? player.match(/sts[:=](\d+)/)?.[1]));
|
|
2290
|
+
if (!timestamp) {
|
|
2291
|
+
throw new Error("YouTube signature timestamp not found");
|
|
2292
|
+
}
|
|
2293
|
+
return timestamp;
|
|
2213
2294
|
}
|
|
2214
|
-
function
|
|
2215
|
-
const
|
|
2216
|
-
if (
|
|
2217
|
-
|
|
2218
|
-
return audio ? [...images, audio] : images;
|
|
2295
|
+
function youtubeVideoId(input) {
|
|
2296
|
+
const url = asUrl(input);
|
|
2297
|
+
if (url.hostname === "youtu.be") {
|
|
2298
|
+
return cleanId(url.pathname.split("/").filter(Boolean)[0]);
|
|
2219
2299
|
}
|
|
2220
|
-
const
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
platform: "tiktok",
|
|
2228
|
-
url
|
|
2229
|
-
}] : [];
|
|
2300
|
+
const fromQuery = cleanId(url.searchParams.get("v"));
|
|
2301
|
+
if (fromQuery) {
|
|
2302
|
+
return fromQuery;
|
|
2303
|
+
}
|
|
2304
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
2305
|
+
const index = parts.findIndex((part) => part === "shorts" || part === "live" || part === "embed");
|
|
2306
|
+
return index >= 0 ? cleanId(parts[index + 1]) : null;
|
|
2230
2307
|
}
|
|
2231
|
-
function
|
|
2232
|
-
|
|
2233
|
-
const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
|
|
2234
|
-
return images.flatMap((image, index) => {
|
|
2235
|
-
const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
|
|
2236
|
-
const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
|
|
2237
|
-
const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
|
|
2238
|
-
return url ? [{
|
|
2239
|
-
filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
|
|
2240
|
-
headers,
|
|
2241
|
-
id,
|
|
2242
|
-
kind: "image",
|
|
2243
|
-
mime: "image/jpeg",
|
|
2244
|
-
platform: "tiktok",
|
|
2245
|
-
url
|
|
2246
|
-
}] : [];
|
|
2247
|
-
});
|
|
2308
|
+
function cleanId(value) {
|
|
2309
|
+
return typeof value === "string" && /^[A-Za-z0-9_-]{11}$/.test(value) ? value : null;
|
|
2248
2310
|
}
|
|
2249
|
-
function
|
|
2250
|
-
const
|
|
2251
|
-
const
|
|
2252
|
-
const
|
|
2253
|
-
const
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2311
|
+
function selectFormat(payload) {
|
|
2312
|
+
const root = object(payload) ? payload : null;
|
|
2313
|
+
const streaming = root && object(root.streamingData) ? root.streamingData : null;
|
|
2314
|
+
const formats = streaming && Array.isArray(streaming.formats) ? streaming.formats.filter(object) : [];
|
|
2315
|
+
const mp4 = formats.filter((format) => string(format.url) && string(format.mimeType)?.startsWith("video/mp4")).sort((left, right) => height(right) - height(left));
|
|
2316
|
+
return mp4[0] ? string(mp4[0].url) : null;
|
|
2317
|
+
}
|
|
2318
|
+
function height(format) {
|
|
2319
|
+
return number(format.height) ?? Number(string(format.qualityLabel)?.match(/(\d+)p/)?.[1] ?? 0);
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
// src/spotify.ts
|
|
2323
|
+
async function resolveSpotify(input) {
|
|
2324
|
+
const url = asUrl(input.url);
|
|
2325
|
+
const id = url.pathname.match(/^\/(?:intl-[a-z-]+\/)?(?:embed\/)?track\/([A-Za-z0-9]{22})\/?$/i)?.[1];
|
|
2326
|
+
if (!id) {
|
|
2327
|
+
throw new PostfetchError(400, "Spotify requires an open.spotify.com track URL");
|
|
2257
2328
|
}
|
|
2258
|
-
const
|
|
2329
|
+
const track = await spotifyTrack(input, id);
|
|
2330
|
+
const youtubeUrl = await matchYoutube(input, track);
|
|
2331
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2259
2332
|
return {
|
|
2260
|
-
|
|
2261
|
-
|
|
2333
|
+
archiveFilename: filename(`spotify_${id}.zip`),
|
|
2334
|
+
comments: [],
|
|
2262
2335
|
id,
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2336
|
+
items: youtube.items,
|
|
2337
|
+
platform: "spotify",
|
|
2338
|
+
metadata: {
|
|
2339
|
+
title: track.title,
|
|
2340
|
+
author: { name: track.artists.join(", ") },
|
|
2341
|
+
extra: { artists: track.artists, durationSeconds: track.durationSeconds, youtubeUrl }
|
|
2342
|
+
}
|
|
2267
2343
|
};
|
|
2268
2344
|
}
|
|
2269
|
-
function
|
|
2270
|
-
const
|
|
2271
|
-
|
|
2345
|
+
async function spotifyTrack(input, id) {
|
|
2346
|
+
const response = await input.net(`https://open.spotify.com/embed/track/${id}`, {
|
|
2347
|
+
headers: { "user-agent": browserUserAgent() }
|
|
2348
|
+
});
|
|
2349
|
+
if (!response.ok) {
|
|
2350
|
+
throw new PostfetchError(response.status === 404 ? 404 : 502, `Spotify track lookup failed: ${response.status}`);
|
|
2351
|
+
}
|
|
2352
|
+
const html = await response.text();
|
|
2353
|
+
const raw = html.match(/<script\b[^>]*\bid=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
2354
|
+
const payload = parseJson2(raw);
|
|
2355
|
+
const props = payload && object(payload.props) ? payload.props : null;
|
|
2356
|
+
const page = props && object(props.pageProps) ? props.pageProps : null;
|
|
2357
|
+
const state = page && object(page.state) ? page.state : null;
|
|
2358
|
+
const data = state && object(state.data) ? state.data : null;
|
|
2359
|
+
const entity = data && object(data.entity) ? data.entity : null;
|
|
2360
|
+
const title = entity && (string(entity.title) ?? string(entity.name));
|
|
2361
|
+
const artists = entity && Array.isArray(entity.artists) ? entity.artists.filter(object).map((artist) => string(artist.name)).filter((name) => name !== null) : [];
|
|
2362
|
+
if (!entity || entity.type !== "track" || entity.id !== id || !title || artists.length === 0) {
|
|
2363
|
+
throw new PostfetchError(502, "Spotify track metadata not found");
|
|
2364
|
+
}
|
|
2365
|
+
const duration = number(entity.duration);
|
|
2366
|
+
return { title, artists, durationSeconds: duration !== null && duration > 0 ? duration / 1000 : undefined };
|
|
2367
|
+
}
|
|
2368
|
+
async function matchYoutube(input, track) {
|
|
2369
|
+
const url = new URL("https://www.youtube.com/results");
|
|
2370
|
+
url.searchParams.set("search_query", `${track.artists.join(" ")} ${track.title} audio`);
|
|
2371
|
+
url.searchParams.set("hl", "en");
|
|
2372
|
+
const response = await input.net(url.href, {
|
|
2373
|
+
headers: { "accept-language": "en-US,en;q=0.9", cookie: "SOCS=CAI", "user-agent": browserUserAgent() }
|
|
2374
|
+
});
|
|
2375
|
+
if (!response.ok) {
|
|
2376
|
+
throw new PostfetchError(502, `YouTube search failed: ${response.status}`);
|
|
2377
|
+
}
|
|
2378
|
+
const html = await response.text();
|
|
2379
|
+
const raw = html.match(/(?:\bytInitialData|window\[["']ytInitialData["']\])\s*=\s*(\{[\s\S]*?)<\/script>/)?.[1];
|
|
2380
|
+
const payload = parseAssignment(raw);
|
|
2381
|
+
if (!payload || !object(payload.contents)) {
|
|
2382
|
+
throw new PostfetchError(502, "YouTube search results not found");
|
|
2383
|
+
}
|
|
2384
|
+
const candidates = videoRenderers(payload.contents);
|
|
2385
|
+
const ranked = candidates.map((candidate, index) => ({ candidate, score: matchScore(track, candidate) - index / 1000 })).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
2386
|
+
const id = ranked[0]?.candidate.videoId;
|
|
2387
|
+
if (typeof id !== "string") {
|
|
2388
|
+
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "notFound");
|
|
2389
|
+
}
|
|
2390
|
+
return `https://www.youtube.com/watch?v=${id}`;
|
|
2391
|
+
}
|
|
2392
|
+
function videoRenderers(value) {
|
|
2393
|
+
if (Array.isArray(value))
|
|
2394
|
+
return value.flatMap(videoRenderers);
|
|
2395
|
+
if (!object(value))
|
|
2396
|
+
return [];
|
|
2397
|
+
if (object(value.videoRenderer))
|
|
2398
|
+
return [value.videoRenderer];
|
|
2399
|
+
return Object.values(value).flatMap(videoRenderers);
|
|
2272
2400
|
}
|
|
2273
|
-
function
|
|
2274
|
-
|
|
2401
|
+
function text(value) {
|
|
2402
|
+
if (!object(value))
|
|
2403
|
+
return "";
|
|
2404
|
+
return string(value.simpleText) ?? (Array.isArray(value.runs) ? value.runs.filter(object).map((run) => typeof run.text === "string" ? run.text : "").join("") : "");
|
|
2275
2405
|
}
|
|
2276
|
-
function
|
|
2277
|
-
|
|
2278
|
-
const setCookie = typeof getter === "function" ? getter.call(headers).join(",") : headers.get("set-cookie");
|
|
2279
|
-
const cookies = setCookie?.split(/,(?=[^;]+?=)/).map((part) => part.split(";")[0]?.trim()).filter((part) => Boolean(part));
|
|
2280
|
-
return cookies && cookies.length > 0 ? cookies.join("; ") : null;
|
|
2406
|
+
function normalize(value) {
|
|
2407
|
+
return value.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
2281
2408
|
}
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2409
|
+
function contains(haystack, needle) {
|
|
2410
|
+
return needle.length > 0 && ` ${haystack} `.includes(` ${needle} `);
|
|
2411
|
+
}
|
|
2412
|
+
function durationSeconds(value) {
|
|
2413
|
+
if (!/^\d+(?::\d{2}){1,2}$/.test(value))
|
|
2414
|
+
return null;
|
|
2415
|
+
return value.split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0);
|
|
2416
|
+
}
|
|
2417
|
+
function matchScore(track, candidate) {
|
|
2418
|
+
if (typeof candidate.videoId !== "string" || !/^[A-Za-z0-9_-]{11}$/.test(candidate.videoId))
|
|
2419
|
+
return -Infinity;
|
|
2420
|
+
const title = normalize(text(candidate.title));
|
|
2421
|
+
const author = normalize(text(candidate.ownerText) || text(candidate.longBylineText) || text(candidate.shortBylineText));
|
|
2422
|
+
const expected = normalize(track.title.replace(/\s*[([](?:feat\.?|ft\.?).*?[)\]]/gi, ""));
|
|
2423
|
+
const tokens = expected.split(" ").filter(Boolean);
|
|
2424
|
+
if (tokens.length === 0)
|
|
2425
|
+
return -Infinity;
|
|
2426
|
+
const coverage = tokens.filter((token) => contains(title, token)).length / tokens.length;
|
|
2427
|
+
if (coverage < (tokens.length <= 3 ? 1 : 0.8))
|
|
2428
|
+
return -Infinity;
|
|
2429
|
+
if (tokens.length <= 3 && !contains(title, expected))
|
|
2430
|
+
return -Infinity;
|
|
2431
|
+
const artists = track.artists.map(normalize);
|
|
2432
|
+
if (!contains(`${title} ${author}`, artists[0]))
|
|
2433
|
+
return -Infinity;
|
|
2434
|
+
const duration = durationSeconds(text(candidate.lengthText));
|
|
2435
|
+
if (duration === null || duration <= 0)
|
|
2436
|
+
return -Infinity;
|
|
2437
|
+
const tolerance = Math.min(30, Math.max(12, (track.durationSeconds ?? duration) * 0.08));
|
|
2438
|
+
const delta = track.durationSeconds === undefined ? 0 : Math.abs(duration - track.durationSeconds);
|
|
2439
|
+
if (delta > tolerance)
|
|
2440
|
+
return -Infinity;
|
|
2441
|
+
let recordingTitle = title;
|
|
2442
|
+
for (const artist of artists)
|
|
2443
|
+
recordingTitle = recordingTitle.replace(artist, "");
|
|
2444
|
+
for (const version of [/\bremix\b/, /\bcover\b/, /\bkaraoke\b/, /\binstrumental\b/, /\bslowed\b/, /\b(?:sped|speed) up\b/, /\bnightcore\b/, /\blive\b/, /\bacoustic\b/]) {
|
|
2445
|
+
if (version.test(recordingTitle) !== version.test(expected))
|
|
2446
|
+
return -Infinity;
|
|
2447
|
+
}
|
|
2448
|
+
const artistUpload = contains(author, artists[0]);
|
|
2449
|
+
const credited = artists.filter((artist) => contains(`${title} ${author}`, artist)).length / artists.length;
|
|
2450
|
+
return coverage * 10 + (artistUpload ? 8 : 0) + credited * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
2451
|
+
}
|
|
2452
|
+
function parseJson2(value) {
|
|
2453
|
+
try {
|
|
2454
|
+
const parsed = JSON.parse(value ?? "");
|
|
2455
|
+
return object(parsed) ? parsed : null;
|
|
2456
|
+
} catch {
|
|
2457
|
+
return null;
|
|
2458
|
+
}
|
|
2459
|
+
}
|
|
2460
|
+
function parseAssignment(value) {
|
|
2461
|
+
if (!value)
|
|
2462
|
+
return null;
|
|
2463
|
+
let depth = 0;
|
|
2464
|
+
let quoted = false;
|
|
2465
|
+
let escaped = false;
|
|
2466
|
+
for (let index = 0;index < value.length; index += 1) {
|
|
2467
|
+
const character = value[index];
|
|
2468
|
+
if (quoted) {
|
|
2469
|
+
if (escaped)
|
|
2470
|
+
escaped = false;
|
|
2471
|
+
else if (character === "\\")
|
|
2472
|
+
escaped = true;
|
|
2473
|
+
else if (character === '"')
|
|
2474
|
+
quoted = false;
|
|
2475
|
+
} else if (character === '"')
|
|
2476
|
+
quoted = true;
|
|
2477
|
+
else if (character === "{")
|
|
2478
|
+
depth += 1;
|
|
2479
|
+
else if (character === "}" && --depth === 0)
|
|
2480
|
+
return parseJson2(value.slice(0, index + 1));
|
|
2481
|
+
}
|
|
2482
|
+
return null;
|
|
2483
|
+
}
|
|
2484
|
+
|
|
2485
|
+
// src/tiktok.ts
|
|
2486
|
+
var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
|
|
2487
|
+
var pageAttemptCount = 12;
|
|
2488
|
+
async function resolveTiktok(input) {
|
|
2489
|
+
const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
|
|
2490
|
+
const id = videoId(pageUrl);
|
|
2286
2491
|
if (!id) {
|
|
2287
|
-
throw new Error("
|
|
2492
|
+
throw new Error("TikTok video id not found");
|
|
2288
2493
|
}
|
|
2289
|
-
const
|
|
2290
|
-
|
|
2291
|
-
|
|
2494
|
+
const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
|
|
2495
|
+
const user = author(page.item) ?? username(pageUrl) ?? "i";
|
|
2496
|
+
const headers = {
|
|
2497
|
+
referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
|
|
2498
|
+
"user-agent": page.userAgent
|
|
2499
|
+
};
|
|
2500
|
+
if (page.cookie) {
|
|
2501
|
+
headers.cookie = page.cookie;
|
|
2292
2502
|
}
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
2297
|
-
return media.flatMap((entry, index) => {
|
|
2298
|
-
const candidates = twitterCandidates(entry, postId2, index + 1);
|
|
2299
|
-
return candidates.length > 0 ? [candidates] : [];
|
|
2300
|
-
});
|
|
2301
|
-
});
|
|
2302
|
-
for (const { id: postId2, tweet: post } of posts.toReversed()) {
|
|
2303
|
-
const card = twitterCardImage(post, postId2);
|
|
2304
|
-
if (card)
|
|
2305
|
-
groups.push([{ bitrate: 0, item: card }]);
|
|
2503
|
+
const items = mediaItems4(page.item, user, id, headers);
|
|
2504
|
+
if (items.length === 0) {
|
|
2505
|
+
throw new Error("TikTok media not found");
|
|
2306
2506
|
}
|
|
2307
|
-
|
|
2308
|
-
return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
2507
|
+
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
2309
2508
|
}
|
|
2310
|
-
function
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2509
|
+
async function videoPage(net, id, photo) {
|
|
2510
|
+
for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
|
|
2511
|
+
const baseHeaders2 = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
|
|
2512
|
+
const userAgent2 = baseHeaders2["user-agent"];
|
|
2513
|
+
const headers2 = {
|
|
2514
|
+
...baseHeaders2,
|
|
2515
|
+
referer: "https://www.tiktok.com/",
|
|
2516
|
+
"sec-fetch-site": "same-origin"
|
|
2517
|
+
};
|
|
2518
|
+
try {
|
|
2519
|
+
return { ...await fetchVideoPage(net, id, headers2), userAgent: userAgent2 };
|
|
2520
|
+
} catch (error) {
|
|
2521
|
+
if (!recoverablePageError(error)) {
|
|
2522
|
+
throw error;
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2321
2525
|
}
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
const replyToId = string(tweet.in_reply_to_status_id_str) ?? parentId;
|
|
2325
|
-
if (replyToId)
|
|
2326
|
-
extra.replyToId = replyToId;
|
|
2327
|
-
if (parent && parentId) {
|
|
2328
|
-
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
2526
|
+
if (!photo) {
|
|
2527
|
+
throw new Error("TikTok hydration not found");
|
|
2329
2528
|
}
|
|
2529
|
+
const baseHeaders = navigationHeaders();
|
|
2530
|
+
const userAgent = baseHeaders["user-agent"];
|
|
2531
|
+
const headers = {
|
|
2532
|
+
...baseHeaders,
|
|
2533
|
+
referer: "https://www.tiktok.com/",
|
|
2534
|
+
"sec-fetch-site": "same-origin"
|
|
2535
|
+
};
|
|
2536
|
+
return { ...await fetchEmbedPage(net, id, headers), userAgent };
|
|
2537
|
+
}
|
|
2538
|
+
function tiktokMetadata(item) {
|
|
2539
|
+
const user = object(item.author) ? item.author : null;
|
|
2540
|
+
const stats = object(item.stats) ? item.stats : null;
|
|
2541
|
+
const music = object(item.music) ? item.music : null;
|
|
2542
|
+
const video = object(item.video) ? item.video : null;
|
|
2330
2543
|
return {
|
|
2331
|
-
text:
|
|
2544
|
+
text: string(item.desc) ?? undefined,
|
|
2332
2545
|
author: user ? {
|
|
2333
|
-
handle: string(user.
|
|
2334
|
-
name: string(user.
|
|
2335
|
-
verified: bool(user.verified)
|
|
2546
|
+
handle: string(user.uniqueId) ?? undefined,
|
|
2547
|
+
name: string(user.nickname) ?? undefined,
|
|
2548
|
+
verified: bool(user.verified)
|
|
2336
2549
|
} : undefined,
|
|
2337
|
-
createdAt:
|
|
2338
|
-
likeCount: count(
|
|
2339
|
-
commentCount: count(
|
|
2340
|
-
shareCount: count(
|
|
2341
|
-
viewCount: count(
|
|
2342
|
-
|
|
2343
|
-
|
|
2550
|
+
createdAt: isoFromEpochSeconds(item.createTime),
|
|
2551
|
+
likeCount: stats ? count(stats.diggCount) : undefined,
|
|
2552
|
+
commentCount: stats ? count(stats.commentCount) : undefined,
|
|
2553
|
+
shareCount: stats ? count(stats.shareCount) : undefined,
|
|
2554
|
+
viewCount: stats ? count(stats.playCount) : undefined,
|
|
2555
|
+
extra: {
|
|
2556
|
+
saveCount: stats ? count(stats.collectCount) : undefined,
|
|
2557
|
+
musicTitle: music ? string(music.title) ?? undefined : undefined,
|
|
2558
|
+
musicAuthor: music ? string(music.authorName) ?? undefined : undefined,
|
|
2559
|
+
region: string(item.locationCreated) ?? undefined,
|
|
2560
|
+
durationSeconds: video ? count(video.duration) : undefined
|
|
2561
|
+
}
|
|
2344
2562
|
};
|
|
2345
2563
|
}
|
|
2346
|
-
function
|
|
2347
|
-
const
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
return
|
|
2564
|
+
async function fetchVideoPage(net, id, headers) {
|
|
2565
|
+
const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
|
|
2566
|
+
headers
|
|
2567
|
+
});
|
|
2568
|
+
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
2351
2569
|
}
|
|
2352
|
-
function
|
|
2353
|
-
|
|
2570
|
+
async function fetchEmbedPage(net, id, headers) {
|
|
2571
|
+
const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
|
|
2572
|
+
headers
|
|
2573
|
+
});
|
|
2574
|
+
return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
|
|
2354
2575
|
}
|
|
2355
|
-
function
|
|
2356
|
-
|
|
2357
|
-
return null;
|
|
2358
|
-
const entities = object(tweet.entities) ? tweet.entities : null;
|
|
2359
|
-
const media = [
|
|
2360
|
-
...entities && Array.isArray(entities.media) ? entities.media : [],
|
|
2361
|
-
...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
|
|
2362
|
-
];
|
|
2363
|
-
const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
|
|
2364
|
-
while (true) {
|
|
2365
|
-
const trailing = text.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
|
|
2366
|
-
if (!trailing || !urls.has(trailing[1]))
|
|
2367
|
-
break;
|
|
2368
|
-
text = text.slice(0, trailing.index).trimEnd();
|
|
2369
|
-
}
|
|
2370
|
-
return text.replace(/&(?:amp|gt|lt);/g, (entity) => entity === "&" ? "&" : entity === ">" ? ">" : "<");
|
|
2576
|
+
function isShortlinkHost(hostname) {
|
|
2577
|
+
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
2371
2578
|
}
|
|
2372
|
-
function
|
|
2373
|
-
|
|
2374
|
-
|
|
2579
|
+
async function followShortlink(net, userAgent, input) {
|
|
2580
|
+
const url = asUrl(input);
|
|
2581
|
+
if (!isShortlinkHost(url.hostname)) {
|
|
2582
|
+
return input;
|
|
2375
2583
|
}
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
for (const [id, post] of posts) {
|
|
2381
|
-
if (!needsFullText(post)) {
|
|
2382
|
-
continue;
|
|
2383
|
-
}
|
|
2384
|
-
try {
|
|
2385
|
-
const response = await net(`https://api.fxtwitter.com/status/${id}`, {
|
|
2386
|
-
headers: { accept: "application/json", "user-agent": browserUserAgent() }
|
|
2387
|
-
}, 1);
|
|
2388
|
-
if (!response.ok) {
|
|
2389
|
-
continue;
|
|
2390
|
-
}
|
|
2391
|
-
const payload = await response.json();
|
|
2392
|
-
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
|
|
2393
|
-
continue;
|
|
2394
|
-
}
|
|
2395
|
-
let current = payload.tweet;
|
|
2396
|
-
const seen = new Set;
|
|
2397
|
-
while (current) {
|
|
2398
|
-
const currentId = string(current.id);
|
|
2399
|
-
if (!currentId || seen.has(currentId)) {
|
|
2400
|
-
break;
|
|
2401
|
-
}
|
|
2402
|
-
seen.add(currentId);
|
|
2403
|
-
const target = posts.get(currentId);
|
|
2404
|
-
const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
|
|
2405
|
-
const text = raw ?? string(current.text);
|
|
2406
|
-
if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
|
|
2407
|
-
target.full_text = text;
|
|
2408
|
-
}
|
|
2409
|
-
current = object(current.quote) ? current.quote : null;
|
|
2410
|
-
}
|
|
2411
|
-
} catch {}
|
|
2584
|
+
const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
|
|
2585
|
+
const location = response.headers.get("location");
|
|
2586
|
+
if (location) {
|
|
2587
|
+
return new URL(location, input).href;
|
|
2412
2588
|
}
|
|
2589
|
+
const html = await response.text();
|
|
2590
|
+
const href = html.match(/<a href="(https:\/\/[^"]+)"/)?.[1];
|
|
2591
|
+
return href ? new URL(href.split("?")[0]).href : input;
|
|
2413
2592
|
}
|
|
2414
|
-
function
|
|
2415
|
-
|
|
2416
|
-
const seen = new Set;
|
|
2417
|
-
const visit = (current, fallbackId) => {
|
|
2418
|
-
const id = string(current.id_str) ?? fallbackId;
|
|
2419
|
-
if (!id || seen.has(id))
|
|
2420
|
-
return;
|
|
2421
|
-
seen.add(id);
|
|
2422
|
-
tweets.push({ id, tweet: current });
|
|
2423
|
-
if (object(current.quoted_tweet))
|
|
2424
|
-
visit(current.quoted_tweet);
|
|
2425
|
-
if (includeParents && object(current.parent))
|
|
2426
|
-
visit(current.parent);
|
|
2427
|
-
};
|
|
2428
|
-
visit(tweet, rootId);
|
|
2429
|
-
return tweets;
|
|
2593
|
+
function videoId(input) {
|
|
2594
|
+
return asUrl(input).pathname.match(/(?:video|photo)\/(\d+)/)?.[1] ?? null;
|
|
2430
2595
|
}
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
if (!response.ok) {
|
|
2435
|
-
throw new Error(`Twitter syndication failed: ${response.status}`);
|
|
2596
|
+
function itemStruct(html) {
|
|
2597
|
+
if (html.includes("SlardarWAF") || html.includes("_wafchallengeid")) {
|
|
2598
|
+
throw new Error("TikTok WAF challenge");
|
|
2436
2599
|
}
|
|
2437
|
-
const
|
|
2438
|
-
|
|
2439
|
-
|
|
2600
|
+
const start = html.indexOf(marker);
|
|
2601
|
+
const end = start === -1 ? -1 : html.indexOf("</script>", start + marker.length);
|
|
2602
|
+
if (start === -1 || end === -1) {
|
|
2603
|
+
throw new Error("TikTok hydration not found");
|
|
2440
2604
|
}
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2605
|
+
const parsed = JSON.parse(html.slice(start + marker.length, end));
|
|
2606
|
+
const scope = object(parsed) && object(parsed.__DEFAULT_SCOPE__) ? parsed.__DEFAULT_SCOPE__ : null;
|
|
2607
|
+
const detail = scope && object(scope["webapp.video-detail"]) ? scope["webapp.video-detail"] : null;
|
|
2608
|
+
const info = detail && object(detail.itemInfo) ? detail.itemInfo : null;
|
|
2609
|
+
const item = info && object(info.itemStruct) ? info.itemStruct : null;
|
|
2610
|
+
if (!item) {
|
|
2611
|
+
throw new Error("TikTok itemStruct not found");
|
|
2612
|
+
}
|
|
2613
|
+
return item;
|
|
2445
2614
|
}
|
|
2446
|
-
function
|
|
2447
|
-
const
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
return videoVariants(entry).map(({ bitrate, url }) => ({
|
|
2451
|
-
bitrate,
|
|
2452
|
-
item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }
|
|
2453
|
-
}));
|
|
2615
|
+
function embedItemStruct(html, id) {
|
|
2616
|
+
const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
2617
|
+
if (!raw) {
|
|
2618
|
+
throw new Error("TikTok embed state not found");
|
|
2454
2619
|
}
|
|
2455
|
-
const
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
const
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
if (!/^(photo_image_full_size|summary_photo_image|thumbnail_image)(_(original|x_large|large|small))?$/.test(key) || !object(value) || !object(value.image_value))
|
|
2465
|
-
return [];
|
|
2466
|
-
const image2 = value.image_value;
|
|
2467
|
-
const rawUrl = string(image2.url);
|
|
2468
|
-
if (!rawUrl)
|
|
2469
|
-
return [];
|
|
2470
|
-
try {
|
|
2471
|
-
const url = new URL(rawUrl);
|
|
2472
|
-
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
2473
|
-
return [];
|
|
2474
|
-
const format = (url.searchParams.get("format") ?? url.pathname.match(/\.(jpe?g|png|webp|gif)$/i)?.[1] ?? "jpg").toLowerCase();
|
|
2475
|
-
const extension = format === "jpeg" ? "jpg" : format;
|
|
2476
|
-
if (!["jpg", "png", "webp", "gif"].includes(extension))
|
|
2477
|
-
return [];
|
|
2478
|
-
return [{
|
|
2479
|
-
url: url.href,
|
|
2480
|
-
extension,
|
|
2481
|
-
original: key.endsWith("_original") ? 1 : 0,
|
|
2482
|
-
area: Math.max(0, number(image2.width) ?? 0) * Math.max(0, number(image2.height) ?? 0)
|
|
2483
|
-
}];
|
|
2484
|
-
} catch {
|
|
2485
|
-
return [];
|
|
2486
|
-
}
|
|
2487
|
-
}).sort((a, b) => b.original - a.original || b.area - a.area);
|
|
2488
|
-
const image = images[0];
|
|
2489
|
-
return image ? {
|
|
2490
|
-
filename: filename(`twitter_${id}_preview.${image.extension}`),
|
|
2491
|
-
headers: { "user-agent": browserUserAgent() },
|
|
2492
|
-
id,
|
|
2493
|
-
kind: "image",
|
|
2494
|
-
mime: `image/${image.extension === "jpg" ? "jpeg" : image.extension}`,
|
|
2495
|
-
platform: "twitter",
|
|
2496
|
-
role: "linkPreview",
|
|
2497
|
-
url: image.url
|
|
2498
|
-
} : null;
|
|
2499
|
-
}
|
|
2500
|
-
function videoVariants(entry) {
|
|
2501
|
-
const info = object(entry.video_info) ? entry.video_info : null;
|
|
2502
|
-
const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
|
|
2503
|
-
const seen = new Set;
|
|
2504
|
-
return variants.flatMap((variant) => {
|
|
2505
|
-
const url = string(variant.url);
|
|
2506
|
-
if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
|
|
2507
|
-
return [];
|
|
2508
|
-
}
|
|
2509
|
-
seen.add(url);
|
|
2510
|
-
return [{ bitrate: number(variant.bitrate) ?? 0, url }];
|
|
2511
|
-
}).sort((left, right) => right.bitrate - left.bitrate);
|
|
2512
|
-
}
|
|
2513
|
-
async function sizeLimitedItems(input, groups) {
|
|
2514
|
-
if (groups.length === 0 || input.tryMaxBytes === undefined) {
|
|
2515
|
-
return groups.map((group) => group[0].item);
|
|
2620
|
+
const parsed = JSON.parse(raw);
|
|
2621
|
+
const source2 = object(parsed) && object(parsed.source) ? parsed.source : null;
|
|
2622
|
+
const data = source2 && object(source2.data) ? source2.data : null;
|
|
2623
|
+
const rawRoute = data ? data[`/embed/v2/${id}`] : null;
|
|
2624
|
+
const route = object(rawRoute) ? rawRoute : null;
|
|
2625
|
+
const videoData = route && object(route.videoData) ? route.videoData : null;
|
|
2626
|
+
const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
|
|
2627
|
+
if (!videoData || !item) {
|
|
2628
|
+
throw new Error("TikTok embed item not found");
|
|
2516
2629
|
}
|
|
2517
|
-
const
|
|
2518
|
-
const
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2630
|
+
const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
|
|
2631
|
+
const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
|
|
2632
|
+
const video = object(item.video) ? item.video : {};
|
|
2633
|
+
const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
|
|
2634
|
+
const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
|
|
2635
|
+
const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
|
|
2636
|
+
if (displayImages.length === 0) {
|
|
2637
|
+
throw new Error("TikTok video hydration not found");
|
|
2638
|
+
}
|
|
2639
|
+
return {
|
|
2640
|
+
...item,
|
|
2641
|
+
author: {
|
|
2642
|
+
nickname: string(authorInfo.nickName),
|
|
2643
|
+
uniqueId: string(authorInfo.uniqueId),
|
|
2644
|
+
verified: bool(authorInfo.verified)
|
|
2645
|
+
},
|
|
2646
|
+
desc: string(item.text),
|
|
2647
|
+
imagePost: {
|
|
2648
|
+
images: displayImages.map((image) => ({
|
|
2649
|
+
imageURL: {
|
|
2650
|
+
urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
|
|
2651
|
+
}
|
|
2652
|
+
}))
|
|
2653
|
+
},
|
|
2654
|
+
music: {
|
|
2655
|
+
authorName: string(musicInfo.authorName),
|
|
2656
|
+
playUrl: firstString(musicInfo.playUrl),
|
|
2657
|
+
title: string(musicInfo.musicName)
|
|
2658
|
+
},
|
|
2659
|
+
stats: {
|
|
2660
|
+
commentCount: item.commentCount,
|
|
2661
|
+
diggCount: item.diggCount,
|
|
2662
|
+
playCount: item.playCount,
|
|
2663
|
+
shareCount: item.shareCount
|
|
2664
|
+
},
|
|
2665
|
+
video: {
|
|
2666
|
+
duration: videoMeta.duration
|
|
2522
2667
|
}
|
|
2523
|
-
const pending = contentLength(input.net, candidate.item);
|
|
2524
|
-
sizes.set(candidate.item.url, pending);
|
|
2525
|
-
return pending;
|
|
2526
2668
|
};
|
|
2527
|
-
const highest = groups.map((group) => group[0]);
|
|
2528
|
-
const highestSizes = await Promise.all(highest.map(size));
|
|
2529
|
-
const highestTotal = knownTotal(highestSizes);
|
|
2530
|
-
if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
|
|
2531
|
-
return highest.map(({ item }) => item);
|
|
2532
|
-
}
|
|
2533
|
-
const sized = await Promise.all(groups.map(async (group) => Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) })))));
|
|
2534
|
-
const fitting = bestFitting(sized, input.tryMaxBytes);
|
|
2535
|
-
return (fitting ?? groups.map((group) => group.at(-1))).map(({ item }) => item);
|
|
2536
2669
|
}
|
|
2537
|
-
function
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
function bestFitting(groups, maxBytes) {
|
|
2541
|
-
let selections = [
|
|
2542
|
-
{ bytes: 0, candidates: [], quality: 0 }
|
|
2543
|
-
];
|
|
2544
|
-
for (const group of groups) {
|
|
2545
|
-
selections = selections.flatMap((selection) => group.flatMap((candidate) => {
|
|
2546
|
-
if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
|
|
2547
|
-
return [];
|
|
2548
|
-
}
|
|
2549
|
-
return [{
|
|
2550
|
-
bytes: selection.bytes + candidate.bytes,
|
|
2551
|
-
candidates: [...selection.candidates, candidate],
|
|
2552
|
-
quality: selection.quality + candidate.bitrate
|
|
2553
|
-
}];
|
|
2554
|
-
}));
|
|
2555
|
-
if (selections.length === 0) {
|
|
2556
|
-
return null;
|
|
2557
|
-
}
|
|
2558
|
-
}
|
|
2559
|
-
const best = selections.reduce((current, selection) => selection.quality > current.quality ? selection : current);
|
|
2560
|
-
return best.candidates;
|
|
2670
|
+
function recoverablePageError(error) {
|
|
2671
|
+
const message = error instanceof Error ? error.message : "";
|
|
2672
|
+
return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
|
|
2561
2673
|
}
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
|
|
2565
|
-
const raw = response.ok ? response.headers.get("content-length") : null;
|
|
2566
|
-
if (!raw || !/^\d+$/.test(raw)) {
|
|
2567
|
-
return null;
|
|
2568
|
-
}
|
|
2569
|
-
const bytes = Number(raw);
|
|
2570
|
-
return Number.isSafeInteger(bytes) ? bytes : null;
|
|
2571
|
-
} catch {
|
|
2572
|
-
return null;
|
|
2573
|
-
}
|
|
2674
|
+
function firstString(value) {
|
|
2675
|
+
return Array.isArray(value) ? value.map(string).find((candidate) => candidate !== null) ?? null : string(value);
|
|
2574
2676
|
}
|
|
2575
|
-
function
|
|
2576
|
-
|
|
2677
|
+
function downloadUrl(item) {
|
|
2678
|
+
const video = object(item.video) ? item.video : null;
|
|
2679
|
+
return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
|
|
2577
2680
|
}
|
|
2578
|
-
|
|
2579
|
-
const
|
|
2580
|
-
if (
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
const comments = [];
|
|
2584
|
-
const seen = new Set;
|
|
2585
|
-
const cursors = new Set;
|
|
2586
|
-
let cursor = null;
|
|
2587
|
-
do {
|
|
2588
|
-
const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
|
|
2589
|
-
if (cursor)
|
|
2590
|
-
url.searchParams.set("cursor", cursor);
|
|
2591
|
-
const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
|
|
2592
|
-
if (!response.ok)
|
|
2593
|
-
throw new Error("Comment lookup failed");
|
|
2594
|
-
const payload = await response.json();
|
|
2595
|
-
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
|
|
2596
|
-
throw new Error("Invalid comments");
|
|
2597
|
-
const previousCount = comments.length;
|
|
2598
|
-
for (const reply of payload.replies) {
|
|
2599
|
-
if (!object(reply))
|
|
2600
|
-
continue;
|
|
2601
|
-
const replyId = string(reply.id);
|
|
2602
|
-
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
2603
|
-
if (!replyId || parent !== id || seen.has(replyId))
|
|
2604
|
-
continue;
|
|
2605
|
-
seen.add(replyId);
|
|
2606
|
-
comments.push(twitterComment(reply, replyId));
|
|
2607
|
-
if (comments.length >= limit)
|
|
2608
|
-
return comments;
|
|
2609
|
-
}
|
|
2610
|
-
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
2611
|
-
if (!cursor || cursors.has(cursor) || comments.length === previousCount)
|
|
2612
|
-
break;
|
|
2613
|
-
cursors.add(cursor);
|
|
2614
|
-
} while (comments.length < limit);
|
|
2615
|
-
return comments;
|
|
2616
|
-
} catch {
|
|
2617
|
-
return [];
|
|
2681
|
+
function mediaItems4(item, user, id, headers) {
|
|
2682
|
+
const images = imageItems3(item, user, id, headers);
|
|
2683
|
+
if (images.length > 0) {
|
|
2684
|
+
const audio = audioItem2(item, user, id, headers);
|
|
2685
|
+
return audio ? [...images, audio] : images;
|
|
2618
2686
|
}
|
|
2687
|
+
const url = downloadUrl(item);
|
|
2688
|
+
return url ? [{
|
|
2689
|
+
filename: filename(`tiktok_${user}_${id}.mp4`),
|
|
2690
|
+
headers,
|
|
2691
|
+
id,
|
|
2692
|
+
kind: "video",
|
|
2693
|
+
mime: "video/mp4",
|
|
2694
|
+
platform: "tiktok",
|
|
2695
|
+
url
|
|
2696
|
+
}] : [];
|
|
2619
2697
|
}
|
|
2620
|
-
function
|
|
2621
|
-
const
|
|
2622
|
-
const
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
const
|
|
2626
|
-
const
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2698
|
+
function imageItems3(item, user, id, headers) {
|
|
2699
|
+
const imagePost = object(item.imagePost) ? item.imagePost : null;
|
|
2700
|
+
const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
|
|
2701
|
+
return images.flatMap((image, index) => {
|
|
2702
|
+
const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
|
|
2703
|
+
const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
|
|
2704
|
+
const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
|
|
2705
|
+
return url ? [{
|
|
2706
|
+
filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
|
|
2707
|
+
headers,
|
|
2630
2708
|
id,
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
headers: { "user-agent": browserUserAgent() }
|
|
2637
|
-
}];
|
|
2709
|
+
kind: "image",
|
|
2710
|
+
mime: "image/jpeg",
|
|
2711
|
+
platform: "tiktok",
|
|
2712
|
+
url
|
|
2713
|
+
}] : [];
|
|
2638
2714
|
});
|
|
2715
|
+
}
|
|
2716
|
+
function audioItem2(item, user, id, headers) {
|
|
2717
|
+
const video = object(item.video) ? item.video : null;
|
|
2718
|
+
const music = object(item.music) ? item.music : null;
|
|
2719
|
+
const url = video ? string(video.playAddr) : null;
|
|
2720
|
+
const fallback = music ? string(music.playUrl) : null;
|
|
2721
|
+
const selected = url ?? fallback;
|
|
2722
|
+
if (!selected) {
|
|
2723
|
+
return null;
|
|
2724
|
+
}
|
|
2725
|
+
const extension = selected.includes("mime_type=audio_mpeg") ? "mp3" : "m4a";
|
|
2639
2726
|
return {
|
|
2727
|
+
filename: filename(`tiktok_${user}_${id}_audio.${extension}`),
|
|
2728
|
+
headers,
|
|
2640
2729
|
id,
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
author: {
|
|
2646
|
-
handle: string(author2.screen_name) ?? undefined,
|
|
2647
|
-
name: string(author2.name) ?? undefined,
|
|
2648
|
-
verified: bool(verification.verified)
|
|
2649
|
-
},
|
|
2650
|
-
createdAt: isoFromDateString(reply.created_at),
|
|
2651
|
-
likeCount: count(reply.likes),
|
|
2652
|
-
commentCount: count(reply.replies),
|
|
2653
|
-
shareCount: count(reply.reposts),
|
|
2654
|
-
viewCount: count(reply.views),
|
|
2655
|
-
nsfw: bool(reply.possibly_sensitive)
|
|
2656
|
-
}
|
|
2730
|
+
kind: "audio",
|
|
2731
|
+
mime: extension === "mp3" ? "audio/mpeg" : "audio/mp4",
|
|
2732
|
+
platform: "tiktok",
|
|
2733
|
+
url: selected
|
|
2657
2734
|
};
|
|
2658
2735
|
}
|
|
2736
|
+
function author(item) {
|
|
2737
|
+
const user = object(item.author) ? item.author : null;
|
|
2738
|
+
return user ? string(user.uniqueId) : null;
|
|
2739
|
+
}
|
|
2740
|
+
function username(input) {
|
|
2741
|
+
return asUrl(input).pathname.match(/@([^/]+)/)?.[1] ?? null;
|
|
2742
|
+
}
|
|
2743
|
+
function cookieHeader(headers) {
|
|
2744
|
+
const getter = headers.getSetCookie;
|
|
2745
|
+
const setCookie = typeof getter === "function" ? getter.call(headers).join(",") : headers.get("set-cookie");
|
|
2746
|
+
const cookies = setCookie?.split(/,(?=[^;]+?=)/).map((part) => part.split(";")[0]?.trim()).filter((part) => Boolean(part));
|
|
2747
|
+
return cookies && cookies.length > 0 ? cookies.join("; ") : null;
|
|
2748
|
+
}
|
|
2659
2749
|
|
|
2660
|
-
// src/
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
number: "28",
|
|
2664
|
-
userAgent: "com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip",
|
|
2665
|
-
version: "1.65.10"
|
|
2666
|
-
};
|
|
2667
|
-
var visionOsClient = {
|
|
2668
|
-
clientName: "VISIONOS",
|
|
2669
|
-
clientVersion: "1.02",
|
|
2670
|
-
deviceMake: "Apple",
|
|
2671
|
-
deviceModel: "RealityDevice17,1",
|
|
2672
|
-
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 15_7_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
|
|
2673
|
-
osName: "visionOS",
|
|
2674
|
-
osVersion: "26.5.23O471"
|
|
2675
|
-
};
|
|
2676
|
-
var browserCookie = "PREF=hl=en&tz=UTC; SOCS=CAI";
|
|
2677
|
-
async function resolveYoutube(input) {
|
|
2678
|
-
const id = youtubeVideoId(input.url);
|
|
2750
|
+
// src/twitter.ts
|
|
2751
|
+
async function resolveTwitter(input) {
|
|
2752
|
+
const id = tweetId(input.url);
|
|
2679
2753
|
if (!id) {
|
|
2680
|
-
throw new Error("
|
|
2754
|
+
throw new Error("Tweet id not found");
|
|
2681
2755
|
}
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
const payload = await youtubePlayer(input.net, playerBody(id, session), playerHeaders(session));
|
|
2686
|
-
const streams = selectStreams(payload, input.preferredWidth);
|
|
2687
|
-
if (!streams) {
|
|
2688
|
-
throw new Error("YouTube mp4 stream not found");
|
|
2689
|
-
}
|
|
2690
|
-
const headers = { "user-agent": androidVrClient.userAgent };
|
|
2691
|
-
await Promise.all([streams.video, streams.audio].filter((url) => url !== null).map((url) => probeStream(input.net, url, headers)));
|
|
2692
|
-
return youtubeResult(id, payload, streams, headers);
|
|
2693
|
-
} catch (primaryError) {
|
|
2694
|
-
try {
|
|
2695
|
-
return await resolveVisionOs(input, id, session?.visitorData);
|
|
2696
|
-
} catch (fallbackError) {
|
|
2697
|
-
const message = (error) => error instanceof Error ? error.message : String(error);
|
|
2698
|
-
throw new Error(`YouTube resolution failed: ${message(primaryError)}; visionOS HLS: ${message(fallbackError)}`, { cause: fallbackError });
|
|
2699
|
-
}
|
|
2756
|
+
const tweet = await syndication(input.net, id);
|
|
2757
|
+
if (string(tweet.__typename) === "TweetTombstone") {
|
|
2758
|
+
throw new Error("Tweet is unavailable or age-restricted");
|
|
2700
2759
|
}
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
const
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2760
|
+
await expandTwitterText(input.net, tweet, id);
|
|
2761
|
+
const posts = twitterTweets(tweet, id);
|
|
2762
|
+
const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
|
|
2763
|
+
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
2764
|
+
return media.flatMap((entry, index) => {
|
|
2765
|
+
const candidates = twitterCandidates(entry, postId2, index + 1);
|
|
2766
|
+
return candidates.length > 0 ? [candidates] : [];
|
|
2767
|
+
});
|
|
2707
2768
|
});
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
const status = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.status) : null;
|
|
2713
|
-
if (status !== "OK") {
|
|
2714
|
-
const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
|
|
2715
|
-
throw new Error(reason ?? "YouTube video unavailable");
|
|
2716
|
-
}
|
|
2717
|
-
return payload;
|
|
2718
|
-
}
|
|
2719
|
-
function youtubeResult(id, payload, streams, headers) {
|
|
2720
|
-
const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
|
|
2721
|
-
const media = {
|
|
2722
|
-
filename: filename(`youtube_${title ?? id}_${id}.mp4`),
|
|
2723
|
-
headers,
|
|
2724
|
-
id,
|
|
2725
|
-
kind: "video",
|
|
2726
|
-
mime: "video/mp4",
|
|
2727
|
-
platform: "youtube",
|
|
2728
|
-
url: streams.video,
|
|
2729
|
-
...streams.hls ? { hls: true } : {},
|
|
2730
|
-
...streams.audio ? { audio: { headers, url: streams.audio } } : {}
|
|
2731
|
-
};
|
|
2732
|
-
return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
2733
|
-
}
|
|
2734
|
-
async function probeStream(net, url, headers) {
|
|
2735
|
-
const response = await net(url, { headers }, 1);
|
|
2736
|
-
await response.body?.cancel();
|
|
2737
|
-
if (!response.ok) {
|
|
2738
|
-
throw new Error(`YouTube stream failed: ${response.status}`);
|
|
2769
|
+
for (const { id: postId2, tweet: post } of posts.toReversed()) {
|
|
2770
|
+
const card = twitterCardImage(post, postId2);
|
|
2771
|
+
if (card)
|
|
2772
|
+
groups.push([{ bitrate: 0, item: card }]);
|
|
2739
2773
|
}
|
|
2774
|
+
const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
|
|
2775
|
+
return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
2740
2776
|
}
|
|
2741
|
-
|
|
2742
|
-
const
|
|
2743
|
-
const
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
context: { client: { ...visionOsClient, hl: "en", gl: "US", ...visitorData ? { visitorData } : {} } }
|
|
2748
|
-
}, {
|
|
2749
|
-
...headers,
|
|
2750
|
-
"content-type": "application/json",
|
|
2751
|
-
"x-youtube-client-name": "101",
|
|
2752
|
-
"x-youtube-client-version": visionOsClient.clientVersion,
|
|
2753
|
-
...visitorData ? { "x-goog-visitor-id": visitorData } : {}
|
|
2754
|
-
});
|
|
2755
|
-
const streaming = object(payload) && object(payload.streamingData) ? payload.streamingData : null;
|
|
2756
|
-
const manifestUrl = streaming ? string(streaming.hlsManifestUrl) : null;
|
|
2757
|
-
if (!manifestUrl) {
|
|
2758
|
-
throw new Error("YouTube HLS manifest not found");
|
|
2777
|
+
function twitterMetadata(tweet) {
|
|
2778
|
+
const user = object(tweet.user) ? tweet.user : null;
|
|
2779
|
+
const extra = {};
|
|
2780
|
+
const lang = string(tweet.lang);
|
|
2781
|
+
if (lang) {
|
|
2782
|
+
extra.lang = lang;
|
|
2759
2783
|
}
|
|
2760
|
-
const
|
|
2761
|
-
|
|
2762
|
-
|
|
2784
|
+
const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
|
|
2785
|
+
const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
|
|
2786
|
+
if (quotedTweet && quotedId) {
|
|
2787
|
+
extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
|
|
2763
2788
|
}
|
|
2764
|
-
const
|
|
2765
|
-
const
|
|
2766
|
-
const
|
|
2767
|
-
if (
|
|
2768
|
-
|
|
2789
|
+
const parent = object(tweet.parent) ? tweet.parent : null;
|
|
2790
|
+
const parentId = parent ? string(parent.id_str) : null;
|
|
2791
|
+
const replyToId = string(tweet.in_reply_to_status_id_str) ?? parentId;
|
|
2792
|
+
if (replyToId)
|
|
2793
|
+
extra.replyToId = replyToId;
|
|
2794
|
+
if (parent && parentId) {
|
|
2795
|
+
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
2769
2796
|
}
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2797
|
+
return {
|
|
2798
|
+
text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
|
|
2799
|
+
author: user ? {
|
|
2800
|
+
handle: string(user.screen_name) ?? undefined,
|
|
2801
|
+
name: string(user.name) ?? undefined,
|
|
2802
|
+
verified: bool(user.verified) ?? bool(user.is_blue_verified)
|
|
2803
|
+
} : undefined,
|
|
2804
|
+
createdAt: isoFromDateString(tweet.created_at),
|
|
2805
|
+
likeCount: count(tweet.favorite_count),
|
|
2806
|
+
commentCount: count(tweet.conversation_count),
|
|
2807
|
+
shareCount: count(tweet.retweet_count),
|
|
2808
|
+
viewCount: count(tweet.view_count),
|
|
2809
|
+
nsfw: bool(tweet.possibly_sensitive),
|
|
2810
|
+
extra
|
|
2811
|
+
};
|
|
2812
|
+
}
|
|
2813
|
+
function noteText(tweet) {
|
|
2814
|
+
const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
|
|
2815
|
+
const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
|
|
2816
|
+
const result = results && object(results.result) ? results.result : null;
|
|
2817
|
+
return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
|
|
2818
|
+
}
|
|
2819
|
+
function twitterText(tweet) {
|
|
2820
|
+
return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
|
|
2821
|
+
}
|
|
2822
|
+
function cleanTwitterText(text2, tweet) {
|
|
2823
|
+
if (text2 === null)
|
|
2824
|
+
return null;
|
|
2825
|
+
const entities = object(tweet.entities) ? tweet.entities : null;
|
|
2826
|
+
const media = [
|
|
2827
|
+
...entities && Array.isArray(entities.media) ? entities.media : [],
|
|
2828
|
+
...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
|
|
2829
|
+
];
|
|
2830
|
+
const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
|
|
2831
|
+
while (true) {
|
|
2832
|
+
const trailing = text2.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
|
|
2833
|
+
if (!trailing || !urls.has(trailing[1]))
|
|
2834
|
+
break;
|
|
2835
|
+
text2 = text2.slice(0, trailing.index).trimEnd();
|
|
2773
2836
|
}
|
|
2774
|
-
|
|
2775
|
-
|
|
2837
|
+
return text2.replace(/&(?:amp|gt|lt);/g, (entity) => entity === "&" ? "&" : entity === ">" ? ">" : "<");
|
|
2838
|
+
}
|
|
2839
|
+
function needsFullText(tweet) {
|
|
2840
|
+
if (noteText(tweet) || string(tweet.full_text)) {
|
|
2841
|
+
return false;
|
|
2776
2842
|
}
|
|
2777
|
-
return
|
|
2843
|
+
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
2778
2844
|
}
|
|
2779
|
-
function
|
|
2780
|
-
const
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
return {
|
|
2785
|
-
title: details ? string(details.title) ?? undefined : undefined,
|
|
2786
|
-
text: details ? string(details.shortDescription) ?? undefined : undefined,
|
|
2787
|
-
author: author2 ? { name: author2 } : undefined,
|
|
2788
|
-
createdAt: renderer ? isoFromDateString(renderer.publishDate) : undefined,
|
|
2789
|
-
viewCount: details ? count(details.viewCount) : undefined,
|
|
2790
|
-
extra: {
|
|
2791
|
-
channelId: details ? string(details.channelId) ?? undefined : undefined,
|
|
2792
|
-
durationSeconds: details ? count(details.lengthSeconds) : undefined,
|
|
2793
|
-
keywords: keywords.length > 0 ? keywords : undefined
|
|
2845
|
+
async function expandTwitterText(net, tweet, rootId) {
|
|
2846
|
+
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
2847
|
+
for (const [id, post] of posts) {
|
|
2848
|
+
if (!needsFullText(post)) {
|
|
2849
|
+
continue;
|
|
2794
2850
|
}
|
|
2851
|
+
try {
|
|
2852
|
+
const response = await net(`https://api.fxtwitter.com/status/${id}`, {
|
|
2853
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() }
|
|
2854
|
+
}, 1);
|
|
2855
|
+
if (!response.ok) {
|
|
2856
|
+
continue;
|
|
2857
|
+
}
|
|
2858
|
+
const payload = await response.json();
|
|
2859
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
|
|
2860
|
+
continue;
|
|
2861
|
+
}
|
|
2862
|
+
let current = payload.tweet;
|
|
2863
|
+
const seen = new Set;
|
|
2864
|
+
while (current) {
|
|
2865
|
+
const currentId = string(current.id);
|
|
2866
|
+
if (!currentId || seen.has(currentId)) {
|
|
2867
|
+
break;
|
|
2868
|
+
}
|
|
2869
|
+
seen.add(currentId);
|
|
2870
|
+
const target = posts.get(currentId);
|
|
2871
|
+
const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
|
|
2872
|
+
const text2 = raw ?? string(current.text);
|
|
2873
|
+
if (target && needsFullText(target) && text2 && text2.length > (twitterText(target)?.length ?? 0)) {
|
|
2874
|
+
target.full_text = text2;
|
|
2875
|
+
}
|
|
2876
|
+
current = object(current.quote) ? current.quote : null;
|
|
2877
|
+
}
|
|
2878
|
+
} catch {}
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
function twitterTweets(tweet, rootId, includeParents = false) {
|
|
2882
|
+
const tweets = [];
|
|
2883
|
+
const seen = new Set;
|
|
2884
|
+
const visit = (current, fallbackId) => {
|
|
2885
|
+
const id = string(current.id_str) ?? fallbackId;
|
|
2886
|
+
if (!id || seen.has(id))
|
|
2887
|
+
return;
|
|
2888
|
+
seen.add(id);
|
|
2889
|
+
tweets.push({ id, tweet: current });
|
|
2890
|
+
if (object(current.quoted_tweet))
|
|
2891
|
+
visit(current.quoted_tweet);
|
|
2892
|
+
if (includeParents && object(current.parent))
|
|
2893
|
+
visit(current.parent);
|
|
2795
2894
|
};
|
|
2895
|
+
visit(tweet, rootId);
|
|
2896
|
+
return tweets;
|
|
2796
2897
|
}
|
|
2797
|
-
function
|
|
2798
|
-
const
|
|
2799
|
-
const
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
return { video, audio };
|
|
2898
|
+
async function syndication(net, id) {
|
|
2899
|
+
const url = `https://cdn.syndication.twimg.com/tweet-result?id=${id}&lang=en&token=${syndicationToken(id)}`;
|
|
2900
|
+
const response = await net(url, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
|
|
2901
|
+
if (!response.ok) {
|
|
2902
|
+
throw new Error(`Twitter syndication failed: ${response.status}`);
|
|
2803
2903
|
}
|
|
2804
|
-
const
|
|
2805
|
-
|
|
2904
|
+
const payload = await response.json();
|
|
2905
|
+
if (!object(payload)) {
|
|
2906
|
+
throw new Error("Twitter response invalid");
|
|
2907
|
+
}
|
|
2908
|
+
return payload;
|
|
2806
2909
|
}
|
|
2807
|
-
function
|
|
2808
|
-
|
|
2809
|
-
const streaming = root && object(root.streamingData) ? root.streamingData : null;
|
|
2810
|
-
return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
|
|
2910
|
+
function syndicationToken(id) {
|
|
2911
|
+
return (Number(id) / 1000000000000000 * Math.PI).toString(36).replace(/(0+|\.)/g, "");
|
|
2811
2912
|
}
|
|
2812
|
-
function
|
|
2813
|
-
const
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
|
|
2824
|
-
}, null);
|
|
2825
|
-
return best ? string(best.url) : null;
|
|
2913
|
+
function twitterCandidates(entry, id, index) {
|
|
2914
|
+
const headers = { "user-agent": browserUserAgent() };
|
|
2915
|
+
const type = string(entry.type);
|
|
2916
|
+
if (type === "video" || type === "animated_gif") {
|
|
2917
|
+
return videoVariants(entry).map(({ bitrate, url }) => ({
|
|
2918
|
+
bitrate,
|
|
2919
|
+
item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }
|
|
2920
|
+
}));
|
|
2921
|
+
}
|
|
2922
|
+
const photo = string(entry.media_url_https);
|
|
2923
|
+
return photo ? [{ bitrate: 0, item: { filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` } }] : [];
|
|
2826
2924
|
}
|
|
2827
|
-
function
|
|
2828
|
-
const
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2925
|
+
function twitterCardImage(tweet, id) {
|
|
2926
|
+
const card = object(tweet.card) ? tweet.card : null;
|
|
2927
|
+
if (!card || card.name !== "summary" && card.name !== "summary_large_image")
|
|
2928
|
+
return null;
|
|
2929
|
+
const bindings = object(card.binding_values) ? card.binding_values : {};
|
|
2930
|
+
const images = Object.entries(bindings).flatMap(([key, value]) => {
|
|
2931
|
+
if (!/^(photo_image_full_size|summary_photo_image|thumbnail_image)(_(original|x_large|large|small))?$/.test(key) || !object(value) || !object(value.image_value))
|
|
2932
|
+
return [];
|
|
2933
|
+
const image2 = value.image_value;
|
|
2934
|
+
const rawUrl = string(image2.url);
|
|
2935
|
+
if (!rawUrl)
|
|
2936
|
+
return [];
|
|
2937
|
+
try {
|
|
2938
|
+
const url = new URL(rawUrl);
|
|
2939
|
+
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
2940
|
+
return [];
|
|
2941
|
+
const format = (url.searchParams.get("format") ?? url.pathname.match(/\.(jpe?g|png|webp|gif)$/i)?.[1] ?? "jpg").toLowerCase();
|
|
2942
|
+
const extension = format === "jpeg" ? "jpg" : format;
|
|
2943
|
+
if (!["jpg", "png", "webp", "gif"].includes(extension))
|
|
2944
|
+
return [];
|
|
2945
|
+
return [{
|
|
2946
|
+
url: url.href,
|
|
2947
|
+
extension,
|
|
2948
|
+
original: key.endsWith("_original") ? 1 : 0,
|
|
2949
|
+
area: Math.max(0, number(image2.width) ?? 0) * Math.max(0, number(image2.height) ?? 0)
|
|
2950
|
+
}];
|
|
2951
|
+
} catch {
|
|
2952
|
+
return [];
|
|
2832
2953
|
}
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2954
|
+
}).sort((a, b) => b.original - a.original || b.area - a.area);
|
|
2955
|
+
const image = images[0];
|
|
2956
|
+
return image ? {
|
|
2957
|
+
filename: filename(`twitter_${id}_preview.${image.extension}`),
|
|
2958
|
+
headers: { "user-agent": browserUserAgent() },
|
|
2959
|
+
id,
|
|
2960
|
+
kind: "image",
|
|
2961
|
+
mime: `image/${image.extension === "jpg" ? "jpeg" : image.extension}`,
|
|
2962
|
+
platform: "twitter",
|
|
2963
|
+
role: "linkPreview",
|
|
2964
|
+
url: image.url
|
|
2965
|
+
} : null;
|
|
2966
|
+
}
|
|
2967
|
+
function videoVariants(entry) {
|
|
2968
|
+
const info = object(entry.video_info) ? entry.video_info : null;
|
|
2969
|
+
const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
|
|
2970
|
+
const seen = new Set;
|
|
2971
|
+
return variants.flatMap((variant) => {
|
|
2972
|
+
const url = string(variant.url);
|
|
2973
|
+
if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
|
|
2974
|
+
return [];
|
|
2836
2975
|
}
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2976
|
+
seen.add(url);
|
|
2977
|
+
return [{ bitrate: number(variant.bitrate) ?? 0, url }];
|
|
2978
|
+
}).sort((left, right) => right.bitrate - left.bitrate);
|
|
2840
2979
|
}
|
|
2841
|
-
function
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
return 2;
|
|
2980
|
+
async function sizeLimitedItems(input, groups) {
|
|
2981
|
+
if (groups.length === 0 || input.tryMaxBytes === undefined) {
|
|
2982
|
+
return groups.map((group) => group[0].item);
|
|
2845
2983
|
}
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
context: {
|
|
2852
|
-
client: {
|
|
2853
|
-
androidSdkVersion: 32,
|
|
2854
|
-
clientName: androidVrClient.name,
|
|
2855
|
-
clientVersion: androidVrClient.version,
|
|
2856
|
-
deviceMake: "Oculus",
|
|
2857
|
-
deviceModel: "Quest 3",
|
|
2858
|
-
gl: "US",
|
|
2859
|
-
hl: "en",
|
|
2860
|
-
osName: "Android",
|
|
2861
|
-
osVersion: "12L",
|
|
2862
|
-
timeZone: "UTC",
|
|
2863
|
-
userAgent: androidVrClient.userAgent,
|
|
2864
|
-
utcOffsetMinutes: 0,
|
|
2865
|
-
visitorData: session.visitorData
|
|
2866
|
-
}
|
|
2867
|
-
},
|
|
2868
|
-
playbackContext: {
|
|
2869
|
-
contentPlaybackContext: {
|
|
2870
|
-
html5Preference: "HTML5_PREF_WANTS",
|
|
2871
|
-
signatureTimestamp: session.signatureTimestamp
|
|
2872
|
-
}
|
|
2873
|
-
},
|
|
2874
|
-
racyCheckOk: true,
|
|
2875
|
-
videoId: id
|
|
2876
|
-
};
|
|
2877
|
-
}
|
|
2878
|
-
async function youtubeSession(net, id) {
|
|
2879
|
-
const response = await net(`https://www.youtube.com/watch?v=${id}&bpctr=9999999999&has_verified=1`, {
|
|
2880
|
-
headers: {
|
|
2881
|
-
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
2882
|
-
"accept-language": "en-us,en;q=0.5",
|
|
2883
|
-
cookie: browserCookie,
|
|
2884
|
-
"sec-fetch-mode": "navigate",
|
|
2885
|
-
"user-agent": browserUserAgent()
|
|
2984
|
+
const sizes = new Map;
|
|
2985
|
+
const size = (candidate) => {
|
|
2986
|
+
const existing = sizes.get(candidate.item.url);
|
|
2987
|
+
if (existing) {
|
|
2988
|
+
return existing;
|
|
2886
2989
|
}
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
}
|
|
2891
|
-
const page = await response.text();
|
|
2892
|
-
const visitorData = string(page.match(/"visitorData":"([^"]+)"/)?.[1]);
|
|
2893
|
-
if (!visitorData) {
|
|
2894
|
-
throw new Error("YouTube visitor data not found");
|
|
2895
|
-
}
|
|
2896
|
-
const playerPath = string(page.match(/"jsUrl":"([^"]+)"/)?.[1]) ?? string(page.match(/"PLAYER_JS_URL":"([^"]+)"/)?.[1]);
|
|
2897
|
-
if (!playerPath) {
|
|
2898
|
-
throw new Error("YouTube player url not found");
|
|
2899
|
-
}
|
|
2900
|
-
return {
|
|
2901
|
-
cookie: youtubeCookie(response.headers),
|
|
2902
|
-
signatureTimestamp: await signatureTimestamp(net, playerPath),
|
|
2903
|
-
visitorData
|
|
2990
|
+
const pending = contentLength(input.net, candidate.item);
|
|
2991
|
+
sizes.set(candidate.item.url, pending);
|
|
2992
|
+
return pending;
|
|
2904
2993
|
};
|
|
2994
|
+
const highest = groups.map((group) => group[0]);
|
|
2995
|
+
const highestSizes = await Promise.all(highest.map(size));
|
|
2996
|
+
const highestTotal = knownTotal(highestSizes);
|
|
2997
|
+
if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
|
|
2998
|
+
return highest.map(({ item }) => item);
|
|
2999
|
+
}
|
|
3000
|
+
const sized = await Promise.all(groups.map(async (group) => Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) })))));
|
|
3001
|
+
const fitting = bestFitting(sized, input.tryMaxBytes);
|
|
3002
|
+
return (fitting ?? groups.map((group) => group.at(-1))).map(({ item }) => item);
|
|
2905
3003
|
}
|
|
2906
|
-
function
|
|
2907
|
-
return
|
|
2908
|
-
"content-type": "application/json",
|
|
2909
|
-
cookie: `${browserCookie}; ${session.cookie}`,
|
|
2910
|
-
origin: "https://www.youtube.com",
|
|
2911
|
-
"user-agent": androidVrClient.userAgent,
|
|
2912
|
-
"x-goog-visitor-id": session.visitorData,
|
|
2913
|
-
"x-youtube-client-name": androidVrClient.number,
|
|
2914
|
-
"x-youtube-client-version": androidVrClient.version
|
|
2915
|
-
});
|
|
2916
|
-
}
|
|
2917
|
-
function youtubeCookie(headers) {
|
|
2918
|
-
const readable = headers;
|
|
2919
|
-
const fallback = headers.get("set-cookie");
|
|
2920
|
-
const cookieHeaders = readable.getSetCookie ? readable.getSetCookie() : fallback ? [fallback] : [];
|
|
2921
|
-
return cookieHeaders.map((header) => string(header.split(";", 1)[0])).filter(string).join("; ");
|
|
3004
|
+
function knownTotal(sizes) {
|
|
3005
|
+
return sizes.some((size) => size === null) ? null : sizes.reduce((total, size) => total + (size ?? 0), 0);
|
|
2922
3006
|
}
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
3007
|
+
function bestFitting(groups, maxBytes) {
|
|
3008
|
+
let selections = [
|
|
3009
|
+
{ bytes: 0, candidates: [], quality: 0 }
|
|
3010
|
+
];
|
|
3011
|
+
for (const group of groups) {
|
|
3012
|
+
selections = selections.flatMap((selection) => group.flatMap((candidate) => {
|
|
3013
|
+
if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
|
|
3014
|
+
return [];
|
|
3015
|
+
}
|
|
3016
|
+
return [{
|
|
3017
|
+
bytes: selection.bytes + candidate.bytes,
|
|
3018
|
+
candidates: [...selection.candidates, candidate],
|
|
3019
|
+
quality: selection.quality + candidate.bitrate
|
|
3020
|
+
}];
|
|
3021
|
+
}));
|
|
3022
|
+
if (selections.length === 0) {
|
|
3023
|
+
return null;
|
|
3024
|
+
}
|
|
2934
3025
|
}
|
|
2935
|
-
|
|
3026
|
+
const best = selections.reduce((current, selection) => selection.quality > current.quality ? selection : current);
|
|
3027
|
+
return best.candidates;
|
|
2936
3028
|
}
|
|
2937
|
-
function
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
3029
|
+
async function contentLength(net, item) {
|
|
3030
|
+
try {
|
|
3031
|
+
const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
|
|
3032
|
+
const raw = response.ok ? response.headers.get("content-length") : null;
|
|
3033
|
+
if (!raw || !/^\d+$/.test(raw)) {
|
|
3034
|
+
return null;
|
|
3035
|
+
}
|
|
3036
|
+
const bytes = Number(raw);
|
|
3037
|
+
return Number.isSafeInteger(bytes) ? bytes : null;
|
|
3038
|
+
} catch {
|
|
3039
|
+
return null;
|
|
2945
3040
|
}
|
|
2946
|
-
const parts = url.pathname.split("/").filter(Boolean);
|
|
2947
|
-
const index = parts.findIndex((part) => part === "shorts" || part === "live" || part === "embed");
|
|
2948
|
-
return index >= 0 ? cleanId(parts[index + 1]) : null;
|
|
2949
3041
|
}
|
|
2950
|
-
function
|
|
2951
|
-
return
|
|
3042
|
+
function tweetId(input) {
|
|
3043
|
+
return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
|
|
2952
3044
|
}
|
|
2953
|
-
function
|
|
2954
|
-
const
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
3045
|
+
async function twitterComments(input, id) {
|
|
3046
|
+
const limit = input.comments;
|
|
3047
|
+
if (!limit || !Number.isSafeInteger(limit) || limit < 0)
|
|
3048
|
+
return [];
|
|
3049
|
+
try {
|
|
3050
|
+
const comments = [];
|
|
3051
|
+
const seen = new Set;
|
|
3052
|
+
const cursors = new Set;
|
|
3053
|
+
let cursor = null;
|
|
3054
|
+
do {
|
|
3055
|
+
const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
|
|
3056
|
+
if (cursor)
|
|
3057
|
+
url.searchParams.set("cursor", cursor);
|
|
3058
|
+
const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
|
|
3059
|
+
if (!response.ok)
|
|
3060
|
+
throw new Error("Comment lookup failed");
|
|
3061
|
+
const payload = await response.json();
|
|
3062
|
+
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
|
|
3063
|
+
throw new Error("Invalid comments");
|
|
3064
|
+
const previousCount = comments.length;
|
|
3065
|
+
for (const reply of payload.replies) {
|
|
3066
|
+
if (!object(reply))
|
|
3067
|
+
continue;
|
|
3068
|
+
const replyId = string(reply.id);
|
|
3069
|
+
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
3070
|
+
if (!replyId || parent !== id || seen.has(replyId))
|
|
3071
|
+
continue;
|
|
3072
|
+
seen.add(replyId);
|
|
3073
|
+
comments.push(twitterComment(reply, replyId));
|
|
3074
|
+
if (comments.length >= limit)
|
|
3075
|
+
return comments;
|
|
3076
|
+
}
|
|
3077
|
+
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
3078
|
+
if (!cursor || cursors.has(cursor) || comments.length === previousCount)
|
|
3079
|
+
break;
|
|
3080
|
+
cursors.add(cursor);
|
|
3081
|
+
} while (comments.length < limit);
|
|
3082
|
+
return comments;
|
|
3083
|
+
} catch {
|
|
3084
|
+
return [];
|
|
3085
|
+
}
|
|
2959
3086
|
}
|
|
2960
|
-
function
|
|
2961
|
-
|
|
3087
|
+
function twitterComment(reply, id) {
|
|
3088
|
+
const author2 = object(reply.author) ? reply.author : {};
|
|
3089
|
+
const verification = object(author2.verification) ? author2.verification : {};
|
|
3090
|
+
const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
|
|
3091
|
+
const items = media.flatMap((entry, index) => {
|
|
3092
|
+
const url = string(entry.url);
|
|
3093
|
+
const photo = entry.type === "photo";
|
|
3094
|
+
if (!url || !photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")
|
|
3095
|
+
return [];
|
|
3096
|
+
return [{
|
|
3097
|
+
id,
|
|
3098
|
+
platform: "twitter",
|
|
3099
|
+
filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
|
|
3100
|
+
kind: photo ? "image" : "video",
|
|
3101
|
+
mime: photo ? "image/jpeg" : "video/mp4",
|
|
3102
|
+
url,
|
|
3103
|
+
headers: { "user-agent": browserUserAgent() }
|
|
3104
|
+
}];
|
|
3105
|
+
});
|
|
3106
|
+
return {
|
|
3107
|
+
id,
|
|
3108
|
+
url: `https://x.com/i/status/${id}`,
|
|
3109
|
+
items,
|
|
3110
|
+
metadata: {
|
|
3111
|
+
text: cleanTwitterText(string(reply.text), reply) ?? undefined,
|
|
3112
|
+
author: {
|
|
3113
|
+
handle: string(author2.screen_name) ?? undefined,
|
|
3114
|
+
name: string(author2.name) ?? undefined,
|
|
3115
|
+
verified: bool(verification.verified)
|
|
3116
|
+
},
|
|
3117
|
+
createdAt: isoFromDateString(reply.created_at),
|
|
3118
|
+
likeCount: count(reply.likes),
|
|
3119
|
+
commentCount: count(reply.replies),
|
|
3120
|
+
shareCount: count(reply.reposts),
|
|
3121
|
+
viewCount: count(reply.views),
|
|
3122
|
+
nsfw: bool(reply.possibly_sensitive)
|
|
3123
|
+
}
|
|
3124
|
+
};
|
|
2962
3125
|
}
|
|
2963
3126
|
|
|
2964
3127
|
// src/postfetch.ts
|
|
@@ -2994,6 +3157,8 @@ async function resolve(context) {
|
|
|
2994
3157
|
return resolveReddit(context);
|
|
2995
3158
|
case "soundcloud":
|
|
2996
3159
|
return resolveSoundcloud(context);
|
|
3160
|
+
case "spotify":
|
|
3161
|
+
return resolveSpotify(context);
|
|
2997
3162
|
case "tiktok":
|
|
2998
3163
|
return resolveTiktok(context);
|
|
2999
3164
|
case "twitter":
|
|
@@ -3074,6 +3239,9 @@ function detect(url) {
|
|
|
3074
3239
|
if (host.includes("soundcloud.com") || host === "snd.sc") {
|
|
3075
3240
|
return "soundcloud";
|
|
3076
3241
|
}
|
|
3242
|
+
if (host === "open.spotify.com") {
|
|
3243
|
+
return "spotify";
|
|
3244
|
+
}
|
|
3077
3245
|
if (host.includes("youtube.com") || host === "youtu.be") {
|
|
3078
3246
|
return "youtube";
|
|
3079
3247
|
}
|
|
@@ -3083,7 +3251,7 @@ function detect(url) {
|
|
|
3083
3251
|
if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
|
|
3084
3252
|
return "twitter";
|
|
3085
3253
|
}
|
|
3086
|
-
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
3254
|
+
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube URLs are supported");
|
|
3087
3255
|
}
|
|
3088
3256
|
// src/mp4-remux.ts
|
|
3089
3257
|
async function prepareMp4(bytes, ffmpegPath = "ffmpeg", ffprobePath = "ffprobe", runtime) {
|
|
@@ -3505,10 +3673,15 @@ async function archive(result, options = {}) {
|
|
|
3505
3673
|
return { bytes: zip(files), filename: result.archiveFilename, mime: "application/zip" };
|
|
3506
3674
|
}
|
|
3507
3675
|
async function toResponse(result, options = {}) {
|
|
3508
|
-
|
|
3509
|
-
|
|
3676
|
+
const response = result.items.length === 1 ? await singleResponse(result.items[0], options) : await archiveResponse(result, options);
|
|
3677
|
+
if (result.platform === "spotify") {
|
|
3678
|
+
response.headers.set("x-media-platform", result.platform);
|
|
3679
|
+
response.headers.set("x-media-id", result.id);
|
|
3680
|
+
const youtubeUrl = result.metadata?.extra?.youtubeUrl;
|
|
3681
|
+
if (youtubeUrl)
|
|
3682
|
+
response.headers.set("x-media-youtube-url", youtubeUrl);
|
|
3510
3683
|
}
|
|
3511
|
-
return
|
|
3684
|
+
return response;
|
|
3512
3685
|
}
|
|
3513
3686
|
async function singleResponse(item, options) {
|
|
3514
3687
|
if (buffered(item)) {
|