open-nami 1.0.4 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +731 -142
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -9,7 +9,7 @@ var require_package = __commonJS({
|
|
|
9
9
|
"package.json"(exports, module) {
|
|
10
10
|
module.exports = {
|
|
11
11
|
name: "open-nami",
|
|
12
|
-
version: "1.0.
|
|
12
|
+
version: "1.0.5",
|
|
13
13
|
description: "Open Nami \u2014 terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
|
|
14
14
|
type: "module",
|
|
15
15
|
bin: {
|
|
@@ -1407,10 +1407,11 @@ function parseProfileInput(input) {
|
|
|
1407
1407
|
const reserved = ["home", "explore", "notifications", "messages", "i", "settings"];
|
|
1408
1408
|
if (firstPart && !reserved.includes(firstPart.toLowerCase())) {
|
|
1409
1409
|
const username = firstPart.replace(/^@/, "");
|
|
1410
|
+
const isLikesUrl = pathParts[1]?.toLowerCase() === "likes";
|
|
1410
1411
|
return {
|
|
1411
1412
|
platform: "x",
|
|
1412
1413
|
username: sanitizeUsername(username),
|
|
1413
|
-
cleanUrl: `https://x.com/${username}`,
|
|
1414
|
+
cleanUrl: isLikesUrl ? `https://x.com/${username}/likes` : `https://x.com/${username}`,
|
|
1414
1415
|
rawInput: input
|
|
1415
1416
|
};
|
|
1416
1417
|
}
|
|
@@ -1575,6 +1576,9 @@ function resolvePlatformDownloadsDir(options = {}) {
|
|
|
1575
1576
|
return defaultFallback;
|
|
1576
1577
|
}
|
|
1577
1578
|
|
|
1579
|
+
// src/lib/engines/extractors/instagram.ts
|
|
1580
|
+
import crypto from "crypto";
|
|
1581
|
+
|
|
1578
1582
|
// src/lib/engines/extractors/http.ts
|
|
1579
1583
|
import fs5 from "fs/promises";
|
|
1580
1584
|
import { setTimeout as setTimeoutPromise } from "timers/promises";
|
|
@@ -1762,7 +1766,9 @@ var HttpClient = class {
|
|
|
1762
1766
|
}
|
|
1763
1767
|
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1764
1768
|
}
|
|
1765
|
-
|
|
1769
|
+
const text = await res.text();
|
|
1770
|
+
const cleanText = text.replace(/^\s*(?:for\s*\(\s*;\s*;\s*\);|\)\]\}',)/, "").trim();
|
|
1771
|
+
return JSON.parse(cleanText);
|
|
1766
1772
|
}
|
|
1767
1773
|
async fetchText(url, options = {}) {
|
|
1768
1774
|
const res = await this.request(url, options);
|
|
@@ -1781,13 +1787,13 @@ var HttpClient = class {
|
|
|
1781
1787
|
var IG_APP_ID = "936619743392459";
|
|
1782
1788
|
var IG_TIMELINE_QUERY_HASH = "69cba40317214236af40e7efa697781d";
|
|
1783
1789
|
var IG_QUERIES = {
|
|
1784
|
-
PolarisProfilePostsTabContentQuery_connection: "
|
|
1790
|
+
PolarisProfilePostsTabContentQuery_connection: "39229011733413759",
|
|
1785
1791
|
PolarisProfileReelsTabContentQuery_connection: "28989741540630190",
|
|
1786
1792
|
PolarisProfileTaggedTabContentQuery_connection: "28794602806842110",
|
|
1787
1793
|
PolarisProfileStoryHighlightsTrayContentQuery: "26970053832668570",
|
|
1788
1794
|
PolarisStoriesV3ReelPageGalleryQuery: "28262315486766731",
|
|
1789
1795
|
PolarisStoriesV3TrayContainerQuery: "27703822975903310",
|
|
1790
|
-
PolarisProfilePageContentQuery: "
|
|
1796
|
+
PolarisProfilePageContentQuery: "28649589871350527"
|
|
1791
1797
|
};
|
|
1792
1798
|
var IG_POLARIS_DOC_ID = IG_QUERIES.PolarisProfilePostsTabContentQuery_connection;
|
|
1793
1799
|
function logExtractorDebug(endpoint, err) {
|
|
@@ -1831,6 +1837,21 @@ function parseIgNodeToMediaItems(node, username) {
|
|
|
1831
1837
|
height: video?.height ?? node.dimensions?.height,
|
|
1832
1838
|
thumbnailUrl: node.display_url
|
|
1833
1839
|
});
|
|
1840
|
+
} else {
|
|
1841
|
+
const candidate = node.image_versions2?.candidates?.[0];
|
|
1842
|
+
const photoUrl = candidate?.url || node.display_url;
|
|
1843
|
+
if (photoUrl) {
|
|
1844
|
+
items.push({
|
|
1845
|
+
id: baseId,
|
|
1846
|
+
url: photoUrl,
|
|
1847
|
+
filename: `${username}_${date || baseId}`,
|
|
1848
|
+
extension: "jpg",
|
|
1849
|
+
type: "photo",
|
|
1850
|
+
date,
|
|
1851
|
+
width: candidate?.width ?? node.dimensions?.width,
|
|
1852
|
+
height: candidate?.height ?? node.dimensions?.height
|
|
1853
|
+
});
|
|
1854
|
+
}
|
|
1834
1855
|
}
|
|
1835
1856
|
} else {
|
|
1836
1857
|
const candidate = node.image_versions2?.candidates?.[0];
|
|
@@ -1910,7 +1931,15 @@ var InstagramExtractor = class {
|
|
|
1910
1931
|
if (!res.data?.user) {
|
|
1911
1932
|
throw new Error(`Instagram user not found or private profile: ${username}`);
|
|
1912
1933
|
}
|
|
1913
|
-
|
|
1934
|
+
const user = res.data.user;
|
|
1935
|
+
if (user.is_private) {
|
|
1936
|
+
const fs13 = user.friendship_status;
|
|
1937
|
+
const isFollowing = fs13 ? Boolean(fs13.following) : Boolean(user.followed_by_viewer);
|
|
1938
|
+
if (!isFollowing) {
|
|
1939
|
+
throw new Error(`Instagram profile @${username} is private. You must provide a session cookie jar of an account following this profile.`);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
return user;
|
|
1914
1943
|
}
|
|
1915
1944
|
async fetchHighlightsTray(userId, options) {
|
|
1916
1945
|
try {
|
|
@@ -1932,6 +1961,10 @@ var InstagramExtractor = class {
|
|
|
1932
1961
|
"Referer": "https://www.instagram.com/"
|
|
1933
1962
|
}
|
|
1934
1963
|
});
|
|
1964
|
+
if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
|
|
1965
|
+
const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
|
|
1966
|
+
throw new Error(`Instagram GraphQL query error: ${errMsg}`);
|
|
1967
|
+
}
|
|
1935
1968
|
const edges = res?.data?.highlights?.edges;
|
|
1936
1969
|
if (Array.isArray(edges) && edges.length > 0) {
|
|
1937
1970
|
return edges.map((e) => e.node);
|
|
@@ -1975,87 +2008,207 @@ var InstagramExtractor = class {
|
|
|
1975
2008
|
return [];
|
|
1976
2009
|
}
|
|
1977
2010
|
}
|
|
1978
|
-
async
|
|
1979
|
-
|
|
1980
|
-
|
|
2011
|
+
async fetchWebTokens(username, options) {
|
|
2012
|
+
let csrfToken;
|
|
2013
|
+
if (options.cookiePath) {
|
|
2014
|
+
try {
|
|
2015
|
+
const cookies = await loadCookiesFromPath(options.cookiePath, "instagram.com");
|
|
2016
|
+
csrfToken = cookies["csrftoken"];
|
|
2017
|
+
} catch {
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
if (!csrfToken) {
|
|
2021
|
+
csrfToken = crypto.randomBytes(16).toString("hex");
|
|
2022
|
+
}
|
|
1981
2023
|
try {
|
|
1982
|
-
const
|
|
2024
|
+
const html = await this.httpClient.fetchText(`https://www.instagram.com/${encodeURIComponent(username)}/`, {
|
|
1983
2025
|
cookiePath: options.cookiePath,
|
|
1984
2026
|
cookieDomain: "instagram.com",
|
|
1985
2027
|
signal: options.signal,
|
|
1986
2028
|
headers: {
|
|
1987
|
-
"
|
|
1988
|
-
"Referer": `https://www.instagram.com/${username}/`
|
|
2029
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
|
1989
2030
|
}
|
|
1990
2031
|
});
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
2032
|
+
let lsd;
|
|
2033
|
+
let fbDtsg;
|
|
2034
|
+
let userId;
|
|
2035
|
+
const eqmcMatch = html.match(/id="__eqmc"[^>]*>([^<]+)<\/script>/);
|
|
2036
|
+
if (eqmcMatch) {
|
|
2037
|
+
try {
|
|
2038
|
+
const eqmc = JSON.parse(eqmcMatch[1]);
|
|
2039
|
+
if (eqmc.l) lsd = eqmc.l;
|
|
2040
|
+
if (eqmc.f) fbDtsg = eqmc.f;
|
|
2041
|
+
} catch {
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
if (!lsd) {
|
|
2045
|
+
const m = html.match(/"lsd":"([^"]+)"/) || html.match(/"LSD",\[\],\{"token":"([^"]+)"/);
|
|
2046
|
+
if (m) lsd = m[1];
|
|
2047
|
+
}
|
|
2048
|
+
if (!fbDtsg) {
|
|
2049
|
+
const m = html.match(/"dtsg":\{"token":"([^"]+)"/);
|
|
2050
|
+
if (m) fbDtsg = m[1];
|
|
1998
2051
|
}
|
|
2052
|
+
const uMatch = html.match(/"profile_id":"([^"]+)"/) || html.match(/"user_id":"([^"]+)"/);
|
|
2053
|
+
if (uMatch) userId = uMatch[1];
|
|
2054
|
+
return { lsd, fbDtsg, csrfToken, userId };
|
|
1999
2055
|
} catch (err) {
|
|
2000
|
-
logExtractorDebug("
|
|
2056
|
+
logExtractorDebug("fetchWebTokens", err);
|
|
2057
|
+
return { csrfToken };
|
|
2058
|
+
}
|
|
2059
|
+
}
|
|
2060
|
+
async fetchPostsTabGraphQL(username, cursor, tokens, options) {
|
|
2061
|
+
const postVariables = JSON.stringify({
|
|
2062
|
+
after: cursor || null,
|
|
2063
|
+
before: null,
|
|
2064
|
+
data: {
|
|
2065
|
+
count: 12,
|
|
2066
|
+
include_reel_media_seen_timestamp: true,
|
|
2067
|
+
include_relationship_info: true,
|
|
2068
|
+
latest_besties_reel_media: true,
|
|
2069
|
+
latest_reel_media: true
|
|
2070
|
+
},
|
|
2071
|
+
first: 12,
|
|
2072
|
+
include_multi_captions: true,
|
|
2073
|
+
last: null,
|
|
2074
|
+
username,
|
|
2075
|
+
__relay_internal__pv__PolarisMultiCaptionCarouselEnabledrelayprovider: true,
|
|
2076
|
+
__relay_internal__pv__PolarisShortDramaEnabledrelayprovider: true,
|
|
2077
|
+
__relay_internal__pv__PolarisThreadsStyleCommentsEnabledrelayprovider: false,
|
|
2078
|
+
__relay_internal__pv__PolarisReelsRecoDebugOverlayEnabledrelayprovider: false
|
|
2079
|
+
});
|
|
2080
|
+
const bodyParams = new URLSearchParams({
|
|
2081
|
+
av: "17841415137994167",
|
|
2082
|
+
__d: "www",
|
|
2083
|
+
__user: "0",
|
|
2084
|
+
__a: "1",
|
|
2085
|
+
__req: "0",
|
|
2086
|
+
__hs: "20731.HYP:instagram_web_pkg.2.1...0",
|
|
2087
|
+
dpr: "1",
|
|
2088
|
+
__ccg: "EXCELLENT",
|
|
2089
|
+
__rev: "1049285836",
|
|
2090
|
+
__hsi: "7693193318680449885",
|
|
2091
|
+
__comet_req: "7",
|
|
2092
|
+
jazoest: "26461",
|
|
2093
|
+
fb_api_caller_class: "RelayModern",
|
|
2094
|
+
fb_api_req_friendly_name: "PolarisProfilePostsTabContentQuery_connection",
|
|
2095
|
+
server_timestamps: "true",
|
|
2096
|
+
variables: postVariables,
|
|
2097
|
+
doc_id: IG_POLARIS_DOC_ID
|
|
2098
|
+
});
|
|
2099
|
+
if (tokens.fbDtsg) bodyParams.set("fb_dtsg", tokens.fbDtsg);
|
|
2100
|
+
if (tokens.lsd) bodyParams.set("lsd", tokens.lsd);
|
|
2101
|
+
const headers = {
|
|
2102
|
+
"Accept": "*/*",
|
|
2103
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
2104
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2105
|
+
"X-IG-Max-Touch-Points": "0",
|
|
2106
|
+
"X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
|
|
2107
|
+
"Referer": `https://www.instagram.com/${username}/`,
|
|
2108
|
+
"Origin": "https://www.instagram.com",
|
|
2109
|
+
"X-ASBD-ID": "359341",
|
|
2110
|
+
"Sec-Fetch-Dest": "empty",
|
|
2111
|
+
"Sec-Fetch-Mode": "cors",
|
|
2112
|
+
"Sec-Fetch-Site": "same-origin"
|
|
2113
|
+
};
|
|
2114
|
+
if (tokens.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
|
|
2115
|
+
if (tokens.lsd) headers["X-FB-LSD"] = tokens.lsd;
|
|
2116
|
+
const res = await this.httpClient.fetchJson("https://www.instagram.com/graphql/query", {
|
|
2117
|
+
method: "POST",
|
|
2118
|
+
body: bodyParams.toString(),
|
|
2119
|
+
cookiePath: options.cookiePath,
|
|
2120
|
+
cookieDomain: "instagram.com",
|
|
2121
|
+
signal: options.signal,
|
|
2122
|
+
headers
|
|
2123
|
+
});
|
|
2124
|
+
if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
|
|
2125
|
+
const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
|
|
2126
|
+
throw new Error(`Instagram GraphQL query error: ${errMsg}`);
|
|
2127
|
+
}
|
|
2128
|
+
const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
|
|
2129
|
+
if (connection?.edges && Array.isArray(connection.edges)) {
|
|
2130
|
+
let resolvedUserId;
|
|
2131
|
+
for (const edge of connection.edges) {
|
|
2132
|
+
const node = edge?.node;
|
|
2133
|
+
if (node?.owner_id) {
|
|
2134
|
+
resolvedUserId = String(node.owner_id);
|
|
2135
|
+
break;
|
|
2136
|
+
}
|
|
2137
|
+
if (node?.user?.pk || node?.user?.id) {
|
|
2138
|
+
resolvedUserId = String(node.user.pk || node.user.id);
|
|
2139
|
+
break;
|
|
2140
|
+
}
|
|
2141
|
+
if (typeof node?.id === "string" && node.id.includes("_")) {
|
|
2142
|
+
resolvedUserId = node.id.split("_")[1];
|
|
2143
|
+
break;
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
return {
|
|
2147
|
+
edges: connection.edges,
|
|
2148
|
+
hasNextPage: Boolean(connection.page_info?.has_next_page),
|
|
2149
|
+
nextCursor: connection.page_info?.end_cursor,
|
|
2150
|
+
userId: resolvedUserId
|
|
2151
|
+
};
|
|
2001
2152
|
}
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2153
|
+
return { edges: [], hasNextPage: false };
|
|
2154
|
+
}
|
|
2155
|
+
async fetchNextPostsPage(userId, username, cursor, options, tokens) {
|
|
2156
|
+
if (userId) {
|
|
2157
|
+
const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
|
|
2158
|
+
const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
|
|
2159
|
+
try {
|
|
2160
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
2161
|
+
cookiePath: options.cookiePath,
|
|
2162
|
+
cookieDomain: "instagram.com",
|
|
2163
|
+
signal: options.signal,
|
|
2164
|
+
headers: {
|
|
2165
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2166
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
2167
|
+
}
|
|
2168
|
+
});
|
|
2169
|
+
const data = res?.data?.user?.edge_owner_to_timeline_media;
|
|
2170
|
+
if (data?.edges && data.edges.length > 0) {
|
|
2171
|
+
return {
|
|
2172
|
+
edges: data.edges,
|
|
2173
|
+
hasNextPage: Boolean(data.page_info?.has_next_page),
|
|
2174
|
+
nextCursor: data.page_info?.end_cursor
|
|
2175
|
+
};
|
|
2025
2176
|
}
|
|
2026
|
-
})
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2177
|
+
} catch (err) {
|
|
2178
|
+
logExtractorDebug("edge_owner_to_timeline_media", err);
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
try {
|
|
2182
|
+
const page = await this.fetchPostsTabGraphQL(username, cursor, tokens || {}, options);
|
|
2183
|
+
if (page.edges && page.edges.length > 0) {
|
|
2184
|
+
return page;
|
|
2034
2185
|
}
|
|
2035
2186
|
} catch (err) {
|
|
2036
2187
|
logExtractorDebug("PolarisProfilePostsTabContentQuery_connection", err);
|
|
2037
2188
|
}
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2189
|
+
if (userId) {
|
|
2190
|
+
try {
|
|
2191
|
+
const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
|
|
2192
|
+
const res = await this.httpClient.fetchJson(restUrl, {
|
|
2193
|
+
cookiePath: options.cookiePath,
|
|
2194
|
+
cookieDomain: "instagram.com",
|
|
2195
|
+
signal: options.signal,
|
|
2196
|
+
headers: {
|
|
2197
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2198
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
2199
|
+
}
|
|
2200
|
+
});
|
|
2201
|
+
if (Array.isArray(res?.items) && res.items.length > 0) {
|
|
2202
|
+
const edges = res.items.map((item) => ({ node: item }));
|
|
2203
|
+
return {
|
|
2204
|
+
edges,
|
|
2205
|
+
hasNextPage: Boolean(res.more_available),
|
|
2206
|
+
nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
|
|
2207
|
+
};
|
|
2047
2208
|
}
|
|
2048
|
-
})
|
|
2049
|
-
|
|
2050
|
-
const edges = res.items.map((item) => ({ node: item }));
|
|
2051
|
-
return {
|
|
2052
|
-
edges,
|
|
2053
|
-
hasNextPage: Boolean(res.more_available),
|
|
2054
|
-
nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
|
|
2055
|
-
};
|
|
2209
|
+
} catch (err) {
|
|
2210
|
+
logExtractorDebug("v1 user feed REST", err);
|
|
2056
2211
|
}
|
|
2057
|
-
} catch (err) {
|
|
2058
|
-
logExtractorDebug("v1 user feed REST", err);
|
|
2059
2212
|
}
|
|
2060
2213
|
return { edges: [], hasNextPage: false };
|
|
2061
2214
|
}
|
|
@@ -2089,6 +2242,10 @@ var InstagramExtractor = class {
|
|
|
2089
2242
|
"Referer": "https://www.instagram.com/"
|
|
2090
2243
|
}
|
|
2091
2244
|
});
|
|
2245
|
+
if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
|
|
2246
|
+
const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
|
|
2247
|
+
throw new Error(`Instagram GraphQL query error: ${errMsg}`);
|
|
2248
|
+
}
|
|
2092
2249
|
const reelsData = res?.data?.fetch__XDTUserDict;
|
|
2093
2250
|
let connection = reelsData?.reels;
|
|
2094
2251
|
if (!connection && reelsData) {
|
|
@@ -2119,9 +2276,29 @@ var InstagramExtractor = class {
|
|
|
2119
2276
|
found: 0,
|
|
2120
2277
|
message: `Fetching Instagram profile @${username}...`
|
|
2121
2278
|
});
|
|
2122
|
-
|
|
2123
|
-
|
|
2279
|
+
let user = null;
|
|
2280
|
+
try {
|
|
2281
|
+
user = await this.fetchUserProfile(username, options);
|
|
2282
|
+
} catch (err) {
|
|
2283
|
+
logExtractorDebug("fetchUserProfile", err);
|
|
2284
|
+
}
|
|
2285
|
+
let webTokens = {};
|
|
2286
|
+
if (!user) {
|
|
2287
|
+
webTokens = await this.fetchWebTokens(username, options);
|
|
2288
|
+
}
|
|
2289
|
+
let userId = user?.id || webTokens.userId;
|
|
2290
|
+
if (!userId && (subDir === "Stories" || subDir === "Highlights")) {
|
|
2291
|
+
try {
|
|
2292
|
+
const probe = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
|
|
2293
|
+
if (probe.userId) {
|
|
2294
|
+
userId = probe.userId;
|
|
2295
|
+
}
|
|
2296
|
+
} catch (err) {
|
|
2297
|
+
logExtractorDebug("userId probe", err);
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2124
2300
|
if (subDir === "Stories") {
|
|
2301
|
+
if (!userId) return;
|
|
2125
2302
|
options.onProgress?.({
|
|
2126
2303
|
stage: "Stories",
|
|
2127
2304
|
found: 0,
|
|
@@ -2137,6 +2314,7 @@ var InstagramExtractor = class {
|
|
|
2137
2314
|
return;
|
|
2138
2315
|
}
|
|
2139
2316
|
if (subDir === "Highlights") {
|
|
2317
|
+
if (!userId) return;
|
|
2140
2318
|
options.onProgress?.({
|
|
2141
2319
|
stage: "Highlights",
|
|
2142
2320
|
found: 0,
|
|
@@ -2163,9 +2341,22 @@ var InstagramExtractor = class {
|
|
|
2163
2341
|
}
|
|
2164
2342
|
let found = 0;
|
|
2165
2343
|
const emittedItemIds = /* @__PURE__ */ new Set();
|
|
2166
|
-
let currentEdges = user
|
|
2167
|
-
let hasNextPage = Boolean(user
|
|
2168
|
-
let endCursor = user
|
|
2344
|
+
let currentEdges = user?.edge_owner_to_timeline_media?.edges ?? [];
|
|
2345
|
+
let hasNextPage = Boolean(user?.edge_owner_to_timeline_media?.page_info?.has_next_page);
|
|
2346
|
+
let endCursor = user?.edge_owner_to_timeline_media?.page_info?.end_cursor;
|
|
2347
|
+
if (!user) {
|
|
2348
|
+
try {
|
|
2349
|
+
const firstPage = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
|
|
2350
|
+
currentEdges = firstPage.edges;
|
|
2351
|
+
hasNextPage = firstPage.hasNextPage;
|
|
2352
|
+
endCursor = firstPage.nextCursor;
|
|
2353
|
+
if (!userId && firstPage.userId) {
|
|
2354
|
+
userId = firstPage.userId;
|
|
2355
|
+
}
|
|
2356
|
+
} catch (err) {
|
|
2357
|
+
logExtractorDebug("fetchPostsTabGraphQL initial", err);
|
|
2358
|
+
}
|
|
2359
|
+
}
|
|
2169
2360
|
while (true) {
|
|
2170
2361
|
for (const edge of currentEdges) {
|
|
2171
2362
|
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
@@ -2188,7 +2379,7 @@ var InstagramExtractor = class {
|
|
|
2188
2379
|
}
|
|
2189
2380
|
await sleep(1500, 2500, options.signal);
|
|
2190
2381
|
try {
|
|
2191
|
-
const nextPage = await this.fetchNextPostsPage(userId, username, endCursor, options);
|
|
2382
|
+
const nextPage = await this.fetchNextPostsPage(userId || "", username, endCursor, options, webTokens);
|
|
2192
2383
|
currentEdges = nextPage.edges;
|
|
2193
2384
|
hasNextPage = nextPage.hasNextPage;
|
|
2194
2385
|
endCursor = nextPage.nextCursor;
|
|
@@ -2199,8 +2390,8 @@ var InstagramExtractor = class {
|
|
|
2199
2390
|
break;
|
|
2200
2391
|
}
|
|
2201
2392
|
}
|
|
2202
|
-
if (subDir === "Videos") {
|
|
2203
|
-
const videoEdges = user
|
|
2393
|
+
if (subDir === "Videos" && userId) {
|
|
2394
|
+
const videoEdges = user?.edge_felix_video_timeline?.edges ?? [];
|
|
2204
2395
|
for (const edge of videoEdges) {
|
|
2205
2396
|
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2206
2397
|
for (const item of items) {
|
|
@@ -2272,8 +2463,43 @@ var TWITTER_FEATURES = {
|
|
|
2272
2463
|
responsive_web_text_conversations_enabled: false,
|
|
2273
2464
|
longform_notetweets_rich_text_read_enabled: true,
|
|
2274
2465
|
longform_notetweets_inline_media_enabled: true,
|
|
2275
|
-
responsive_web_enhance_cards_enabled: false
|
|
2466
|
+
responsive_web_enhance_cards_enabled: false,
|
|
2467
|
+
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
2468
|
+
communities_web_enable_tweet_community_results_fetch: true,
|
|
2469
|
+
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
|
2470
|
+
articles_preview_enabled: true,
|
|
2471
|
+
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
|
2472
|
+
responsive_web_grok_analyze_post_followups_enabled: true,
|
|
2473
|
+
responsive_web_jetfuel_frame: true,
|
|
2474
|
+
responsive_web_grok_share_attachment_enabled: true,
|
|
2475
|
+
responsive_web_grok_analysis_button_from_backend: true,
|
|
2476
|
+
responsive_web_grok_image_annotation_enabled: true,
|
|
2477
|
+
responsive_web_grok_imagine_annotation_enabled: true,
|
|
2478
|
+
profile_label_improvements_pcf_label_in_post_enabled: true,
|
|
2479
|
+
rweb_tipjar_consumption_enabled: true
|
|
2276
2480
|
};
|
|
2481
|
+
function shouldSkipTweetReply(tweet, targetUserId, mode = "timeline") {
|
|
2482
|
+
const tweetResult = tweet.tweet || tweet;
|
|
2483
|
+
const legacy = tweetResult.legacy || tweetResult;
|
|
2484
|
+
const replyToUserId = legacy.in_reply_to_user_id_str;
|
|
2485
|
+
if (!replyToUserId) {
|
|
2486
|
+
return false;
|
|
2487
|
+
}
|
|
2488
|
+
const authorId = legacy.user_id_str || tweetResult.core?.user_results?.result?.rest_id;
|
|
2489
|
+
if (mode === "timeline") {
|
|
2490
|
+
if (targetUserId) {
|
|
2491
|
+
if (replyToUserId !== targetUserId || authorId && authorId !== targetUserId) {
|
|
2492
|
+
return true;
|
|
2493
|
+
}
|
|
2494
|
+
return false;
|
|
2495
|
+
}
|
|
2496
|
+
return Boolean(authorId && replyToUserId !== authorId);
|
|
2497
|
+
}
|
|
2498
|
+
if (authorId && replyToUserId === authorId) {
|
|
2499
|
+
return false;
|
|
2500
|
+
}
|
|
2501
|
+
return true;
|
|
2502
|
+
}
|
|
2277
2503
|
function parseTweetMediaEntities(tweet, username) {
|
|
2278
2504
|
const items = [];
|
|
2279
2505
|
const tweetResult = tweet.tweet || tweet;
|
|
@@ -2368,11 +2594,17 @@ var XExtractor = class {
|
|
|
2368
2594
|
}
|
|
2369
2595
|
if (username === "i_history_likes") {
|
|
2370
2596
|
if (options.cookiePath) {
|
|
2371
|
-
|
|
2597
|
+
let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
|
|
2598
|
+
if (!cookies["twid"]) {
|
|
2599
|
+
cookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
|
|
2600
|
+
}
|
|
2601
|
+
if (!cookies["twid"]) {
|
|
2602
|
+
cookies = await loadCookiesFromPath(options.cookiePath);
|
|
2603
|
+
}
|
|
2372
2604
|
const twid = cookies["twid"];
|
|
2373
2605
|
if (twid) {
|
|
2374
2606
|
const decoded = decodeURIComponent(twid).replace(/^"|"$/g, "").trim();
|
|
2375
|
-
const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded;
|
|
2607
|
+
const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded.includes("=") ? decoded.split("=").pop()?.trim() : decoded;
|
|
2376
2608
|
if (rawId && /^[a-zA-Z0-9_-]+$/.test(rawId)) {
|
|
2377
2609
|
return rawId;
|
|
2378
2610
|
}
|
|
@@ -2414,7 +2646,21 @@ var XExtractor = class {
|
|
|
2414
2646
|
async *extract(profile, options) {
|
|
2415
2647
|
const username = profile.username;
|
|
2416
2648
|
const subDir = options.subDir;
|
|
2417
|
-
const isLikes = username === "i_history_likes";
|
|
2649
|
+
const isLikes = username === "i_history_likes" || profile.cleanUrl?.endsWith("/likes");
|
|
2650
|
+
const authHeaders = {};
|
|
2651
|
+
if (options.cookiePath) {
|
|
2652
|
+
let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
|
|
2653
|
+
if (!cookies["ct0"] && !cookies["auth_token"]) {
|
|
2654
|
+
const twCookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
|
|
2655
|
+
cookies = { ...twCookies, ...cookies };
|
|
2656
|
+
}
|
|
2657
|
+
if (cookies["ct0"]) {
|
|
2658
|
+
authHeaders["x-csrf-token"] = cookies["ct0"];
|
|
2659
|
+
}
|
|
2660
|
+
if (cookies["auth_token"]) {
|
|
2661
|
+
authHeaders["x-twitter-auth-type"] = "OAuth2Session";
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2418
2664
|
options.onProgress?.({
|
|
2419
2665
|
stage: isLikes ? "Likes" : "Profile",
|
|
2420
2666
|
found: 0,
|
|
@@ -2445,11 +2691,15 @@ var XExtractor = class {
|
|
|
2445
2691
|
res = await this.httpClient.fetchJson(url, {
|
|
2446
2692
|
cookiePath: options.cookiePath,
|
|
2447
2693
|
cookieDomain: "x.com",
|
|
2448
|
-
signal: options.signal
|
|
2694
|
+
signal: options.signal,
|
|
2695
|
+
headers: authHeaders
|
|
2449
2696
|
});
|
|
2450
2697
|
} catch (err) {
|
|
2451
2698
|
break;
|
|
2452
2699
|
}
|
|
2700
|
+
if (res?.errors?.length && !res?.data?.user?.result) {
|
|
2701
|
+
throw new Error(`X/Twitter API error: ${res.errors[0]?.message || "GraphQL query failed"}`);
|
|
2702
|
+
}
|
|
2453
2703
|
const instructions = res?.data?.user?.result?.timeline_v2?.timeline?.instructions || [];
|
|
2454
2704
|
let newCursor;
|
|
2455
2705
|
let pageItemCount = 0;
|
|
@@ -2462,6 +2712,9 @@ var XExtractor = class {
|
|
|
2462
2712
|
}
|
|
2463
2713
|
const tweetContent = entry.content?.itemContent?.tweet_results?.result;
|
|
2464
2714
|
if (tweetContent) {
|
|
2715
|
+
if (shouldSkipTweetReply(tweetContent, userId, isLikes ? "likes" : "timeline")) {
|
|
2716
|
+
continue;
|
|
2717
|
+
}
|
|
2465
2718
|
const mediaItems = parseTweetMediaEntities(tweetContent, username);
|
|
2466
2719
|
for (const item of mediaItems) {
|
|
2467
2720
|
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
@@ -2490,16 +2743,86 @@ var XExtractor = class {
|
|
|
2490
2743
|
|
|
2491
2744
|
// src/lib/engines/extractors/tiktok.ts
|
|
2492
2745
|
function parseTikTokRehydrationJson(html) {
|
|
2493
|
-
const
|
|
2746
|
+
const universalMatch = html.match(
|
|
2494
2747
|
/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
|
|
2495
2748
|
);
|
|
2496
|
-
if (
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2749
|
+
if (universalMatch) {
|
|
2750
|
+
try {
|
|
2751
|
+
const raw = JSON.parse(universalMatch[1]);
|
|
2752
|
+
const scope = raw?.["__DEFAULT_SCOPE__"] || raw;
|
|
2753
|
+
if (scope) return scope;
|
|
2754
|
+
} catch {
|
|
2755
|
+
}
|
|
2502
2756
|
}
|
|
2757
|
+
const sigiTagMatch = html.match(
|
|
2758
|
+
/<script id="SIGI_STATE"[^>]*>([\s\S]*?)<\/script>/
|
|
2759
|
+
);
|
|
2760
|
+
if (sigiTagMatch) {
|
|
2761
|
+
try {
|
|
2762
|
+
const raw = JSON.parse(sigiTagMatch[1]);
|
|
2763
|
+
const user = Object.values(raw?.UserModule?.users || {})[0];
|
|
2764
|
+
const items = Object.values(raw?.ItemModule || {});
|
|
2765
|
+
return {
|
|
2766
|
+
"webapp.user-detail": {
|
|
2767
|
+
userInfo: { user },
|
|
2768
|
+
itemList: items
|
|
2769
|
+
},
|
|
2770
|
+
SIGI_STATE: raw
|
|
2771
|
+
};
|
|
2772
|
+
} catch {
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2775
|
+
const sigiWindowMatch = html.match(
|
|
2776
|
+
/(?:window\[['"]SIGI_STATE['"]\]|SIGI_STATE)\s*=\s*(\{[\s\S]*?\})\s*;/i
|
|
2777
|
+
);
|
|
2778
|
+
if (sigiWindowMatch) {
|
|
2779
|
+
try {
|
|
2780
|
+
const raw = JSON.parse(sigiWindowMatch[1]);
|
|
2781
|
+
const user = Object.values(raw?.UserModule?.users || {})[0];
|
|
2782
|
+
const items = Object.values(raw?.ItemModule || {});
|
|
2783
|
+
return {
|
|
2784
|
+
"webapp.user-detail": {
|
|
2785
|
+
userInfo: { user },
|
|
2786
|
+
itemList: items
|
|
2787
|
+
},
|
|
2788
|
+
SIGI_STATE: raw
|
|
2789
|
+
};
|
|
2790
|
+
} catch {
|
|
2791
|
+
}
|
|
2792
|
+
}
|
|
2793
|
+
const nextDataMatch = html.match(
|
|
2794
|
+
/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/
|
|
2795
|
+
);
|
|
2796
|
+
if (nextDataMatch) {
|
|
2797
|
+
try {
|
|
2798
|
+
const raw = JSON.parse(nextDataMatch[1]);
|
|
2799
|
+
const pageProps = raw?.props?.pageProps;
|
|
2800
|
+
if (pageProps) {
|
|
2801
|
+
return {
|
|
2802
|
+
"webapp.user-detail": {
|
|
2803
|
+
userInfo: pageProps.userInfo,
|
|
2804
|
+
itemList: pageProps.items || pageProps.itemList || []
|
|
2805
|
+
},
|
|
2806
|
+
__NEXT_DATA__: raw
|
|
2807
|
+
};
|
|
2808
|
+
}
|
|
2809
|
+
} catch {
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
const secUidMatch = html.match(/(?:secUid|sec_uid)["':=\s]+(MS4wLjABAAAA[\w-]{32,64})/i);
|
|
2813
|
+
if (secUidMatch) {
|
|
2814
|
+
return {
|
|
2815
|
+
"webapp.user-detail": {
|
|
2816
|
+
userInfo: {
|
|
2817
|
+
user: {
|
|
2818
|
+
secUid: secUidMatch[1]
|
|
2819
|
+
}
|
|
2820
|
+
},
|
|
2821
|
+
itemList: []
|
|
2822
|
+
}
|
|
2823
|
+
};
|
|
2824
|
+
}
|
|
2825
|
+
return null;
|
|
2503
2826
|
}
|
|
2504
2827
|
function parseTikTokItem(item, username) {
|
|
2505
2828
|
const items = [];
|
|
@@ -2596,13 +2919,28 @@ var TikTokExtractor = class {
|
|
|
2596
2919
|
if (options.signal?.aborted) break;
|
|
2597
2920
|
if (visitedCursors.has(cursor)) break;
|
|
2598
2921
|
visitedCursors.add(cursor);
|
|
2599
|
-
const
|
|
2922
|
+
const deviceId = `${Math.floor(725e16 + Math.random() * 75e15)}`;
|
|
2923
|
+
const queryParams = new URLSearchParams({
|
|
2924
|
+
aid: "1988",
|
|
2925
|
+
app_name: "tiktok_web",
|
|
2926
|
+
device_platform: "web_pc",
|
|
2927
|
+
channel: "tiktok_web",
|
|
2928
|
+
device_id: deviceId,
|
|
2929
|
+
secUid,
|
|
2930
|
+
count: "30",
|
|
2931
|
+
cursor,
|
|
2932
|
+
post_item_list_request_type: "0"
|
|
2933
|
+
});
|
|
2934
|
+
const apiUrl = `https://www.tiktok.com/api/post/item_list/?${queryParams.toString()}`;
|
|
2600
2935
|
let res;
|
|
2601
2936
|
try {
|
|
2602
2937
|
res = await this.httpClient.fetchJson(apiUrl, {
|
|
2603
2938
|
cookiePath: options.cookiePath,
|
|
2604
2939
|
cookieDomain: "tiktok.com",
|
|
2605
|
-
signal: options.signal
|
|
2940
|
+
signal: options.signal,
|
|
2941
|
+
headers: {
|
|
2942
|
+
"Referer": profileUrl
|
|
2943
|
+
}
|
|
2606
2944
|
});
|
|
2607
2945
|
} catch {
|
|
2608
2946
|
break;
|
|
@@ -2654,28 +2992,55 @@ function decodeEscapedUrl(raw) {
|
|
|
2654
2992
|
function extractFacebookMediaFromHtml(html, username) {
|
|
2655
2993
|
const items = [];
|
|
2656
2994
|
const seenUrls = /* @__PURE__ */ new Set();
|
|
2657
|
-
const
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2995
|
+
const videoPatterns = [
|
|
2996
|
+
/"browser_native_hd_url":"([^"]+)"/g,
|
|
2997
|
+
/"browser_native_sd_url":"([^"]+)"/g,
|
|
2998
|
+
/"playable_url_quality_hd":"([^"]+)"/g,
|
|
2999
|
+
/"playable_url":"([^"]+)"/g,
|
|
3000
|
+
/<video[^>]+src=["'](https?:[^"']+)["']/gi
|
|
3001
|
+
];
|
|
3002
|
+
let videoIndex = 1;
|
|
3003
|
+
for (const regex of videoPatterns) {
|
|
3004
|
+
let vMatch;
|
|
3005
|
+
while ((vMatch = regex.exec(html)) !== null) {
|
|
3006
|
+
const rawUrl = vMatch[1];
|
|
3007
|
+
const videoUrl = decodeEscapedUrl(rawUrl).replace(/&/g, "&");
|
|
3008
|
+
if (videoUrl.startsWith("http") && !seenUrls.has(videoUrl)) {
|
|
3009
|
+
seenUrls.add(videoUrl);
|
|
3010
|
+
items.push({
|
|
3011
|
+
id: `fb_video_${videoIndex}`,
|
|
3012
|
+
url: videoUrl,
|
|
3013
|
+
filename: `${username}_video_${videoIndex}`,
|
|
3014
|
+
extension: "mp4",
|
|
3015
|
+
type: "video"
|
|
3016
|
+
});
|
|
3017
|
+
videoIndex++;
|
|
3018
|
+
}
|
|
2671
3019
|
}
|
|
2672
3020
|
}
|
|
2673
|
-
const photoRegex = /,"image":\{"uri":"([^"]+)"/g;
|
|
3021
|
+
const photoRegex = /,"(?:image|viewer_image|full_size_image|display_image)":\{"uri":"([^"]+)"/g;
|
|
2674
3022
|
let match;
|
|
2675
3023
|
let photoIndex = 1;
|
|
2676
3024
|
while ((match = photoRegex.exec(html)) !== null) {
|
|
2677
3025
|
const rawUrl = match[1];
|
|
2678
|
-
const photoUrl = decodeEscapedUrl(rawUrl);
|
|
3026
|
+
const photoUrl = decodeEscapedUrl(rawUrl).replace(/&/g, "&");
|
|
3027
|
+
if (!seenUrls.has(photoUrl)) {
|
|
3028
|
+
seenUrls.add(photoUrl);
|
|
3029
|
+
items.push({
|
|
3030
|
+
id: `fb_photo_${photoIndex}`,
|
|
3031
|
+
url: photoUrl,
|
|
3032
|
+
filename: `${username}_photo_${photoIndex}`,
|
|
3033
|
+
extension: "jpg",
|
|
3034
|
+
type: "photo"
|
|
3035
|
+
});
|
|
3036
|
+
photoIndex++;
|
|
3037
|
+
}
|
|
3038
|
+
}
|
|
3039
|
+
const mbasicImgRegex = /<img[^>]+src=["'](https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*)["']/gi;
|
|
3040
|
+
let mMatch;
|
|
3041
|
+
while ((mMatch = mbasicImgRegex.exec(html)) !== null) {
|
|
3042
|
+
const rawUrl = mMatch[1];
|
|
3043
|
+
const photoUrl = decodeEscapedUrl(rawUrl).replace(/&/g, "&");
|
|
2679
3044
|
if (!seenUrls.has(photoUrl)) {
|
|
2680
3045
|
seenUrls.add(photoUrl);
|
|
2681
3046
|
items.push({
|
|
@@ -2689,10 +3054,10 @@ function extractFacebookMediaFromHtml(html, username) {
|
|
|
2689
3054
|
}
|
|
2690
3055
|
}
|
|
2691
3056
|
if (items.length === 0) {
|
|
2692
|
-
const scontentRegex = /https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*/gi;
|
|
3057
|
+
const scontentRegex = /https:\/\/[^"'\s<>]*scontent[^"'\s<>]*\.(?:jpg|png|webp)[^"'\s<>]*/gi;
|
|
2693
3058
|
let sMatch;
|
|
2694
3059
|
while ((sMatch = scontentRegex.exec(html)) !== null) {
|
|
2695
|
-
const cleanUrl = sMatch[0].replace(/\\/g, "");
|
|
3060
|
+
const cleanUrl = sMatch[0].replace(/\\/g, "").replace(/&/g, "&");
|
|
2696
3061
|
if (!seenUrls.has(cleanUrl)) {
|
|
2697
3062
|
seenUrls.add(cleanUrl);
|
|
2698
3063
|
items.push({
|
|
@@ -2708,6 +3073,59 @@ function extractFacebookMediaFromHtml(html, username) {
|
|
|
2708
3073
|
}
|
|
2709
3074
|
return items;
|
|
2710
3075
|
}
|
|
3076
|
+
function extractFacebookPaginationCursor(html) {
|
|
3077
|
+
const jsonCursorMatch = html.match(/"end_cursor":"([^"]+)"/) || html.match(/"cursor":"([^"]+)"/);
|
|
3078
|
+
if (jsonCursorMatch && jsonCursorMatch[1]) {
|
|
3079
|
+
return jsonCursorMatch[1];
|
|
3080
|
+
}
|
|
3081
|
+
const linkCursorMatch = html.match(/[?&](?:cursor|after)=([^"'\s&<>]+)/i);
|
|
3082
|
+
if (linkCursorMatch && linkCursorMatch[1]) {
|
|
3083
|
+
try {
|
|
3084
|
+
return decodeURIComponent(linkCursorMatch[1]);
|
|
3085
|
+
} catch {
|
|
3086
|
+
return linkCursorMatch[1];
|
|
3087
|
+
}
|
|
3088
|
+
}
|
|
3089
|
+
return null;
|
|
3090
|
+
}
|
|
3091
|
+
function resolveFacebookUrl(href, currentUrl) {
|
|
3092
|
+
const cleanHref = href.replace(/\\/g, "").replace(/&/g, "&");
|
|
3093
|
+
try {
|
|
3094
|
+
return new URL(cleanHref, currentUrl).toString();
|
|
3095
|
+
} catch {
|
|
3096
|
+
return cleanHref;
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
function extractFacebookNextPageUrl(html, currentUrl) {
|
|
3100
|
+
const moreItemMatch = html.match(/<div[^>]*id=["'](?:m_more_item|m_more_posts)["'][^>]*>\s*<a[^>]+href=["']([^"']+)["']/i);
|
|
3101
|
+
if (moreItemMatch && moreItemMatch[1]) {
|
|
3102
|
+
return resolveFacebookUrl(moreItemMatch[1], currentUrl);
|
|
3103
|
+
}
|
|
3104
|
+
const cursorLabelMatch = html.match(/<a[^>]+href=["']([^"']*(?:cursor=|after=)[^"']+)["'][^>]*>[\s\S]*?(?:See [Mm]ore|More [Pp]hotos|More|Next|Older [Pp]osts)[\s\S]*?<\/a>/i);
|
|
3105
|
+
if (cursorLabelMatch && cursorLabelMatch[1]) {
|
|
3106
|
+
return resolveFacebookUrl(cursorLabelMatch[1], currentUrl);
|
|
3107
|
+
}
|
|
3108
|
+
const genericCursorMatch = html.match(/<a[^>]+href=["']([^"']*(?:[?&]|&)(?:cursor|after)=[^"']+)["']/i);
|
|
3109
|
+
if (genericCursorMatch && genericCursorMatch[1]) {
|
|
3110
|
+
return resolveFacebookUrl(genericCursorMatch[1], currentUrl);
|
|
3111
|
+
}
|
|
3112
|
+
const cursor = extractFacebookPaginationCursor(html);
|
|
3113
|
+
if (cursor) {
|
|
3114
|
+
try {
|
|
3115
|
+
const u = new URL(currentUrl);
|
|
3116
|
+
if (u.searchParams.get("cursor") !== cursor) {
|
|
3117
|
+
u.searchParams.set("cursor", cursor);
|
|
3118
|
+
return u.toString();
|
|
3119
|
+
}
|
|
3120
|
+
} catch {
|
|
3121
|
+
if (!currentUrl.includes(cursor)) {
|
|
3122
|
+
const sep = currentUrl.includes("?") ? "&" : "?";
|
|
3123
|
+
return `${currentUrl}${sep}cursor=${encodeURIComponent(cursor)}`;
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
return null;
|
|
3128
|
+
}
|
|
2711
3129
|
var FacebookExtractor = class {
|
|
2712
3130
|
platform = "facebook";
|
|
2713
3131
|
httpClient;
|
|
@@ -2724,26 +3142,120 @@ var FacebookExtractor = class {
|
|
|
2724
3142
|
found: 0,
|
|
2725
3143
|
message: `Fetching Facebook profile @${username}...`
|
|
2726
3144
|
});
|
|
2727
|
-
const
|
|
2728
|
-
const
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
3145
|
+
const targetPath = subDir === "Photos" ? "photos" : "videos";
|
|
3146
|
+
const desktopUrl = `https://www.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
|
|
3147
|
+
const mbasicUrl = `https://mbasic.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
|
|
3148
|
+
const mobileUrl = `https://m.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
|
|
3149
|
+
let currentUrl = desktopUrl;
|
|
3150
|
+
let html = "";
|
|
3151
|
+
try {
|
|
3152
|
+
html = await this.httpClient.fetchText(desktopUrl, {
|
|
3153
|
+
cookiePath: options.cookiePath,
|
|
3154
|
+
cookieDomain: "facebook.com",
|
|
3155
|
+
signal: options.signal,
|
|
3156
|
+
retries: 0,
|
|
3157
|
+
timeoutMs: 4e3
|
|
3158
|
+
});
|
|
3159
|
+
} catch {
|
|
3160
|
+
html = "";
|
|
3161
|
+
}
|
|
3162
|
+
let initialItems = extractFacebookMediaFromHtml(html, username);
|
|
3163
|
+
if (initialItems.length === 0) {
|
|
2741
3164
|
options.onProgress?.({
|
|
2742
|
-
stage:
|
|
2743
|
-
found,
|
|
2744
|
-
message: `
|
|
3165
|
+
stage: "Profile",
|
|
3166
|
+
found: 0,
|
|
3167
|
+
message: `Desktop yielded 0 items. Falling back to mbasic.facebook.com for @${username}...`
|
|
2745
3168
|
});
|
|
2746
|
-
|
|
3169
|
+
try {
|
|
3170
|
+
const mbasicHtml = await this.httpClient.fetchText(mbasicUrl, {
|
|
3171
|
+
cookiePath: options.cookiePath,
|
|
3172
|
+
cookieDomain: "facebook.com",
|
|
3173
|
+
signal: options.signal,
|
|
3174
|
+
retries: 1,
|
|
3175
|
+
timeoutMs: 5e3,
|
|
3176
|
+
headers: {
|
|
3177
|
+
"Referer": "https://mbasic.facebook.com/"
|
|
3178
|
+
}
|
|
3179
|
+
});
|
|
3180
|
+
const mbasicItems = extractFacebookMediaFromHtml(mbasicHtml, username);
|
|
3181
|
+
if (mbasicItems.length > 0 || mbasicHtml.length > 200) {
|
|
3182
|
+
html = mbasicHtml;
|
|
3183
|
+
currentUrl = mbasicUrl;
|
|
3184
|
+
initialItems = mbasicItems;
|
|
3185
|
+
}
|
|
3186
|
+
} catch {
|
|
3187
|
+
}
|
|
3188
|
+
if (initialItems.length === 0) {
|
|
3189
|
+
try {
|
|
3190
|
+
const mobileHtml = await this.httpClient.fetchText(mobileUrl, {
|
|
3191
|
+
cookiePath: options.cookiePath,
|
|
3192
|
+
cookieDomain: "facebook.com",
|
|
3193
|
+
signal: options.signal,
|
|
3194
|
+
retries: 0,
|
|
3195
|
+
timeoutMs: 5e3,
|
|
3196
|
+
headers: {
|
|
3197
|
+
"User-Agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
|
|
3198
|
+
"Referer": "https://m.facebook.com/"
|
|
3199
|
+
}
|
|
3200
|
+
});
|
|
3201
|
+
const mobileItems = extractFacebookMediaFromHtml(mobileHtml, username);
|
|
3202
|
+
if (mobileItems.length > 0) {
|
|
3203
|
+
html = mobileHtml;
|
|
3204
|
+
currentUrl = mobileUrl;
|
|
3205
|
+
initialItems = mobileItems;
|
|
3206
|
+
}
|
|
3207
|
+
} catch {
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
}
|
|
3211
|
+
const seenUrls = /* @__PURE__ */ new Set();
|
|
3212
|
+
let found = 0;
|
|
3213
|
+
let page = 1;
|
|
3214
|
+
const maxPages = 5;
|
|
3215
|
+
while (true) {
|
|
3216
|
+
const items = extractFacebookMediaFromHtml(html, username);
|
|
3217
|
+
let pageNewCount = 0;
|
|
3218
|
+
for (const item of items) {
|
|
3219
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
3220
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
3221
|
+
if (seenUrls.has(item.url)) continue;
|
|
3222
|
+
seenUrls.add(item.url);
|
|
3223
|
+
found++;
|
|
3224
|
+
pageNewCount++;
|
|
3225
|
+
options.onProgress?.({
|
|
3226
|
+
stage: subDir,
|
|
3227
|
+
found,
|
|
3228
|
+
message: `Found item: ${item.filename}`
|
|
3229
|
+
});
|
|
3230
|
+
yield item;
|
|
3231
|
+
}
|
|
3232
|
+
if (page >= maxPages || options.signal?.aborted) {
|
|
3233
|
+
break;
|
|
3234
|
+
}
|
|
3235
|
+
const nextUrl = extractFacebookNextPageUrl(html, currentUrl);
|
|
3236
|
+
if (!nextUrl || nextUrl === currentUrl) {
|
|
3237
|
+
break;
|
|
3238
|
+
}
|
|
3239
|
+
if (pageNewCount === 0 && page > 1) {
|
|
3240
|
+
break;
|
|
3241
|
+
}
|
|
3242
|
+
await sleep(1e3, 2e3, options.signal);
|
|
3243
|
+
try {
|
|
3244
|
+
currentUrl = nextUrl;
|
|
3245
|
+
html = await this.httpClient.fetchText(currentUrl, {
|
|
3246
|
+
cookiePath: options.cookiePath,
|
|
3247
|
+
cookieDomain: "facebook.com",
|
|
3248
|
+
signal: options.signal,
|
|
3249
|
+
retries: 1,
|
|
3250
|
+
timeoutMs: 5e3,
|
|
3251
|
+
headers: {
|
|
3252
|
+
"Referer": currentUrl.includes("mbasic") ? "https://mbasic.facebook.com/" : "https://www.facebook.com/"
|
|
3253
|
+
}
|
|
3254
|
+
});
|
|
3255
|
+
page++;
|
|
3256
|
+
} catch {
|
|
3257
|
+
break;
|
|
3258
|
+
}
|
|
2747
3259
|
}
|
|
2748
3260
|
}
|
|
2749
3261
|
};
|
|
@@ -2771,17 +3283,48 @@ import path5 from "path";
|
|
|
2771
3283
|
import { Readable } from "stream";
|
|
2772
3284
|
import { pipeline } from "stream/promises";
|
|
2773
3285
|
var ARCHIVE_FILENAME = ".open-nami-archive.json";
|
|
3286
|
+
var YTDLP_ARCHIVE_FILENAME = "archive_yt-dlp.txt";
|
|
3287
|
+
var TEXT_ARCHIVE_FILENAME = "archive.txt";
|
|
2774
3288
|
async function loadArchive(destDir) {
|
|
2775
|
-
const
|
|
3289
|
+
const archive = /* @__PURE__ */ new Set();
|
|
3290
|
+
const jsonPath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
2776
3291
|
try {
|
|
2777
|
-
const raw = await fsPromises2.readFile(
|
|
3292
|
+
const raw = await fsPromises2.readFile(jsonPath, "utf-8");
|
|
2778
3293
|
const parsed = JSON.parse(raw);
|
|
2779
3294
|
if (Array.isArray(parsed)) {
|
|
2780
|
-
|
|
3295
|
+
for (const id of parsed) {
|
|
3296
|
+
if (id) archive.add(String(id));
|
|
3297
|
+
}
|
|
2781
3298
|
}
|
|
2782
3299
|
} catch {
|
|
2783
3300
|
}
|
|
2784
|
-
|
|
3301
|
+
const ytdlpPath = path5.join(destDir, YTDLP_ARCHIVE_FILENAME);
|
|
3302
|
+
try {
|
|
3303
|
+
const raw = await fsPromises2.readFile(ytdlpPath, "utf-8");
|
|
3304
|
+
const lines = raw.split(/\r?\n/);
|
|
3305
|
+
for (const rawLine of lines) {
|
|
3306
|
+
const line = rawLine.trim();
|
|
3307
|
+
if (!line || line.startsWith("#")) continue;
|
|
3308
|
+
const parts = line.split(/\s+/);
|
|
3309
|
+
const id = parts[parts.length - 1];
|
|
3310
|
+
if (id) archive.add(id);
|
|
3311
|
+
}
|
|
3312
|
+
} catch {
|
|
3313
|
+
}
|
|
3314
|
+
const textPath = path5.join(destDir, TEXT_ARCHIVE_FILENAME);
|
|
3315
|
+
try {
|
|
3316
|
+
const raw = await fsPromises2.readFile(textPath, "utf-8");
|
|
3317
|
+
const lines = raw.split(/\r?\n/);
|
|
3318
|
+
for (const rawLine of lines) {
|
|
3319
|
+
const line = rawLine.trim();
|
|
3320
|
+
if (!line || line.startsWith("#")) continue;
|
|
3321
|
+
const parts = line.split(/\s+/);
|
|
3322
|
+
const id = parts[parts.length - 1];
|
|
3323
|
+
if (id) archive.add(id);
|
|
3324
|
+
}
|
|
3325
|
+
} catch {
|
|
3326
|
+
}
|
|
3327
|
+
return archive;
|
|
2785
3328
|
}
|
|
2786
3329
|
async function saveArchive(destDir, archive) {
|
|
2787
3330
|
const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
@@ -2791,6 +3334,18 @@ async function saveArchive(destDir, archive) {
|
|
|
2791
3334
|
await fsPromises2.rename(tmpPath, archivePath);
|
|
2792
3335
|
} catch {
|
|
2793
3336
|
}
|
|
3337
|
+
const textPaths = [
|
|
3338
|
+
path5.join(destDir, YTDLP_ARCHIVE_FILENAME),
|
|
3339
|
+
path5.join(destDir, TEXT_ARCHIVE_FILENAME)
|
|
3340
|
+
];
|
|
3341
|
+
const content = [...archive].map((id) => `${id}
|
|
3342
|
+
`).join("");
|
|
3343
|
+
for (const tPath of textPaths) {
|
|
3344
|
+
try {
|
|
3345
|
+
await fsPromises2.writeFile(tPath, content, "utf-8");
|
|
3346
|
+
} catch {
|
|
3347
|
+
}
|
|
3348
|
+
}
|
|
2794
3349
|
}
|
|
2795
3350
|
function sanitizeFilename(rawFilename, fallbackId, ext) {
|
|
2796
3351
|
const cleanExt = ext.replace(/^\./, "");
|
|
@@ -2843,12 +3398,13 @@ async function downloadMediaStream(items, options) {
|
|
|
2843
3398
|
consecutiveErrors++;
|
|
2844
3399
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2845
3400
|
errors.push(`Extraction stream error: ${errMsg}`);
|
|
3401
|
+
const cleanErr = errMsg.includes("429") ? "rate limited by platform (HTTP 429)" : errMsg.length > 50 ? `${errMsg.slice(0, 47)}\u2026` : errMsg;
|
|
2846
3402
|
onProgress?.({
|
|
2847
3403
|
job: jobName,
|
|
2848
3404
|
downloadedCount: downloaded,
|
|
2849
3405
|
skippedCount: skipped,
|
|
2850
3406
|
currentFile: "",
|
|
2851
|
-
statusText: `Extractor
|
|
3407
|
+
statusText: `Extractor error: ${cleanErr}`
|
|
2852
3408
|
});
|
|
2853
3409
|
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS || signal?.aborted) {
|
|
2854
3410
|
isDone = true;
|
|
@@ -3318,8 +3874,9 @@ async function runGalleryDl(options) {
|
|
|
3318
3874
|
let downloadedCount = 0;
|
|
3319
3875
|
let skippedCount = 0;
|
|
3320
3876
|
let errorOutput = "";
|
|
3877
|
+
let engineCmdStr = "gallery-dl";
|
|
3321
3878
|
try {
|
|
3322
|
-
await ensureGalleryDl((status) => {
|
|
3879
|
+
engineCmdStr = await ensureGalleryDl((status) => {
|
|
3323
3880
|
onProgress?.({
|
|
3324
3881
|
job: subDir,
|
|
3325
3882
|
downloadedCount: 0,
|
|
@@ -3335,9 +3892,15 @@ async function runGalleryDl(options) {
|
|
|
3335
3892
|
error: err.message || String(err)
|
|
3336
3893
|
};
|
|
3337
3894
|
}
|
|
3338
|
-
const engineCmd = resolveEngineCommand("gallery-dl");
|
|
3895
|
+
const engineCmd = engineCmdStr.includes(path8.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("gallery-dl");
|
|
3339
3896
|
const fullArgs = [...engineCmd.baseArgs, ...args];
|
|
3340
3897
|
return new Promise((resolve) => {
|
|
3898
|
+
onProgress?.({
|
|
3899
|
+
job: subDir,
|
|
3900
|
+
downloadedCount,
|
|
3901
|
+
skippedCount,
|
|
3902
|
+
statusText: "downloading with gallery-dl\u2026"
|
|
3903
|
+
});
|
|
3341
3904
|
const child = spawn2(engineCmd.command, fullArgs, {
|
|
3342
3905
|
windowsHide: true,
|
|
3343
3906
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -3461,8 +4024,9 @@ async function runYtDlp(options) {
|
|
|
3461
4024
|
let downloadedCount = 0;
|
|
3462
4025
|
let skippedCount = 0;
|
|
3463
4026
|
let errorOutput = "";
|
|
4027
|
+
let engineCmdStr = "yt-dlp";
|
|
3464
4028
|
try {
|
|
3465
|
-
await ensureYtDlp((status) => {
|
|
4029
|
+
engineCmdStr = await ensureYtDlp((status) => {
|
|
3466
4030
|
onProgress?.({
|
|
3467
4031
|
job: "Videos",
|
|
3468
4032
|
downloadedCount: 0,
|
|
@@ -3478,9 +4042,15 @@ async function runYtDlp(options) {
|
|
|
3478
4042
|
error: err.message || String(err)
|
|
3479
4043
|
};
|
|
3480
4044
|
}
|
|
3481
|
-
const engineCmd = resolveEngineCommand("yt-dlp");
|
|
4045
|
+
const engineCmd = engineCmdStr.includes(path9.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("yt-dlp");
|
|
3482
4046
|
const fullArgs = [...engineCmd.baseArgs, ...args];
|
|
3483
4047
|
return new Promise((resolve) => {
|
|
4048
|
+
onProgress?.({
|
|
4049
|
+
job: "Videos",
|
|
4050
|
+
downloadedCount,
|
|
4051
|
+
skippedCount,
|
|
4052
|
+
statusText: "downloading with yt-dlp\u2026"
|
|
4053
|
+
});
|
|
3484
4054
|
const child = spawn3(engineCmd.command, fullArgs, {
|
|
3485
4055
|
windowsHide: true,
|
|
3486
4056
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -3617,7 +4187,7 @@ async function dispatchDownload(options) {
|
|
|
3617
4187
|
jobs.push({ type: "highlights" });
|
|
3618
4188
|
}
|
|
3619
4189
|
}
|
|
3620
|
-
const
|
|
4190
|
+
const defaultRunNative = async (subDir) => {
|
|
3621
4191
|
const dest = path10.join(targetDir, subDir);
|
|
3622
4192
|
try {
|
|
3623
4193
|
const extractor = getExtractor(profile.platform);
|
|
@@ -3650,27 +4220,34 @@ async function dispatchDownload(options) {
|
|
|
3650
4220
|
};
|
|
3651
4221
|
}
|
|
3652
4222
|
};
|
|
3653
|
-
const
|
|
4223
|
+
const runNative = options.runnerOverrides?.nativeRunner ?? defaultRunNative;
|
|
4224
|
+
const runGdl = options.runnerOverrides?.gdlRunner ?? ((subDir) => runGalleryDl({
|
|
3654
4225
|
profile,
|
|
3655
4226
|
targetDir,
|
|
3656
4227
|
subDir,
|
|
3657
4228
|
cookiePath: cookieCopy?.filePath,
|
|
3658
4229
|
signal: options.signal,
|
|
3659
4230
|
onProgress
|
|
3660
|
-
});
|
|
3661
|
-
const runYt = () => runYtDlp({
|
|
4231
|
+
}));
|
|
4232
|
+
const runYt = options.runnerOverrides?.ytRunner ?? (() => runYtDlp({
|
|
3662
4233
|
profile,
|
|
3663
4234
|
targetDir,
|
|
3664
4235
|
cookiePath: cookieCopy?.filePath,
|
|
3665
4236
|
signal: options.signal,
|
|
3666
4237
|
onProgress
|
|
3667
|
-
});
|
|
4238
|
+
}));
|
|
3668
4239
|
for (const job of jobs) {
|
|
3669
4240
|
if (options.signal?.aborted) break;
|
|
3670
4241
|
if (job.type === "photos" || job.type === "stories" || job.type === "highlights") {
|
|
3671
4242
|
const subDir = job.type === "photos" ? "Photos" : job.type === "stories" ? "Stories" : "Highlights";
|
|
3672
4243
|
let res = await runNative(subDir);
|
|
3673
4244
|
if ((!res.success || res.downloaded === 0 && res.skipped === 0) && !options.signal?.aborted) {
|
|
4245
|
+
onProgress?.({
|
|
4246
|
+
job: subDir,
|
|
4247
|
+
downloadedCount: totalDownloaded,
|
|
4248
|
+
skippedCount: totalSkipped,
|
|
4249
|
+
statusText: "switching to gallery-dl engine\u2026"
|
|
4250
|
+
});
|
|
3674
4251
|
const gdlRes = await runGdl(subDir);
|
|
3675
4252
|
if (gdlRes.downloaded > 0 || gdlRes.skipped > 0 || gdlRes.success) {
|
|
3676
4253
|
res = gdlRes;
|
|
@@ -3685,12 +4262,24 @@ async function dispatchDownload(options) {
|
|
|
3685
4262
|
totalDownloaded += res.downloaded;
|
|
3686
4263
|
totalSkipped += res.skipped;
|
|
3687
4264
|
} else if (!options.signal?.aborted) {
|
|
4265
|
+
onProgress?.({
|
|
4266
|
+
job: "Videos",
|
|
4267
|
+
downloadedCount: totalDownloaded,
|
|
4268
|
+
skippedCount: totalSkipped,
|
|
4269
|
+
statusText: "switching to yt-dlp engine\u2026"
|
|
4270
|
+
});
|
|
3688
4271
|
const fbRes = await runYt();
|
|
3689
4272
|
if (fbRes.downloaded > 0 || fbRes.skipped > 0 || fbRes.success) {
|
|
3690
4273
|
totalDownloaded += fbRes.downloaded;
|
|
3691
4274
|
totalSkipped += fbRes.skipped;
|
|
3692
4275
|
if (!fbRes.success && fbRes.error) errors.push(`[Videos] ${fbRes.error}`);
|
|
3693
4276
|
} else if (!options.signal?.aborted) {
|
|
4277
|
+
onProgress?.({
|
|
4278
|
+
job: "Videos",
|
|
4279
|
+
downloadedCount: totalDownloaded,
|
|
4280
|
+
skippedCount: totalSkipped,
|
|
4281
|
+
statusText: "switching to gallery-dl engine\u2026"
|
|
4282
|
+
});
|
|
3694
4283
|
const gdlRes = await runGdl("Videos");
|
|
3695
4284
|
totalDownloaded += gdlRes.downloaded;
|
|
3696
4285
|
totalSkipped += gdlRes.skipped;
|