open-nami 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +749 -142
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -9,8 +9,8 @@ var require_package = __commonJS({
|
|
|
9
9
|
"package.json"(exports, module) {
|
|
10
10
|
module.exports = {
|
|
11
11
|
name: "open-nami",
|
|
12
|
-
version: "1.0.
|
|
13
|
-
description: "Open Nami
|
|
12
|
+
version: "1.0.6",
|
|
13
|
+
description: "Open Nami: terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
|
|
14
14
|
type: "module",
|
|
15
15
|
bin: {
|
|
16
16
|
"open-nami": "dist/cli.js",
|
|
@@ -1407,10 +1407,11 @@ function parseProfileInput(input) {
|
|
|
1407
1407
|
const reserved = ["home", "explore", "notifications", "messages", "i", "settings"];
|
|
1408
1408
|
if (firstPart && !reserved.includes(firstPart.toLowerCase())) {
|
|
1409
1409
|
const username = firstPart.replace(/^@/, "");
|
|
1410
|
+
const isLikesUrl = pathParts[1]?.toLowerCase() === "likes";
|
|
1410
1411
|
return {
|
|
1411
1412
|
platform: "x",
|
|
1412
1413
|
username: sanitizeUsername(username),
|
|
1413
|
-
cleanUrl: `https://x.com/${username}`,
|
|
1414
|
+
cleanUrl: isLikesUrl ? `https://x.com/${username}/likes` : `https://x.com/${username}`,
|
|
1414
1415
|
rawInput: input
|
|
1415
1416
|
};
|
|
1416
1417
|
}
|
|
@@ -1575,6 +1576,9 @@ function resolvePlatformDownloadsDir(options = {}) {
|
|
|
1575
1576
|
return defaultFallback;
|
|
1576
1577
|
}
|
|
1577
1578
|
|
|
1579
|
+
// src/lib/engines/extractors/instagram.ts
|
|
1580
|
+
import crypto from "crypto";
|
|
1581
|
+
|
|
1578
1582
|
// src/lib/engines/extractors/http.ts
|
|
1579
1583
|
import fs5 from "fs/promises";
|
|
1580
1584
|
import { setTimeout as setTimeoutPromise } from "timers/promises";
|
|
@@ -1762,7 +1766,9 @@ var HttpClient = class {
|
|
|
1762
1766
|
}
|
|
1763
1767
|
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1764
1768
|
}
|
|
1765
|
-
|
|
1769
|
+
const text = await res.text();
|
|
1770
|
+
const cleanText = text.replace(/^\s*(?:for\s*\(\s*;\s*;\s*\);|\)\]\}',)/, "").trim();
|
|
1771
|
+
return JSON.parse(cleanText);
|
|
1766
1772
|
}
|
|
1767
1773
|
async fetchText(url, options = {}) {
|
|
1768
1774
|
const res = await this.request(url, options);
|
|
@@ -1781,13 +1787,13 @@ var HttpClient = class {
|
|
|
1781
1787
|
var IG_APP_ID = "936619743392459";
|
|
1782
1788
|
var IG_TIMELINE_QUERY_HASH = "69cba40317214236af40e7efa697781d";
|
|
1783
1789
|
var IG_QUERIES = {
|
|
1784
|
-
PolarisProfilePostsTabContentQuery_connection: "
|
|
1790
|
+
PolarisProfilePostsTabContentQuery_connection: "39229011733413759",
|
|
1785
1791
|
PolarisProfileReelsTabContentQuery_connection: "28989741540630190",
|
|
1786
1792
|
PolarisProfileTaggedTabContentQuery_connection: "28794602806842110",
|
|
1787
1793
|
PolarisProfileStoryHighlightsTrayContentQuery: "26970053832668570",
|
|
1788
1794
|
PolarisStoriesV3ReelPageGalleryQuery: "28262315486766731",
|
|
1789
1795
|
PolarisStoriesV3TrayContainerQuery: "27703822975903310",
|
|
1790
|
-
PolarisProfilePageContentQuery: "
|
|
1796
|
+
PolarisProfilePageContentQuery: "28649589871350527"
|
|
1791
1797
|
};
|
|
1792
1798
|
var IG_POLARIS_DOC_ID = IG_QUERIES.PolarisProfilePostsTabContentQuery_connection;
|
|
1793
1799
|
function logExtractorDebug(endpoint, err) {
|
|
@@ -1831,6 +1837,21 @@ function parseIgNodeToMediaItems(node, username) {
|
|
|
1831
1837
|
height: video?.height ?? node.dimensions?.height,
|
|
1832
1838
|
thumbnailUrl: node.display_url
|
|
1833
1839
|
});
|
|
1840
|
+
} else {
|
|
1841
|
+
const candidate = node.image_versions2?.candidates?.[0];
|
|
1842
|
+
const photoUrl = candidate?.url || node.display_url;
|
|
1843
|
+
if (photoUrl) {
|
|
1844
|
+
items.push({
|
|
1845
|
+
id: baseId,
|
|
1846
|
+
url: photoUrl,
|
|
1847
|
+
filename: `${username}_${date || baseId}`,
|
|
1848
|
+
extension: "jpg",
|
|
1849
|
+
type: "photo",
|
|
1850
|
+
date,
|
|
1851
|
+
width: candidate?.width ?? node.dimensions?.width,
|
|
1852
|
+
height: candidate?.height ?? node.dimensions?.height
|
|
1853
|
+
});
|
|
1854
|
+
}
|
|
1834
1855
|
}
|
|
1835
1856
|
} else {
|
|
1836
1857
|
const candidate = node.image_versions2?.candidates?.[0];
|
|
@@ -1910,7 +1931,15 @@ var InstagramExtractor = class {
|
|
|
1910
1931
|
if (!res.data?.user) {
|
|
1911
1932
|
throw new Error(`Instagram user not found or private profile: ${username}`);
|
|
1912
1933
|
}
|
|
1913
|
-
|
|
1934
|
+
const user = res.data.user;
|
|
1935
|
+
if (user.is_private) {
|
|
1936
|
+
const fs13 = user.friendship_status;
|
|
1937
|
+
const isFollowing = fs13 ? Boolean(fs13.following) : Boolean(user.followed_by_viewer);
|
|
1938
|
+
if (!isFollowing) {
|
|
1939
|
+
throw new Error(`Instagram profile @${username} is private. You must provide a session cookie jar of an account following this profile.`);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
return user;
|
|
1914
1943
|
}
|
|
1915
1944
|
async fetchHighlightsTray(userId, options) {
|
|
1916
1945
|
try {
|
|
@@ -1932,6 +1961,10 @@ var InstagramExtractor = class {
|
|
|
1932
1961
|
"Referer": "https://www.instagram.com/"
|
|
1933
1962
|
}
|
|
1934
1963
|
});
|
|
1964
|
+
if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
|
|
1965
|
+
const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
|
|
1966
|
+
throw new Error(`Instagram GraphQL query error: ${errMsg}`);
|
|
1967
|
+
}
|
|
1935
1968
|
const edges = res?.data?.highlights?.edges;
|
|
1936
1969
|
if (Array.isArray(edges) && edges.length > 0) {
|
|
1937
1970
|
return edges.map((e) => e.node);
|
|
@@ -1975,87 +2008,211 @@ var InstagramExtractor = class {
|
|
|
1975
2008
|
return [];
|
|
1976
2009
|
}
|
|
1977
2010
|
}
|
|
1978
|
-
async
|
|
1979
|
-
|
|
1980
|
-
|
|
2011
|
+
async fetchWebTokens(username, options) {
|
|
2012
|
+
let csrfToken;
|
|
2013
|
+
if (options.cookiePath) {
|
|
2014
|
+
try {
|
|
2015
|
+
const cookies = await loadCookiesFromPath(options.cookiePath, "instagram.com");
|
|
2016
|
+
csrfToken = cookies["csrftoken"];
|
|
2017
|
+
} catch {
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
if (!csrfToken) {
|
|
2021
|
+
csrfToken = crypto.randomBytes(16).toString("hex");
|
|
2022
|
+
}
|
|
1981
2023
|
try {
|
|
1982
|
-
const
|
|
2024
|
+
const html = await this.httpClient.fetchText(`https://www.instagram.com/${encodeURIComponent(username)}/`, {
|
|
1983
2025
|
cookiePath: options.cookiePath,
|
|
1984
2026
|
cookieDomain: "instagram.com",
|
|
1985
2027
|
signal: options.signal,
|
|
1986
2028
|
headers: {
|
|
1987
|
-
"
|
|
1988
|
-
"Referer": `https://www.instagram.com/${username}/`
|
|
2029
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
|
1989
2030
|
}
|
|
1990
2031
|
});
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
2032
|
+
let lsd;
|
|
2033
|
+
let fbDtsg;
|
|
2034
|
+
let userId;
|
|
2035
|
+
const eqmcMatch = html.match(/id="__eqmc"[^>]*>([^<]+)<\/script>/);
|
|
2036
|
+
if (eqmcMatch) {
|
|
2037
|
+
try {
|
|
2038
|
+
const eqmc = JSON.parse(eqmcMatch[1]);
|
|
2039
|
+
if (eqmc.l) lsd = eqmc.l;
|
|
2040
|
+
if (eqmc.f) fbDtsg = eqmc.f;
|
|
2041
|
+
} catch {
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
if (!lsd) {
|
|
2045
|
+
const m = html.match(/"lsd":"([^"]+)"/) || html.match(/"LSD",\[\],\{"token":"([^"]+)"/);
|
|
2046
|
+
if (m) lsd = m[1];
|
|
2047
|
+
}
|
|
2048
|
+
if (!fbDtsg) {
|
|
2049
|
+
const m = html.match(/"dtsg":\{"token":"([^"]+)"/);
|
|
2050
|
+
if (m) fbDtsg = m[1];
|
|
1998
2051
|
}
|
|
2052
|
+
const uMatch = html.match(/"profile_id":"([^"]+)"/) || html.match(/"user_id":"([^"]+)"/);
|
|
2053
|
+
if (uMatch) userId = uMatch[1];
|
|
2054
|
+
return { lsd, fbDtsg, csrfToken, userId };
|
|
1999
2055
|
} catch (err) {
|
|
2000
|
-
logExtractorDebug("
|
|
2056
|
+
logExtractorDebug("fetchWebTokens", err);
|
|
2057
|
+
return { csrfToken };
|
|
2058
|
+
}
|
|
2059
|
+
}
|
|
2060
|
+
async fetchPostsTabGraphQL(username, cursor, tokens, options) {
|
|
2061
|
+
const postVariables = JSON.stringify({
|
|
2062
|
+
after: cursor || null,
|
|
2063
|
+
before: null,
|
|
2064
|
+
data: {
|
|
2065
|
+
count: 12,
|
|
2066
|
+
include_reel_media_seen_timestamp: true,
|
|
2067
|
+
include_relationship_info: true,
|
|
2068
|
+
latest_besties_reel_media: true,
|
|
2069
|
+
latest_reel_media: true
|
|
2070
|
+
},
|
|
2071
|
+
first: 12,
|
|
2072
|
+
include_multi_captions: true,
|
|
2073
|
+
last: null,
|
|
2074
|
+
username,
|
|
2075
|
+
__relay_internal__pv__PolarisMultiCaptionCarouselEnabledrelayprovider: true,
|
|
2076
|
+
__relay_internal__pv__PolarisShortDramaEnabledrelayprovider: true,
|
|
2077
|
+
__relay_internal__pv__PolarisThreadsStyleCommentsEnabledrelayprovider: false,
|
|
2078
|
+
__relay_internal__pv__PolarisReelsRecoDebugOverlayEnabledrelayprovider: false
|
|
2079
|
+
});
|
|
2080
|
+
const bodyParams = new URLSearchParams({
|
|
2081
|
+
av: "17841415137994167",
|
|
2082
|
+
__d: "www",
|
|
2083
|
+
__user: "0",
|
|
2084
|
+
__a: "1",
|
|
2085
|
+
__req: "0",
|
|
2086
|
+
__hs: "20731.HYP:instagram_web_pkg.2.1...0",
|
|
2087
|
+
dpr: "1",
|
|
2088
|
+
__ccg: "EXCELLENT",
|
|
2089
|
+
__rev: "1049285836",
|
|
2090
|
+
__hsi: "7693193318680449885",
|
|
2091
|
+
__comet_req: "7",
|
|
2092
|
+
jazoest: "26461",
|
|
2093
|
+
fb_api_caller_class: "RelayModern",
|
|
2094
|
+
fb_api_req_friendly_name: "PolarisProfilePostsTabContentQuery_connection",
|
|
2095
|
+
server_timestamps: "true",
|
|
2096
|
+
variables: postVariables,
|
|
2097
|
+
doc_id: IG_POLARIS_DOC_ID
|
|
2098
|
+
});
|
|
2099
|
+
if (tokens.fbDtsg) bodyParams.set("fb_dtsg", tokens.fbDtsg);
|
|
2100
|
+
if (tokens.lsd) bodyParams.set("lsd", tokens.lsd);
|
|
2101
|
+
const headers = {
|
|
2102
|
+
"Accept": "*/*",
|
|
2103
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
2104
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2105
|
+
"X-IG-Max-Touch-Points": "0",
|
|
2106
|
+
"X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
|
|
2107
|
+
"Referer": `https://www.instagram.com/${username}/`,
|
|
2108
|
+
"Origin": "https://www.instagram.com",
|
|
2109
|
+
"X-ASBD-ID": "359341",
|
|
2110
|
+
"Sec-Fetch-Dest": "empty",
|
|
2111
|
+
"Sec-Fetch-Mode": "cors",
|
|
2112
|
+
"Sec-Fetch-Site": "same-origin"
|
|
2113
|
+
};
|
|
2114
|
+
if (tokens.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
|
|
2115
|
+
if (tokens.lsd) headers["X-FB-LSD"] = tokens.lsd;
|
|
2116
|
+
const res = await this.httpClient.fetchJson("https://www.instagram.com/graphql/query", {
|
|
2117
|
+
method: "POST",
|
|
2118
|
+
body: bodyParams.toString(),
|
|
2119
|
+
cookiePath: options.cookiePath,
|
|
2120
|
+
cookieDomain: "instagram.com",
|
|
2121
|
+
signal: options.signal,
|
|
2122
|
+
headers
|
|
2123
|
+
});
|
|
2124
|
+
if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
|
|
2125
|
+
const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
|
|
2126
|
+
throw new Error(`Instagram GraphQL query error: ${errMsg}`);
|
|
2127
|
+
}
|
|
2128
|
+
const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
|
|
2129
|
+
if (connection?.edges && Array.isArray(connection.edges)) {
|
|
2130
|
+
let resolvedUserId;
|
|
2131
|
+
for (const edge of connection.edges) {
|
|
2132
|
+
const node = edge?.node;
|
|
2133
|
+
if (node?.owner_id) {
|
|
2134
|
+
resolvedUserId = String(node.owner_id);
|
|
2135
|
+
break;
|
|
2136
|
+
}
|
|
2137
|
+
if (node?.user?.pk || node?.user?.id) {
|
|
2138
|
+
resolvedUserId = String(node.user.pk || node.user.id);
|
|
2139
|
+
break;
|
|
2140
|
+
}
|
|
2141
|
+
if (typeof node?.id === "string" && node.id.includes("_")) {
|
|
2142
|
+
resolvedUserId = node.id.split("_")[1];
|
|
2143
|
+
break;
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
return {
|
|
2147
|
+
edges: connection.edges,
|
|
2148
|
+
hasNextPage: Boolean(connection.page_info?.has_next_page),
|
|
2149
|
+
nextCursor: connection.page_info?.end_cursor,
|
|
2150
|
+
userId: resolvedUserId
|
|
2151
|
+
};
|
|
2001
2152
|
}
|
|
2153
|
+
return { edges: [], hasNextPage: false };
|
|
2154
|
+
}
|
|
2155
|
+
async fetchNextPostsPage(userId, username, cursor, options, tokens) {
|
|
2002
2156
|
try {
|
|
2003
|
-
const
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
username,
|
|
2007
|
-
data: { count: 12 }
|
|
2008
|
-
});
|
|
2009
|
-
const postUrl = "https://www.instagram.com/graphql/query";
|
|
2010
|
-
const body = new URLSearchParams({
|
|
2011
|
-
doc_id: IG_POLARIS_DOC_ID,
|
|
2012
|
-
variables: postVariables
|
|
2013
|
-
}).toString();
|
|
2014
|
-
const res = await this.httpClient.fetchJson(postUrl, {
|
|
2015
|
-
method: "POST",
|
|
2016
|
-
body,
|
|
2017
|
-
cookiePath: options.cookiePath,
|
|
2018
|
-
cookieDomain: "instagram.com",
|
|
2019
|
-
signal: options.signal,
|
|
2020
|
-
headers: {
|
|
2021
|
-
"Content-Type": "application/x-www-form-urlencoded",
|
|
2022
|
-
"X-IG-App-ID": IG_APP_ID,
|
|
2023
|
-
"X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
|
|
2024
|
-
"Referer": `https://www.instagram.com/${username}/`
|
|
2025
|
-
}
|
|
2026
|
-
});
|
|
2027
|
-
const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
|
|
2028
|
-
if (connection?.edges && connection.edges.length > 0) {
|
|
2029
|
-
return {
|
|
2030
|
-
edges: connection.edges,
|
|
2031
|
-
hasNextPage: Boolean(connection.page_info?.has_next_page),
|
|
2032
|
-
nextCursor: connection.page_info?.end_cursor
|
|
2033
|
-
};
|
|
2157
|
+
const page = await this.fetchPostsTabGraphQL(username, cursor, tokens || {}, options);
|
|
2158
|
+
if (page.edges && page.edges.length > 0) {
|
|
2159
|
+
return page;
|
|
2034
2160
|
}
|
|
2035
2161
|
} catch (err) {
|
|
2036
2162
|
logExtractorDebug("PolarisProfilePostsTabContentQuery_connection", err);
|
|
2037
2163
|
}
|
|
2038
|
-
|
|
2039
|
-
const
|
|
2040
|
-
const
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
signal: options.signal,
|
|
2044
|
-
headers: {
|
|
2164
|
+
if (userId) {
|
|
2165
|
+
const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
|
|
2166
|
+
const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
|
|
2167
|
+
try {
|
|
2168
|
+
const headers = {
|
|
2045
2169
|
"X-IG-App-ID": IG_APP_ID,
|
|
2046
2170
|
"Referer": `https://www.instagram.com/${username}/`
|
|
2171
|
+
};
|
|
2172
|
+
if (tokens?.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
|
|
2173
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
2174
|
+
cookiePath: options.cookiePath,
|
|
2175
|
+
cookieDomain: "instagram.com",
|
|
2176
|
+
signal: options.signal,
|
|
2177
|
+
headers
|
|
2178
|
+
});
|
|
2179
|
+
const data = res?.data?.user?.edge_owner_to_timeline_media;
|
|
2180
|
+
if (data?.edges && data.edges.length > 0) {
|
|
2181
|
+
return {
|
|
2182
|
+
edges: data.edges,
|
|
2183
|
+
hasNextPage: Boolean(data.page_info?.has_next_page),
|
|
2184
|
+
nextCursor: data.page_info?.end_cursor
|
|
2185
|
+
};
|
|
2047
2186
|
}
|
|
2048
|
-
})
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2187
|
+
} catch (err) {
|
|
2188
|
+
logExtractorDebug("edge_owner_to_timeline_media", err);
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
if (userId) {
|
|
2192
|
+
try {
|
|
2193
|
+
const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
|
|
2194
|
+
const headers = {
|
|
2195
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2196
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
2055
2197
|
};
|
|
2198
|
+
if (tokens?.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
|
|
2199
|
+
const res = await this.httpClient.fetchJson(restUrl, {
|
|
2200
|
+
cookiePath: options.cookiePath,
|
|
2201
|
+
cookieDomain: "instagram.com",
|
|
2202
|
+
signal: options.signal,
|
|
2203
|
+
headers
|
|
2204
|
+
});
|
|
2205
|
+
if (Array.isArray(res?.items) && res.items.length > 0) {
|
|
2206
|
+
const edges = res.items.map((item) => ({ node: item }));
|
|
2207
|
+
return {
|
|
2208
|
+
edges,
|
|
2209
|
+
hasNextPage: Boolean(res.more_available),
|
|
2210
|
+
nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
|
|
2211
|
+
};
|
|
2212
|
+
}
|
|
2213
|
+
} catch (err) {
|
|
2214
|
+
logExtractorDebug("v1 user feed REST", err);
|
|
2056
2215
|
}
|
|
2057
|
-
} catch (err) {
|
|
2058
|
-
logExtractorDebug("v1 user feed REST", err);
|
|
2059
2216
|
}
|
|
2060
2217
|
return { edges: [], hasNextPage: false };
|
|
2061
2218
|
}
|
|
@@ -2089,6 +2246,10 @@ var InstagramExtractor = class {
|
|
|
2089
2246
|
"Referer": "https://www.instagram.com/"
|
|
2090
2247
|
}
|
|
2091
2248
|
});
|
|
2249
|
+
if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
|
|
2250
|
+
const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
|
|
2251
|
+
throw new Error(`Instagram GraphQL query error: ${errMsg}`);
|
|
2252
|
+
}
|
|
2092
2253
|
const reelsData = res?.data?.fetch__XDTUserDict;
|
|
2093
2254
|
let connection = reelsData?.reels;
|
|
2094
2255
|
if (!connection && reelsData) {
|
|
@@ -2119,9 +2280,31 @@ var InstagramExtractor = class {
|
|
|
2119
2280
|
found: 0,
|
|
2120
2281
|
message: `Fetching Instagram profile @${username}...`
|
|
2121
2282
|
});
|
|
2122
|
-
|
|
2123
|
-
|
|
2283
|
+
let user = null;
|
|
2284
|
+
try {
|
|
2285
|
+
user = await this.fetchUserProfile(username, options);
|
|
2286
|
+
} catch (err) {
|
|
2287
|
+
logExtractorDebug("fetchUserProfile", err);
|
|
2288
|
+
}
|
|
2289
|
+
let webTokens = {};
|
|
2290
|
+
try {
|
|
2291
|
+
webTokens = await this.fetchWebTokens(username, options);
|
|
2292
|
+
} catch (err) {
|
|
2293
|
+
logExtractorDebug("fetchWebTokens", err);
|
|
2294
|
+
}
|
|
2295
|
+
let userId = user?.id || webTokens.userId;
|
|
2296
|
+
if (!userId && (subDir === "Stories" || subDir === "Highlights")) {
|
|
2297
|
+
try {
|
|
2298
|
+
const probe = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
|
|
2299
|
+
if (probe.userId) {
|
|
2300
|
+
userId = probe.userId;
|
|
2301
|
+
}
|
|
2302
|
+
} catch (err) {
|
|
2303
|
+
logExtractorDebug("userId probe", err);
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2124
2306
|
if (subDir === "Stories") {
|
|
2307
|
+
if (!userId) return;
|
|
2125
2308
|
options.onProgress?.({
|
|
2126
2309
|
stage: "Stories",
|
|
2127
2310
|
found: 0,
|
|
@@ -2137,6 +2320,7 @@ var InstagramExtractor = class {
|
|
|
2137
2320
|
return;
|
|
2138
2321
|
}
|
|
2139
2322
|
if (subDir === "Highlights") {
|
|
2323
|
+
if (!userId) return;
|
|
2140
2324
|
options.onProgress?.({
|
|
2141
2325
|
stage: "Highlights",
|
|
2142
2326
|
found: 0,
|
|
@@ -2163,9 +2347,22 @@ var InstagramExtractor = class {
|
|
|
2163
2347
|
}
|
|
2164
2348
|
let found = 0;
|
|
2165
2349
|
const emittedItemIds = /* @__PURE__ */ new Set();
|
|
2166
|
-
let currentEdges = user
|
|
2167
|
-
let hasNextPage = Boolean(user
|
|
2168
|
-
let endCursor = user
|
|
2350
|
+
let currentEdges = user?.edge_owner_to_timeline_media?.edges ?? [];
|
|
2351
|
+
let hasNextPage = Boolean(user?.edge_owner_to_timeline_media?.page_info?.has_next_page);
|
|
2352
|
+
let endCursor = user?.edge_owner_to_timeline_media?.page_info?.end_cursor;
|
|
2353
|
+
if (!user) {
|
|
2354
|
+
try {
|
|
2355
|
+
const firstPage = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
|
|
2356
|
+
currentEdges = firstPage.edges;
|
|
2357
|
+
hasNextPage = firstPage.hasNextPage;
|
|
2358
|
+
endCursor = firstPage.nextCursor;
|
|
2359
|
+
if (!userId && firstPage.userId) {
|
|
2360
|
+
userId = firstPage.userId;
|
|
2361
|
+
}
|
|
2362
|
+
} catch (err) {
|
|
2363
|
+
logExtractorDebug("fetchPostsTabGraphQL initial", err);
|
|
2364
|
+
}
|
|
2365
|
+
}
|
|
2169
2366
|
while (true) {
|
|
2170
2367
|
for (const edge of currentEdges) {
|
|
2171
2368
|
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
@@ -2188,7 +2385,7 @@ var InstagramExtractor = class {
|
|
|
2188
2385
|
}
|
|
2189
2386
|
await sleep(1500, 2500, options.signal);
|
|
2190
2387
|
try {
|
|
2191
|
-
const nextPage = await this.fetchNextPostsPage(userId, username, endCursor, options);
|
|
2388
|
+
const nextPage = await this.fetchNextPostsPage(userId || "", username, endCursor, options, webTokens);
|
|
2192
2389
|
currentEdges = nextPage.edges;
|
|
2193
2390
|
hasNextPage = nextPage.hasNextPage;
|
|
2194
2391
|
endCursor = nextPage.nextCursor;
|
|
@@ -2199,8 +2396,8 @@ var InstagramExtractor = class {
|
|
|
2199
2396
|
break;
|
|
2200
2397
|
}
|
|
2201
2398
|
}
|
|
2202
|
-
if (subDir === "Videos") {
|
|
2203
|
-
const videoEdges = user
|
|
2399
|
+
if (subDir === "Videos" && userId) {
|
|
2400
|
+
const videoEdges = user?.edge_felix_video_timeline?.edges ?? [];
|
|
2204
2401
|
for (const edge of videoEdges) {
|
|
2205
2402
|
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2206
2403
|
for (const item of items) {
|
|
@@ -2272,8 +2469,43 @@ var TWITTER_FEATURES = {
|
|
|
2272
2469
|
responsive_web_text_conversations_enabled: false,
|
|
2273
2470
|
longform_notetweets_rich_text_read_enabled: true,
|
|
2274
2471
|
longform_notetweets_inline_media_enabled: true,
|
|
2275
|
-
responsive_web_enhance_cards_enabled: false
|
|
2472
|
+
responsive_web_enhance_cards_enabled: false,
|
|
2473
|
+
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
2474
|
+
communities_web_enable_tweet_community_results_fetch: true,
|
|
2475
|
+
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
|
2476
|
+
articles_preview_enabled: true,
|
|
2477
|
+
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
|
2478
|
+
responsive_web_grok_analyze_post_followups_enabled: true,
|
|
2479
|
+
responsive_web_jetfuel_frame: true,
|
|
2480
|
+
responsive_web_grok_share_attachment_enabled: true,
|
|
2481
|
+
responsive_web_grok_analysis_button_from_backend: true,
|
|
2482
|
+
responsive_web_grok_image_annotation_enabled: true,
|
|
2483
|
+
responsive_web_grok_imagine_annotation_enabled: true,
|
|
2484
|
+
profile_label_improvements_pcf_label_in_post_enabled: true,
|
|
2485
|
+
rweb_tipjar_consumption_enabled: true
|
|
2276
2486
|
};
|
|
2487
|
+
function shouldSkipTweetReply(tweet, targetUserId, mode = "timeline") {
|
|
2488
|
+
const tweetResult = tweet.tweet || tweet;
|
|
2489
|
+
const legacy = tweetResult.legacy || tweetResult;
|
|
2490
|
+
const replyToUserId = legacy.in_reply_to_user_id_str;
|
|
2491
|
+
if (!replyToUserId) {
|
|
2492
|
+
return false;
|
|
2493
|
+
}
|
|
2494
|
+
const authorId = legacy.user_id_str || tweetResult.core?.user_results?.result?.rest_id;
|
|
2495
|
+
if (mode === "timeline") {
|
|
2496
|
+
if (targetUserId) {
|
|
2497
|
+
if (replyToUserId !== targetUserId || authorId && authorId !== targetUserId) {
|
|
2498
|
+
return true;
|
|
2499
|
+
}
|
|
2500
|
+
return false;
|
|
2501
|
+
}
|
|
2502
|
+
return Boolean(authorId && replyToUserId !== authorId);
|
|
2503
|
+
}
|
|
2504
|
+
if (authorId && replyToUserId === authorId) {
|
|
2505
|
+
return false;
|
|
2506
|
+
}
|
|
2507
|
+
return true;
|
|
2508
|
+
}
|
|
2277
2509
|
function parseTweetMediaEntities(tweet, username) {
|
|
2278
2510
|
const items = [];
|
|
2279
2511
|
const tweetResult = tweet.tweet || tweet;
|
|
@@ -2368,11 +2600,17 @@ var XExtractor = class {
|
|
|
2368
2600
|
}
|
|
2369
2601
|
if (username === "i_history_likes") {
|
|
2370
2602
|
if (options.cookiePath) {
|
|
2371
|
-
|
|
2603
|
+
let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
|
|
2604
|
+
if (!cookies["twid"]) {
|
|
2605
|
+
cookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
|
|
2606
|
+
}
|
|
2607
|
+
if (!cookies["twid"]) {
|
|
2608
|
+
cookies = await loadCookiesFromPath(options.cookiePath);
|
|
2609
|
+
}
|
|
2372
2610
|
const twid = cookies["twid"];
|
|
2373
2611
|
if (twid) {
|
|
2374
2612
|
const decoded = decodeURIComponent(twid).replace(/^"|"$/g, "").trim();
|
|
2375
|
-
const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded;
|
|
2613
|
+
const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded.includes("=") ? decoded.split("=").pop()?.trim() : decoded;
|
|
2376
2614
|
if (rawId && /^[a-zA-Z0-9_-]+$/.test(rawId)) {
|
|
2377
2615
|
return rawId;
|
|
2378
2616
|
}
|
|
@@ -2414,7 +2652,21 @@ var XExtractor = class {
|
|
|
2414
2652
|
async *extract(profile, options) {
|
|
2415
2653
|
const username = profile.username;
|
|
2416
2654
|
const subDir = options.subDir;
|
|
2417
|
-
const isLikes = username === "i_history_likes";
|
|
2655
|
+
const isLikes = username === "i_history_likes" || profile.cleanUrl?.endsWith("/likes");
|
|
2656
|
+
const authHeaders = {};
|
|
2657
|
+
if (options.cookiePath) {
|
|
2658
|
+
let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
|
|
2659
|
+
if (!cookies["ct0"] && !cookies["auth_token"]) {
|
|
2660
|
+
const twCookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
|
|
2661
|
+
cookies = { ...twCookies, ...cookies };
|
|
2662
|
+
}
|
|
2663
|
+
if (cookies["ct0"]) {
|
|
2664
|
+
authHeaders["x-csrf-token"] = cookies["ct0"];
|
|
2665
|
+
}
|
|
2666
|
+
if (cookies["auth_token"]) {
|
|
2667
|
+
authHeaders["x-twitter-auth-type"] = "OAuth2Session";
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2418
2670
|
options.onProgress?.({
|
|
2419
2671
|
stage: isLikes ? "Likes" : "Profile",
|
|
2420
2672
|
found: 0,
|
|
@@ -2445,11 +2697,15 @@ var XExtractor = class {
|
|
|
2445
2697
|
res = await this.httpClient.fetchJson(url, {
|
|
2446
2698
|
cookiePath: options.cookiePath,
|
|
2447
2699
|
cookieDomain: "x.com",
|
|
2448
|
-
signal: options.signal
|
|
2700
|
+
signal: options.signal,
|
|
2701
|
+
headers: authHeaders
|
|
2449
2702
|
});
|
|
2450
2703
|
} catch (err) {
|
|
2451
2704
|
break;
|
|
2452
2705
|
}
|
|
2706
|
+
if (res?.errors?.length && !res?.data?.user?.result) {
|
|
2707
|
+
throw new Error(`X/Twitter API error: ${res.errors[0]?.message || "GraphQL query failed"}`);
|
|
2708
|
+
}
|
|
2453
2709
|
const instructions = res?.data?.user?.result?.timeline_v2?.timeline?.instructions || [];
|
|
2454
2710
|
let newCursor;
|
|
2455
2711
|
let pageItemCount = 0;
|
|
@@ -2462,6 +2718,9 @@ var XExtractor = class {
|
|
|
2462
2718
|
}
|
|
2463
2719
|
const tweetContent = entry.content?.itemContent?.tweet_results?.result;
|
|
2464
2720
|
if (tweetContent) {
|
|
2721
|
+
if (shouldSkipTweetReply(tweetContent, userId, isLikes ? "likes" : "timeline")) {
|
|
2722
|
+
continue;
|
|
2723
|
+
}
|
|
2465
2724
|
const mediaItems = parseTweetMediaEntities(tweetContent, username);
|
|
2466
2725
|
for (const item of mediaItems) {
|
|
2467
2726
|
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
@@ -2490,16 +2749,86 @@ var XExtractor = class {
|
|
|
2490
2749
|
|
|
2491
2750
|
// src/lib/engines/extractors/tiktok.ts
|
|
2492
2751
|
function parseTikTokRehydrationJson(html) {
|
|
2493
|
-
const
|
|
2752
|
+
const universalMatch = html.match(
|
|
2494
2753
|
/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
|
|
2495
2754
|
);
|
|
2496
|
-
if (
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2755
|
+
if (universalMatch) {
|
|
2756
|
+
try {
|
|
2757
|
+
const raw = JSON.parse(universalMatch[1]);
|
|
2758
|
+
const scope = raw?.["__DEFAULT_SCOPE__"] || raw;
|
|
2759
|
+
if (scope) return scope;
|
|
2760
|
+
} catch {
|
|
2761
|
+
}
|
|
2502
2762
|
}
|
|
2763
|
+
const sigiTagMatch = html.match(
|
|
2764
|
+
/<script id="SIGI_STATE"[^>]*>([\s\S]*?)<\/script>/
|
|
2765
|
+
);
|
|
2766
|
+
if (sigiTagMatch) {
|
|
2767
|
+
try {
|
|
2768
|
+
const raw = JSON.parse(sigiTagMatch[1]);
|
|
2769
|
+
const user = Object.values(raw?.UserModule?.users || {})[0];
|
|
2770
|
+
const items = Object.values(raw?.ItemModule || {});
|
|
2771
|
+
return {
|
|
2772
|
+
"webapp.user-detail": {
|
|
2773
|
+
userInfo: { user },
|
|
2774
|
+
itemList: items
|
|
2775
|
+
},
|
|
2776
|
+
SIGI_STATE: raw
|
|
2777
|
+
};
|
|
2778
|
+
} catch {
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
const sigiWindowMatch = html.match(
|
|
2782
|
+
/(?:window\[['"]SIGI_STATE['"]\]|SIGI_STATE)\s*=\s*(\{[\s\S]*?\})\s*;/i
|
|
2783
|
+
);
|
|
2784
|
+
if (sigiWindowMatch) {
|
|
2785
|
+
try {
|
|
2786
|
+
const raw = JSON.parse(sigiWindowMatch[1]);
|
|
2787
|
+
const user = Object.values(raw?.UserModule?.users || {})[0];
|
|
2788
|
+
const items = Object.values(raw?.ItemModule || {});
|
|
2789
|
+
return {
|
|
2790
|
+
"webapp.user-detail": {
|
|
2791
|
+
userInfo: { user },
|
|
2792
|
+
itemList: items
|
|
2793
|
+
},
|
|
2794
|
+
SIGI_STATE: raw
|
|
2795
|
+
};
|
|
2796
|
+
} catch {
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2799
|
+
const nextDataMatch = html.match(
|
|
2800
|
+
/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/
|
|
2801
|
+
);
|
|
2802
|
+
if (nextDataMatch) {
|
|
2803
|
+
try {
|
|
2804
|
+
const raw = JSON.parse(nextDataMatch[1]);
|
|
2805
|
+
const pageProps = raw?.props?.pageProps;
|
|
2806
|
+
if (pageProps) {
|
|
2807
|
+
return {
|
|
2808
|
+
"webapp.user-detail": {
|
|
2809
|
+
userInfo: pageProps.userInfo,
|
|
2810
|
+
itemList: pageProps.items || pageProps.itemList || []
|
|
2811
|
+
},
|
|
2812
|
+
__NEXT_DATA__: raw
|
|
2813
|
+
};
|
|
2814
|
+
}
|
|
2815
|
+
} catch {
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2818
|
+
const secUidMatch = html.match(/(?:secUid|sec_uid)["':=\s]+(MS4wLjABAAAA[\w-]{32,64})/i);
|
|
2819
|
+
if (secUidMatch) {
|
|
2820
|
+
return {
|
|
2821
|
+
"webapp.user-detail": {
|
|
2822
|
+
userInfo: {
|
|
2823
|
+
user: {
|
|
2824
|
+
secUid: secUidMatch[1]
|
|
2825
|
+
}
|
|
2826
|
+
},
|
|
2827
|
+
itemList: []
|
|
2828
|
+
}
|
|
2829
|
+
};
|
|
2830
|
+
}
|
|
2831
|
+
return null;
|
|
2503
2832
|
}
|
|
2504
2833
|
function parseTikTokItem(item, username) {
|
|
2505
2834
|
const items = [];
|
|
@@ -2596,13 +2925,28 @@ var TikTokExtractor = class {
|
|
|
2596
2925
|
if (options.signal?.aborted) break;
|
|
2597
2926
|
if (visitedCursors.has(cursor)) break;
|
|
2598
2927
|
visitedCursors.add(cursor);
|
|
2599
|
-
const
|
|
2928
|
+
const deviceId = `${Math.floor(725e16 + Math.random() * 75e15)}`;
|
|
2929
|
+
const queryParams = new URLSearchParams({
|
|
2930
|
+
aid: "1988",
|
|
2931
|
+
app_name: "tiktok_web",
|
|
2932
|
+
device_platform: "web_pc",
|
|
2933
|
+
channel: "tiktok_web",
|
|
2934
|
+
device_id: deviceId,
|
|
2935
|
+
secUid,
|
|
2936
|
+
count: "30",
|
|
2937
|
+
cursor,
|
|
2938
|
+
post_item_list_request_type: "0"
|
|
2939
|
+
});
|
|
2940
|
+
const apiUrl = `https://www.tiktok.com/api/post/item_list/?${queryParams.toString()}`;
|
|
2600
2941
|
let res;
|
|
2601
2942
|
try {
|
|
2602
2943
|
res = await this.httpClient.fetchJson(apiUrl, {
|
|
2603
2944
|
cookiePath: options.cookiePath,
|
|
2604
2945
|
cookieDomain: "tiktok.com",
|
|
2605
|
-
signal: options.signal
|
|
2946
|
+
signal: options.signal,
|
|
2947
|
+
headers: {
|
|
2948
|
+
"Referer": profileUrl
|
|
2949
|
+
}
|
|
2606
2950
|
});
|
|
2607
2951
|
} catch {
|
|
2608
2952
|
break;
|
|
@@ -2654,28 +2998,55 @@ function decodeEscapedUrl(raw) {
|
|
|
2654
2998
|
function extractFacebookMediaFromHtml(html, username) {
|
|
2655
2999
|
const items = [];
|
|
2656
3000
|
const seenUrls = /* @__PURE__ */ new Set();
|
|
2657
|
-
const
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
3001
|
+
const videoPatterns = [
|
|
3002
|
+
/"browser_native_hd_url":"([^"]+)"/g,
|
|
3003
|
+
/"browser_native_sd_url":"([^"]+)"/g,
|
|
3004
|
+
/"playable_url_quality_hd":"([^"]+)"/g,
|
|
3005
|
+
/"playable_url":"([^"]+)"/g,
|
|
3006
|
+
/<video[^>]+src=["'](https?:[^"']+)["']/gi
|
|
3007
|
+
];
|
|
3008
|
+
let videoIndex = 1;
|
|
3009
|
+
for (const regex of videoPatterns) {
|
|
3010
|
+
let vMatch;
|
|
3011
|
+
while ((vMatch = regex.exec(html)) !== null) {
|
|
3012
|
+
const rawUrl = vMatch[1];
|
|
3013
|
+
const videoUrl = decodeEscapedUrl(rawUrl).replace(/&/g, "&");
|
|
3014
|
+
if (videoUrl.startsWith("http") && !seenUrls.has(videoUrl)) {
|
|
3015
|
+
seenUrls.add(videoUrl);
|
|
3016
|
+
items.push({
|
|
3017
|
+
id: `fb_video_${videoIndex}`,
|
|
3018
|
+
url: videoUrl,
|
|
3019
|
+
filename: `${username}_video_${videoIndex}`,
|
|
3020
|
+
extension: "mp4",
|
|
3021
|
+
type: "video"
|
|
3022
|
+
});
|
|
3023
|
+
videoIndex++;
|
|
3024
|
+
}
|
|
2671
3025
|
}
|
|
2672
3026
|
}
|
|
2673
|
-
const photoRegex = /,"image":\{"uri":"([^"]+)"/g;
|
|
3027
|
+
const photoRegex = /,"(?:image|viewer_image|full_size_image|display_image)":\{"uri":"([^"]+)"/g;
|
|
2674
3028
|
let match;
|
|
2675
3029
|
let photoIndex = 1;
|
|
2676
3030
|
while ((match = photoRegex.exec(html)) !== null) {
|
|
2677
3031
|
const rawUrl = match[1];
|
|
2678
|
-
const photoUrl = decodeEscapedUrl(rawUrl);
|
|
3032
|
+
const photoUrl = decodeEscapedUrl(rawUrl).replace(/&/g, "&");
|
|
3033
|
+
if (!seenUrls.has(photoUrl)) {
|
|
3034
|
+
seenUrls.add(photoUrl);
|
|
3035
|
+
items.push({
|
|
3036
|
+
id: `fb_photo_${photoIndex}`,
|
|
3037
|
+
url: photoUrl,
|
|
3038
|
+
filename: `${username}_photo_${photoIndex}`,
|
|
3039
|
+
extension: "jpg",
|
|
3040
|
+
type: "photo"
|
|
3041
|
+
});
|
|
3042
|
+
photoIndex++;
|
|
3043
|
+
}
|
|
3044
|
+
}
|
|
3045
|
+
const mbasicImgRegex = /<img[^>]+src=["'](https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*)["']/gi;
|
|
3046
|
+
let mMatch;
|
|
3047
|
+
while ((mMatch = mbasicImgRegex.exec(html)) !== null) {
|
|
3048
|
+
const rawUrl = mMatch[1];
|
|
3049
|
+
const photoUrl = decodeEscapedUrl(rawUrl).replace(/&/g, "&");
|
|
2679
3050
|
if (!seenUrls.has(photoUrl)) {
|
|
2680
3051
|
seenUrls.add(photoUrl);
|
|
2681
3052
|
items.push({
|
|
@@ -2689,10 +3060,10 @@ function extractFacebookMediaFromHtml(html, username) {
|
|
|
2689
3060
|
}
|
|
2690
3061
|
}
|
|
2691
3062
|
if (items.length === 0) {
|
|
2692
|
-
const scontentRegex = /https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*/gi;
|
|
3063
|
+
const scontentRegex = /https:\/\/[^"'\s<>]*scontent[^"'\s<>]*\.(?:jpg|png|webp)[^"'\s<>]*/gi;
|
|
2693
3064
|
let sMatch;
|
|
2694
3065
|
while ((sMatch = scontentRegex.exec(html)) !== null) {
|
|
2695
|
-
const cleanUrl = sMatch[0].replace(/\\/g, "");
|
|
3066
|
+
const cleanUrl = sMatch[0].replace(/\\/g, "").replace(/&/g, "&");
|
|
2696
3067
|
if (!seenUrls.has(cleanUrl)) {
|
|
2697
3068
|
seenUrls.add(cleanUrl);
|
|
2698
3069
|
items.push({
|
|
@@ -2708,6 +3079,59 @@ function extractFacebookMediaFromHtml(html, username) {
|
|
|
2708
3079
|
}
|
|
2709
3080
|
return items;
|
|
2710
3081
|
}
|
|
3082
|
+
function extractFacebookPaginationCursor(html) {
|
|
3083
|
+
const jsonCursorMatch = html.match(/"end_cursor":"([^"]+)"/) || html.match(/"cursor":"([^"]+)"/);
|
|
3084
|
+
if (jsonCursorMatch && jsonCursorMatch[1]) {
|
|
3085
|
+
return jsonCursorMatch[1];
|
|
3086
|
+
}
|
|
3087
|
+
const linkCursorMatch = html.match(/[?&](?:cursor|after)=([^"'\s&<>]+)/i);
|
|
3088
|
+
if (linkCursorMatch && linkCursorMatch[1]) {
|
|
3089
|
+
try {
|
|
3090
|
+
return decodeURIComponent(linkCursorMatch[1]);
|
|
3091
|
+
} catch {
|
|
3092
|
+
return linkCursorMatch[1];
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
return null;
|
|
3096
|
+
}
|
|
3097
|
+
function resolveFacebookUrl(href, currentUrl) {
|
|
3098
|
+
const cleanHref = href.replace(/\\/g, "").replace(/&/g, "&");
|
|
3099
|
+
try {
|
|
3100
|
+
return new URL(cleanHref, currentUrl).toString();
|
|
3101
|
+
} catch {
|
|
3102
|
+
return cleanHref;
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
function extractFacebookNextPageUrl(html, currentUrl) {
|
|
3106
|
+
const moreItemMatch = html.match(/<div[^>]*id=["'](?:m_more_item|m_more_posts)["'][^>]*>\s*<a[^>]+href=["']([^"']+)["']/i);
|
|
3107
|
+
if (moreItemMatch && moreItemMatch[1]) {
|
|
3108
|
+
return resolveFacebookUrl(moreItemMatch[1], currentUrl);
|
|
3109
|
+
}
|
|
3110
|
+
const cursorLabelMatch = html.match(/<a[^>]+href=["']([^"']*(?:cursor=|after=)[^"']+)["'][^>]*>[\s\S]*?(?:See [Mm]ore|More [Pp]hotos|More|Next|Older [Pp]osts)[\s\S]*?<\/a>/i);
|
|
3111
|
+
if (cursorLabelMatch && cursorLabelMatch[1]) {
|
|
3112
|
+
return resolveFacebookUrl(cursorLabelMatch[1], currentUrl);
|
|
3113
|
+
}
|
|
3114
|
+
const genericCursorMatch = html.match(/<a[^>]+href=["']([^"']*(?:[?&]|&)(?:cursor|after)=[^"']+)["']/i);
|
|
3115
|
+
if (genericCursorMatch && genericCursorMatch[1]) {
|
|
3116
|
+
return resolveFacebookUrl(genericCursorMatch[1], currentUrl);
|
|
3117
|
+
}
|
|
3118
|
+
const cursor = extractFacebookPaginationCursor(html);
|
|
3119
|
+
if (cursor) {
|
|
3120
|
+
try {
|
|
3121
|
+
const u = new URL(currentUrl);
|
|
3122
|
+
if (u.searchParams.get("cursor") !== cursor) {
|
|
3123
|
+
u.searchParams.set("cursor", cursor);
|
|
3124
|
+
return u.toString();
|
|
3125
|
+
}
|
|
3126
|
+
} catch {
|
|
3127
|
+
if (!currentUrl.includes(cursor)) {
|
|
3128
|
+
const sep = currentUrl.includes("?") ? "&" : "?";
|
|
3129
|
+
return `${currentUrl}${sep}cursor=${encodeURIComponent(cursor)}`;
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
return null;
|
|
3134
|
+
}
|
|
2711
3135
|
var FacebookExtractor = class {
|
|
2712
3136
|
platform = "facebook";
|
|
2713
3137
|
httpClient;
|
|
@@ -2724,26 +3148,120 @@ var FacebookExtractor = class {
|
|
|
2724
3148
|
found: 0,
|
|
2725
3149
|
message: `Fetching Facebook profile @${username}...`
|
|
2726
3150
|
});
|
|
2727
|
-
const
|
|
2728
|
-
const
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
3151
|
+
const targetPath = subDir === "Photos" ? "photos" : "videos";
|
|
3152
|
+
const desktopUrl = `https://www.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
|
|
3153
|
+
const mbasicUrl = `https://mbasic.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
|
|
3154
|
+
const mobileUrl = `https://m.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
|
|
3155
|
+
let currentUrl = desktopUrl;
|
|
3156
|
+
let html = "";
|
|
3157
|
+
try {
|
|
3158
|
+
html = await this.httpClient.fetchText(desktopUrl, {
|
|
3159
|
+
cookiePath: options.cookiePath,
|
|
3160
|
+
cookieDomain: "facebook.com",
|
|
3161
|
+
signal: options.signal,
|
|
3162
|
+
retries: 0,
|
|
3163
|
+
timeoutMs: 4e3
|
|
3164
|
+
});
|
|
3165
|
+
} catch {
|
|
3166
|
+
html = "";
|
|
3167
|
+
}
|
|
3168
|
+
let initialItems = extractFacebookMediaFromHtml(html, username);
|
|
3169
|
+
if (initialItems.length === 0) {
|
|
2741
3170
|
options.onProgress?.({
|
|
2742
|
-
stage:
|
|
2743
|
-
found,
|
|
2744
|
-
message: `
|
|
3171
|
+
stage: "Profile",
|
|
3172
|
+
found: 0,
|
|
3173
|
+
message: `Desktop yielded 0 items. Falling back to mbasic.facebook.com for @${username}...`
|
|
2745
3174
|
});
|
|
2746
|
-
|
|
3175
|
+
try {
|
|
3176
|
+
const mbasicHtml = await this.httpClient.fetchText(mbasicUrl, {
|
|
3177
|
+
cookiePath: options.cookiePath,
|
|
3178
|
+
cookieDomain: "facebook.com",
|
|
3179
|
+
signal: options.signal,
|
|
3180
|
+
retries: 1,
|
|
3181
|
+
timeoutMs: 5e3,
|
|
3182
|
+
headers: {
|
|
3183
|
+
"Referer": "https://mbasic.facebook.com/"
|
|
3184
|
+
}
|
|
3185
|
+
});
|
|
3186
|
+
const mbasicItems = extractFacebookMediaFromHtml(mbasicHtml, username);
|
|
3187
|
+
if (mbasicItems.length > 0 || mbasicHtml.length > 200) {
|
|
3188
|
+
html = mbasicHtml;
|
|
3189
|
+
currentUrl = mbasicUrl;
|
|
3190
|
+
initialItems = mbasicItems;
|
|
3191
|
+
}
|
|
3192
|
+
} catch {
|
|
3193
|
+
}
|
|
3194
|
+
if (initialItems.length === 0) {
|
|
3195
|
+
try {
|
|
3196
|
+
const mobileHtml = await this.httpClient.fetchText(mobileUrl, {
|
|
3197
|
+
cookiePath: options.cookiePath,
|
|
3198
|
+
cookieDomain: "facebook.com",
|
|
3199
|
+
signal: options.signal,
|
|
3200
|
+
retries: 0,
|
|
3201
|
+
timeoutMs: 5e3,
|
|
3202
|
+
headers: {
|
|
3203
|
+
"User-Agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
|
|
3204
|
+
"Referer": "https://m.facebook.com/"
|
|
3205
|
+
}
|
|
3206
|
+
});
|
|
3207
|
+
const mobileItems = extractFacebookMediaFromHtml(mobileHtml, username);
|
|
3208
|
+
if (mobileItems.length > 0) {
|
|
3209
|
+
html = mobileHtml;
|
|
3210
|
+
currentUrl = mobileUrl;
|
|
3211
|
+
initialItems = mobileItems;
|
|
3212
|
+
}
|
|
3213
|
+
} catch {
|
|
3214
|
+
}
|
|
3215
|
+
}
|
|
3216
|
+
}
|
|
3217
|
+
const seenUrls = /* @__PURE__ */ new Set();
|
|
3218
|
+
let found = 0;
|
|
3219
|
+
let page = 1;
|
|
3220
|
+
const maxPages = 5;
|
|
3221
|
+
while (true) {
|
|
3222
|
+
const items = extractFacebookMediaFromHtml(html, username);
|
|
3223
|
+
let pageNewCount = 0;
|
|
3224
|
+
for (const item of items) {
|
|
3225
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
3226
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
3227
|
+
if (seenUrls.has(item.url)) continue;
|
|
3228
|
+
seenUrls.add(item.url);
|
|
3229
|
+
found++;
|
|
3230
|
+
pageNewCount++;
|
|
3231
|
+
options.onProgress?.({
|
|
3232
|
+
stage: subDir,
|
|
3233
|
+
found,
|
|
3234
|
+
message: `Found item: ${item.filename}`
|
|
3235
|
+
});
|
|
3236
|
+
yield item;
|
|
3237
|
+
}
|
|
3238
|
+
if (page >= maxPages || options.signal?.aborted) {
|
|
3239
|
+
break;
|
|
3240
|
+
}
|
|
3241
|
+
const nextUrl = extractFacebookNextPageUrl(html, currentUrl);
|
|
3242
|
+
if (!nextUrl || nextUrl === currentUrl) {
|
|
3243
|
+
break;
|
|
3244
|
+
}
|
|
3245
|
+
if (pageNewCount === 0 && page > 1) {
|
|
3246
|
+
break;
|
|
3247
|
+
}
|
|
3248
|
+
await sleep(1e3, 2e3, options.signal);
|
|
3249
|
+
try {
|
|
3250
|
+
currentUrl = nextUrl;
|
|
3251
|
+
html = await this.httpClient.fetchText(currentUrl, {
|
|
3252
|
+
cookiePath: options.cookiePath,
|
|
3253
|
+
cookieDomain: "facebook.com",
|
|
3254
|
+
signal: options.signal,
|
|
3255
|
+
retries: 1,
|
|
3256
|
+
timeoutMs: 5e3,
|
|
3257
|
+
headers: {
|
|
3258
|
+
"Referer": currentUrl.includes("mbasic") ? "https://mbasic.facebook.com/" : "https://www.facebook.com/"
|
|
3259
|
+
}
|
|
3260
|
+
});
|
|
3261
|
+
page++;
|
|
3262
|
+
} catch {
|
|
3263
|
+
break;
|
|
3264
|
+
}
|
|
2747
3265
|
}
|
|
2748
3266
|
}
|
|
2749
3267
|
};
|
|
@@ -2771,17 +3289,48 @@ import path5 from "path";
|
|
|
2771
3289
|
import { Readable } from "stream";
|
|
2772
3290
|
import { pipeline } from "stream/promises";
|
|
2773
3291
|
var ARCHIVE_FILENAME = ".open-nami-archive.json";
|
|
3292
|
+
var YTDLP_ARCHIVE_FILENAME = "archive_yt-dlp.txt";
|
|
3293
|
+
var TEXT_ARCHIVE_FILENAME = "archive.txt";
|
|
2774
3294
|
async function loadArchive(destDir) {
|
|
2775
|
-
const
|
|
3295
|
+
const archive = /* @__PURE__ */ new Set();
|
|
3296
|
+
const jsonPath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
2776
3297
|
try {
|
|
2777
|
-
const raw = await fsPromises2.readFile(
|
|
3298
|
+
const raw = await fsPromises2.readFile(jsonPath, "utf-8");
|
|
2778
3299
|
const parsed = JSON.parse(raw);
|
|
2779
3300
|
if (Array.isArray(parsed)) {
|
|
2780
|
-
|
|
3301
|
+
for (const id of parsed) {
|
|
3302
|
+
if (id) archive.add(String(id));
|
|
3303
|
+
}
|
|
3304
|
+
}
|
|
3305
|
+
} catch {
|
|
3306
|
+
}
|
|
3307
|
+
const ytdlpPath = path5.join(destDir, YTDLP_ARCHIVE_FILENAME);
|
|
3308
|
+
try {
|
|
3309
|
+
const raw = await fsPromises2.readFile(ytdlpPath, "utf-8");
|
|
3310
|
+
const lines = raw.split(/\r?\n/);
|
|
3311
|
+
for (const rawLine of lines) {
|
|
3312
|
+
const line = rawLine.trim();
|
|
3313
|
+
if (!line || line.startsWith("#")) continue;
|
|
3314
|
+
const parts = line.split(/\s+/);
|
|
3315
|
+
const id = parts[parts.length - 1];
|
|
3316
|
+
if (id) archive.add(id);
|
|
3317
|
+
}
|
|
3318
|
+
} catch {
|
|
3319
|
+
}
|
|
3320
|
+
const textPath = path5.join(destDir, TEXT_ARCHIVE_FILENAME);
|
|
3321
|
+
try {
|
|
3322
|
+
const raw = await fsPromises2.readFile(textPath, "utf-8");
|
|
3323
|
+
const lines = raw.split(/\r?\n/);
|
|
3324
|
+
for (const rawLine of lines) {
|
|
3325
|
+
const line = rawLine.trim();
|
|
3326
|
+
if (!line || line.startsWith("#")) continue;
|
|
3327
|
+
const parts = line.split(/\s+/);
|
|
3328
|
+
const id = parts[parts.length - 1];
|
|
3329
|
+
if (id) archive.add(id);
|
|
2781
3330
|
}
|
|
2782
3331
|
} catch {
|
|
2783
3332
|
}
|
|
2784
|
-
return
|
|
3333
|
+
return archive;
|
|
2785
3334
|
}
|
|
2786
3335
|
async function saveArchive(destDir, archive) {
|
|
2787
3336
|
const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
@@ -2791,6 +3340,18 @@ async function saveArchive(destDir, archive) {
|
|
|
2791
3340
|
await fsPromises2.rename(tmpPath, archivePath);
|
|
2792
3341
|
} catch {
|
|
2793
3342
|
}
|
|
3343
|
+
const textPaths = [
|
|
3344
|
+
path5.join(destDir, YTDLP_ARCHIVE_FILENAME),
|
|
3345
|
+
path5.join(destDir, TEXT_ARCHIVE_FILENAME)
|
|
3346
|
+
];
|
|
3347
|
+
const content = [...archive].map((id) => `${id}
|
|
3348
|
+
`).join("");
|
|
3349
|
+
for (const tPath of textPaths) {
|
|
3350
|
+
try {
|
|
3351
|
+
await fsPromises2.writeFile(tPath, content, "utf-8");
|
|
3352
|
+
} catch {
|
|
3353
|
+
}
|
|
3354
|
+
}
|
|
2794
3355
|
}
|
|
2795
3356
|
function sanitizeFilename(rawFilename, fallbackId, ext) {
|
|
2796
3357
|
const cleanExt = ext.replace(/^\./, "");
|
|
@@ -2804,6 +3365,14 @@ function sanitizeFilename(rawFilename, fallbackId, ext) {
|
|
|
2804
3365
|
}
|
|
2805
3366
|
return sanitizedBase.includes(".") ? sanitizedBase : `${sanitizedBase}.${cleanExt}`;
|
|
2806
3367
|
}
|
|
3368
|
+
function isCdnUrl(rawUrl) {
|
|
3369
|
+
try {
|
|
3370
|
+
const hostname = new URL(rawUrl).hostname.toLowerCase();
|
|
3371
|
+
return hostname.endsWith(".cdninstagram.com") || hostname.endsWith(".fbcdn.net") || hostname.endsWith(".tiktokcdn.com") || hostname.endsWith(".akamaized.net") || hostname === "pbs.twimg.com" || hostname === "video.twimg.com";
|
|
3372
|
+
} catch {
|
|
3373
|
+
return false;
|
|
3374
|
+
}
|
|
3375
|
+
}
|
|
2807
3376
|
async function downloadMediaStream(items, options) {
|
|
2808
3377
|
const { jobName, destDir, signal, onProgress } = options;
|
|
2809
3378
|
const httpClient = options.httpClient ?? new HttpClient();
|
|
@@ -2843,12 +3412,13 @@ async function downloadMediaStream(items, options) {
|
|
|
2843
3412
|
consecutiveErrors++;
|
|
2844
3413
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2845
3414
|
errors.push(`Extraction stream error: ${errMsg}`);
|
|
3415
|
+
const cleanErr = errMsg.includes("429") ? "rate limited by platform (HTTP 429)" : errMsg.length > 50 ? `${errMsg.slice(0, 47)}\u2026` : errMsg;
|
|
2846
3416
|
onProgress?.({
|
|
2847
3417
|
job: jobName,
|
|
2848
3418
|
downloadedCount: downloaded,
|
|
2849
3419
|
skippedCount: skipped,
|
|
2850
3420
|
currentFile: "",
|
|
2851
|
-
statusText: `Extractor
|
|
3421
|
+
statusText: `Extractor error: ${cleanErr}`
|
|
2852
3422
|
});
|
|
2853
3423
|
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS || signal?.aborted) {
|
|
2854
3424
|
isDone = true;
|
|
@@ -2883,10 +3453,13 @@ async function downloadMediaStream(items, options) {
|
|
|
2883
3453
|
if (options.referer) {
|
|
2884
3454
|
reqHeaders["Referer"] = options.referer;
|
|
2885
3455
|
}
|
|
3456
|
+
const isCdn = isCdnUrl(item.url);
|
|
2886
3457
|
const response = await httpClient.request(item.url, {
|
|
2887
3458
|
signal,
|
|
2888
|
-
cookiePath: options.cookiePath,
|
|
2889
|
-
headers: reqHeaders
|
|
3459
|
+
cookiePath: isCdn ? void 0 : options.cookiePath,
|
|
3460
|
+
headers: reqHeaders,
|
|
3461
|
+
retries: 2,
|
|
3462
|
+
retryDelayMs: 1500
|
|
2890
3463
|
});
|
|
2891
3464
|
if (!response.ok || !response.body) {
|
|
2892
3465
|
try {
|
|
@@ -2907,6 +3480,7 @@ async function downloadMediaStream(items, options) {
|
|
|
2907
3480
|
await fsPromises2.rename(tmpPath, finalPath);
|
|
2908
3481
|
archive.add(item.id);
|
|
2909
3482
|
downloaded++;
|
|
3483
|
+
consecutiveErrors = 0;
|
|
2910
3484
|
onProgress?.({
|
|
2911
3485
|
job: jobName,
|
|
2912
3486
|
downloadedCount: downloaded,
|
|
@@ -3318,8 +3892,9 @@ async function runGalleryDl(options) {
|
|
|
3318
3892
|
let downloadedCount = 0;
|
|
3319
3893
|
let skippedCount = 0;
|
|
3320
3894
|
let errorOutput = "";
|
|
3895
|
+
let engineCmdStr = "gallery-dl";
|
|
3321
3896
|
try {
|
|
3322
|
-
await ensureGalleryDl((status) => {
|
|
3897
|
+
engineCmdStr = await ensureGalleryDl((status) => {
|
|
3323
3898
|
onProgress?.({
|
|
3324
3899
|
job: subDir,
|
|
3325
3900
|
downloadedCount: 0,
|
|
@@ -3335,9 +3910,15 @@ async function runGalleryDl(options) {
|
|
|
3335
3910
|
error: err.message || String(err)
|
|
3336
3911
|
};
|
|
3337
3912
|
}
|
|
3338
|
-
const engineCmd = resolveEngineCommand("gallery-dl");
|
|
3913
|
+
const engineCmd = engineCmdStr.includes(path8.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("gallery-dl");
|
|
3339
3914
|
const fullArgs = [...engineCmd.baseArgs, ...args];
|
|
3340
3915
|
return new Promise((resolve) => {
|
|
3916
|
+
onProgress?.({
|
|
3917
|
+
job: subDir,
|
|
3918
|
+
downloadedCount,
|
|
3919
|
+
skippedCount,
|
|
3920
|
+
statusText: "downloading with gallery-dl\u2026"
|
|
3921
|
+
});
|
|
3341
3922
|
const child = spawn2(engineCmd.command, fullArgs, {
|
|
3342
3923
|
windowsHide: true,
|
|
3343
3924
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -3461,8 +4042,9 @@ async function runYtDlp(options) {
|
|
|
3461
4042
|
let downloadedCount = 0;
|
|
3462
4043
|
let skippedCount = 0;
|
|
3463
4044
|
let errorOutput = "";
|
|
4045
|
+
let engineCmdStr = "yt-dlp";
|
|
3464
4046
|
try {
|
|
3465
|
-
await ensureYtDlp((status) => {
|
|
4047
|
+
engineCmdStr = await ensureYtDlp((status) => {
|
|
3466
4048
|
onProgress?.({
|
|
3467
4049
|
job: "Videos",
|
|
3468
4050
|
downloadedCount: 0,
|
|
@@ -3478,9 +4060,15 @@ async function runYtDlp(options) {
|
|
|
3478
4060
|
error: err.message || String(err)
|
|
3479
4061
|
};
|
|
3480
4062
|
}
|
|
3481
|
-
const engineCmd = resolveEngineCommand("yt-dlp");
|
|
4063
|
+
const engineCmd = engineCmdStr.includes(path9.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("yt-dlp");
|
|
3482
4064
|
const fullArgs = [...engineCmd.baseArgs, ...args];
|
|
3483
4065
|
return new Promise((resolve) => {
|
|
4066
|
+
onProgress?.({
|
|
4067
|
+
job: "Videos",
|
|
4068
|
+
downloadedCount,
|
|
4069
|
+
skippedCount,
|
|
4070
|
+
statusText: "downloading with yt-dlp\u2026"
|
|
4071
|
+
});
|
|
3484
4072
|
const child = spawn3(engineCmd.command, fullArgs, {
|
|
3485
4073
|
windowsHide: true,
|
|
3486
4074
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -3617,7 +4205,7 @@ async function dispatchDownload(options) {
|
|
|
3617
4205
|
jobs.push({ type: "highlights" });
|
|
3618
4206
|
}
|
|
3619
4207
|
}
|
|
3620
|
-
const
|
|
4208
|
+
const defaultRunNative = async (subDir) => {
|
|
3621
4209
|
const dest = path10.join(targetDir, subDir);
|
|
3622
4210
|
try {
|
|
3623
4211
|
const extractor = getExtractor(profile.platform);
|
|
@@ -3650,27 +4238,34 @@ async function dispatchDownload(options) {
|
|
|
3650
4238
|
};
|
|
3651
4239
|
}
|
|
3652
4240
|
};
|
|
3653
|
-
const
|
|
4241
|
+
const runNative = options.runnerOverrides?.nativeRunner ?? defaultRunNative;
|
|
4242
|
+
const runGdl = options.runnerOverrides?.gdlRunner ?? ((subDir) => runGalleryDl({
|
|
3654
4243
|
profile,
|
|
3655
4244
|
targetDir,
|
|
3656
4245
|
subDir,
|
|
3657
4246
|
cookiePath: cookieCopy?.filePath,
|
|
3658
4247
|
signal: options.signal,
|
|
3659
4248
|
onProgress
|
|
3660
|
-
});
|
|
3661
|
-
const runYt = () => runYtDlp({
|
|
4249
|
+
}));
|
|
4250
|
+
const runYt = options.runnerOverrides?.ytRunner ?? (() => runYtDlp({
|
|
3662
4251
|
profile,
|
|
3663
4252
|
targetDir,
|
|
3664
4253
|
cookiePath: cookieCopy?.filePath,
|
|
3665
4254
|
signal: options.signal,
|
|
3666
4255
|
onProgress
|
|
3667
|
-
});
|
|
4256
|
+
}));
|
|
3668
4257
|
for (const job of jobs) {
|
|
3669
4258
|
if (options.signal?.aborted) break;
|
|
3670
4259
|
if (job.type === "photos" || job.type === "stories" || job.type === "highlights") {
|
|
3671
4260
|
const subDir = job.type === "photos" ? "Photos" : job.type === "stories" ? "Stories" : "Highlights";
|
|
3672
4261
|
let res = await runNative(subDir);
|
|
3673
4262
|
if ((!res.success || res.downloaded === 0 && res.skipped === 0) && !options.signal?.aborted) {
|
|
4263
|
+
onProgress?.({
|
|
4264
|
+
job: subDir,
|
|
4265
|
+
downloadedCount: totalDownloaded,
|
|
4266
|
+
skippedCount: totalSkipped,
|
|
4267
|
+
statusText: "switching to gallery-dl engine\u2026"
|
|
4268
|
+
});
|
|
3674
4269
|
const gdlRes = await runGdl(subDir);
|
|
3675
4270
|
if (gdlRes.downloaded > 0 || gdlRes.skipped > 0 || gdlRes.success) {
|
|
3676
4271
|
res = gdlRes;
|
|
@@ -3685,12 +4280,24 @@ async function dispatchDownload(options) {
|
|
|
3685
4280
|
totalDownloaded += res.downloaded;
|
|
3686
4281
|
totalSkipped += res.skipped;
|
|
3687
4282
|
} else if (!options.signal?.aborted) {
|
|
4283
|
+
onProgress?.({
|
|
4284
|
+
job: "Videos",
|
|
4285
|
+
downloadedCount: totalDownloaded,
|
|
4286
|
+
skippedCount: totalSkipped,
|
|
4287
|
+
statusText: "switching to yt-dlp engine\u2026"
|
|
4288
|
+
});
|
|
3688
4289
|
const fbRes = await runYt();
|
|
3689
4290
|
if (fbRes.downloaded > 0 || fbRes.skipped > 0 || fbRes.success) {
|
|
3690
4291
|
totalDownloaded += fbRes.downloaded;
|
|
3691
4292
|
totalSkipped += fbRes.skipped;
|
|
3692
4293
|
if (!fbRes.success && fbRes.error) errors.push(`[Videos] ${fbRes.error}`);
|
|
3693
4294
|
} else if (!options.signal?.aborted) {
|
|
4295
|
+
onProgress?.({
|
|
4296
|
+
job: "Videos",
|
|
4297
|
+
downloadedCount: totalDownloaded,
|
|
4298
|
+
skippedCount: totalSkipped,
|
|
4299
|
+
statusText: "switching to gallery-dl engine\u2026"
|
|
4300
|
+
});
|
|
3694
4301
|
const gdlRes = await runGdl("Videos");
|
|
3695
4302
|
totalDownloaded += gdlRes.downloaded;
|
|
3696
4303
|
totalSkipped += gdlRes.skipped;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "open-nami",
|
|
3
|
-
"version": "1.0.
|
|
4
|
-
"description": "Open Nami
|
|
3
|
+
"version": "1.0.6",
|
|
4
|
+
"description": "Open Nami: terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"open-nami": "dist/cli.js",
|