open-nami 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/cli.js +749 -142
  2. package/package.json +2 -2
package/dist/cli.js CHANGED
@@ -9,8 +9,8 @@ var require_package = __commonJS({
9
9
  "package.json"(exports, module) {
10
10
  module.exports = {
11
11
  name: "open-nami",
12
- version: "1.0.4",
13
- description: "Open Nami \u2014 terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
12
+ version: "1.0.6",
13
+ description: "Open Nami: terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
14
14
  type: "module",
15
15
  bin: {
16
16
  "open-nami": "dist/cli.js",
@@ -1407,10 +1407,11 @@ function parseProfileInput(input) {
1407
1407
  const reserved = ["home", "explore", "notifications", "messages", "i", "settings"];
1408
1408
  if (firstPart && !reserved.includes(firstPart.toLowerCase())) {
1409
1409
  const username = firstPart.replace(/^@/, "");
1410
+ const isLikesUrl = pathParts[1]?.toLowerCase() === "likes";
1410
1411
  return {
1411
1412
  platform: "x",
1412
1413
  username: sanitizeUsername(username),
1413
- cleanUrl: `https://x.com/${username}`,
1414
+ cleanUrl: isLikesUrl ? `https://x.com/${username}/likes` : `https://x.com/${username}`,
1414
1415
  rawInput: input
1415
1416
  };
1416
1417
  }
@@ -1575,6 +1576,9 @@ function resolvePlatformDownloadsDir(options = {}) {
1575
1576
  return defaultFallback;
1576
1577
  }
1577
1578
 
1579
+ // src/lib/engines/extractors/instagram.ts
1580
+ import crypto from "crypto";
1581
+
1578
1582
  // src/lib/engines/extractors/http.ts
1579
1583
  import fs5 from "fs/promises";
1580
1584
  import { setTimeout as setTimeoutPromise } from "timers/promises";
@@ -1762,7 +1766,9 @@ var HttpClient = class {
1762
1766
  }
1763
1767
  throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
1764
1768
  }
1765
- return await res.json();
1769
+ const text = await res.text();
1770
+ const cleanText = text.replace(/^\s*(?:for\s*\(\s*;\s*;\s*\);|\)\]\}',)/, "").trim();
1771
+ return JSON.parse(cleanText);
1766
1772
  }
1767
1773
  async fetchText(url, options = {}) {
1768
1774
  const res = await this.request(url, options);
@@ -1781,13 +1787,13 @@ var HttpClient = class {
1781
1787
  var IG_APP_ID = "936619743392459";
1782
1788
  var IG_TIMELINE_QUERY_HASH = "69cba40317214236af40e7efa697781d";
1783
1789
  var IG_QUERIES = {
1784
- PolarisProfilePostsTabContentQuery_connection: "28142289515441884",
1790
+ PolarisProfilePostsTabContentQuery_connection: "39229011733413759",
1785
1791
  PolarisProfileReelsTabContentQuery_connection: "28989741540630190",
1786
1792
  PolarisProfileTaggedTabContentQuery_connection: "28794602806842110",
1787
1793
  PolarisProfileStoryHighlightsTrayContentQuery: "26970053832668570",
1788
1794
  PolarisStoriesV3ReelPageGalleryQuery: "28262315486766731",
1789
1795
  PolarisStoriesV3TrayContainerQuery: "27703822975903310",
1790
- PolarisProfilePageContentQuery: "28036671149327607"
1796
+ PolarisProfilePageContentQuery: "28649589871350527"
1791
1797
  };
1792
1798
  var IG_POLARIS_DOC_ID = IG_QUERIES.PolarisProfilePostsTabContentQuery_connection;
1793
1799
  function logExtractorDebug(endpoint, err) {
@@ -1831,6 +1837,21 @@ function parseIgNodeToMediaItems(node, username) {
1831
1837
  height: video?.height ?? node.dimensions?.height,
1832
1838
  thumbnailUrl: node.display_url
1833
1839
  });
1840
+ } else {
1841
+ const candidate = node.image_versions2?.candidates?.[0];
1842
+ const photoUrl = candidate?.url || node.display_url;
1843
+ if (photoUrl) {
1844
+ items.push({
1845
+ id: baseId,
1846
+ url: photoUrl,
1847
+ filename: `${username}_${date || baseId}`,
1848
+ extension: "jpg",
1849
+ type: "photo",
1850
+ date,
1851
+ width: candidate?.width ?? node.dimensions?.width,
1852
+ height: candidate?.height ?? node.dimensions?.height
1853
+ });
1854
+ }
1834
1855
  }
1835
1856
  } else {
1836
1857
  const candidate = node.image_versions2?.candidates?.[0];
@@ -1910,7 +1931,15 @@ var InstagramExtractor = class {
1910
1931
  if (!res.data?.user) {
1911
1932
  throw new Error(`Instagram user not found or private profile: ${username}`);
1912
1933
  }
1913
- return res.data.user;
1934
+ const user = res.data.user;
1935
+ if (user.is_private) {
1936
+ const fs13 = user.friendship_status;
1937
+ const isFollowing = fs13 ? Boolean(fs13.following) : Boolean(user.followed_by_viewer);
1938
+ if (!isFollowing) {
1939
+ throw new Error(`Instagram profile @${username} is private. You must provide a session cookie jar of an account following this profile.`);
1940
+ }
1941
+ }
1942
+ return user;
1914
1943
  }
1915
1944
  async fetchHighlightsTray(userId, options) {
1916
1945
  try {
@@ -1932,6 +1961,10 @@ var InstagramExtractor = class {
1932
1961
  "Referer": "https://www.instagram.com/"
1933
1962
  }
1934
1963
  });
1964
+ if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
1965
+ const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
1966
+ throw new Error(`Instagram GraphQL query error: ${errMsg}`);
1967
+ }
1935
1968
  const edges = res?.data?.highlights?.edges;
1936
1969
  if (Array.isArray(edges) && edges.length > 0) {
1937
1970
  return edges.map((e) => e.node);
@@ -1975,87 +2008,211 @@ var InstagramExtractor = class {
1975
2008
  return [];
1976
2009
  }
1977
2010
  }
1978
- async fetchNextPostsPage(userId, username, cursor, options) {
1979
- const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
1980
- const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
2011
+ async fetchWebTokens(username, options) {
2012
+ let csrfToken;
2013
+ if (options.cookiePath) {
2014
+ try {
2015
+ const cookies = await loadCookiesFromPath(options.cookiePath, "instagram.com");
2016
+ csrfToken = cookies["csrftoken"];
2017
+ } catch {
2018
+ }
2019
+ }
2020
+ if (!csrfToken) {
2021
+ csrfToken = crypto.randomBytes(16).toString("hex");
2022
+ }
1981
2023
  try {
1982
- const res = await this.httpClient.fetchJson(url, {
2024
+ const html = await this.httpClient.fetchText(`https://www.instagram.com/${encodeURIComponent(username)}/`, {
1983
2025
  cookiePath: options.cookiePath,
1984
2026
  cookieDomain: "instagram.com",
1985
2027
  signal: options.signal,
1986
2028
  headers: {
1987
- "X-IG-App-ID": IG_APP_ID,
1988
- "Referer": `https://www.instagram.com/${username}/`
2029
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
1989
2030
  }
1990
2031
  });
1991
- const data = res?.data?.user?.edge_owner_to_timeline_media;
1992
- if (data?.edges && data.edges.length > 0) {
1993
- return {
1994
- edges: data.edges,
1995
- hasNextPage: Boolean(data.page_info?.has_next_page),
1996
- nextCursor: data.page_info?.end_cursor
1997
- };
2032
+ let lsd;
2033
+ let fbDtsg;
2034
+ let userId;
2035
+ const eqmcMatch = html.match(/id="__eqmc"[^>]*>([^<]+)<\/script>/);
2036
+ if (eqmcMatch) {
2037
+ try {
2038
+ const eqmc = JSON.parse(eqmcMatch[1]);
2039
+ if (eqmc.l) lsd = eqmc.l;
2040
+ if (eqmc.f) fbDtsg = eqmc.f;
2041
+ } catch {
2042
+ }
2043
+ }
2044
+ if (!lsd) {
2045
+ const m = html.match(/"lsd":"([^"]+)"/) || html.match(/"LSD",\[\],\{"token":"([^"]+)"/);
2046
+ if (m) lsd = m[1];
2047
+ }
2048
+ if (!fbDtsg) {
2049
+ const m = html.match(/"dtsg":\{"token":"([^"]+)"/);
2050
+ if (m) fbDtsg = m[1];
1998
2051
  }
2052
+ const uMatch = html.match(/"profile_id":"([^"]+)"/) || html.match(/"user_id":"([^"]+)"/);
2053
+ if (uMatch) userId = uMatch[1];
2054
+ return { lsd, fbDtsg, csrfToken, userId };
1999
2055
  } catch (err) {
2000
- logExtractorDebug("edge_owner_to_timeline_media", err);
2056
+ logExtractorDebug("fetchWebTokens", err);
2057
+ return { csrfToken };
2058
+ }
2059
+ }
2060
+ async fetchPostsTabGraphQL(username, cursor, tokens, options) {
2061
+ const postVariables = JSON.stringify({
2062
+ after: cursor || null,
2063
+ before: null,
2064
+ data: {
2065
+ count: 12,
2066
+ include_reel_media_seen_timestamp: true,
2067
+ include_relationship_info: true,
2068
+ latest_besties_reel_media: true,
2069
+ latest_reel_media: true
2070
+ },
2071
+ first: 12,
2072
+ include_multi_captions: true,
2073
+ last: null,
2074
+ username,
2075
+ __relay_internal__pv__PolarisMultiCaptionCarouselEnabledrelayprovider: true,
2076
+ __relay_internal__pv__PolarisShortDramaEnabledrelayprovider: true,
2077
+ __relay_internal__pv__PolarisThreadsStyleCommentsEnabledrelayprovider: false,
2078
+ __relay_internal__pv__PolarisReelsRecoDebugOverlayEnabledrelayprovider: false
2079
+ });
2080
+ const bodyParams = new URLSearchParams({
2081
+ av: "17841415137994167",
2082
+ __d: "www",
2083
+ __user: "0",
2084
+ __a: "1",
2085
+ __req: "0",
2086
+ __hs: "20731.HYP:instagram_web_pkg.2.1...0",
2087
+ dpr: "1",
2088
+ __ccg: "EXCELLENT",
2089
+ __rev: "1049285836",
2090
+ __hsi: "7693193318680449885",
2091
+ __comet_req: "7",
2092
+ jazoest: "26461",
2093
+ fb_api_caller_class: "RelayModern",
2094
+ fb_api_req_friendly_name: "PolarisProfilePostsTabContentQuery_connection",
2095
+ server_timestamps: "true",
2096
+ variables: postVariables,
2097
+ doc_id: IG_POLARIS_DOC_ID
2098
+ });
2099
+ if (tokens.fbDtsg) bodyParams.set("fb_dtsg", tokens.fbDtsg);
2100
+ if (tokens.lsd) bodyParams.set("lsd", tokens.lsd);
2101
+ const headers = {
2102
+ "Accept": "*/*",
2103
+ "Content-Type": "application/x-www-form-urlencoded",
2104
+ "X-IG-App-ID": IG_APP_ID,
2105
+ "X-IG-Max-Touch-Points": "0",
2106
+ "X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
2107
+ "Referer": `https://www.instagram.com/${username}/`,
2108
+ "Origin": "https://www.instagram.com",
2109
+ "X-ASBD-ID": "359341",
2110
+ "Sec-Fetch-Dest": "empty",
2111
+ "Sec-Fetch-Mode": "cors",
2112
+ "Sec-Fetch-Site": "same-origin"
2113
+ };
2114
+ if (tokens.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
2115
+ if (tokens.lsd) headers["X-FB-LSD"] = tokens.lsd;
2116
+ const res = await this.httpClient.fetchJson("https://www.instagram.com/graphql/query", {
2117
+ method: "POST",
2118
+ body: bodyParams.toString(),
2119
+ cookiePath: options.cookiePath,
2120
+ cookieDomain: "instagram.com",
2121
+ signal: options.signal,
2122
+ headers
2123
+ });
2124
+ if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
2125
+ const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
2126
+ throw new Error(`Instagram GraphQL query error: ${errMsg}`);
2127
+ }
2128
+ const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
2129
+ if (connection?.edges && Array.isArray(connection.edges)) {
2130
+ let resolvedUserId;
2131
+ for (const edge of connection.edges) {
2132
+ const node = edge?.node;
2133
+ if (node?.owner_id) {
2134
+ resolvedUserId = String(node.owner_id);
2135
+ break;
2136
+ }
2137
+ if (node?.user?.pk || node?.user?.id) {
2138
+ resolvedUserId = String(node.user.pk || node.user.id);
2139
+ break;
2140
+ }
2141
+ if (typeof node?.id === "string" && node.id.includes("_")) {
2142
+ resolvedUserId = node.id.split("_")[1];
2143
+ break;
2144
+ }
2145
+ }
2146
+ return {
2147
+ edges: connection.edges,
2148
+ hasNextPage: Boolean(connection.page_info?.has_next_page),
2149
+ nextCursor: connection.page_info?.end_cursor,
2150
+ userId: resolvedUserId
2151
+ };
2001
2152
  }
2153
+ return { edges: [], hasNextPage: false };
2154
+ }
2155
+ async fetchNextPostsPage(userId, username, cursor, options, tokens) {
2002
2156
  try {
2003
- const postVariables = JSON.stringify({
2004
- after: cursor,
2005
- first: 12,
2006
- username,
2007
- data: { count: 12 }
2008
- });
2009
- const postUrl = "https://www.instagram.com/graphql/query";
2010
- const body = new URLSearchParams({
2011
- doc_id: IG_POLARIS_DOC_ID,
2012
- variables: postVariables
2013
- }).toString();
2014
- const res = await this.httpClient.fetchJson(postUrl, {
2015
- method: "POST",
2016
- body,
2017
- cookiePath: options.cookiePath,
2018
- cookieDomain: "instagram.com",
2019
- signal: options.signal,
2020
- headers: {
2021
- "Content-Type": "application/x-www-form-urlencoded",
2022
- "X-IG-App-ID": IG_APP_ID,
2023
- "X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
2024
- "Referer": `https://www.instagram.com/${username}/`
2025
- }
2026
- });
2027
- const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
2028
- if (connection?.edges && connection.edges.length > 0) {
2029
- return {
2030
- edges: connection.edges,
2031
- hasNextPage: Boolean(connection.page_info?.has_next_page),
2032
- nextCursor: connection.page_info?.end_cursor
2033
- };
2157
+ const page = await this.fetchPostsTabGraphQL(username, cursor, tokens || {}, options);
2158
+ if (page.edges && page.edges.length > 0) {
2159
+ return page;
2034
2160
  }
2035
2161
  } catch (err) {
2036
2162
  logExtractorDebug("PolarisProfilePostsTabContentQuery_connection", err);
2037
2163
  }
2038
- try {
2039
- const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
2040
- const res = await this.httpClient.fetchJson(restUrl, {
2041
- cookiePath: options.cookiePath,
2042
- cookieDomain: "instagram.com",
2043
- signal: options.signal,
2044
- headers: {
2164
+ if (userId) {
2165
+ const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
2166
+ const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
2167
+ try {
2168
+ const headers = {
2045
2169
  "X-IG-App-ID": IG_APP_ID,
2046
2170
  "Referer": `https://www.instagram.com/${username}/`
2171
+ };
2172
+ if (tokens?.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
2173
+ const res = await this.httpClient.fetchJson(url, {
2174
+ cookiePath: options.cookiePath,
2175
+ cookieDomain: "instagram.com",
2176
+ signal: options.signal,
2177
+ headers
2178
+ });
2179
+ const data = res?.data?.user?.edge_owner_to_timeline_media;
2180
+ if (data?.edges && data.edges.length > 0) {
2181
+ return {
2182
+ edges: data.edges,
2183
+ hasNextPage: Boolean(data.page_info?.has_next_page),
2184
+ nextCursor: data.page_info?.end_cursor
2185
+ };
2047
2186
  }
2048
- });
2049
- if (Array.isArray(res?.items) && res.items.length > 0) {
2050
- const edges = res.items.map((item) => ({ node: item }));
2051
- return {
2052
- edges,
2053
- hasNextPage: Boolean(res.more_available),
2054
- nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
2187
+ } catch (err) {
2188
+ logExtractorDebug("edge_owner_to_timeline_media", err);
2189
+ }
2190
+ }
2191
+ if (userId) {
2192
+ try {
2193
+ const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
2194
+ const headers = {
2195
+ "X-IG-App-ID": IG_APP_ID,
2196
+ "Referer": `https://www.instagram.com/${username}/`
2055
2197
  };
2198
+ if (tokens?.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
2199
+ const res = await this.httpClient.fetchJson(restUrl, {
2200
+ cookiePath: options.cookiePath,
2201
+ cookieDomain: "instagram.com",
2202
+ signal: options.signal,
2203
+ headers
2204
+ });
2205
+ if (Array.isArray(res?.items) && res.items.length > 0) {
2206
+ const edges = res.items.map((item) => ({ node: item }));
2207
+ return {
2208
+ edges,
2209
+ hasNextPage: Boolean(res.more_available),
2210
+ nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
2211
+ };
2212
+ }
2213
+ } catch (err) {
2214
+ logExtractorDebug("v1 user feed REST", err);
2056
2215
  }
2057
- } catch (err) {
2058
- logExtractorDebug("v1 user feed REST", err);
2059
2216
  }
2060
2217
  return { edges: [], hasNextPage: false };
2061
2218
  }
@@ -2089,6 +2246,10 @@ var InstagramExtractor = class {
2089
2246
  "Referer": "https://www.instagram.com/"
2090
2247
  }
2091
2248
  });
2249
+ if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
2250
+ const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
2251
+ throw new Error(`Instagram GraphQL query error: ${errMsg}`);
2252
+ }
2092
2253
  const reelsData = res?.data?.fetch__XDTUserDict;
2093
2254
  let connection = reelsData?.reels;
2094
2255
  if (!connection && reelsData) {
@@ -2119,9 +2280,31 @@ var InstagramExtractor = class {
2119
2280
  found: 0,
2120
2281
  message: `Fetching Instagram profile @${username}...`
2121
2282
  });
2122
- const user = await this.fetchUserProfile(username, options);
2123
- const userId = user.id;
2283
+ let user = null;
2284
+ try {
2285
+ user = await this.fetchUserProfile(username, options);
2286
+ } catch (err) {
2287
+ logExtractorDebug("fetchUserProfile", err);
2288
+ }
2289
+ let webTokens = {};
2290
+ try {
2291
+ webTokens = await this.fetchWebTokens(username, options);
2292
+ } catch (err) {
2293
+ logExtractorDebug("fetchWebTokens", err);
2294
+ }
2295
+ let userId = user?.id || webTokens.userId;
2296
+ if (!userId && (subDir === "Stories" || subDir === "Highlights")) {
2297
+ try {
2298
+ const probe = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
2299
+ if (probe.userId) {
2300
+ userId = probe.userId;
2301
+ }
2302
+ } catch (err) {
2303
+ logExtractorDebug("userId probe", err);
2304
+ }
2305
+ }
2124
2306
  if (subDir === "Stories") {
2307
+ if (!userId) return;
2125
2308
  options.onProgress?.({
2126
2309
  stage: "Stories",
2127
2310
  found: 0,
@@ -2137,6 +2320,7 @@ var InstagramExtractor = class {
2137
2320
  return;
2138
2321
  }
2139
2322
  if (subDir === "Highlights") {
2323
+ if (!userId) return;
2140
2324
  options.onProgress?.({
2141
2325
  stage: "Highlights",
2142
2326
  found: 0,
@@ -2163,9 +2347,22 @@ var InstagramExtractor = class {
2163
2347
  }
2164
2348
  let found = 0;
2165
2349
  const emittedItemIds = /* @__PURE__ */ new Set();
2166
- let currentEdges = user.edge_owner_to_timeline_media?.edges ?? [];
2167
- let hasNextPage = Boolean(user.edge_owner_to_timeline_media?.page_info?.has_next_page);
2168
- let endCursor = user.edge_owner_to_timeline_media?.page_info?.end_cursor;
2350
+ let currentEdges = user?.edge_owner_to_timeline_media?.edges ?? [];
2351
+ let hasNextPage = Boolean(user?.edge_owner_to_timeline_media?.page_info?.has_next_page);
2352
+ let endCursor = user?.edge_owner_to_timeline_media?.page_info?.end_cursor;
2353
+ if (!user) {
2354
+ try {
2355
+ const firstPage = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
2356
+ currentEdges = firstPage.edges;
2357
+ hasNextPage = firstPage.hasNextPage;
2358
+ endCursor = firstPage.nextCursor;
2359
+ if (!userId && firstPage.userId) {
2360
+ userId = firstPage.userId;
2361
+ }
2362
+ } catch (err) {
2363
+ logExtractorDebug("fetchPostsTabGraphQL initial", err);
2364
+ }
2365
+ }
2169
2366
  while (true) {
2170
2367
  for (const edge of currentEdges) {
2171
2368
  const items = parseIgNodeToMediaItems(edge.node, username);
@@ -2188,7 +2385,7 @@ var InstagramExtractor = class {
2188
2385
  }
2189
2386
  await sleep(1500, 2500, options.signal);
2190
2387
  try {
2191
- const nextPage = await this.fetchNextPostsPage(userId, username, endCursor, options);
2388
+ const nextPage = await this.fetchNextPostsPage(userId || "", username, endCursor, options, webTokens);
2192
2389
  currentEdges = nextPage.edges;
2193
2390
  hasNextPage = nextPage.hasNextPage;
2194
2391
  endCursor = nextPage.nextCursor;
@@ -2199,8 +2396,8 @@ var InstagramExtractor = class {
2199
2396
  break;
2200
2397
  }
2201
2398
  }
2202
- if (subDir === "Videos") {
2203
- const videoEdges = user.edge_felix_video_timeline?.edges ?? [];
2399
+ if (subDir === "Videos" && userId) {
2400
+ const videoEdges = user?.edge_felix_video_timeline?.edges ?? [];
2204
2401
  for (const edge of videoEdges) {
2205
2402
  const items = parseIgNodeToMediaItems(edge.node, username);
2206
2403
  for (const item of items) {
@@ -2272,8 +2469,43 @@ var TWITTER_FEATURES = {
2272
2469
  responsive_web_text_conversations_enabled: false,
2273
2470
  longform_notetweets_rich_text_read_enabled: true,
2274
2471
  longform_notetweets_inline_media_enabled: true,
2275
- responsive_web_enhance_cards_enabled: false
2472
+ responsive_web_enhance_cards_enabled: false,
2473
+ creator_subscriptions_tweet_preview_api_enabled: true,
2474
+ communities_web_enable_tweet_community_results_fetch: true,
2475
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
2476
+ articles_preview_enabled: true,
2477
+ responsive_web_twitter_article_tweet_consumption_enabled: true,
2478
+ responsive_web_grok_analyze_post_followups_enabled: true,
2479
+ responsive_web_jetfuel_frame: true,
2480
+ responsive_web_grok_share_attachment_enabled: true,
2481
+ responsive_web_grok_analysis_button_from_backend: true,
2482
+ responsive_web_grok_image_annotation_enabled: true,
2483
+ responsive_web_grok_imagine_annotation_enabled: true,
2484
+ profile_label_improvements_pcf_label_in_post_enabled: true,
2485
+ rweb_tipjar_consumption_enabled: true
2276
2486
  };
2487
+ function shouldSkipTweetReply(tweet, targetUserId, mode = "timeline") {
2488
+ const tweetResult = tweet.tweet || tweet;
2489
+ const legacy = tweetResult.legacy || tweetResult;
2490
+ const replyToUserId = legacy.in_reply_to_user_id_str;
2491
+ if (!replyToUserId) {
2492
+ return false;
2493
+ }
2494
+ const authorId = legacy.user_id_str || tweetResult.core?.user_results?.result?.rest_id;
2495
+ if (mode === "timeline") {
2496
+ if (targetUserId) {
2497
+ if (replyToUserId !== targetUserId || authorId && authorId !== targetUserId) {
2498
+ return true;
2499
+ }
2500
+ return false;
2501
+ }
2502
+ return Boolean(authorId && replyToUserId !== authorId);
2503
+ }
2504
+ if (authorId && replyToUserId === authorId) {
2505
+ return false;
2506
+ }
2507
+ return true;
2508
+ }
2277
2509
  function parseTweetMediaEntities(tweet, username) {
2278
2510
  const items = [];
2279
2511
  const tweetResult = tweet.tweet || tweet;
@@ -2368,11 +2600,17 @@ var XExtractor = class {
2368
2600
  }
2369
2601
  if (username === "i_history_likes") {
2370
2602
  if (options.cookiePath) {
2371
- const cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
2603
+ let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
2604
+ if (!cookies["twid"]) {
2605
+ cookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
2606
+ }
2607
+ if (!cookies["twid"]) {
2608
+ cookies = await loadCookiesFromPath(options.cookiePath);
2609
+ }
2372
2610
  const twid = cookies["twid"];
2373
2611
  if (twid) {
2374
2612
  const decoded = decodeURIComponent(twid).replace(/^"|"$/g, "").trim();
2375
- const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded;
2613
+ const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded.includes("=") ? decoded.split("=").pop()?.trim() : decoded;
2376
2614
  if (rawId && /^[a-zA-Z0-9_-]+$/.test(rawId)) {
2377
2615
  return rawId;
2378
2616
  }
@@ -2414,7 +2652,21 @@ var XExtractor = class {
2414
2652
  async *extract(profile, options) {
2415
2653
  const username = profile.username;
2416
2654
  const subDir = options.subDir;
2417
- const isLikes = username === "i_history_likes";
2655
+ const isLikes = username === "i_history_likes" || profile.cleanUrl?.endsWith("/likes");
2656
+ const authHeaders = {};
2657
+ if (options.cookiePath) {
2658
+ let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
2659
+ if (!cookies["ct0"] && !cookies["auth_token"]) {
2660
+ const twCookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
2661
+ cookies = { ...twCookies, ...cookies };
2662
+ }
2663
+ if (cookies["ct0"]) {
2664
+ authHeaders["x-csrf-token"] = cookies["ct0"];
2665
+ }
2666
+ if (cookies["auth_token"]) {
2667
+ authHeaders["x-twitter-auth-type"] = "OAuth2Session";
2668
+ }
2669
+ }
2418
2670
  options.onProgress?.({
2419
2671
  stage: isLikes ? "Likes" : "Profile",
2420
2672
  found: 0,
@@ -2445,11 +2697,15 @@ var XExtractor = class {
2445
2697
  res = await this.httpClient.fetchJson(url, {
2446
2698
  cookiePath: options.cookiePath,
2447
2699
  cookieDomain: "x.com",
2448
- signal: options.signal
2700
+ signal: options.signal,
2701
+ headers: authHeaders
2449
2702
  });
2450
2703
  } catch (err) {
2451
2704
  break;
2452
2705
  }
2706
+ if (res?.errors?.length && !res?.data?.user?.result) {
2707
+ throw new Error(`X/Twitter API error: ${res.errors[0]?.message || "GraphQL query failed"}`);
2708
+ }
2453
2709
  const instructions = res?.data?.user?.result?.timeline_v2?.timeline?.instructions || [];
2454
2710
  let newCursor;
2455
2711
  let pageItemCount = 0;
@@ -2462,6 +2718,9 @@ var XExtractor = class {
2462
2718
  }
2463
2719
  const tweetContent = entry.content?.itemContent?.tweet_results?.result;
2464
2720
  if (tweetContent) {
2721
+ if (shouldSkipTweetReply(tweetContent, userId, isLikes ? "likes" : "timeline")) {
2722
+ continue;
2723
+ }
2465
2724
  const mediaItems = parseTweetMediaEntities(tweetContent, username);
2466
2725
  for (const item of mediaItems) {
2467
2726
  if (subDir === "Photos" && item.type !== "photo") continue;
@@ -2490,16 +2749,86 @@ var XExtractor = class {
2490
2749
 
2491
2750
  // src/lib/engines/extractors/tiktok.ts
2492
2751
  function parseTikTokRehydrationJson(html) {
2493
- const match = html.match(
2752
+ const universalMatch = html.match(
2494
2753
  /<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
2495
2754
  );
2496
- if (!match) return null;
2497
- try {
2498
- const raw = JSON.parse(match[1]);
2499
- return raw?.["__DEFAULT_SCOPE__"];
2500
- } catch {
2501
- return null;
2755
+ if (universalMatch) {
2756
+ try {
2757
+ const raw = JSON.parse(universalMatch[1]);
2758
+ const scope = raw?.["__DEFAULT_SCOPE__"] || raw;
2759
+ if (scope) return scope;
2760
+ } catch {
2761
+ }
2502
2762
  }
2763
+ const sigiTagMatch = html.match(
2764
+ /<script id="SIGI_STATE"[^>]*>([\s\S]*?)<\/script>/
2765
+ );
2766
+ if (sigiTagMatch) {
2767
+ try {
2768
+ const raw = JSON.parse(sigiTagMatch[1]);
2769
+ const user = Object.values(raw?.UserModule?.users || {})[0];
2770
+ const items = Object.values(raw?.ItemModule || {});
2771
+ return {
2772
+ "webapp.user-detail": {
2773
+ userInfo: { user },
2774
+ itemList: items
2775
+ },
2776
+ SIGI_STATE: raw
2777
+ };
2778
+ } catch {
2779
+ }
2780
+ }
2781
+ const sigiWindowMatch = html.match(
2782
+ /(?:window\[['"]SIGI_STATE['"]\]|SIGI_STATE)\s*=\s*(\{[\s\S]*?\})\s*;/i
2783
+ );
2784
+ if (sigiWindowMatch) {
2785
+ try {
2786
+ const raw = JSON.parse(sigiWindowMatch[1]);
2787
+ const user = Object.values(raw?.UserModule?.users || {})[0];
2788
+ const items = Object.values(raw?.ItemModule || {});
2789
+ return {
2790
+ "webapp.user-detail": {
2791
+ userInfo: { user },
2792
+ itemList: items
2793
+ },
2794
+ SIGI_STATE: raw
2795
+ };
2796
+ } catch {
2797
+ }
2798
+ }
2799
+ const nextDataMatch = html.match(
2800
+ /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/
2801
+ );
2802
+ if (nextDataMatch) {
2803
+ try {
2804
+ const raw = JSON.parse(nextDataMatch[1]);
2805
+ const pageProps = raw?.props?.pageProps;
2806
+ if (pageProps) {
2807
+ return {
2808
+ "webapp.user-detail": {
2809
+ userInfo: pageProps.userInfo,
2810
+ itemList: pageProps.items || pageProps.itemList || []
2811
+ },
2812
+ __NEXT_DATA__: raw
2813
+ };
2814
+ }
2815
+ } catch {
2816
+ }
2817
+ }
2818
+ const secUidMatch = html.match(/(?:secUid|sec_uid)["':=\s]+(MS4wLjABAAAA[\w-]{32,64})/i);
2819
+ if (secUidMatch) {
2820
+ return {
2821
+ "webapp.user-detail": {
2822
+ userInfo: {
2823
+ user: {
2824
+ secUid: secUidMatch[1]
2825
+ }
2826
+ },
2827
+ itemList: []
2828
+ }
2829
+ };
2830
+ }
2831
+ return null;
2503
2832
  }
2504
2833
  function parseTikTokItem(item, username) {
2505
2834
  const items = [];
@@ -2596,13 +2925,28 @@ var TikTokExtractor = class {
2596
2925
  if (options.signal?.aborted) break;
2597
2926
  if (visitedCursors.has(cursor)) break;
2598
2927
  visitedCursors.add(cursor);
2599
- const apiUrl = `https://www.tiktok.com/api/post/item_list/?secUid=${encodeURIComponent(secUid)}&count=30&cursor=${cursor}&post_item_list_request_type=0`;
2928
+ const deviceId = `${Math.floor(725e16 + Math.random() * 75e15)}`;
2929
+ const queryParams = new URLSearchParams({
2930
+ aid: "1988",
2931
+ app_name: "tiktok_web",
2932
+ device_platform: "web_pc",
2933
+ channel: "tiktok_web",
2934
+ device_id: deviceId,
2935
+ secUid,
2936
+ count: "30",
2937
+ cursor,
2938
+ post_item_list_request_type: "0"
2939
+ });
2940
+ const apiUrl = `https://www.tiktok.com/api/post/item_list/?${queryParams.toString()}`;
2600
2941
  let res;
2601
2942
  try {
2602
2943
  res = await this.httpClient.fetchJson(apiUrl, {
2603
2944
  cookiePath: options.cookiePath,
2604
2945
  cookieDomain: "tiktok.com",
2605
- signal: options.signal
2946
+ signal: options.signal,
2947
+ headers: {
2948
+ "Referer": profileUrl
2949
+ }
2606
2950
  });
2607
2951
  } catch {
2608
2952
  break;
@@ -2654,28 +2998,55 @@ function decodeEscapedUrl(raw) {
2654
2998
  function extractFacebookMediaFromHtml(html, username) {
2655
2999
  const items = [];
2656
3000
  const seenUrls = /* @__PURE__ */ new Set();
2657
- const hdMatch = html.match(/"browser_native_hd_url":"([^"]+)"/);
2658
- const sdMatch = html.match(/"browser_native_sd_url":"([^"]+)"/);
2659
- const rawVideoUrl = hdMatch?.[1] || sdMatch?.[1];
2660
- if (rawVideoUrl) {
2661
- const videoUrl = decodeEscapedUrl(rawVideoUrl);
2662
- if (!seenUrls.has(videoUrl)) {
2663
- seenUrls.add(videoUrl);
2664
- items.push({
2665
- id: `fb_video_${Date.now()}`,
2666
- url: videoUrl,
2667
- filename: `${username}_video_${Date.now()}`,
2668
- extension: "mp4",
2669
- type: "video"
2670
- });
3001
+ const videoPatterns = [
3002
+ /"browser_native_hd_url":"([^"]+)"/g,
3003
+ /"browser_native_sd_url":"([^"]+)"/g,
3004
+ /"playable_url_quality_hd":"([^"]+)"/g,
3005
+ /"playable_url":"([^"]+)"/g,
3006
+ /<video[^>]+src=["'](https?:[^"']+)["']/gi
3007
+ ];
3008
+ let videoIndex = 1;
3009
+ for (const regex of videoPatterns) {
3010
+ let vMatch;
3011
+ while ((vMatch = regex.exec(html)) !== null) {
3012
+ const rawUrl = vMatch[1];
3013
+ const videoUrl = decodeEscapedUrl(rawUrl).replace(/&amp;/g, "&");
3014
+ if (videoUrl.startsWith("http") && !seenUrls.has(videoUrl)) {
3015
+ seenUrls.add(videoUrl);
3016
+ items.push({
3017
+ id: `fb_video_${videoIndex}`,
3018
+ url: videoUrl,
3019
+ filename: `${username}_video_${videoIndex}`,
3020
+ extension: "mp4",
3021
+ type: "video"
3022
+ });
3023
+ videoIndex++;
3024
+ }
2671
3025
  }
2672
3026
  }
2673
- const photoRegex = /,"image":\{"uri":"([^"]+)"/g;
3027
+ const photoRegex = /,"(?:image|viewer_image|full_size_image|display_image)":\{"uri":"([^"]+)"/g;
2674
3028
  let match;
2675
3029
  let photoIndex = 1;
2676
3030
  while ((match = photoRegex.exec(html)) !== null) {
2677
3031
  const rawUrl = match[1];
2678
- const photoUrl = decodeEscapedUrl(rawUrl);
3032
+ const photoUrl = decodeEscapedUrl(rawUrl).replace(/&amp;/g, "&");
3033
+ if (!seenUrls.has(photoUrl)) {
3034
+ seenUrls.add(photoUrl);
3035
+ items.push({
3036
+ id: `fb_photo_${photoIndex}`,
3037
+ url: photoUrl,
3038
+ filename: `${username}_photo_${photoIndex}`,
3039
+ extension: "jpg",
3040
+ type: "photo"
3041
+ });
3042
+ photoIndex++;
3043
+ }
3044
+ }
3045
+ const mbasicImgRegex = /<img[^>]+src=["'](https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*)["']/gi;
3046
+ let mMatch;
3047
+ while ((mMatch = mbasicImgRegex.exec(html)) !== null) {
3048
+ const rawUrl = mMatch[1];
3049
+ const photoUrl = decodeEscapedUrl(rawUrl).replace(/&amp;/g, "&");
2679
3050
  if (!seenUrls.has(photoUrl)) {
2680
3051
  seenUrls.add(photoUrl);
2681
3052
  items.push({
@@ -2689,10 +3060,10 @@ function extractFacebookMediaFromHtml(html, username) {
2689
3060
  }
2690
3061
  }
2691
3062
  if (items.length === 0) {
2692
- const scontentRegex = /https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*/gi;
3063
+ const scontentRegex = /https:\/\/[^"'\s<>]*scontent[^"'\s<>]*\.(?:jpg|png|webp)[^"'\s<>]*/gi;
2693
3064
  let sMatch;
2694
3065
  while ((sMatch = scontentRegex.exec(html)) !== null) {
2695
- const cleanUrl = sMatch[0].replace(/\\/g, "");
3066
+ const cleanUrl = sMatch[0].replace(/\\/g, "").replace(/&amp;/g, "&");
2696
3067
  if (!seenUrls.has(cleanUrl)) {
2697
3068
  seenUrls.add(cleanUrl);
2698
3069
  items.push({
@@ -2708,6 +3079,59 @@ function extractFacebookMediaFromHtml(html, username) {
2708
3079
  }
2709
3080
  return items;
2710
3081
  }
3082
+ function extractFacebookPaginationCursor(html) {
3083
+ const jsonCursorMatch = html.match(/"end_cursor":"([^"]+)"/) || html.match(/"cursor":"([^"]+)"/);
3084
+ if (jsonCursorMatch && jsonCursorMatch[1]) {
3085
+ return jsonCursorMatch[1];
3086
+ }
3087
+ const linkCursorMatch = html.match(/[?&](?:cursor|after)=([^"'\s&<>]+)/i);
3088
+ if (linkCursorMatch && linkCursorMatch[1]) {
3089
+ try {
3090
+ return decodeURIComponent(linkCursorMatch[1]);
3091
+ } catch {
3092
+ return linkCursorMatch[1];
3093
+ }
3094
+ }
3095
+ return null;
3096
+ }
3097
+ function resolveFacebookUrl(href, currentUrl) {
3098
+ const cleanHref = href.replace(/\\/g, "").replace(/&amp;/g, "&");
3099
+ try {
3100
+ return new URL(cleanHref, currentUrl).toString();
3101
+ } catch {
3102
+ return cleanHref;
3103
+ }
3104
+ }
3105
+ function extractFacebookNextPageUrl(html, currentUrl) {
3106
+ const moreItemMatch = html.match(/<div[^>]*id=["'](?:m_more_item|m_more_posts)["'][^>]*>\s*<a[^>]+href=["']([^"']+)["']/i);
3107
+ if (moreItemMatch && moreItemMatch[1]) {
3108
+ return resolveFacebookUrl(moreItemMatch[1], currentUrl);
3109
+ }
3110
+ const cursorLabelMatch = html.match(/<a[^>]+href=["']([^"']*(?:cursor=|after=)[^"']+)["'][^>]*>[\s\S]*?(?:See [Mm]ore|More [Pp]hotos|More|Next|Older [Pp]osts)[\s\S]*?<\/a>/i);
3111
+ if (cursorLabelMatch && cursorLabelMatch[1]) {
3112
+ return resolveFacebookUrl(cursorLabelMatch[1], currentUrl);
3113
+ }
3114
+ const genericCursorMatch = html.match(/<a[^>]+href=["']([^"']*(?:[?&]|&amp;)(?:cursor|after)=[^"']+)["']/i);
3115
+ if (genericCursorMatch && genericCursorMatch[1]) {
3116
+ return resolveFacebookUrl(genericCursorMatch[1], currentUrl);
3117
+ }
3118
+ const cursor = extractFacebookPaginationCursor(html);
3119
+ if (cursor) {
3120
+ try {
3121
+ const u = new URL(currentUrl);
3122
+ if (u.searchParams.get("cursor") !== cursor) {
3123
+ u.searchParams.set("cursor", cursor);
3124
+ return u.toString();
3125
+ }
3126
+ } catch {
3127
+ if (!currentUrl.includes(cursor)) {
3128
+ const sep = currentUrl.includes("?") ? "&" : "?";
3129
+ return `${currentUrl}${sep}cursor=${encodeURIComponent(cursor)}`;
3130
+ }
3131
+ }
3132
+ }
3133
+ return null;
3134
+ }
2711
3135
  var FacebookExtractor = class {
2712
3136
  platform = "facebook";
2713
3137
  httpClient;
@@ -2724,26 +3148,120 @@ var FacebookExtractor = class {
2724
3148
  found: 0,
2725
3149
  message: `Fetching Facebook profile @${username}...`
2726
3150
  });
2727
- const targetUrl = subDir === "Photos" ? `https://www.facebook.com/${encodeURIComponent(username)}/photos` : `https://www.facebook.com/${encodeURIComponent(username)}/videos`;
2728
- const html = await this.httpClient.fetchText(targetUrl, {
2729
- cookiePath: options.cookiePath,
2730
- cookieDomain: "facebook.com",
2731
- signal: options.signal,
2732
- retries: 0,
2733
- timeoutMs: 4e3
2734
- });
2735
- const items = extractFacebookMediaFromHtml(html, username);
2736
- let found = 0;
2737
- for (const item of items) {
2738
- if (subDir === "Photos" && item.type !== "photo") continue;
2739
- if (subDir === "Videos" && item.type !== "video") continue;
2740
- found++;
3151
+ const targetPath = subDir === "Photos" ? "photos" : "videos";
3152
+ const desktopUrl = `https://www.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
3153
+ const mbasicUrl = `https://mbasic.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
3154
+ const mobileUrl = `https://m.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
3155
+ let currentUrl = desktopUrl;
3156
+ let html = "";
3157
+ try {
3158
+ html = await this.httpClient.fetchText(desktopUrl, {
3159
+ cookiePath: options.cookiePath,
3160
+ cookieDomain: "facebook.com",
3161
+ signal: options.signal,
3162
+ retries: 0,
3163
+ timeoutMs: 4e3
3164
+ });
3165
+ } catch {
3166
+ html = "";
3167
+ }
3168
+ let initialItems = extractFacebookMediaFromHtml(html, username);
3169
+ if (initialItems.length === 0) {
2741
3170
  options.onProgress?.({
2742
- stage: subDir,
2743
- found,
2744
- message: `Found item: ${item.filename}`
3171
+ stage: "Profile",
3172
+ found: 0,
3173
+ message: `Desktop yielded 0 items. Falling back to mbasic.facebook.com for @${username}...`
2745
3174
  });
2746
- yield item;
3175
+ try {
3176
+ const mbasicHtml = await this.httpClient.fetchText(mbasicUrl, {
3177
+ cookiePath: options.cookiePath,
3178
+ cookieDomain: "facebook.com",
3179
+ signal: options.signal,
3180
+ retries: 1,
3181
+ timeoutMs: 5e3,
3182
+ headers: {
3183
+ "Referer": "https://mbasic.facebook.com/"
3184
+ }
3185
+ });
3186
+ const mbasicItems = extractFacebookMediaFromHtml(mbasicHtml, username);
3187
+ if (mbasicItems.length > 0 || mbasicHtml.length > 200) {
3188
+ html = mbasicHtml;
3189
+ currentUrl = mbasicUrl;
3190
+ initialItems = mbasicItems;
3191
+ }
3192
+ } catch {
3193
+ }
3194
+ if (initialItems.length === 0) {
3195
+ try {
3196
+ const mobileHtml = await this.httpClient.fetchText(mobileUrl, {
3197
+ cookiePath: options.cookiePath,
3198
+ cookieDomain: "facebook.com",
3199
+ signal: options.signal,
3200
+ retries: 0,
3201
+ timeoutMs: 5e3,
3202
+ headers: {
3203
+ "User-Agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
3204
+ "Referer": "https://m.facebook.com/"
3205
+ }
3206
+ });
3207
+ const mobileItems = extractFacebookMediaFromHtml(mobileHtml, username);
3208
+ if (mobileItems.length > 0) {
3209
+ html = mobileHtml;
3210
+ currentUrl = mobileUrl;
3211
+ initialItems = mobileItems;
3212
+ }
3213
+ } catch {
3214
+ }
3215
+ }
3216
+ }
3217
+ const seenUrls = /* @__PURE__ */ new Set();
3218
+ let found = 0;
3219
+ let page = 1;
3220
+ const maxPages = 5;
3221
+ while (true) {
3222
+ const items = extractFacebookMediaFromHtml(html, username);
3223
+ let pageNewCount = 0;
3224
+ for (const item of items) {
3225
+ if (subDir === "Photos" && item.type !== "photo") continue;
3226
+ if (subDir === "Videos" && item.type !== "video") continue;
3227
+ if (seenUrls.has(item.url)) continue;
3228
+ seenUrls.add(item.url);
3229
+ found++;
3230
+ pageNewCount++;
3231
+ options.onProgress?.({
3232
+ stage: subDir,
3233
+ found,
3234
+ message: `Found item: ${item.filename}`
3235
+ });
3236
+ yield item;
3237
+ }
3238
+ if (page >= maxPages || options.signal?.aborted) {
3239
+ break;
3240
+ }
3241
+ const nextUrl = extractFacebookNextPageUrl(html, currentUrl);
3242
+ if (!nextUrl || nextUrl === currentUrl) {
3243
+ break;
3244
+ }
3245
+ if (pageNewCount === 0 && page > 1) {
3246
+ break;
3247
+ }
3248
+ await sleep(1e3, 2e3, options.signal);
3249
+ try {
3250
+ currentUrl = nextUrl;
3251
+ html = await this.httpClient.fetchText(currentUrl, {
3252
+ cookiePath: options.cookiePath,
3253
+ cookieDomain: "facebook.com",
3254
+ signal: options.signal,
3255
+ retries: 1,
3256
+ timeoutMs: 5e3,
3257
+ headers: {
3258
+ "Referer": currentUrl.includes("mbasic") ? "https://mbasic.facebook.com/" : "https://www.facebook.com/"
3259
+ }
3260
+ });
3261
+ page++;
3262
+ } catch {
3263
+ break;
3264
+ }
2747
3265
  }
2748
3266
  }
2749
3267
  };
@@ -2771,17 +3289,48 @@ import path5 from "path";
2771
3289
  import { Readable } from "stream";
2772
3290
  import { pipeline } from "stream/promises";
2773
3291
  var ARCHIVE_FILENAME = ".open-nami-archive.json";
3292
+ var YTDLP_ARCHIVE_FILENAME = "archive_yt-dlp.txt";
3293
+ var TEXT_ARCHIVE_FILENAME = "archive.txt";
2774
3294
  async function loadArchive(destDir) {
2775
- const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
3295
+ const archive = /* @__PURE__ */ new Set();
3296
+ const jsonPath = path5.join(destDir, ARCHIVE_FILENAME);
2776
3297
  try {
2777
- const raw = await fsPromises2.readFile(archivePath, "utf-8");
3298
+ const raw = await fsPromises2.readFile(jsonPath, "utf-8");
2778
3299
  const parsed = JSON.parse(raw);
2779
3300
  if (Array.isArray(parsed)) {
2780
- return new Set(parsed);
3301
+ for (const id of parsed) {
3302
+ if (id) archive.add(String(id));
3303
+ }
3304
+ }
3305
+ } catch {
3306
+ }
3307
+ const ytdlpPath = path5.join(destDir, YTDLP_ARCHIVE_FILENAME);
3308
+ try {
3309
+ const raw = await fsPromises2.readFile(ytdlpPath, "utf-8");
3310
+ const lines = raw.split(/\r?\n/);
3311
+ for (const rawLine of lines) {
3312
+ const line = rawLine.trim();
3313
+ if (!line || line.startsWith("#")) continue;
3314
+ const parts = line.split(/\s+/);
3315
+ const id = parts[parts.length - 1];
3316
+ if (id) archive.add(id);
3317
+ }
3318
+ } catch {
3319
+ }
3320
+ const textPath = path5.join(destDir, TEXT_ARCHIVE_FILENAME);
3321
+ try {
3322
+ const raw = await fsPromises2.readFile(textPath, "utf-8");
3323
+ const lines = raw.split(/\r?\n/);
3324
+ for (const rawLine of lines) {
3325
+ const line = rawLine.trim();
3326
+ if (!line || line.startsWith("#")) continue;
3327
+ const parts = line.split(/\s+/);
3328
+ const id = parts[parts.length - 1];
3329
+ if (id) archive.add(id);
2781
3330
  }
2782
3331
  } catch {
2783
3332
  }
2784
- return /* @__PURE__ */ new Set();
3333
+ return archive;
2785
3334
  }
2786
3335
  async function saveArchive(destDir, archive) {
2787
3336
  const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
@@ -2791,6 +3340,18 @@ async function saveArchive(destDir, archive) {
2791
3340
  await fsPromises2.rename(tmpPath, archivePath);
2792
3341
  } catch {
2793
3342
  }
3343
+ const textPaths = [
3344
+ path5.join(destDir, YTDLP_ARCHIVE_FILENAME),
3345
+ path5.join(destDir, TEXT_ARCHIVE_FILENAME)
3346
+ ];
3347
+ const content = [...archive].map((id) => `${id}
3348
+ `).join("");
3349
+ for (const tPath of textPaths) {
3350
+ try {
3351
+ await fsPromises2.writeFile(tPath, content, "utf-8");
3352
+ } catch {
3353
+ }
3354
+ }
2794
3355
  }
2795
3356
  function sanitizeFilename(rawFilename, fallbackId, ext) {
2796
3357
  const cleanExt = ext.replace(/^\./, "");
@@ -2804,6 +3365,14 @@ function sanitizeFilename(rawFilename, fallbackId, ext) {
2804
3365
  }
2805
3366
  return sanitizedBase.includes(".") ? sanitizedBase : `${sanitizedBase}.${cleanExt}`;
2806
3367
  }
3368
+ function isCdnUrl(rawUrl) {
3369
+ try {
3370
+ const hostname = new URL(rawUrl).hostname.toLowerCase();
3371
+ return hostname.endsWith(".cdninstagram.com") || hostname.endsWith(".fbcdn.net") || hostname.endsWith(".tiktokcdn.com") || hostname.endsWith(".akamaized.net") || hostname === "pbs.twimg.com" || hostname === "video.twimg.com";
3372
+ } catch {
3373
+ return false;
3374
+ }
3375
+ }
2807
3376
  async function downloadMediaStream(items, options) {
2808
3377
  const { jobName, destDir, signal, onProgress } = options;
2809
3378
  const httpClient = options.httpClient ?? new HttpClient();
@@ -2843,12 +3412,13 @@ async function downloadMediaStream(items, options) {
2843
3412
  consecutiveErrors++;
2844
3413
  const errMsg = err instanceof Error ? err.message : String(err);
2845
3414
  errors.push(`Extraction stream error: ${errMsg}`);
3415
+ const cleanErr = errMsg.includes("429") ? "rate limited by platform (HTTP 429)" : errMsg.length > 50 ? `${errMsg.slice(0, 47)}\u2026` : errMsg;
2846
3416
  onProgress?.({
2847
3417
  job: jobName,
2848
3418
  downloadedCount: downloaded,
2849
3419
  skippedCount: skipped,
2850
3420
  currentFile: "",
2851
- statusText: `Extractor glitch: ${errMsg}`
3421
+ statusText: `Extractor error: ${cleanErr}`
2852
3422
  });
2853
3423
  if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS || signal?.aborted) {
2854
3424
  isDone = true;
@@ -2883,10 +3453,13 @@ async function downloadMediaStream(items, options) {
2883
3453
  if (options.referer) {
2884
3454
  reqHeaders["Referer"] = options.referer;
2885
3455
  }
3456
+ const isCdn = isCdnUrl(item.url);
2886
3457
  const response = await httpClient.request(item.url, {
2887
3458
  signal,
2888
- cookiePath: options.cookiePath,
2889
- headers: reqHeaders
3459
+ cookiePath: isCdn ? void 0 : options.cookiePath,
3460
+ headers: reqHeaders,
3461
+ retries: 2,
3462
+ retryDelayMs: 1500
2890
3463
  });
2891
3464
  if (!response.ok || !response.body) {
2892
3465
  try {
@@ -2907,6 +3480,7 @@ async function downloadMediaStream(items, options) {
2907
3480
  await fsPromises2.rename(tmpPath, finalPath);
2908
3481
  archive.add(item.id);
2909
3482
  downloaded++;
3483
+ consecutiveErrors = 0;
2910
3484
  onProgress?.({
2911
3485
  job: jobName,
2912
3486
  downloadedCount: downloaded,
@@ -3318,8 +3892,9 @@ async function runGalleryDl(options) {
3318
3892
  let downloadedCount = 0;
3319
3893
  let skippedCount = 0;
3320
3894
  let errorOutput = "";
3895
+ let engineCmdStr = "gallery-dl";
3321
3896
  try {
3322
- await ensureGalleryDl((status) => {
3897
+ engineCmdStr = await ensureGalleryDl((status) => {
3323
3898
  onProgress?.({
3324
3899
  job: subDir,
3325
3900
  downloadedCount: 0,
@@ -3335,9 +3910,15 @@ async function runGalleryDl(options) {
3335
3910
  error: err.message || String(err)
3336
3911
  };
3337
3912
  }
3338
- const engineCmd = resolveEngineCommand("gallery-dl");
3913
+ const engineCmd = engineCmdStr.includes(path8.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("gallery-dl");
3339
3914
  const fullArgs = [...engineCmd.baseArgs, ...args];
3340
3915
  return new Promise((resolve) => {
3916
+ onProgress?.({
3917
+ job: subDir,
3918
+ downloadedCount,
3919
+ skippedCount,
3920
+ statusText: "downloading with gallery-dl\u2026"
3921
+ });
3341
3922
  const child = spawn2(engineCmd.command, fullArgs, {
3342
3923
  windowsHide: true,
3343
3924
  stdio: ["ignore", "pipe", "pipe"],
@@ -3461,8 +4042,9 @@ async function runYtDlp(options) {
3461
4042
  let downloadedCount = 0;
3462
4043
  let skippedCount = 0;
3463
4044
  let errorOutput = "";
4045
+ let engineCmdStr = "yt-dlp";
3464
4046
  try {
3465
- await ensureYtDlp((status) => {
4047
+ engineCmdStr = await ensureYtDlp((status) => {
3466
4048
  onProgress?.({
3467
4049
  job: "Videos",
3468
4050
  downloadedCount: 0,
@@ -3478,9 +4060,15 @@ async function runYtDlp(options) {
3478
4060
  error: err.message || String(err)
3479
4061
  };
3480
4062
  }
3481
- const engineCmd = resolveEngineCommand("yt-dlp");
4063
+ const engineCmd = engineCmdStr.includes(path9.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("yt-dlp");
3482
4064
  const fullArgs = [...engineCmd.baseArgs, ...args];
3483
4065
  return new Promise((resolve) => {
4066
+ onProgress?.({
4067
+ job: "Videos",
4068
+ downloadedCount,
4069
+ skippedCount,
4070
+ statusText: "downloading with yt-dlp\u2026"
4071
+ });
3484
4072
  const child = spawn3(engineCmd.command, fullArgs, {
3485
4073
  windowsHide: true,
3486
4074
  stdio: ["ignore", "pipe", "pipe"],
@@ -3617,7 +4205,7 @@ async function dispatchDownload(options) {
3617
4205
  jobs.push({ type: "highlights" });
3618
4206
  }
3619
4207
  }
3620
- const runNative = async (subDir) => {
4208
+ const defaultRunNative = async (subDir) => {
3621
4209
  const dest = path10.join(targetDir, subDir);
3622
4210
  try {
3623
4211
  const extractor = getExtractor(profile.platform);
@@ -3650,27 +4238,34 @@ async function dispatchDownload(options) {
3650
4238
  };
3651
4239
  }
3652
4240
  };
3653
- const runGdl = (subDir) => runGalleryDl({
4241
+ const runNative = options.runnerOverrides?.nativeRunner ?? defaultRunNative;
4242
+ const runGdl = options.runnerOverrides?.gdlRunner ?? ((subDir) => runGalleryDl({
3654
4243
  profile,
3655
4244
  targetDir,
3656
4245
  subDir,
3657
4246
  cookiePath: cookieCopy?.filePath,
3658
4247
  signal: options.signal,
3659
4248
  onProgress
3660
- });
3661
- const runYt = () => runYtDlp({
4249
+ }));
4250
+ const runYt = options.runnerOverrides?.ytRunner ?? (() => runYtDlp({
3662
4251
  profile,
3663
4252
  targetDir,
3664
4253
  cookiePath: cookieCopy?.filePath,
3665
4254
  signal: options.signal,
3666
4255
  onProgress
3667
- });
4256
+ }));
3668
4257
  for (const job of jobs) {
3669
4258
  if (options.signal?.aborted) break;
3670
4259
  if (job.type === "photos" || job.type === "stories" || job.type === "highlights") {
3671
4260
  const subDir = job.type === "photos" ? "Photos" : job.type === "stories" ? "Stories" : "Highlights";
3672
4261
  let res = await runNative(subDir);
3673
4262
  if ((!res.success || res.downloaded === 0 && res.skipped === 0) && !options.signal?.aborted) {
4263
+ onProgress?.({
4264
+ job: subDir,
4265
+ downloadedCount: totalDownloaded,
4266
+ skippedCount: totalSkipped,
4267
+ statusText: "switching to gallery-dl engine\u2026"
4268
+ });
3674
4269
  const gdlRes = await runGdl(subDir);
3675
4270
  if (gdlRes.downloaded > 0 || gdlRes.skipped > 0 || gdlRes.success) {
3676
4271
  res = gdlRes;
@@ -3685,12 +4280,24 @@ async function dispatchDownload(options) {
3685
4280
  totalDownloaded += res.downloaded;
3686
4281
  totalSkipped += res.skipped;
3687
4282
  } else if (!options.signal?.aborted) {
4283
+ onProgress?.({
4284
+ job: "Videos",
4285
+ downloadedCount: totalDownloaded,
4286
+ skippedCount: totalSkipped,
4287
+ statusText: "switching to yt-dlp engine\u2026"
4288
+ });
3688
4289
  const fbRes = await runYt();
3689
4290
  if (fbRes.downloaded > 0 || fbRes.skipped > 0 || fbRes.success) {
3690
4291
  totalDownloaded += fbRes.downloaded;
3691
4292
  totalSkipped += fbRes.skipped;
3692
4293
  if (!fbRes.success && fbRes.error) errors.push(`[Videos] ${fbRes.error}`);
3693
4294
  } else if (!options.signal?.aborted) {
4295
+ onProgress?.({
4296
+ job: "Videos",
4297
+ downloadedCount: totalDownloaded,
4298
+ skippedCount: totalSkipped,
4299
+ statusText: "switching to gallery-dl engine\u2026"
4300
+ });
3694
4301
  const gdlRes = await runGdl("Videos");
3695
4302
  totalDownloaded += gdlRes.downloaded;
3696
4303
  totalSkipped += gdlRes.skipped;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "open-nami",
3
- "version": "1.0.4",
4
- "description": "Open Nami — terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
3
+ "version": "1.0.6",
4
+ "description": "Open Nami: terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
5
5
  "type": "module",
6
6
  "bin": {
7
7
  "open-nami": "dist/cli.js",