open-nami 1.0.4 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/cli.js +731 -142
  2. package/package.json +1 -1
package/dist/cli.js CHANGED
@@ -9,7 +9,7 @@ var require_package = __commonJS({
9
9
  "package.json"(exports, module) {
10
10
  module.exports = {
11
11
  name: "open-nami",
12
- version: "1.0.4",
12
+ version: "1.0.5",
13
13
  description: "Open Nami \u2014 terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
14
14
  type: "module",
15
15
  bin: {
@@ -1407,10 +1407,11 @@ function parseProfileInput(input) {
1407
1407
  const reserved = ["home", "explore", "notifications", "messages", "i", "settings"];
1408
1408
  if (firstPart && !reserved.includes(firstPart.toLowerCase())) {
1409
1409
  const username = firstPart.replace(/^@/, "");
1410
+ const isLikesUrl = pathParts[1]?.toLowerCase() === "likes";
1410
1411
  return {
1411
1412
  platform: "x",
1412
1413
  username: sanitizeUsername(username),
1413
- cleanUrl: `https://x.com/${username}`,
1414
+ cleanUrl: isLikesUrl ? `https://x.com/${username}/likes` : `https://x.com/${username}`,
1414
1415
  rawInput: input
1415
1416
  };
1416
1417
  }
@@ -1575,6 +1576,9 @@ function resolvePlatformDownloadsDir(options = {}) {
1575
1576
  return defaultFallback;
1576
1577
  }
1577
1578
 
1579
+ // src/lib/engines/extractors/instagram.ts
1580
+ import crypto from "crypto";
1581
+
1578
1582
  // src/lib/engines/extractors/http.ts
1579
1583
  import fs5 from "fs/promises";
1580
1584
  import { setTimeout as setTimeoutPromise } from "timers/promises";
@@ -1762,7 +1766,9 @@ var HttpClient = class {
1762
1766
  }
1763
1767
  throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
1764
1768
  }
1765
- return await res.json();
1769
+ const text = await res.text();
1770
+ const cleanText = text.replace(/^\s*(?:for\s*\(\s*;\s*;\s*\);|\)\]\}',)/, "").trim();
1771
+ return JSON.parse(cleanText);
1766
1772
  }
1767
1773
  async fetchText(url, options = {}) {
1768
1774
  const res = await this.request(url, options);
@@ -1781,13 +1787,13 @@ var HttpClient = class {
1781
1787
  var IG_APP_ID = "936619743392459";
1782
1788
  var IG_TIMELINE_QUERY_HASH = "69cba40317214236af40e7efa697781d";
1783
1789
  var IG_QUERIES = {
1784
- PolarisProfilePostsTabContentQuery_connection: "28142289515441884",
1790
+ PolarisProfilePostsTabContentQuery_connection: "39229011733413759",
1785
1791
  PolarisProfileReelsTabContentQuery_connection: "28989741540630190",
1786
1792
  PolarisProfileTaggedTabContentQuery_connection: "28794602806842110",
1787
1793
  PolarisProfileStoryHighlightsTrayContentQuery: "26970053832668570",
1788
1794
  PolarisStoriesV3ReelPageGalleryQuery: "28262315486766731",
1789
1795
  PolarisStoriesV3TrayContainerQuery: "27703822975903310",
1790
- PolarisProfilePageContentQuery: "28036671149327607"
1796
+ PolarisProfilePageContentQuery: "28649589871350527"
1791
1797
  };
1792
1798
  var IG_POLARIS_DOC_ID = IG_QUERIES.PolarisProfilePostsTabContentQuery_connection;
1793
1799
  function logExtractorDebug(endpoint, err) {
@@ -1831,6 +1837,21 @@ function parseIgNodeToMediaItems(node, username) {
1831
1837
  height: video?.height ?? node.dimensions?.height,
1832
1838
  thumbnailUrl: node.display_url
1833
1839
  });
1840
+ } else {
1841
+ const candidate = node.image_versions2?.candidates?.[0];
1842
+ const photoUrl = candidate?.url || node.display_url;
1843
+ if (photoUrl) {
1844
+ items.push({
1845
+ id: baseId,
1846
+ url: photoUrl,
1847
+ filename: `${username}_${date || baseId}`,
1848
+ extension: "jpg",
1849
+ type: "photo",
1850
+ date,
1851
+ width: candidate?.width ?? node.dimensions?.width,
1852
+ height: candidate?.height ?? node.dimensions?.height
1853
+ });
1854
+ }
1834
1855
  }
1835
1856
  } else {
1836
1857
  const candidate = node.image_versions2?.candidates?.[0];
@@ -1910,7 +1931,15 @@ var InstagramExtractor = class {
1910
1931
  if (!res.data?.user) {
1911
1932
  throw new Error(`Instagram user not found or private profile: ${username}`);
1912
1933
  }
1913
- return res.data.user;
1934
+ const user = res.data.user;
1935
+ if (user.is_private) {
1936
+ const fs13 = user.friendship_status;
1937
+ const isFollowing = fs13 ? Boolean(fs13.following) : Boolean(user.followed_by_viewer);
1938
+ if (!isFollowing) {
1939
+ throw new Error(`Instagram profile @${username} is private. You must provide a session cookie jar of an account following this profile.`);
1940
+ }
1941
+ }
1942
+ return user;
1914
1943
  }
1915
1944
  async fetchHighlightsTray(userId, options) {
1916
1945
  try {
@@ -1932,6 +1961,10 @@ var InstagramExtractor = class {
1932
1961
  "Referer": "https://www.instagram.com/"
1933
1962
  }
1934
1963
  });
1964
+ if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
1965
+ const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
1966
+ throw new Error(`Instagram GraphQL query error: ${errMsg}`);
1967
+ }
1935
1968
  const edges = res?.data?.highlights?.edges;
1936
1969
  if (Array.isArray(edges) && edges.length > 0) {
1937
1970
  return edges.map((e) => e.node);
@@ -1975,87 +2008,207 @@ var InstagramExtractor = class {
1975
2008
  return [];
1976
2009
  }
1977
2010
  }
1978
- async fetchNextPostsPage(userId, username, cursor, options) {
1979
- const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
1980
- const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
2011
+ async fetchWebTokens(username, options) {
2012
+ let csrfToken;
2013
+ if (options.cookiePath) {
2014
+ try {
2015
+ const cookies = await loadCookiesFromPath(options.cookiePath, "instagram.com");
2016
+ csrfToken = cookies["csrftoken"];
2017
+ } catch {
2018
+ }
2019
+ }
2020
+ if (!csrfToken) {
2021
+ csrfToken = crypto.randomBytes(16).toString("hex");
2022
+ }
1981
2023
  try {
1982
- const res = await this.httpClient.fetchJson(url, {
2024
+ const html = await this.httpClient.fetchText(`https://www.instagram.com/${encodeURIComponent(username)}/`, {
1983
2025
  cookiePath: options.cookiePath,
1984
2026
  cookieDomain: "instagram.com",
1985
2027
  signal: options.signal,
1986
2028
  headers: {
1987
- "X-IG-App-ID": IG_APP_ID,
1988
- "Referer": `https://www.instagram.com/${username}/`
2029
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
1989
2030
  }
1990
2031
  });
1991
- const data = res?.data?.user?.edge_owner_to_timeline_media;
1992
- if (data?.edges && data.edges.length > 0) {
1993
- return {
1994
- edges: data.edges,
1995
- hasNextPage: Boolean(data.page_info?.has_next_page),
1996
- nextCursor: data.page_info?.end_cursor
1997
- };
2032
+ let lsd;
2033
+ let fbDtsg;
2034
+ let userId;
2035
+ const eqmcMatch = html.match(/id="__eqmc"[^>]*>([^<]+)<\/script>/);
2036
+ if (eqmcMatch) {
2037
+ try {
2038
+ const eqmc = JSON.parse(eqmcMatch[1]);
2039
+ if (eqmc.l) lsd = eqmc.l;
2040
+ if (eqmc.f) fbDtsg = eqmc.f;
2041
+ } catch {
2042
+ }
2043
+ }
2044
+ if (!lsd) {
2045
+ const m = html.match(/"lsd":"([^"]+)"/) || html.match(/"LSD",\[\],\{"token":"([^"]+)"/);
2046
+ if (m) lsd = m[1];
2047
+ }
2048
+ if (!fbDtsg) {
2049
+ const m = html.match(/"dtsg":\{"token":"([^"]+)"/);
2050
+ if (m) fbDtsg = m[1];
1998
2051
  }
2052
+ const uMatch = html.match(/"profile_id":"([^"]+)"/) || html.match(/"user_id":"([^"]+)"/);
2053
+ if (uMatch) userId = uMatch[1];
2054
+ return { lsd, fbDtsg, csrfToken, userId };
1999
2055
  } catch (err) {
2000
- logExtractorDebug("edge_owner_to_timeline_media", err);
2056
+ logExtractorDebug("fetchWebTokens", err);
2057
+ return { csrfToken };
2058
+ }
2059
+ }
2060
+ async fetchPostsTabGraphQL(username, cursor, tokens, options) {
2061
+ const postVariables = JSON.stringify({
2062
+ after: cursor || null,
2063
+ before: null,
2064
+ data: {
2065
+ count: 12,
2066
+ include_reel_media_seen_timestamp: true,
2067
+ include_relationship_info: true,
2068
+ latest_besties_reel_media: true,
2069
+ latest_reel_media: true
2070
+ },
2071
+ first: 12,
2072
+ include_multi_captions: true,
2073
+ last: null,
2074
+ username,
2075
+ __relay_internal__pv__PolarisMultiCaptionCarouselEnabledrelayprovider: true,
2076
+ __relay_internal__pv__PolarisShortDramaEnabledrelayprovider: true,
2077
+ __relay_internal__pv__PolarisThreadsStyleCommentsEnabledrelayprovider: false,
2078
+ __relay_internal__pv__PolarisReelsRecoDebugOverlayEnabledrelayprovider: false
2079
+ });
2080
+ const bodyParams = new URLSearchParams({
2081
+ av: "17841415137994167",
2082
+ __d: "www",
2083
+ __user: "0",
2084
+ __a: "1",
2085
+ __req: "0",
2086
+ __hs: "20731.HYP:instagram_web_pkg.2.1...0",
2087
+ dpr: "1",
2088
+ __ccg: "EXCELLENT",
2089
+ __rev: "1049285836",
2090
+ __hsi: "7693193318680449885",
2091
+ __comet_req: "7",
2092
+ jazoest: "26461",
2093
+ fb_api_caller_class: "RelayModern",
2094
+ fb_api_req_friendly_name: "PolarisProfilePostsTabContentQuery_connection",
2095
+ server_timestamps: "true",
2096
+ variables: postVariables,
2097
+ doc_id: IG_POLARIS_DOC_ID
2098
+ });
2099
+ if (tokens.fbDtsg) bodyParams.set("fb_dtsg", tokens.fbDtsg);
2100
+ if (tokens.lsd) bodyParams.set("lsd", tokens.lsd);
2101
+ const headers = {
2102
+ "Accept": "*/*",
2103
+ "Content-Type": "application/x-www-form-urlencoded",
2104
+ "X-IG-App-ID": IG_APP_ID,
2105
+ "X-IG-Max-Touch-Points": "0",
2106
+ "X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
2107
+ "Referer": `https://www.instagram.com/${username}/`,
2108
+ "Origin": "https://www.instagram.com",
2109
+ "X-ASBD-ID": "359341",
2110
+ "Sec-Fetch-Dest": "empty",
2111
+ "Sec-Fetch-Mode": "cors",
2112
+ "Sec-Fetch-Site": "same-origin"
2113
+ };
2114
+ if (tokens.csrfToken) headers["X-CSRFToken"] = tokens.csrfToken;
2115
+ if (tokens.lsd) headers["X-FB-LSD"] = tokens.lsd;
2116
+ const res = await this.httpClient.fetchJson("https://www.instagram.com/graphql/query", {
2117
+ method: "POST",
2118
+ body: bodyParams.toString(),
2119
+ cookiePath: options.cookiePath,
2120
+ cookieDomain: "instagram.com",
2121
+ signal: options.signal,
2122
+ headers
2123
+ });
2124
+ if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
2125
+ const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
2126
+ throw new Error(`Instagram GraphQL query error: ${errMsg}`);
2127
+ }
2128
+ const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
2129
+ if (connection?.edges && Array.isArray(connection.edges)) {
2130
+ let resolvedUserId;
2131
+ for (const edge of connection.edges) {
2132
+ const node = edge?.node;
2133
+ if (node?.owner_id) {
2134
+ resolvedUserId = String(node.owner_id);
2135
+ break;
2136
+ }
2137
+ if (node?.user?.pk || node?.user?.id) {
2138
+ resolvedUserId = String(node.user.pk || node.user.id);
2139
+ break;
2140
+ }
2141
+ if (typeof node?.id === "string" && node.id.includes("_")) {
2142
+ resolvedUserId = node.id.split("_")[1];
2143
+ break;
2144
+ }
2145
+ }
2146
+ return {
2147
+ edges: connection.edges,
2148
+ hasNextPage: Boolean(connection.page_info?.has_next_page),
2149
+ nextCursor: connection.page_info?.end_cursor,
2150
+ userId: resolvedUserId
2151
+ };
2001
2152
  }
2002
- try {
2003
- const postVariables = JSON.stringify({
2004
- after: cursor,
2005
- first: 12,
2006
- username,
2007
- data: { count: 12 }
2008
- });
2009
- const postUrl = "https://www.instagram.com/graphql/query";
2010
- const body = new URLSearchParams({
2011
- doc_id: IG_POLARIS_DOC_ID,
2012
- variables: postVariables
2013
- }).toString();
2014
- const res = await this.httpClient.fetchJson(postUrl, {
2015
- method: "POST",
2016
- body,
2017
- cookiePath: options.cookiePath,
2018
- cookieDomain: "instagram.com",
2019
- signal: options.signal,
2020
- headers: {
2021
- "Content-Type": "application/x-www-form-urlencoded",
2022
- "X-IG-App-ID": IG_APP_ID,
2023
- "X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
2024
- "Referer": `https://www.instagram.com/${username}/`
2153
+ return { edges: [], hasNextPage: false };
2154
+ }
2155
+ async fetchNextPostsPage(userId, username, cursor, options, tokens) {
2156
+ if (userId) {
2157
+ const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
2158
+ const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
2159
+ try {
2160
+ const res = await this.httpClient.fetchJson(url, {
2161
+ cookiePath: options.cookiePath,
2162
+ cookieDomain: "instagram.com",
2163
+ signal: options.signal,
2164
+ headers: {
2165
+ "X-IG-App-ID": IG_APP_ID,
2166
+ "Referer": `https://www.instagram.com/${username}/`
2167
+ }
2168
+ });
2169
+ const data = res?.data?.user?.edge_owner_to_timeline_media;
2170
+ if (data?.edges && data.edges.length > 0) {
2171
+ return {
2172
+ edges: data.edges,
2173
+ hasNextPage: Boolean(data.page_info?.has_next_page),
2174
+ nextCursor: data.page_info?.end_cursor
2175
+ };
2025
2176
  }
2026
- });
2027
- const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
2028
- if (connection?.edges && connection.edges.length > 0) {
2029
- return {
2030
- edges: connection.edges,
2031
- hasNextPage: Boolean(connection.page_info?.has_next_page),
2032
- nextCursor: connection.page_info?.end_cursor
2033
- };
2177
+ } catch (err) {
2178
+ logExtractorDebug("edge_owner_to_timeline_media", err);
2179
+ }
2180
+ }
2181
+ try {
2182
+ const page = await this.fetchPostsTabGraphQL(username, cursor, tokens || {}, options);
2183
+ if (page.edges && page.edges.length > 0) {
2184
+ return page;
2034
2185
  }
2035
2186
  } catch (err) {
2036
2187
  logExtractorDebug("PolarisProfilePostsTabContentQuery_connection", err);
2037
2188
  }
2038
- try {
2039
- const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
2040
- const res = await this.httpClient.fetchJson(restUrl, {
2041
- cookiePath: options.cookiePath,
2042
- cookieDomain: "instagram.com",
2043
- signal: options.signal,
2044
- headers: {
2045
- "X-IG-App-ID": IG_APP_ID,
2046
- "Referer": `https://www.instagram.com/${username}/`
2189
+ if (userId) {
2190
+ try {
2191
+ const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
2192
+ const res = await this.httpClient.fetchJson(restUrl, {
2193
+ cookiePath: options.cookiePath,
2194
+ cookieDomain: "instagram.com",
2195
+ signal: options.signal,
2196
+ headers: {
2197
+ "X-IG-App-ID": IG_APP_ID,
2198
+ "Referer": `https://www.instagram.com/${username}/`
2199
+ }
2200
+ });
2201
+ if (Array.isArray(res?.items) && res.items.length > 0) {
2202
+ const edges = res.items.map((item) => ({ node: item }));
2203
+ return {
2204
+ edges,
2205
+ hasNextPage: Boolean(res.more_available),
2206
+ nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
2207
+ };
2047
2208
  }
2048
- });
2049
- if (Array.isArray(res?.items) && res.items.length > 0) {
2050
- const edges = res.items.map((item) => ({ node: item }));
2051
- return {
2052
- edges,
2053
- hasNextPage: Boolean(res.more_available),
2054
- nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
2055
- };
2209
+ } catch (err) {
2210
+ logExtractorDebug("v1 user feed REST", err);
2056
2211
  }
2057
- } catch (err) {
2058
- logExtractorDebug("v1 user feed REST", err);
2059
2212
  }
2060
2213
  return { edges: [], hasNextPage: false };
2061
2214
  }
@@ -2089,6 +2242,10 @@ var InstagramExtractor = class {
2089
2242
  "Referer": "https://www.instagram.com/"
2090
2243
  }
2091
2244
  });
2245
+ if (res?.errors && Array.isArray(res.errors) && res.errors.length > 0) {
2246
+ const errMsg = res.errors.map((e) => e.message || JSON.stringify(e)).join("; ");
2247
+ throw new Error(`Instagram GraphQL query error: ${errMsg}`);
2248
+ }
2092
2249
  const reelsData = res?.data?.fetch__XDTUserDict;
2093
2250
  let connection = reelsData?.reels;
2094
2251
  if (!connection && reelsData) {
@@ -2119,9 +2276,29 @@ var InstagramExtractor = class {
2119
2276
  found: 0,
2120
2277
  message: `Fetching Instagram profile @${username}...`
2121
2278
  });
2122
- const user = await this.fetchUserProfile(username, options);
2123
- const userId = user.id;
2279
+ let user = null;
2280
+ try {
2281
+ user = await this.fetchUserProfile(username, options);
2282
+ } catch (err) {
2283
+ logExtractorDebug("fetchUserProfile", err);
2284
+ }
2285
+ let webTokens = {};
2286
+ if (!user) {
2287
+ webTokens = await this.fetchWebTokens(username, options);
2288
+ }
2289
+ let userId = user?.id || webTokens.userId;
2290
+ if (!userId && (subDir === "Stories" || subDir === "Highlights")) {
2291
+ try {
2292
+ const probe = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
2293
+ if (probe.userId) {
2294
+ userId = probe.userId;
2295
+ }
2296
+ } catch (err) {
2297
+ logExtractorDebug("userId probe", err);
2298
+ }
2299
+ }
2124
2300
  if (subDir === "Stories") {
2301
+ if (!userId) return;
2125
2302
  options.onProgress?.({
2126
2303
  stage: "Stories",
2127
2304
  found: 0,
@@ -2137,6 +2314,7 @@ var InstagramExtractor = class {
2137
2314
  return;
2138
2315
  }
2139
2316
  if (subDir === "Highlights") {
2317
+ if (!userId) return;
2140
2318
  options.onProgress?.({
2141
2319
  stage: "Highlights",
2142
2320
  found: 0,
@@ -2163,9 +2341,22 @@ var InstagramExtractor = class {
2163
2341
  }
2164
2342
  let found = 0;
2165
2343
  const emittedItemIds = /* @__PURE__ */ new Set();
2166
- let currentEdges = user.edge_owner_to_timeline_media?.edges ?? [];
2167
- let hasNextPage = Boolean(user.edge_owner_to_timeline_media?.page_info?.has_next_page);
2168
- let endCursor = user.edge_owner_to_timeline_media?.page_info?.end_cursor;
2344
+ let currentEdges = user?.edge_owner_to_timeline_media?.edges ?? [];
2345
+ let hasNextPage = Boolean(user?.edge_owner_to_timeline_media?.page_info?.has_next_page);
2346
+ let endCursor = user?.edge_owner_to_timeline_media?.page_info?.end_cursor;
2347
+ if (!user) {
2348
+ try {
2349
+ const firstPage = await this.fetchPostsTabGraphQL(username, void 0, webTokens, options);
2350
+ currentEdges = firstPage.edges;
2351
+ hasNextPage = firstPage.hasNextPage;
2352
+ endCursor = firstPage.nextCursor;
2353
+ if (!userId && firstPage.userId) {
2354
+ userId = firstPage.userId;
2355
+ }
2356
+ } catch (err) {
2357
+ logExtractorDebug("fetchPostsTabGraphQL initial", err);
2358
+ }
2359
+ }
2169
2360
  while (true) {
2170
2361
  for (const edge of currentEdges) {
2171
2362
  const items = parseIgNodeToMediaItems(edge.node, username);
@@ -2188,7 +2379,7 @@ var InstagramExtractor = class {
2188
2379
  }
2189
2380
  await sleep(1500, 2500, options.signal);
2190
2381
  try {
2191
- const nextPage = await this.fetchNextPostsPage(userId, username, endCursor, options);
2382
+ const nextPage = await this.fetchNextPostsPage(userId || "", username, endCursor, options, webTokens);
2192
2383
  currentEdges = nextPage.edges;
2193
2384
  hasNextPage = nextPage.hasNextPage;
2194
2385
  endCursor = nextPage.nextCursor;
@@ -2199,8 +2390,8 @@ var InstagramExtractor = class {
2199
2390
  break;
2200
2391
  }
2201
2392
  }
2202
- if (subDir === "Videos") {
2203
- const videoEdges = user.edge_felix_video_timeline?.edges ?? [];
2393
+ if (subDir === "Videos" && userId) {
2394
+ const videoEdges = user?.edge_felix_video_timeline?.edges ?? [];
2204
2395
  for (const edge of videoEdges) {
2205
2396
  const items = parseIgNodeToMediaItems(edge.node, username);
2206
2397
  for (const item of items) {
@@ -2272,8 +2463,43 @@ var TWITTER_FEATURES = {
2272
2463
  responsive_web_text_conversations_enabled: false,
2273
2464
  longform_notetweets_rich_text_read_enabled: true,
2274
2465
  longform_notetweets_inline_media_enabled: true,
2275
- responsive_web_enhance_cards_enabled: false
2466
+ responsive_web_enhance_cards_enabled: false,
2467
+ creator_subscriptions_tweet_preview_api_enabled: true,
2468
+ communities_web_enable_tweet_community_results_fetch: true,
2469
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
2470
+ articles_preview_enabled: true,
2471
+ responsive_web_twitter_article_tweet_consumption_enabled: true,
2472
+ responsive_web_grok_analyze_post_followups_enabled: true,
2473
+ responsive_web_jetfuel_frame: true,
2474
+ responsive_web_grok_share_attachment_enabled: true,
2475
+ responsive_web_grok_analysis_button_from_backend: true,
2476
+ responsive_web_grok_image_annotation_enabled: true,
2477
+ responsive_web_grok_imagine_annotation_enabled: true,
2478
+ profile_label_improvements_pcf_label_in_post_enabled: true,
2479
+ rweb_tipjar_consumption_enabled: true
2276
2480
  };
2481
+ function shouldSkipTweetReply(tweet, targetUserId, mode = "timeline") {
2482
+ const tweetResult = tweet.tweet || tweet;
2483
+ const legacy = tweetResult.legacy || tweetResult;
2484
+ const replyToUserId = legacy.in_reply_to_user_id_str;
2485
+ if (!replyToUserId) {
2486
+ return false;
2487
+ }
2488
+ const authorId = legacy.user_id_str || tweetResult.core?.user_results?.result?.rest_id;
2489
+ if (mode === "timeline") {
2490
+ if (targetUserId) {
2491
+ if (replyToUserId !== targetUserId || authorId && authorId !== targetUserId) {
2492
+ return true;
2493
+ }
2494
+ return false;
2495
+ }
2496
+ return Boolean(authorId && replyToUserId !== authorId);
2497
+ }
2498
+ if (authorId && replyToUserId === authorId) {
2499
+ return false;
2500
+ }
2501
+ return true;
2502
+ }
2277
2503
  function parseTweetMediaEntities(tweet, username) {
2278
2504
  const items = [];
2279
2505
  const tweetResult = tweet.tweet || tweet;
@@ -2368,11 +2594,17 @@ var XExtractor = class {
2368
2594
  }
2369
2595
  if (username === "i_history_likes") {
2370
2596
  if (options.cookiePath) {
2371
- const cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
2597
+ let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
2598
+ if (!cookies["twid"]) {
2599
+ cookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
2600
+ }
2601
+ if (!cookies["twid"]) {
2602
+ cookies = await loadCookiesFromPath(options.cookiePath);
2603
+ }
2372
2604
  const twid = cookies["twid"];
2373
2605
  if (twid) {
2374
2606
  const decoded = decodeURIComponent(twid).replace(/^"|"$/g, "").trim();
2375
- const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded;
2607
+ const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded.includes("=") ? decoded.split("=").pop()?.trim() : decoded;
2376
2608
  if (rawId && /^[a-zA-Z0-9_-]+$/.test(rawId)) {
2377
2609
  return rawId;
2378
2610
  }
@@ -2414,7 +2646,21 @@ var XExtractor = class {
2414
2646
  async *extract(profile, options) {
2415
2647
  const username = profile.username;
2416
2648
  const subDir = options.subDir;
2417
- const isLikes = username === "i_history_likes";
2649
+ const isLikes = username === "i_history_likes" || profile.cleanUrl?.endsWith("/likes");
2650
+ const authHeaders = {};
2651
+ if (options.cookiePath) {
2652
+ let cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
2653
+ if (!cookies["ct0"] && !cookies["auth_token"]) {
2654
+ const twCookies = await loadCookiesFromPath(options.cookiePath, "twitter.com");
2655
+ cookies = { ...twCookies, ...cookies };
2656
+ }
2657
+ if (cookies["ct0"]) {
2658
+ authHeaders["x-csrf-token"] = cookies["ct0"];
2659
+ }
2660
+ if (cookies["auth_token"]) {
2661
+ authHeaders["x-twitter-auth-type"] = "OAuth2Session";
2662
+ }
2663
+ }
2418
2664
  options.onProgress?.({
2419
2665
  stage: isLikes ? "Likes" : "Profile",
2420
2666
  found: 0,
@@ -2445,11 +2691,15 @@ var XExtractor = class {
2445
2691
  res = await this.httpClient.fetchJson(url, {
2446
2692
  cookiePath: options.cookiePath,
2447
2693
  cookieDomain: "x.com",
2448
- signal: options.signal
2694
+ signal: options.signal,
2695
+ headers: authHeaders
2449
2696
  });
2450
2697
  } catch (err) {
2451
2698
  break;
2452
2699
  }
2700
+ if (res?.errors?.length && !res?.data?.user?.result) {
2701
+ throw new Error(`X/Twitter API error: ${res.errors[0]?.message || "GraphQL query failed"}`);
2702
+ }
2453
2703
  const instructions = res?.data?.user?.result?.timeline_v2?.timeline?.instructions || [];
2454
2704
  let newCursor;
2455
2705
  let pageItemCount = 0;
@@ -2462,6 +2712,9 @@ var XExtractor = class {
2462
2712
  }
2463
2713
  const tweetContent = entry.content?.itemContent?.tweet_results?.result;
2464
2714
  if (tweetContent) {
2715
+ if (shouldSkipTweetReply(tweetContent, userId, isLikes ? "likes" : "timeline")) {
2716
+ continue;
2717
+ }
2465
2718
  const mediaItems = parseTweetMediaEntities(tweetContent, username);
2466
2719
  for (const item of mediaItems) {
2467
2720
  if (subDir === "Photos" && item.type !== "photo") continue;
@@ -2490,16 +2743,86 @@ var XExtractor = class {
2490
2743
 
2491
2744
  // src/lib/engines/extractors/tiktok.ts
2492
2745
  function parseTikTokRehydrationJson(html) {
2493
- const match = html.match(
2746
+ const universalMatch = html.match(
2494
2747
  /<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
2495
2748
  );
2496
- if (!match) return null;
2497
- try {
2498
- const raw = JSON.parse(match[1]);
2499
- return raw?.["__DEFAULT_SCOPE__"];
2500
- } catch {
2501
- return null;
2749
+ if (universalMatch) {
2750
+ try {
2751
+ const raw = JSON.parse(universalMatch[1]);
2752
+ const scope = raw?.["__DEFAULT_SCOPE__"] || raw;
2753
+ if (scope) return scope;
2754
+ } catch {
2755
+ }
2502
2756
  }
2757
+ const sigiTagMatch = html.match(
2758
+ /<script id="SIGI_STATE"[^>]*>([\s\S]*?)<\/script>/
2759
+ );
2760
+ if (sigiTagMatch) {
2761
+ try {
2762
+ const raw = JSON.parse(sigiTagMatch[1]);
2763
+ const user = Object.values(raw?.UserModule?.users || {})[0];
2764
+ const items = Object.values(raw?.ItemModule || {});
2765
+ return {
2766
+ "webapp.user-detail": {
2767
+ userInfo: { user },
2768
+ itemList: items
2769
+ },
2770
+ SIGI_STATE: raw
2771
+ };
2772
+ } catch {
2773
+ }
2774
+ }
2775
+ const sigiWindowMatch = html.match(
2776
+ /(?:window\[['"]SIGI_STATE['"]\]|SIGI_STATE)\s*=\s*(\{[\s\S]*?\})\s*;/i
2777
+ );
2778
+ if (sigiWindowMatch) {
2779
+ try {
2780
+ const raw = JSON.parse(sigiWindowMatch[1]);
2781
+ const user = Object.values(raw?.UserModule?.users || {})[0];
2782
+ const items = Object.values(raw?.ItemModule || {});
2783
+ return {
2784
+ "webapp.user-detail": {
2785
+ userInfo: { user },
2786
+ itemList: items
2787
+ },
2788
+ SIGI_STATE: raw
2789
+ };
2790
+ } catch {
2791
+ }
2792
+ }
2793
+ const nextDataMatch = html.match(
2794
+ /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/
2795
+ );
2796
+ if (nextDataMatch) {
2797
+ try {
2798
+ const raw = JSON.parse(nextDataMatch[1]);
2799
+ const pageProps = raw?.props?.pageProps;
2800
+ if (pageProps) {
2801
+ return {
2802
+ "webapp.user-detail": {
2803
+ userInfo: pageProps.userInfo,
2804
+ itemList: pageProps.items || pageProps.itemList || []
2805
+ },
2806
+ __NEXT_DATA__: raw
2807
+ };
2808
+ }
2809
+ } catch {
2810
+ }
2811
+ }
2812
+ const secUidMatch = html.match(/(?:secUid|sec_uid)["':=\s]+(MS4wLjABAAAA[\w-]{32,64})/i);
2813
+ if (secUidMatch) {
2814
+ return {
2815
+ "webapp.user-detail": {
2816
+ userInfo: {
2817
+ user: {
2818
+ secUid: secUidMatch[1]
2819
+ }
2820
+ },
2821
+ itemList: []
2822
+ }
2823
+ };
2824
+ }
2825
+ return null;
2503
2826
  }
2504
2827
  function parseTikTokItem(item, username) {
2505
2828
  const items = [];
@@ -2596,13 +2919,28 @@ var TikTokExtractor = class {
2596
2919
  if (options.signal?.aborted) break;
2597
2920
  if (visitedCursors.has(cursor)) break;
2598
2921
  visitedCursors.add(cursor);
2599
- const apiUrl = `https://www.tiktok.com/api/post/item_list/?secUid=${encodeURIComponent(secUid)}&count=30&cursor=${cursor}&post_item_list_request_type=0`;
2922
+ const deviceId = `${Math.floor(725e16 + Math.random() * 75e15)}`;
2923
+ const queryParams = new URLSearchParams({
2924
+ aid: "1988",
2925
+ app_name: "tiktok_web",
2926
+ device_platform: "web_pc",
2927
+ channel: "tiktok_web",
2928
+ device_id: deviceId,
2929
+ secUid,
2930
+ count: "30",
2931
+ cursor,
2932
+ post_item_list_request_type: "0"
2933
+ });
2934
+ const apiUrl = `https://www.tiktok.com/api/post/item_list/?${queryParams.toString()}`;
2600
2935
  let res;
2601
2936
  try {
2602
2937
  res = await this.httpClient.fetchJson(apiUrl, {
2603
2938
  cookiePath: options.cookiePath,
2604
2939
  cookieDomain: "tiktok.com",
2605
- signal: options.signal
2940
+ signal: options.signal,
2941
+ headers: {
2942
+ "Referer": profileUrl
2943
+ }
2606
2944
  });
2607
2945
  } catch {
2608
2946
  break;
@@ -2654,28 +2992,55 @@ function decodeEscapedUrl(raw) {
2654
2992
  function extractFacebookMediaFromHtml(html, username) {
2655
2993
  const items = [];
2656
2994
  const seenUrls = /* @__PURE__ */ new Set();
2657
- const hdMatch = html.match(/"browser_native_hd_url":"([^"]+)"/);
2658
- const sdMatch = html.match(/"browser_native_sd_url":"([^"]+)"/);
2659
- const rawVideoUrl = hdMatch?.[1] || sdMatch?.[1];
2660
- if (rawVideoUrl) {
2661
- const videoUrl = decodeEscapedUrl(rawVideoUrl);
2662
- if (!seenUrls.has(videoUrl)) {
2663
- seenUrls.add(videoUrl);
2664
- items.push({
2665
- id: `fb_video_${Date.now()}`,
2666
- url: videoUrl,
2667
- filename: `${username}_video_${Date.now()}`,
2668
- extension: "mp4",
2669
- type: "video"
2670
- });
2995
+ const videoPatterns = [
2996
+ /"browser_native_hd_url":"([^"]+)"/g,
2997
+ /"browser_native_sd_url":"([^"]+)"/g,
2998
+ /"playable_url_quality_hd":"([^"]+)"/g,
2999
+ /"playable_url":"([^"]+)"/g,
3000
+ /<video[^>]+src=["'](https?:[^"']+)["']/gi
3001
+ ];
3002
+ let videoIndex = 1;
3003
+ for (const regex of videoPatterns) {
3004
+ let vMatch;
3005
+ while ((vMatch = regex.exec(html)) !== null) {
3006
+ const rawUrl = vMatch[1];
3007
+ const videoUrl = decodeEscapedUrl(rawUrl).replace(/&amp;/g, "&");
3008
+ if (videoUrl.startsWith("http") && !seenUrls.has(videoUrl)) {
3009
+ seenUrls.add(videoUrl);
3010
+ items.push({
3011
+ id: `fb_video_${videoIndex}`,
3012
+ url: videoUrl,
3013
+ filename: `${username}_video_${videoIndex}`,
3014
+ extension: "mp4",
3015
+ type: "video"
3016
+ });
3017
+ videoIndex++;
3018
+ }
2671
3019
  }
2672
3020
  }
2673
- const photoRegex = /,"image":\{"uri":"([^"]+)"/g;
3021
+ const photoRegex = /,"(?:image|viewer_image|full_size_image|display_image)":\{"uri":"([^"]+)"/g;
2674
3022
  let match;
2675
3023
  let photoIndex = 1;
2676
3024
  while ((match = photoRegex.exec(html)) !== null) {
2677
3025
  const rawUrl = match[1];
2678
- const photoUrl = decodeEscapedUrl(rawUrl);
3026
+ const photoUrl = decodeEscapedUrl(rawUrl).replace(/&amp;/g, "&");
3027
+ if (!seenUrls.has(photoUrl)) {
3028
+ seenUrls.add(photoUrl);
3029
+ items.push({
3030
+ id: `fb_photo_${photoIndex}`,
3031
+ url: photoUrl,
3032
+ filename: `${username}_photo_${photoIndex}`,
3033
+ extension: "jpg",
3034
+ type: "photo"
3035
+ });
3036
+ photoIndex++;
3037
+ }
3038
+ }
3039
+ const mbasicImgRegex = /<img[^>]+src=["'](https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*)["']/gi;
3040
+ let mMatch;
3041
+ while ((mMatch = mbasicImgRegex.exec(html)) !== null) {
3042
+ const rawUrl = mMatch[1];
3043
+ const photoUrl = decodeEscapedUrl(rawUrl).replace(/&amp;/g, "&");
2679
3044
  if (!seenUrls.has(photoUrl)) {
2680
3045
  seenUrls.add(photoUrl);
2681
3046
  items.push({
@@ -2689,10 +3054,10 @@ function extractFacebookMediaFromHtml(html, username) {
2689
3054
  }
2690
3055
  }
2691
3056
  if (items.length === 0) {
2692
- const scontentRegex = /https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*/gi;
3057
+ const scontentRegex = /https:\/\/[^"'\s<>]*scontent[^"'\s<>]*\.(?:jpg|png|webp)[^"'\s<>]*/gi;
2693
3058
  let sMatch;
2694
3059
  while ((sMatch = scontentRegex.exec(html)) !== null) {
2695
- const cleanUrl = sMatch[0].replace(/\\/g, "");
3060
+ const cleanUrl = sMatch[0].replace(/\\/g, "").replace(/&amp;/g, "&");
2696
3061
  if (!seenUrls.has(cleanUrl)) {
2697
3062
  seenUrls.add(cleanUrl);
2698
3063
  items.push({
@@ -2708,6 +3073,59 @@ function extractFacebookMediaFromHtml(html, username) {
2708
3073
  }
2709
3074
  return items;
2710
3075
  }
3076
+ function extractFacebookPaginationCursor(html) {
3077
+ const jsonCursorMatch = html.match(/"end_cursor":"([^"]+)"/) || html.match(/"cursor":"([^"]+)"/);
3078
+ if (jsonCursorMatch && jsonCursorMatch[1]) {
3079
+ return jsonCursorMatch[1];
3080
+ }
3081
+ const linkCursorMatch = html.match(/[?&](?:cursor|after)=([^"'\s&<>]+)/i);
3082
+ if (linkCursorMatch && linkCursorMatch[1]) {
3083
+ try {
3084
+ return decodeURIComponent(linkCursorMatch[1]);
3085
+ } catch {
3086
+ return linkCursorMatch[1];
3087
+ }
3088
+ }
3089
+ return null;
3090
+ }
3091
+ function resolveFacebookUrl(href, currentUrl) {
3092
+ const cleanHref = href.replace(/\\/g, "").replace(/&amp;/g, "&");
3093
+ try {
3094
+ return new URL(cleanHref, currentUrl).toString();
3095
+ } catch {
3096
+ return cleanHref;
3097
+ }
3098
+ }
3099
+ function extractFacebookNextPageUrl(html, currentUrl) {
3100
+ const moreItemMatch = html.match(/<div[^>]*id=["'](?:m_more_item|m_more_posts)["'][^>]*>\s*<a[^>]+href=["']([^"']+)["']/i);
3101
+ if (moreItemMatch && moreItemMatch[1]) {
3102
+ return resolveFacebookUrl(moreItemMatch[1], currentUrl);
3103
+ }
3104
+ const cursorLabelMatch = html.match(/<a[^>]+href=["']([^"']*(?:cursor=|after=)[^"']+)["'][^>]*>[\s\S]*?(?:See [Mm]ore|More [Pp]hotos|More|Next|Older [Pp]osts)[\s\S]*?<\/a>/i);
3105
+ if (cursorLabelMatch && cursorLabelMatch[1]) {
3106
+ return resolveFacebookUrl(cursorLabelMatch[1], currentUrl);
3107
+ }
3108
+ const genericCursorMatch = html.match(/<a[^>]+href=["']([^"']*(?:[?&]|&amp;)(?:cursor|after)=[^"']+)["']/i);
3109
+ if (genericCursorMatch && genericCursorMatch[1]) {
3110
+ return resolveFacebookUrl(genericCursorMatch[1], currentUrl);
3111
+ }
3112
+ const cursor = extractFacebookPaginationCursor(html);
3113
+ if (cursor) {
3114
+ try {
3115
+ const u = new URL(currentUrl);
3116
+ if (u.searchParams.get("cursor") !== cursor) {
3117
+ u.searchParams.set("cursor", cursor);
3118
+ return u.toString();
3119
+ }
3120
+ } catch {
3121
+ if (!currentUrl.includes(cursor)) {
3122
+ const sep = currentUrl.includes("?") ? "&" : "?";
3123
+ return `${currentUrl}${sep}cursor=${encodeURIComponent(cursor)}`;
3124
+ }
3125
+ }
3126
+ }
3127
+ return null;
3128
+ }
2711
3129
  var FacebookExtractor = class {
2712
3130
  platform = "facebook";
2713
3131
  httpClient;
@@ -2724,26 +3142,120 @@ var FacebookExtractor = class {
2724
3142
  found: 0,
2725
3143
  message: `Fetching Facebook profile @${username}...`
2726
3144
  });
2727
- const targetUrl = subDir === "Photos" ? `https://www.facebook.com/${encodeURIComponent(username)}/photos` : `https://www.facebook.com/${encodeURIComponent(username)}/videos`;
2728
- const html = await this.httpClient.fetchText(targetUrl, {
2729
- cookiePath: options.cookiePath,
2730
- cookieDomain: "facebook.com",
2731
- signal: options.signal,
2732
- retries: 0,
2733
- timeoutMs: 4e3
2734
- });
2735
- const items = extractFacebookMediaFromHtml(html, username);
2736
- let found = 0;
2737
- for (const item of items) {
2738
- if (subDir === "Photos" && item.type !== "photo") continue;
2739
- if (subDir === "Videos" && item.type !== "video") continue;
2740
- found++;
3145
+ const targetPath = subDir === "Photos" ? "photos" : "videos";
3146
+ const desktopUrl = `https://www.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
3147
+ const mbasicUrl = `https://mbasic.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
3148
+ const mobileUrl = `https://m.facebook.com/${encodeURIComponent(username)}/${targetPath}`;
3149
+ let currentUrl = desktopUrl;
3150
+ let html = "";
3151
+ try {
3152
+ html = await this.httpClient.fetchText(desktopUrl, {
3153
+ cookiePath: options.cookiePath,
3154
+ cookieDomain: "facebook.com",
3155
+ signal: options.signal,
3156
+ retries: 0,
3157
+ timeoutMs: 4e3
3158
+ });
3159
+ } catch {
3160
+ html = "";
3161
+ }
3162
+ let initialItems = extractFacebookMediaFromHtml(html, username);
3163
+ if (initialItems.length === 0) {
2741
3164
  options.onProgress?.({
2742
- stage: subDir,
2743
- found,
2744
- message: `Found item: ${item.filename}`
3165
+ stage: "Profile",
3166
+ found: 0,
3167
+ message: `Desktop yielded 0 items. Falling back to mbasic.facebook.com for @${username}...`
2745
3168
  });
2746
- yield item;
3169
+ try {
3170
+ const mbasicHtml = await this.httpClient.fetchText(mbasicUrl, {
3171
+ cookiePath: options.cookiePath,
3172
+ cookieDomain: "facebook.com",
3173
+ signal: options.signal,
3174
+ retries: 1,
3175
+ timeoutMs: 5e3,
3176
+ headers: {
3177
+ "Referer": "https://mbasic.facebook.com/"
3178
+ }
3179
+ });
3180
+ const mbasicItems = extractFacebookMediaFromHtml(mbasicHtml, username);
3181
+ if (mbasicItems.length > 0 || mbasicHtml.length > 200) {
3182
+ html = mbasicHtml;
3183
+ currentUrl = mbasicUrl;
3184
+ initialItems = mbasicItems;
3185
+ }
3186
+ } catch {
3187
+ }
3188
+ if (initialItems.length === 0) {
3189
+ try {
3190
+ const mobileHtml = await this.httpClient.fetchText(mobileUrl, {
3191
+ cookiePath: options.cookiePath,
3192
+ cookieDomain: "facebook.com",
3193
+ signal: options.signal,
3194
+ retries: 0,
3195
+ timeoutMs: 5e3,
3196
+ headers: {
3197
+ "User-Agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
3198
+ "Referer": "https://m.facebook.com/"
3199
+ }
3200
+ });
3201
+ const mobileItems = extractFacebookMediaFromHtml(mobileHtml, username);
3202
+ if (mobileItems.length > 0) {
3203
+ html = mobileHtml;
3204
+ currentUrl = mobileUrl;
3205
+ initialItems = mobileItems;
3206
+ }
3207
+ } catch {
3208
+ }
3209
+ }
3210
+ }
3211
+ const seenUrls = /* @__PURE__ */ new Set();
3212
+ let found = 0;
3213
+ let page = 1;
3214
+ const maxPages = 5;
3215
+ while (true) {
3216
+ const items = extractFacebookMediaFromHtml(html, username);
3217
+ let pageNewCount = 0;
3218
+ for (const item of items) {
3219
+ if (subDir === "Photos" && item.type !== "photo") continue;
3220
+ if (subDir === "Videos" && item.type !== "video") continue;
3221
+ if (seenUrls.has(item.url)) continue;
3222
+ seenUrls.add(item.url);
3223
+ found++;
3224
+ pageNewCount++;
3225
+ options.onProgress?.({
3226
+ stage: subDir,
3227
+ found,
3228
+ message: `Found item: ${item.filename}`
3229
+ });
3230
+ yield item;
3231
+ }
3232
+ if (page >= maxPages || options.signal?.aborted) {
3233
+ break;
3234
+ }
3235
+ const nextUrl = extractFacebookNextPageUrl(html, currentUrl);
3236
+ if (!nextUrl || nextUrl === currentUrl) {
3237
+ break;
3238
+ }
3239
+ if (pageNewCount === 0 && page > 1) {
3240
+ break;
3241
+ }
3242
+ await sleep(1e3, 2e3, options.signal);
3243
+ try {
3244
+ currentUrl = nextUrl;
3245
+ html = await this.httpClient.fetchText(currentUrl, {
3246
+ cookiePath: options.cookiePath,
3247
+ cookieDomain: "facebook.com",
3248
+ signal: options.signal,
3249
+ retries: 1,
3250
+ timeoutMs: 5e3,
3251
+ headers: {
3252
+ "Referer": currentUrl.includes("mbasic") ? "https://mbasic.facebook.com/" : "https://www.facebook.com/"
3253
+ }
3254
+ });
3255
+ page++;
3256
+ } catch {
3257
+ break;
3258
+ }
2747
3259
  }
2748
3260
  }
2749
3261
  };
@@ -2771,17 +3283,48 @@ import path5 from "path";
2771
3283
  import { Readable } from "stream";
2772
3284
  import { pipeline } from "stream/promises";
2773
3285
  var ARCHIVE_FILENAME = ".open-nami-archive.json";
3286
+ var YTDLP_ARCHIVE_FILENAME = "archive_yt-dlp.txt";
3287
+ var TEXT_ARCHIVE_FILENAME = "archive.txt";
2774
3288
  async function loadArchive(destDir) {
2775
- const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
3289
+ const archive = /* @__PURE__ */ new Set();
3290
+ const jsonPath = path5.join(destDir, ARCHIVE_FILENAME);
2776
3291
  try {
2777
- const raw = await fsPromises2.readFile(archivePath, "utf-8");
3292
+ const raw = await fsPromises2.readFile(jsonPath, "utf-8");
2778
3293
  const parsed = JSON.parse(raw);
2779
3294
  if (Array.isArray(parsed)) {
2780
- return new Set(parsed);
3295
+ for (const id of parsed) {
3296
+ if (id) archive.add(String(id));
3297
+ }
2781
3298
  }
2782
3299
  } catch {
2783
3300
  }
2784
- return /* @__PURE__ */ new Set();
3301
+ const ytdlpPath = path5.join(destDir, YTDLP_ARCHIVE_FILENAME);
3302
+ try {
3303
+ const raw = await fsPromises2.readFile(ytdlpPath, "utf-8");
3304
+ const lines = raw.split(/\r?\n/);
3305
+ for (const rawLine of lines) {
3306
+ const line = rawLine.trim();
3307
+ if (!line || line.startsWith("#")) continue;
3308
+ const parts = line.split(/\s+/);
3309
+ const id = parts[parts.length - 1];
3310
+ if (id) archive.add(id);
3311
+ }
3312
+ } catch {
3313
+ }
3314
+ const textPath = path5.join(destDir, TEXT_ARCHIVE_FILENAME);
3315
+ try {
3316
+ const raw = await fsPromises2.readFile(textPath, "utf-8");
3317
+ const lines = raw.split(/\r?\n/);
3318
+ for (const rawLine of lines) {
3319
+ const line = rawLine.trim();
3320
+ if (!line || line.startsWith("#")) continue;
3321
+ const parts = line.split(/\s+/);
3322
+ const id = parts[parts.length - 1];
3323
+ if (id) archive.add(id);
3324
+ }
3325
+ } catch {
3326
+ }
3327
+ return archive;
2785
3328
  }
2786
3329
  async function saveArchive(destDir, archive) {
2787
3330
  const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
@@ -2791,6 +3334,18 @@ async function saveArchive(destDir, archive) {
2791
3334
  await fsPromises2.rename(tmpPath, archivePath);
2792
3335
  } catch {
2793
3336
  }
3337
+ const textPaths = [
3338
+ path5.join(destDir, YTDLP_ARCHIVE_FILENAME),
3339
+ path5.join(destDir, TEXT_ARCHIVE_FILENAME)
3340
+ ];
3341
+ const content = [...archive].map((id) => `${id}
3342
+ `).join("");
3343
+ for (const tPath of textPaths) {
3344
+ try {
3345
+ await fsPromises2.writeFile(tPath, content, "utf-8");
3346
+ } catch {
3347
+ }
3348
+ }
2794
3349
  }
2795
3350
  function sanitizeFilename(rawFilename, fallbackId, ext) {
2796
3351
  const cleanExt = ext.replace(/^\./, "");
@@ -2843,12 +3398,13 @@ async function downloadMediaStream(items, options) {
2843
3398
  consecutiveErrors++;
2844
3399
  const errMsg = err instanceof Error ? err.message : String(err);
2845
3400
  errors.push(`Extraction stream error: ${errMsg}`);
3401
+ const cleanErr = errMsg.includes("429") ? "rate limited by platform (HTTP 429)" : errMsg.length > 50 ? `${errMsg.slice(0, 47)}\u2026` : errMsg;
2846
3402
  onProgress?.({
2847
3403
  job: jobName,
2848
3404
  downloadedCount: downloaded,
2849
3405
  skippedCount: skipped,
2850
3406
  currentFile: "",
2851
- statusText: `Extractor glitch: ${errMsg}`
3407
+ statusText: `Extractor error: ${cleanErr}`
2852
3408
  });
2853
3409
  if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS || signal?.aborted) {
2854
3410
  isDone = true;
@@ -3318,8 +3874,9 @@ async function runGalleryDl(options) {
3318
3874
  let downloadedCount = 0;
3319
3875
  let skippedCount = 0;
3320
3876
  let errorOutput = "";
3877
+ let engineCmdStr = "gallery-dl";
3321
3878
  try {
3322
- await ensureGalleryDl((status) => {
3879
+ engineCmdStr = await ensureGalleryDl((status) => {
3323
3880
  onProgress?.({
3324
3881
  job: subDir,
3325
3882
  downloadedCount: 0,
@@ -3335,9 +3892,15 @@ async function runGalleryDl(options) {
3335
3892
  error: err.message || String(err)
3336
3893
  };
3337
3894
  }
3338
- const engineCmd = resolveEngineCommand("gallery-dl");
3895
+ const engineCmd = engineCmdStr.includes(path8.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("gallery-dl");
3339
3896
  const fullArgs = [...engineCmd.baseArgs, ...args];
3340
3897
  return new Promise((resolve) => {
3898
+ onProgress?.({
3899
+ job: subDir,
3900
+ downloadedCount,
3901
+ skippedCount,
3902
+ statusText: "downloading with gallery-dl\u2026"
3903
+ });
3341
3904
  const child = spawn2(engineCmd.command, fullArgs, {
3342
3905
  windowsHide: true,
3343
3906
  stdio: ["ignore", "pipe", "pipe"],
@@ -3461,8 +4024,9 @@ async function runYtDlp(options) {
3461
4024
  let downloadedCount = 0;
3462
4025
  let skippedCount = 0;
3463
4026
  let errorOutput = "";
4027
+ let engineCmdStr = "yt-dlp";
3464
4028
  try {
3465
- await ensureYtDlp((status) => {
4029
+ engineCmdStr = await ensureYtDlp((status) => {
3466
4030
  onProgress?.({
3467
4031
  job: "Videos",
3468
4032
  downloadedCount: 0,
@@ -3478,9 +4042,15 @@ async function runYtDlp(options) {
3478
4042
  error: err.message || String(err)
3479
4043
  };
3480
4044
  }
3481
- const engineCmd = resolveEngineCommand("yt-dlp");
4045
+ const engineCmd = engineCmdStr.includes(path9.sep) ? { command: engineCmdStr, baseArgs: [] } : resolveEngineCommand("yt-dlp");
3482
4046
  const fullArgs = [...engineCmd.baseArgs, ...args];
3483
4047
  return new Promise((resolve) => {
4048
+ onProgress?.({
4049
+ job: "Videos",
4050
+ downloadedCount,
4051
+ skippedCount,
4052
+ statusText: "downloading with yt-dlp\u2026"
4053
+ });
3484
4054
  const child = spawn3(engineCmd.command, fullArgs, {
3485
4055
  windowsHide: true,
3486
4056
  stdio: ["ignore", "pipe", "pipe"],
@@ -3617,7 +4187,7 @@ async function dispatchDownload(options) {
3617
4187
  jobs.push({ type: "highlights" });
3618
4188
  }
3619
4189
  }
3620
- const runNative = async (subDir) => {
4190
+ const defaultRunNative = async (subDir) => {
3621
4191
  const dest = path10.join(targetDir, subDir);
3622
4192
  try {
3623
4193
  const extractor = getExtractor(profile.platform);
@@ -3650,27 +4220,34 @@ async function dispatchDownload(options) {
3650
4220
  };
3651
4221
  }
3652
4222
  };
3653
- const runGdl = (subDir) => runGalleryDl({
4223
+ const runNative = options.runnerOverrides?.nativeRunner ?? defaultRunNative;
4224
+ const runGdl = options.runnerOverrides?.gdlRunner ?? ((subDir) => runGalleryDl({
3654
4225
  profile,
3655
4226
  targetDir,
3656
4227
  subDir,
3657
4228
  cookiePath: cookieCopy?.filePath,
3658
4229
  signal: options.signal,
3659
4230
  onProgress
3660
- });
3661
- const runYt = () => runYtDlp({
4231
+ }));
4232
+ const runYt = options.runnerOverrides?.ytRunner ?? (() => runYtDlp({
3662
4233
  profile,
3663
4234
  targetDir,
3664
4235
  cookiePath: cookieCopy?.filePath,
3665
4236
  signal: options.signal,
3666
4237
  onProgress
3667
- });
4238
+ }));
3668
4239
  for (const job of jobs) {
3669
4240
  if (options.signal?.aborted) break;
3670
4241
  if (job.type === "photos" || job.type === "stories" || job.type === "highlights") {
3671
4242
  const subDir = job.type === "photos" ? "Photos" : job.type === "stories" ? "Stories" : "Highlights";
3672
4243
  let res = await runNative(subDir);
3673
4244
  if ((!res.success || res.downloaded === 0 && res.skipped === 0) && !options.signal?.aborted) {
4245
+ onProgress?.({
4246
+ job: subDir,
4247
+ downloadedCount: totalDownloaded,
4248
+ skippedCount: totalSkipped,
4249
+ statusText: "switching to gallery-dl engine\u2026"
4250
+ });
3674
4251
  const gdlRes = await runGdl(subDir);
3675
4252
  if (gdlRes.downloaded > 0 || gdlRes.skipped > 0 || gdlRes.success) {
3676
4253
  res = gdlRes;
@@ -3685,12 +4262,24 @@ async function dispatchDownload(options) {
3685
4262
  totalDownloaded += res.downloaded;
3686
4263
  totalSkipped += res.skipped;
3687
4264
  } else if (!options.signal?.aborted) {
4265
+ onProgress?.({
4266
+ job: "Videos",
4267
+ downloadedCount: totalDownloaded,
4268
+ skippedCount: totalSkipped,
4269
+ statusText: "switching to yt-dlp engine\u2026"
4270
+ });
3688
4271
  const fbRes = await runYt();
3689
4272
  if (fbRes.downloaded > 0 || fbRes.skipped > 0 || fbRes.success) {
3690
4273
  totalDownloaded += fbRes.downloaded;
3691
4274
  totalSkipped += fbRes.skipped;
3692
4275
  if (!fbRes.success && fbRes.error) errors.push(`[Videos] ${fbRes.error}`);
3693
4276
  } else if (!options.signal?.aborted) {
4277
+ onProgress?.({
4278
+ job: "Videos",
4279
+ downloadedCount: totalDownloaded,
4280
+ skippedCount: totalSkipped,
4281
+ statusText: "switching to gallery-dl engine\u2026"
4282
+ });
3694
4283
  const gdlRes = await runGdl("Videos");
3695
4284
  totalDownloaded += gdlRes.downloaded;
3696
4285
  totalSkipped += gdlRes.skipped;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "open-nami",
3
- "version": "1.0.4",
3
+ "version": "1.0.5",
4
4
  "description": "Open Nami — terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
5
5
  "type": "module",
6
6
  "bin": {