open-nami 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +377 -118
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -9,7 +9,7 @@ var require_package = __commonJS({
|
|
|
9
9
|
"package.json"(exports, module) {
|
|
10
10
|
module.exports = {
|
|
11
11
|
name: "open-nami",
|
|
12
|
-
version: "1.0.
|
|
12
|
+
version: "1.0.4",
|
|
13
13
|
description: "Open Nami \u2014 terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
|
|
14
14
|
type: "module",
|
|
15
15
|
bin: {
|
|
@@ -1387,6 +1387,22 @@ function parseProfileInput(input) {
|
|
|
1387
1387
|
}
|
|
1388
1388
|
}
|
|
1389
1389
|
if (host.includes("x.com") || host.includes("twitter.com")) {
|
|
1390
|
+
if (pathParts[0]?.toLowerCase() === "i" && pathParts[1]?.toLowerCase() === "history" && pathParts[2]?.toLowerCase() === "likes") {
|
|
1391
|
+
return {
|
|
1392
|
+
platform: "x",
|
|
1393
|
+
username: "i_history_likes",
|
|
1394
|
+
cleanUrl: "https://x.com/i/history/likes",
|
|
1395
|
+
rawInput: input
|
|
1396
|
+
};
|
|
1397
|
+
}
|
|
1398
|
+
if (pathParts[0]?.toLowerCase() === "i" && pathParts[1]?.toLowerCase() === "bookmarks") {
|
|
1399
|
+
return {
|
|
1400
|
+
platform: "x",
|
|
1401
|
+
username: "i_bookmarks",
|
|
1402
|
+
cleanUrl: "https://x.com/i/bookmarks",
|
|
1403
|
+
rawInput: input
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1390
1406
|
const firstPart = pathParts[0] || "";
|
|
1391
1407
|
const reserved = ["home", "explore", "notifications", "messages", "i", "settings"];
|
|
1392
1408
|
if (firstPart && !reserved.includes(firstPart.toLowerCase())) {
|
|
@@ -1627,6 +1643,11 @@ async function sleep(minMs, maxMs, signal) {
|
|
|
1627
1643
|
throw err;
|
|
1628
1644
|
}
|
|
1629
1645
|
}
|
|
1646
|
+
function combineTimeoutSignal(signal, timeoutMs) {
|
|
1647
|
+
if (!timeoutMs || timeoutMs <= 0) return signal;
|
|
1648
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
1649
|
+
return signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
|
1650
|
+
}
|
|
1630
1651
|
var HttpClient = class {
|
|
1631
1652
|
defaultCookies = {};
|
|
1632
1653
|
defaultHeaders;
|
|
@@ -1661,8 +1682,8 @@ var HttpClient = class {
|
|
|
1661
1682
|
Object.assign(this.defaultCookies, loaded);
|
|
1662
1683
|
}
|
|
1663
1684
|
async request(url, options = {}) {
|
|
1664
|
-
const retries = options.retries ??
|
|
1665
|
-
const retryDelay = options.retryDelayMs ??
|
|
1685
|
+
const retries = options.retries ?? 1;
|
|
1686
|
+
const retryDelay = options.retryDelayMs ?? 1e3;
|
|
1666
1687
|
let cookieHeader = "";
|
|
1667
1688
|
if (options.cookiePath) {
|
|
1668
1689
|
const fromPath = await this.getCachedCookies(options.cookiePath, options.cookieDomain);
|
|
@@ -1688,9 +1709,11 @@ var HttpClient = class {
|
|
|
1688
1709
|
if (options.signal?.aborted) {
|
|
1689
1710
|
throw options.signal.reason ?? new Error("Aborted");
|
|
1690
1711
|
}
|
|
1712
|
+
const requestSignal = combineTimeoutSignal(options.signal || void 0, options.timeoutMs);
|
|
1691
1713
|
try {
|
|
1692
1714
|
const response = await fetch(url, {
|
|
1693
1715
|
...options,
|
|
1716
|
+
signal: requestSignal,
|
|
1694
1717
|
headers
|
|
1695
1718
|
});
|
|
1696
1719
|
if (response.status === 429) {
|
|
@@ -1699,7 +1722,8 @@ var HttpClient = class {
|
|
|
1699
1722
|
await response.body?.cancel();
|
|
1700
1723
|
} catch {
|
|
1701
1724
|
}
|
|
1702
|
-
|
|
1725
|
+
const backoffMs = Math.min(retryDelay * (attempt + 1), 2500);
|
|
1726
|
+
await sleep(backoffMs, void 0, options.signal || void 0);
|
|
1703
1727
|
continue;
|
|
1704
1728
|
}
|
|
1705
1729
|
}
|
|
@@ -1708,7 +1732,8 @@ var HttpClient = class {
|
|
|
1708
1732
|
await response.body?.cancel();
|
|
1709
1733
|
} catch {
|
|
1710
1734
|
}
|
|
1711
|
-
|
|
1735
|
+
const backoffMs = Math.min(retryDelay * (attempt + 1), 2e3);
|
|
1736
|
+
await sleep(backoffMs, void 0, options.signal || void 0);
|
|
1712
1737
|
continue;
|
|
1713
1738
|
}
|
|
1714
1739
|
return response;
|
|
@@ -1731,6 +1756,10 @@ var HttpClient = class {
|
|
|
1731
1756
|
}
|
|
1732
1757
|
});
|
|
1733
1758
|
if (!res.ok) {
|
|
1759
|
+
try {
|
|
1760
|
+
await res.body?.cancel();
|
|
1761
|
+
} catch {
|
|
1762
|
+
}
|
|
1734
1763
|
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1735
1764
|
}
|
|
1736
1765
|
return await res.json();
|
|
@@ -1738,6 +1767,10 @@ var HttpClient = class {
|
|
|
1738
1767
|
async fetchText(url, options = {}) {
|
|
1739
1768
|
const res = await this.request(url, options);
|
|
1740
1769
|
if (!res.ok) {
|
|
1770
|
+
try {
|
|
1771
|
+
await res.body?.cancel();
|
|
1772
|
+
} catch {
|
|
1773
|
+
}
|
|
1741
1774
|
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1742
1775
|
}
|
|
1743
1776
|
return await res.text();
|
|
@@ -1747,7 +1780,22 @@ var HttpClient = class {
|
|
|
1747
1780
|
// src/lib/engines/extractors/instagram.ts
|
|
1748
1781
|
var IG_APP_ID = "936619743392459";
|
|
1749
1782
|
var IG_TIMELINE_QUERY_HASH = "69cba40317214236af40e7efa697781d";
|
|
1750
|
-
var
|
|
1783
|
+
var IG_QUERIES = {
|
|
1784
|
+
PolarisProfilePostsTabContentQuery_connection: "28142289515441884",
|
|
1785
|
+
PolarisProfileReelsTabContentQuery_connection: "28989741540630190",
|
|
1786
|
+
PolarisProfileTaggedTabContentQuery_connection: "28794602806842110",
|
|
1787
|
+
PolarisProfileStoryHighlightsTrayContentQuery: "26970053832668570",
|
|
1788
|
+
PolarisStoriesV3ReelPageGalleryQuery: "28262315486766731",
|
|
1789
|
+
PolarisStoriesV3TrayContainerQuery: "27703822975903310",
|
|
1790
|
+
PolarisProfilePageContentQuery: "28036671149327607"
|
|
1791
|
+
};
|
|
1792
|
+
var IG_POLARIS_DOC_ID = IG_QUERIES.PolarisProfilePostsTabContentQuery_connection;
|
|
1793
|
+
function logExtractorDebug(endpoint, err) {
|
|
1794
|
+
if (process.env.DEBUG || process.env.OPEN_NAMI_DEBUG) {
|
|
1795
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1796
|
+
console.debug(`[open-nami:instagram] ${endpoint} query fallback: ${message}`);
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1751
1799
|
function parseIgNodeToMediaItems(node, username) {
|
|
1752
1800
|
const items = [];
|
|
1753
1801
|
const anyNode = node;
|
|
@@ -1852,6 +1900,8 @@ var InstagramExtractor = class {
|
|
|
1852
1900
|
cookiePath: options.cookiePath,
|
|
1853
1901
|
cookieDomain: "instagram.com",
|
|
1854
1902
|
signal: options.signal,
|
|
1903
|
+
retries: 0,
|
|
1904
|
+
timeoutMs: 4e3,
|
|
1855
1905
|
headers: {
|
|
1856
1906
|
"X-IG-App-ID": IG_APP_ID,
|
|
1857
1907
|
"Referer": `https://www.instagram.com/${username}/`
|
|
@@ -1863,6 +1913,32 @@ var InstagramExtractor = class {
|
|
|
1863
1913
|
return res.data.user;
|
|
1864
1914
|
}
|
|
1865
1915
|
async fetchHighlightsTray(userId, options) {
|
|
1916
|
+
try {
|
|
1917
|
+
const variables = JSON.stringify({ user_id: userId });
|
|
1918
|
+
const body = new URLSearchParams({
|
|
1919
|
+
doc_id: IG_QUERIES.PolarisProfileStoryHighlightsTrayContentQuery,
|
|
1920
|
+
variables
|
|
1921
|
+
}).toString();
|
|
1922
|
+
const res = await this.httpClient.fetchJson("https://www.instagram.com/graphql/query", {
|
|
1923
|
+
method: "POST",
|
|
1924
|
+
body,
|
|
1925
|
+
cookiePath: options.cookiePath,
|
|
1926
|
+
cookieDomain: "instagram.com",
|
|
1927
|
+
signal: options.signal,
|
|
1928
|
+
headers: {
|
|
1929
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
1930
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1931
|
+
"X-FB-Friendly-Name": "PolarisProfileStoryHighlightsTrayContentQuery",
|
|
1932
|
+
"Referer": "https://www.instagram.com/"
|
|
1933
|
+
}
|
|
1934
|
+
});
|
|
1935
|
+
const edges = res?.data?.highlights?.edges;
|
|
1936
|
+
if (Array.isArray(edges) && edges.length > 0) {
|
|
1937
|
+
return edges.map((e) => e.node);
|
|
1938
|
+
}
|
|
1939
|
+
} catch (err) {
|
|
1940
|
+
logExtractorDebug("PolarisProfileStoryHighlightsTrayContentQuery", err);
|
|
1941
|
+
}
|
|
1866
1942
|
const url = `https://www.instagram.com/api/v1/highlights/${userId}/highlights_tray/`;
|
|
1867
1943
|
try {
|
|
1868
1944
|
const res = await this.httpClient.fetchJson(url, {
|
|
@@ -1874,7 +1950,8 @@ var InstagramExtractor = class {
|
|
|
1874
1950
|
}
|
|
1875
1951
|
});
|
|
1876
1952
|
return res.tray ?? [];
|
|
1877
|
-
} catch {
|
|
1953
|
+
} catch (err) {
|
|
1954
|
+
logExtractorDebug("Highlights legacy REST", err);
|
|
1878
1955
|
return [];
|
|
1879
1956
|
}
|
|
1880
1957
|
}
|
|
@@ -1919,7 +1996,8 @@ var InstagramExtractor = class {
|
|
|
1919
1996
|
nextCursor: data.page_info?.end_cursor
|
|
1920
1997
|
};
|
|
1921
1998
|
}
|
|
1922
|
-
} catch {
|
|
1999
|
+
} catch (err) {
|
|
2000
|
+
logExtractorDebug("edge_owner_to_timeline_media", err);
|
|
1923
2001
|
}
|
|
1924
2002
|
try {
|
|
1925
2003
|
const postVariables = JSON.stringify({
|
|
@@ -1954,7 +2032,8 @@ var InstagramExtractor = class {
|
|
|
1954
2032
|
nextCursor: connection.page_info?.end_cursor
|
|
1955
2033
|
};
|
|
1956
2034
|
}
|
|
1957
|
-
} catch {
|
|
2035
|
+
} catch (err) {
|
|
2036
|
+
logExtractorDebug("PolarisProfilePostsTabContentQuery_connection", err);
|
|
1958
2037
|
}
|
|
1959
2038
|
try {
|
|
1960
2039
|
const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
|
|
@@ -1975,7 +2054,60 @@ var InstagramExtractor = class {
|
|
|
1975
2054
|
nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
|
|
1976
2055
|
};
|
|
1977
2056
|
}
|
|
1978
|
-
} catch {
|
|
2057
|
+
} catch (err) {
|
|
2058
|
+
logExtractorDebug("v1 user feed REST", err);
|
|
2059
|
+
}
|
|
2060
|
+
return { edges: [], hasNextPage: false };
|
|
2061
|
+
}
|
|
2062
|
+
async fetchNextReelsPage(userId, cursor, options) {
|
|
2063
|
+
try {
|
|
2064
|
+
const variables = JSON.stringify({
|
|
2065
|
+
after: cursor,
|
|
2066
|
+
first: 12,
|
|
2067
|
+
id: userId,
|
|
2068
|
+
data: {
|
|
2069
|
+
include_feed_video: true,
|
|
2070
|
+
page_size: 12,
|
|
2071
|
+
target_user_id: userId
|
|
2072
|
+
}
|
|
2073
|
+
});
|
|
2074
|
+
const postUrl = "https://www.instagram.com/graphql/query";
|
|
2075
|
+
const body = new URLSearchParams({
|
|
2076
|
+
doc_id: IG_QUERIES.PolarisProfileReelsTabContentQuery_connection,
|
|
2077
|
+
variables
|
|
2078
|
+
}).toString();
|
|
2079
|
+
const res = await this.httpClient.fetchJson(postUrl, {
|
|
2080
|
+
method: "POST",
|
|
2081
|
+
body,
|
|
2082
|
+
cookiePath: options.cookiePath,
|
|
2083
|
+
cookieDomain: "instagram.com",
|
|
2084
|
+
signal: options.signal,
|
|
2085
|
+
headers: {
|
|
2086
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
2087
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2088
|
+
"X-FB-Friendly-Name": "PolarisProfileReelsTabContentQuery_connection",
|
|
2089
|
+
"Referer": "https://www.instagram.com/"
|
|
2090
|
+
}
|
|
2091
|
+
});
|
|
2092
|
+
const reelsData = res?.data?.fetch__XDTUserDict;
|
|
2093
|
+
let connection = reelsData?.reels;
|
|
2094
|
+
if (!connection && reelsData) {
|
|
2095
|
+
for (const val of Object.values(reelsData)) {
|
|
2096
|
+
if (val && typeof val === "object" && "edges" in val) {
|
|
2097
|
+
connection = val;
|
|
2098
|
+
break;
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
if (connection?.edges && Array.isArray(connection.edges)) {
|
|
2103
|
+
return {
|
|
2104
|
+
edges: connection.edges,
|
|
2105
|
+
hasNextPage: Boolean(connection.page_info?.has_next_page),
|
|
2106
|
+
nextCursor: connection.page_info?.end_cursor
|
|
2107
|
+
};
|
|
2108
|
+
}
|
|
2109
|
+
} catch (err) {
|
|
2110
|
+
logExtractorDebug("PolarisProfileReelsTabContentQuery_connection", err);
|
|
1979
2111
|
}
|
|
1980
2112
|
return { edges: [], hasNextPage: false };
|
|
1981
2113
|
}
|
|
@@ -2030,6 +2162,7 @@ var InstagramExtractor = class {
|
|
|
2030
2162
|
return;
|
|
2031
2163
|
}
|
|
2032
2164
|
let found = 0;
|
|
2165
|
+
const emittedItemIds = /* @__PURE__ */ new Set();
|
|
2033
2166
|
let currentEdges = user.edge_owner_to_timeline_media?.edges ?? [];
|
|
2034
2167
|
let hasNextPage = Boolean(user.edge_owner_to_timeline_media?.page_info?.has_next_page);
|
|
2035
2168
|
let endCursor = user.edge_owner_to_timeline_media?.page_info?.end_cursor;
|
|
@@ -2039,6 +2172,8 @@ var InstagramExtractor = class {
|
|
|
2039
2172
|
for (const item of items) {
|
|
2040
2173
|
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2041
2174
|
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2175
|
+
if (emittedItemIds.has(item.id)) continue;
|
|
2176
|
+
emittedItemIds.add(item.id);
|
|
2042
2177
|
found++;
|
|
2043
2178
|
options.onProgress?.({
|
|
2044
2179
|
stage: subDir,
|
|
@@ -2070,6 +2205,8 @@ var InstagramExtractor = class {
|
|
|
2070
2205
|
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2071
2206
|
for (const item of items) {
|
|
2072
2207
|
if (item.type !== "video") continue;
|
|
2208
|
+
if (emittedItemIds.has(item.id)) continue;
|
|
2209
|
+
emittedItemIds.add(item.id);
|
|
2073
2210
|
found++;
|
|
2074
2211
|
options.onProgress?.({
|
|
2075
2212
|
stage: "Videos",
|
|
@@ -2079,6 +2216,37 @@ var InstagramExtractor = class {
|
|
|
2079
2216
|
yield item;
|
|
2080
2217
|
}
|
|
2081
2218
|
}
|
|
2219
|
+
let reelsCursor = "";
|
|
2220
|
+
let hasMoreReels = true;
|
|
2221
|
+
while (hasMoreReels && !options.signal?.aborted) {
|
|
2222
|
+
try {
|
|
2223
|
+
const reelsPage = await this.fetchNextReelsPage(userId, reelsCursor, options);
|
|
2224
|
+
if (!reelsPage.edges || reelsPage.edges.length === 0) break;
|
|
2225
|
+
for (const edge of reelsPage.edges) {
|
|
2226
|
+
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2227
|
+
for (const item of items) {
|
|
2228
|
+
if (item.type !== "video") continue;
|
|
2229
|
+
if (emittedItemIds.has(item.id)) continue;
|
|
2230
|
+
emittedItemIds.add(item.id);
|
|
2231
|
+
found++;
|
|
2232
|
+
options.onProgress?.({
|
|
2233
|
+
stage: "Videos",
|
|
2234
|
+
found,
|
|
2235
|
+
message: `Found reel: ${item.filename}`
|
|
2236
|
+
});
|
|
2237
|
+
yield item;
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
if (reelsPage.hasNextPage && reelsPage.nextCursor && reelsPage.nextCursor !== reelsCursor) {
|
|
2241
|
+
reelsCursor = reelsPage.nextCursor;
|
|
2242
|
+
await sleep(1500, 2500, options.signal);
|
|
2243
|
+
} else {
|
|
2244
|
+
hasMoreReels = false;
|
|
2245
|
+
}
|
|
2246
|
+
} catch {
|
|
2247
|
+
break;
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2082
2250
|
}
|
|
2083
2251
|
}
|
|
2084
2252
|
};
|
|
@@ -2175,6 +2343,8 @@ var XExtractor = class {
|
|
|
2175
2343
|
cookiePath: options.cookiePath,
|
|
2176
2344
|
cookieDomain: "x.com",
|
|
2177
2345
|
signal: options.signal,
|
|
2346
|
+
retries: 0,
|
|
2347
|
+
timeoutMs: 3e3,
|
|
2178
2348
|
headers: {
|
|
2179
2349
|
"Authorization": `Bearer ${TWITTER_BEARER_TOKEN}`
|
|
2180
2350
|
}
|
|
@@ -2193,6 +2363,23 @@ var XExtractor = class {
|
|
|
2193
2363
|
throw new Error("Failed to activate X/Twitter guest session: no guest token returned");
|
|
2194
2364
|
}
|
|
2195
2365
|
async fetchUserRestId(username, options) {
|
|
2366
|
+
if (username.startsWith("id:")) {
|
|
2367
|
+
return username.slice(3);
|
|
2368
|
+
}
|
|
2369
|
+
if (username === "i_history_likes") {
|
|
2370
|
+
if (options.cookiePath) {
|
|
2371
|
+
const cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
|
|
2372
|
+
const twid = cookies["twid"];
|
|
2373
|
+
if (twid) {
|
|
2374
|
+
const decoded = decodeURIComponent(twid).replace(/^"|"$/g, "").trim();
|
|
2375
|
+
const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded;
|
|
2376
|
+
if (rawId && /^[a-zA-Z0-9_-]+$/.test(rawId)) {
|
|
2377
|
+
return rawId;
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
throw new Error("Authentication required for X/Twitter likes history: missing 'twid' cookie");
|
|
2382
|
+
}
|
|
2196
2383
|
await this.ensureGuestToken(options);
|
|
2197
2384
|
const variables = JSON.stringify({
|
|
2198
2385
|
screen_name: username,
|
|
@@ -2204,7 +2391,9 @@ var XExtractor = class {
|
|
|
2204
2391
|
const res = await this.httpClient.fetchJson(url, {
|
|
2205
2392
|
cookiePath: options.cookiePath,
|
|
2206
2393
|
cookieDomain: "x.com",
|
|
2207
|
-
signal: options.signal
|
|
2394
|
+
signal: options.signal,
|
|
2395
|
+
retries: 0,
|
|
2396
|
+
timeoutMs: 4e3
|
|
2208
2397
|
});
|
|
2209
2398
|
const restId = res.data?.user?.result?.rest_id;
|
|
2210
2399
|
if (restId) return restId;
|
|
@@ -2212,7 +2401,11 @@ var XExtractor = class {
|
|
|
2212
2401
|
}
|
|
2213
2402
|
const syndiUrl = `https://cdn.syndication.twimg.com/widgets/followbutton/info.json?screen_names=${encodeURIComponent(username)}`;
|
|
2214
2403
|
try {
|
|
2215
|
-
const res = await this.httpClient.fetchJson(syndiUrl, {
|
|
2404
|
+
const res = await this.httpClient.fetchJson(syndiUrl, {
|
|
2405
|
+
signal: options.signal,
|
|
2406
|
+
retries: 0,
|
|
2407
|
+
timeoutMs: 3e3
|
|
2408
|
+
});
|
|
2216
2409
|
if (res[0]?.id) return String(res[0].id);
|
|
2217
2410
|
} catch {
|
|
2218
2411
|
}
|
|
@@ -2221,13 +2414,14 @@ var XExtractor = class {
|
|
|
2221
2414
|
async *extract(profile, options) {
|
|
2222
2415
|
const username = profile.username;
|
|
2223
2416
|
const subDir = options.subDir;
|
|
2417
|
+
const isLikes = username === "i_history_likes";
|
|
2224
2418
|
options.onProgress?.({
|
|
2225
|
-
stage: "Profile",
|
|
2419
|
+
stage: isLikes ? "Likes" : "Profile",
|
|
2226
2420
|
found: 0,
|
|
2227
|
-
message: `Fetching X profile @${username}...`
|
|
2421
|
+
message: isLikes ? "Fetching liked posts from X..." : `Fetching X profile @${username}...`
|
|
2228
2422
|
});
|
|
2229
2423
|
const userId = await this.fetchUserRestId(username, options);
|
|
2230
|
-
const endpoint = "https://x.com/i/api/graphql/jCRhbOzdgOHp6u9H4g2tEg/UserMedia";
|
|
2424
|
+
const endpoint = isLikes ? "https://x.com/i/api/graphql/TGEKkJG_meudeaFcqaxM-Q/Likes" : "https://x.com/i/api/graphql/jCRhbOzdgOHp6u9H4g2tEg/UserMedia";
|
|
2231
2425
|
let cursor;
|
|
2232
2426
|
let hasMore = true;
|
|
2233
2427
|
let found = 0;
|
|
@@ -2368,7 +2562,9 @@ var TikTokExtractor = class {
|
|
|
2368
2562
|
const html = await this.httpClient.fetchText(profileUrl, {
|
|
2369
2563
|
cookiePath: options.cookiePath,
|
|
2370
2564
|
cookieDomain: "tiktok.com",
|
|
2371
|
-
signal: options.signal
|
|
2565
|
+
signal: options.signal,
|
|
2566
|
+
retries: 0,
|
|
2567
|
+
timeoutMs: 4e3
|
|
2372
2568
|
});
|
|
2373
2569
|
const scope = parseTikTokRehydrationJson(html);
|
|
2374
2570
|
const userDetail = scope?.["webapp.user-detail"];
|
|
@@ -2435,6 +2631,26 @@ var TikTokExtractor = class {
|
|
|
2435
2631
|
};
|
|
2436
2632
|
|
|
2437
2633
|
// src/lib/engines/extractors/facebook.ts
|
|
2634
|
+
function decodeEscapedUrl(raw) {
|
|
2635
|
+
const sanitized = raw.replace(/[\r\n\t]/g, "").trim();
|
|
2636
|
+
try {
|
|
2637
|
+
return JSON.parse(`"${sanitized.replace(/"/g, '\\"')}"`);
|
|
2638
|
+
} catch {
|
|
2639
|
+
return sanitized.replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))).replace(/\\(["\\\/bfnrt])/g, (_, ch) => {
|
|
2640
|
+
const escapes = {
|
|
2641
|
+
'"': '"',
|
|
2642
|
+
"\\": "\\",
|
|
2643
|
+
"/": "/",
|
|
2644
|
+
b: "\b",
|
|
2645
|
+
f: "\f",
|
|
2646
|
+
n: "\n",
|
|
2647
|
+
r: "\r",
|
|
2648
|
+
t: " "
|
|
2649
|
+
};
|
|
2650
|
+
return escapes[ch] || ch;
|
|
2651
|
+
});
|
|
2652
|
+
}
|
|
2653
|
+
}
|
|
2438
2654
|
function extractFacebookMediaFromHtml(html, username) {
|
|
2439
2655
|
const items = [];
|
|
2440
2656
|
const seenUrls = /* @__PURE__ */ new Set();
|
|
@@ -2442,40 +2658,34 @@ function extractFacebookMediaFromHtml(html, username) {
|
|
|
2442
2658
|
const sdMatch = html.match(/"browser_native_sd_url":"([^"]+)"/);
|
|
2443
2659
|
const rawVideoUrl = hdMatch?.[1] || sdMatch?.[1];
|
|
2444
2660
|
if (rawVideoUrl) {
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
});
|
|
2456
|
-
}
|
|
2457
|
-
} catch {
|
|
2661
|
+
const videoUrl = decodeEscapedUrl(rawVideoUrl);
|
|
2662
|
+
if (!seenUrls.has(videoUrl)) {
|
|
2663
|
+
seenUrls.add(videoUrl);
|
|
2664
|
+
items.push({
|
|
2665
|
+
id: `fb_video_${Date.now()}`,
|
|
2666
|
+
url: videoUrl,
|
|
2667
|
+
filename: `${username}_video_${Date.now()}`,
|
|
2668
|
+
extension: "mp4",
|
|
2669
|
+
type: "video"
|
|
2670
|
+
});
|
|
2458
2671
|
}
|
|
2459
2672
|
}
|
|
2460
2673
|
const photoRegex = /,"image":\{"uri":"([^"]+)"/g;
|
|
2461
2674
|
let match;
|
|
2462
2675
|
let photoIndex = 1;
|
|
2463
2676
|
while ((match = photoRegex.exec(html)) !== null) {
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
photoIndex++;
|
|
2477
|
-
}
|
|
2478
|
-
} catch {
|
|
2677
|
+
const rawUrl = match[1];
|
|
2678
|
+
const photoUrl = decodeEscapedUrl(rawUrl);
|
|
2679
|
+
if (!seenUrls.has(photoUrl)) {
|
|
2680
|
+
seenUrls.add(photoUrl);
|
|
2681
|
+
items.push({
|
|
2682
|
+
id: `fb_photo_${photoIndex}`,
|
|
2683
|
+
url: photoUrl,
|
|
2684
|
+
filename: `${username}_photo_${photoIndex}`,
|
|
2685
|
+
extension: "jpg",
|
|
2686
|
+
type: "photo"
|
|
2687
|
+
});
|
|
2688
|
+
photoIndex++;
|
|
2479
2689
|
}
|
|
2480
2690
|
}
|
|
2481
2691
|
if (items.length === 0) {
|
|
@@ -2518,7 +2728,9 @@ var FacebookExtractor = class {
|
|
|
2518
2728
|
const html = await this.httpClient.fetchText(targetUrl, {
|
|
2519
2729
|
cookiePath: options.cookiePath,
|
|
2520
2730
|
cookieDomain: "facebook.com",
|
|
2521
|
-
signal: options.signal
|
|
2731
|
+
signal: options.signal,
|
|
2732
|
+
retries: 0,
|
|
2733
|
+
timeoutMs: 4e3
|
|
2522
2734
|
});
|
|
2523
2735
|
const items = extractFacebookMediaFromHtml(html, username);
|
|
2524
2736
|
let found = 0;
|
|
@@ -2595,90 +2807,137 @@ function sanitizeFilename(rawFilename, fallbackId, ext) {
|
|
|
2595
2807
|
async function downloadMediaStream(items, options) {
|
|
2596
2808
|
const { jobName, destDir, signal, onProgress } = options;
|
|
2597
2809
|
const httpClient = options.httpClient ?? new HttpClient();
|
|
2810
|
+
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2598
2811
|
await fsPromises2.mkdir(destDir, { recursive: true });
|
|
2599
2812
|
const archive = await loadArchive(destDir);
|
|
2600
2813
|
let downloaded = 0;
|
|
2601
2814
|
let skipped = 0;
|
|
2602
2815
|
const errors = [];
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
const
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
downloadedCount: downloaded,
|
|
2615
|
-
skippedCount: skipped,
|
|
2616
|
-
currentFile: finalFilename,
|
|
2617
|
-
statusText: `Skipping already downloaded: ${finalFilename}`
|
|
2618
|
-
});
|
|
2619
|
-
continue;
|
|
2620
|
-
}
|
|
2621
|
-
onProgress?.({
|
|
2622
|
-
job: jobName,
|
|
2623
|
-
downloadedCount: downloaded,
|
|
2624
|
-
skippedCount: skipped,
|
|
2625
|
-
currentFile: finalFilename,
|
|
2626
|
-
statusText: `Downloading: ${finalFilename}`
|
|
2627
|
-
});
|
|
2628
|
-
const tmpPath = `${finalPath}.download`;
|
|
2629
|
-
try {
|
|
2630
|
-
const reqHeaders = {};
|
|
2631
|
-
if (options.referer) {
|
|
2632
|
-
reqHeaders["Referer"] = options.referer;
|
|
2633
|
-
}
|
|
2634
|
-
const response = await httpClient.request(item.url, {
|
|
2635
|
-
signal,
|
|
2636
|
-
cookiePath: options.cookiePath,
|
|
2637
|
-
headers: reqHeaders
|
|
2638
|
-
});
|
|
2639
|
-
if (!response.ok || !response.body) {
|
|
2640
|
-
throw new Error(`HTTP ${response.status} (${response.statusText}) for ${item.url}`);
|
|
2816
|
+
const iterator = items[Symbol.asyncIterator]();
|
|
2817
|
+
let queueLock = Promise.resolve();
|
|
2818
|
+
let isDone = false;
|
|
2819
|
+
let consecutiveErrors = 0;
|
|
2820
|
+
const MAX_CONSECUTIVE_ERRORS = 5;
|
|
2821
|
+
function getNextItem() {
|
|
2822
|
+
const nextItem = queueLock.then(() => iterator.next());
|
|
2823
|
+
queueLock = nextItem.then(
|
|
2824
|
+
() => {
|
|
2825
|
+
},
|
|
2826
|
+
() => {
|
|
2641
2827
|
}
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2828
|
+
);
|
|
2829
|
+
return nextItem;
|
|
2830
|
+
}
|
|
2831
|
+
async function downloadWorker() {
|
|
2832
|
+
while (!isDone && !signal?.aborted) {
|
|
2833
|
+
let item;
|
|
2834
|
+
try {
|
|
2835
|
+
const next = await getNextItem();
|
|
2836
|
+
if (next.done) {
|
|
2837
|
+
isDone = true;
|
|
2838
|
+
break;
|
|
2649
2839
|
}
|
|
2840
|
+
consecutiveErrors = 0;
|
|
2841
|
+
item = next.value;
|
|
2842
|
+
} catch (err) {
|
|
2843
|
+
consecutiveErrors++;
|
|
2844
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2845
|
+
errors.push(`Extraction stream error: ${errMsg}`);
|
|
2846
|
+
onProgress?.({
|
|
2847
|
+
job: jobName,
|
|
2848
|
+
downloadedCount: downloaded,
|
|
2849
|
+
skippedCount: skipped,
|
|
2850
|
+
currentFile: "",
|
|
2851
|
+
statusText: `Extractor glitch: ${errMsg}`
|
|
2852
|
+
});
|
|
2853
|
+
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS || signal?.aborted) {
|
|
2854
|
+
isDone = true;
|
|
2855
|
+
break;
|
|
2856
|
+
}
|
|
2857
|
+
continue;
|
|
2858
|
+
}
|
|
2859
|
+
const ext = item.extension.replace(/^\./, "") || (item.type === "video" ? "mp4" : "jpg");
|
|
2860
|
+
const finalFilename = sanitizeFilename(item.filename, item.id, ext);
|
|
2861
|
+
const finalPath = path5.join(destDir, finalFilename);
|
|
2862
|
+
if (archive.has(item.id) && fs6.existsSync(finalPath)) {
|
|
2863
|
+
skipped++;
|
|
2864
|
+
onProgress?.({
|
|
2865
|
+
job: jobName,
|
|
2866
|
+
downloadedCount: downloaded,
|
|
2867
|
+
skippedCount: skipped,
|
|
2868
|
+
currentFile: finalFilename,
|
|
2869
|
+
statusText: `Skipping already downloaded: ${finalFilename}`
|
|
2870
|
+
});
|
|
2871
|
+
continue;
|
|
2650
2872
|
}
|
|
2651
|
-
await fsPromises2.rename(tmpPath, finalPath);
|
|
2652
|
-
archive.add(item.id);
|
|
2653
|
-
downloaded++;
|
|
2654
2873
|
onProgress?.({
|
|
2655
2874
|
job: jobName,
|
|
2656
2875
|
downloadedCount: downloaded,
|
|
2657
2876
|
skippedCount: skipped,
|
|
2658
2877
|
currentFile: finalFilename,
|
|
2659
|
-
statusText: `
|
|
2878
|
+
statusText: `Downloading: ${finalFilename}`
|
|
2660
2879
|
});
|
|
2661
|
-
|
|
2880
|
+
const tmpPath = `${finalPath}.download`;
|
|
2662
2881
|
try {
|
|
2663
|
-
|
|
2664
|
-
|
|
2882
|
+
const reqHeaders = {};
|
|
2883
|
+
if (options.referer) {
|
|
2884
|
+
reqHeaders["Referer"] = options.referer;
|
|
2665
2885
|
}
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2886
|
+
const response = await httpClient.request(item.url, {
|
|
2887
|
+
signal,
|
|
2888
|
+
cookiePath: options.cookiePath,
|
|
2889
|
+
headers: reqHeaders
|
|
2890
|
+
});
|
|
2891
|
+
if (!response.ok || !response.body) {
|
|
2892
|
+
try {
|
|
2893
|
+
await response.body?.cancel();
|
|
2894
|
+
} catch {
|
|
2895
|
+
}
|
|
2896
|
+
throw new Error(`HTTP ${response.status} (${response.statusText}) for ${item.url}`);
|
|
2897
|
+
}
|
|
2898
|
+
const fileStream = fs6.createWriteStream(tmpPath);
|
|
2899
|
+
await pipeline(Readable.fromWeb(response.body), fileStream, { signal });
|
|
2900
|
+
if (item.date) {
|
|
2901
|
+
try {
|
|
2902
|
+
const mtime = new Date(item.date * 1e3);
|
|
2903
|
+
await fsPromises2.utimes(tmpPath, mtime, mtime);
|
|
2904
|
+
} catch {
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
await fsPromises2.rename(tmpPath, finalPath);
|
|
2908
|
+
archive.add(item.id);
|
|
2909
|
+
downloaded++;
|
|
2910
|
+
onProgress?.({
|
|
2911
|
+
job: jobName,
|
|
2912
|
+
downloadedCount: downloaded,
|
|
2913
|
+
skippedCount: skipped,
|
|
2914
|
+
currentFile: finalFilename,
|
|
2915
|
+
statusText: `Saved: ${finalFilename}`
|
|
2916
|
+
});
|
|
2917
|
+
} catch (err) {
|
|
2918
|
+
try {
|
|
2919
|
+
if (fs6.existsSync(tmpPath)) {
|
|
2920
|
+
await fsPromises2.unlink(tmpPath);
|
|
2921
|
+
}
|
|
2922
|
+
} catch {
|
|
2923
|
+
}
|
|
2924
|
+
if (signal?.aborted) {
|
|
2925
|
+
break;
|
|
2926
|
+
}
|
|
2927
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2928
|
+
errors.push(`${finalFilename}: ${errMsg}`);
|
|
2929
|
+
onProgress?.({
|
|
2930
|
+
job: jobName,
|
|
2931
|
+
downloadedCount: downloaded,
|
|
2932
|
+
skippedCount: skipped,
|
|
2933
|
+
currentFile: finalFilename,
|
|
2934
|
+
statusText: `Failed: ${finalFilename} (${errMsg})`
|
|
2935
|
+
});
|
|
2670
2936
|
}
|
|
2671
|
-
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2672
|
-
errors.push(`${finalFilename}: ${errMsg}`);
|
|
2673
|
-
onProgress?.({
|
|
2674
|
-
job: jobName,
|
|
2675
|
-
downloadedCount: downloaded,
|
|
2676
|
-
skippedCount: skipped,
|
|
2677
|
-
currentFile: finalFilename,
|
|
2678
|
-
statusText: `Failed: ${finalFilename} (${errMsg})`
|
|
2679
|
-
});
|
|
2680
2937
|
}
|
|
2681
2938
|
}
|
|
2939
|
+
const workers = Array.from({ length: concurrency }, () => downloadWorker());
|
|
2940
|
+
await Promise.all(workers);
|
|
2682
2941
|
await saveArchive(destDir, archive);
|
|
2683
2942
|
return { downloaded, skipped, errors };
|
|
2684
2943
|
}
|
|
@@ -2989,8 +3248,8 @@ var GDL_ARCHIVE = "archive_gallery-dl.sqlite3";
|
|
|
2989
3248
|
var UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
2990
3249
|
var PLATFORM_SLEEP = {
|
|
2991
3250
|
tiktok: "0.6-1.2",
|
|
2992
|
-
instagram: "
|
|
2993
|
-
facebook: "1.
|
|
3251
|
+
instagram: "1.0-1.8",
|
|
3252
|
+
facebook: "1.0-1.8"
|
|
2994
3253
|
};
|
|
2995
3254
|
function buildGalleryDlArgs(options) {
|
|
2996
3255
|
const { destDir, subDir, platform, cleanUrl, username, cookiePath } = options;
|
|
@@ -3030,12 +3289,12 @@ function buildGalleryDlArgs(options) {
|
|
|
3030
3289
|
} else if (subDir === "Stories") {
|
|
3031
3290
|
if (platform === "instagram") {
|
|
3032
3291
|
targetUrl = `https://www.instagram.com/stories/${encodeURIComponent(username)}/`;
|
|
3033
|
-
args.push("--sleep-request", "
|
|
3292
|
+
args.push("--sleep-request", "1.0-1.8");
|
|
3034
3293
|
}
|
|
3035
3294
|
} else if (subDir === "Highlights") {
|
|
3036
3295
|
if (platform === "instagram") {
|
|
3037
3296
|
targetUrl = `https://www.instagram.com/${encodeURIComponent(username)}/highlights/`;
|
|
3038
|
-
args.push("--sleep-request", "
|
|
3297
|
+
args.push("--sleep-request", "1.0-1.8");
|
|
3039
3298
|
}
|
|
3040
3299
|
}
|
|
3041
3300
|
if (cookiePath) {
|
|
@@ -3313,14 +3572,14 @@ function resolveDownloadBaseDir(customBaseDir) {
|
|
|
3313
3572
|
if (customBaseDir && customBaseDir.trim()) {
|
|
3314
3573
|
return path10.resolve(expandPath(customBaseDir.trim()));
|
|
3315
3574
|
}
|
|
3316
|
-
const configDir = loadConfig().downloadDir;
|
|
3317
|
-
if (configDir && configDir.trim()) {
|
|
3318
|
-
return path10.resolve(expandPath(configDir.trim()));
|
|
3319
|
-
}
|
|
3320
3575
|
const envDir = process.env.OPEN_NAMI_DIR || process.env.NAMI_DIR;
|
|
3321
3576
|
if (envDir && envDir.trim()) {
|
|
3322
3577
|
return path10.resolve(expandPath(envDir.trim()));
|
|
3323
3578
|
}
|
|
3579
|
+
const configDir = loadConfig().downloadDir;
|
|
3580
|
+
if (configDir && configDir.trim()) {
|
|
3581
|
+
return path10.resolve(expandPath(configDir.trim()));
|
|
3582
|
+
}
|
|
3324
3583
|
if (process.env.NAMI_BASE_DIR && process.env.NAMI_BASE_DIR.trim()) {
|
|
3325
3584
|
const raw = process.env.NAMI_BASE_DIR.trim();
|
|
3326
3585
|
return path10.resolve(expandPath(raw.endsWith("downloads") ? raw : path10.join(raw, "downloads")));
|