open-nami 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli.js +1520 -76
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -9,7 +9,7 @@ var require_package = __commonJS({
|
|
|
9
9
|
"package.json"(exports, module) {
|
|
10
10
|
module.exports = {
|
|
11
11
|
name: "open-nami",
|
|
12
|
-
version: "1.0.
|
|
12
|
+
version: "1.0.4",
|
|
13
13
|
description: "Open Nami \u2014 terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
|
|
14
14
|
type: "module",
|
|
15
15
|
bin: {
|
|
@@ -1044,9 +1044,9 @@ import fsPromises from "fs/promises";
|
|
|
1044
1044
|
import path2 from "path";
|
|
1045
1045
|
import os4 from "os";
|
|
1046
1046
|
var CANDIDATE_DIRS = [
|
|
1047
|
-
path2.join(process.cwd(), "cookies"),
|
|
1048
1047
|
path2.join(os4.homedir(), ".open-nami", "cookies"),
|
|
1049
|
-
path2.join(os4.homedir(), ".nami", "cookies")
|
|
1048
|
+
path2.join(os4.homedir(), ".nami", "cookies"),
|
|
1049
|
+
path2.join(process.cwd(), "cookies")
|
|
1050
1050
|
];
|
|
1051
1051
|
function getCandidateDirs(customDir) {
|
|
1052
1052
|
const dirs = [];
|
|
@@ -1387,6 +1387,22 @@ function parseProfileInput(input) {
|
|
|
1387
1387
|
}
|
|
1388
1388
|
}
|
|
1389
1389
|
if (host.includes("x.com") || host.includes("twitter.com")) {
|
|
1390
|
+
if (pathParts[0]?.toLowerCase() === "i" && pathParts[1]?.toLowerCase() === "history" && pathParts[2]?.toLowerCase() === "likes") {
|
|
1391
|
+
return {
|
|
1392
|
+
platform: "x",
|
|
1393
|
+
username: "i_history_likes",
|
|
1394
|
+
cleanUrl: "https://x.com/i/history/likes",
|
|
1395
|
+
rawInput: input
|
|
1396
|
+
};
|
|
1397
|
+
}
|
|
1398
|
+
if (pathParts[0]?.toLowerCase() === "i" && pathParts[1]?.toLowerCase() === "bookmarks") {
|
|
1399
|
+
return {
|
|
1400
|
+
platform: "x",
|
|
1401
|
+
username: "i_bookmarks",
|
|
1402
|
+
cleanUrl: "https://x.com/i/bookmarks",
|
|
1403
|
+
rawInput: input
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1390
1406
|
const firstPart = pathParts[0] || "";
|
|
1391
1407
|
const reserved = ["home", "explore", "notifications", "messages", "i", "settings"];
|
|
1392
1408
|
if (firstPart && !reserved.includes(firstPart.toLowerCase())) {
|
|
@@ -1420,9 +1436,9 @@ function parseProfileInput(input) {
|
|
|
1420
1436
|
}
|
|
1421
1437
|
|
|
1422
1438
|
// src/lib/engines/dispatcher.ts
|
|
1423
|
-
import
|
|
1439
|
+
import path10 from "path";
|
|
1424
1440
|
import os8 from "os";
|
|
1425
|
-
import
|
|
1441
|
+
import fs11 from "fs/promises";
|
|
1426
1442
|
|
|
1427
1443
|
// src/lib/known-folders.ts
|
|
1428
1444
|
import { execFileSync } from "child_process";
|
|
@@ -1559,29 +1575,1396 @@ function resolvePlatformDownloadsDir(options = {}) {
|
|
|
1559
1575
|
return defaultFallback;
|
|
1560
1576
|
}
|
|
1561
1577
|
|
|
1578
|
+
// src/lib/engines/extractors/http.ts
|
|
1579
|
+
import fs5 from "fs/promises";
|
|
1580
|
+
import { setTimeout as setTimeoutPromise } from "timers/promises";
|
|
1581
|
+
var DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
1582
|
+
var DEFAULT_CHROME_HEADERS = {
|
|
1583
|
+
"User-Agent": DEFAULT_USER_AGENT,
|
|
1584
|
+
"sec-ch-ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
|
1585
|
+
"sec-ch-ua-mobile": "?0",
|
|
1586
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
1587
|
+
"Accept": "*/*",
|
|
1588
|
+
"Accept-Language": "en-US,en;q=0.9"
|
|
1589
|
+
};
|
|
1590
|
+
function parseNetscapeCookies(content, filterDomain) {
|
|
1591
|
+
const cookies = {};
|
|
1592
|
+
const lines = content.split(/\r?\n/);
|
|
1593
|
+
for (const rawLine of lines) {
|
|
1594
|
+
let line = rawLine.trim();
|
|
1595
|
+
if (!line) continue;
|
|
1596
|
+
if (line.startsWith("#HttpOnly_")) {
|
|
1597
|
+
line = line.slice("#HttpOnly_".length);
|
|
1598
|
+
} else if (line.startsWith("#")) {
|
|
1599
|
+
continue;
|
|
1600
|
+
}
|
|
1601
|
+
const parts = line.split(" ");
|
|
1602
|
+
if (parts.length >= 7) {
|
|
1603
|
+
const domain = parts[0].toLowerCase();
|
|
1604
|
+
const name = parts[5];
|
|
1605
|
+
const value = parts[6];
|
|
1606
|
+
if (filterDomain) {
|
|
1607
|
+
const cleanDomain = filterDomain.toLowerCase().replace(/^\./, "");
|
|
1608
|
+
const lineDomain = domain.replace(/^\./, "");
|
|
1609
|
+
if (!lineDomain.endsWith(cleanDomain) && !cleanDomain.endsWith(lineDomain)) {
|
|
1610
|
+
continue;
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
if (name) {
|
|
1614
|
+
cookies[name] = value;
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
return cookies;
|
|
1619
|
+
}
|
|
1620
|
+
async function loadCookiesFromPath(filePath, filterDomain) {
|
|
1621
|
+
try {
|
|
1622
|
+
const content = await fs5.readFile(filePath, "utf-8");
|
|
1623
|
+
return parseNetscapeCookies(content, filterDomain);
|
|
1624
|
+
} catch {
|
|
1625
|
+
return {};
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
function formatCookieHeader(cookies) {
|
|
1629
|
+
return Object.entries(cookies).map(([k, v]) => `${k}=${v}`).join("; ");
|
|
1630
|
+
}
|
|
1631
|
+
async function sleep(minMs, maxMs, signal) {
|
|
1632
|
+
const ms = maxMs !== void 0 ? Math.floor(minMs + Math.random() * (maxMs - minMs)) : minMs;
|
|
1633
|
+
if (ms <= 0) return;
|
|
1634
|
+
if (signal?.aborted) {
|
|
1635
|
+
throw signal.reason ?? new Error("Aborted");
|
|
1636
|
+
}
|
|
1637
|
+
try {
|
|
1638
|
+
await setTimeoutPromise(ms, void 0, { signal });
|
|
1639
|
+
} catch (err) {
|
|
1640
|
+
if (signal?.aborted && signal.reason) {
|
|
1641
|
+
throw signal.reason;
|
|
1642
|
+
}
|
|
1643
|
+
throw err;
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
function combineTimeoutSignal(signal, timeoutMs) {
|
|
1647
|
+
if (!timeoutMs || timeoutMs <= 0) return signal;
|
|
1648
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
1649
|
+
return signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
|
1650
|
+
}
|
|
1651
|
+
var HttpClient = class {
|
|
1652
|
+
defaultCookies = {};
|
|
1653
|
+
defaultHeaders;
|
|
1654
|
+
cookieCache = /* @__PURE__ */ new Map();
|
|
1655
|
+
constructor(headers, cookies) {
|
|
1656
|
+
this.defaultHeaders = { ...DEFAULT_CHROME_HEADERS, ...headers };
|
|
1657
|
+
if (cookies) {
|
|
1658
|
+
this.defaultCookies = { ...cookies };
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
1661
|
+
setCookie(name, value) {
|
|
1662
|
+
this.defaultCookies[name] = value;
|
|
1663
|
+
}
|
|
1664
|
+
setHeader(name, value) {
|
|
1665
|
+
this.defaultHeaders[name] = value;
|
|
1666
|
+
}
|
|
1667
|
+
clearCookieCache() {
|
|
1668
|
+
this.cookieCache.clear();
|
|
1669
|
+
}
|
|
1670
|
+
async getCachedCookies(filePath, domain) {
|
|
1671
|
+
const cacheKey = `${filePath}::${domain || ""}`;
|
|
1672
|
+
const cached = this.cookieCache.get(cacheKey);
|
|
1673
|
+
if (cached) {
|
|
1674
|
+
return cached;
|
|
1675
|
+
}
|
|
1676
|
+
const loaded = await loadCookiesFromPath(filePath, domain);
|
|
1677
|
+
this.cookieCache.set(cacheKey, loaded);
|
|
1678
|
+
return loaded;
|
|
1679
|
+
}
|
|
1680
|
+
async loadCookies(filePath, domain) {
|
|
1681
|
+
const loaded = await this.getCachedCookies(filePath, domain);
|
|
1682
|
+
Object.assign(this.defaultCookies, loaded);
|
|
1683
|
+
}
|
|
1684
|
+
async request(url, options = {}) {
|
|
1685
|
+
const retries = options.retries ?? 1;
|
|
1686
|
+
const retryDelay = options.retryDelayMs ?? 1e3;
|
|
1687
|
+
let cookieHeader = "";
|
|
1688
|
+
if (options.cookiePath) {
|
|
1689
|
+
const fromPath = await this.getCachedCookies(options.cookiePath, options.cookieDomain);
|
|
1690
|
+
const merged = { ...this.defaultCookies, ...fromPath };
|
|
1691
|
+
cookieHeader = formatCookieHeader(merged);
|
|
1692
|
+
} else if (typeof options.cookies === "string") {
|
|
1693
|
+
cookieHeader = options.cookies;
|
|
1694
|
+
} else if (options.cookies) {
|
|
1695
|
+
const merged = { ...this.defaultCookies, ...options.cookies };
|
|
1696
|
+
cookieHeader = formatCookieHeader(merged);
|
|
1697
|
+
} else if (Object.keys(this.defaultCookies).length > 0) {
|
|
1698
|
+
cookieHeader = formatCookieHeader(this.defaultCookies);
|
|
1699
|
+
}
|
|
1700
|
+
const headers = {
|
|
1701
|
+
...this.defaultHeaders,
|
|
1702
|
+
...options.headers
|
|
1703
|
+
};
|
|
1704
|
+
if (cookieHeader) {
|
|
1705
|
+
headers["Cookie"] = cookieHeader;
|
|
1706
|
+
}
|
|
1707
|
+
let lastError;
|
|
1708
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
1709
|
+
if (options.signal?.aborted) {
|
|
1710
|
+
throw options.signal.reason ?? new Error("Aborted");
|
|
1711
|
+
}
|
|
1712
|
+
const requestSignal = combineTimeoutSignal(options.signal || void 0, options.timeoutMs);
|
|
1713
|
+
try {
|
|
1714
|
+
const response = await fetch(url, {
|
|
1715
|
+
...options,
|
|
1716
|
+
signal: requestSignal,
|
|
1717
|
+
headers
|
|
1718
|
+
});
|
|
1719
|
+
if (response.status === 429) {
|
|
1720
|
+
if (attempt < retries) {
|
|
1721
|
+
try {
|
|
1722
|
+
await response.body?.cancel();
|
|
1723
|
+
} catch {
|
|
1724
|
+
}
|
|
1725
|
+
const backoffMs = Math.min(retryDelay * (attempt + 1), 2500);
|
|
1726
|
+
await sleep(backoffMs, void 0, options.signal || void 0);
|
|
1727
|
+
continue;
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
if (response.status >= 500 && attempt < retries) {
|
|
1731
|
+
try {
|
|
1732
|
+
await response.body?.cancel();
|
|
1733
|
+
} catch {
|
|
1734
|
+
}
|
|
1735
|
+
const backoffMs = Math.min(retryDelay * (attempt + 1), 2e3);
|
|
1736
|
+
await sleep(backoffMs, void 0, options.signal || void 0);
|
|
1737
|
+
continue;
|
|
1738
|
+
}
|
|
1739
|
+
return response;
|
|
1740
|
+
} catch (err) {
|
|
1741
|
+
lastError = err;
|
|
1742
|
+
if (options.signal?.aborted) throw err;
|
|
1743
|
+
if (attempt < retries) {
|
|
1744
|
+
await sleep(retryDelay * (attempt + 1), void 0, options.signal || void 0);
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
throw lastError ?? new Error(`Request failed after ${retries} attempts: ${url}`);
|
|
1749
|
+
}
|
|
1750
|
+
async fetchJson(url, options = {}) {
|
|
1751
|
+
const res = await this.request(url, {
|
|
1752
|
+
...options,
|
|
1753
|
+
headers: {
|
|
1754
|
+
"Accept": "application/json, text/plain, */*",
|
|
1755
|
+
...options.headers
|
|
1756
|
+
}
|
|
1757
|
+
});
|
|
1758
|
+
if (!res.ok) {
|
|
1759
|
+
try {
|
|
1760
|
+
await res.body?.cancel();
|
|
1761
|
+
} catch {
|
|
1762
|
+
}
|
|
1763
|
+
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1764
|
+
}
|
|
1765
|
+
return await res.json();
|
|
1766
|
+
}
|
|
1767
|
+
async fetchText(url, options = {}) {
|
|
1768
|
+
const res = await this.request(url, options);
|
|
1769
|
+
if (!res.ok) {
|
|
1770
|
+
try {
|
|
1771
|
+
await res.body?.cancel();
|
|
1772
|
+
} catch {
|
|
1773
|
+
}
|
|
1774
|
+
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1775
|
+
}
|
|
1776
|
+
return await res.text();
|
|
1777
|
+
}
|
|
1778
|
+
};
|
|
1779
|
+
|
|
1780
|
+
// src/lib/engines/extractors/instagram.ts
|
|
1781
|
+
var IG_APP_ID = "936619743392459";
|
|
1782
|
+
var IG_TIMELINE_QUERY_HASH = "69cba40317214236af40e7efa697781d";
|
|
1783
|
+
var IG_QUERIES = {
|
|
1784
|
+
PolarisProfilePostsTabContentQuery_connection: "28142289515441884",
|
|
1785
|
+
PolarisProfileReelsTabContentQuery_connection: "28989741540630190",
|
|
1786
|
+
PolarisProfileTaggedTabContentQuery_connection: "28794602806842110",
|
|
1787
|
+
PolarisProfileStoryHighlightsTrayContentQuery: "26970053832668570",
|
|
1788
|
+
PolarisStoriesV3ReelPageGalleryQuery: "28262315486766731",
|
|
1789
|
+
PolarisStoriesV3TrayContainerQuery: "27703822975903310",
|
|
1790
|
+
PolarisProfilePageContentQuery: "28036671149327607"
|
|
1791
|
+
};
|
|
1792
|
+
var IG_POLARIS_DOC_ID = IG_QUERIES.PolarisProfilePostsTabContentQuery_connection;
|
|
1793
|
+
function logExtractorDebug(endpoint, err) {
|
|
1794
|
+
if (process.env.DEBUG || process.env.OPEN_NAMI_DEBUG) {
|
|
1795
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1796
|
+
console.debug(`[open-nami:instagram] ${endpoint} query fallback: ${message}`);
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
function parseIgNodeToMediaItems(node, username) {
|
|
1800
|
+
const items = [];
|
|
1801
|
+
const anyNode = node;
|
|
1802
|
+
const baseId = node.shortcode || node.id || anyNode.pk;
|
|
1803
|
+
const date = node.taken_at_timestamp || anyNode.taken_at;
|
|
1804
|
+
const childrenEdges = node.edge_sidecar_to_children?.edges;
|
|
1805
|
+
const carouselMedia = anyNode.carousel_media;
|
|
1806
|
+
const childrenList = childrenEdges ? childrenEdges.map((e) => e.node) : Array.isArray(carouselMedia) ? carouselMedia : [];
|
|
1807
|
+
if (childrenList.length > 0) {
|
|
1808
|
+
childrenList.forEach((childNode, index) => {
|
|
1809
|
+
const subItems = parseIgNodeToMediaItems(childNode, username);
|
|
1810
|
+
for (const item of subItems) {
|
|
1811
|
+
item.id = `${baseId}_${index + 1}`;
|
|
1812
|
+
item.filename = `${username}_${date || baseId}_${index + 1}`;
|
|
1813
|
+
items.push(item);
|
|
1814
|
+
}
|
|
1815
|
+
});
|
|
1816
|
+
return items;
|
|
1817
|
+
}
|
|
1818
|
+
const isVideo = Boolean(node.is_video || node.video_url || node.video_versions && node.video_versions.length > 0);
|
|
1819
|
+
if (isVideo) {
|
|
1820
|
+
const video = node.video_versions?.[0];
|
|
1821
|
+
const videoUrl = node.video_url || video?.url;
|
|
1822
|
+
if (videoUrl) {
|
|
1823
|
+
items.push({
|
|
1824
|
+
id: baseId,
|
|
1825
|
+
url: videoUrl,
|
|
1826
|
+
filename: `${username}_${date || baseId}`,
|
|
1827
|
+
extension: "mp4",
|
|
1828
|
+
type: "video",
|
|
1829
|
+
date,
|
|
1830
|
+
width: video?.width ?? node.dimensions?.width,
|
|
1831
|
+
height: video?.height ?? node.dimensions?.height,
|
|
1832
|
+
thumbnailUrl: node.display_url
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
} else {
|
|
1836
|
+
const candidate = node.image_versions2?.candidates?.[0];
|
|
1837
|
+
const photoUrl = candidate?.url || node.display_url;
|
|
1838
|
+
if (photoUrl) {
|
|
1839
|
+
items.push({
|
|
1840
|
+
id: baseId,
|
|
1841
|
+
url: photoUrl,
|
|
1842
|
+
filename: `${username}_${date || baseId}`,
|
|
1843
|
+
extension: "jpg",
|
|
1844
|
+
type: "photo",
|
|
1845
|
+
date,
|
|
1846
|
+
width: candidate?.width ?? node.dimensions?.width,
|
|
1847
|
+
height: candidate?.height ?? node.dimensions?.height
|
|
1848
|
+
});
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
return items;
|
|
1852
|
+
}
|
|
1853
|
+
function parseReelItemToMediaItem(item, username) {
|
|
1854
|
+
const id = item.id || item.pk || String(Date.now());
|
|
1855
|
+
const date = item.taken_at || item.created_at;
|
|
1856
|
+
const isVideo = item.media_type === 2 || Boolean(item.video_versions && item.video_versions.length > 0);
|
|
1857
|
+
if (isVideo) {
|
|
1858
|
+
const video = item.video_versions?.[0];
|
|
1859
|
+
const videoUrl = video?.url;
|
|
1860
|
+
if (!videoUrl) return null;
|
|
1861
|
+
return {
|
|
1862
|
+
id,
|
|
1863
|
+
url: videoUrl,
|
|
1864
|
+
filename: `${username}_${date || id}`,
|
|
1865
|
+
extension: "mp4",
|
|
1866
|
+
type: "video",
|
|
1867
|
+
date,
|
|
1868
|
+
width: video?.width ?? item.original_width,
|
|
1869
|
+
height: video?.height ?? item.original_height,
|
|
1870
|
+
thumbnailUrl: item.image_versions2?.candidates?.[0]?.url
|
|
1871
|
+
};
|
|
1872
|
+
} else {
|
|
1873
|
+
const candidate = item.image_versions2?.candidates?.[0];
|
|
1874
|
+
const photoUrl = candidate?.url;
|
|
1875
|
+
if (!photoUrl) return null;
|
|
1876
|
+
return {
|
|
1877
|
+
id,
|
|
1878
|
+
url: photoUrl,
|
|
1879
|
+
filename: `${username}_${date || id}`,
|
|
1880
|
+
extension: "jpg",
|
|
1881
|
+
type: "photo",
|
|
1882
|
+
date,
|
|
1883
|
+
width: candidate?.width ?? item.original_width,
|
|
1884
|
+
height: candidate?.height ?? item.original_height
|
|
1885
|
+
};
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
var InstagramExtractor = class {
|
|
1889
|
+
platform = "instagram";
|
|
1890
|
+
httpClient;
|
|
1891
|
+
constructor(httpClient) {
|
|
1892
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
1893
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1894
|
+
"Referer": "https://www.instagram.com/"
|
|
1895
|
+
});
|
|
1896
|
+
}
|
|
1897
|
+
async fetchUserProfile(username, options) {
|
|
1898
|
+
const url = `https://www.instagram.com/api/v1/users/web_profile_info/?username=${encodeURIComponent(username)}`;
|
|
1899
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1900
|
+
cookiePath: options.cookiePath,
|
|
1901
|
+
cookieDomain: "instagram.com",
|
|
1902
|
+
signal: options.signal,
|
|
1903
|
+
retries: 0,
|
|
1904
|
+
timeoutMs: 4e3,
|
|
1905
|
+
headers: {
|
|
1906
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1907
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
1908
|
+
}
|
|
1909
|
+
});
|
|
1910
|
+
if (!res.data?.user) {
|
|
1911
|
+
throw new Error(`Instagram user not found or private profile: ${username}`);
|
|
1912
|
+
}
|
|
1913
|
+
return res.data.user;
|
|
1914
|
+
}
|
|
1915
|
+
async fetchHighlightsTray(userId, options) {
|
|
1916
|
+
try {
|
|
1917
|
+
const variables = JSON.stringify({ user_id: userId });
|
|
1918
|
+
const body = new URLSearchParams({
|
|
1919
|
+
doc_id: IG_QUERIES.PolarisProfileStoryHighlightsTrayContentQuery,
|
|
1920
|
+
variables
|
|
1921
|
+
}).toString();
|
|
1922
|
+
const res = await this.httpClient.fetchJson("https://www.instagram.com/graphql/query", {
|
|
1923
|
+
method: "POST",
|
|
1924
|
+
body,
|
|
1925
|
+
cookiePath: options.cookiePath,
|
|
1926
|
+
cookieDomain: "instagram.com",
|
|
1927
|
+
signal: options.signal,
|
|
1928
|
+
headers: {
|
|
1929
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
1930
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1931
|
+
"X-FB-Friendly-Name": "PolarisProfileStoryHighlightsTrayContentQuery",
|
|
1932
|
+
"Referer": "https://www.instagram.com/"
|
|
1933
|
+
}
|
|
1934
|
+
});
|
|
1935
|
+
const edges = res?.data?.highlights?.edges;
|
|
1936
|
+
if (Array.isArray(edges) && edges.length > 0) {
|
|
1937
|
+
return edges.map((e) => e.node);
|
|
1938
|
+
}
|
|
1939
|
+
} catch (err) {
|
|
1940
|
+
logExtractorDebug("PolarisProfileStoryHighlightsTrayContentQuery", err);
|
|
1941
|
+
}
|
|
1942
|
+
const url = `https://www.instagram.com/api/v1/highlights/${userId}/highlights_tray/`;
|
|
1943
|
+
try {
|
|
1944
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1945
|
+
cookiePath: options.cookiePath,
|
|
1946
|
+
cookieDomain: "instagram.com",
|
|
1947
|
+
signal: options.signal,
|
|
1948
|
+
headers: {
|
|
1949
|
+
"X-IG-App-ID": IG_APP_ID
|
|
1950
|
+
}
|
|
1951
|
+
});
|
|
1952
|
+
return res.tray ?? [];
|
|
1953
|
+
} catch (err) {
|
|
1954
|
+
logExtractorDebug("Highlights legacy REST", err);
|
|
1955
|
+
return [];
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
async fetchReelsMedia(reelIds, options) {
|
|
1959
|
+
if (reelIds.length === 0) return [];
|
|
1960
|
+
const idsParam = reelIds.join(",");
|
|
1961
|
+
const url = `https://www.instagram.com/api/v1/feed/reels_media/?reel_ids=${encodeURIComponent(idsParam)}`;
|
|
1962
|
+
try {
|
|
1963
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1964
|
+
cookiePath: options.cookiePath,
|
|
1965
|
+
cookieDomain: "instagram.com",
|
|
1966
|
+
signal: options.signal,
|
|
1967
|
+
headers: {
|
|
1968
|
+
"X-IG-App-ID": IG_APP_ID
|
|
1969
|
+
}
|
|
1970
|
+
});
|
|
1971
|
+
if (res.reels_media) return res.reels_media;
|
|
1972
|
+
if (res.reels) return Object.values(res.reels);
|
|
1973
|
+
return [];
|
|
1974
|
+
} catch {
|
|
1975
|
+
return [];
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
async fetchNextPostsPage(userId, username, cursor, options) {
|
|
1979
|
+
const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
|
|
1980
|
+
const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
|
|
1981
|
+
try {
|
|
1982
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1983
|
+
cookiePath: options.cookiePath,
|
|
1984
|
+
cookieDomain: "instagram.com",
|
|
1985
|
+
signal: options.signal,
|
|
1986
|
+
headers: {
|
|
1987
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1988
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
1989
|
+
}
|
|
1990
|
+
});
|
|
1991
|
+
const data = res?.data?.user?.edge_owner_to_timeline_media;
|
|
1992
|
+
if (data?.edges && data.edges.length > 0) {
|
|
1993
|
+
return {
|
|
1994
|
+
edges: data.edges,
|
|
1995
|
+
hasNextPage: Boolean(data.page_info?.has_next_page),
|
|
1996
|
+
nextCursor: data.page_info?.end_cursor
|
|
1997
|
+
};
|
|
1998
|
+
}
|
|
1999
|
+
} catch (err) {
|
|
2000
|
+
logExtractorDebug("edge_owner_to_timeline_media", err);
|
|
2001
|
+
}
|
|
2002
|
+
try {
|
|
2003
|
+
const postVariables = JSON.stringify({
|
|
2004
|
+
after: cursor,
|
|
2005
|
+
first: 12,
|
|
2006
|
+
username,
|
|
2007
|
+
data: { count: 12 }
|
|
2008
|
+
});
|
|
2009
|
+
const postUrl = "https://www.instagram.com/graphql/query";
|
|
2010
|
+
const body = new URLSearchParams({
|
|
2011
|
+
doc_id: IG_POLARIS_DOC_ID,
|
|
2012
|
+
variables: postVariables
|
|
2013
|
+
}).toString();
|
|
2014
|
+
const res = await this.httpClient.fetchJson(postUrl, {
|
|
2015
|
+
method: "POST",
|
|
2016
|
+
body,
|
|
2017
|
+
cookiePath: options.cookiePath,
|
|
2018
|
+
cookieDomain: "instagram.com",
|
|
2019
|
+
signal: options.signal,
|
|
2020
|
+
headers: {
|
|
2021
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
2022
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2023
|
+
"X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
|
|
2024
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
2025
|
+
}
|
|
2026
|
+
});
|
|
2027
|
+
const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
|
|
2028
|
+
if (connection?.edges && connection.edges.length > 0) {
|
|
2029
|
+
return {
|
|
2030
|
+
edges: connection.edges,
|
|
2031
|
+
hasNextPage: Boolean(connection.page_info?.has_next_page),
|
|
2032
|
+
nextCursor: connection.page_info?.end_cursor
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
} catch (err) {
|
|
2036
|
+
logExtractorDebug("PolarisProfilePostsTabContentQuery_connection", err);
|
|
2037
|
+
}
|
|
2038
|
+
try {
|
|
2039
|
+
const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
|
|
2040
|
+
const res = await this.httpClient.fetchJson(restUrl, {
|
|
2041
|
+
cookiePath: options.cookiePath,
|
|
2042
|
+
cookieDomain: "instagram.com",
|
|
2043
|
+
signal: options.signal,
|
|
2044
|
+
headers: {
|
|
2045
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2046
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
2047
|
+
}
|
|
2048
|
+
});
|
|
2049
|
+
if (Array.isArray(res?.items) && res.items.length > 0) {
|
|
2050
|
+
const edges = res.items.map((item) => ({ node: item }));
|
|
2051
|
+
return {
|
|
2052
|
+
edges,
|
|
2053
|
+
hasNextPage: Boolean(res.more_available),
|
|
2054
|
+
nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
|
|
2055
|
+
};
|
|
2056
|
+
}
|
|
2057
|
+
} catch (err) {
|
|
2058
|
+
logExtractorDebug("v1 user feed REST", err);
|
|
2059
|
+
}
|
|
2060
|
+
return { edges: [], hasNextPage: false };
|
|
2061
|
+
}
|
|
2062
|
+
async fetchNextReelsPage(userId, cursor, options) {
|
|
2063
|
+
try {
|
|
2064
|
+
const variables = JSON.stringify({
|
|
2065
|
+
after: cursor,
|
|
2066
|
+
first: 12,
|
|
2067
|
+
id: userId,
|
|
2068
|
+
data: {
|
|
2069
|
+
include_feed_video: true,
|
|
2070
|
+
page_size: 12,
|
|
2071
|
+
target_user_id: userId
|
|
2072
|
+
}
|
|
2073
|
+
});
|
|
2074
|
+
const postUrl = "https://www.instagram.com/graphql/query";
|
|
2075
|
+
const body = new URLSearchParams({
|
|
2076
|
+
doc_id: IG_QUERIES.PolarisProfileReelsTabContentQuery_connection,
|
|
2077
|
+
variables
|
|
2078
|
+
}).toString();
|
|
2079
|
+
const res = await this.httpClient.fetchJson(postUrl, {
|
|
2080
|
+
method: "POST",
|
|
2081
|
+
body,
|
|
2082
|
+
cookiePath: options.cookiePath,
|
|
2083
|
+
cookieDomain: "instagram.com",
|
|
2084
|
+
signal: options.signal,
|
|
2085
|
+
headers: {
|
|
2086
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
2087
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
2088
|
+
"X-FB-Friendly-Name": "PolarisProfileReelsTabContentQuery_connection",
|
|
2089
|
+
"Referer": "https://www.instagram.com/"
|
|
2090
|
+
}
|
|
2091
|
+
});
|
|
2092
|
+
const reelsData = res?.data?.fetch__XDTUserDict;
|
|
2093
|
+
let connection = reelsData?.reels;
|
|
2094
|
+
if (!connection && reelsData) {
|
|
2095
|
+
for (const val of Object.values(reelsData)) {
|
|
2096
|
+
if (val && typeof val === "object" && "edges" in val) {
|
|
2097
|
+
connection = val;
|
|
2098
|
+
break;
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
if (connection?.edges && Array.isArray(connection.edges)) {
|
|
2103
|
+
return {
|
|
2104
|
+
edges: connection.edges,
|
|
2105
|
+
hasNextPage: Boolean(connection.page_info?.has_next_page),
|
|
2106
|
+
nextCursor: connection.page_info?.end_cursor
|
|
2107
|
+
};
|
|
2108
|
+
}
|
|
2109
|
+
} catch (err) {
|
|
2110
|
+
logExtractorDebug("PolarisProfileReelsTabContentQuery_connection", err);
|
|
2111
|
+
}
|
|
2112
|
+
return { edges: [], hasNextPage: false };
|
|
2113
|
+
}
|
|
2114
|
+
async *extract(profile, options) {
|
|
2115
|
+
const username = profile.username;
|
|
2116
|
+
const subDir = options.subDir;
|
|
2117
|
+
options.onProgress?.({
|
|
2118
|
+
stage: "Profile",
|
|
2119
|
+
found: 0,
|
|
2120
|
+
message: `Fetching Instagram profile @${username}...`
|
|
2121
|
+
});
|
|
2122
|
+
const user = await this.fetchUserProfile(username, options);
|
|
2123
|
+
const userId = user.id;
|
|
2124
|
+
if (subDir === "Stories") {
|
|
2125
|
+
options.onProgress?.({
|
|
2126
|
+
stage: "Stories",
|
|
2127
|
+
found: 0,
|
|
2128
|
+
message: `Fetching stories for @${username}...`
|
|
2129
|
+
});
|
|
2130
|
+
const reels = await this.fetchReelsMedia([userId], options);
|
|
2131
|
+
for (const reel of reels) {
|
|
2132
|
+
for (const rawItem of reel.items || []) {
|
|
2133
|
+
const item = parseReelItemToMediaItem(rawItem, username);
|
|
2134
|
+
if (item) yield item;
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
return;
|
|
2138
|
+
}
|
|
2139
|
+
if (subDir === "Highlights") {
|
|
2140
|
+
options.onProgress?.({
|
|
2141
|
+
stage: "Highlights",
|
|
2142
|
+
found: 0,
|
|
2143
|
+
message: `Fetching highlights tray for @${username}...`
|
|
2144
|
+
});
|
|
2145
|
+
const tray = await this.fetchHighlightsTray(userId, options);
|
|
2146
|
+
const highlightIds = tray.map((t) => t.id).filter(Boolean);
|
|
2147
|
+
if (highlightIds.length > 0) {
|
|
2148
|
+
const chunkSize = 5;
|
|
2149
|
+
for (let i = 0; i < highlightIds.length; i += chunkSize) {
|
|
2150
|
+
if (options.signal?.aborted) break;
|
|
2151
|
+
const chunk = highlightIds.slice(i, i + chunkSize);
|
|
2152
|
+
const reels = await this.fetchReelsMedia(chunk, options);
|
|
2153
|
+
for (const reel of reels) {
|
|
2154
|
+
for (const rawItem of reel.items || []) {
|
|
2155
|
+
const item = parseReelItemToMediaItem(rawItem, username);
|
|
2156
|
+
if (item) yield item;
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
await sleep(1e3, 1500, options.signal);
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
return;
|
|
2163
|
+
}
|
|
2164
|
+
let found = 0;
|
|
2165
|
+
const emittedItemIds = /* @__PURE__ */ new Set();
|
|
2166
|
+
let currentEdges = user.edge_owner_to_timeline_media?.edges ?? [];
|
|
2167
|
+
let hasNextPage = Boolean(user.edge_owner_to_timeline_media?.page_info?.has_next_page);
|
|
2168
|
+
let endCursor = user.edge_owner_to_timeline_media?.page_info?.end_cursor;
|
|
2169
|
+
while (true) {
|
|
2170
|
+
for (const edge of currentEdges) {
|
|
2171
|
+
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2172
|
+
for (const item of items) {
|
|
2173
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2174
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2175
|
+
if (emittedItemIds.has(item.id)) continue;
|
|
2176
|
+
emittedItemIds.add(item.id);
|
|
2177
|
+
found++;
|
|
2178
|
+
options.onProgress?.({
|
|
2179
|
+
stage: subDir,
|
|
2180
|
+
found,
|
|
2181
|
+
message: `Found item: ${item.filename}`
|
|
2182
|
+
});
|
|
2183
|
+
yield item;
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
if (!hasNextPage || !endCursor || options.signal?.aborted) {
|
|
2187
|
+
break;
|
|
2188
|
+
}
|
|
2189
|
+
await sleep(1500, 2500, options.signal);
|
|
2190
|
+
try {
|
|
2191
|
+
const nextPage = await this.fetchNextPostsPage(userId, username, endCursor, options);
|
|
2192
|
+
currentEdges = nextPage.edges;
|
|
2193
|
+
hasNextPage = nextPage.hasNextPage;
|
|
2194
|
+
endCursor = nextPage.nextCursor;
|
|
2195
|
+
if (currentEdges.length === 0) {
|
|
2196
|
+
break;
|
|
2197
|
+
}
|
|
2198
|
+
} catch {
|
|
2199
|
+
break;
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
if (subDir === "Videos") {
|
|
2203
|
+
const videoEdges = user.edge_felix_video_timeline?.edges ?? [];
|
|
2204
|
+
for (const edge of videoEdges) {
|
|
2205
|
+
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2206
|
+
for (const item of items) {
|
|
2207
|
+
if (item.type !== "video") continue;
|
|
2208
|
+
if (emittedItemIds.has(item.id)) continue;
|
|
2209
|
+
emittedItemIds.add(item.id);
|
|
2210
|
+
found++;
|
|
2211
|
+
options.onProgress?.({
|
|
2212
|
+
stage: "Videos",
|
|
2213
|
+
found,
|
|
2214
|
+
message: `Found reel: ${item.filename}`
|
|
2215
|
+
});
|
|
2216
|
+
yield item;
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
let reelsCursor = "";
|
|
2220
|
+
let hasMoreReels = true;
|
|
2221
|
+
while (hasMoreReels && !options.signal?.aborted) {
|
|
2222
|
+
try {
|
|
2223
|
+
const reelsPage = await this.fetchNextReelsPage(userId, reelsCursor, options);
|
|
2224
|
+
if (!reelsPage.edges || reelsPage.edges.length === 0) break;
|
|
2225
|
+
for (const edge of reelsPage.edges) {
|
|
2226
|
+
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2227
|
+
for (const item of items) {
|
|
2228
|
+
if (item.type !== "video") continue;
|
|
2229
|
+
if (emittedItemIds.has(item.id)) continue;
|
|
2230
|
+
emittedItemIds.add(item.id);
|
|
2231
|
+
found++;
|
|
2232
|
+
options.onProgress?.({
|
|
2233
|
+
stage: "Videos",
|
|
2234
|
+
found,
|
|
2235
|
+
message: `Found reel: ${item.filename}`
|
|
2236
|
+
});
|
|
2237
|
+
yield item;
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
if (reelsPage.hasNextPage && reelsPage.nextCursor && reelsPage.nextCursor !== reelsCursor) {
|
|
2241
|
+
reelsCursor = reelsPage.nextCursor;
|
|
2242
|
+
await sleep(1500, 2500, options.signal);
|
|
2243
|
+
} else {
|
|
2244
|
+
hasMoreReels = false;
|
|
2245
|
+
}
|
|
2246
|
+
} catch {
|
|
2247
|
+
break;
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
};
|
|
2253
|
+
|
|
2254
|
+
// src/lib/engines/extractors/x.ts
|
|
2255
|
+
var TWITTER_BEARER_TOKEN = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
|
2256
|
+
var TWITTER_FEATURES = {
|
|
2257
|
+
responsive_web_graphql_exclude_directive_enabled: true,
|
|
2258
|
+
verified_phone_label_enabled: false,
|
|
2259
|
+
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
2260
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
2261
|
+
tweetypie_unmention_optimization_enabled: true,
|
|
2262
|
+
vibe_api_enabled: true,
|
|
2263
|
+
responsive_web_edit_tweet_api_enabled: true,
|
|
2264
|
+
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
2265
|
+
view_counts_everywhere_api_enabled: true,
|
|
2266
|
+
longform_notetweets_consumption_enabled: true,
|
|
2267
|
+
tweet_awards_web_tipping_enabled: false,
|
|
2268
|
+
freedom_of_speech_not_reach_fetch_enabled: true,
|
|
2269
|
+
standardized_nudges_misinfo: true,
|
|
2270
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
2271
|
+
interactive_text_enabled: true,
|
|
2272
|
+
responsive_web_text_conversations_enabled: false,
|
|
2273
|
+
longform_notetweets_rich_text_read_enabled: true,
|
|
2274
|
+
longform_notetweets_inline_media_enabled: true,
|
|
2275
|
+
responsive_web_enhance_cards_enabled: false
|
|
2276
|
+
};
|
|
2277
|
+
function parseTweetMediaEntities(tweet, username) {
|
|
2278
|
+
const items = [];
|
|
2279
|
+
const tweetResult = tweet.tweet || tweet;
|
|
2280
|
+
const legacy = tweetResult.legacy || tweetResult;
|
|
2281
|
+
const entities = legacy.extended_entities?.media || legacy.entities?.media || [];
|
|
2282
|
+
const tweetId = tweetResult.rest_id || legacy.id_str || String(Date.now());
|
|
2283
|
+
const rawDate = legacy.created_at ? new Date(legacy.created_at).getTime() / 1e3 : void 0;
|
|
2284
|
+
for (let idx = 0; idx < entities.length; idx++) {
|
|
2285
|
+
const media = entities[idx];
|
|
2286
|
+
const subIndex = idx + 1;
|
|
2287
|
+
const baseFilename = `${username}_${tweetId}_${subIndex}`;
|
|
2288
|
+
if (media.type === "video" || media.type === "animated_gif") {
|
|
2289
|
+
const variants = (media.video_info?.variants || []).filter((v) => v.content_type === "video/mp4").sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0));
|
|
2290
|
+
const bestVariant = variants[0];
|
|
2291
|
+
if (bestVariant?.url) {
|
|
2292
|
+
items.push({
|
|
2293
|
+
id: `${tweetId}_${media.id_str}`,
|
|
2294
|
+
url: bestVariant.url,
|
|
2295
|
+
filename: baseFilename,
|
|
2296
|
+
extension: "mp4",
|
|
2297
|
+
type: "video",
|
|
2298
|
+
date: rawDate,
|
|
2299
|
+
thumbnailUrl: media.media_url_https
|
|
2300
|
+
});
|
|
2301
|
+
}
|
|
2302
|
+
} else {
|
|
2303
|
+
let photoUrl = media.media_url_https;
|
|
2304
|
+
if (photoUrl) {
|
|
2305
|
+
const cleanUrl = photoUrl.replace(/:[a-zA-Z]+$/, "");
|
|
2306
|
+
const urlWithOrig = cleanUrl.includes("?") ? `${cleanUrl}&name=orig` : `${cleanUrl}?name=orig`;
|
|
2307
|
+
const extMatch = photoUrl.match(/\.([a-zA-Z0-9]+)(?:$|\?)/);
|
|
2308
|
+
const ext = extMatch ? extMatch[1] : "jpg";
|
|
2309
|
+
items.push({
|
|
2310
|
+
id: `${tweetId}_${media.id_str}`,
|
|
2311
|
+
url: urlWithOrig,
|
|
2312
|
+
filename: baseFilename,
|
|
2313
|
+
extension: ext,
|
|
2314
|
+
type: "photo",
|
|
2315
|
+
date: rawDate,
|
|
2316
|
+
width: media.original_info?.width,
|
|
2317
|
+
height: media.original_info?.height
|
|
2318
|
+
});
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
return items;
|
|
2323
|
+
}
|
|
2324
|
+
var XExtractor = class {
|
|
2325
|
+
platform = "x";
|
|
2326
|
+
httpClient;
|
|
2327
|
+
guestToken;
|
|
2328
|
+
constructor(httpClient) {
|
|
2329
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
2330
|
+
"Authorization": `Bearer ${TWITTER_BEARER_TOKEN}`,
|
|
2331
|
+
"Referer": "https://x.com/",
|
|
2332
|
+
"x-twitter-active-user": "yes",
|
|
2333
|
+
"x-twitter-client-language": "en"
|
|
2334
|
+
});
|
|
2335
|
+
}
|
|
2336
|
+
async ensureGuestToken(options) {
|
|
2337
|
+
if (this.guestToken) return this.guestToken;
|
|
2338
|
+
try {
|
|
2339
|
+
const res = await this.httpClient.fetchJson(
|
|
2340
|
+
"https://api.twitter.com/1.1/guest/activate.json",
|
|
2341
|
+
{
|
|
2342
|
+
method: "POST",
|
|
2343
|
+
cookiePath: options.cookiePath,
|
|
2344
|
+
cookieDomain: "x.com",
|
|
2345
|
+
signal: options.signal,
|
|
2346
|
+
retries: 0,
|
|
2347
|
+
timeoutMs: 3e3,
|
|
2348
|
+
headers: {
|
|
2349
|
+
"Authorization": `Bearer ${TWITTER_BEARER_TOKEN}`
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
);
|
|
2353
|
+
if (res.guest_token) {
|
|
2354
|
+
this.guestToken = res.guest_token;
|
|
2355
|
+
this.httpClient.setHeader("x-guest-token", res.guest_token);
|
|
2356
|
+
return res.guest_token;
|
|
2357
|
+
}
|
|
2358
|
+
} catch (err) {
|
|
2359
|
+
if (options.signal?.aborted) throw err;
|
|
2360
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2361
|
+
throw new Error(`Failed to activate X/Twitter guest session: ${msg}`);
|
|
2362
|
+
}
|
|
2363
|
+
throw new Error("Failed to activate X/Twitter guest session: no guest token returned");
|
|
2364
|
+
}
|
|
2365
|
+
async fetchUserRestId(username, options) {
|
|
2366
|
+
if (username.startsWith("id:")) {
|
|
2367
|
+
return username.slice(3);
|
|
2368
|
+
}
|
|
2369
|
+
if (username === "i_history_likes") {
|
|
2370
|
+
if (options.cookiePath) {
|
|
2371
|
+
const cookies = await loadCookiesFromPath(options.cookiePath, "x.com");
|
|
2372
|
+
const twid = cookies["twid"];
|
|
2373
|
+
if (twid) {
|
|
2374
|
+
const decoded = decodeURIComponent(twid).replace(/^"|"$/g, "").trim();
|
|
2375
|
+
const rawId = decoded.startsWith("u=") ? decoded.slice(2) : decoded;
|
|
2376
|
+
if (rawId && /^[a-zA-Z0-9_-]+$/.test(rawId)) {
|
|
2377
|
+
return rawId;
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
throw new Error("Authentication required for X/Twitter likes history: missing 'twid' cookie");
|
|
2382
|
+
}
|
|
2383
|
+
await this.ensureGuestToken(options);
|
|
2384
|
+
const variables = JSON.stringify({
|
|
2385
|
+
screen_name: username,
|
|
2386
|
+
withSafetyModeUserFields: true
|
|
2387
|
+
});
|
|
2388
|
+
const features = JSON.stringify(TWITTER_FEATURES);
|
|
2389
|
+
const url = `https://x.com/i/api/graphql/ck5KkZ8t5cOmoLssopN99Q/UserByScreenName?variables=${encodeURIComponent(variables)}&features=${encodeURIComponent(features)}`;
|
|
2390
|
+
try {
|
|
2391
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
2392
|
+
cookiePath: options.cookiePath,
|
|
2393
|
+
cookieDomain: "x.com",
|
|
2394
|
+
signal: options.signal,
|
|
2395
|
+
retries: 0,
|
|
2396
|
+
timeoutMs: 4e3
|
|
2397
|
+
});
|
|
2398
|
+
const restId = res.data?.user?.result?.rest_id;
|
|
2399
|
+
if (restId) return restId;
|
|
2400
|
+
} catch {
|
|
2401
|
+
}
|
|
2402
|
+
const syndiUrl = `https://cdn.syndication.twimg.com/widgets/followbutton/info.json?screen_names=${encodeURIComponent(username)}`;
|
|
2403
|
+
try {
|
|
2404
|
+
const res = await this.httpClient.fetchJson(syndiUrl, {
|
|
2405
|
+
signal: options.signal,
|
|
2406
|
+
retries: 0,
|
|
2407
|
+
timeoutMs: 3e3
|
|
2408
|
+
});
|
|
2409
|
+
if (res[0]?.id) return String(res[0].id);
|
|
2410
|
+
} catch {
|
|
2411
|
+
}
|
|
2412
|
+
throw new Error(`Failed to resolve X/Twitter user ID for @${username}`);
|
|
2413
|
+
}
|
|
2414
|
+
async *extract(profile, options) {
|
|
2415
|
+
const username = profile.username;
|
|
2416
|
+
const subDir = options.subDir;
|
|
2417
|
+
const isLikes = username === "i_history_likes";
|
|
2418
|
+
options.onProgress?.({
|
|
2419
|
+
stage: isLikes ? "Likes" : "Profile",
|
|
2420
|
+
found: 0,
|
|
2421
|
+
message: isLikes ? "Fetching liked posts from X..." : `Fetching X profile @${username}...`
|
|
2422
|
+
});
|
|
2423
|
+
const userId = await this.fetchUserRestId(username, options);
|
|
2424
|
+
const endpoint = isLikes ? "https://x.com/i/api/graphql/TGEKkJG_meudeaFcqaxM-Q/Likes" : "https://x.com/i/api/graphql/jCRhbOzdgOHp6u9H4g2tEg/UserMedia";
|
|
2425
|
+
let cursor;
|
|
2426
|
+
let hasMore = true;
|
|
2427
|
+
let found = 0;
|
|
2428
|
+
while (hasMore) {
|
|
2429
|
+
if (options.signal?.aborted) break;
|
|
2430
|
+
const variables = {
|
|
2431
|
+
userId,
|
|
2432
|
+
count: 50,
|
|
2433
|
+
includePromotedContent: false,
|
|
2434
|
+
withClientEventToken: false,
|
|
2435
|
+
withBirdwatchNotes: false,
|
|
2436
|
+
withVoice: true
|
|
2437
|
+
};
|
|
2438
|
+
if (cursor) {
|
|
2439
|
+
variables.cursor = cursor;
|
|
2440
|
+
}
|
|
2441
|
+
const features = JSON.stringify(TWITTER_FEATURES);
|
|
2442
|
+
const url = `${endpoint}?variables=${encodeURIComponent(JSON.stringify(variables))}&features=${encodeURIComponent(features)}`;
|
|
2443
|
+
let res;
|
|
2444
|
+
try {
|
|
2445
|
+
res = await this.httpClient.fetchJson(url, {
|
|
2446
|
+
cookiePath: options.cookiePath,
|
|
2447
|
+
cookieDomain: "x.com",
|
|
2448
|
+
signal: options.signal
|
|
2449
|
+
});
|
|
2450
|
+
} catch (err) {
|
|
2451
|
+
break;
|
|
2452
|
+
}
|
|
2453
|
+
const instructions = res?.data?.user?.result?.timeline_v2?.timeline?.instructions || [];
|
|
2454
|
+
let newCursor;
|
|
2455
|
+
let pageItemCount = 0;
|
|
2456
|
+
for (const inst of instructions) {
|
|
2457
|
+
const entries = inst.entries || (inst.entry ? [inst.entry] : []);
|
|
2458
|
+
for (const entry of entries) {
|
|
2459
|
+
if (entry.entryId?.startsWith("cursor-bottom-")) {
|
|
2460
|
+
newCursor = entry.content?.value;
|
|
2461
|
+
continue;
|
|
2462
|
+
}
|
|
2463
|
+
const tweetContent = entry.content?.itemContent?.tweet_results?.result;
|
|
2464
|
+
if (tweetContent) {
|
|
2465
|
+
const mediaItems = parseTweetMediaEntities(tweetContent, username);
|
|
2466
|
+
for (const item of mediaItems) {
|
|
2467
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2468
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2469
|
+
found++;
|
|
2470
|
+
pageItemCount++;
|
|
2471
|
+
options.onProgress?.({
|
|
2472
|
+
stage: subDir,
|
|
2473
|
+
found,
|
|
2474
|
+
message: `Found item: ${item.filename}`
|
|
2475
|
+
});
|
|
2476
|
+
yield item;
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
}
|
|
2480
|
+
}
|
|
2481
|
+
if (!newCursor || newCursor === cursor || pageItemCount === 0) {
|
|
2482
|
+
hasMore = false;
|
|
2483
|
+
} else {
|
|
2484
|
+
cursor = newCursor;
|
|
2485
|
+
await sleep(1500, 2500, options.signal);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
};
|
|
2490
|
+
|
|
2491
|
+
// src/lib/engines/extractors/tiktok.ts
|
|
2492
|
+
function parseTikTokRehydrationJson(html) {
|
|
2493
|
+
const match = html.match(
|
|
2494
|
+
/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
|
|
2495
|
+
);
|
|
2496
|
+
if (!match) return null;
|
|
2497
|
+
try {
|
|
2498
|
+
const raw = JSON.parse(match[1]);
|
|
2499
|
+
return raw?.["__DEFAULT_SCOPE__"];
|
|
2500
|
+
} catch {
|
|
2501
|
+
return null;
|
|
2502
|
+
}
|
|
2503
|
+
}
|
|
2504
|
+
function parseTikTokItem(item, username) {
|
|
2505
|
+
const items = [];
|
|
2506
|
+
const id = item.id || item.itemId || String(Date.now());
|
|
2507
|
+
const date = Number(item.createTime) || void 0;
|
|
2508
|
+
const caption = item.desc;
|
|
2509
|
+
const imageList = item.imagePost?.images;
|
|
2510
|
+
if (Array.isArray(imageList) && imageList.length > 0) {
|
|
2511
|
+
imageList.forEach((img, idx) => {
|
|
2512
|
+
const imgUrl = img.imageURL?.urlList?.[0] || img.displayImage?.urlList?.[0];
|
|
2513
|
+
if (imgUrl) {
|
|
2514
|
+
items.push({
|
|
2515
|
+
id: `${id}_${idx + 1}`,
|
|
2516
|
+
url: imgUrl,
|
|
2517
|
+
filename: `${username}_${id}_${idx + 1}`,
|
|
2518
|
+
extension: "jpg",
|
|
2519
|
+
type: "photo",
|
|
2520
|
+
caption,
|
|
2521
|
+
date
|
|
2522
|
+
});
|
|
2523
|
+
}
|
|
2524
|
+
});
|
|
2525
|
+
return items;
|
|
2526
|
+
}
|
|
2527
|
+
const video = item.video;
|
|
2528
|
+
const videoUrl = video?.playAddr || video?.downloadAddr;
|
|
2529
|
+
if (videoUrl) {
|
|
2530
|
+
items.push({
|
|
2531
|
+
id,
|
|
2532
|
+
url: videoUrl,
|
|
2533
|
+
filename: `${username}_${id}`,
|
|
2534
|
+
extension: "mp4",
|
|
2535
|
+
type: "video",
|
|
2536
|
+
caption,
|
|
2537
|
+
date,
|
|
2538
|
+
width: video?.width,
|
|
2539
|
+
height: video?.height,
|
|
2540
|
+
thumbnailUrl: video?.cover || video?.originCover
|
|
2541
|
+
});
|
|
2542
|
+
}
|
|
2543
|
+
return items;
|
|
2544
|
+
}
|
|
2545
|
+
var TikTokExtractor = class {
|
|
2546
|
+
platform = "tiktok";
|
|
2547
|
+
httpClient;
|
|
2548
|
+
constructor(httpClient) {
|
|
2549
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
2550
|
+
"Referer": "https://www.tiktok.com/"
|
|
2551
|
+
});
|
|
2552
|
+
}
|
|
2553
|
+
async *extract(profile, options) {
|
|
2554
|
+
const cleanUsername = profile.username.replace(/^@/, "");
|
|
2555
|
+
const subDir = options.subDir;
|
|
2556
|
+
options.onProgress?.({
|
|
2557
|
+
stage: "Profile",
|
|
2558
|
+
found: 0,
|
|
2559
|
+
message: `Fetching TikTok profile @${cleanUsername}...`
|
|
2560
|
+
});
|
|
2561
|
+
const profileUrl = `https://www.tiktok.com/@${cleanUsername}`;
|
|
2562
|
+
const html = await this.httpClient.fetchText(profileUrl, {
|
|
2563
|
+
cookiePath: options.cookiePath,
|
|
2564
|
+
cookieDomain: "tiktok.com",
|
|
2565
|
+
signal: options.signal,
|
|
2566
|
+
retries: 0,
|
|
2567
|
+
timeoutMs: 4e3
|
|
2568
|
+
});
|
|
2569
|
+
const scope = parseTikTokRehydrationJson(html);
|
|
2570
|
+
const userDetail = scope?.["webapp.user-detail"];
|
|
2571
|
+
const userInfo = userDetail?.userInfo?.user;
|
|
2572
|
+
const secUid = userInfo?.secUid;
|
|
2573
|
+
let found = 0;
|
|
2574
|
+
const initialItems = userDetail?.itemList || [];
|
|
2575
|
+
for (const rawItem of initialItems) {
|
|
2576
|
+
const mediaList = parseTikTokItem(rawItem, cleanUsername);
|
|
2577
|
+
for (const item of mediaList) {
|
|
2578
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2579
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2580
|
+
found++;
|
|
2581
|
+
options.onProgress?.({
|
|
2582
|
+
stage: subDir,
|
|
2583
|
+
found,
|
|
2584
|
+
message: `Found item: ${item.filename}`
|
|
2585
|
+
});
|
|
2586
|
+
yield item;
|
|
2587
|
+
}
|
|
2588
|
+
}
|
|
2589
|
+
if (!secUid) {
|
|
2590
|
+
return;
|
|
2591
|
+
}
|
|
2592
|
+
let cursor = "0";
|
|
2593
|
+
let hasMore = true;
|
|
2594
|
+
const visitedCursors = /* @__PURE__ */ new Set();
|
|
2595
|
+
while (hasMore) {
|
|
2596
|
+
if (options.signal?.aborted) break;
|
|
2597
|
+
if (visitedCursors.has(cursor)) break;
|
|
2598
|
+
visitedCursors.add(cursor);
|
|
2599
|
+
const apiUrl = `https://www.tiktok.com/api/post/item_list/?secUid=${encodeURIComponent(secUid)}&count=30&cursor=${cursor}&post_item_list_request_type=0`;
|
|
2600
|
+
let res;
|
|
2601
|
+
try {
|
|
2602
|
+
res = await this.httpClient.fetchJson(apiUrl, {
|
|
2603
|
+
cookiePath: options.cookiePath,
|
|
2604
|
+
cookieDomain: "tiktok.com",
|
|
2605
|
+
signal: options.signal
|
|
2606
|
+
});
|
|
2607
|
+
} catch {
|
|
2608
|
+
break;
|
|
2609
|
+
}
|
|
2610
|
+
const items = res?.itemList || [];
|
|
2611
|
+
if (items.length === 0) break;
|
|
2612
|
+
for (const rawItem of items) {
|
|
2613
|
+
const mediaList = parseTikTokItem(rawItem, cleanUsername);
|
|
2614
|
+
for (const item of mediaList) {
|
|
2615
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2616
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2617
|
+
found++;
|
|
2618
|
+
options.onProgress?.({
|
|
2619
|
+
stage: subDir,
|
|
2620
|
+
found,
|
|
2621
|
+
message: `Found item: ${item.filename}`
|
|
2622
|
+
});
|
|
2623
|
+
yield item;
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
hasMore = Boolean(res?.hasMore);
|
|
2627
|
+
cursor = String(res?.cursor ?? "0");
|
|
2628
|
+
await sleep(1e3, 2e3, options.signal);
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
};
|
|
2632
|
+
|
|
2633
|
+
// src/lib/engines/extractors/facebook.ts
|
|
2634
|
+
function decodeEscapedUrl(raw) {
|
|
2635
|
+
const sanitized = raw.replace(/[\r\n\t]/g, "").trim();
|
|
2636
|
+
try {
|
|
2637
|
+
return JSON.parse(`"${sanitized.replace(/"/g, '\\"')}"`);
|
|
2638
|
+
} catch {
|
|
2639
|
+
return sanitized.replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))).replace(/\\(["\\\/bfnrt])/g, (_, ch) => {
|
|
2640
|
+
const escapes = {
|
|
2641
|
+
'"': '"',
|
|
2642
|
+
"\\": "\\",
|
|
2643
|
+
"/": "/",
|
|
2644
|
+
b: "\b",
|
|
2645
|
+
f: "\f",
|
|
2646
|
+
n: "\n",
|
|
2647
|
+
r: "\r",
|
|
2648
|
+
t: " "
|
|
2649
|
+
};
|
|
2650
|
+
return escapes[ch] || ch;
|
|
2651
|
+
});
|
|
2652
|
+
}
|
|
2653
|
+
}
|
|
2654
|
+
function extractFacebookMediaFromHtml(html, username) {
|
|
2655
|
+
const items = [];
|
|
2656
|
+
const seenUrls = /* @__PURE__ */ new Set();
|
|
2657
|
+
const hdMatch = html.match(/"browser_native_hd_url":"([^"]+)"/);
|
|
2658
|
+
const sdMatch = html.match(/"browser_native_sd_url":"([^"]+)"/);
|
|
2659
|
+
const rawVideoUrl = hdMatch?.[1] || sdMatch?.[1];
|
|
2660
|
+
if (rawVideoUrl) {
|
|
2661
|
+
const videoUrl = decodeEscapedUrl(rawVideoUrl);
|
|
2662
|
+
if (!seenUrls.has(videoUrl)) {
|
|
2663
|
+
seenUrls.add(videoUrl);
|
|
2664
|
+
items.push({
|
|
2665
|
+
id: `fb_video_${Date.now()}`,
|
|
2666
|
+
url: videoUrl,
|
|
2667
|
+
filename: `${username}_video_${Date.now()}`,
|
|
2668
|
+
extension: "mp4",
|
|
2669
|
+
type: "video"
|
|
2670
|
+
});
|
|
2671
|
+
}
|
|
2672
|
+
}
|
|
2673
|
+
const photoRegex = /,"image":\{"uri":"([^"]+)"/g;
|
|
2674
|
+
let match;
|
|
2675
|
+
let photoIndex = 1;
|
|
2676
|
+
while ((match = photoRegex.exec(html)) !== null) {
|
|
2677
|
+
const rawUrl = match[1];
|
|
2678
|
+
const photoUrl = decodeEscapedUrl(rawUrl);
|
|
2679
|
+
if (!seenUrls.has(photoUrl)) {
|
|
2680
|
+
seenUrls.add(photoUrl);
|
|
2681
|
+
items.push({
|
|
2682
|
+
id: `fb_photo_${photoIndex}`,
|
|
2683
|
+
url: photoUrl,
|
|
2684
|
+
filename: `${username}_photo_${photoIndex}`,
|
|
2685
|
+
extension: "jpg",
|
|
2686
|
+
type: "photo"
|
|
2687
|
+
});
|
|
2688
|
+
photoIndex++;
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2691
|
+
if (items.length === 0) {
|
|
2692
|
+
const scontentRegex = /https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*/gi;
|
|
2693
|
+
let sMatch;
|
|
2694
|
+
while ((sMatch = scontentRegex.exec(html)) !== null) {
|
|
2695
|
+
const cleanUrl = sMatch[0].replace(/\\/g, "");
|
|
2696
|
+
if (!seenUrls.has(cleanUrl)) {
|
|
2697
|
+
seenUrls.add(cleanUrl);
|
|
2698
|
+
items.push({
|
|
2699
|
+
id: `fb_photo_${photoIndex}`,
|
|
2700
|
+
url: cleanUrl,
|
|
2701
|
+
filename: `${username}_photo_${photoIndex}`,
|
|
2702
|
+
extension: "jpg",
|
|
2703
|
+
type: "photo"
|
|
2704
|
+
});
|
|
2705
|
+
photoIndex++;
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
return items;
|
|
2710
|
+
}
|
|
2711
|
+
var FacebookExtractor = class {
|
|
2712
|
+
platform = "facebook";
|
|
2713
|
+
httpClient;
|
|
2714
|
+
constructor(httpClient) {
|
|
2715
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
2716
|
+
"Referer": "https://www.facebook.com/"
|
|
2717
|
+
});
|
|
2718
|
+
}
|
|
2719
|
+
async *extract(profile, options) {
|
|
2720
|
+
const username = profile.username;
|
|
2721
|
+
const subDir = options.subDir;
|
|
2722
|
+
options.onProgress?.({
|
|
2723
|
+
stage: "Profile",
|
|
2724
|
+
found: 0,
|
|
2725
|
+
message: `Fetching Facebook profile @${username}...`
|
|
2726
|
+
});
|
|
2727
|
+
const targetUrl = subDir === "Photos" ? `https://www.facebook.com/${encodeURIComponent(username)}/photos` : `https://www.facebook.com/${encodeURIComponent(username)}/videos`;
|
|
2728
|
+
const html = await this.httpClient.fetchText(targetUrl, {
|
|
2729
|
+
cookiePath: options.cookiePath,
|
|
2730
|
+
cookieDomain: "facebook.com",
|
|
2731
|
+
signal: options.signal,
|
|
2732
|
+
retries: 0,
|
|
2733
|
+
timeoutMs: 4e3
|
|
2734
|
+
});
|
|
2735
|
+
const items = extractFacebookMediaFromHtml(html, username);
|
|
2736
|
+
let found = 0;
|
|
2737
|
+
for (const item of items) {
|
|
2738
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2739
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2740
|
+
found++;
|
|
2741
|
+
options.onProgress?.({
|
|
2742
|
+
stage: subDir,
|
|
2743
|
+
found,
|
|
2744
|
+
message: `Found item: ${item.filename}`
|
|
2745
|
+
});
|
|
2746
|
+
yield item;
|
|
2747
|
+
}
|
|
2748
|
+
}
|
|
2749
|
+
};
|
|
2750
|
+
|
|
2751
|
+
// src/lib/engines/extractors/index.ts
|
|
2752
|
+
function getExtractor(platform) {
|
|
2753
|
+
switch (platform) {
|
|
2754
|
+
case "instagram":
|
|
2755
|
+
return new InstagramExtractor();
|
|
2756
|
+
case "x":
|
|
2757
|
+
return new XExtractor();
|
|
2758
|
+
case "tiktok":
|
|
2759
|
+
return new TikTokExtractor();
|
|
2760
|
+
case "facebook":
|
|
2761
|
+
return new FacebookExtractor();
|
|
2762
|
+
default:
|
|
2763
|
+
throw new Error(`Unsupported extractor platform: ${platform}`);
|
|
2764
|
+
}
|
|
2765
|
+
}
|
|
2766
|
+
|
|
2767
|
+
// src/lib/engines/downloader.ts
|
|
2768
|
+
import fs6 from "fs";
|
|
2769
|
+
import fsPromises2 from "fs/promises";
|
|
2770
|
+
import path5 from "path";
|
|
2771
|
+
import { Readable } from "stream";
|
|
2772
|
+
import { pipeline } from "stream/promises";
|
|
2773
|
+
var ARCHIVE_FILENAME = ".open-nami-archive.json";
|
|
2774
|
+
async function loadArchive(destDir) {
|
|
2775
|
+
const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
2776
|
+
try {
|
|
2777
|
+
const raw = await fsPromises2.readFile(archivePath, "utf-8");
|
|
2778
|
+
const parsed = JSON.parse(raw);
|
|
2779
|
+
if (Array.isArray(parsed)) {
|
|
2780
|
+
return new Set(parsed);
|
|
2781
|
+
}
|
|
2782
|
+
} catch {
|
|
2783
|
+
}
|
|
2784
|
+
return /* @__PURE__ */ new Set();
|
|
2785
|
+
}
|
|
2786
|
+
async function saveArchive(destDir, archive) {
|
|
2787
|
+
const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
2788
|
+
const tmpPath = `${archivePath}.tmp`;
|
|
2789
|
+
try {
|
|
2790
|
+
await fsPromises2.writeFile(tmpPath, JSON.stringify([...archive]), "utf-8");
|
|
2791
|
+
await fsPromises2.rename(tmpPath, archivePath);
|
|
2792
|
+
} catch {
|
|
2793
|
+
}
|
|
2794
|
+
}
|
|
2795
|
+
function sanitizeFilename(rawFilename, fallbackId, ext) {
|
|
2796
|
+
const cleanExt = ext.replace(/^\./, "");
|
|
2797
|
+
const baseOnly = path5.basename(rawFilename);
|
|
2798
|
+
let sanitizedBase = baseOnly.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_").replace(/[. ]+$/, "").trim();
|
|
2799
|
+
if (!sanitizedBase || /^_+$/.test(sanitizedBase)) {
|
|
2800
|
+
sanitizedBase = `item_${fallbackId}`;
|
|
2801
|
+
}
|
|
2802
|
+
if (/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$/i.test(sanitizedBase)) {
|
|
2803
|
+
sanitizedBase = `_${sanitizedBase}`;
|
|
2804
|
+
}
|
|
2805
|
+
return sanitizedBase.includes(".") ? sanitizedBase : `${sanitizedBase}.${cleanExt}`;
|
|
2806
|
+
}
|
|
2807
|
+
async function downloadMediaStream(items, options) {
|
|
2808
|
+
const { jobName, destDir, signal, onProgress } = options;
|
|
2809
|
+
const httpClient = options.httpClient ?? new HttpClient();
|
|
2810
|
+
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2811
|
+
await fsPromises2.mkdir(destDir, { recursive: true });
|
|
2812
|
+
const archive = await loadArchive(destDir);
|
|
2813
|
+
let downloaded = 0;
|
|
2814
|
+
let skipped = 0;
|
|
2815
|
+
const errors = [];
|
|
2816
|
+
const iterator = items[Symbol.asyncIterator]();
|
|
2817
|
+
let queueLock = Promise.resolve();
|
|
2818
|
+
let isDone = false;
|
|
2819
|
+
let consecutiveErrors = 0;
|
|
2820
|
+
const MAX_CONSECUTIVE_ERRORS = 5;
|
|
2821
|
+
function getNextItem() {
|
|
2822
|
+
const nextItem = queueLock.then(() => iterator.next());
|
|
2823
|
+
queueLock = nextItem.then(
|
|
2824
|
+
() => {
|
|
2825
|
+
},
|
|
2826
|
+
() => {
|
|
2827
|
+
}
|
|
2828
|
+
);
|
|
2829
|
+
return nextItem;
|
|
2830
|
+
}
|
|
2831
|
+
async function downloadWorker() {
|
|
2832
|
+
while (!isDone && !signal?.aborted) {
|
|
2833
|
+
let item;
|
|
2834
|
+
try {
|
|
2835
|
+
const next = await getNextItem();
|
|
2836
|
+
if (next.done) {
|
|
2837
|
+
isDone = true;
|
|
2838
|
+
break;
|
|
2839
|
+
}
|
|
2840
|
+
consecutiveErrors = 0;
|
|
2841
|
+
item = next.value;
|
|
2842
|
+
} catch (err) {
|
|
2843
|
+
consecutiveErrors++;
|
|
2844
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2845
|
+
errors.push(`Extraction stream error: ${errMsg}`);
|
|
2846
|
+
onProgress?.({
|
|
2847
|
+
job: jobName,
|
|
2848
|
+
downloadedCount: downloaded,
|
|
2849
|
+
skippedCount: skipped,
|
|
2850
|
+
currentFile: "",
|
|
2851
|
+
statusText: `Extractor glitch: ${errMsg}`
|
|
2852
|
+
});
|
|
2853
|
+
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS || signal?.aborted) {
|
|
2854
|
+
isDone = true;
|
|
2855
|
+
break;
|
|
2856
|
+
}
|
|
2857
|
+
continue;
|
|
2858
|
+
}
|
|
2859
|
+
const ext = item.extension.replace(/^\./, "") || (item.type === "video" ? "mp4" : "jpg");
|
|
2860
|
+
const finalFilename = sanitizeFilename(item.filename, item.id, ext);
|
|
2861
|
+
const finalPath = path5.join(destDir, finalFilename);
|
|
2862
|
+
if (archive.has(item.id) && fs6.existsSync(finalPath)) {
|
|
2863
|
+
skipped++;
|
|
2864
|
+
onProgress?.({
|
|
2865
|
+
job: jobName,
|
|
2866
|
+
downloadedCount: downloaded,
|
|
2867
|
+
skippedCount: skipped,
|
|
2868
|
+
currentFile: finalFilename,
|
|
2869
|
+
statusText: `Skipping already downloaded: ${finalFilename}`
|
|
2870
|
+
});
|
|
2871
|
+
continue;
|
|
2872
|
+
}
|
|
2873
|
+
onProgress?.({
|
|
2874
|
+
job: jobName,
|
|
2875
|
+
downloadedCount: downloaded,
|
|
2876
|
+
skippedCount: skipped,
|
|
2877
|
+
currentFile: finalFilename,
|
|
2878
|
+
statusText: `Downloading: ${finalFilename}`
|
|
2879
|
+
});
|
|
2880
|
+
const tmpPath = `${finalPath}.download`;
|
|
2881
|
+
try {
|
|
2882
|
+
const reqHeaders = {};
|
|
2883
|
+
if (options.referer) {
|
|
2884
|
+
reqHeaders["Referer"] = options.referer;
|
|
2885
|
+
}
|
|
2886
|
+
const response = await httpClient.request(item.url, {
|
|
2887
|
+
signal,
|
|
2888
|
+
cookiePath: options.cookiePath,
|
|
2889
|
+
headers: reqHeaders
|
|
2890
|
+
});
|
|
2891
|
+
if (!response.ok || !response.body) {
|
|
2892
|
+
try {
|
|
2893
|
+
await response.body?.cancel();
|
|
2894
|
+
} catch {
|
|
2895
|
+
}
|
|
2896
|
+
throw new Error(`HTTP ${response.status} (${response.statusText}) for ${item.url}`);
|
|
2897
|
+
}
|
|
2898
|
+
const fileStream = fs6.createWriteStream(tmpPath);
|
|
2899
|
+
await pipeline(Readable.fromWeb(response.body), fileStream, { signal });
|
|
2900
|
+
if (item.date) {
|
|
2901
|
+
try {
|
|
2902
|
+
const mtime = new Date(item.date * 1e3);
|
|
2903
|
+
await fsPromises2.utimes(tmpPath, mtime, mtime);
|
|
2904
|
+
} catch {
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
await fsPromises2.rename(tmpPath, finalPath);
|
|
2908
|
+
archive.add(item.id);
|
|
2909
|
+
downloaded++;
|
|
2910
|
+
onProgress?.({
|
|
2911
|
+
job: jobName,
|
|
2912
|
+
downloadedCount: downloaded,
|
|
2913
|
+
skippedCount: skipped,
|
|
2914
|
+
currentFile: finalFilename,
|
|
2915
|
+
statusText: `Saved: ${finalFilename}`
|
|
2916
|
+
});
|
|
2917
|
+
} catch (err) {
|
|
2918
|
+
try {
|
|
2919
|
+
if (fs6.existsSync(tmpPath)) {
|
|
2920
|
+
await fsPromises2.unlink(tmpPath);
|
|
2921
|
+
}
|
|
2922
|
+
} catch {
|
|
2923
|
+
}
|
|
2924
|
+
if (signal?.aborted) {
|
|
2925
|
+
break;
|
|
2926
|
+
}
|
|
2927
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2928
|
+
errors.push(`${finalFilename}: ${errMsg}`);
|
|
2929
|
+
onProgress?.({
|
|
2930
|
+
job: jobName,
|
|
2931
|
+
downloadedCount: downloaded,
|
|
2932
|
+
skippedCount: skipped,
|
|
2933
|
+
currentFile: finalFilename,
|
|
2934
|
+
statusText: `Failed: ${finalFilename} (${errMsg})`
|
|
2935
|
+
});
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
const workers = Array.from({ length: concurrency }, () => downloadWorker());
|
|
2940
|
+
await Promise.all(workers);
|
|
2941
|
+
await saveArchive(destDir, archive);
|
|
2942
|
+
return { downloaded, skipped, errors };
|
|
2943
|
+
}
|
|
2944
|
+
|
|
1562
2945
|
// src/lib/engines/gallery-dl.ts
|
|
1563
2946
|
import { spawn as spawn2 } from "child_process";
|
|
1564
|
-
import
|
|
1565
|
-
import
|
|
2947
|
+
import path8 from "path";
|
|
2948
|
+
import fs9 from "fs/promises";
|
|
1566
2949
|
|
|
1567
2950
|
// src/lib/engines/exec.ts
|
|
1568
|
-
import
|
|
1569
|
-
import
|
|
2951
|
+
import fs8 from "fs";
|
|
2952
|
+
import path7 from "path";
|
|
1570
2953
|
|
|
1571
2954
|
// src/lib/engines/fetcher.ts
|
|
1572
2955
|
import { spawn } from "child_process";
|
|
1573
2956
|
import { createWriteStream } from "fs";
|
|
1574
|
-
import
|
|
2957
|
+
import fs7 from "fs/promises";
|
|
1575
2958
|
import os7 from "os";
|
|
1576
|
-
import
|
|
1577
|
-
import { Readable } from "stream";
|
|
1578
|
-
import { pipeline } from "stream/promises";
|
|
2959
|
+
import path6 from "path";
|
|
2960
|
+
import { Readable as Readable2 } from "stream";
|
|
2961
|
+
import { pipeline as pipeline2 } from "stream/promises";
|
|
1579
2962
|
function getOpenNamiBinDir(env = process.env) {
|
|
1580
2963
|
if (env.OPEN_NAMI_BIN_DIR && env.OPEN_NAMI_BIN_DIR.trim()) {
|
|
1581
|
-
return
|
|
2964
|
+
return path6.resolve(env.OPEN_NAMI_BIN_DIR.trim());
|
|
1582
2965
|
}
|
|
1583
2966
|
const base = env.OPEN_NAMI_DIR && env.OPEN_NAMI_DIR.trim() ? env.OPEN_NAMI_DIR.trim() : os7.homedir();
|
|
1584
|
-
return
|
|
2967
|
+
return path6.join(base, ".open-nami", "bin");
|
|
1585
2968
|
}
|
|
1586
2969
|
var YTDLP_RELEASE_BASE = "https://github.com/yt-dlp/yt-dlp/releases/latest/download";
|
|
1587
2970
|
var GALLERYDL_CODEBERG_API = "https://codeberg.org/api/v1/repos/mikf/gallery-dl/releases/latest";
|
|
@@ -1632,7 +3015,7 @@ function getBinaryVersion(executablePath) {
|
|
|
1632
3015
|
});
|
|
1633
3016
|
}
|
|
1634
3017
|
async function downloadBinary(url, targetFile, signal) {
|
|
1635
|
-
await
|
|
3018
|
+
await fs7.mkdir(path6.dirname(targetFile), { recursive: true });
|
|
1636
3019
|
const response = await fetch(url, {
|
|
1637
3020
|
signal,
|
|
1638
3021
|
headers: { "User-Agent": "open-nami" }
|
|
@@ -1641,18 +3024,30 @@ async function downloadBinary(url, targetFile, signal) {
|
|
|
1641
3024
|
throw new Error(`Failed to download binary from ${url} (HTTP ${response.status}). Check your connection and try again.`);
|
|
1642
3025
|
}
|
|
1643
3026
|
const tmp = `${targetFile}.download`;
|
|
1644
|
-
await pipeline(Readable.fromWeb(response.body), createWriteStream(tmp), { signal });
|
|
1645
3027
|
try {
|
|
1646
|
-
await
|
|
1647
|
-
|
|
3028
|
+
await pipeline2(Readable2.fromWeb(response.body), createWriteStream(tmp), { signal });
|
|
3029
|
+
try {
|
|
3030
|
+
await fs7.chmod(tmp, 493);
|
|
3031
|
+
} catch {
|
|
3032
|
+
}
|
|
3033
|
+
const stats = await fs7.stat(tmp);
|
|
3034
|
+
if (stats.size < 100 * 1024) {
|
|
3035
|
+
throw new Error(`Downloaded binary from ${url} is unexpectedly small (${stats.size} bytes). File may be incomplete or corrupted.`);
|
|
3036
|
+
}
|
|
3037
|
+
await fs7.rename(tmp, targetFile);
|
|
3038
|
+
return targetFile;
|
|
3039
|
+
} catch (err) {
|
|
3040
|
+
try {
|
|
3041
|
+
await fs7.unlink(tmp);
|
|
3042
|
+
} catch {
|
|
3043
|
+
}
|
|
3044
|
+
throw err;
|
|
1648
3045
|
}
|
|
1649
|
-
await fs5.rename(tmp, targetFile);
|
|
1650
|
-
return targetFile;
|
|
1651
3046
|
}
|
|
1652
3047
|
async function downloadLatestYtDlp(targetDir, signal, onStatus) {
|
|
1653
3048
|
const dir = targetDir || getOpenNamiBinDir();
|
|
1654
3049
|
const binaryName = process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp";
|
|
1655
|
-
const local =
|
|
3050
|
+
const local = path6.join(dir, binaryName);
|
|
1656
3051
|
const asset = ytDlpAssetName();
|
|
1657
3052
|
const url = `${YTDLP_RELEASE_BASE}/${asset}`;
|
|
1658
3053
|
onStatus?.(`fetching latest yt-dlp (${asset})\u2026`);
|
|
@@ -1688,7 +3083,7 @@ async function downloadLatestGalleryDl(targetDir, signal, onStatus) {
|
|
|
1688
3083
|
}
|
|
1689
3084
|
const dir = targetDir || getOpenNamiBinDir();
|
|
1690
3085
|
const binaryName = process.platform === "win32" ? "gallery-dl.exe" : "gallery-dl";
|
|
1691
|
-
const local =
|
|
3086
|
+
const local = path6.join(dir, binaryName);
|
|
1692
3087
|
const asset = galleryDlAssetName();
|
|
1693
3088
|
const candidateUrls = await getGalleryDlDownloadUrls(asset, fetch, signal);
|
|
1694
3089
|
let lastError;
|
|
@@ -1706,7 +3101,7 @@ async function downloadLatestGalleryDl(targetDir, signal, onStatus) {
|
|
|
1706
3101
|
async function ensureYtDlp(onStatus, signal) {
|
|
1707
3102
|
if (await commandWorks("yt-dlp", ["--version"])) return "yt-dlp";
|
|
1708
3103
|
const binaryName = process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp";
|
|
1709
|
-
const local =
|
|
3104
|
+
const local = path6.join(getOpenNamiBinDir(), binaryName);
|
|
1710
3105
|
if (await commandWorks(local, ["--version"])) return local;
|
|
1711
3106
|
if (await commandWorks("python", ["-m", "yt_dlp", "--version"])) return "python";
|
|
1712
3107
|
if (await commandWorks("python3", ["-m", "yt_dlp", "--version"])) return "python3";
|
|
@@ -1716,7 +3111,7 @@ async function ensureYtDlp(onStatus, signal) {
|
|
|
1716
3111
|
async function ensureGalleryDl(onStatus, signal) {
|
|
1717
3112
|
if (await commandWorks("gallery-dl", ["--version"])) return "gallery-dl";
|
|
1718
3113
|
const binaryName = process.platform === "win32" ? "gallery-dl.exe" : "gallery-dl";
|
|
1719
|
-
const local =
|
|
3114
|
+
const local = path6.join(getOpenNamiBinDir(), binaryName);
|
|
1720
3115
|
if (await commandWorks(local, ["--version"])) return local;
|
|
1721
3116
|
if (await commandWorks("python", ["-m", "gallery_dl", "--version"])) return "python";
|
|
1722
3117
|
if (await commandWorks("python3", ["-m", "gallery_dl", "--version"])) return "python3";
|
|
@@ -1729,7 +3124,7 @@ async function updateYtDlp(options) {
|
|
|
1729
3124
|
const signal = options?.signal;
|
|
1730
3125
|
const dir = getOpenNamiBinDir();
|
|
1731
3126
|
const binaryName = process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp";
|
|
1732
|
-
const local =
|
|
3127
|
+
const local = path6.join(dir, binaryName);
|
|
1733
3128
|
let previousVersion;
|
|
1734
3129
|
if (await commandWorks(local, ["--version"])) {
|
|
1735
3130
|
previousVersion = await getBinaryVersion(local);
|
|
@@ -1752,7 +3147,7 @@ async function updateGalleryDl(options) {
|
|
|
1752
3147
|
const signal = options?.signal;
|
|
1753
3148
|
const dir = getOpenNamiBinDir();
|
|
1754
3149
|
const binaryName = process.platform === "win32" ? "gallery-dl.exe" : "gallery-dl";
|
|
1755
|
-
const local =
|
|
3150
|
+
const local = path6.join(dir, binaryName);
|
|
1756
3151
|
let previousVersion;
|
|
1757
3152
|
if (await commandWorks(local, ["--version"])) {
|
|
1758
3153
|
previousVersion = await getBinaryVersion(local);
|
|
@@ -1773,16 +3168,16 @@ async function updateGalleryDl(options) {
|
|
|
1773
3168
|
// src/lib/engines/exec.ts
|
|
1774
3169
|
function findExecutable(binName, pathEnv = process.env.PATH || "") {
|
|
1775
3170
|
if (!pathEnv) return null;
|
|
1776
|
-
const delimiter =
|
|
3171
|
+
const delimiter = path7.delimiter;
|
|
1777
3172
|
const dirs = pathEnv.split(delimiter).filter(Boolean);
|
|
1778
3173
|
const isWin = process.platform === "win32";
|
|
1779
3174
|
const extensions = isWin ? [".exe", ".cmd", ".bat", ""] : [""];
|
|
1780
3175
|
for (const dir of dirs) {
|
|
1781
3176
|
for (const ext of extensions) {
|
|
1782
|
-
const fullPath =
|
|
3177
|
+
const fullPath = path7.join(dir, `${binName}${ext}`);
|
|
1783
3178
|
try {
|
|
1784
|
-
if (
|
|
1785
|
-
const stat =
|
|
3179
|
+
if (fs8.existsSync(fullPath)) {
|
|
3180
|
+
const stat = fs8.statSync(fullPath);
|
|
1786
3181
|
if (stat.isFile()) {
|
|
1787
3182
|
return fullPath;
|
|
1788
3183
|
}
|
|
@@ -1812,10 +3207,10 @@ function resolveEngineCommand(engine, options = {}) {
|
|
|
1812
3207
|
}
|
|
1813
3208
|
const binDir = getOpenNamiBinDir(env);
|
|
1814
3209
|
const isWin = process.platform === "win32";
|
|
1815
|
-
const localBin =
|
|
3210
|
+
const localBin = path7.join(binDir, `${engine}${isWin ? ".exe" : ""}`);
|
|
1816
3211
|
try {
|
|
1817
|
-
if (
|
|
1818
|
-
const stat =
|
|
3212
|
+
if (fs8.existsSync(localBin)) {
|
|
3213
|
+
const stat = fs8.statSync(localBin);
|
|
1819
3214
|
if (stat.isFile()) {
|
|
1820
3215
|
return {
|
|
1821
3216
|
command: localBin,
|
|
@@ -1853,8 +3248,8 @@ var GDL_ARCHIVE = "archive_gallery-dl.sqlite3";
|
|
|
1853
3248
|
var UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
1854
3249
|
var PLATFORM_SLEEP = {
|
|
1855
3250
|
tiktok: "0.6-1.2",
|
|
1856
|
-
instagram: "
|
|
1857
|
-
facebook: "1.
|
|
3251
|
+
instagram: "1.0-1.8",
|
|
3252
|
+
facebook: "1.0-1.8"
|
|
1858
3253
|
};
|
|
1859
3254
|
function buildGalleryDlArgs(options) {
|
|
1860
3255
|
const { destDir, subDir, platform, cleanUrl, username, cookiePath } = options;
|
|
@@ -1872,7 +3267,7 @@ function buildGalleryDlArgs(options) {
|
|
|
1872
3267
|
"-o",
|
|
1873
3268
|
'headers.sec-ch-ua-platform="Windows"',
|
|
1874
3269
|
"--download-archive",
|
|
1875
|
-
|
|
3270
|
+
path8.join(destDir, GDL_ARCHIVE),
|
|
1876
3271
|
"--retries",
|
|
1877
3272
|
"3",
|
|
1878
3273
|
"--sleep-429",
|
|
@@ -1894,12 +3289,12 @@ function buildGalleryDlArgs(options) {
|
|
|
1894
3289
|
} else if (subDir === "Stories") {
|
|
1895
3290
|
if (platform === "instagram") {
|
|
1896
3291
|
targetUrl = `https://www.instagram.com/stories/${encodeURIComponent(username)}/`;
|
|
1897
|
-
args.push("--sleep-request", "
|
|
3292
|
+
args.push("--sleep-request", "1.0-1.8");
|
|
1898
3293
|
}
|
|
1899
3294
|
} else if (subDir === "Highlights") {
|
|
1900
3295
|
if (platform === "instagram") {
|
|
1901
3296
|
targetUrl = `https://www.instagram.com/${encodeURIComponent(username)}/highlights/`;
|
|
1902
|
-
args.push("--sleep-request", "
|
|
3297
|
+
args.push("--sleep-request", "1.0-1.8");
|
|
1903
3298
|
}
|
|
1904
3299
|
}
|
|
1905
3300
|
if (cookiePath) {
|
|
@@ -1910,8 +3305,8 @@ function buildGalleryDlArgs(options) {
|
|
|
1910
3305
|
}
|
|
1911
3306
|
async function runGalleryDl(options) {
|
|
1912
3307
|
const { profile, targetDir, subDir, cookiePath, onProgress } = options;
|
|
1913
|
-
const destDir =
|
|
1914
|
-
await
|
|
3308
|
+
const destDir = path8.join(targetDir, subDir);
|
|
3309
|
+
await fs9.mkdir(destDir, { recursive: true });
|
|
1915
3310
|
const args = buildGalleryDlArgs({
|
|
1916
3311
|
destDir,
|
|
1917
3312
|
subDir,
|
|
@@ -1953,7 +3348,7 @@ async function runGalleryDl(options) {
|
|
|
1953
3348
|
if (!trimmed) return;
|
|
1954
3349
|
if (trimmed.startsWith("#")) {
|
|
1955
3350
|
skippedCount++;
|
|
1956
|
-
const filename =
|
|
3351
|
+
const filename = path8.basename(trimmed.slice(1).trim());
|
|
1957
3352
|
onProgress?.({
|
|
1958
3353
|
job: subDir,
|
|
1959
3354
|
downloadedCount,
|
|
@@ -1961,9 +3356,9 @@ async function runGalleryDl(options) {
|
|
|
1961
3356
|
currentFile: filename,
|
|
1962
3357
|
statusText: `Skipped: ${filename}`
|
|
1963
3358
|
});
|
|
1964
|
-
} else if (trimmed.match(/\.(jpe?g|png|gif|webp|bmp|jfif|heic|avif|tiff|svg|mp4|mov|webm|mkv|avi|flv|m4v)$/i) || trimmed.includes(
|
|
3359
|
+
} else if (trimmed.match(/\.(jpe?g|png|gif|webp|bmp|jfif|heic|avif|tiff|svg|mp4|mov|webm|mkv|avi|flv|m4v)$/i) || trimmed.includes(path8.sep)) {
|
|
1965
3360
|
downloadedCount++;
|
|
1966
|
-
const filename =
|
|
3361
|
+
const filename = path8.basename(trimmed);
|
|
1967
3362
|
onProgress?.({
|
|
1968
3363
|
job: subDir,
|
|
1969
3364
|
downloadedCount,
|
|
@@ -2014,15 +3409,15 @@ async function runGalleryDl(options) {
|
|
|
2014
3409
|
|
|
2015
3410
|
// src/lib/engines/yt-dlp.ts
|
|
2016
3411
|
import { spawn as spawn3 } from "child_process";
|
|
2017
|
-
import
|
|
2018
|
-
import
|
|
3412
|
+
import path9 from "path";
|
|
3413
|
+
import fs10 from "fs/promises";
|
|
2019
3414
|
var YTDLP_ARCHIVE = "archive_yt-dlp.txt";
|
|
2020
3415
|
var UA2 = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
2021
3416
|
function buildYtDlpArgs(options) {
|
|
2022
3417
|
const { destDir, platform, cleanUrl, cookiePath } = options;
|
|
2023
3418
|
const args = [
|
|
2024
3419
|
"-o",
|
|
2025
|
-
|
|
3420
|
+
path9.join(destDir, "%(title).150B [%(id)s].%(ext)s"),
|
|
2026
3421
|
"--windows-filenames",
|
|
2027
3422
|
"--newline",
|
|
2028
3423
|
"--ignore-errors",
|
|
@@ -2030,7 +3425,7 @@ function buildYtDlpArgs(options) {
|
|
|
2030
3425
|
"--user-agent",
|
|
2031
3426
|
UA2,
|
|
2032
3427
|
"--download-archive",
|
|
2033
|
-
|
|
3428
|
+
path9.join(destDir, YTDLP_ARCHIVE),
|
|
2034
3429
|
"--concurrent-fragments",
|
|
2035
3430
|
"4",
|
|
2036
3431
|
"--retries",
|
|
@@ -2055,8 +3450,8 @@ function buildYtDlpArgs(options) {
|
|
|
2055
3450
|
}
|
|
2056
3451
|
async function runYtDlp(options) {
|
|
2057
3452
|
const { profile, targetDir, cookiePath, onProgress } = options;
|
|
2058
|
-
const destDir =
|
|
2059
|
-
await
|
|
3453
|
+
const destDir = path9.join(targetDir, "Videos");
|
|
3454
|
+
await fs10.mkdir(destDir, { recursive: true });
|
|
2060
3455
|
const args = buildYtDlpArgs({
|
|
2061
3456
|
destDir,
|
|
2062
3457
|
platform: profile.platform,
|
|
@@ -2104,7 +3499,7 @@ async function runYtDlp(options) {
|
|
|
2104
3499
|
});
|
|
2105
3500
|
} else if (trimmed.includes("[download] Destination:")) {
|
|
2106
3501
|
const dest = trimmed.replace("[download] Destination:", "").trim();
|
|
2107
|
-
const filename =
|
|
3502
|
+
const filename = path9.basename(dest);
|
|
2108
3503
|
onProgress?.({
|
|
2109
3504
|
job: "Videos",
|
|
2110
3505
|
downloadedCount,
|
|
@@ -2161,38 +3556,44 @@ async function runYtDlp(options) {
|
|
|
2161
3556
|
}
|
|
2162
3557
|
|
|
2163
3558
|
// src/lib/engines/dispatcher.ts
|
|
3559
|
+
var REFERER_MAP = {
|
|
3560
|
+
tiktok: "https://www.tiktok.com/",
|
|
3561
|
+
instagram: "https://www.instagram.com/",
|
|
3562
|
+
x: "https://x.com/",
|
|
3563
|
+
facebook: "https://www.facebook.com/"
|
|
3564
|
+
};
|
|
2164
3565
|
function expandPath(dir) {
|
|
2165
3566
|
if (dir.startsWith("~/") || dir.startsWith("~\\") || dir === "~") {
|
|
2166
|
-
return
|
|
3567
|
+
return path10.join(os8.homedir(), dir.slice(1));
|
|
2167
3568
|
}
|
|
2168
3569
|
return dir;
|
|
2169
3570
|
}
|
|
2170
3571
|
function resolveDownloadBaseDir(customBaseDir) {
|
|
2171
3572
|
if (customBaseDir && customBaseDir.trim()) {
|
|
2172
|
-
return
|
|
2173
|
-
}
|
|
2174
|
-
const configDir = loadConfig().downloadDir;
|
|
2175
|
-
if (configDir && configDir.trim()) {
|
|
2176
|
-
return path9.resolve(expandPath(configDir.trim()));
|
|
3573
|
+
return path10.resolve(expandPath(customBaseDir.trim()));
|
|
2177
3574
|
}
|
|
2178
3575
|
const envDir = process.env.OPEN_NAMI_DIR || process.env.NAMI_DIR;
|
|
2179
3576
|
if (envDir && envDir.trim()) {
|
|
2180
|
-
return
|
|
3577
|
+
return path10.resolve(expandPath(envDir.trim()));
|
|
3578
|
+
}
|
|
3579
|
+
const configDir = loadConfig().downloadDir;
|
|
3580
|
+
if (configDir && configDir.trim()) {
|
|
3581
|
+
return path10.resolve(expandPath(configDir.trim()));
|
|
2181
3582
|
}
|
|
2182
3583
|
if (process.env.NAMI_BASE_DIR && process.env.NAMI_BASE_DIR.trim()) {
|
|
2183
3584
|
const raw = process.env.NAMI_BASE_DIR.trim();
|
|
2184
|
-
return
|
|
3585
|
+
return path10.resolve(expandPath(raw.endsWith("downloads") ? raw : path10.join(raw, "downloads")));
|
|
2185
3586
|
}
|
|
2186
|
-
return
|
|
3587
|
+
return path10.join(resolvePlatformDownloadsDir(), "Open Nami");
|
|
2187
3588
|
}
|
|
2188
3589
|
function resolveDefaultDownloadDir(platform, username, baseDir) {
|
|
2189
3590
|
const root = resolveDownloadBaseDir(baseDir);
|
|
2190
|
-
return
|
|
3591
|
+
return path10.join(root, platform, username);
|
|
2191
3592
|
}
|
|
2192
3593
|
async function dispatchDownload(options) {
|
|
2193
3594
|
const { profile, choice, onProgress } = options;
|
|
2194
3595
|
const targetDir = options.outputDir || resolveDefaultDownloadDir(profile.platform, profile.username);
|
|
2195
|
-
await
|
|
3596
|
+
await fs11.mkdir(targetDir, { recursive: true });
|
|
2196
3597
|
const masterCookie = findCookieFile(profile.platform);
|
|
2197
3598
|
const cookieCopy = masterCookie ? await createSafeCookieCopy(masterCookie) : null;
|
|
2198
3599
|
let totalDownloaded = 0;
|
|
@@ -2216,6 +3617,39 @@ async function dispatchDownload(options) {
|
|
|
2216
3617
|
jobs.push({ type: "highlights" });
|
|
2217
3618
|
}
|
|
2218
3619
|
}
|
|
3620
|
+
const runNative = async (subDir) => {
|
|
3621
|
+
const dest = path10.join(targetDir, subDir);
|
|
3622
|
+
try {
|
|
3623
|
+
const extractor = getExtractor(profile.platform);
|
|
3624
|
+
const items = extractor.extract(profile, {
|
|
3625
|
+
subDir,
|
|
3626
|
+
cookiePath: cookieCopy?.filePath,
|
|
3627
|
+
signal: options.signal
|
|
3628
|
+
});
|
|
3629
|
+
const res = await downloadMediaStream(items, {
|
|
3630
|
+
jobName: `${profile.username} ${subDir}`,
|
|
3631
|
+
destDir: dest,
|
|
3632
|
+
cookiePath: cookieCopy?.filePath,
|
|
3633
|
+
referer: REFERER_MAP[profile.platform],
|
|
3634
|
+
signal: options.signal,
|
|
3635
|
+
onProgress
|
|
3636
|
+
});
|
|
3637
|
+
return {
|
|
3638
|
+
success: res.errors.length === 0,
|
|
3639
|
+
downloaded: res.downloaded,
|
|
3640
|
+
skipped: res.skipped,
|
|
3641
|
+
...res.errors.length > 0 ? { error: res.errors.join("; ") } : {}
|
|
3642
|
+
};
|
|
3643
|
+
} catch (err) {
|
|
3644
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
3645
|
+
return {
|
|
3646
|
+
success: false,
|
|
3647
|
+
downloaded: 0,
|
|
3648
|
+
skipped: 0,
|
|
3649
|
+
error: errMsg
|
|
3650
|
+
};
|
|
3651
|
+
}
|
|
3652
|
+
};
|
|
2219
3653
|
const runGdl = (subDir) => runGalleryDl({
|
|
2220
3654
|
profile,
|
|
2221
3655
|
targetDir,
|
|
@@ -2235,23 +3669,33 @@ async function dispatchDownload(options) {
|
|
|
2235
3669
|
if (options.signal?.aborted) break;
|
|
2236
3670
|
if (job.type === "photos" || job.type === "stories" || job.type === "highlights") {
|
|
2237
3671
|
const subDir = job.type === "photos" ? "Photos" : job.type === "stories" ? "Stories" : "Highlights";
|
|
2238
|
-
|
|
3672
|
+
let res = await runNative(subDir);
|
|
3673
|
+
if ((!res.success || res.downloaded === 0 && res.skipped === 0) && !options.signal?.aborted) {
|
|
3674
|
+
const gdlRes = await runGdl(subDir);
|
|
3675
|
+
if (gdlRes.downloaded > 0 || gdlRes.skipped > 0 || gdlRes.success) {
|
|
3676
|
+
res = gdlRes;
|
|
3677
|
+
}
|
|
3678
|
+
}
|
|
2239
3679
|
totalDownloaded += res.downloaded;
|
|
2240
3680
|
totalSkipped += res.skipped;
|
|
2241
3681
|
if (!res.success && res.error) errors.push(`[${subDir}] ${res.error}`);
|
|
2242
3682
|
} else if (job.type === "videos") {
|
|
2243
|
-
|
|
2244
|
-
const primary = prefersGdl ? () => runGdl("Videos") : runYt;
|
|
2245
|
-
const fallback = prefersGdl ? runYt : () => runGdl("Videos");
|
|
2246
|
-
let res = await primary();
|
|
3683
|
+
let res = await runNative("Videos");
|
|
2247
3684
|
if (res.downloaded > 0 || res.skipped > 0) {
|
|
2248
3685
|
totalDownloaded += res.downloaded;
|
|
2249
3686
|
totalSkipped += res.skipped;
|
|
2250
3687
|
} else if (!options.signal?.aborted) {
|
|
2251
|
-
const fbRes = await
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
3688
|
+
const fbRes = await runYt();
|
|
3689
|
+
if (fbRes.downloaded > 0 || fbRes.skipped > 0 || fbRes.success) {
|
|
3690
|
+
totalDownloaded += fbRes.downloaded;
|
|
3691
|
+
totalSkipped += fbRes.skipped;
|
|
3692
|
+
if (!fbRes.success && fbRes.error) errors.push(`[Videos] ${fbRes.error}`);
|
|
3693
|
+
} else if (!options.signal?.aborted) {
|
|
3694
|
+
const gdlRes = await runGdl("Videos");
|
|
3695
|
+
totalDownloaded += gdlRes.downloaded;
|
|
3696
|
+
totalSkipped += gdlRes.skipped;
|
|
3697
|
+
if (!gdlRes.success && gdlRes.error) errors.push(`[Videos] ${gdlRes.error}`);
|
|
3698
|
+
}
|
|
2255
3699
|
}
|
|
2256
3700
|
}
|
|
2257
3701
|
}
|
|
@@ -2290,9 +3734,9 @@ function readClipboard(execFn = execFileSync2) {
|
|
|
2290
3734
|
}
|
|
2291
3735
|
|
|
2292
3736
|
// src/lib/reveal.ts
|
|
2293
|
-
import
|
|
3737
|
+
import fs12 from "fs";
|
|
2294
3738
|
import { spawn as spawn4 } from "child_process";
|
|
2295
|
-
import
|
|
3739
|
+
import path11 from "path";
|
|
2296
3740
|
function buildOpenBrowserCommand(url, platform = process.platform) {
|
|
2297
3741
|
try {
|
|
2298
3742
|
const parsed = new URL(url.trim());
|
|
@@ -2319,12 +3763,12 @@ function openBrowser(url) {
|
|
|
2319
3763
|
}
|
|
2320
3764
|
}
|
|
2321
3765
|
function getRevealCommand(targetPath, platform = process.platform, isDirectory) {
|
|
2322
|
-
const pathModule = platform === "win32" ?
|
|
3766
|
+
const pathModule = platform === "win32" ? path11.win32 : platform === "darwin" ? path11.posix : path11;
|
|
2323
3767
|
const resolved = pathModule.resolve(targetPath);
|
|
2324
3768
|
const isDir = isDirectory !== void 0 ? isDirectory : (() => {
|
|
2325
3769
|
try {
|
|
2326
|
-
if (
|
|
2327
|
-
return
|
|
3770
|
+
if (fs12.existsSync(resolved)) {
|
|
3771
|
+
return fs12.statSync(resolved).isDirectory();
|
|
2328
3772
|
}
|
|
2329
3773
|
} catch {
|
|
2330
3774
|
}
|