open-nami 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1254 -69
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -9,7 +9,7 @@ var require_package = __commonJS({
|
|
|
9
9
|
"package.json"(exports, module) {
|
|
10
10
|
module.exports = {
|
|
11
11
|
name: "open-nami",
|
|
12
|
-
version: "1.0.
|
|
12
|
+
version: "1.0.3",
|
|
13
13
|
description: "Open Nami \u2014 terminal-first bulk social media profile downloader for Instagram, TikTok, Facebook & X.",
|
|
14
14
|
type: "module",
|
|
15
15
|
bin: {
|
|
@@ -1044,9 +1044,9 @@ import fsPromises from "fs/promises";
|
|
|
1044
1044
|
import path2 from "path";
|
|
1045
1045
|
import os4 from "os";
|
|
1046
1046
|
var CANDIDATE_DIRS = [
|
|
1047
|
-
path2.join(process.cwd(), "cookies"),
|
|
1048
1047
|
path2.join(os4.homedir(), ".open-nami", "cookies"),
|
|
1049
|
-
path2.join(os4.homedir(), ".nami", "cookies")
|
|
1048
|
+
path2.join(os4.homedir(), ".nami", "cookies"),
|
|
1049
|
+
path2.join(process.cwd(), "cookies")
|
|
1050
1050
|
];
|
|
1051
1051
|
function getCandidateDirs(customDir) {
|
|
1052
1052
|
const dirs = [];
|
|
@@ -1420,9 +1420,9 @@ function parseProfileInput(input) {
|
|
|
1420
1420
|
}
|
|
1421
1421
|
|
|
1422
1422
|
// src/lib/engines/dispatcher.ts
|
|
1423
|
-
import
|
|
1423
|
+
import path10 from "path";
|
|
1424
1424
|
import os8 from "os";
|
|
1425
|
-
import
|
|
1425
|
+
import fs11 from "fs/promises";
|
|
1426
1426
|
|
|
1427
1427
|
// src/lib/known-folders.ts
|
|
1428
1428
|
import { execFileSync } from "child_process";
|
|
@@ -1559,29 +1559,1153 @@ function resolvePlatformDownloadsDir(options = {}) {
|
|
|
1559
1559
|
return defaultFallback;
|
|
1560
1560
|
}
|
|
1561
1561
|
|
|
1562
|
+
// src/lib/engines/extractors/http.ts
|
|
1563
|
+
import fs5 from "fs/promises";
|
|
1564
|
+
import { setTimeout as setTimeoutPromise } from "timers/promises";
|
|
1565
|
+
var DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
1566
|
+
var DEFAULT_CHROME_HEADERS = {
|
|
1567
|
+
"User-Agent": DEFAULT_USER_AGENT,
|
|
1568
|
+
"sec-ch-ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
|
1569
|
+
"sec-ch-ua-mobile": "?0",
|
|
1570
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
1571
|
+
"Accept": "*/*",
|
|
1572
|
+
"Accept-Language": "en-US,en;q=0.9"
|
|
1573
|
+
};
|
|
1574
|
+
function parseNetscapeCookies(content, filterDomain) {
|
|
1575
|
+
const cookies = {};
|
|
1576
|
+
const lines = content.split(/\r?\n/);
|
|
1577
|
+
for (const rawLine of lines) {
|
|
1578
|
+
let line = rawLine.trim();
|
|
1579
|
+
if (!line) continue;
|
|
1580
|
+
if (line.startsWith("#HttpOnly_")) {
|
|
1581
|
+
line = line.slice("#HttpOnly_".length);
|
|
1582
|
+
} else if (line.startsWith("#")) {
|
|
1583
|
+
continue;
|
|
1584
|
+
}
|
|
1585
|
+
const parts = line.split(" ");
|
|
1586
|
+
if (parts.length >= 7) {
|
|
1587
|
+
const domain = parts[0].toLowerCase();
|
|
1588
|
+
const name = parts[5];
|
|
1589
|
+
const value = parts[6];
|
|
1590
|
+
if (filterDomain) {
|
|
1591
|
+
const cleanDomain = filterDomain.toLowerCase().replace(/^\./, "");
|
|
1592
|
+
const lineDomain = domain.replace(/^\./, "");
|
|
1593
|
+
if (!lineDomain.endsWith(cleanDomain) && !cleanDomain.endsWith(lineDomain)) {
|
|
1594
|
+
continue;
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
if (name) {
|
|
1598
|
+
cookies[name] = value;
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
return cookies;
|
|
1603
|
+
}
|
|
1604
|
+
async function loadCookiesFromPath(filePath, filterDomain) {
|
|
1605
|
+
try {
|
|
1606
|
+
const content = await fs5.readFile(filePath, "utf-8");
|
|
1607
|
+
return parseNetscapeCookies(content, filterDomain);
|
|
1608
|
+
} catch {
|
|
1609
|
+
return {};
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
function formatCookieHeader(cookies) {
|
|
1613
|
+
return Object.entries(cookies).map(([k, v]) => `${k}=${v}`).join("; ");
|
|
1614
|
+
}
|
|
1615
|
+
async function sleep(minMs, maxMs, signal) {
|
|
1616
|
+
const ms = maxMs !== void 0 ? Math.floor(minMs + Math.random() * (maxMs - minMs)) : minMs;
|
|
1617
|
+
if (ms <= 0) return;
|
|
1618
|
+
if (signal?.aborted) {
|
|
1619
|
+
throw signal.reason ?? new Error("Aborted");
|
|
1620
|
+
}
|
|
1621
|
+
try {
|
|
1622
|
+
await setTimeoutPromise(ms, void 0, { signal });
|
|
1623
|
+
} catch (err) {
|
|
1624
|
+
if (signal?.aborted && signal.reason) {
|
|
1625
|
+
throw signal.reason;
|
|
1626
|
+
}
|
|
1627
|
+
throw err;
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
var HttpClient = class {
|
|
1631
|
+
defaultCookies = {};
|
|
1632
|
+
defaultHeaders;
|
|
1633
|
+
cookieCache = /* @__PURE__ */ new Map();
|
|
1634
|
+
constructor(headers, cookies) {
|
|
1635
|
+
this.defaultHeaders = { ...DEFAULT_CHROME_HEADERS, ...headers };
|
|
1636
|
+
if (cookies) {
|
|
1637
|
+
this.defaultCookies = { ...cookies };
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
setCookie(name, value) {
|
|
1641
|
+
this.defaultCookies[name] = value;
|
|
1642
|
+
}
|
|
1643
|
+
setHeader(name, value) {
|
|
1644
|
+
this.defaultHeaders[name] = value;
|
|
1645
|
+
}
|
|
1646
|
+
clearCookieCache() {
|
|
1647
|
+
this.cookieCache.clear();
|
|
1648
|
+
}
|
|
1649
|
+
async getCachedCookies(filePath, domain) {
|
|
1650
|
+
const cacheKey = `${filePath}::${domain || ""}`;
|
|
1651
|
+
const cached = this.cookieCache.get(cacheKey);
|
|
1652
|
+
if (cached) {
|
|
1653
|
+
return cached;
|
|
1654
|
+
}
|
|
1655
|
+
const loaded = await loadCookiesFromPath(filePath, domain);
|
|
1656
|
+
this.cookieCache.set(cacheKey, loaded);
|
|
1657
|
+
return loaded;
|
|
1658
|
+
}
|
|
1659
|
+
async loadCookies(filePath, domain) {
|
|
1660
|
+
const loaded = await this.getCachedCookies(filePath, domain);
|
|
1661
|
+
Object.assign(this.defaultCookies, loaded);
|
|
1662
|
+
}
|
|
1663
|
+
async request(url, options = {}) {
|
|
1664
|
+
const retries = options.retries ?? 3;
|
|
1665
|
+
const retryDelay = options.retryDelayMs ?? 1500;
|
|
1666
|
+
let cookieHeader = "";
|
|
1667
|
+
if (options.cookiePath) {
|
|
1668
|
+
const fromPath = await this.getCachedCookies(options.cookiePath, options.cookieDomain);
|
|
1669
|
+
const merged = { ...this.defaultCookies, ...fromPath };
|
|
1670
|
+
cookieHeader = formatCookieHeader(merged);
|
|
1671
|
+
} else if (typeof options.cookies === "string") {
|
|
1672
|
+
cookieHeader = options.cookies;
|
|
1673
|
+
} else if (options.cookies) {
|
|
1674
|
+
const merged = { ...this.defaultCookies, ...options.cookies };
|
|
1675
|
+
cookieHeader = formatCookieHeader(merged);
|
|
1676
|
+
} else if (Object.keys(this.defaultCookies).length > 0) {
|
|
1677
|
+
cookieHeader = formatCookieHeader(this.defaultCookies);
|
|
1678
|
+
}
|
|
1679
|
+
const headers = {
|
|
1680
|
+
...this.defaultHeaders,
|
|
1681
|
+
...options.headers
|
|
1682
|
+
};
|
|
1683
|
+
if (cookieHeader) {
|
|
1684
|
+
headers["Cookie"] = cookieHeader;
|
|
1685
|
+
}
|
|
1686
|
+
let lastError;
|
|
1687
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
1688
|
+
if (options.signal?.aborted) {
|
|
1689
|
+
throw options.signal.reason ?? new Error("Aborted");
|
|
1690
|
+
}
|
|
1691
|
+
try {
|
|
1692
|
+
const response = await fetch(url, {
|
|
1693
|
+
...options,
|
|
1694
|
+
headers
|
|
1695
|
+
});
|
|
1696
|
+
if (response.status === 429) {
|
|
1697
|
+
if (attempt < retries) {
|
|
1698
|
+
try {
|
|
1699
|
+
await response.body?.cancel();
|
|
1700
|
+
} catch {
|
|
1701
|
+
}
|
|
1702
|
+
await sleep(retryDelay * (attempt + 1) * 2, void 0, options.signal || void 0);
|
|
1703
|
+
continue;
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
if (response.status >= 500 && attempt < retries) {
|
|
1707
|
+
try {
|
|
1708
|
+
await response.body?.cancel();
|
|
1709
|
+
} catch {
|
|
1710
|
+
}
|
|
1711
|
+
await sleep(retryDelay * (attempt + 1), void 0, options.signal || void 0);
|
|
1712
|
+
continue;
|
|
1713
|
+
}
|
|
1714
|
+
return response;
|
|
1715
|
+
} catch (err) {
|
|
1716
|
+
lastError = err;
|
|
1717
|
+
if (options.signal?.aborted) throw err;
|
|
1718
|
+
if (attempt < retries) {
|
|
1719
|
+
await sleep(retryDelay * (attempt + 1), void 0, options.signal || void 0);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
throw lastError ?? new Error(`Request failed after ${retries} attempts: ${url}`);
|
|
1724
|
+
}
|
|
1725
|
+
async fetchJson(url, options = {}) {
|
|
1726
|
+
const res = await this.request(url, {
|
|
1727
|
+
...options,
|
|
1728
|
+
headers: {
|
|
1729
|
+
"Accept": "application/json, text/plain, */*",
|
|
1730
|
+
...options.headers
|
|
1731
|
+
}
|
|
1732
|
+
});
|
|
1733
|
+
if (!res.ok) {
|
|
1734
|
+
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1735
|
+
}
|
|
1736
|
+
return await res.json();
|
|
1737
|
+
}
|
|
1738
|
+
async fetchText(url, options = {}) {
|
|
1739
|
+
const res = await this.request(url, options);
|
|
1740
|
+
if (!res.ok) {
|
|
1741
|
+
throw new Error(`HTTP ${res.status} (${res.statusText}) for ${url}`);
|
|
1742
|
+
}
|
|
1743
|
+
return await res.text();
|
|
1744
|
+
}
|
|
1745
|
+
};
|
|
1746
|
+
|
|
1747
|
+
// src/lib/engines/extractors/instagram.ts
|
|
1748
|
+
var IG_APP_ID = "936619743392459";
|
|
1749
|
+
var IG_TIMELINE_QUERY_HASH = "69cba40317214236af40e7efa697781d";
|
|
1750
|
+
var IG_POLARIS_DOC_ID = "28975909992013618";
|
|
1751
|
+
function parseIgNodeToMediaItems(node, username) {
|
|
1752
|
+
const items = [];
|
|
1753
|
+
const anyNode = node;
|
|
1754
|
+
const baseId = node.shortcode || node.id || anyNode.pk;
|
|
1755
|
+
const date = node.taken_at_timestamp || anyNode.taken_at;
|
|
1756
|
+
const childrenEdges = node.edge_sidecar_to_children?.edges;
|
|
1757
|
+
const carouselMedia = anyNode.carousel_media;
|
|
1758
|
+
const childrenList = childrenEdges ? childrenEdges.map((e) => e.node) : Array.isArray(carouselMedia) ? carouselMedia : [];
|
|
1759
|
+
if (childrenList.length > 0) {
|
|
1760
|
+
childrenList.forEach((childNode, index) => {
|
|
1761
|
+
const subItems = parseIgNodeToMediaItems(childNode, username);
|
|
1762
|
+
for (const item of subItems) {
|
|
1763
|
+
item.id = `${baseId}_${index + 1}`;
|
|
1764
|
+
item.filename = `${username}_${date || baseId}_${index + 1}`;
|
|
1765
|
+
items.push(item);
|
|
1766
|
+
}
|
|
1767
|
+
});
|
|
1768
|
+
return items;
|
|
1769
|
+
}
|
|
1770
|
+
const isVideo = Boolean(node.is_video || node.video_url || node.video_versions && node.video_versions.length > 0);
|
|
1771
|
+
if (isVideo) {
|
|
1772
|
+
const video = node.video_versions?.[0];
|
|
1773
|
+
const videoUrl = node.video_url || video?.url;
|
|
1774
|
+
if (videoUrl) {
|
|
1775
|
+
items.push({
|
|
1776
|
+
id: baseId,
|
|
1777
|
+
url: videoUrl,
|
|
1778
|
+
filename: `${username}_${date || baseId}`,
|
|
1779
|
+
extension: "mp4",
|
|
1780
|
+
type: "video",
|
|
1781
|
+
date,
|
|
1782
|
+
width: video?.width ?? node.dimensions?.width,
|
|
1783
|
+
height: video?.height ?? node.dimensions?.height,
|
|
1784
|
+
thumbnailUrl: node.display_url
|
|
1785
|
+
});
|
|
1786
|
+
}
|
|
1787
|
+
} else {
|
|
1788
|
+
const candidate = node.image_versions2?.candidates?.[0];
|
|
1789
|
+
const photoUrl = candidate?.url || node.display_url;
|
|
1790
|
+
if (photoUrl) {
|
|
1791
|
+
items.push({
|
|
1792
|
+
id: baseId,
|
|
1793
|
+
url: photoUrl,
|
|
1794
|
+
filename: `${username}_${date || baseId}`,
|
|
1795
|
+
extension: "jpg",
|
|
1796
|
+
type: "photo",
|
|
1797
|
+
date,
|
|
1798
|
+
width: candidate?.width ?? node.dimensions?.width,
|
|
1799
|
+
height: candidate?.height ?? node.dimensions?.height
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
return items;
|
|
1804
|
+
}
|
|
1805
|
+
function parseReelItemToMediaItem(item, username) {
|
|
1806
|
+
const id = item.id || item.pk || String(Date.now());
|
|
1807
|
+
const date = item.taken_at || item.created_at;
|
|
1808
|
+
const isVideo = item.media_type === 2 || Boolean(item.video_versions && item.video_versions.length > 0);
|
|
1809
|
+
if (isVideo) {
|
|
1810
|
+
const video = item.video_versions?.[0];
|
|
1811
|
+
const videoUrl = video?.url;
|
|
1812
|
+
if (!videoUrl) return null;
|
|
1813
|
+
return {
|
|
1814
|
+
id,
|
|
1815
|
+
url: videoUrl,
|
|
1816
|
+
filename: `${username}_${date || id}`,
|
|
1817
|
+
extension: "mp4",
|
|
1818
|
+
type: "video",
|
|
1819
|
+
date,
|
|
1820
|
+
width: video?.width ?? item.original_width,
|
|
1821
|
+
height: video?.height ?? item.original_height,
|
|
1822
|
+
thumbnailUrl: item.image_versions2?.candidates?.[0]?.url
|
|
1823
|
+
};
|
|
1824
|
+
} else {
|
|
1825
|
+
const candidate = item.image_versions2?.candidates?.[0];
|
|
1826
|
+
const photoUrl = candidate?.url;
|
|
1827
|
+
if (!photoUrl) return null;
|
|
1828
|
+
return {
|
|
1829
|
+
id,
|
|
1830
|
+
url: photoUrl,
|
|
1831
|
+
filename: `${username}_${date || id}`,
|
|
1832
|
+
extension: "jpg",
|
|
1833
|
+
type: "photo",
|
|
1834
|
+
date,
|
|
1835
|
+
width: candidate?.width ?? item.original_width,
|
|
1836
|
+
height: candidate?.height ?? item.original_height
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
var InstagramExtractor = class {
|
|
1841
|
+
platform = "instagram";
|
|
1842
|
+
httpClient;
|
|
1843
|
+
constructor(httpClient) {
|
|
1844
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
1845
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1846
|
+
"Referer": "https://www.instagram.com/"
|
|
1847
|
+
});
|
|
1848
|
+
}
|
|
1849
|
+
async fetchUserProfile(username, options) {
|
|
1850
|
+
const url = `https://www.instagram.com/api/v1/users/web_profile_info/?username=${encodeURIComponent(username)}`;
|
|
1851
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1852
|
+
cookiePath: options.cookiePath,
|
|
1853
|
+
cookieDomain: "instagram.com",
|
|
1854
|
+
signal: options.signal,
|
|
1855
|
+
headers: {
|
|
1856
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1857
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
1858
|
+
}
|
|
1859
|
+
});
|
|
1860
|
+
if (!res.data?.user) {
|
|
1861
|
+
throw new Error(`Instagram user not found or private profile: ${username}`);
|
|
1862
|
+
}
|
|
1863
|
+
return res.data.user;
|
|
1864
|
+
}
|
|
1865
|
+
async fetchHighlightsTray(userId, options) {
|
|
1866
|
+
const url = `https://www.instagram.com/api/v1/highlights/${userId}/highlights_tray/`;
|
|
1867
|
+
try {
|
|
1868
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1869
|
+
cookiePath: options.cookiePath,
|
|
1870
|
+
cookieDomain: "instagram.com",
|
|
1871
|
+
signal: options.signal,
|
|
1872
|
+
headers: {
|
|
1873
|
+
"X-IG-App-ID": IG_APP_ID
|
|
1874
|
+
}
|
|
1875
|
+
});
|
|
1876
|
+
return res.tray ?? [];
|
|
1877
|
+
} catch {
|
|
1878
|
+
return [];
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
async fetchReelsMedia(reelIds, options) {
|
|
1882
|
+
if (reelIds.length === 0) return [];
|
|
1883
|
+
const idsParam = reelIds.join(",");
|
|
1884
|
+
const url = `https://www.instagram.com/api/v1/feed/reels_media/?reel_ids=${encodeURIComponent(idsParam)}`;
|
|
1885
|
+
try {
|
|
1886
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1887
|
+
cookiePath: options.cookiePath,
|
|
1888
|
+
cookieDomain: "instagram.com",
|
|
1889
|
+
signal: options.signal,
|
|
1890
|
+
headers: {
|
|
1891
|
+
"X-IG-App-ID": IG_APP_ID
|
|
1892
|
+
}
|
|
1893
|
+
});
|
|
1894
|
+
if (res.reels_media) return res.reels_media;
|
|
1895
|
+
if (res.reels) return Object.values(res.reels);
|
|
1896
|
+
return [];
|
|
1897
|
+
} catch {
|
|
1898
|
+
return [];
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
async fetchNextPostsPage(userId, username, cursor, options) {
|
|
1902
|
+
const variables = JSON.stringify({ id: userId, first: 12, after: cursor });
|
|
1903
|
+
const url = `https://www.instagram.com/graphql/query/?query_hash=${IG_TIMELINE_QUERY_HASH}&variables=${encodeURIComponent(variables)}`;
|
|
1904
|
+
try {
|
|
1905
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
1906
|
+
cookiePath: options.cookiePath,
|
|
1907
|
+
cookieDomain: "instagram.com",
|
|
1908
|
+
signal: options.signal,
|
|
1909
|
+
headers: {
|
|
1910
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1911
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
1912
|
+
}
|
|
1913
|
+
});
|
|
1914
|
+
const data = res?.data?.user?.edge_owner_to_timeline_media;
|
|
1915
|
+
if (data?.edges && data.edges.length > 0) {
|
|
1916
|
+
return {
|
|
1917
|
+
edges: data.edges,
|
|
1918
|
+
hasNextPage: Boolean(data.page_info?.has_next_page),
|
|
1919
|
+
nextCursor: data.page_info?.end_cursor
|
|
1920
|
+
};
|
|
1921
|
+
}
|
|
1922
|
+
} catch {
|
|
1923
|
+
}
|
|
1924
|
+
try {
|
|
1925
|
+
const postVariables = JSON.stringify({
|
|
1926
|
+
after: cursor,
|
|
1927
|
+
first: 12,
|
|
1928
|
+
username,
|
|
1929
|
+
data: { count: 12 }
|
|
1930
|
+
});
|
|
1931
|
+
const postUrl = "https://www.instagram.com/graphql/query";
|
|
1932
|
+
const body = new URLSearchParams({
|
|
1933
|
+
doc_id: IG_POLARIS_DOC_ID,
|
|
1934
|
+
variables: postVariables
|
|
1935
|
+
}).toString();
|
|
1936
|
+
const res = await this.httpClient.fetchJson(postUrl, {
|
|
1937
|
+
method: "POST",
|
|
1938
|
+
body,
|
|
1939
|
+
cookiePath: options.cookiePath,
|
|
1940
|
+
cookieDomain: "instagram.com",
|
|
1941
|
+
signal: options.signal,
|
|
1942
|
+
headers: {
|
|
1943
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
1944
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1945
|
+
"X-FB-Friendly-Name": "PolarisProfilePostsTabContentQuery_connection",
|
|
1946
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
1947
|
+
}
|
|
1948
|
+
});
|
|
1949
|
+
const connection = res?.data?.xdt_api__v1__feed__user_timeline_graphql_connection || res?.data?.user?.edge_owner_to_timeline_media;
|
|
1950
|
+
if (connection?.edges && connection.edges.length > 0) {
|
|
1951
|
+
return {
|
|
1952
|
+
edges: connection.edges,
|
|
1953
|
+
hasNextPage: Boolean(connection.page_info?.has_next_page),
|
|
1954
|
+
nextCursor: connection.page_info?.end_cursor
|
|
1955
|
+
};
|
|
1956
|
+
}
|
|
1957
|
+
} catch {
|
|
1958
|
+
}
|
|
1959
|
+
try {
|
|
1960
|
+
const restUrl = `https://www.instagram.com/api/v1/feed/user/${userId}/?count=12&max_id=${encodeURIComponent(cursor)}`;
|
|
1961
|
+
const res = await this.httpClient.fetchJson(restUrl, {
|
|
1962
|
+
cookiePath: options.cookiePath,
|
|
1963
|
+
cookieDomain: "instagram.com",
|
|
1964
|
+
signal: options.signal,
|
|
1965
|
+
headers: {
|
|
1966
|
+
"X-IG-App-ID": IG_APP_ID,
|
|
1967
|
+
"Referer": `https://www.instagram.com/${username}/`
|
|
1968
|
+
}
|
|
1969
|
+
});
|
|
1970
|
+
if (Array.isArray(res?.items) && res.items.length > 0) {
|
|
1971
|
+
const edges = res.items.map((item) => ({ node: item }));
|
|
1972
|
+
return {
|
|
1973
|
+
edges,
|
|
1974
|
+
hasNextPage: Boolean(res.more_available),
|
|
1975
|
+
nextCursor: res.next_max_id ? String(res.next_max_id) : void 0
|
|
1976
|
+
};
|
|
1977
|
+
}
|
|
1978
|
+
} catch {
|
|
1979
|
+
}
|
|
1980
|
+
return { edges: [], hasNextPage: false };
|
|
1981
|
+
}
|
|
1982
|
+
async *extract(profile, options) {
|
|
1983
|
+
const username = profile.username;
|
|
1984
|
+
const subDir = options.subDir;
|
|
1985
|
+
options.onProgress?.({
|
|
1986
|
+
stage: "Profile",
|
|
1987
|
+
found: 0,
|
|
1988
|
+
message: `Fetching Instagram profile @${username}...`
|
|
1989
|
+
});
|
|
1990
|
+
const user = await this.fetchUserProfile(username, options);
|
|
1991
|
+
const userId = user.id;
|
|
1992
|
+
if (subDir === "Stories") {
|
|
1993
|
+
options.onProgress?.({
|
|
1994
|
+
stage: "Stories",
|
|
1995
|
+
found: 0,
|
|
1996
|
+
message: `Fetching stories for @${username}...`
|
|
1997
|
+
});
|
|
1998
|
+
const reels = await this.fetchReelsMedia([userId], options);
|
|
1999
|
+
for (const reel of reels) {
|
|
2000
|
+
for (const rawItem of reel.items || []) {
|
|
2001
|
+
const item = parseReelItemToMediaItem(rawItem, username);
|
|
2002
|
+
if (item) yield item;
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
if (subDir === "Highlights") {
|
|
2008
|
+
options.onProgress?.({
|
|
2009
|
+
stage: "Highlights",
|
|
2010
|
+
found: 0,
|
|
2011
|
+
message: `Fetching highlights tray for @${username}...`
|
|
2012
|
+
});
|
|
2013
|
+
const tray = await this.fetchHighlightsTray(userId, options);
|
|
2014
|
+
const highlightIds = tray.map((t) => t.id).filter(Boolean);
|
|
2015
|
+
if (highlightIds.length > 0) {
|
|
2016
|
+
const chunkSize = 5;
|
|
2017
|
+
for (let i = 0; i < highlightIds.length; i += chunkSize) {
|
|
2018
|
+
if (options.signal?.aborted) break;
|
|
2019
|
+
const chunk = highlightIds.slice(i, i + chunkSize);
|
|
2020
|
+
const reels = await this.fetchReelsMedia(chunk, options);
|
|
2021
|
+
for (const reel of reels) {
|
|
2022
|
+
for (const rawItem of reel.items || []) {
|
|
2023
|
+
const item = parseReelItemToMediaItem(rawItem, username);
|
|
2024
|
+
if (item) yield item;
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
await sleep(1e3, 1500, options.signal);
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
return;
|
|
2031
|
+
}
|
|
2032
|
+
let found = 0;
|
|
2033
|
+
let currentEdges = user.edge_owner_to_timeline_media?.edges ?? [];
|
|
2034
|
+
let hasNextPage = Boolean(user.edge_owner_to_timeline_media?.page_info?.has_next_page);
|
|
2035
|
+
let endCursor = user.edge_owner_to_timeline_media?.page_info?.end_cursor;
|
|
2036
|
+
while (true) {
|
|
2037
|
+
for (const edge of currentEdges) {
|
|
2038
|
+
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2039
|
+
for (const item of items) {
|
|
2040
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2041
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2042
|
+
found++;
|
|
2043
|
+
options.onProgress?.({
|
|
2044
|
+
stage: subDir,
|
|
2045
|
+
found,
|
|
2046
|
+
message: `Found item: ${item.filename}`
|
|
2047
|
+
});
|
|
2048
|
+
yield item;
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
if (!hasNextPage || !endCursor || options.signal?.aborted) {
|
|
2052
|
+
break;
|
|
2053
|
+
}
|
|
2054
|
+
await sleep(1500, 2500, options.signal);
|
|
2055
|
+
try {
|
|
2056
|
+
const nextPage = await this.fetchNextPostsPage(userId, username, endCursor, options);
|
|
2057
|
+
currentEdges = nextPage.edges;
|
|
2058
|
+
hasNextPage = nextPage.hasNextPage;
|
|
2059
|
+
endCursor = nextPage.nextCursor;
|
|
2060
|
+
if (currentEdges.length === 0) {
|
|
2061
|
+
break;
|
|
2062
|
+
}
|
|
2063
|
+
} catch {
|
|
2064
|
+
break;
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
if (subDir === "Videos") {
|
|
2068
|
+
const videoEdges = user.edge_felix_video_timeline?.edges ?? [];
|
|
2069
|
+
for (const edge of videoEdges) {
|
|
2070
|
+
const items = parseIgNodeToMediaItems(edge.node, username);
|
|
2071
|
+
for (const item of items) {
|
|
2072
|
+
if (item.type !== "video") continue;
|
|
2073
|
+
found++;
|
|
2074
|
+
options.onProgress?.({
|
|
2075
|
+
stage: "Videos",
|
|
2076
|
+
found,
|
|
2077
|
+
message: `Found reel: ${item.filename}`
|
|
2078
|
+
});
|
|
2079
|
+
yield item;
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
};
|
|
2085
|
+
|
|
2086
|
+
// src/lib/engines/extractors/x.ts
|
|
2087
|
+
var TWITTER_BEARER_TOKEN = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
|
2088
|
+
var TWITTER_FEATURES = {
|
|
2089
|
+
responsive_web_graphql_exclude_directive_enabled: true,
|
|
2090
|
+
verified_phone_label_enabled: false,
|
|
2091
|
+
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
2092
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
2093
|
+
tweetypie_unmention_optimization_enabled: true,
|
|
2094
|
+
vibe_api_enabled: true,
|
|
2095
|
+
responsive_web_edit_tweet_api_enabled: true,
|
|
2096
|
+
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
2097
|
+
view_counts_everywhere_api_enabled: true,
|
|
2098
|
+
longform_notetweets_consumption_enabled: true,
|
|
2099
|
+
tweet_awards_web_tipping_enabled: false,
|
|
2100
|
+
freedom_of_speech_not_reach_fetch_enabled: true,
|
|
2101
|
+
standardized_nudges_misinfo: true,
|
|
2102
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
2103
|
+
interactive_text_enabled: true,
|
|
2104
|
+
responsive_web_text_conversations_enabled: false,
|
|
2105
|
+
longform_notetweets_rich_text_read_enabled: true,
|
|
2106
|
+
longform_notetweets_inline_media_enabled: true,
|
|
2107
|
+
responsive_web_enhance_cards_enabled: false
|
|
2108
|
+
};
|
|
2109
|
+
function parseTweetMediaEntities(tweet, username) {
|
|
2110
|
+
const items = [];
|
|
2111
|
+
const tweetResult = tweet.tweet || tweet;
|
|
2112
|
+
const legacy = tweetResult.legacy || tweetResult;
|
|
2113
|
+
const entities = legacy.extended_entities?.media || legacy.entities?.media || [];
|
|
2114
|
+
const tweetId = tweetResult.rest_id || legacy.id_str || String(Date.now());
|
|
2115
|
+
const rawDate = legacy.created_at ? new Date(legacy.created_at).getTime() / 1e3 : void 0;
|
|
2116
|
+
for (let idx = 0; idx < entities.length; idx++) {
|
|
2117
|
+
const media = entities[idx];
|
|
2118
|
+
const subIndex = idx + 1;
|
|
2119
|
+
const baseFilename = `${username}_${tweetId}_${subIndex}`;
|
|
2120
|
+
if (media.type === "video" || media.type === "animated_gif") {
|
|
2121
|
+
const variants = (media.video_info?.variants || []).filter((v) => v.content_type === "video/mp4").sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0));
|
|
2122
|
+
const bestVariant = variants[0];
|
|
2123
|
+
if (bestVariant?.url) {
|
|
2124
|
+
items.push({
|
|
2125
|
+
id: `${tweetId}_${media.id_str}`,
|
|
2126
|
+
url: bestVariant.url,
|
|
2127
|
+
filename: baseFilename,
|
|
2128
|
+
extension: "mp4",
|
|
2129
|
+
type: "video",
|
|
2130
|
+
date: rawDate,
|
|
2131
|
+
thumbnailUrl: media.media_url_https
|
|
2132
|
+
});
|
|
2133
|
+
}
|
|
2134
|
+
} else {
|
|
2135
|
+
let photoUrl = media.media_url_https;
|
|
2136
|
+
if (photoUrl) {
|
|
2137
|
+
const cleanUrl = photoUrl.replace(/:[a-zA-Z]+$/, "");
|
|
2138
|
+
const urlWithOrig = cleanUrl.includes("?") ? `${cleanUrl}&name=orig` : `${cleanUrl}?name=orig`;
|
|
2139
|
+
const extMatch = photoUrl.match(/\.([a-zA-Z0-9]+)(?:$|\?)/);
|
|
2140
|
+
const ext = extMatch ? extMatch[1] : "jpg";
|
|
2141
|
+
items.push({
|
|
2142
|
+
id: `${tweetId}_${media.id_str}`,
|
|
2143
|
+
url: urlWithOrig,
|
|
2144
|
+
filename: baseFilename,
|
|
2145
|
+
extension: ext,
|
|
2146
|
+
type: "photo",
|
|
2147
|
+
date: rawDate,
|
|
2148
|
+
width: media.original_info?.width,
|
|
2149
|
+
height: media.original_info?.height
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
return items;
|
|
2155
|
+
}
|
|
2156
|
+
var XExtractor = class {
|
|
2157
|
+
platform = "x";
|
|
2158
|
+
httpClient;
|
|
2159
|
+
guestToken;
|
|
2160
|
+
constructor(httpClient) {
|
|
2161
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
2162
|
+
"Authorization": `Bearer ${TWITTER_BEARER_TOKEN}`,
|
|
2163
|
+
"Referer": "https://x.com/",
|
|
2164
|
+
"x-twitter-active-user": "yes",
|
|
2165
|
+
"x-twitter-client-language": "en"
|
|
2166
|
+
});
|
|
2167
|
+
}
|
|
2168
|
+
async ensureGuestToken(options) {
|
|
2169
|
+
if (this.guestToken) return this.guestToken;
|
|
2170
|
+
try {
|
|
2171
|
+
const res = await this.httpClient.fetchJson(
|
|
2172
|
+
"https://api.twitter.com/1.1/guest/activate.json",
|
|
2173
|
+
{
|
|
2174
|
+
method: "POST",
|
|
2175
|
+
cookiePath: options.cookiePath,
|
|
2176
|
+
cookieDomain: "x.com",
|
|
2177
|
+
signal: options.signal,
|
|
2178
|
+
headers: {
|
|
2179
|
+
"Authorization": `Bearer ${TWITTER_BEARER_TOKEN}`
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
);
|
|
2183
|
+
if (res.guest_token) {
|
|
2184
|
+
this.guestToken = res.guest_token;
|
|
2185
|
+
this.httpClient.setHeader("x-guest-token", res.guest_token);
|
|
2186
|
+
return res.guest_token;
|
|
2187
|
+
}
|
|
2188
|
+
} catch (err) {
|
|
2189
|
+
if (options.signal?.aborted) throw err;
|
|
2190
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2191
|
+
throw new Error(`Failed to activate X/Twitter guest session: ${msg}`);
|
|
2192
|
+
}
|
|
2193
|
+
throw new Error("Failed to activate X/Twitter guest session: no guest token returned");
|
|
2194
|
+
}
|
|
2195
|
+
async fetchUserRestId(username, options) {
|
|
2196
|
+
await this.ensureGuestToken(options);
|
|
2197
|
+
const variables = JSON.stringify({
|
|
2198
|
+
screen_name: username,
|
|
2199
|
+
withSafetyModeUserFields: true
|
|
2200
|
+
});
|
|
2201
|
+
const features = JSON.stringify(TWITTER_FEATURES);
|
|
2202
|
+
const url = `https://x.com/i/api/graphql/ck5KkZ8t5cOmoLssopN99Q/UserByScreenName?variables=${encodeURIComponent(variables)}&features=${encodeURIComponent(features)}`;
|
|
2203
|
+
try {
|
|
2204
|
+
const res = await this.httpClient.fetchJson(url, {
|
|
2205
|
+
cookiePath: options.cookiePath,
|
|
2206
|
+
cookieDomain: "x.com",
|
|
2207
|
+
signal: options.signal
|
|
2208
|
+
});
|
|
2209
|
+
const restId = res.data?.user?.result?.rest_id;
|
|
2210
|
+
if (restId) return restId;
|
|
2211
|
+
} catch {
|
|
2212
|
+
}
|
|
2213
|
+
const syndiUrl = `https://cdn.syndication.twimg.com/widgets/followbutton/info.json?screen_names=${encodeURIComponent(username)}`;
|
|
2214
|
+
try {
|
|
2215
|
+
const res = await this.httpClient.fetchJson(syndiUrl, { signal: options.signal });
|
|
2216
|
+
if (res[0]?.id) return String(res[0].id);
|
|
2217
|
+
} catch {
|
|
2218
|
+
}
|
|
2219
|
+
throw new Error(`Failed to resolve X/Twitter user ID for @${username}`);
|
|
2220
|
+
}
|
|
2221
|
+
async *extract(profile, options) {
|
|
2222
|
+
const username = profile.username;
|
|
2223
|
+
const subDir = options.subDir;
|
|
2224
|
+
options.onProgress?.({
|
|
2225
|
+
stage: "Profile",
|
|
2226
|
+
found: 0,
|
|
2227
|
+
message: `Fetching X profile @${username}...`
|
|
2228
|
+
});
|
|
2229
|
+
const userId = await this.fetchUserRestId(username, options);
|
|
2230
|
+
const endpoint = "https://x.com/i/api/graphql/jCRhbOzdgOHp6u9H4g2tEg/UserMedia";
|
|
2231
|
+
let cursor;
|
|
2232
|
+
let hasMore = true;
|
|
2233
|
+
let found = 0;
|
|
2234
|
+
while (hasMore) {
|
|
2235
|
+
if (options.signal?.aborted) break;
|
|
2236
|
+
const variables = {
|
|
2237
|
+
userId,
|
|
2238
|
+
count: 50,
|
|
2239
|
+
includePromotedContent: false,
|
|
2240
|
+
withClientEventToken: false,
|
|
2241
|
+
withBirdwatchNotes: false,
|
|
2242
|
+
withVoice: true
|
|
2243
|
+
};
|
|
2244
|
+
if (cursor) {
|
|
2245
|
+
variables.cursor = cursor;
|
|
2246
|
+
}
|
|
2247
|
+
const features = JSON.stringify(TWITTER_FEATURES);
|
|
2248
|
+
const url = `${endpoint}?variables=${encodeURIComponent(JSON.stringify(variables))}&features=${encodeURIComponent(features)}`;
|
|
2249
|
+
let res;
|
|
2250
|
+
try {
|
|
2251
|
+
res = await this.httpClient.fetchJson(url, {
|
|
2252
|
+
cookiePath: options.cookiePath,
|
|
2253
|
+
cookieDomain: "x.com",
|
|
2254
|
+
signal: options.signal
|
|
2255
|
+
});
|
|
2256
|
+
} catch (err) {
|
|
2257
|
+
break;
|
|
2258
|
+
}
|
|
2259
|
+
const instructions = res?.data?.user?.result?.timeline_v2?.timeline?.instructions || [];
|
|
2260
|
+
let newCursor;
|
|
2261
|
+
let pageItemCount = 0;
|
|
2262
|
+
for (const inst of instructions) {
|
|
2263
|
+
const entries = inst.entries || (inst.entry ? [inst.entry] : []);
|
|
2264
|
+
for (const entry of entries) {
|
|
2265
|
+
if (entry.entryId?.startsWith("cursor-bottom-")) {
|
|
2266
|
+
newCursor = entry.content?.value;
|
|
2267
|
+
continue;
|
|
2268
|
+
}
|
|
2269
|
+
const tweetContent = entry.content?.itemContent?.tweet_results?.result;
|
|
2270
|
+
if (tweetContent) {
|
|
2271
|
+
const mediaItems = parseTweetMediaEntities(tweetContent, username);
|
|
2272
|
+
for (const item of mediaItems) {
|
|
2273
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2274
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2275
|
+
found++;
|
|
2276
|
+
pageItemCount++;
|
|
2277
|
+
options.onProgress?.({
|
|
2278
|
+
stage: subDir,
|
|
2279
|
+
found,
|
|
2280
|
+
message: `Found item: ${item.filename}`
|
|
2281
|
+
});
|
|
2282
|
+
yield item;
|
|
2283
|
+
}
|
|
2284
|
+
}
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2287
|
+
if (!newCursor || newCursor === cursor || pageItemCount === 0) {
|
|
2288
|
+
hasMore = false;
|
|
2289
|
+
} else {
|
|
2290
|
+
cursor = newCursor;
|
|
2291
|
+
await sleep(1500, 2500, options.signal);
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
};
|
|
2296
|
+
|
|
2297
|
+
// src/lib/engines/extractors/tiktok.ts
|
|
2298
|
+
function parseTikTokRehydrationJson(html) {
|
|
2299
|
+
const match = html.match(
|
|
2300
|
+
/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/
|
|
2301
|
+
);
|
|
2302
|
+
if (!match) return null;
|
|
2303
|
+
try {
|
|
2304
|
+
const raw = JSON.parse(match[1]);
|
|
2305
|
+
return raw?.["__DEFAULT_SCOPE__"];
|
|
2306
|
+
} catch {
|
|
2307
|
+
return null;
|
|
2308
|
+
}
|
|
2309
|
+
}
|
|
2310
|
+
function parseTikTokItem(item, username) {
|
|
2311
|
+
const items = [];
|
|
2312
|
+
const id = item.id || item.itemId || String(Date.now());
|
|
2313
|
+
const date = Number(item.createTime) || void 0;
|
|
2314
|
+
const caption = item.desc;
|
|
2315
|
+
const imageList = item.imagePost?.images;
|
|
2316
|
+
if (Array.isArray(imageList) && imageList.length > 0) {
|
|
2317
|
+
imageList.forEach((img, idx) => {
|
|
2318
|
+
const imgUrl = img.imageURL?.urlList?.[0] || img.displayImage?.urlList?.[0];
|
|
2319
|
+
if (imgUrl) {
|
|
2320
|
+
items.push({
|
|
2321
|
+
id: `${id}_${idx + 1}`,
|
|
2322
|
+
url: imgUrl,
|
|
2323
|
+
filename: `${username}_${id}_${idx + 1}`,
|
|
2324
|
+
extension: "jpg",
|
|
2325
|
+
type: "photo",
|
|
2326
|
+
caption,
|
|
2327
|
+
date
|
|
2328
|
+
});
|
|
2329
|
+
}
|
|
2330
|
+
});
|
|
2331
|
+
return items;
|
|
2332
|
+
}
|
|
2333
|
+
const video = item.video;
|
|
2334
|
+
const videoUrl = video?.playAddr || video?.downloadAddr;
|
|
2335
|
+
if (videoUrl) {
|
|
2336
|
+
items.push({
|
|
2337
|
+
id,
|
|
2338
|
+
url: videoUrl,
|
|
2339
|
+
filename: `${username}_${id}`,
|
|
2340
|
+
extension: "mp4",
|
|
2341
|
+
type: "video",
|
|
2342
|
+
caption,
|
|
2343
|
+
date,
|
|
2344
|
+
width: video?.width,
|
|
2345
|
+
height: video?.height,
|
|
2346
|
+
thumbnailUrl: video?.cover || video?.originCover
|
|
2347
|
+
});
|
|
2348
|
+
}
|
|
2349
|
+
return items;
|
|
2350
|
+
}
|
|
2351
|
+
var TikTokExtractor = class {
|
|
2352
|
+
platform = "tiktok";
|
|
2353
|
+
httpClient;
|
|
2354
|
+
constructor(httpClient) {
|
|
2355
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
2356
|
+
"Referer": "https://www.tiktok.com/"
|
|
2357
|
+
});
|
|
2358
|
+
}
|
|
2359
|
+
async *extract(profile, options) {
|
|
2360
|
+
const cleanUsername = profile.username.replace(/^@/, "");
|
|
2361
|
+
const subDir = options.subDir;
|
|
2362
|
+
options.onProgress?.({
|
|
2363
|
+
stage: "Profile",
|
|
2364
|
+
found: 0,
|
|
2365
|
+
message: `Fetching TikTok profile @${cleanUsername}...`
|
|
2366
|
+
});
|
|
2367
|
+
const profileUrl = `https://www.tiktok.com/@${cleanUsername}`;
|
|
2368
|
+
const html = await this.httpClient.fetchText(profileUrl, {
|
|
2369
|
+
cookiePath: options.cookiePath,
|
|
2370
|
+
cookieDomain: "tiktok.com",
|
|
2371
|
+
signal: options.signal
|
|
2372
|
+
});
|
|
2373
|
+
const scope = parseTikTokRehydrationJson(html);
|
|
2374
|
+
const userDetail = scope?.["webapp.user-detail"];
|
|
2375
|
+
const userInfo = userDetail?.userInfo?.user;
|
|
2376
|
+
const secUid = userInfo?.secUid;
|
|
2377
|
+
let found = 0;
|
|
2378
|
+
const initialItems = userDetail?.itemList || [];
|
|
2379
|
+
for (const rawItem of initialItems) {
|
|
2380
|
+
const mediaList = parseTikTokItem(rawItem, cleanUsername);
|
|
2381
|
+
for (const item of mediaList) {
|
|
2382
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2383
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2384
|
+
found++;
|
|
2385
|
+
options.onProgress?.({
|
|
2386
|
+
stage: subDir,
|
|
2387
|
+
found,
|
|
2388
|
+
message: `Found item: ${item.filename}`
|
|
2389
|
+
});
|
|
2390
|
+
yield item;
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
if (!secUid) {
|
|
2394
|
+
return;
|
|
2395
|
+
}
|
|
2396
|
+
let cursor = "0";
|
|
2397
|
+
let hasMore = true;
|
|
2398
|
+
const visitedCursors = /* @__PURE__ */ new Set();
|
|
2399
|
+
while (hasMore) {
|
|
2400
|
+
if (options.signal?.aborted) break;
|
|
2401
|
+
if (visitedCursors.has(cursor)) break;
|
|
2402
|
+
visitedCursors.add(cursor);
|
|
2403
|
+
const apiUrl = `https://www.tiktok.com/api/post/item_list/?secUid=${encodeURIComponent(secUid)}&count=30&cursor=${cursor}&post_item_list_request_type=0`;
|
|
2404
|
+
let res;
|
|
2405
|
+
try {
|
|
2406
|
+
res = await this.httpClient.fetchJson(apiUrl, {
|
|
2407
|
+
cookiePath: options.cookiePath,
|
|
2408
|
+
cookieDomain: "tiktok.com",
|
|
2409
|
+
signal: options.signal
|
|
2410
|
+
});
|
|
2411
|
+
} catch {
|
|
2412
|
+
break;
|
|
2413
|
+
}
|
|
2414
|
+
const items = res?.itemList || [];
|
|
2415
|
+
if (items.length === 0) break;
|
|
2416
|
+
for (const rawItem of items) {
|
|
2417
|
+
const mediaList = parseTikTokItem(rawItem, cleanUsername);
|
|
2418
|
+
for (const item of mediaList) {
|
|
2419
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2420
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2421
|
+
found++;
|
|
2422
|
+
options.onProgress?.({
|
|
2423
|
+
stage: subDir,
|
|
2424
|
+
found,
|
|
2425
|
+
message: `Found item: ${item.filename}`
|
|
2426
|
+
});
|
|
2427
|
+
yield item;
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
hasMore = Boolean(res?.hasMore);
|
|
2431
|
+
cursor = String(res?.cursor ?? "0");
|
|
2432
|
+
await sleep(1e3, 2e3, options.signal);
|
|
2433
|
+
}
|
|
2434
|
+
}
|
|
2435
|
+
};
|
|
2436
|
+
|
|
2437
|
+
// src/lib/engines/extractors/facebook.ts
|
|
2438
|
+
function extractFacebookMediaFromHtml(html, username) {
|
|
2439
|
+
const items = [];
|
|
2440
|
+
const seenUrls = /* @__PURE__ */ new Set();
|
|
2441
|
+
const hdMatch = html.match(/"browser_native_hd_url":"([^"]+)"/);
|
|
2442
|
+
const sdMatch = html.match(/"browser_native_sd_url":"([^"]+)"/);
|
|
2443
|
+
const rawVideoUrl = hdMatch?.[1] || sdMatch?.[1];
|
|
2444
|
+
if (rawVideoUrl) {
|
|
2445
|
+
try {
|
|
2446
|
+
const videoUrl = JSON.parse(`"${rawVideoUrl}"`);
|
|
2447
|
+
if (!seenUrls.has(videoUrl)) {
|
|
2448
|
+
seenUrls.add(videoUrl);
|
|
2449
|
+
items.push({
|
|
2450
|
+
id: `fb_video_${Date.now()}`,
|
|
2451
|
+
url: videoUrl,
|
|
2452
|
+
filename: `${username}_video_${Date.now()}`,
|
|
2453
|
+
extension: "mp4",
|
|
2454
|
+
type: "video"
|
|
2455
|
+
});
|
|
2456
|
+
}
|
|
2457
|
+
} catch {
|
|
2458
|
+
}
|
|
2459
|
+
}
|
|
2460
|
+
const photoRegex = /,"image":\{"uri":"([^"]+)"/g;
|
|
2461
|
+
let match;
|
|
2462
|
+
let photoIndex = 1;
|
|
2463
|
+
while ((match = photoRegex.exec(html)) !== null) {
|
|
2464
|
+
try {
|
|
2465
|
+
const rawUrl = match[1];
|
|
2466
|
+
const photoUrl = JSON.parse(`"${rawUrl}"`);
|
|
2467
|
+
if (!seenUrls.has(photoUrl)) {
|
|
2468
|
+
seenUrls.add(photoUrl);
|
|
2469
|
+
items.push({
|
|
2470
|
+
id: `fb_photo_${photoIndex}`,
|
|
2471
|
+
url: photoUrl,
|
|
2472
|
+
filename: `${username}_photo_${photoIndex}`,
|
|
2473
|
+
extension: "jpg",
|
|
2474
|
+
type: "photo"
|
|
2475
|
+
});
|
|
2476
|
+
photoIndex++;
|
|
2477
|
+
}
|
|
2478
|
+
} catch {
|
|
2479
|
+
}
|
|
2480
|
+
}
|
|
2481
|
+
if (items.length === 0) {
|
|
2482
|
+
const scontentRegex = /https:\/\/[^"'\s]*scontent[^"'\s]*\.(?:jpg|png|webp)[^"'\s]*/gi;
|
|
2483
|
+
let sMatch;
|
|
2484
|
+
while ((sMatch = scontentRegex.exec(html)) !== null) {
|
|
2485
|
+
const cleanUrl = sMatch[0].replace(/\\/g, "");
|
|
2486
|
+
if (!seenUrls.has(cleanUrl)) {
|
|
2487
|
+
seenUrls.add(cleanUrl);
|
|
2488
|
+
items.push({
|
|
2489
|
+
id: `fb_photo_${photoIndex}`,
|
|
2490
|
+
url: cleanUrl,
|
|
2491
|
+
filename: `${username}_photo_${photoIndex}`,
|
|
2492
|
+
extension: "jpg",
|
|
2493
|
+
type: "photo"
|
|
2494
|
+
});
|
|
2495
|
+
photoIndex++;
|
|
2496
|
+
}
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
return items;
|
|
2500
|
+
}
|
|
2501
|
+
var FacebookExtractor = class {
|
|
2502
|
+
platform = "facebook";
|
|
2503
|
+
httpClient;
|
|
2504
|
+
constructor(httpClient) {
|
|
2505
|
+
this.httpClient = httpClient ?? new HttpClient({
|
|
2506
|
+
"Referer": "https://www.facebook.com/"
|
|
2507
|
+
});
|
|
2508
|
+
}
|
|
2509
|
+
async *extract(profile, options) {
|
|
2510
|
+
const username = profile.username;
|
|
2511
|
+
const subDir = options.subDir;
|
|
2512
|
+
options.onProgress?.({
|
|
2513
|
+
stage: "Profile",
|
|
2514
|
+
found: 0,
|
|
2515
|
+
message: `Fetching Facebook profile @${username}...`
|
|
2516
|
+
});
|
|
2517
|
+
const targetUrl = subDir === "Photos" ? `https://www.facebook.com/${encodeURIComponent(username)}/photos` : `https://www.facebook.com/${encodeURIComponent(username)}/videos`;
|
|
2518
|
+
const html = await this.httpClient.fetchText(targetUrl, {
|
|
2519
|
+
cookiePath: options.cookiePath,
|
|
2520
|
+
cookieDomain: "facebook.com",
|
|
2521
|
+
signal: options.signal
|
|
2522
|
+
});
|
|
2523
|
+
const items = extractFacebookMediaFromHtml(html, username);
|
|
2524
|
+
let found = 0;
|
|
2525
|
+
for (const item of items) {
|
|
2526
|
+
if (subDir === "Photos" && item.type !== "photo") continue;
|
|
2527
|
+
if (subDir === "Videos" && item.type !== "video") continue;
|
|
2528
|
+
found++;
|
|
2529
|
+
options.onProgress?.({
|
|
2530
|
+
stage: subDir,
|
|
2531
|
+
found,
|
|
2532
|
+
message: `Found item: ${item.filename}`
|
|
2533
|
+
});
|
|
2534
|
+
yield item;
|
|
2535
|
+
}
|
|
2536
|
+
}
|
|
2537
|
+
};
|
|
2538
|
+
|
|
2539
|
+
// src/lib/engines/extractors/index.ts
|
|
2540
|
+
function getExtractor(platform) {
|
|
2541
|
+
switch (platform) {
|
|
2542
|
+
case "instagram":
|
|
2543
|
+
return new InstagramExtractor();
|
|
2544
|
+
case "x":
|
|
2545
|
+
return new XExtractor();
|
|
2546
|
+
case "tiktok":
|
|
2547
|
+
return new TikTokExtractor();
|
|
2548
|
+
case "facebook":
|
|
2549
|
+
return new FacebookExtractor();
|
|
2550
|
+
default:
|
|
2551
|
+
throw new Error(`Unsupported extractor platform: ${platform}`);
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2555
|
+
// src/lib/engines/downloader.ts
|
|
2556
|
+
import fs6 from "fs";
|
|
2557
|
+
import fsPromises2 from "fs/promises";
|
|
2558
|
+
import path5 from "path";
|
|
2559
|
+
import { Readable } from "stream";
|
|
2560
|
+
import { pipeline } from "stream/promises";
|
|
2561
|
+
var ARCHIVE_FILENAME = ".open-nami-archive.json";
|
|
2562
|
+
async function loadArchive(destDir) {
|
|
2563
|
+
const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
2564
|
+
try {
|
|
2565
|
+
const raw = await fsPromises2.readFile(archivePath, "utf-8");
|
|
2566
|
+
const parsed = JSON.parse(raw);
|
|
2567
|
+
if (Array.isArray(parsed)) {
|
|
2568
|
+
return new Set(parsed);
|
|
2569
|
+
}
|
|
2570
|
+
} catch {
|
|
2571
|
+
}
|
|
2572
|
+
return /* @__PURE__ */ new Set();
|
|
2573
|
+
}
|
|
2574
|
+
async function saveArchive(destDir, archive) {
|
|
2575
|
+
const archivePath = path5.join(destDir, ARCHIVE_FILENAME);
|
|
2576
|
+
const tmpPath = `${archivePath}.tmp`;
|
|
2577
|
+
try {
|
|
2578
|
+
await fsPromises2.writeFile(tmpPath, JSON.stringify([...archive]), "utf-8");
|
|
2579
|
+
await fsPromises2.rename(tmpPath, archivePath);
|
|
2580
|
+
} catch {
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
function sanitizeFilename(rawFilename, fallbackId, ext) {
|
|
2584
|
+
const cleanExt = ext.replace(/^\./, "");
|
|
2585
|
+
const baseOnly = path5.basename(rawFilename);
|
|
2586
|
+
let sanitizedBase = baseOnly.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_").replace(/[. ]+$/, "").trim();
|
|
2587
|
+
if (!sanitizedBase || /^_+$/.test(sanitizedBase)) {
|
|
2588
|
+
sanitizedBase = `item_${fallbackId}`;
|
|
2589
|
+
}
|
|
2590
|
+
if (/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$/i.test(sanitizedBase)) {
|
|
2591
|
+
sanitizedBase = `_${sanitizedBase}`;
|
|
2592
|
+
}
|
|
2593
|
+
return sanitizedBase.includes(".") ? sanitizedBase : `${sanitizedBase}.${cleanExt}`;
|
|
2594
|
+
}
|
|
2595
|
+
async function downloadMediaStream(items, options) {
|
|
2596
|
+
const { jobName, destDir, signal, onProgress } = options;
|
|
2597
|
+
const httpClient = options.httpClient ?? new HttpClient();
|
|
2598
|
+
await fsPromises2.mkdir(destDir, { recursive: true });
|
|
2599
|
+
const archive = await loadArchive(destDir);
|
|
2600
|
+
let downloaded = 0;
|
|
2601
|
+
let skipped = 0;
|
|
2602
|
+
const errors = [];
|
|
2603
|
+
for await (const item of items) {
|
|
2604
|
+
if (signal?.aborted) {
|
|
2605
|
+
break;
|
|
2606
|
+
}
|
|
2607
|
+
const ext = item.extension.replace(/^\./, "") || (item.type === "video" ? "mp4" : "jpg");
|
|
2608
|
+
const finalFilename = sanitizeFilename(item.filename, item.id, ext);
|
|
2609
|
+
const finalPath = path5.join(destDir, finalFilename);
|
|
2610
|
+
if (archive.has(item.id) && fs6.existsSync(finalPath)) {
|
|
2611
|
+
skipped++;
|
|
2612
|
+
onProgress?.({
|
|
2613
|
+
job: jobName,
|
|
2614
|
+
downloadedCount: downloaded,
|
|
2615
|
+
skippedCount: skipped,
|
|
2616
|
+
currentFile: finalFilename,
|
|
2617
|
+
statusText: `Skipping already downloaded: ${finalFilename}`
|
|
2618
|
+
});
|
|
2619
|
+
continue;
|
|
2620
|
+
}
|
|
2621
|
+
onProgress?.({
|
|
2622
|
+
job: jobName,
|
|
2623
|
+
downloadedCount: downloaded,
|
|
2624
|
+
skippedCount: skipped,
|
|
2625
|
+
currentFile: finalFilename,
|
|
2626
|
+
statusText: `Downloading: ${finalFilename}`
|
|
2627
|
+
});
|
|
2628
|
+
const tmpPath = `${finalPath}.download`;
|
|
2629
|
+
try {
|
|
2630
|
+
const reqHeaders = {};
|
|
2631
|
+
if (options.referer) {
|
|
2632
|
+
reqHeaders["Referer"] = options.referer;
|
|
2633
|
+
}
|
|
2634
|
+
const response = await httpClient.request(item.url, {
|
|
2635
|
+
signal,
|
|
2636
|
+
cookiePath: options.cookiePath,
|
|
2637
|
+
headers: reqHeaders
|
|
2638
|
+
});
|
|
2639
|
+
if (!response.ok || !response.body) {
|
|
2640
|
+
throw new Error(`HTTP ${response.status} (${response.statusText}) for ${item.url}`);
|
|
2641
|
+
}
|
|
2642
|
+
const fileStream = fs6.createWriteStream(tmpPath);
|
|
2643
|
+
await pipeline(Readable.fromWeb(response.body), fileStream, { signal });
|
|
2644
|
+
if (item.date) {
|
|
2645
|
+
try {
|
|
2646
|
+
const mtime = new Date(item.date * 1e3);
|
|
2647
|
+
await fsPromises2.utimes(tmpPath, mtime, mtime);
|
|
2648
|
+
} catch {
|
|
2649
|
+
}
|
|
2650
|
+
}
|
|
2651
|
+
await fsPromises2.rename(tmpPath, finalPath);
|
|
2652
|
+
archive.add(item.id);
|
|
2653
|
+
downloaded++;
|
|
2654
|
+
onProgress?.({
|
|
2655
|
+
job: jobName,
|
|
2656
|
+
downloadedCount: downloaded,
|
|
2657
|
+
skippedCount: skipped,
|
|
2658
|
+
currentFile: finalFilename,
|
|
2659
|
+
statusText: `Saved: ${finalFilename}`
|
|
2660
|
+
});
|
|
2661
|
+
} catch (err) {
|
|
2662
|
+
try {
|
|
2663
|
+
if (fs6.existsSync(tmpPath)) {
|
|
2664
|
+
await fsPromises2.unlink(tmpPath);
|
|
2665
|
+
}
|
|
2666
|
+
} catch {
|
|
2667
|
+
}
|
|
2668
|
+
if (signal?.aborted) {
|
|
2669
|
+
break;
|
|
2670
|
+
}
|
|
2671
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2672
|
+
errors.push(`${finalFilename}: ${errMsg}`);
|
|
2673
|
+
onProgress?.({
|
|
2674
|
+
job: jobName,
|
|
2675
|
+
downloadedCount: downloaded,
|
|
2676
|
+
skippedCount: skipped,
|
|
2677
|
+
currentFile: finalFilename,
|
|
2678
|
+
statusText: `Failed: ${finalFilename} (${errMsg})`
|
|
2679
|
+
});
|
|
2680
|
+
}
|
|
2681
|
+
}
|
|
2682
|
+
await saveArchive(destDir, archive);
|
|
2683
|
+
return { downloaded, skipped, errors };
|
|
2684
|
+
}
|
|
2685
|
+
|
|
1562
2686
|
// src/lib/engines/gallery-dl.ts
|
|
1563
2687
|
import { spawn as spawn2 } from "child_process";
|
|
1564
|
-
import
|
|
1565
|
-
import
|
|
2688
|
+
import path8 from "path";
|
|
2689
|
+
import fs9 from "fs/promises";
|
|
1566
2690
|
|
|
1567
2691
|
// src/lib/engines/exec.ts
|
|
1568
|
-
import
|
|
1569
|
-
import
|
|
2692
|
+
import fs8 from "fs";
|
|
2693
|
+
import path7 from "path";
|
|
1570
2694
|
|
|
1571
2695
|
// src/lib/engines/fetcher.ts
|
|
1572
2696
|
import { spawn } from "child_process";
|
|
1573
2697
|
import { createWriteStream } from "fs";
|
|
1574
|
-
import
|
|
2698
|
+
import fs7 from "fs/promises";
|
|
1575
2699
|
import os7 from "os";
|
|
1576
|
-
import
|
|
1577
|
-
import { Readable } from "stream";
|
|
1578
|
-
import { pipeline } from "stream/promises";
|
|
2700
|
+
import path6 from "path";
|
|
2701
|
+
import { Readable as Readable2 } from "stream";
|
|
2702
|
+
import { pipeline as pipeline2 } from "stream/promises";
|
|
1579
2703
|
function getOpenNamiBinDir(env = process.env) {
|
|
1580
2704
|
if (env.OPEN_NAMI_BIN_DIR && env.OPEN_NAMI_BIN_DIR.trim()) {
|
|
1581
|
-
return
|
|
2705
|
+
return path6.resolve(env.OPEN_NAMI_BIN_DIR.trim());
|
|
1582
2706
|
}
|
|
1583
2707
|
const base = env.OPEN_NAMI_DIR && env.OPEN_NAMI_DIR.trim() ? env.OPEN_NAMI_DIR.trim() : os7.homedir();
|
|
1584
|
-
return
|
|
2708
|
+
return path6.join(base, ".open-nami", "bin");
|
|
1585
2709
|
}
|
|
1586
2710
|
var YTDLP_RELEASE_BASE = "https://github.com/yt-dlp/yt-dlp/releases/latest/download";
|
|
1587
2711
|
var GALLERYDL_CODEBERG_API = "https://codeberg.org/api/v1/repos/mikf/gallery-dl/releases/latest";
|
|
@@ -1632,7 +2756,7 @@ function getBinaryVersion(executablePath) {
|
|
|
1632
2756
|
});
|
|
1633
2757
|
}
|
|
1634
2758
|
async function downloadBinary(url, targetFile, signal) {
|
|
1635
|
-
await
|
|
2759
|
+
await fs7.mkdir(path6.dirname(targetFile), { recursive: true });
|
|
1636
2760
|
const response = await fetch(url, {
|
|
1637
2761
|
signal,
|
|
1638
2762
|
headers: { "User-Agent": "open-nami" }
|
|
@@ -1641,18 +2765,30 @@ async function downloadBinary(url, targetFile, signal) {
|
|
|
1641
2765
|
throw new Error(`Failed to download binary from ${url} (HTTP ${response.status}). Check your connection and try again.`);
|
|
1642
2766
|
}
|
|
1643
2767
|
const tmp = `${targetFile}.download`;
|
|
1644
|
-
await pipeline(Readable.fromWeb(response.body), createWriteStream(tmp), { signal });
|
|
1645
2768
|
try {
|
|
1646
|
-
await
|
|
1647
|
-
|
|
2769
|
+
await pipeline2(Readable2.fromWeb(response.body), createWriteStream(tmp), { signal });
|
|
2770
|
+
try {
|
|
2771
|
+
await fs7.chmod(tmp, 493);
|
|
2772
|
+
} catch {
|
|
2773
|
+
}
|
|
2774
|
+
const stats = await fs7.stat(tmp);
|
|
2775
|
+
if (stats.size < 100 * 1024) {
|
|
2776
|
+
throw new Error(`Downloaded binary from ${url} is unexpectedly small (${stats.size} bytes). File may be incomplete or corrupted.`);
|
|
2777
|
+
}
|
|
2778
|
+
await fs7.rename(tmp, targetFile);
|
|
2779
|
+
return targetFile;
|
|
2780
|
+
} catch (err) {
|
|
2781
|
+
try {
|
|
2782
|
+
await fs7.unlink(tmp);
|
|
2783
|
+
} catch {
|
|
2784
|
+
}
|
|
2785
|
+
throw err;
|
|
1648
2786
|
}
|
|
1649
|
-
await fs5.rename(tmp, targetFile);
|
|
1650
|
-
return targetFile;
|
|
1651
2787
|
}
|
|
1652
2788
|
async function downloadLatestYtDlp(targetDir, signal, onStatus) {
|
|
1653
2789
|
const dir = targetDir || getOpenNamiBinDir();
|
|
1654
2790
|
const binaryName = process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp";
|
|
1655
|
-
const local =
|
|
2791
|
+
const local = path6.join(dir, binaryName);
|
|
1656
2792
|
const asset = ytDlpAssetName();
|
|
1657
2793
|
const url = `${YTDLP_RELEASE_BASE}/${asset}`;
|
|
1658
2794
|
onStatus?.(`fetching latest yt-dlp (${asset})\u2026`);
|
|
@@ -1688,7 +2824,7 @@ async function downloadLatestGalleryDl(targetDir, signal, onStatus) {
|
|
|
1688
2824
|
}
|
|
1689
2825
|
const dir = targetDir || getOpenNamiBinDir();
|
|
1690
2826
|
const binaryName = process.platform === "win32" ? "gallery-dl.exe" : "gallery-dl";
|
|
1691
|
-
const local =
|
|
2827
|
+
const local = path6.join(dir, binaryName);
|
|
1692
2828
|
const asset = galleryDlAssetName();
|
|
1693
2829
|
const candidateUrls = await getGalleryDlDownloadUrls(asset, fetch, signal);
|
|
1694
2830
|
let lastError;
|
|
@@ -1706,7 +2842,7 @@ async function downloadLatestGalleryDl(targetDir, signal, onStatus) {
|
|
|
1706
2842
|
async function ensureYtDlp(onStatus, signal) {
|
|
1707
2843
|
if (await commandWorks("yt-dlp", ["--version"])) return "yt-dlp";
|
|
1708
2844
|
const binaryName = process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp";
|
|
1709
|
-
const local =
|
|
2845
|
+
const local = path6.join(getOpenNamiBinDir(), binaryName);
|
|
1710
2846
|
if (await commandWorks(local, ["--version"])) return local;
|
|
1711
2847
|
if (await commandWorks("python", ["-m", "yt_dlp", "--version"])) return "python";
|
|
1712
2848
|
if (await commandWorks("python3", ["-m", "yt_dlp", "--version"])) return "python3";
|
|
@@ -1716,7 +2852,7 @@ async function ensureYtDlp(onStatus, signal) {
|
|
|
1716
2852
|
async function ensureGalleryDl(onStatus, signal) {
|
|
1717
2853
|
if (await commandWorks("gallery-dl", ["--version"])) return "gallery-dl";
|
|
1718
2854
|
const binaryName = process.platform === "win32" ? "gallery-dl.exe" : "gallery-dl";
|
|
1719
|
-
const local =
|
|
2855
|
+
const local = path6.join(getOpenNamiBinDir(), binaryName);
|
|
1720
2856
|
if (await commandWorks(local, ["--version"])) return local;
|
|
1721
2857
|
if (await commandWorks("python", ["-m", "gallery_dl", "--version"])) return "python";
|
|
1722
2858
|
if (await commandWorks("python3", ["-m", "gallery_dl", "--version"])) return "python3";
|
|
@@ -1729,7 +2865,7 @@ async function updateYtDlp(options) {
|
|
|
1729
2865
|
const signal = options?.signal;
|
|
1730
2866
|
const dir = getOpenNamiBinDir();
|
|
1731
2867
|
const binaryName = process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp";
|
|
1732
|
-
const local =
|
|
2868
|
+
const local = path6.join(dir, binaryName);
|
|
1733
2869
|
let previousVersion;
|
|
1734
2870
|
if (await commandWorks(local, ["--version"])) {
|
|
1735
2871
|
previousVersion = await getBinaryVersion(local);
|
|
@@ -1752,7 +2888,7 @@ async function updateGalleryDl(options) {
|
|
|
1752
2888
|
const signal = options?.signal;
|
|
1753
2889
|
const dir = getOpenNamiBinDir();
|
|
1754
2890
|
const binaryName = process.platform === "win32" ? "gallery-dl.exe" : "gallery-dl";
|
|
1755
|
-
const local =
|
|
2891
|
+
const local = path6.join(dir, binaryName);
|
|
1756
2892
|
let previousVersion;
|
|
1757
2893
|
if (await commandWorks(local, ["--version"])) {
|
|
1758
2894
|
previousVersion = await getBinaryVersion(local);
|
|
@@ -1773,16 +2909,16 @@ async function updateGalleryDl(options) {
|
|
|
1773
2909
|
// src/lib/engines/exec.ts
|
|
1774
2910
|
function findExecutable(binName, pathEnv = process.env.PATH || "") {
|
|
1775
2911
|
if (!pathEnv) return null;
|
|
1776
|
-
const delimiter =
|
|
2912
|
+
const delimiter = path7.delimiter;
|
|
1777
2913
|
const dirs = pathEnv.split(delimiter).filter(Boolean);
|
|
1778
2914
|
const isWin = process.platform === "win32";
|
|
1779
2915
|
const extensions = isWin ? [".exe", ".cmd", ".bat", ""] : [""];
|
|
1780
2916
|
for (const dir of dirs) {
|
|
1781
2917
|
for (const ext of extensions) {
|
|
1782
|
-
const fullPath =
|
|
2918
|
+
const fullPath = path7.join(dir, `${binName}${ext}`);
|
|
1783
2919
|
try {
|
|
1784
|
-
if (
|
|
1785
|
-
const stat =
|
|
2920
|
+
if (fs8.existsSync(fullPath)) {
|
|
2921
|
+
const stat = fs8.statSync(fullPath);
|
|
1786
2922
|
if (stat.isFile()) {
|
|
1787
2923
|
return fullPath;
|
|
1788
2924
|
}
|
|
@@ -1812,10 +2948,10 @@ function resolveEngineCommand(engine, options = {}) {
|
|
|
1812
2948
|
}
|
|
1813
2949
|
const binDir = getOpenNamiBinDir(env);
|
|
1814
2950
|
const isWin = process.platform === "win32";
|
|
1815
|
-
const localBin =
|
|
2951
|
+
const localBin = path7.join(binDir, `${engine}${isWin ? ".exe" : ""}`);
|
|
1816
2952
|
try {
|
|
1817
|
-
if (
|
|
1818
|
-
const stat =
|
|
2953
|
+
if (fs8.existsSync(localBin)) {
|
|
2954
|
+
const stat = fs8.statSync(localBin);
|
|
1819
2955
|
if (stat.isFile()) {
|
|
1820
2956
|
return {
|
|
1821
2957
|
command: localBin,
|
|
@@ -1872,7 +3008,7 @@ function buildGalleryDlArgs(options) {
|
|
|
1872
3008
|
"-o",
|
|
1873
3009
|
'headers.sec-ch-ua-platform="Windows"',
|
|
1874
3010
|
"--download-archive",
|
|
1875
|
-
|
|
3011
|
+
path8.join(destDir, GDL_ARCHIVE),
|
|
1876
3012
|
"--retries",
|
|
1877
3013
|
"3",
|
|
1878
3014
|
"--sleep-429",
|
|
@@ -1910,8 +3046,8 @@ function buildGalleryDlArgs(options) {
|
|
|
1910
3046
|
}
|
|
1911
3047
|
async function runGalleryDl(options) {
|
|
1912
3048
|
const { profile, targetDir, subDir, cookiePath, onProgress } = options;
|
|
1913
|
-
const destDir =
|
|
1914
|
-
await
|
|
3049
|
+
const destDir = path8.join(targetDir, subDir);
|
|
3050
|
+
await fs9.mkdir(destDir, { recursive: true });
|
|
1915
3051
|
const args = buildGalleryDlArgs({
|
|
1916
3052
|
destDir,
|
|
1917
3053
|
subDir,
|
|
@@ -1953,7 +3089,7 @@ async function runGalleryDl(options) {
|
|
|
1953
3089
|
if (!trimmed) return;
|
|
1954
3090
|
if (trimmed.startsWith("#")) {
|
|
1955
3091
|
skippedCount++;
|
|
1956
|
-
const filename =
|
|
3092
|
+
const filename = path8.basename(trimmed.slice(1).trim());
|
|
1957
3093
|
onProgress?.({
|
|
1958
3094
|
job: subDir,
|
|
1959
3095
|
downloadedCount,
|
|
@@ -1961,9 +3097,9 @@ async function runGalleryDl(options) {
|
|
|
1961
3097
|
currentFile: filename,
|
|
1962
3098
|
statusText: `Skipped: ${filename}`
|
|
1963
3099
|
});
|
|
1964
|
-
} else if (trimmed.match(/\.(jpe?g|png|gif|webp|bmp|jfif|heic|avif|tiff|svg|mp4|mov|webm|mkv|avi|flv|m4v)$/i) || trimmed.includes(
|
|
3100
|
+
} else if (trimmed.match(/\.(jpe?g|png|gif|webp|bmp|jfif|heic|avif|tiff|svg|mp4|mov|webm|mkv|avi|flv|m4v)$/i) || trimmed.includes(path8.sep)) {
|
|
1965
3101
|
downloadedCount++;
|
|
1966
|
-
const filename =
|
|
3102
|
+
const filename = path8.basename(trimmed);
|
|
1967
3103
|
onProgress?.({
|
|
1968
3104
|
job: subDir,
|
|
1969
3105
|
downloadedCount,
|
|
@@ -2014,15 +3150,15 @@ async function runGalleryDl(options) {
|
|
|
2014
3150
|
|
|
2015
3151
|
// src/lib/engines/yt-dlp.ts
|
|
2016
3152
|
import { spawn as spawn3 } from "child_process";
|
|
2017
|
-
import
|
|
2018
|
-
import
|
|
3153
|
+
import path9 from "path";
|
|
3154
|
+
import fs10 from "fs/promises";
|
|
2019
3155
|
var YTDLP_ARCHIVE = "archive_yt-dlp.txt";
|
|
2020
3156
|
var UA2 = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
2021
3157
|
function buildYtDlpArgs(options) {
|
|
2022
3158
|
const { destDir, platform, cleanUrl, cookiePath } = options;
|
|
2023
3159
|
const args = [
|
|
2024
3160
|
"-o",
|
|
2025
|
-
|
|
3161
|
+
path9.join(destDir, "%(title).150B [%(id)s].%(ext)s"),
|
|
2026
3162
|
"--windows-filenames",
|
|
2027
3163
|
"--newline",
|
|
2028
3164
|
"--ignore-errors",
|
|
@@ -2030,7 +3166,7 @@ function buildYtDlpArgs(options) {
|
|
|
2030
3166
|
"--user-agent",
|
|
2031
3167
|
UA2,
|
|
2032
3168
|
"--download-archive",
|
|
2033
|
-
|
|
3169
|
+
path9.join(destDir, YTDLP_ARCHIVE),
|
|
2034
3170
|
"--concurrent-fragments",
|
|
2035
3171
|
"4",
|
|
2036
3172
|
"--retries",
|
|
@@ -2055,8 +3191,8 @@ function buildYtDlpArgs(options) {
|
|
|
2055
3191
|
}
|
|
2056
3192
|
async function runYtDlp(options) {
|
|
2057
3193
|
const { profile, targetDir, cookiePath, onProgress } = options;
|
|
2058
|
-
const destDir =
|
|
2059
|
-
await
|
|
3194
|
+
const destDir = path9.join(targetDir, "Videos");
|
|
3195
|
+
await fs10.mkdir(destDir, { recursive: true });
|
|
2060
3196
|
const args = buildYtDlpArgs({
|
|
2061
3197
|
destDir,
|
|
2062
3198
|
platform: profile.platform,
|
|
@@ -2104,7 +3240,7 @@ async function runYtDlp(options) {
|
|
|
2104
3240
|
});
|
|
2105
3241
|
} else if (trimmed.includes("[download] Destination:")) {
|
|
2106
3242
|
const dest = trimmed.replace("[download] Destination:", "").trim();
|
|
2107
|
-
const filename =
|
|
3243
|
+
const filename = path9.basename(dest);
|
|
2108
3244
|
onProgress?.({
|
|
2109
3245
|
job: "Videos",
|
|
2110
3246
|
downloadedCount,
|
|
@@ -2161,38 +3297,44 @@ async function runYtDlp(options) {
|
|
|
2161
3297
|
}
|
|
2162
3298
|
|
|
2163
3299
|
// src/lib/engines/dispatcher.ts
|
|
3300
|
+
var REFERER_MAP = {
|
|
3301
|
+
tiktok: "https://www.tiktok.com/",
|
|
3302
|
+
instagram: "https://www.instagram.com/",
|
|
3303
|
+
x: "https://x.com/",
|
|
3304
|
+
facebook: "https://www.facebook.com/"
|
|
3305
|
+
};
|
|
2164
3306
|
function expandPath(dir) {
|
|
2165
3307
|
if (dir.startsWith("~/") || dir.startsWith("~\\") || dir === "~") {
|
|
2166
|
-
return
|
|
3308
|
+
return path10.join(os8.homedir(), dir.slice(1));
|
|
2167
3309
|
}
|
|
2168
3310
|
return dir;
|
|
2169
3311
|
}
|
|
2170
3312
|
function resolveDownloadBaseDir(customBaseDir) {
|
|
2171
3313
|
if (customBaseDir && customBaseDir.trim()) {
|
|
2172
|
-
return
|
|
3314
|
+
return path10.resolve(expandPath(customBaseDir.trim()));
|
|
2173
3315
|
}
|
|
2174
3316
|
const configDir = loadConfig().downloadDir;
|
|
2175
3317
|
if (configDir && configDir.trim()) {
|
|
2176
|
-
return
|
|
3318
|
+
return path10.resolve(expandPath(configDir.trim()));
|
|
2177
3319
|
}
|
|
2178
3320
|
const envDir = process.env.OPEN_NAMI_DIR || process.env.NAMI_DIR;
|
|
2179
3321
|
if (envDir && envDir.trim()) {
|
|
2180
|
-
return
|
|
3322
|
+
return path10.resolve(expandPath(envDir.trim()));
|
|
2181
3323
|
}
|
|
2182
3324
|
if (process.env.NAMI_BASE_DIR && process.env.NAMI_BASE_DIR.trim()) {
|
|
2183
3325
|
const raw = process.env.NAMI_BASE_DIR.trim();
|
|
2184
|
-
return
|
|
3326
|
+
return path10.resolve(expandPath(raw.endsWith("downloads") ? raw : path10.join(raw, "downloads")));
|
|
2185
3327
|
}
|
|
2186
|
-
return
|
|
3328
|
+
return path10.join(resolvePlatformDownloadsDir(), "Open Nami");
|
|
2187
3329
|
}
|
|
2188
3330
|
function resolveDefaultDownloadDir(platform, username, baseDir) {
|
|
2189
3331
|
const root = resolveDownloadBaseDir(baseDir);
|
|
2190
|
-
return
|
|
3332
|
+
return path10.join(root, platform, username);
|
|
2191
3333
|
}
|
|
2192
3334
|
async function dispatchDownload(options) {
|
|
2193
3335
|
const { profile, choice, onProgress } = options;
|
|
2194
3336
|
const targetDir = options.outputDir || resolveDefaultDownloadDir(profile.platform, profile.username);
|
|
2195
|
-
await
|
|
3337
|
+
await fs11.mkdir(targetDir, { recursive: true });
|
|
2196
3338
|
const masterCookie = findCookieFile(profile.platform);
|
|
2197
3339
|
const cookieCopy = masterCookie ? await createSafeCookieCopy(masterCookie) : null;
|
|
2198
3340
|
let totalDownloaded = 0;
|
|
@@ -2216,6 +3358,39 @@ async function dispatchDownload(options) {
|
|
|
2216
3358
|
jobs.push({ type: "highlights" });
|
|
2217
3359
|
}
|
|
2218
3360
|
}
|
|
3361
|
+
const runNative = async (subDir) => {
|
|
3362
|
+
const dest = path10.join(targetDir, subDir);
|
|
3363
|
+
try {
|
|
3364
|
+
const extractor = getExtractor(profile.platform);
|
|
3365
|
+
const items = extractor.extract(profile, {
|
|
3366
|
+
subDir,
|
|
3367
|
+
cookiePath: cookieCopy?.filePath,
|
|
3368
|
+
signal: options.signal
|
|
3369
|
+
});
|
|
3370
|
+
const res = await downloadMediaStream(items, {
|
|
3371
|
+
jobName: `${profile.username} ${subDir}`,
|
|
3372
|
+
destDir: dest,
|
|
3373
|
+
cookiePath: cookieCopy?.filePath,
|
|
3374
|
+
referer: REFERER_MAP[profile.platform],
|
|
3375
|
+
signal: options.signal,
|
|
3376
|
+
onProgress
|
|
3377
|
+
});
|
|
3378
|
+
return {
|
|
3379
|
+
success: res.errors.length === 0,
|
|
3380
|
+
downloaded: res.downloaded,
|
|
3381
|
+
skipped: res.skipped,
|
|
3382
|
+
...res.errors.length > 0 ? { error: res.errors.join("; ") } : {}
|
|
3383
|
+
};
|
|
3384
|
+
} catch (err) {
|
|
3385
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
3386
|
+
return {
|
|
3387
|
+
success: false,
|
|
3388
|
+
downloaded: 0,
|
|
3389
|
+
skipped: 0,
|
|
3390
|
+
error: errMsg
|
|
3391
|
+
};
|
|
3392
|
+
}
|
|
3393
|
+
};
|
|
2219
3394
|
const runGdl = (subDir) => runGalleryDl({
|
|
2220
3395
|
profile,
|
|
2221
3396
|
targetDir,
|
|
@@ -2235,23 +3410,33 @@ async function dispatchDownload(options) {
|
|
|
2235
3410
|
if (options.signal?.aborted) break;
|
|
2236
3411
|
if (job.type === "photos" || job.type === "stories" || job.type === "highlights") {
|
|
2237
3412
|
const subDir = job.type === "photos" ? "Photos" : job.type === "stories" ? "Stories" : "Highlights";
|
|
2238
|
-
|
|
3413
|
+
let res = await runNative(subDir);
|
|
3414
|
+
if ((!res.success || res.downloaded === 0 && res.skipped === 0) && !options.signal?.aborted) {
|
|
3415
|
+
const gdlRes = await runGdl(subDir);
|
|
3416
|
+
if (gdlRes.downloaded > 0 || gdlRes.skipped > 0 || gdlRes.success) {
|
|
3417
|
+
res = gdlRes;
|
|
3418
|
+
}
|
|
3419
|
+
}
|
|
2239
3420
|
totalDownloaded += res.downloaded;
|
|
2240
3421
|
totalSkipped += res.skipped;
|
|
2241
3422
|
if (!res.success && res.error) errors.push(`[${subDir}] ${res.error}`);
|
|
2242
3423
|
} else if (job.type === "videos") {
|
|
2243
|
-
|
|
2244
|
-
const primary = prefersGdl ? () => runGdl("Videos") : runYt;
|
|
2245
|
-
const fallback = prefersGdl ? runYt : () => runGdl("Videos");
|
|
2246
|
-
let res = await primary();
|
|
3424
|
+
let res = await runNative("Videos");
|
|
2247
3425
|
if (res.downloaded > 0 || res.skipped > 0) {
|
|
2248
3426
|
totalDownloaded += res.downloaded;
|
|
2249
3427
|
totalSkipped += res.skipped;
|
|
2250
3428
|
} else if (!options.signal?.aborted) {
|
|
2251
|
-
const fbRes = await
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
3429
|
+
const fbRes = await runYt();
|
|
3430
|
+
if (fbRes.downloaded > 0 || fbRes.skipped > 0 || fbRes.success) {
|
|
3431
|
+
totalDownloaded += fbRes.downloaded;
|
|
3432
|
+
totalSkipped += fbRes.skipped;
|
|
3433
|
+
if (!fbRes.success && fbRes.error) errors.push(`[Videos] ${fbRes.error}`);
|
|
3434
|
+
} else if (!options.signal?.aborted) {
|
|
3435
|
+
const gdlRes = await runGdl("Videos");
|
|
3436
|
+
totalDownloaded += gdlRes.downloaded;
|
|
3437
|
+
totalSkipped += gdlRes.skipped;
|
|
3438
|
+
if (!gdlRes.success && gdlRes.error) errors.push(`[Videos] ${gdlRes.error}`);
|
|
3439
|
+
}
|
|
2255
3440
|
}
|
|
2256
3441
|
}
|
|
2257
3442
|
}
|
|
@@ -2290,9 +3475,9 @@ function readClipboard(execFn = execFileSync2) {
|
|
|
2290
3475
|
}
|
|
2291
3476
|
|
|
2292
3477
|
// src/lib/reveal.ts
|
|
2293
|
-
import
|
|
3478
|
+
import fs12 from "fs";
|
|
2294
3479
|
import { spawn as spawn4 } from "child_process";
|
|
2295
|
-
import
|
|
3480
|
+
import path11 from "path";
|
|
2296
3481
|
function buildOpenBrowserCommand(url, platform = process.platform) {
|
|
2297
3482
|
try {
|
|
2298
3483
|
const parsed = new URL(url.trim());
|
|
@@ -2319,12 +3504,12 @@ function openBrowser(url) {
|
|
|
2319
3504
|
}
|
|
2320
3505
|
}
|
|
2321
3506
|
function getRevealCommand(targetPath, platform = process.platform, isDirectory) {
|
|
2322
|
-
const pathModule = platform === "win32" ?
|
|
3507
|
+
const pathModule = platform === "win32" ? path11.win32 : platform === "darwin" ? path11.posix : path11;
|
|
2323
3508
|
const resolved = pathModule.resolve(targetPath);
|
|
2324
3509
|
const isDir = isDirectory !== void 0 ? isDirectory : (() => {
|
|
2325
3510
|
try {
|
|
2326
|
-
if (
|
|
2327
|
-
return
|
|
3511
|
+
if (fs12.existsSync(resolved)) {
|
|
3512
|
+
return fs12.statSync(resolved).isDirectory();
|
|
2328
3513
|
}
|
|
2329
3514
|
} catch {
|
|
2330
3515
|
}
|