@postfetch/core 0.13.5 → 0.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +131 -1
- package/package.json +1 -1
- package/src/reddit.ts +125 -1
package/dist/index.js
CHANGED
|
@@ -1706,7 +1706,12 @@ async function mediaItems3(input, post, id) {
|
|
|
1706
1706
|
return own;
|
|
1707
1707
|
}
|
|
1708
1708
|
const parents = Array.isArray(post.crosspost_parent_list) ? post.crosspost_parent_list.filter(object) : [];
|
|
1709
|
-
|
|
1709
|
+
const parent = parents[0];
|
|
1710
|
+
const inherited = parent ? await extract(input, parent, id) : [];
|
|
1711
|
+
if (inherited.length > 0) {
|
|
1712
|
+
return inherited;
|
|
1713
|
+
}
|
|
1714
|
+
return linkPreviewItems(post, id).concat(parent ? linkPreviewItems(parent, id) : []).slice(0, 1);
|
|
1710
1715
|
}
|
|
1711
1716
|
async function extract(input, post, id) {
|
|
1712
1717
|
const gallery = galleryItems(post, id);
|
|
@@ -1830,6 +1835,50 @@ function imageItems2(post, id) {
|
|
|
1830
1835
|
const isImage = string(post.post_hint) === "image" || asUrl(url).hostname === "i.redd.it" || /\.(jpe?g|png|webp|gif)$/i.test(path);
|
|
1831
1836
|
return isImage ? [mediaItem(id, null, "image", mimeOf(null, url), url)] : [];
|
|
1832
1837
|
}
|
|
1838
|
+
function linkPreviewItems(post, id) {
|
|
1839
|
+
const destination = string(post.url_overridden_by_dest) ?? string(post.url);
|
|
1840
|
+
if (post.is_self === true || !destination)
|
|
1841
|
+
return [];
|
|
1842
|
+
try {
|
|
1843
|
+
const url = new URL(destination);
|
|
1844
|
+
if (!/^https?:$/.test(url.protocol) || /(^|\.)(reddit\.com|redd\.it)$/.test(url.hostname))
|
|
1845
|
+
return [];
|
|
1846
|
+
} catch {
|
|
1847
|
+
return [];
|
|
1848
|
+
}
|
|
1849
|
+
const preview = object(post.preview) ? post.preview : null;
|
|
1850
|
+
const images = preview && Array.isArray(preview.images) ? preview.images.filter(object) : [];
|
|
1851
|
+
const candidates = images.flatMap((image) => {
|
|
1852
|
+
const source2 = object(image.source) ? [image.source] : [];
|
|
1853
|
+
const resolutions = Array.isArray(image.resolutions) ? image.resolutions.filter(object) : [];
|
|
1854
|
+
return [...source2, ...resolutions].flatMap((rendition, index) => {
|
|
1855
|
+
const rawUrl = string(rendition.url);
|
|
1856
|
+
if (!rawUrl)
|
|
1857
|
+
return [];
|
|
1858
|
+
try {
|
|
1859
|
+
const url = new URL(rawUrl.replace(/&/g, "&"));
|
|
1860
|
+
if (!/^https?:$/.test(url.protocol))
|
|
1861
|
+
return [];
|
|
1862
|
+
return [{
|
|
1863
|
+
url: url.href,
|
|
1864
|
+
original: source2.length > 0 && index === 0 ? 1 : 0,
|
|
1865
|
+
area: Math.max(0, number(rendition.width) ?? 0) * Math.max(0, number(rendition.height) ?? 0)
|
|
1866
|
+
}];
|
|
1867
|
+
} catch {
|
|
1868
|
+
return [];
|
|
1869
|
+
}
|
|
1870
|
+
});
|
|
1871
|
+
}).sort((a, b) => b.original - a.original || b.area - a.area);
|
|
1872
|
+
const best = candidates[0];
|
|
1873
|
+
if (!best)
|
|
1874
|
+
return [];
|
|
1875
|
+
const mime = mimeOf(null, best.url);
|
|
1876
|
+
return [{
|
|
1877
|
+
...mediaItem(id, null, "image", mime, best.url),
|
|
1878
|
+
filename: filename(`reddit_${id}_preview.${extensionOf(mime)}`),
|
|
1879
|
+
role: "linkPreview"
|
|
1880
|
+
}];
|
|
1881
|
+
}
|
|
1833
1882
|
function mediaItem(id, index, kind, mime, url) {
|
|
1834
1883
|
const suffix = index === null ? "" : `_${index}`;
|
|
1835
1884
|
return {
|
|
@@ -1880,6 +1929,10 @@ async function redditComments(input, token, id) {
|
|
|
1880
1929
|
try {
|
|
1881
1930
|
const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
|
|
1882
1931
|
const response = await input.net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=${Math.min(limit, 100)}&depth=1&sort=top`, { headers }, 1);
|
|
1932
|
+
if (response.status === 403) {
|
|
1933
|
+
await response.body?.cancel();
|
|
1934
|
+
return await graphqlComments(input.net, headers, id, limit);
|
|
1935
|
+
}
|
|
1883
1936
|
if (!response.ok)
|
|
1884
1937
|
throw new Error("Reddit comments failed");
|
|
1885
1938
|
const payload = await response.json();
|
|
@@ -1922,6 +1975,10 @@ async function redditComments(input, token, id) {
|
|
|
1922
1975
|
const url = new URL("https://oauth.reddit.com/api/morechildren");
|
|
1923
1976
|
url.search = new URLSearchParams({ api_type: "json", raw_json: "1", link_id: `t3_${id}`, children: children.join(","), sort: "top", depth: "1", limit_children: "true" }).toString();
|
|
1924
1977
|
const more = await input.net(url.href, { headers }, 1);
|
|
1978
|
+
if (more.status === 403) {
|
|
1979
|
+
await more.body?.cancel();
|
|
1980
|
+
return await graphqlComments(input.net, headers, id, limit);
|
|
1981
|
+
}
|
|
1925
1982
|
if (!more.ok)
|
|
1926
1983
|
throw new Error("Reddit more comments failed");
|
|
1927
1984
|
const result = await more.json();
|
|
@@ -1936,6 +1993,79 @@ async function redditComments(input, token, id) {
|
|
|
1936
1993
|
return [];
|
|
1937
1994
|
}
|
|
1938
1995
|
}
|
|
1996
|
+
var postCommentsHash = "314f29c2494fd78c7844ef2c6b09acb4bef5b37ab50b4ddc01691d6d5c8433f8";
|
|
1997
|
+
async function graphqlComments(net, headers, id, limit) {
|
|
1998
|
+
const comments = [];
|
|
1999
|
+
const seen = new Set;
|
|
2000
|
+
const cursors = new Set;
|
|
2001
|
+
let after;
|
|
2002
|
+
do {
|
|
2003
|
+
const response = await net("https://gql-fed.reddit.com/", {
|
|
2004
|
+
method: "POST",
|
|
2005
|
+
headers: { ...headers, "content-type": "application/json" },
|
|
2006
|
+
body: JSON.stringify({
|
|
2007
|
+
operationName: "PostComments",
|
|
2008
|
+
variables: {
|
|
2009
|
+
id: `t3_${id}`,
|
|
2010
|
+
sortType: "TOP",
|
|
2011
|
+
count: Math.min(200, limit - comments.length),
|
|
2012
|
+
maxDepth: 1,
|
|
2013
|
+
includeConsistencyToken: false,
|
|
2014
|
+
includePremiumAvatarTreatment: false,
|
|
2015
|
+
...after && { after }
|
|
2016
|
+
},
|
|
2017
|
+
extensions: { persistedQuery: { version: 1, sha256Hash: postCommentsHash } }
|
|
2018
|
+
})
|
|
2019
|
+
}, 1);
|
|
2020
|
+
if (!response.ok)
|
|
2021
|
+
throw new Error(`Reddit GraphQL comments failed: ${response.status}`);
|
|
2022
|
+
const payload = await response.json();
|
|
2023
|
+
const data = object(payload) && object(payload.data) ? payload.data : null;
|
|
2024
|
+
const post = data && object(data.postInfoById) ? data.postInfoById : null;
|
|
2025
|
+
const forest = post && object(post.commentForest) ? post.commentForest : null;
|
|
2026
|
+
if (!forest || !Array.isArray(forest.trees))
|
|
2027
|
+
throw new Error("Invalid Reddit GraphQL comments");
|
|
2028
|
+
after = undefined;
|
|
2029
|
+
for (const tree of forest.trees) {
|
|
2030
|
+
if (!object(tree) || tree.depth !== 0 || tree.parentId != null && tree.parentId !== `t3_${id}`)
|
|
2031
|
+
continue;
|
|
2032
|
+
if (object(tree.more)) {
|
|
2033
|
+
after = string(tree.more.cursor) ?? undefined;
|
|
2034
|
+
continue;
|
|
2035
|
+
}
|
|
2036
|
+
if (!object(tree.node) || comments.length >= limit)
|
|
2037
|
+
continue;
|
|
2038
|
+
const node = tree.node;
|
|
2039
|
+
const fullname = string(node.id);
|
|
2040
|
+
const content = object(node.content) ? node.content : null;
|
|
2041
|
+
const body = content && string(content.markdown);
|
|
2042
|
+
if (!fullname || !/^t1_[a-z0-9]+$/i.test(fullname) || seen.has(fullname) || !body || body === "[deleted]" || body === "[removed]" || node.__typename === "DeletedComment" || node.isRemoved === true || node.isAdminTakedown === true)
|
|
2043
|
+
continue;
|
|
2044
|
+
seen.add(fullname);
|
|
2045
|
+
const metadata = {};
|
|
2046
|
+
const media = Array.isArray(content.richtextMedia) ? content.richtextMedia.filter(object) : [];
|
|
2047
|
+
for (const asset of media) {
|
|
2048
|
+
const mediaId2 = string(asset.id);
|
|
2049
|
+
const url = string(asset.url);
|
|
2050
|
+
if (!mediaId2 || !url || asset.status && asset.status !== "VALID")
|
|
2051
|
+
continue;
|
|
2052
|
+
const mime = string(asset.mimetype);
|
|
2053
|
+
metadata[mediaId2] = { m: mime, s: mime === "video/mp4" ? { mp4: url } : { u: url } };
|
|
2054
|
+
}
|
|
2055
|
+
const createdAt = string(node.createdAt)?.replace(/(\.\d{3})\d+/, "$1").replace(/([+-]\d{2})(\d{2})$/, "$1:$2");
|
|
2056
|
+
comments.push(redditComment({
|
|
2057
|
+
author: object(node.authorInfo) ? node.authorInfo.name : undefined,
|
|
2058
|
+
created_utc: createdAt ? Math.floor(Date.parse(createdAt) / 1000) : undefined,
|
|
2059
|
+
ups: node.score,
|
|
2060
|
+
media_metadata: metadata
|
|
2061
|
+
}, fullname.slice(3), id, body));
|
|
2062
|
+
}
|
|
2063
|
+
if (!after || cursors.has(after))
|
|
2064
|
+
break;
|
|
2065
|
+
cursors.add(after);
|
|
2066
|
+
} while (comments.length < limit);
|
|
2067
|
+
return comments;
|
|
2068
|
+
}
|
|
1939
2069
|
function redditComment(data, id, postId2, body) {
|
|
1940
2070
|
const metadata = object(data.media_metadata) ? data.media_metadata : {};
|
|
1941
2071
|
const mediaIds = Object.keys(metadata).filter((key) => object(metadata[key])).sort((left, right) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.7",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/reddit.ts
CHANGED
|
@@ -128,7 +128,12 @@ async function mediaItems(input: ResolveContext, post: Json, id: string): Promis
|
|
|
128
128
|
return own;
|
|
129
129
|
}
|
|
130
130
|
const parents = Array.isArray(post.crosspost_parent_list) ? post.crosspost_parent_list.filter(object) : [];
|
|
131
|
-
|
|
131
|
+
const parent = parents[0];
|
|
132
|
+
const inherited = parent ? await extract(input, parent, id) : [];
|
|
133
|
+
if (inherited.length > 0) {
|
|
134
|
+
return inherited;
|
|
135
|
+
}
|
|
136
|
+
return linkPreviewItems(post, id).concat(parent ? linkPreviewItems(parent, id) : []).slice(0, 1);
|
|
132
137
|
}
|
|
133
138
|
|
|
134
139
|
async function extract(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
|
|
@@ -277,6 +282,48 @@ function imageItems(post: Json, id: string): MediaItem[] {
|
|
|
277
282
|
return isImage ? [mediaItem(id, null, "image", mimeOf(null, url), url)] : [];
|
|
278
283
|
}
|
|
279
284
|
|
|
285
|
+
function linkPreviewItems(post: Json, id: string): MediaItem[] {
|
|
286
|
+
// Reddit also supplies preview images for photos/videos and self posts. Only
|
|
287
|
+
// external link cards belong here; native attachments take precedence above.
|
|
288
|
+
const destination = string(post.url_overridden_by_dest) ?? string(post.url);
|
|
289
|
+
if (post.is_self === true || !destination) return [];
|
|
290
|
+
try {
|
|
291
|
+
const url = new URL(destination);
|
|
292
|
+
if (!/^https?:$/.test(url.protocol) || /(^|\.)(reddit\.com|redd\.it)$/.test(url.hostname)) return [];
|
|
293
|
+
} catch {
|
|
294
|
+
return [];
|
|
295
|
+
}
|
|
296
|
+
const preview = object(post.preview) ? post.preview : null;
|
|
297
|
+
const images = preview && Array.isArray(preview.images) ? preview.images.filter(object) : [];
|
|
298
|
+
const candidates = images.flatMap((image) => {
|
|
299
|
+
const source = object(image.source) ? [image.source] : [];
|
|
300
|
+
const resolutions = Array.isArray(image.resolutions) ? image.resolutions.filter(object) : [];
|
|
301
|
+
return [...source, ...resolutions].flatMap((rendition, index) => {
|
|
302
|
+
const rawUrl = string(rendition.url);
|
|
303
|
+
if (!rawUrl) return [];
|
|
304
|
+
try {
|
|
305
|
+
const url = new URL(rawUrl.replace(/&/g, "&"));
|
|
306
|
+
if (!/^https?:$/.test(url.protocol)) return [];
|
|
307
|
+
return [{
|
|
308
|
+
url: url.href,
|
|
309
|
+
original: source.length > 0 && index === 0 ? 1 : 0,
|
|
310
|
+
area: Math.max(0, number(rendition.width) ?? 0) * Math.max(0, number(rendition.height) ?? 0),
|
|
311
|
+
}];
|
|
312
|
+
} catch {
|
|
313
|
+
return [];
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
}).sort((a, b) => b.original - a.original || b.area - a.area);
|
|
317
|
+
const best = candidates[0];
|
|
318
|
+
if (!best) return [];
|
|
319
|
+
const mime = mimeOf(null, best.url);
|
|
320
|
+
return [{
|
|
321
|
+
...mediaItem(id, null, "image", mime, best.url),
|
|
322
|
+
filename: filename(`reddit_${id}_preview.${extensionOf(mime)}`),
|
|
323
|
+
role: "linkPreview",
|
|
324
|
+
}];
|
|
325
|
+
}
|
|
326
|
+
|
|
280
327
|
function mediaItem(id: string, index: number | null, kind: MediaItem["kind"], mime: string, url: string): MediaItem {
|
|
281
328
|
const suffix = index === null ? "" : `_${index}`;
|
|
282
329
|
return {
|
|
@@ -329,6 +376,10 @@ async function redditComments(input: ResolveContext, token: string, id: string):
|
|
|
329
376
|
try {
|
|
330
377
|
const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
|
|
331
378
|
const response = await input.net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=${Math.min(limit, 100)}&depth=1&sort=top`, { headers }, 1);
|
|
379
|
+
if (response.status === 403) {
|
|
380
|
+
await response.body?.cancel();
|
|
381
|
+
return await graphqlComments(input.net, headers, id, limit);
|
|
382
|
+
}
|
|
332
383
|
if (!response.ok) throw new Error("Reddit comments failed");
|
|
333
384
|
const payload: unknown = await response.json();
|
|
334
385
|
const listing = Array.isArray(payload) ? payload[1] : null;
|
|
@@ -364,6 +415,10 @@ async function redditComments(input: ResolveContext, token: string, id: string):
|
|
|
364
415
|
const url = new URL("https://oauth.reddit.com/api/morechildren");
|
|
365
416
|
url.search = new URLSearchParams({ api_type: "json", raw_json: "1", link_id: `t3_${id}`, children: children.join(","), sort: "top", depth: "1", limit_children: "true" }).toString();
|
|
366
417
|
const more = await input.net(url.href, { headers }, 1);
|
|
418
|
+
if (more.status === 403) {
|
|
419
|
+
await more.body?.cancel();
|
|
420
|
+
return await graphqlComments(input.net, headers, id, limit);
|
|
421
|
+
}
|
|
367
422
|
if (!more.ok) throw new Error("Reddit more comments failed");
|
|
368
423
|
const result: unknown = await more.json();
|
|
369
424
|
const json = object(result) && object(result.json) ? result.json : null;
|
|
@@ -377,6 +432,75 @@ async function redditComments(input: ResolveContext, token: string, id: string):
|
|
|
377
432
|
}
|
|
378
433
|
}
|
|
379
434
|
|
|
435
|
+
// Reddit blocks anonymous OAuth comment routes while its mobile GraphQL API
|
|
436
|
+
// still accepts the same token. This is the app's registered PostComments query;
|
|
437
|
+
// gql-fed rejects arbitrary query text. Keep the JSON API as the first choice.
|
|
438
|
+
const postCommentsHash = "314f29c2494fd78c7844ef2c6b09acb4bef5b37ab50b4ddc01691d6d5c8433f8";
|
|
439
|
+
|
|
440
|
+
async function graphqlComments(net: Net, headers: Record<string, string>, id: string, limit: number): Promise<PostComment[]> {
|
|
441
|
+
const comments: PostComment[] = [];
|
|
442
|
+
const seen = new Set<string>();
|
|
443
|
+
const cursors = new Set<string>();
|
|
444
|
+
let after: string | undefined;
|
|
445
|
+
do {
|
|
446
|
+
const response = await net("https://gql-fed.reddit.com/", {
|
|
447
|
+
method: "POST",
|
|
448
|
+
headers: { ...headers, "content-type": "application/json" },
|
|
449
|
+
body: JSON.stringify({
|
|
450
|
+
operationName: "PostComments",
|
|
451
|
+
variables: {
|
|
452
|
+
id: `t3_${id}`, sortType: "TOP", count: Math.min(200, limit - comments.length), maxDepth: 1,
|
|
453
|
+
includeConsistencyToken: false, includePremiumAvatarTreatment: false,
|
|
454
|
+
...(after && { after }),
|
|
455
|
+
},
|
|
456
|
+
extensions: { persistedQuery: { version: 1, sha256Hash: postCommentsHash } },
|
|
457
|
+
}),
|
|
458
|
+
}, 1);
|
|
459
|
+
if (!response.ok) throw new Error(`Reddit GraphQL comments failed: ${response.status}`);
|
|
460
|
+
const payload: unknown = await response.json();
|
|
461
|
+
const data = object(payload) && object(payload.data) ? payload.data : null;
|
|
462
|
+
const post = data && object(data.postInfoById) ? data.postInfoById : null;
|
|
463
|
+
const forest = post && object(post.commentForest) ? post.commentForest : null;
|
|
464
|
+
if (!forest || !Array.isArray(forest.trees)) throw new Error("Invalid Reddit GraphQL comments");
|
|
465
|
+
after = undefined;
|
|
466
|
+
for (const tree of forest.trees) {
|
|
467
|
+
if (!object(tree) || tree.depth !== 0 || (tree.parentId != null && tree.parentId !== `t3_${id}`)) continue;
|
|
468
|
+
if (object(tree.more)) {
|
|
469
|
+
after = string(tree.more.cursor) ?? undefined;
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
if (!object(tree.node) || comments.length >= limit) continue;
|
|
473
|
+
const node = tree.node;
|
|
474
|
+
const fullname = string(node.id);
|
|
475
|
+
const content = object(node.content) ? node.content : null;
|
|
476
|
+
const body = content && string(content.markdown);
|
|
477
|
+
if (!fullname || !/^t1_[a-z0-9]+$/i.test(fullname) || seen.has(fullname) || !body
|
|
478
|
+
|| body === "[deleted]" || body === "[removed]" || node.__typename === "DeletedComment" || node.isRemoved === true || node.isAdminTakedown === true) continue;
|
|
479
|
+
seen.add(fullname);
|
|
480
|
+
const metadata: Json = {};
|
|
481
|
+
const media = Array.isArray(content.richtextMedia) ? content.richtextMedia.filter(object) : [];
|
|
482
|
+
for (const asset of media) {
|
|
483
|
+
const mediaId = string(asset.id);
|
|
484
|
+
const url = string(asset.url);
|
|
485
|
+
if (!mediaId || !url || (asset.status && asset.status !== "VALID")) continue;
|
|
486
|
+
const mime = string(asset.mimetype);
|
|
487
|
+
metadata[mediaId] = { m: mime, s: mime === "video/mp4" ? { mp4: url } : { u: url } };
|
|
488
|
+
}
|
|
489
|
+
// The API uses microseconds and offsets like +0000; normalize for JS dates.
|
|
490
|
+
const createdAt = string(node.createdAt)?.replace(/(\.\d{3})\d+/, "$1").replace(/([+-]\d{2})(\d{2})$/, "$1:$2");
|
|
491
|
+
comments.push(redditComment({
|
|
492
|
+
author: object(node.authorInfo) ? node.authorInfo.name : undefined,
|
|
493
|
+
created_utc: createdAt ? Math.floor(Date.parse(createdAt) / 1000) : undefined,
|
|
494
|
+
ups: node.score,
|
|
495
|
+
media_metadata: metadata,
|
|
496
|
+
}, fullname.slice(3), id, body));
|
|
497
|
+
}
|
|
498
|
+
if (!after || cursors.has(after)) break;
|
|
499
|
+
cursors.add(after);
|
|
500
|
+
} while (comments.length < limit);
|
|
501
|
+
return comments;
|
|
502
|
+
}
|
|
503
|
+
|
|
380
504
|
function redditComment(data: Json, id: string, postId: string, body: string): PostComment {
|
|
381
505
|
const metadata = object(data.media_metadata) ? data.media_metadata : {};
|
|
382
506
|
// Comment images/GIFs use the same rendition metadata as Reddit galleries.
|