@postfetch/core 0.14.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1693,6 +1693,64 @@ async function fetchWebPost(net, id, input, initial) {
1693
1693
  }
1694
1694
  return null;
1695
1695
  }
1696
+ async function fetchEmbeddedPost(net, id) {
1697
+ const page = await fetchWebPage(net, `https://www.reddit.com/comments/${id}?embedded=true`, browserUserAgent());
1698
+ if (!page.ok)
1699
+ return null;
1700
+ for (const match of page.html.matchAll(/<div\s([^>]+)>/gi)) {
1701
+ const post = attributes(match[1]);
1702
+ if (post.id !== `thing_t3_${id}` || post["data-fullname"] !== `t3_${id}` || post["data-is-gallery"] !== "true")
1703
+ continue;
1704
+ const html = divContent(page.html, match.index + match[0].length);
1705
+ if (html === null)
1706
+ return null;
1707
+ const title = [...html.matchAll(/<a\s([^>]+)>([\s\S]*?)<\/a>/gi)].find((link) => attributes(link[1])["data-event-action"] === "title")?.[2];
1708
+ const preview = [...html.matchAll(/<div\s([^>]+)>/gi)].map((tag) => attributes(tag[1])).find((attrs) => attrs.id === `media-preview-${id}`);
1709
+ const ids = preview?.["data-media-ids"]?.split(",") ?? [];
1710
+ if (!title || !ids.length || ids.some((key) => !/^[a-z0-9]+$/i.test(key)) || new Set(ids).size !== ids.length)
1711
+ return null;
1712
+ const metadata = {};
1713
+ for (const link of html.matchAll(/<a\s([^>]+)>/gi)) {
1714
+ const attrs = attributes(link[1]);
1715
+ const url = new URL(attrs.href ?? "", "https://www.reddit.com");
1716
+ if (url.protocol !== "https:")
1717
+ continue;
1718
+ if (attrs.class?.split(/\s+/).includes("gallery-item-thumbnail-link")) {
1719
+ const index = Number(attrs["data-position"]) - 1;
1720
+ const key = ids[index];
1721
+ if (!Number.isSafeInteger(index) || !key || !/^(?:preview|i)\.redd\.it$/.test(url.hostname) || !new RegExp(`^/${key}\\.(?:png|jpe?g|webp)$`, "i").test(url.pathname))
1722
+ continue;
1723
+ metadata[key] = { s: { u: `https://i.redd.it${url.pathname}` } };
1724
+ } else if (/^(?:www\.)?reddit\.com$/.test(url.hostname)) {
1725
+ const video = url.pathname.match(/^\/link\/([a-z0-9]+)\/video\/([a-z0-9]+)\/player\/?$/i);
1726
+ if (video?.[1] === id && ids.includes(video[2])) {
1727
+ metadata[video[2]] = { e: "RedditVideo", dashUrl: `https://v.redd.it/${video[2]}/DASHPlaylist.mpd` };
1728
+ }
1729
+ }
1730
+ }
1731
+ if (ids.some((key) => !metadata[key]))
1732
+ return null;
1733
+ const body = [...html.matchAll(/<div\s([^>]+)>/gi)].find((tag) => attributes(tag[1]).class?.split(/\s+/).includes("usertext-body"));
1734
+ const text = body ? divContent(html, body.index + body[0].length) : null;
1735
+ const timestamp = finiteNumber(post["data-timestamp"]);
1736
+ return {
1737
+ id,
1738
+ title: plainText(title),
1739
+ selftext: text === null ? undefined : plainText(text.replace(/<a\s[^>]*>https:\/\/(?:www\.)?reddit\.com\/link\/[a-z0-9]+\/video\/[a-z0-9]+\/player<\/a>/gi, "")),
1740
+ author: post["data-author"],
1741
+ subreddit: post["data-subreddit"],
1742
+ permalink: post["data-permalink"],
1743
+ created_utc: timestamp === undefined ? undefined : Math.floor(timestamp / 1000),
1744
+ ups: finiteNumber(post["data-score"]),
1745
+ score: finiteNumber(post["data-score"]),
1746
+ num_comments: finiteNumber(post["data-comments-count"]),
1747
+ over_18: post["data-nsfw"] === "true" ? true : post["data-nsfw"] === "false" ? false : undefined,
1748
+ gallery_data: { items: ids.map((media_id) => ({ media_id })) },
1749
+ media_metadata: metadata
1750
+ };
1751
+ }
1752
+ return null;
1753
+ }
1696
1754
  async function fetchRssPost(net, id) {
1697
1755
  const url = `https://www.reddit.com/comments/${id}/.rss?limit=1`;
1698
1756
  const headers = { "user-agent": browserUserAgent(), accept: "application/atom+xml" };
@@ -1852,18 +1910,23 @@ function postText(html, id) {
1852
1910
  const attrs = attributes(match[1]);
1853
1911
  if (attrs.id !== `t3_${id}-post-rtjson-content`)
1854
1912
  continue;
1855
- const start = match.index + match[0].length;
1856
- const tags = /<div\b[^>]*>|<\/div\s*>/gi;
1857
- tags.lastIndex = start;
1858
- let depth = 1;
1859
- for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
1860
- depth += tag[0].startsWith("</") ? -1 : 1;
1861
- if (depth === 0)
1862
- return plainText(html.slice(start, tag.index));
1863
- }
1913
+ const content = divContent(html, match.index + match[0].length);
1914
+ if (content !== null)
1915
+ return plainText(content);
1864
1916
  }
1865
1917
  return;
1866
1918
  }
1919
+ function divContent(html, start) {
1920
+ const tags = /<div\b[^>]*>|<\/div\s*>/gi;
1921
+ tags.lastIndex = start;
1922
+ let depth = 1;
1923
+ for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
1924
+ depth += tag[0].startsWith("</") ? -1 : 1;
1925
+ if (depth === 0)
1926
+ return html.slice(start, tag.index);
1927
+ }
1928
+ return null;
1929
+ }
1867
1930
  function plainText(html) {
1868
1931
  return decodeHtml3(html.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<\/?(?:p|div|li|blockquote|pre|h[1-6])\b[^>]*>|<br\b[^>]*>/gi, `
1869
1932
  `).replace(/<[^>]*>/g, "")).replace(/[\t ]+/g, " ").replace(/ *\n */g, `
@@ -2004,6 +2067,9 @@ async function fetchPost(net, token, id, url, page) {
2004
2067
  const post2 = await fetchWebPost(net, id, url, page).catch(() => null);
2005
2068
  if (post2)
2006
2069
  return post2;
2070
+ const embedded = await fetchEmbeddedPost(net, id).catch(() => null);
2071
+ if (embedded)
2072
+ return embedded;
2007
2073
  const rss = await fetchRssPost(net, id).catch(() => null);
2008
2074
  if (rss)
2009
2075
  return rss;
@@ -2034,7 +2100,7 @@ async function mediaItems3(input, post, id) {
2034
2100
  return linkPreviewItems(post, id).concat(parent ? linkPreviewItems(parent, id) : []).slice(0, 1);
2035
2101
  }
2036
2102
  async function extract(input, post, id) {
2037
- const gallery = galleryItems(post, id);
2103
+ const gallery = await galleryMedia(input, post, id);
2038
2104
  if (gallery.length > 0) {
2039
2105
  return gallery;
2040
2106
  }
@@ -2044,7 +2110,28 @@ async function extract(input, post, id) {
2044
2110
  }
2045
2111
  return imageItems2(post, id);
2046
2112
  }
2047
- function galleryItems(post, id) {
2113
+ async function galleryMedia(input, post, id) {
2114
+ const metadata = object(post.media_metadata) ? post.media_metadata : {};
2115
+ if (!Object.values(metadata).some((meta) => object(meta) && meta.e === "RedditVideo"))
2116
+ return galleryItems(post, id);
2117
+ const data = object(post.gallery_data) ? post.gallery_data : null;
2118
+ const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
2119
+ const items = [];
2120
+ for (const [index, entry] of order.entries()) {
2121
+ const key = string(entry.media_id);
2122
+ const meta = key && object(metadata[key]) ? metadata[key] : null;
2123
+ if (meta?.e === "RedditVideo") {
2124
+ const videos = await videoItems(input, { secure_media: { reddit_video: { dash_url: meta.dashUrl } } }, id);
2125
+ if (!videos.length)
2126
+ throw new Error("Reddit gallery video unavailable");
2127
+ items.push(...videos.map((video) => ({ ...video, filename: filename(`reddit_${id}_${index + 1}.mp4`) })));
2128
+ } else {
2129
+ items.push(...galleryItems({ ...post, gallery_data: { items: [entry] } }, id, index + 1));
2130
+ }
2131
+ }
2132
+ return items;
2133
+ }
2134
+ function galleryItems(post, id, startIndex = 1) {
2048
2135
  const data = object(post.gallery_data) ? post.gallery_data : null;
2049
2136
  const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
2050
2137
  const metadata = object(post.media_metadata) ? post.media_metadata : {};
@@ -2057,10 +2144,10 @@ function galleryItems(post, id) {
2057
2144
  }
2058
2145
  const video = string(best.mp4);
2059
2146
  if (video) {
2060
- return [mediaItem(id, index + 1, "video", "video/mp4", video)];
2147
+ return [mediaItem(id, index + startIndex, "video", "video/mp4", video)];
2061
2148
  }
2062
2149
  const image = string(best.u) ?? string(best.gif);
2063
- return image ? [mediaItem(id, index + 1, "image", mimeOf(string(meta?.m), image), image)] : [];
2150
+ return image ? [mediaItem(id, index + startIndex, "image", mimeOf(string(meta?.m), image), image)] : [];
2064
2151
  });
2065
2152
  }
2066
2153
  async function videoItems(input, post, id) {
@@ -9,4 +9,5 @@ export type RedditPage = {
9
9
  export declare function readWebPage(response: Response, userAgent: string): Promise<RedditPage>;
10
10
  export declare function fetchWebPage(net: Net, input: string, userAgent: string, cookies?: string[]): Promise<RedditPage>;
11
11
  export declare function fetchWebPost(net: Net, id: string, input: string, initial?: RedditPage): Promise<Json | null>;
12
+ export declare function fetchEmbeddedPost(net: Net, id: string): Promise<Json | null>;
12
13
  export declare function fetchRssPost(net: Net, id: string): Promise<Json | null>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@postfetch/core",
3
- "version": "0.14.0",
3
+ "version": "0.14.1",
4
4
  "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
5
5
  "license": "MIT",
6
6
  "type": "module",
package/src/reddit-web.ts CHANGED
@@ -79,6 +79,59 @@ export async function fetchWebPost(net: Net, id: string, input: string, initial?
79
79
  return null;
80
80
  }
81
81
 
82
+ // The embedded view still serves legacy gallery markup when the main page is
83
+ // a CAPTCHA. Its media-id list includes videos that have no gallery thumbnail.
84
+ export async function fetchEmbeddedPost(net: Net, id: string): Promise<Json | null> {
85
+ const page = await fetchWebPage(net, `https://www.reddit.com/comments/${id}?embedded=true`, browserUserAgent());
86
+ if (!page.ok) return null;
87
+ for (const match of page.html.matchAll(/<div\s([^>]+)>/gi)) {
88
+ const post = attributes(match[1]);
89
+ if (post.id !== `thing_t3_${id}` || post["data-fullname"] !== `t3_${id}` || post["data-is-gallery"] !== "true") continue;
90
+ const html = divContent(page.html, match.index + match[0].length);
91
+ if (html === null) return null;
92
+ const title = [...html.matchAll(/<a\s([^>]+)>([\s\S]*?)<\/a>/gi)]
93
+ .find((link) => attributes(link[1])["data-event-action"] === "title")?.[2];
94
+ const preview = [...html.matchAll(/<div\s([^>]+)>/gi)]
95
+ .map((tag) => attributes(tag[1])).find((attrs) => attrs.id === `media-preview-${id}`);
96
+ const ids = preview?.["data-media-ids"]?.split(",") ?? [];
97
+ if (!title || !ids.length || ids.some((key) => !/^[a-z0-9]+$/i.test(key)) || new Set(ids).size !== ids.length) return null;
98
+ const metadata: Record<string, Json> = {};
99
+ for (const link of html.matchAll(/<a\s([^>]+)>/gi)) {
100
+ const attrs = attributes(link[1]);
101
+ const url = new URL(attrs.href ?? "", "https://www.reddit.com");
102
+ if (url.protocol !== "https:") continue;
103
+ if (attrs.class?.split(/\s+/).includes("gallery-item-thumbnail-link")) {
104
+ const index = Number(attrs["data-position"]) - 1;
105
+ const key = ids[index];
106
+ // Only the gallery's full-size link identifies an original. Never
107
+ // promote a tile, cover, comment image, or recommendation to an item.
108
+ if (!Number.isSafeInteger(index) || !key || !/^(?:preview|i)\.redd\.it$/.test(url.hostname)
109
+ || !new RegExp(`^/${key}\\.(?:png|jpe?g|webp)$`, "i").test(url.pathname)) continue;
110
+ metadata[key] = { s: { u: `https://i.redd.it${url.pathname}` } };
111
+ } else if (/^(?:www\.)?reddit\.com$/.test(url.hostname)) {
112
+ const video = url.pathname.match(/^\/link\/([a-z0-9]+)\/video\/([a-z0-9]+)\/player\/?$/i);
113
+ if (video?.[1] === id && ids.includes(video[2])) {
114
+ metadata[video[2]] = { e: "RedditVideo", dashUrl: `https://v.redd.it/${video[2]}/DASHPlaylist.mpd` };
115
+ }
116
+ }
117
+ }
118
+ if (ids.some((key) => !metadata[key])) return null;
119
+ const body = [...html.matchAll(/<div\s([^>]+)>/gi)]
120
+ .find((tag) => attributes(tag[1]).class?.split(/\s+/).includes("usertext-body"));
121
+ const text = body ? divContent(html, body.index + body[0].length) : null;
122
+ const timestamp = finiteNumber(post["data-timestamp"]);
123
+ return {
124
+ id, title: plainText(title), selftext: text === null ? undefined : plainText(text.replace(/<a\s[^>]*>https:\/\/(?:www\.)?reddit\.com\/link\/[a-z0-9]+\/video\/[a-z0-9]+\/player<\/a>/gi, "")),
125
+ author: post["data-author"], subreddit: post["data-subreddit"], permalink: post["data-permalink"],
126
+ created_utc: timestamp === undefined ? undefined : Math.floor(timestamp / 1000),
127
+ ups: finiteNumber(post["data-score"]), score: finiteNumber(post["data-score"]), num_comments: finiteNumber(post["data-comments-count"]),
128
+ over_18: post["data-nsfw"] === "true" ? true : post["data-nsfw"] === "false" ? false : undefined,
129
+ gallery_data: { items: ids.map((media_id) => ({ media_id })) }, media_metadata: metadata,
130
+ };
131
+ }
132
+ return null;
133
+ }
134
+
82
135
  // The public Atom feed remains usable on some routes that reject OAuth and
83
136
  // browser pages. Match the root entry explicitly: replies have their own media.
84
137
  export async function fetchRssPost(net: Net, id: string): Promise<Json | null> {
@@ -222,18 +275,23 @@ function postText(html: string, id: string): string | undefined {
222
275
  for (const match of html.matchAll(/<div\s([^>]+)>/gi)) {
223
276
  const attrs = attributes(match[1]);
224
277
  if (attrs.id !== `t3_${id}-post-rtjson-content`) continue;
225
- const start = match.index + match[0].length;
226
- const tags = /<div\b[^>]*>|<\/div\s*>/gi;
227
- tags.lastIndex = start;
228
- let depth = 1;
229
- for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
230
- depth += tag[0].startsWith("</") ? -1 : 1;
231
- if (depth === 0) return plainText(html.slice(start, tag.index));
232
- }
278
+ const content = divContent(html, match.index + match[0].length);
279
+ if (content !== null) return plainText(content);
233
280
  }
234
281
  return undefined;
235
282
  }
236
283
 
284
+ function divContent(html: string, start: number): string | null {
285
+ const tags = /<div\b[^>]*>|<\/div\s*>/gi;
286
+ tags.lastIndex = start;
287
+ let depth = 1;
288
+ for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
289
+ depth += tag[0].startsWith("</") ? -1 : 1;
290
+ if (depth === 0) return html.slice(start, tag.index);
291
+ }
292
+ return null;
293
+ }
294
+
237
295
  function plainText(html: string): string {
238
296
  return decodeHtml(html
239
297
  .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "")
package/src/reddit.ts CHANGED
@@ -17,7 +17,7 @@ import {
17
17
  type MediaItem,
18
18
  } from "./internal";
19
19
  import { browserUserAgent } from "./fingerprint";
20
- import { fetchWebPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
20
+ import { fetchWebPost, fetchEmbeddedPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
21
21
 
22
22
  // The installed-app token can still be issued while both OAuth post routes
23
23
  // reject it with Reddit's network-security page. Keep the public page as a
@@ -116,6 +116,8 @@ async function fetchPost(net: Net, token: string, id: string, url: string, page?
116
116
  // blocked. Keep the original failure if the page cannot supply the post.
117
117
  const post = await fetchWebPost(net, id, url, page).catch(() => null);
118
118
  if (post) return post;
119
+ const embedded = await fetchEmbeddedPost(net, id).catch(() => null);
120
+ if (embedded) return embedded;
119
121
  const rss = await fetchRssPost(net, id).catch(() => null);
120
122
  if (rss) return rss;
121
123
  }
@@ -147,7 +149,7 @@ async function mediaItems(input: ResolveContext, post: Json, id: string): Promis
147
149
  }
148
150
 
149
151
  async function extract(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
150
- const gallery = galleryItems(post, id);
152
+ const gallery = await galleryMedia(input, post, id);
151
153
  if (gallery.length > 0) {
152
154
  return gallery;
153
155
  }
@@ -158,7 +160,28 @@ async function extract(input: ResolveContext, post: Json, id: string): Promise<M
158
160
  return imageItems(post, id);
159
161
  }
160
162
 
161
- function galleryItems(post: Json, id: string): MediaItem[] {
163
+ async function galleryMedia(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
164
+ const metadata = object(post.media_metadata) ? post.media_metadata : {};
165
+ if (!Object.values(metadata).some((meta) => object(meta) && meta.e === "RedditVideo")) return galleryItems(post, id);
166
+ const data = object(post.gallery_data) ? post.gallery_data : null;
167
+ const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
168
+ const items: MediaItem[] = [];
169
+ for (const [index, entry] of order.entries()) {
170
+ const key = string(entry.media_id);
171
+ const meta = key && object(metadata[key]) ? metadata[key] : null;
172
+ if (meta?.e === "RedditVideo") {
173
+ const videos = await videoItems(input, { secure_media: { reddit_video: { dash_url: meta.dashUrl } } }, id);
174
+ if (!videos.length) throw new Error("Reddit gallery video unavailable");
175
+ items.push(...videos.map((video) => ({ ...video, filename: filename(`reddit_${id}_${index + 1}.mp4`) })));
176
+ } else {
177
+ // Retain the original position in filenames when videos split the images.
178
+ items.push(...galleryItems({ ...post, gallery_data: { items: [entry] } }, id, index + 1));
179
+ }
180
+ }
181
+ return items;
182
+ }
183
+
184
+ function galleryItems(post: Json, id: string, startIndex = 1): MediaItem[] {
162
185
  const data = object(post.gallery_data) ? post.gallery_data : null;
163
186
  const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
164
187
  const metadata = object(post.media_metadata) ? post.media_metadata : {};
@@ -171,10 +194,10 @@ function galleryItems(post: Json, id: string): MediaItem[] {
171
194
  }
172
195
  const video = string(best.mp4);
173
196
  if (video) {
174
- return [mediaItem(id, index + 1, "video", "video/mp4", video)];
197
+ return [mediaItem(id, index + startIndex, "video", "video/mp4", video)];
175
198
  }
176
199
  const image = string(best.u) ?? string(best.gif);
177
- return image ? [mediaItem(id, index + 1, "image", mimeOf(string(meta?.m), image), image)] : [];
200
+ return image ? [mediaItem(id, index + startIndex, "image", mimeOf(string(meta?.m), image), image)] : [];
178
201
  });
179
202
  }
180
203