@postfetch/core 0.11.0 → 0.12.1

package/dist/index.js

@@ -1540,7 +1540,7 @@ async function resolveReddit(input) {
   const token = await appToken(input.net);
   const post = await fetchPost(input.net, token, id);
   const items = await mediaItems3(input, post, id);
-  return { archiveFilename: filename(`reddit_${id}.zip`), comments: [], id, items, metadata: redditMetadata(post), platform: "reddit" };
+  return { archiveFilename: filename(`reddit_${id}.zip`), comments: await redditComments(input, token, id), id, items, metadata: redditMetadata(post), platform: "reddit" };
 }
 function redditMetadata(post) {
   const author = string(post.author);
@@ -1790,6 +1790,102 @@ function extensionOf(mime) {
   }
   return "jpg";
 }
+async function redditComments(input, token, id) {
+  const limit = input.comments;
+  if (!limit || !Number.isSafeInteger(limit) || limit < 0)
+    return [];
+  try {
+    const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
+    const response = await input.net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=${Math.min(limit, 100)}&depth=1&sort=top`, { headers }, 1);
+    if (!response.ok)
+      throw new Error("Reddit comments failed");
+    const payload = await response.json();
+    const listing = Array.isArray(payload) ? payload[1] : null;
+    if (!object(listing) || !object(listing.data) || !Array.isArray(listing.data.children))
+      throw new Error("Invalid Reddit comments");
+    const comments = [];
+    const seen = new Set;
+    const queued = new Set;
+    const pending = [];
+    const collect = (things) => {
+      for (const thing of things) {
+        if (!object(thing) || !object(thing.data))
+          continue;
+        const data = thing.data;
+        if (data.parent_id !== `t3_${id}`)
+          continue;
+        if (thing.kind === "more" && Array.isArray(data.children)) {
+          for (const child of data.children) {
+            if (typeof child !== "string" || !/^[a-z0-9]+$/i.test(child) || queued.has(child))
+              continue;
+            queued.add(child);
+            pending.push(child);
+          }
+        } else if (thing.kind === "t1" && comments.length < limit) {
+          const commentId = string(data.id);
+          if (!commentId || seen.has(commentId))
+            continue;
+          seen.add(commentId);
+          const body = string(data.body);
+          if (!body || body === "[deleted]" || body === "[removed]")
+            continue;
+          comments.push(redditComment(data, commentId, id, body));
+        }
+      }
+    };
+    collect(listing.data.children);
+    while (comments.length < limit && pending.length > 0) {
+      const children = pending.splice(0, Math.min(100, limit - comments.length));
+      const url = new URL("https://oauth.reddit.com/api/morechildren");
+      url.search = new URLSearchParams({ api_type: "json", raw_json: "1", link_id: `t3_${id}`, children: children.join(","), sort: "top", depth: "1", limit_children: "true" }).toString();
+      const more = await input.net(url.href, { headers }, 1);
+      if (!more.ok)
+        throw new Error("Reddit more comments failed");
+      const result = await more.json();
+      const json = object(result) && object(result.json) ? result.json : null;
+      const data = json && object(json.data) ? json.data : null;
+      if (!data || !Array.isArray(data.things) || Array.isArray(json?.errors) && json.errors.length > 0)
+        throw new Error("Invalid additional Reddit comments");
+      collect(data.things);
+    }
+    return comments;
+  } catch {
+    return [];
+  }
+}
+function redditComment(data, id, postId2, body) {
+  const metadata = object(data.media_metadata) ? data.media_metadata : {};
+  const mediaIds = Object.keys(metadata).filter((key) => object(metadata[key])).sort((left, right) => {
+    const position = (key) => {
+      const meta = metadata[key];
+      const best = object(meta) && object(meta.s) ? meta.s : {};
+      const tokens = [`![img](${key})`, `![gif](${key})`, string(best.u), string(best.gif), string(best.mp4)];
+      const positions = tokens.flatMap((token) => token && body.includes(token) ? [body.indexOf(token)] : []);
+      return Math.min(...positions);
+    };
+    return position(left) - position(right);
+  });
+  const items = galleryItems({ media_metadata: metadata, gallery_data: { items: mediaIds.map((media_id) => ({ media_id })) } }, id);
+  const text = body.replace(/!\[(?:img|gif)\]\(([^)]+)\)/g, (match, key) => {
+    const meta = metadata[key];
+    return object(meta) && object(meta.s) && (string(meta.s.u) || string(meta.s.gif) || string(meta.s.mp4)) ? "" : match;
+  }).split(`
+`).filter((line) => !items.some((item) => line.trim() === item.url)).join(`
+`).trim();
+  const author = string(data.author);
+  return {
+    id,
+    url: `https://www.reddit.com/comments/${postId2}/_/${id}/`,
+    items,
+    metadata: {
+      text,
+      author: author && author !== "[deleted]" ? { handle: author } : undefined,
+      createdAt: isoFromEpochSeconds(data.created_utc),
+      likeCount: count(data.ups),
+      nsfw: bool(data.over_18)
+    }
+  };
+}
 
 // src/soundcloud.ts
 async function resolveSoundcloud(input) {
@@ -2212,11 +2308,14 @@ function twitterMetadata(tweet) {
   }
   const parent = object(tweet.parent) ? tweet.parent : null;
   const parentId = parent ? string(parent.id_str) : null;
+  const replyToId = string(tweet.in_reply_to_status_id_str) ?? parentId;
+  if (replyToId)
+    extra.replyToId = replyToId;
   if (parent && parentId) {
     extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
   }
   return {
-    text: twitterText(tweet) ?? undefined,
+    text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
     author: user ? {
       handle: string(user.screen_name) ?? undefined,
       name: string(user.name) ?? undefined,
@@ -2240,6 +2339,23 @@ function noteText(tweet) {
 function twitterText(tweet) {
   return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
 }
+function cleanTwitterText(text, tweet) {
+  if (text === null)
+    return null;
+  const entities = object(tweet.entities) ? tweet.entities : null;
+  const media = [
+    ...entities && Array.isArray(entities.media) ? entities.media : [],
+    ...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
+  ];
+  const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
+  while (true) {
+    const trailing = text.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
+    if (!trailing || !urls.has(trailing[1]))
+      break;
+    text = text.slice(0, trailing.index).trimEnd();
+  }
+  return text.replace(/&(?:gt|lt);/g, (entity) => entity === "&gt;" ? ">" : "<");
+}
 function needsFullText(tweet) {
   if (noteText(tweet) || string(tweet.full_text)) {
     return false;
@@ -2470,7 +2586,7 @@ function twitterComment(reply, id) {
     url: `https://x.com/i/status/${id}`,
     items,
     metadata: {
-      text: string(reply.text) ?? undefined,
+      text: cleanTwitterText(string(reply.text), reply) ?? undefined,
       author: {
         handle: string(author2.screen_name) ?? undefined,
         name: string(author2.name) ?? undefined,

package/dist/internal.d.ts

@@ -97,6 +97,8 @@ export type TiktokExtra = {
 };
 /** X (Twitter)-specific metadata. */
 export type TwitterExtra = {
+    /** Parent status id when this post is a reply, even if the parent metadata is unavailable. */
+    replyToId?: string;
     /** BCP 47 language tag detected for the tweet. */
     lang?: string;
     /** The post quoted by this post, when X includes it in the resolved payload. */

package/dist/postfetch.d.ts

@@ -3,7 +3,7 @@ import { type Platform, type PostfetchResult } from "./internal";
 export type PostfetchOptions = {
     /** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
     fetch?: typeof fetch;
-    /** Maximum replies to fetch for X. Other platforms ignore this option. Failures return an empty array. */
+    /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
     comments?: number;
     /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
     preferredWidth?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@postfetch/core",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
   "license": "MIT",
   "type": "module",

package/src/internal.ts

@@ -101,6 +101,8 @@ export type TiktokExtra = {
 
 /** X (Twitter)-specific metadata. */
 export type TwitterExtra = {
+  /** Parent status id when this post is a reply, even if the parent metadata is unavailable. */
+  replyToId?: string;
   /** BCP 47 language tag detected for the tweet. */
   lang?: string;
   /** The post quoted by this post, when X includes it in the resolved payload. */

package/src/postfetch.ts

@@ -13,7 +13,7 @@ import { resolveYoutube } from "./youtube";
 export type PostfetchOptions = {
   /** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
   fetch?: typeof fetch;
-  /** Maximum replies to fetch for X. Other platforms ignore this option. Failures return an empty array. */
+  /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
   comments?: number;
   /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
   preferredWidth?: number;

package/src/reddit.ts

@@ -8,6 +8,7 @@ import {
   object,
   string,
   type PostMetadata,
+  type PostComment,
   type RedditExtra,
   type ResolveContext,
   type Json,
@@ -31,7 +32,7 @@ export async function resolveReddit(input: ResolveContext): Promise<PostfetchRes
   // already thrown if the post itself is missing, so empty items here mean a
   // self/text post, not a failure.
   const items = await mediaItems(input, post, id);
-  return { archiveFilename: filename(`reddit_${id}.zip`), comments: [], id, items, metadata: redditMetadata(post), platform: "reddit" };
+  return { archiveFilename: filename(`reddit_${id}.zip`), comments: await redditComments(input, token, id), id, items, metadata: redditMetadata(post), platform: "reddit" };
 }
 
 export function redditMetadata(post: Json): PostMetadata & { extra?: RedditExtra } {
@@ -316,3 +317,91 @@ function extensionOf(mime: string): string {
   }
   return "jpg";
 }
+
+async function redditComments(input: ResolveContext, token: string, id: string): Promise<PostComment[]> {
+  const limit = input.comments;
+  if (!limit || !Number.isSafeInteger(limit) || limit < 0) return [];
+  try {
+    const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
+    const response = await input.net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=${Math.min(limit, 100)}&depth=1&sort=top`, { headers }, 1);
+    if (!response.ok) throw new Error("Reddit comments failed");
+    const payload: unknown = await response.json();
+    const listing = Array.isArray(payload) ? payload[1] : null;
+    if (!object(listing) || !object(listing.data) || !Array.isArray(listing.data.children)) throw new Error("Invalid Reddit comments");
+    const comments: PostComment[] = [];
+    const seen = new Set<string>();
+    const queued = new Set<string>();
+    const pending: string[] = [];
+    const collect = (things: unknown[]) => {
+      for (const thing of things) {
+        if (!object(thing) || !object(thing.data)) continue;
+        const data = thing.data;
+        if (data.parent_id !== `t3_${id}`) continue;
+        if (thing.kind === "more" && Array.isArray(data.children)) {
+          for (const child of data.children) {
+            if (typeof child !== "string" || !/^[a-z0-9]+$/i.test(child) || queued.has(child)) continue;
+            queued.add(child);
+            pending.push(child);
+          }
+        } else if (thing.kind === "t1" && comments.length < limit) {
+          const commentId = string(data.id);
+          if (!commentId || seen.has(commentId)) continue;
+          seen.add(commentId);
+          const body = string(data.body);
+          if (!body || body === "[deleted]" || body === "[removed]") continue;
+          comments.push(redditComment(data, commentId, id, body));
+        }
+      }
+    };
+    collect(listing.data.children);
+    while (comments.length < limit && pending.length > 0) {
+      const children = pending.splice(0, Math.min(100, limit - comments.length));
+      const url = new URL("https://oauth.reddit.com/api/morechildren");
+      url.search = new URLSearchParams({ api_type: "json", raw_json: "1", link_id: `t3_${id}`, children: children.join(","), sort: "top", depth: "1", limit_children: "true" }).toString();
+      const more = await input.net(url.href, { headers }, 1);
+      if (!more.ok) throw new Error("Reddit more comments failed");
+      const result: unknown = await more.json();
+      const json = object(result) && object(result.json) ? result.json : null;
+      const data = json && object(json.data) ? json.data : null;
+      if (!data || !Array.isArray(data.things) || (Array.isArray(json?.errors) && json.errors.length > 0)) throw new Error("Invalid additional Reddit comments");
+      collect(data.things);
+    }
+    return comments;
+  } catch {
+    return [];
+  }
+}
+
+function redditComment(data: Json, id: string, postId: string, body: string): PostComment {
+  const metadata = object(data.media_metadata) ? data.media_metadata : {};
+  // Comment images/GIFs use the same rendition metadata as Reddit galleries.
+  // Keep their inline order; do not treat arbitrary links as attachments.
+  const mediaIds = Object.keys(metadata).filter((key) => object(metadata[key])).sort((left, right) => {
+    const position = (key: string) => {
+      const meta = metadata[key];
+      const best = object(meta) && object(meta.s) ? meta.s : {};
+      const tokens = [`![img](${key})`, `![gif](${key})`, string(best.u), string(best.gif), string(best.mp4)];
+      const positions = tokens.flatMap((token) => token && body.includes(token) ? [body.indexOf(token)] : []);
+      return Math.min(...positions);
+    };
+    return position(left) - position(right);
+  });
+  const items = galleryItems({ media_metadata: metadata, gallery_data: { items: mediaIds.map((media_id) => ({ media_id })) } }, id);
+  const text = body.replace(/!\[(?:img|gif)\]\(([^)]+)\)/g, (match, key: string) => {
+    const meta = metadata[key];
+    return object(meta) && object(meta.s) && (string(meta.s.u) || string(meta.s.gif) || string(meta.s.mp4)) ? "" : match;
+  }).split("\n").filter((line) => !items.some((item) => line.trim() === item.url)).join("\n").trim();
+  const author = string(data.author);
+  return {
+    id,
+    url: `https://www.reddit.com/comments/${postId}/_/${id}/`,
+    items,
+    metadata: {
+      text,
+      author: author && author !== "[deleted]" ? { handle: author } : undefined,
+      createdAt: isoFromEpochSeconds(data.created_utc),
+      likeCount: count(data.ups),
+      nsfw: bool(data.over_18),
+    },
+  };
+}

package/src/twitter.ts

@@ -61,11 +61,13 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
   }
   const parent = object(tweet.parent) ? tweet.parent : null;
   const parentId = parent ? string(parent.id_str) : null;
+  const replyToId = string(tweet.in_reply_to_status_id_str) ?? parentId;
+  if (replyToId) extra.replyToId = replyToId;
   if (parent && parentId) {
     extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
   }
   return {
-    text: twitterText(tweet) ?? undefined,
+    text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
     author: user
       ? {
           handle: string(user.screen_name) ?? undefined,
@@ -94,6 +96,24 @@ function twitterText(tweet: Json): string | null {
   return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
 }
 
+// X serializes attachments as URLs in text. Only remove a trailing URL when
+// the same post explicitly identifies it as media; ordinary links stay intact.
+function cleanTwitterText(text: string | null, tweet: Json): string | null {
+  if (text === null) return null;
+  const entities = object(tweet.entities) ? tweet.entities : null;
+  const media = [
+    ...(entities && Array.isArray(entities.media) ? entities.media : []),
+    ...(Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []),
+  ];
+  const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
+  while (true) {
+    const trailing: RegExpMatchArray | null = text.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
+    if (!trailing || !urls.has(trailing[1])) break;
+    text = text.slice(0, trailing.index).trimEnd();
+  }
+  return text.replace(/&(?:gt|lt);/g, (entity) => entity === "&gt;" ? ">" : "<");
+}
+
 function needsFullText(tweet: Json): boolean {
   if (noteText(tweet) || string(tweet.full_text)) {
     return false;
@@ -348,7 +368,7 @@ function twitterComment(reply: Json, id: string): PostComment {
     url: `https://x.com/i/status/${id}`,
     items,
     metadata: {
-      text: string(reply.text) ?? undefined,
+      text: cleanTwitterText(string(reply.text), reply) ?? undefined,
       author: {
         handle: string(author.screen_name) ?? undefined,
         name: string(author.name) ?? undefined,