@postfetch/core 0.12.0 → 0.12.2

package/dist/index.js

@@ -365,8 +365,9 @@ function embedHeaders() {
 async function resolveInstagram(input) {
   const code = shortcode(input.url);
   const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
+  const page = await pageMedia(input.net, code, input.preferredWidth);
   const resolvers = [
-    () => pageMedia(input.net, code, input.preferredWidth),
+    async () => page.media,
     () => mobileMedia(input.net, code, input.preferredWidth),
     () => embedMedia(input.net, code),
     () => graphqlMedia(input.net, code)
@@ -380,7 +381,7 @@ async function resolveInstagram(input) {
     }
   }
   if (!media) {
-    throw await instagramUnavailable(input.net, code);
+    throw await instagramUnavailable(input.net, code, page.reason);
   }
   const items = mediaItems(media, code, input.preferredWidth);
   if (items.length === 0) {
@@ -388,12 +389,12 @@ async function resolveInstagram(input) {
   }
   return { archiveFilename: filename(`instagram_${code}.zip`), comments: [], id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
 }
-async function instagramUnavailable(net, code) {
+async function instagramUnavailable(net, code, pageReason) {
   const url = new URL("https://i.instagram.com/api/v1/oembed/");
   url.searchParams.set("url", `https://www.instagram.com/p/${code}/`);
   const response = await net(url.href, { headers: mobileHeaders() }, 1).catch(() => null);
   const body = response ? await response.text().catch(() => "") : "";
-  const reason = instagramUnavailableReason(response?.status ?? 0, body);
+  const reason = pageReason ?? instagramUnavailableReason(response?.status ?? 0, body);
   const probe = response ? `oembed ${response.status}` : "oembed unreachable";
   return new PostfetchError(reasonStatus(reason), `Instagram post unavailable: ${reason} (${probe})`, reason);
 }
@@ -407,7 +408,7 @@ function instagramUnavailableReason(status, body) {
   if (status !== 0 && status < 400) {
     return "unavailable";
   }
-  if (/MIN_AGE_ACCOUNT|under 18/i.test(body)) {
+  if (/MIN_AGE_ACCOUNT|under 18|age[- ]restricted/i.test(body)) {
     return "ageRestricted";
   }
   if (/private account|"is_private":\s*true|account is private/i.test(body)) {
@@ -480,11 +481,15 @@ function shortcode(input) {
 async function pageMedia(net, code, preferredWidth) {
   const response = await net(`https://www.instagram.com/p/${code}/`, { headers: navigationHeaders() });
   if (!response.ok) {
-    return null;
+    return { media: null };
   }
   const html = await response.text();
   const media = inlineMedia(html, code);
-  return media && mediaItems(media, code, preferredWidth).length > 0 ? media : null;
+  const reason = media && object(media.gating_ruling) ? instagramUnavailableReason(400, JSON.stringify(media.gating_ruling)) : "unavailable";
+  return {
+    media: media && mediaItems(media, code, preferredWidth).length > 0 ? media : null,
+    reason: reason === "unavailable" ? undefined : reason
+  };
 }
 function inlineMedia(html, code) {
   for (const match of html.matchAll(/<script type="application\/json"[^>]*>(.*?)<\/script>/gs)) {
@@ -515,7 +520,7 @@ function searchMedia(node, code) {
     return null;
   }
   const hasMedia = Array.isArray(node.video_versions) || Array.isArray(node.carousel_media) || object(node.image_versions2) && Array.isArray(node.image_versions2.candidates);
-  if (hasMedia && node.code === code) {
+  if ((hasMedia || object(node.gating_ruling)) && node.code === code) {
     return node;
   }
   for (const key in node) {
@@ -2315,7 +2320,7 @@ function twitterMetadata(tweet) {
     extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
   }
   return {
-    text: twitterText(tweet) ?? undefined,
+    text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
     author: user ? {
       handle: string(user.screen_name) ?? undefined,
       name: string(user.name) ?? undefined,
@@ -2339,6 +2344,23 @@ function noteText(tweet) {
 function twitterText(tweet) {
   return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
 }
+function cleanTwitterText(text, tweet) {
+  if (text === null)
+    return null;
+  const entities = object(tweet.entities) ? tweet.entities : null;
+  const media = [
+    ...entities && Array.isArray(entities.media) ? entities.media : [],
+    ...Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []
+  ];
+  const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
+  while (true) {
+    const trailing = text.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
+    if (!trailing || !urls.has(trailing[1]))
+      break;
+    text = text.slice(0, trailing.index).trimEnd();
+  }
+  return text.replace(/&(?:gt|lt);/g, (entity) => entity === "&gt;" ? ">" : "<");
+}
 function needsFullText(tweet) {
   if (noteText(tweet) || string(tweet.full_text)) {
     return false;
@@ -2569,7 +2591,7 @@ function twitterComment(reply, id) {
     url: `https://x.com/i/status/${id}`,
     items,
     metadata: {
-      text: string(reply.text) ?? undefined,
+      text: cleanTwitterText(string(reply.text), reply) ?? undefined,
       author: {
         handle: string(author2.screen_name) ?? undefined,
         name: string(author2.name) ?? undefined,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@postfetch/core",
-  "version": "0.12.0",
+  "version": "0.12.2",
   "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
   "license": "MIT",
   "type": "module",

package/src/instagram.ts

@@ -59,8 +59,9 @@ function embedHeaders(): Record<string, string> {
 export async function resolveInstagram(input: ResolveContext): Promise<PostfetchResult> {
   const code = shortcode(input.url);
   const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
+  const page = await pageMedia(input.net, code, input.preferredWidth);
   const resolvers: Array<() => Promise<Json | null>> = [
-    () => pageMedia(input.net, code, input.preferredWidth),
+    async () => page.media,
     () => mobileMedia(input.net, code, input.preferredWidth),
     () => embedMedia(input.net, code),
     () => graphqlMedia(input.net, code),
@@ -74,7 +75,7 @@ export async function resolveInstagram(input: ResolveContext): Promise<Postfetch
     }
   }
   if (!media) {
-    throw await instagramUnavailable(input.net, code);
+    throw await instagramUnavailable(input.net, code, page.reason);
   }
   const items = mediaItems(media, code, input.preferredWidth);
   if (items.length === 0) {
@@ -83,12 +84,14 @@ export async function resolveInstagram(input: ResolveContext): Promise<Postfetch
   return { archiveFilename: filename(`instagram_${code}.zip`), comments: [], id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
 }
 
-async function instagramUnavailable(net: Net, code: string): Promise<PostfetchError> {
+async function instagramUnavailable(net: Net, code: string, pageReason?: PostfetchReason): Promise<PostfetchError> {
   const url = new URL("https://i.instagram.com/api/v1/oembed/");
   url.searchParams.set("url", `https://www.instagram.com/p/${code}/`);
   const response = await net(url.href, { headers: mobileHeaders() }, 1).catch(() => null);
   const body = response ? await response.text().catch(() => "") : "";
-  const reason = instagramUnavailableReason(response?.status ?? 0, body);
+  // The page can identify an age gate even when oembed only reports a generic
+  // geoblock_required error with no MIN_AGE_ACCOUNT marker.
+  const reason = pageReason ?? instagramUnavailableReason(response?.status ?? 0, body);
   // Carry the oembed status so an unclassified failure still leaves a raw clue
   // (200 = post is public but no media could be parsed; 429 = throttled; …).
   const probe = response ? `oembed ${response.status}` : "oembed unreachable";
@@ -109,7 +112,7 @@ export function instagramUnavailableReason(status: number, body: string): Postfe
   if (status !== 0 && status < 400) {
     return "unavailable";
   }
-  if (/MIN_AGE_ACCOUNT|under 18/i.test(body)) {
+  if (/MIN_AGE_ACCOUNT|under 18|age[- ]restricted/i.test(body)) {
     return "ageRestricted";
   }
   if (/private account|"is_private":\s*true|account is private/i.test(body)) {
@@ -190,14 +193,20 @@ function shortcode(input: string): string {
   return code;
 }
 
-async function pageMedia(net: Net, code: string, preferredWidth: number): Promise<Json | null> {
+async function pageMedia(net: Net, code: string, preferredWidth: number): Promise<{ media: Json | null; reason?: PostfetchReason }> {
   const response = await net(`https://www.instagram.com/p/${code}/`, { headers: navigationHeaders() });
   if (!response.ok) {
-    return null;
+    return { media: null };
   }
   const html = await response.text();
   const media = inlineMedia(html, code);
-  return media && mediaItems(media, code, preferredWidth).length > 0 ? media : null;
+  const reason = media && object(media.gating_ruling)
+    ? instagramUnavailableReason(400, JSON.stringify(media.gating_ruling))
+    : "unavailable";
+  return {
+    media: media && mediaItems(media, code, preferredWidth).length > 0 ? media : null,
+    reason: reason === "unavailable" ? undefined : reason,
+  };
 }
 
 function inlineMedia(html: string, code: string): Json | null {
@@ -233,7 +242,7 @@ function searchMedia(node: unknown, code: string): Json | null {
     Array.isArray(node.video_versions) ||
     Array.isArray(node.carousel_media) ||
     (object(node.image_versions2) && Array.isArray(node.image_versions2.candidates));
-  if (hasMedia && node.code === code) {
+  if ((hasMedia || object(node.gating_ruling)) && node.code === code) {
     return node;
   }
   for (const key in node) {

package/src/twitter.ts

@@ -67,7 +67,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
     extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
   }
   return {
-    text: twitterText(tweet) ?? undefined,
+    text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
     author: user
       ? {
           handle: string(user.screen_name) ?? undefined,
@@ -96,6 +96,24 @@ function twitterText(tweet: Json): string | null {
   return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
 }
 
+// X serializes attachments as URLs in text. Only remove a trailing URL when
+// the same post explicitly identifies it as media; ordinary links stay intact.
+function cleanTwitterText(text: string | null, tweet: Json): string | null {
+  if (text === null) return null;
+  const entities = object(tweet.entities) ? tweet.entities : null;
+  const media = [
+    ...(entities && Array.isArray(entities.media) ? entities.media : []),
+    ...(Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails : []),
+  ];
+  const urls = new Set(media.filter(object).map((entry) => string(entry.url)).filter(Boolean));
+  while (true) {
+    const trailing: RegExpMatchArray | null = text.match(/(?:^|\s)(https?:\/\/\S+)\s*$/);
+    if (!trailing || !urls.has(trailing[1])) break;
+    text = text.slice(0, trailing.index).trimEnd();
+  }
+  return text.replace(/&(?:gt|lt);/g, (entity) => entity === "&gt;" ? ">" : "<");
+}
+
 function needsFullText(tweet: Json): boolean {
   if (noteText(tweet) || string(tweet.full_text)) {
     return false;
@@ -350,7 +368,7 @@ function twitterComment(reply: Json, id: string): PostComment {
     url: `https://x.com/i/status/${id}`,
     items,
     metadata: {
-      text: string(reply.text) ?? undefined,
+      text: cleanTwitterText(string(reply.text), reply) ?? undefined,
       author: {
         handle: string(author.screen_name) ?? undefined,
         name: string(author.name) ?? undefined,