@postfetch/core 0.5.1 → 0.6.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@postfetch/core",
-  "version": "0.5.1",
-  "description": "Zero-dependency typed core that turns Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
+  "version": "0.6.0",
+  "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
   "license": "MIT",
   "type": "module",
   "exports": {
@@ -30,6 +30,7 @@
     "downloader",
     "facebook",
     "instagram",
+    "linkedin",
     "media",
     "pinterest",
     "reddit",

package/src/facebook.ts

@@ -1,10 +1,14 @@
-import { asUrl, filename, type ResolveContext, type Json, type Net, type PostfetchResult, type MediaItem } from "./internal";
+import { asUrl, count, filename, isoFromEpochSeconds, type PostMetadata, type ResolveContext, type Json, type Net, type PostfetchResult, type MediaItem } from "./internal";
 import { browserUserAgent } from "./fingerprint";
 
 export async function resolveFacebook(input: ResolveContext): Promise<PostfetchResult> {
   const canonical = await canonicalUrl(input.net, input.url);
   const id = facebookId(canonical) ?? facebookId(input.url) ?? "video";
-  const url = await embedVideo(input.net, canonical);
+  // The embed player only exposes /share/v/ videos; reels (and anything it
+  // misses) come from the watch page, which the &_rdr flag serves logged-out.
+  const embed = await embedVideo(input.net, canonical);
+  const watch = !embed && /^\d+$/.test(id) ? await watchVideo(input.net, id) : null;
+  const url = embed ?? watch?.url ?? null;
   if (!url) {
     throw new Error("Facebook video not found");
   }
@@ -17,7 +21,7 @@ export async function resolveFacebook(input: ResolveContext): Promise<PostfetchR
     platform: "facebook",
     url,
   };
-  return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [item], platform: "facebook" };
+  return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [item], metadata: watch?.metadata, platform: "facebook" };
 }
 
 function navigationHeaders(): Record<string, string> {
@@ -49,6 +53,39 @@ async function embedVideo(net: Net, canonical: string): Promise<string | null> {
   return source(html, "hd_src") ?? source(html, "sd_src");
 }
 
+// The logged-out watch page (with &_rdr) embeds the progressive video URLs in
+// its ScheduledServerJS blocks; these are the same fields yt-dlp reads.
+async function watchVideo(net: Net, id: string): Promise<{ url: string; metadata: PostMetadata } | null> {
+  // Without the document Accept header Facebook serves a JS shell without the
+  // media JSON, so ask for HTML explicitly here.
+  const headers = { ...navigationHeaders(), accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" };
+  const response = await net(`https://www.facebook.com/watch/?v=${id}&_rdr`, { headers });
+  if (!response.ok) {
+    return null;
+  }
+  const html = await response.text();
+  const url =
+    source(html, "playable_url_quality_hd") ??
+    source(html, "browser_native_hd_url") ??
+    source(html, "playable_url") ??
+    source(html, "browser_native_sd_url");
+  return url ? { url, metadata: watchMetadata(html) } : null;
+}
+
+// Only the fields that can be read unambiguously from the flat page: the
+// caption (creation_story) and the reaction/comment/share counts appear in
+// shapes shared with related videos and comment threads, so they are skipped
+// rather than risk attributing the wrong value.
+function watchMetadata(html: string): PostMetadata {
+  const owner = html.match(/"name":("(?:\\.|[^"\\])*"),"__isVideoOwner"/);
+  const name = owner ? (JSON.parse(owner[1]) as string) : undefined;
+  return {
+    author: name ? { name } : undefined,
+    createdAt: isoFromEpochSeconds(html.match(/"publish_time":(\d+)/)?.[1]),
+    viewCount: count(html.match(/"video_view_count":(\d+)/)?.[1]),
+  };
+}
+
 function source(html: string, key: string): string | null {
   const match = html.match(new RegExp(`"${key}":("(?:\\\\.|[^"\\\\])*")`));
   if (!match) {

package/src/fingerprint.ts

@@ -32,6 +32,10 @@ export type BrowserFingerprint = {
   userAgent: string;
 };
 
+export type NavigationHeaders = Record<string, string> & {
+  "user-agent": string;
+};
+
 export function browserFingerprint(): BrowserFingerprint {
   const version = pick(chromeVersions);
   const platform = pick(desktopPlatforms);
@@ -43,20 +47,26 @@ export function browserFingerprint(): BrowserFingerprint {
   };
 }
 
-export function navigationHeaders(): Record<string, string> {
-  const fingerprint = browserFingerprint();
+function baseNavigationHeaders(userAgent: string, acceptLanguage: string): NavigationHeaders {
   return {
     accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-    "accept-language": fingerprint.acceptLanguage,
-    "sec-ch-ua": fingerprint.secChUa,
-    "sec-ch-ua-mobile": "?0",
-    "sec-ch-ua-platform": fingerprint.secChUaPlatform,
+    "accept-language": acceptLanguage,
     "sec-fetch-dest": "document",
     "sec-fetch-mode": "navigate",
     "sec-fetch-site": "none",
     "sec-fetch-user": "?1",
     "upgrade-insecure-requests": "1",
-    "user-agent": fingerprint.userAgent,
+    "user-agent": userAgent,
+  };
+}
+
+export function navigationHeaders(): NavigationHeaders {
+  const fingerprint = browserFingerprint();
+  return {
+    ...baseNavigationHeaders(fingerprint.userAgent, fingerprint.acceptLanguage),
+    "sec-ch-ua": fingerprint.secChUa,
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": fingerprint.secChUaPlatform,
   };
 }
 
@@ -70,6 +80,10 @@ export function firefoxUserAgent(): string {
   return `Mozilla/5.0 (${platform.uaToken}; rv:${version}.0) Gecko/20100101 Firefox/${version}.0`;
 }
 
+export function firefoxNavigationHeaders(userAgent = firefoxUserAgent()): NavigationHeaders {
+  return baseNavigationHeaders(userAgent, pick(acceptLanguages));
+}
+
 export function instagramAppUserAgent(): string {
   return pick(instagramAppUserAgents);
 }

package/src/index.ts

@@ -1,7 +1,7 @@
 /**
  * Turn a public social post URL into its media.
  *
- * {@link postfetch} resolves Instagram, TikTok, YouTube, Facebook and X (Twitter)
+ * {@link postfetch} resolves Instagram, LinkedIn, TikTok, YouTube, Facebook and X (Twitter)
  * URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
  * (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
  * Cloudflare Workers and browsers alike. It only resolves the media; the

package/src/instagram.ts

@@ -58,11 +58,21 @@ function embedHeaders(): Record<string, string> {
 
 export async function resolveInstagram(input: ResolveContext): Promise<PostfetchResult> {
   const code = shortcode(input.url);
-  const media =
-    (await pageMedia(input.net, code, input.preferredWidth)) ??
-    (await mobileMedia(input.net, code, input.preferredWidth)) ??
-    (await embedMedia(input.net, code)) ??
-    (await graphqlMedia(input.net, code));
+  const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
+  const resolvers: Array<() => Promise<Json | null>> = [
+    () => pageMedia(input.net, code, input.preferredWidth),
+    () => mobileMedia(input.net, code, input.preferredWidth),
+    () => embedMedia(input.net, code),
+    () => graphqlMedia(input.net, code),
+  ];
+  let media: Json | null = null;
+  for (const resolve of resolvers) {
+    const candidate = await resolve();
+    if (candidate && (!videoRequired || mediaItems(candidate, code, input.preferredWidth).some((item) => item.kind === "video"))) {
+      media = candidate;
+      break;
+    }
+  }
   if (!media) {
     throw await instagramUnavailable(input.net, code);
   }
@@ -301,23 +311,20 @@ async function graphqlMedia(net: Net, code: string): Promise<Json | null> {
   }
   const body = new URLSearchParams({
     ...params.body,
-    doc_id: "8845758582119845",
+    av: "0",
+    doc_id: "26130443479876713",
     fb_api_caller_class: "RelayModern",
-    fb_api_req_friendly_name: "PolarisPostActionLoadPostQueryQuery",
+    fb_api_req_friendly_name: "PolarisPostRootQuery",
     server_timestamps: "true",
-    variables: JSON.stringify({
-      fetch_tagged_user_count: null,
-      hoisted_comment_id: null,
-      hoisted_reply_id: null,
-      shortcode: code,
-    }),
+    variables: JSON.stringify({ shortcode: code }),
   });
   const response = await net("https://www.instagram.com/graphql/query", {
     body,
     headers: {
       ...embedHeaders(),
       ...params.headers,
-      "X-FB-Friendly-Name": "PolarisPostActionLoadPostQueryQuery",
+      "X-FB-Friendly-Name": "PolarisPostRootQuery",
+      "X-Requested-With": "XMLHttpRequest",
       "content-type": "application/x-www-form-urlencoded",
     },
     method: "POST",
@@ -344,14 +351,14 @@ async function graphqlParams(net: Net, code: string): Promise<{ body: Record<str
   const push = entryObject("InstagramWebPushInfo", html);
   const lsd = entryObject("LSD", html)?.token ?? randomToken();
   const csrf = entryObject("InstagramSecurityConfig", html)?.csrf_token;
-  const cookie = [
-    csrf && `csrftoken=${csrf}`,
-    polaris?.device_id && `ig_did=${polaris.device_id}`,
-    polaris?.machine_id && `mid=${polaris.machine_id}`,
-    "wd=1280x720",
-    "dpr=2",
-    "ig_nrcb=1",
-  ].filter((value): value is string => typeof value === "string" && value.length > 0).join("; ");
+  const cookie = instagramCookies(response.headers, {
+    csrftoken: string(csrf),
+    dpr: "2",
+    ig_did: string(polaris?.device_id),
+    ig_nrcb: "1",
+    mid: string(polaris?.machine_id),
+    wd: "1280x720",
+  });
   return {
     headers: {
       "X-CSRFToken": string(csrf) ?? "",
@@ -386,6 +393,12 @@ async function graphqlParams(net: Net, code: string): Promise<{ body: Record<str
 }
 
 function gqlShortcodeMedia(data: Json): Json | null {
+  const webInfo = object(data.xdt_api__v1__media__shortcode__web_info) ? data.xdt_api__v1__media__shortcode__web_info : null;
+  const webItems = webInfo && Array.isArray(webInfo.items) ? webInfo.items : [];
+  const webMedia = webItems[0];
+  if (object(webMedia)) {
+    return webMedia;
+  }
   const media = data.gql_data && object(data.gql_data)
     ? data.gql_data.shortcode_media ?? data.gql_data.xdt_shortcode_media
     : data.shortcode_media ?? data.xdt_shortcode_media;
@@ -411,6 +424,25 @@ function mediaItems(media: Json, code: string, preferredWidth: number): MediaIte
   return instagramItem(media, code, null, preferredWidth);
 }
 
+function instagramCookies(headers: Headers, fallback: Record<string, string | null>): string {
+  const getter = (headers as Headers & { getSetCookie?: () => string[] }).getSetCookie;
+  const raw = typeof getter === "function" ? getter.call(headers) : headers.get("set-cookie")?.split(/,(?=[^;]+?=)/) ?? [];
+  const cookies = new Map<string, string>();
+  for (const value of raw) {
+    const pair = value.split(";", 1)[0]?.trim();
+    const separator = pair?.indexOf("=") ?? -1;
+    if (pair && separator > 0) {
+      cookies.set(pair.slice(0, separator), pair.slice(separator + 1));
+    }
+  }
+  for (const [name, value] of Object.entries(fallback)) {
+    if (value && !cookies.has(name)) {
+      cookies.set(name, value);
+    }
+  }
+  return [...cookies].map(([name, value]) => `${name}=${value}`).join("; ");
+}
+
 function instagramItem(media: Json, code: string, index: number | null, preferredWidth: number): MediaItem[] {
   const video = selectVersion(media.video_versions, preferredWidth) ?? string(media.video_url);
   const suffix = index === null ? "" : `_${index}`;

package/src/internal.ts

@@ -1,5 +1,5 @@
 /** A platform that {@link postfetch} can resolve. */
-export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
+export type Platform = "facebook" | "instagram" | "linkedin" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
 
 /** The kind of a {@link MediaItem}. */
 export type MediaKind = "audio" | "image" | "video";
@@ -149,6 +149,7 @@ export type PinterestExtra = {
 export type PostfetchResult =
   | PlatformResult<"facebook", never>
   | PlatformResult<"instagram", InstagramExtra>
+  | PlatformResult<"linkedin", never>
   | PlatformResult<"pinterest", PinterestExtra>
   | PlatformResult<"reddit", RedditExtra>
   | PlatformResult<"soundcloud", SoundcloudExtra>

package/src/linkedin.ts

@@ -0,0 +1,243 @@
+import {
+  count,
+  filename,
+  isoFromDateString,
+  object,
+  PostfetchError,
+  string,
+  type Json,
+  type MediaItem,
+  type PostMetadata,
+  type PostfetchResult,
+  type ResolveContext,
+} from "./internal";
+import { browserUserAgent, navigationHeaders } from "./fingerprint";
+
+type LinkedinPostKind = "activity" | "share" | "ugcPost";
+
+type LinkedinPostReference = {
+  id: string;
+  kind: LinkedinPostKind;
+};
+
+export async function resolveLinkedin(input: ResolveContext): Promise<PostfetchResult> {
+  const reference = postReference(input.url);
+  const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
+  const response = await input.net(pageUrl, { headers: navigationHeaders() });
+  if (response.status === 404) {
+    throw new PostfetchError(404, "LinkedIn post not found", "notFound");
+  }
+  if (response.status === 429 || response.status === 999) {
+    throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
+  }
+  if (!response.ok) {
+    throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
+  }
+  if (response.url.includes("/signup/") || response.url.includes("/login")) {
+    throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
+  }
+
+  const html = await response.text();
+  const post = linkedinPost(html);
+  if (!post) {
+    throw new PostfetchError(404, "LinkedIn post unavailable", "unavailable");
+  }
+  const canonical = string(post["@id"]);
+  const canonicalReference = canonical ? postReferenceOrNull(canonical) : null;
+  const id = canonicalReference?.id ?? reference.id;
+  const items = linkedinItems(post, html, id, pageUrl);
+  if (items.length === 0) {
+    throw new Error("LinkedIn media url not found");
+  }
+  return {
+    archiveFilename: filename(`linkedin_${id}.zip`),
+    id,
+    items,
+    metadata: linkedinMetadata(post),
+    platform: "linkedin",
+  };
+}
+
+export function linkedinMetadata(post: Json): PostMetadata {
+  const author = linkedinAuthor(post);
+  return {
+    title: string(post.text) ?? string(post.name) ?? string(post.headline) ?? undefined,
+    text: string(post.description) ?? string(post.articleBody) ?? string(post.text) ?? undefined,
+    author: author
+      ? {
+          handle: linkedinHandle(string(author.url)) ?? undefined,
+          name: string(author.name) ?? undefined,
+        }
+      : undefined,
+    createdAt: isoFromDateString(post.datePublished ?? post.uploadDate),
+    likeCount: interactionCount(post.interactionStatistic, "LikeAction"),
+    commentCount: count(post.commentCount) ?? interactionCount(post.interactionStatistic, "CommentAction"),
+  };
+}
+
+function postReference(input: string): LinkedinPostReference {
+  const reference = postReferenceOrNull(input);
+  if (!reference) {
+    throw new PostfetchError(400, "LinkedIn post id not found");
+  }
+  return reference;
+}
+
+function postReferenceOrNull(input: string): LinkedinPostReference | null {
+  let pathname: string;
+  try {
+    pathname = new URL(input).pathname.replace(/%3A/gi, ":");
+  } catch {
+    return null;
+  }
+  const urn = pathname.match(/\/(?:embed\/)?feed\/update\/urn:li:(activity|share|ugcpost):(\d+)\/?$/i);
+  const post = pathname.match(/\/posts\/[^/]*-(activity|share|ugcpost)-(\d+)(?:-[^/]*)?\/?$/i);
+  const match = urn ?? post;
+  if (!match) {
+    return null;
+  }
+  switch (match[1].toLowerCase()) {
+    case "activity":
+      return { id: match[2], kind: "activity" };
+    case "share":
+      return { id: match[2], kind: "share" };
+    case "ugcpost":
+      return { id: match[2], kind: "ugcPost" };
+    default:
+      return null;
+  }
+}
+
+function linkedinAuthor(post: Json): Json | null {
+  if (object(post.creator)) {
+    return post.creator;
+  }
+  if (object(post.author)) {
+    return post.author;
+  }
+  return null;
+}
+
+function linkedinPost(html: string): Json | null {
+  for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
+    if (!/\btype=["']application\/ld\+json["']/i.test(match[1])) {
+      continue;
+    }
+    const parsed = parseJson(match[2]);
+    for (const entry of jsonLdEntries(parsed)) {
+      if (hasSchemaType(entry, "VideoObject") || hasSchemaType(entry, "SocialMediaPosting")) {
+        return entry;
+      }
+    }
+  }
+  return null;
+}
+
+function parseJson(value: string): unknown {
+  try {
+    return JSON.parse(value);
+  } catch {
+    try {
+      return JSON.parse(decodeHtml(value));
+    } catch {
+      return null;
+    }
+  }
+}
+
+function jsonLdEntries(value: unknown): Json[] {
+  if (Array.isArray(value)) {
+    return value.filter(object);
+  }
+  if (!object(value)) {
+    return [];
+  }
+  return Array.isArray(value["@graph"]) ? value["@graph"].filter(object) : [value];
+}
+
+function hasSchemaType(value: Json, expected: string): boolean {
+  const type = value["@type"];
+  return type === expected || (Array.isArray(type) && type.includes(expected));
+}
+
+function linkedinItems(post: Json, html: string, id: string, referer: string): MediaItem[] {
+  const headers = { referer, "user-agent": browserUserAgent() };
+  if (hasSchemaType(post, "VideoObject")) {
+    const url = bestVideoUrl(html) ?? string(post.contentUrl);
+    return url
+      ? [{ filename: filename(`linkedin_${id}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "linkedin", url }]
+      : [];
+  }
+  const url = imageUrl(post.image);
+  return url
+    ? [{ filename: filename(`linkedin_${id}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "linkedin", url }]
+    : [];
+}
+
+function bestVideoUrl(html: string): string | null {
+  const encoded = html.match(/<video\b[^>]*\bdata-sources=(["'])(.*?)\1[^>]*>/is)?.[2];
+  if (!encoded) {
+    return null;
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(decodeHtml(encoded));
+  } catch {
+    return null;
+  }
+  const sources = Array.isArray(parsed) ? parsed.filter(object) : [];
+  const videos = sources.filter((source) => string(source.type) === "video/mp4" && string(source.src));
+  const selected = videos.reduce<Json | null>((best, candidate) => {
+    const bitrate = count(candidate["data-bitrate"]) ?? 0;
+    const bestBitrate = best ? count(best["data-bitrate"]) ?? 0 : -1;
+    return bitrate > bestBitrate ? candidate : best;
+  }, null);
+  return selected ? string(selected.src) : null;
+}
+
+function imageUrl(value: unknown): string | null {
+  if (typeof value === "string") {
+    return string(value);
+  }
+  if (Array.isArray(value)) {
+    return value.map(imageUrl).find((url): url is string => Boolean(url)) ?? null;
+  }
+  return object(value) ? string(value.url) : null;
+}
+
+function interactionCount(value: unknown, action: string): number | undefined {
+  const statistics = Array.isArray(value) ? value.filter(object) : object(value) ? [value] : [];
+  const statistic = statistics.find((entry) => string(entry.interactionType)?.endsWith(`/${action}`));
+  return statistic ? count(statistic.userInteractionCount) : undefined;
+}
+
+function linkedinHandle(value: string | null): string | null {
+  if (!value) {
+    return null;
+  }
+  try {
+    return new URL(value).pathname.match(/^\/(?:company|in|showcase)\/([^/]+)/)?.[1] ?? null;
+  } catch {
+    return null;
+  }
+}
+
+function decodeHtml(value: string): string {
+  return value.replace(/&(quot|amp|lt|gt|#39|#x27|#(\d+)|#x([0-9a-f]+));/gi, (_entity, name: string, decimal: string, hex: string) => {
+    switch (name.toLowerCase()) {
+      case "quot":
+        return '"';
+      case "amp":
+        return "&";
+      case "lt":
+        return "<";
+      case "gt":
+        return ">";
+      case "#39":
+      case "#x27":
+        return "'";
+      default:
+        return String.fromCodePoint(decimal ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
+    }
+  });
+}

package/src/postfetch.ts

@@ -1,6 +1,7 @@
 import { asUrl, createNet, PostfetchError, type Platform, type PostfetchResult } from "./internal";
 import { resolveFacebook } from "./facebook";
 import { resolveInstagram } from "./instagram";
+import { resolveLinkedin } from "./linkedin";
 import { resolvePinterest } from "./pinterest";
 import { resolveReddit } from "./reddit";
 import { resolveSoundcloud } from "./soundcloud";
@@ -24,7 +25,7 @@ export type PostfetchOptions = {
  * performs no side effects beyond the lookup; use {@link download}, {@link archive}
  * or {@link toResponse} to turn the result into bytes or a `Response`.
  *
- * @param url A post URL from Instagram, TikTok, YouTube, Facebook or X.
+ * @param url A post URL from a supported platform.
  * @param options See {@link PostfetchOptions}.
  * @returns The resolved post and its media items.
  * @throws {PostfetchError} If the URL is empty or its host is unsupported.
@@ -52,6 +53,8 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
       return resolveFacebook(context);
     case "instagram":
       return resolveInstagram(context);
+    case "linkedin":
+      return resolveLinkedin(context);
     case "pinterest":
       return resolvePinterest(context);
     case "reddit":
@@ -82,6 +85,9 @@ export function detect(url: string): Platform {
   if (host.includes("instagram.com")) {
     return "instagram";
   }
+  if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
+    return "linkedin";
+  }
   if (host.includes("pinterest.") || host === "pin.it") {
     return "pinterest";
   }
@@ -100,5 +106,5 @@ export function detect(url: string): Platform {
   if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
     return "twitter";
   }
-  throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
+  throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
 }