@postfetch/core 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # @postfetch/core
2
2
 
3
- Zero-dependency typed core that turns Instagram, TikTok and YouTube post URLs into media files. Part of [postfetch](https://github.com/chelokot/postfetch).
3
+ Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files. Part of [postfetch](https://github.com/chelokot/postfetch).
4
4
 
5
5
  ```ts
6
6
  import { postfetch, download, archive, toResponse, PostfetchError } from "@postfetch/core";
@@ -4,8 +4,12 @@ export type BrowserFingerprint = {
4
4
  secChUaPlatform: string;
5
5
  userAgent: string;
6
6
  };
7
+ export type NavigationHeaders = Record<string, string> & {
8
+ "user-agent": string;
9
+ };
7
10
  export declare function browserFingerprint(): BrowserFingerprint;
8
- export declare function navigationHeaders(): Record<string, string>;
11
+ export declare function navigationHeaders(): NavigationHeaders;
9
12
  export declare function browserUserAgent(): string;
10
13
  export declare function firefoxUserAgent(): string;
14
+ export declare function firefoxNavigationHeaders(userAgent?: string): NavigationHeaders;
11
15
  export declare function instagramAppUserAgent(): string;
package/dist/index.d.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Turn a public social post URL into its media.
3
3
  *
4
- * {@link postfetch} resolves Instagram, TikTok, YouTube, Facebook and X (Twitter)
4
+ * {@link postfetch} resolves Instagram, LinkedIn, TikTok, YouTube, Facebook and X (Twitter)
5
5
  * URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
6
6
  * (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
7
7
  * Cloudflare Workers and browsers alike. It only resolves the media; the
package/dist/index.js CHANGED
@@ -124,20 +124,25 @@ function browserFingerprint() {
124
124
  userAgent: `Mozilla/5.0 (${platform.uaToken}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${version}.0.0.0 Safari/537.36`
125
125
  };
126
126
  }
127
- function navigationHeaders() {
128
- const fingerprint = browserFingerprint();
127
+ function baseNavigationHeaders(userAgent, acceptLanguage) {
129
128
  return {
130
129
  accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
131
- "accept-language": fingerprint.acceptLanguage,
132
- "sec-ch-ua": fingerprint.secChUa,
133
- "sec-ch-ua-mobile": "?0",
134
- "sec-ch-ua-platform": fingerprint.secChUaPlatform,
130
+ "accept-language": acceptLanguage,
135
131
  "sec-fetch-dest": "document",
136
132
  "sec-fetch-mode": "navigate",
137
133
  "sec-fetch-site": "none",
138
134
  "sec-fetch-user": "?1",
139
135
  "upgrade-insecure-requests": "1",
140
- "user-agent": fingerprint.userAgent
136
+ "user-agent": userAgent
137
+ };
138
+ }
139
+ function navigationHeaders() {
140
+ const fingerprint = browserFingerprint();
141
+ return {
142
+ ...baseNavigationHeaders(fingerprint.userAgent, fingerprint.acceptLanguage),
143
+ "sec-ch-ua": fingerprint.secChUa,
144
+ "sec-ch-ua-mobile": "?0",
145
+ "sec-ch-ua-platform": fingerprint.secChUaPlatform
141
146
  };
142
147
  }
143
148
  function browserUserAgent() {
@@ -148,6 +153,9 @@ function firefoxUserAgent() {
148
153
  const version = pick(firefoxVersions);
149
154
  return `Mozilla/5.0 (${platform.uaToken}; rv:${version}.0) Gecko/20100101 Firefox/${version}.0`;
150
155
  }
156
+ function firefoxNavigationHeaders(userAgent = firefoxUserAgent()) {
157
+ return baseNavigationHeaders(userAgent, pick(acceptLanguages));
158
+ }
151
159
  function instagramAppUserAgent() {
152
160
  return pick(instagramAppUserAgents);
153
161
  }
@@ -271,7 +279,21 @@ function embedHeaders() {
271
279
  }
272
280
  async function resolveInstagram(input) {
273
281
  const code = shortcode(input.url);
274
- const media = await pageMedia(input.net, code, input.preferredWidth) ?? await mobileMedia(input.net, code, input.preferredWidth) ?? await embedMedia(input.net, code) ?? await graphqlMedia(input.net, code);
282
+ const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
283
+ const resolvers = [
284
+ () => pageMedia(input.net, code, input.preferredWidth),
285
+ () => mobileMedia(input.net, code, input.preferredWidth),
286
+ () => embedMedia(input.net, code),
287
+ () => graphqlMedia(input.net, code)
288
+ ];
289
+ let media = null;
290
+ for (const resolve of resolvers) {
291
+ const candidate = await resolve();
292
+ if (candidate && (!videoRequired || mediaItems(candidate, code, input.preferredWidth).some((item) => item.kind === "video"))) {
293
+ media = candidate;
294
+ break;
295
+ }
296
+ }
275
297
  if (!media) {
276
298
  throw await instagramUnavailable(input.net, code);
277
299
  }
@@ -481,23 +503,20 @@ async function graphqlMedia(net, code) {
481
503
  }
482
504
  const body = new URLSearchParams({
483
505
  ...params.body,
484
- doc_id: "8845758582119845",
506
+ av: "0",
507
+ doc_id: "26130443479876713",
485
508
  fb_api_caller_class: "RelayModern",
486
- fb_api_req_friendly_name: "PolarisPostActionLoadPostQueryQuery",
509
+ fb_api_req_friendly_name: "PolarisPostRootQuery",
487
510
  server_timestamps: "true",
488
- variables: JSON.stringify({
489
- fetch_tagged_user_count: null,
490
- hoisted_comment_id: null,
491
- hoisted_reply_id: null,
492
- shortcode: code
493
- })
511
+ variables: JSON.stringify({ shortcode: code })
494
512
  });
495
513
  const response = await net("https://www.instagram.com/graphql/query", {
496
514
  body,
497
515
  headers: {
498
516
  ...embedHeaders(),
499
517
  ...params.headers,
500
- "X-FB-Friendly-Name": "PolarisPostActionLoadPostQueryQuery",
518
+ "X-FB-Friendly-Name": "PolarisPostRootQuery",
519
+ "X-Requested-With": "XMLHttpRequest",
501
520
  "content-type": "application/x-www-form-urlencoded"
502
521
  },
503
522
  method: "POST"
@@ -523,14 +542,14 @@ async function graphqlParams(net, code) {
523
542
  const push = entryObject("InstagramWebPushInfo", html);
524
543
  const lsd = entryObject("LSD", html)?.token ?? randomToken();
525
544
  const csrf = entryObject("InstagramSecurityConfig", html)?.csrf_token;
526
- const cookie = [
527
- csrf && `csrftoken=${csrf}`,
528
- polaris?.device_id && `ig_did=${polaris.device_id}`,
529
- polaris?.machine_id && `mid=${polaris.machine_id}`,
530
- "wd=1280x720",
531
- "dpr=2",
532
- "ig_nrcb=1"
533
- ].filter((value) => typeof value === "string" && value.length > 0).join("; ");
545
+ const cookie = instagramCookies(response.headers, {
546
+ csrftoken: string(csrf),
547
+ dpr: "2",
548
+ ig_did: string(polaris?.device_id),
549
+ ig_nrcb: "1",
550
+ mid: string(polaris?.machine_id),
551
+ wd: "1280x720"
552
+ });
534
553
  return {
535
554
  headers: {
536
555
  "X-CSRFToken": string(csrf) ?? "",
@@ -564,6 +583,12 @@ async function graphqlParams(net, code) {
564
583
  };
565
584
  }
566
585
  function gqlShortcodeMedia(data) {
586
+ const webInfo = object(data.xdt_api__v1__media__shortcode__web_info) ? data.xdt_api__v1__media__shortcode__web_info : null;
587
+ const webItems = webInfo && Array.isArray(webInfo.items) ? webInfo.items : [];
588
+ const webMedia = webItems[0];
589
+ if (object(webMedia)) {
590
+ return webMedia;
591
+ }
567
592
  const media = data.gql_data && object(data.gql_data) ? data.gql_data.shortcode_media ?? data.gql_data.xdt_shortcode_media : data.shortcode_media ?? data.xdt_shortcode_media;
568
593
  return object(media) ? media : null;
569
594
  }
@@ -583,6 +608,24 @@ function mediaItems(media, code, preferredWidth) {
583
608
  }
584
609
  return instagramItem(media, code, null, preferredWidth);
585
610
  }
611
+ function instagramCookies(headers, fallback) {
612
+ const getter = headers.getSetCookie;
613
+ const raw = typeof getter === "function" ? getter.call(headers) : headers.get("set-cookie")?.split(/,(?=[^;]+?=)/) ?? [];
614
+ const cookies = new Map;
615
+ for (const value of raw) {
616
+ const pair = value.split(";", 1)[0]?.trim();
617
+ const separator = pair?.indexOf("=") ?? -1;
618
+ if (pair && separator > 0) {
619
+ cookies.set(pair.slice(0, separator), pair.slice(separator + 1));
620
+ }
621
+ }
622
+ for (const [name, value] of Object.entries(fallback)) {
623
+ if (value && !cookies.has(name)) {
624
+ cookies.set(name, value);
625
+ }
626
+ }
627
+ return [...cookies].map(([name, value]) => `${name}=${value}`).join("; ");
628
+ }
586
629
  function instagramItem(media, code, index, preferredWidth) {
587
630
  const video = selectVersion(media.video_versions, preferredWidth) ?? string(media.video_url);
588
631
  const suffix = index === null ? "" : `_${index}`;
@@ -646,6 +689,208 @@ function randomToken(length = 8) {
646
689
  return crypto.getRandomValues(new Uint8Array(length)).reduce((value, byte) => value + (byte % 36).toString(36), "");
647
690
  }
648
691
 
692
+ // src/linkedin.ts
693
+ async function resolveLinkedin(input) {
694
+ const reference = postReference(input.url);
695
+ const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
696
+ const response = await input.net(pageUrl, { headers: navigationHeaders() });
697
+ if (response.status === 404) {
698
+ throw new PostfetchError(404, "LinkedIn post not found", "notFound");
699
+ }
700
+ if (response.status === 429 || response.status === 999) {
701
+ throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
702
+ }
703
+ if (!response.ok) {
704
+ throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
705
+ }
706
+ if (response.url.includes("/signup/") || response.url.includes("/login")) {
707
+ throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
708
+ }
709
+ const html = await response.text();
710
+ const post = linkedinPost(html);
711
+ if (!post) {
712
+ throw new PostfetchError(404, "LinkedIn post unavailable", "unavailable");
713
+ }
714
+ const canonical = string(post["@id"]);
715
+ const canonicalReference = canonical ? postReferenceOrNull(canonical) : null;
716
+ const id = canonicalReference?.id ?? reference.id;
717
+ const items = linkedinItems(post, html, id, pageUrl);
718
+ if (items.length === 0) {
719
+ throw new Error("LinkedIn media url not found");
720
+ }
721
+ return {
722
+ archiveFilename: filename(`linkedin_${id}.zip`),
723
+ id,
724
+ items,
725
+ metadata: linkedinMetadata(post),
726
+ platform: "linkedin"
727
+ };
728
+ }
729
+ function linkedinMetadata(post) {
730
+ const author = linkedinAuthor(post);
731
+ return {
732
+ title: string(post.text) ?? string(post.name) ?? string(post.headline) ?? undefined,
733
+ text: string(post.description) ?? string(post.articleBody) ?? string(post.text) ?? undefined,
734
+ author: author ? {
735
+ handle: linkedinHandle(string(author.url)) ?? undefined,
736
+ name: string(author.name) ?? undefined
737
+ } : undefined,
738
+ createdAt: isoFromDateString(post.datePublished ?? post.uploadDate),
739
+ likeCount: interactionCount(post.interactionStatistic, "LikeAction"),
740
+ commentCount: count(post.commentCount) ?? interactionCount(post.interactionStatistic, "CommentAction")
741
+ };
742
+ }
743
+ function postReference(input) {
744
+ const reference = postReferenceOrNull(input);
745
+ if (!reference) {
746
+ throw new PostfetchError(400, "LinkedIn post id not found");
747
+ }
748
+ return reference;
749
+ }
750
+ function postReferenceOrNull(input) {
751
+ let pathname;
752
+ try {
753
+ pathname = new URL(input).pathname.replace(/%3A/gi, ":");
754
+ } catch {
755
+ return null;
756
+ }
757
+ const urn = pathname.match(/\/(?:embed\/)?feed\/update\/urn:li:(activity|share|ugcpost):(\d+)\/?$/i);
758
+ const post = pathname.match(/\/posts\/[^/]*-(activity|share|ugcpost)-(\d+)(?:-[^/]*)?\/?$/i);
759
+ const match = urn ?? post;
760
+ if (!match) {
761
+ return null;
762
+ }
763
+ switch (match[1].toLowerCase()) {
764
+ case "activity":
765
+ return { id: match[2], kind: "activity" };
766
+ case "share":
767
+ return { id: match[2], kind: "share" };
768
+ case "ugcpost":
769
+ return { id: match[2], kind: "ugcPost" };
770
+ default:
771
+ return null;
772
+ }
773
+ }
774
+ function linkedinAuthor(post) {
775
+ if (object(post.creator)) {
776
+ return post.creator;
777
+ }
778
+ if (object(post.author)) {
779
+ return post.author;
780
+ }
781
+ return null;
782
+ }
783
+ function linkedinPost(html) {
784
+ for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
785
+ if (!/\btype=["']application\/ld\+json["']/i.test(match[1])) {
786
+ continue;
787
+ }
788
+ const parsed = parseJson(match[2]);
789
+ for (const entry of jsonLdEntries(parsed)) {
790
+ if (hasSchemaType(entry, "VideoObject") || hasSchemaType(entry, "SocialMediaPosting")) {
791
+ return entry;
792
+ }
793
+ }
794
+ }
795
+ return null;
796
+ }
797
+ function parseJson(value) {
798
+ try {
799
+ return JSON.parse(value);
800
+ } catch {
801
+ try {
802
+ return JSON.parse(decodeHtml(value));
803
+ } catch {
804
+ return null;
805
+ }
806
+ }
807
+ }
808
+ function jsonLdEntries(value) {
809
+ if (Array.isArray(value)) {
810
+ return value.filter(object);
811
+ }
812
+ if (!object(value)) {
813
+ return [];
814
+ }
815
+ return Array.isArray(value["@graph"]) ? value["@graph"].filter(object) : [value];
816
+ }
817
+ function hasSchemaType(value, expected) {
818
+ const type = value["@type"];
819
+ return type === expected || Array.isArray(type) && type.includes(expected);
820
+ }
821
+ function linkedinItems(post, html, id, referer) {
822
+ const headers = { referer, "user-agent": browserUserAgent() };
823
+ if (hasSchemaType(post, "VideoObject")) {
824
+ const url2 = bestVideoUrl(html) ?? string(post.contentUrl);
825
+ return url2 ? [{ filename: filename(`linkedin_${id}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "linkedin", url: url2 }] : [];
826
+ }
827
+ const url = imageUrl(post.image);
828
+ return url ? [{ filename: filename(`linkedin_${id}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "linkedin", url }] : [];
829
+ }
830
+ function bestVideoUrl(html) {
831
+ const encoded = html.match(/<video\b[^>]*\bdata-sources=(["'])(.*?)\1[^>]*>/is)?.[2];
832
+ if (!encoded) {
833
+ return null;
834
+ }
835
+ let parsed;
836
+ try {
837
+ parsed = JSON.parse(decodeHtml(encoded));
838
+ } catch {
839
+ return null;
840
+ }
841
+ const sources = Array.isArray(parsed) ? parsed.filter(object) : [];
842
+ const videos = sources.filter((source2) => string(source2.type) === "video/mp4" && string(source2.src));
843
+ const selected = videos.reduce((best, candidate) => {
844
+ const bitrate = count(candidate["data-bitrate"]) ?? 0;
845
+ const bestBitrate = best ? count(best["data-bitrate"]) ?? 0 : -1;
846
+ return bitrate > bestBitrate ? candidate : best;
847
+ }, null);
848
+ return selected ? string(selected.src) : null;
849
+ }
850
+ function imageUrl(value) {
851
+ if (typeof value === "string") {
852
+ return string(value);
853
+ }
854
+ if (Array.isArray(value)) {
855
+ return value.map(imageUrl).find((url) => Boolean(url)) ?? null;
856
+ }
857
+ return object(value) ? string(value.url) : null;
858
+ }
859
+ function interactionCount(value, action) {
860
+ const statistics = Array.isArray(value) ? value.filter(object) : object(value) ? [value] : [];
861
+ const statistic = statistics.find((entry) => string(entry.interactionType)?.endsWith(`/${action}`));
862
+ return statistic ? count(statistic.userInteractionCount) : undefined;
863
+ }
864
+ function linkedinHandle(value) {
865
+ if (!value) {
866
+ return null;
867
+ }
868
+ try {
869
+ return new URL(value).pathname.match(/^\/(?:company|in|showcase)\/([^/]+)/)?.[1] ?? null;
870
+ } catch {
871
+ return null;
872
+ }
873
+ }
874
+ function decodeHtml(value) {
875
+ return value.replace(/&(quot|amp|lt|gt|#39|#x27|#(\d+)|#x([0-9a-f]+));/gi, (_entity, name, decimal, hex) => {
876
+ switch (name.toLowerCase()) {
877
+ case "quot":
878
+ return '"';
879
+ case "amp":
880
+ return "&";
881
+ case "lt":
882
+ return "<";
883
+ case "gt":
884
+ return ">";
885
+ case "#39":
886
+ case "#x27":
887
+ return "'";
888
+ default:
889
+ return String.fromCodePoint(decimal ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
890
+ }
891
+ });
892
+ }
893
+
649
894
  // src/isobmff.ts
650
895
  function parseBoxes(data, start = 0, end = data.length) {
651
896
  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
@@ -1407,23 +1652,18 @@ function trackId(track) {
1407
1652
 
1408
1653
  // src/tiktok.ts
1409
1654
  var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
1655
+ var pageAttemptCount = 12;
1410
1656
  async function resolveTiktok(input) {
1411
- const userAgent = browserUserAgent();
1412
- const pageUrl = await followShortlink(input.net, userAgent, input.url);
1657
+ const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
1413
1658
  const id = videoId(pageUrl);
1414
1659
  if (!id) {
1415
1660
  throw new Error("TikTok video id not found");
1416
1661
  }
1417
- const page = await fetchVideoPage(input.net, id, userAgent).catch((error) => {
1418
- if (!recoverablePageError(error)) {
1419
- throw error;
1420
- }
1421
- return fetchVideoPage(input.net, id, firefoxUserAgent());
1422
- });
1662
+ const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
1423
1663
  const user = author(page.item) ?? username(pageUrl) ?? "i";
1424
1664
  const headers = {
1425
1665
  referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
1426
- "user-agent": userAgent
1666
+ "user-agent": page.userAgent
1427
1667
  };
1428
1668
  if (page.cookie) {
1429
1669
  headers.cookie = page.cookie;
@@ -1434,6 +1674,35 @@ async function resolveTiktok(input) {
1434
1674
  }
1435
1675
  return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
1436
1676
  }
1677
+ async function videoPage(net, id, photo) {
1678
+ for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
1679
+ const baseHeaders2 = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
1680
+ const userAgent2 = baseHeaders2["user-agent"];
1681
+ const headers2 = {
1682
+ ...baseHeaders2,
1683
+ referer: "https://www.tiktok.com/",
1684
+ "sec-fetch-site": "same-origin"
1685
+ };
1686
+ try {
1687
+ return { ...await fetchVideoPage(net, id, headers2), userAgent: userAgent2 };
1688
+ } catch (error) {
1689
+ if (!recoverablePageError(error)) {
1690
+ throw error;
1691
+ }
1692
+ }
1693
+ }
1694
+ if (!photo) {
1695
+ throw new Error("TikTok hydration not found");
1696
+ }
1697
+ const baseHeaders = navigationHeaders();
1698
+ const userAgent = baseHeaders["user-agent"];
1699
+ const headers = {
1700
+ ...baseHeaders,
1701
+ referer: "https://www.tiktok.com/",
1702
+ "sec-fetch-site": "same-origin"
1703
+ };
1704
+ return { ...await fetchEmbedPage(net, id, headers), userAgent };
1705
+ }
1437
1706
  function tiktokMetadata(item) {
1438
1707
  const user = object(item.author) ? item.author : null;
1439
1708
  const stats = object(item.stats) ? item.stats : null;
@@ -1460,16 +1729,18 @@ function tiktokMetadata(item) {
1460
1729
  }
1461
1730
  };
1462
1731
  }
1463
- async function fetchVideoPage(net, id, userAgent) {
1732
+ async function fetchVideoPage(net, id, headers) {
1464
1733
  const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
1465
- headers: {
1466
- accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
1467
- referer: "https://www.tiktok.com/",
1468
- "user-agent": userAgent
1469
- }
1734
+ headers
1470
1735
  });
1471
1736
  return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
1472
1737
  }
1738
+ async function fetchEmbedPage(net, id, headers) {
1739
+ const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
1740
+ headers
1741
+ });
1742
+ return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
1743
+ }
1473
1744
  function isShortlinkHost(hostname) {
1474
1745
  return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
1475
1746
  }
@@ -1509,10 +1780,68 @@ function itemStruct(html) {
1509
1780
  }
1510
1781
  return item;
1511
1782
  }
1783
+ function embedItemStruct(html, id) {
1784
+ const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
1785
+ if (!raw) {
1786
+ throw new Error("TikTok embed state not found");
1787
+ }
1788
+ const parsed = JSON.parse(raw);
1789
+ const source2 = object(parsed) && object(parsed.source) ? parsed.source : null;
1790
+ const data = source2 && object(source2.data) ? source2.data : null;
1791
+ const rawRoute = data ? data[`/embed/v2/${id}`] : null;
1792
+ const route = object(rawRoute) ? rawRoute : null;
1793
+ const videoData = route && object(route.videoData) ? route.videoData : null;
1794
+ const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
1795
+ if (!videoData || !item) {
1796
+ throw new Error("TikTok embed item not found");
1797
+ }
1798
+ const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
1799
+ const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
1800
+ const video = object(item.video) ? item.video : {};
1801
+ const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
1802
+ const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
1803
+ const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
1804
+ if (displayImages.length === 0) {
1805
+ throw new Error("TikTok video hydration not found");
1806
+ }
1807
+ return {
1808
+ ...item,
1809
+ author: {
1810
+ nickname: string(authorInfo.nickName),
1811
+ uniqueId: string(authorInfo.uniqueId),
1812
+ verified: bool(authorInfo.verified)
1813
+ },
1814
+ desc: string(item.text),
1815
+ imagePost: {
1816
+ images: displayImages.map((image) => ({
1817
+ imageURL: {
1818
+ urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
1819
+ }
1820
+ }))
1821
+ },
1822
+ music: {
1823
+ authorName: string(musicInfo.authorName),
1824
+ playUrl: firstString(musicInfo.playUrl),
1825
+ title: string(musicInfo.musicName)
1826
+ },
1827
+ stats: {
1828
+ commentCount: item.commentCount,
1829
+ diggCount: item.diggCount,
1830
+ playCount: item.playCount,
1831
+ shareCount: item.shareCount
1832
+ },
1833
+ video: {
1834
+ duration: videoMeta.duration
1835
+ }
1836
+ };
1837
+ }
1512
1838
  function recoverablePageError(error) {
1513
1839
  const message = error instanceof Error ? error.message : "";
1514
1840
  return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
1515
1841
  }
1842
+ function firstString(value) {
1843
+ return Array.isArray(value) ? value.map(string).find((candidate) => candidate !== null) ?? null : string(value);
1844
+ }
1516
1845
  function downloadUrl(item) {
1517
1846
  const video = object(item.video) ? item.video : null;
1518
1847
  return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
@@ -1538,8 +1867,8 @@ function imageItems3(item, user, id, headers) {
1538
1867
  const imagePost = object(item.imagePost) ? item.imagePost : null;
1539
1868
  const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
1540
1869
  return images.flatMap((image, index) => {
1541
- const imageUrl = object(image.imageURL) ? image.imageURL : null;
1542
- const list = imageUrl && Array.isArray(imageUrl.urlList) ? imageUrl.urlList : [];
1870
+ const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
1871
+ const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
1543
1872
  const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
1544
1873
  return url ? [{
1545
1874
  filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
@@ -1896,6 +2225,8 @@ async function postfetch(url, options = {}) {
1896
2225
  return resolveFacebook(context);
1897
2226
  case "instagram":
1898
2227
  return resolveInstagram(context);
2228
+ case "linkedin":
2229
+ return resolveLinkedin(context);
1899
2230
  case "pinterest":
1900
2231
  return resolvePinterest(context);
1901
2232
  case "reddit":
@@ -1918,6 +2249,9 @@ function detect(url) {
1918
2249
  if (host.includes("instagram.com")) {
1919
2250
  return "instagram";
1920
2251
  }
2252
+ if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
2253
+ return "linkedin";
2254
+ }
1921
2255
  if (host.includes("pinterest.") || host === "pin.it") {
1922
2256
  return "pinterest";
1923
2257
  }
@@ -1936,7 +2270,7 @@ function detect(url) {
1936
2270
  if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
1937
2271
  return "twitter";
1938
2272
  }
1939
- throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
2273
+ throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
1940
2274
  }
1941
2275
  // src/remux.ts
1942
2276
  function mergeAudioVideo(video, audio) {
@@ -1,5 +1,5 @@
1
1
  /** A platform that {@link postfetch} can resolve. */
2
- export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
2
+ export type Platform = "facebook" | "instagram" | "linkedin" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
3
3
  /** The kind of a {@link MediaItem}. */
4
4
  export type MediaKind = "audio" | "image" | "video";
5
5
  /** A single resolved media file: a direct URL plus everything needed to fetch and name it. */
@@ -137,7 +137,7 @@ export type PinterestExtra = {
137
137
  outboundLink?: string;
138
138
  };
139
139
  /** The result of resolving a post: its platform, id, media items and metadata. */
140
- export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
140
+ export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"linkedin", never> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
141
141
  /** One platform's variant of a {@link PostfetchResult}, carrying that platform's `extra` metadata shape. */
142
142
  export type PlatformResult<P extends Platform, Extra> = {
143
143
  /** Suggested filename when zipping every item. */
@@ -0,0 +1,3 @@
1
+ import { type Json, type PostMetadata, type PostfetchResult, type ResolveContext } from "./internal";
2
+ export declare function resolveLinkedin(input: ResolveContext): Promise<PostfetchResult>;
3
+ export declare function linkedinMetadata(post: Json): PostMetadata;
@@ -14,7 +14,7 @@ export type PostfetchOptions = {
14
14
  * performs no side effects beyond the lookup; use {@link download}, {@link archive}
15
15
  * or {@link toResponse} to turn the result into bytes or a `Response`.
16
16
  *
17
- * @param url A post URL from Instagram, TikTok, YouTube, Facebook or X.
17
+ * @param url A post URL from a supported platform.
18
18
  * @param options See {@link PostfetchOptions}.
19
19
  * @returns The resolved post and its media items.
20
20
  * @throws {PostfetchError} If the URL is empty or its host is unsupported.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@postfetch/core",
3
- "version": "0.5.2",
4
- "description": "Zero-dependency typed core that turns Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
3
+ "version": "0.6.0",
4
+ "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
5
5
  "license": "MIT",
6
6
  "type": "module",
7
7
  "exports": {
@@ -30,6 +30,7 @@
30
30
  "downloader",
31
31
  "facebook",
32
32
  "instagram",
33
+ "linkedin",
33
34
  "media",
34
35
  "pinterest",
35
36
  "reddit",
@@ -32,6 +32,10 @@ export type BrowserFingerprint = {
32
32
  userAgent: string;
33
33
  };
34
34
 
35
+ export type NavigationHeaders = Record<string, string> & {
36
+ "user-agent": string;
37
+ };
38
+
35
39
  export function browserFingerprint(): BrowserFingerprint {
36
40
  const version = pick(chromeVersions);
37
41
  const platform = pick(desktopPlatforms);
@@ -43,20 +47,26 @@ export function browserFingerprint(): BrowserFingerprint {
43
47
  };
44
48
  }
45
49
 
46
- export function navigationHeaders(): Record<string, string> {
47
- const fingerprint = browserFingerprint();
50
+ function baseNavigationHeaders(userAgent: string, acceptLanguage: string): NavigationHeaders {
48
51
  return {
49
52
  accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
50
- "accept-language": fingerprint.acceptLanguage,
51
- "sec-ch-ua": fingerprint.secChUa,
52
- "sec-ch-ua-mobile": "?0",
53
- "sec-ch-ua-platform": fingerprint.secChUaPlatform,
53
+ "accept-language": acceptLanguage,
54
54
  "sec-fetch-dest": "document",
55
55
  "sec-fetch-mode": "navigate",
56
56
  "sec-fetch-site": "none",
57
57
  "sec-fetch-user": "?1",
58
58
  "upgrade-insecure-requests": "1",
59
- "user-agent": fingerprint.userAgent,
59
+ "user-agent": userAgent,
60
+ };
61
+ }
62
+
63
+ export function navigationHeaders(): NavigationHeaders {
64
+ const fingerprint = browserFingerprint();
65
+ return {
66
+ ...baseNavigationHeaders(fingerprint.userAgent, fingerprint.acceptLanguage),
67
+ "sec-ch-ua": fingerprint.secChUa,
68
+ "sec-ch-ua-mobile": "?0",
69
+ "sec-ch-ua-platform": fingerprint.secChUaPlatform,
60
70
  };
61
71
  }
62
72
 
@@ -70,6 +80,10 @@ export function firefoxUserAgent(): string {
70
80
  return `Mozilla/5.0 (${platform.uaToken}; rv:${version}.0) Gecko/20100101 Firefox/${version}.0`;
71
81
  }
72
82
 
83
+ export function firefoxNavigationHeaders(userAgent = firefoxUserAgent()): NavigationHeaders {
84
+ return baseNavigationHeaders(userAgent, pick(acceptLanguages));
85
+ }
86
+
73
87
  export function instagramAppUserAgent(): string {
74
88
  return pick(instagramAppUserAgents);
75
89
  }
package/src/index.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Turn a public social post URL into its media.
3
3
  *
4
- * {@link postfetch} resolves Instagram, TikTok, YouTube, Facebook and X (Twitter)
4
+ * {@link postfetch} resolves Instagram, LinkedIn, TikTok, YouTube, Facebook and X (Twitter)
5
5
  * URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
6
6
  * (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
7
7
  * Cloudflare Workers and browsers alike. It only resolves the media; the
package/src/instagram.ts CHANGED
@@ -58,11 +58,21 @@ function embedHeaders(): Record<string, string> {
58
58
 
59
59
  export async function resolveInstagram(input: ResolveContext): Promise<PostfetchResult> {
60
60
  const code = shortcode(input.url);
61
- const media =
62
- (await pageMedia(input.net, code, input.preferredWidth)) ??
63
- (await mobileMedia(input.net, code, input.preferredWidth)) ??
64
- (await embedMedia(input.net, code)) ??
65
- (await graphqlMedia(input.net, code));
61
+ const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
62
+ const resolvers: Array<() => Promise<Json | null>> = [
63
+ () => pageMedia(input.net, code, input.preferredWidth),
64
+ () => mobileMedia(input.net, code, input.preferredWidth),
65
+ () => embedMedia(input.net, code),
66
+ () => graphqlMedia(input.net, code),
67
+ ];
68
+ let media: Json | null = null;
69
+ for (const resolve of resolvers) {
70
+ const candidate = await resolve();
71
+ if (candidate && (!videoRequired || mediaItems(candidate, code, input.preferredWidth).some((item) => item.kind === "video"))) {
72
+ media = candidate;
73
+ break;
74
+ }
75
+ }
66
76
  if (!media) {
67
77
  throw await instagramUnavailable(input.net, code);
68
78
  }
@@ -301,23 +311,20 @@ async function graphqlMedia(net: Net, code: string): Promise<Json | null> {
301
311
  }
302
312
  const body = new URLSearchParams({
303
313
  ...params.body,
304
- doc_id: "8845758582119845",
314
+ av: "0",
315
+ doc_id: "26130443479876713",
305
316
  fb_api_caller_class: "RelayModern",
306
- fb_api_req_friendly_name: "PolarisPostActionLoadPostQueryQuery",
317
+ fb_api_req_friendly_name: "PolarisPostRootQuery",
307
318
  server_timestamps: "true",
308
- variables: JSON.stringify({
309
- fetch_tagged_user_count: null,
310
- hoisted_comment_id: null,
311
- hoisted_reply_id: null,
312
- shortcode: code,
313
- }),
319
+ variables: JSON.stringify({ shortcode: code }),
314
320
  });
315
321
  const response = await net("https://www.instagram.com/graphql/query", {
316
322
  body,
317
323
  headers: {
318
324
  ...embedHeaders(),
319
325
  ...params.headers,
320
- "X-FB-Friendly-Name": "PolarisPostActionLoadPostQueryQuery",
326
+ "X-FB-Friendly-Name": "PolarisPostRootQuery",
327
+ "X-Requested-With": "XMLHttpRequest",
321
328
  "content-type": "application/x-www-form-urlencoded",
322
329
  },
323
330
  method: "POST",
@@ -344,14 +351,14 @@ async function graphqlParams(net: Net, code: string): Promise<{ body: Record<str
344
351
  const push = entryObject("InstagramWebPushInfo", html);
345
352
  const lsd = entryObject("LSD", html)?.token ?? randomToken();
346
353
  const csrf = entryObject("InstagramSecurityConfig", html)?.csrf_token;
347
- const cookie = [
348
- csrf && `csrftoken=${csrf}`,
349
- polaris?.device_id && `ig_did=${polaris.device_id}`,
350
- polaris?.machine_id && `mid=${polaris.machine_id}`,
351
- "wd=1280x720",
352
- "dpr=2",
353
- "ig_nrcb=1",
354
- ].filter((value): value is string => typeof value === "string" && value.length > 0).join("; ");
354
+ const cookie = instagramCookies(response.headers, {
355
+ csrftoken: string(csrf),
356
+ dpr: "2",
357
+ ig_did: string(polaris?.device_id),
358
+ ig_nrcb: "1",
359
+ mid: string(polaris?.machine_id),
360
+ wd: "1280x720",
361
+ });
355
362
  return {
356
363
  headers: {
357
364
  "X-CSRFToken": string(csrf) ?? "",
@@ -386,6 +393,12 @@ async function graphqlParams(net: Net, code: string): Promise<{ body: Record<str
386
393
  }
387
394
 
388
395
  function gqlShortcodeMedia(data: Json): Json | null {
396
+ const webInfo = object(data.xdt_api__v1__media__shortcode__web_info) ? data.xdt_api__v1__media__shortcode__web_info : null;
397
+ const webItems = webInfo && Array.isArray(webInfo.items) ? webInfo.items : [];
398
+ const webMedia = webItems[0];
399
+ if (object(webMedia)) {
400
+ return webMedia;
401
+ }
389
402
  const media = data.gql_data && object(data.gql_data)
390
403
  ? data.gql_data.shortcode_media ?? data.gql_data.xdt_shortcode_media
391
404
  : data.shortcode_media ?? data.xdt_shortcode_media;
@@ -411,6 +424,25 @@ function mediaItems(media: Json, code: string, preferredWidth: number): MediaIte
411
424
  return instagramItem(media, code, null, preferredWidth);
412
425
  }
413
426
 
427
+ function instagramCookies(headers: Headers, fallback: Record<string, string | null>): string {
428
+ const getter = (headers as Headers & { getSetCookie?: () => string[] }).getSetCookie;
429
+ const raw = typeof getter === "function" ? getter.call(headers) : headers.get("set-cookie")?.split(/,(?=[^;]+?=)/) ?? [];
430
+ const cookies = new Map<string, string>();
431
+ for (const value of raw) {
432
+ const pair = value.split(";", 1)[0]?.trim();
433
+ const separator = pair?.indexOf("=") ?? -1;
434
+ if (pair && separator > 0) {
435
+ cookies.set(pair.slice(0, separator), pair.slice(separator + 1));
436
+ }
437
+ }
438
+ for (const [name, value] of Object.entries(fallback)) {
439
+ if (value && !cookies.has(name)) {
440
+ cookies.set(name, value);
441
+ }
442
+ }
443
+ return [...cookies].map(([name, value]) => `${name}=${value}`).join("; ");
444
+ }
445
+
414
446
  function instagramItem(media: Json, code: string, index: number | null, preferredWidth: number): MediaItem[] {
415
447
  const video = selectVersion(media.video_versions, preferredWidth) ?? string(media.video_url);
416
448
  const suffix = index === null ? "" : `_${index}`;
package/src/internal.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  /** A platform that {@link postfetch} can resolve. */
2
- export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
2
+ export type Platform = "facebook" | "instagram" | "linkedin" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
3
3
 
4
4
  /** The kind of a {@link MediaItem}. */
5
5
  export type MediaKind = "audio" | "image" | "video";
@@ -149,6 +149,7 @@ export type PinterestExtra = {
149
149
  export type PostfetchResult =
150
150
  | PlatformResult<"facebook", never>
151
151
  | PlatformResult<"instagram", InstagramExtra>
152
+ | PlatformResult<"linkedin", never>
152
153
  | PlatformResult<"pinterest", PinterestExtra>
153
154
  | PlatformResult<"reddit", RedditExtra>
154
155
  | PlatformResult<"soundcloud", SoundcloudExtra>
@@ -0,0 +1,243 @@
1
+ import {
2
+ count,
3
+ filename,
4
+ isoFromDateString,
5
+ object,
6
+ PostfetchError,
7
+ string,
8
+ type Json,
9
+ type MediaItem,
10
+ type PostMetadata,
11
+ type PostfetchResult,
12
+ type ResolveContext,
13
+ } from "./internal";
14
+ import { browserUserAgent, navigationHeaders } from "./fingerprint";
15
+
16
+ type LinkedinPostKind = "activity" | "share" | "ugcPost";
17
+
18
+ type LinkedinPostReference = {
19
+ id: string;
20
+ kind: LinkedinPostKind;
21
+ };
22
+
23
+ export async function resolveLinkedin(input: ResolveContext): Promise<PostfetchResult> {
24
+ const reference = postReference(input.url);
25
+ const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
26
+ const response = await input.net(pageUrl, { headers: navigationHeaders() });
27
+ if (response.status === 404) {
28
+ throw new PostfetchError(404, "LinkedIn post not found", "notFound");
29
+ }
30
+ if (response.status === 429 || response.status === 999) {
31
+ throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
32
+ }
33
+ if (!response.ok) {
34
+ throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
35
+ }
36
+ if (response.url.includes("/signup/") || response.url.includes("/login")) {
37
+ throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
38
+ }
39
+
40
+ const html = await response.text();
41
+ const post = linkedinPost(html);
42
+ if (!post) {
43
+ throw new PostfetchError(404, "LinkedIn post unavailable", "unavailable");
44
+ }
45
+ const canonical = string(post["@id"]);
46
+ const canonicalReference = canonical ? postReferenceOrNull(canonical) : null;
47
+ const id = canonicalReference?.id ?? reference.id;
48
+ const items = linkedinItems(post, html, id, pageUrl);
49
+ if (items.length === 0) {
50
+ throw new Error("LinkedIn media url not found");
51
+ }
52
+ return {
53
+ archiveFilename: filename(`linkedin_${id}.zip`),
54
+ id,
55
+ items,
56
+ metadata: linkedinMetadata(post),
57
+ platform: "linkedin",
58
+ };
59
+ }
60
+
61
+ export function linkedinMetadata(post: Json): PostMetadata {
62
+ const author = linkedinAuthor(post);
63
+ return {
64
+ title: string(post.text) ?? string(post.name) ?? string(post.headline) ?? undefined,
65
+ text: string(post.description) ?? string(post.articleBody) ?? string(post.text) ?? undefined,
66
+ author: author
67
+ ? {
68
+ handle: linkedinHandle(string(author.url)) ?? undefined,
69
+ name: string(author.name) ?? undefined,
70
+ }
71
+ : undefined,
72
+ createdAt: isoFromDateString(post.datePublished ?? post.uploadDate),
73
+ likeCount: interactionCount(post.interactionStatistic, "LikeAction"),
74
+ commentCount: count(post.commentCount) ?? interactionCount(post.interactionStatistic, "CommentAction"),
75
+ };
76
+ }
77
+
78
+ function postReference(input: string): LinkedinPostReference {
79
+ const reference = postReferenceOrNull(input);
80
+ if (!reference) {
81
+ throw new PostfetchError(400, "LinkedIn post id not found");
82
+ }
83
+ return reference;
84
+ }
85
+
86
+ function postReferenceOrNull(input: string): LinkedinPostReference | null {
87
+ let pathname: string;
88
+ try {
89
+ pathname = new URL(input).pathname.replace(/%3A/gi, ":");
90
+ } catch {
91
+ return null;
92
+ }
93
+ const urn = pathname.match(/\/(?:embed\/)?feed\/update\/urn:li:(activity|share|ugcpost):(\d+)\/?$/i);
94
+ const post = pathname.match(/\/posts\/[^/]*-(activity|share|ugcpost)-(\d+)(?:-[^/]*)?\/?$/i);
95
+ const match = urn ?? post;
96
+ if (!match) {
97
+ return null;
98
+ }
99
+ switch (match[1].toLowerCase()) {
100
+ case "activity":
101
+ return { id: match[2], kind: "activity" };
102
+ case "share":
103
+ return { id: match[2], kind: "share" };
104
+ case "ugcpost":
105
+ return { id: match[2], kind: "ugcPost" };
106
+ default:
107
+ return null;
108
+ }
109
+ }
110
+
111
+ function linkedinAuthor(post: Json): Json | null {
112
+ if (object(post.creator)) {
113
+ return post.creator;
114
+ }
115
+ if (object(post.author)) {
116
+ return post.author;
117
+ }
118
+ return null;
119
+ }
120
+
121
+ function linkedinPost(html: string): Json | null {
122
+ for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
123
+ if (!/\btype=["']application\/ld\+json["']/i.test(match[1])) {
124
+ continue;
125
+ }
126
+ const parsed = parseJson(match[2]);
127
+ for (const entry of jsonLdEntries(parsed)) {
128
+ if (hasSchemaType(entry, "VideoObject") || hasSchemaType(entry, "SocialMediaPosting")) {
129
+ return entry;
130
+ }
131
+ }
132
+ }
133
+ return null;
134
+ }
135
+
136
+ function parseJson(value: string): unknown {
137
+ try {
138
+ return JSON.parse(value);
139
+ } catch {
140
+ try {
141
+ return JSON.parse(decodeHtml(value));
142
+ } catch {
143
+ return null;
144
+ }
145
+ }
146
+ }
147
+
148
+ function jsonLdEntries(value: unknown): Json[] {
149
+ if (Array.isArray(value)) {
150
+ return value.filter(object);
151
+ }
152
+ if (!object(value)) {
153
+ return [];
154
+ }
155
+ return Array.isArray(value["@graph"]) ? value["@graph"].filter(object) : [value];
156
+ }
157
+
158
+ function hasSchemaType(value: Json, expected: string): boolean {
159
+ const type = value["@type"];
160
+ return type === expected || (Array.isArray(type) && type.includes(expected));
161
+ }
162
+
163
+ function linkedinItems(post: Json, html: string, id: string, referer: string): MediaItem[] {
164
+ const headers = { referer, "user-agent": browserUserAgent() };
165
+ if (hasSchemaType(post, "VideoObject")) {
166
+ const url = bestVideoUrl(html) ?? string(post.contentUrl);
167
+ return url
168
+ ? [{ filename: filename(`linkedin_${id}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "linkedin", url }]
169
+ : [];
170
+ }
171
+ const url = imageUrl(post.image);
172
+ return url
173
+ ? [{ filename: filename(`linkedin_${id}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "linkedin", url }]
174
+ : [];
175
+ }
176
+
177
+ function bestVideoUrl(html: string): string | null {
178
+ const encoded = html.match(/<video\b[^>]*\bdata-sources=(["'])(.*?)\1[^>]*>/is)?.[2];
179
+ if (!encoded) {
180
+ return null;
181
+ }
182
+ let parsed: unknown;
183
+ try {
184
+ parsed = JSON.parse(decodeHtml(encoded));
185
+ } catch {
186
+ return null;
187
+ }
188
+ const sources = Array.isArray(parsed) ? parsed.filter(object) : [];
189
+ const videos = sources.filter((source) => string(source.type) === "video/mp4" && string(source.src));
190
+ const selected = videos.reduce<Json | null>((best, candidate) => {
191
+ const bitrate = count(candidate["data-bitrate"]) ?? 0;
192
+ const bestBitrate = best ? count(best["data-bitrate"]) ?? 0 : -1;
193
+ return bitrate > bestBitrate ? candidate : best;
194
+ }, null);
195
+ return selected ? string(selected.src) : null;
196
+ }
197
+
198
+ function imageUrl(value: unknown): string | null {
199
+ if (typeof value === "string") {
200
+ return string(value);
201
+ }
202
+ if (Array.isArray(value)) {
203
+ return value.map(imageUrl).find((url): url is string => Boolean(url)) ?? null;
204
+ }
205
+ return object(value) ? string(value.url) : null;
206
+ }
207
+
208
+ function interactionCount(value: unknown, action: string): number | undefined {
209
+ const statistics = Array.isArray(value) ? value.filter(object) : object(value) ? [value] : [];
210
+ const statistic = statistics.find((entry) => string(entry.interactionType)?.endsWith(`/${action}`));
211
+ return statistic ? count(statistic.userInteractionCount) : undefined;
212
+ }
213
+
214
+ function linkedinHandle(value: string | null): string | null {
215
+ if (!value) {
216
+ return null;
217
+ }
218
+ try {
219
+ return new URL(value).pathname.match(/^\/(?:company|in|showcase)\/([^/]+)/)?.[1] ?? null;
220
+ } catch {
221
+ return null;
222
+ }
223
+ }
224
+
225
+ function decodeHtml(value: string): string {
226
+ return value.replace(/&(quot|amp|lt|gt|#39|#x27|#(\d+)|#x([0-9a-f]+));/gi, (_entity, name: string, decimal: string, hex: string) => {
227
+ switch (name.toLowerCase()) {
228
+ case "quot":
229
+ return '"';
230
+ case "amp":
231
+ return "&";
232
+ case "lt":
233
+ return "<";
234
+ case "gt":
235
+ return ">";
236
+ case "#39":
237
+ case "#x27":
238
+ return "'";
239
+ default:
240
+ return String.fromCodePoint(decimal ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
241
+ }
242
+ });
243
+ }
package/src/postfetch.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { asUrl, createNet, PostfetchError, type Platform, type PostfetchResult } from "./internal";
2
2
  import { resolveFacebook } from "./facebook";
3
3
  import { resolveInstagram } from "./instagram";
4
+ import { resolveLinkedin } from "./linkedin";
4
5
  import { resolvePinterest } from "./pinterest";
5
6
  import { resolveReddit } from "./reddit";
6
7
  import { resolveSoundcloud } from "./soundcloud";
@@ -24,7 +25,7 @@ export type PostfetchOptions = {
24
25
  * performs no side effects beyond the lookup; use {@link download}, {@link archive}
25
26
  * or {@link toResponse} to turn the result into bytes or a `Response`.
26
27
  *
27
- * @param url A post URL from Instagram, TikTok, YouTube, Facebook or X.
28
+ * @param url A post URL from a supported platform.
28
29
  * @param options See {@link PostfetchOptions}.
29
30
  * @returns The resolved post and its media items.
30
31
  * @throws {PostfetchError} If the URL is empty or its host is unsupported.
@@ -52,6 +53,8 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
52
53
  return resolveFacebook(context);
53
54
  case "instagram":
54
55
  return resolveInstagram(context);
56
+ case "linkedin":
57
+ return resolveLinkedin(context);
55
58
  case "pinterest":
56
59
  return resolvePinterest(context);
57
60
  case "reddit":
@@ -82,6 +85,9 @@ export function detect(url: string): Platform {
82
85
  if (host.includes("instagram.com")) {
83
86
  return "instagram";
84
87
  }
88
+ if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
89
+ return "linkedin";
90
+ }
85
91
  if (host.includes("pinterest.") || host === "pin.it") {
86
92
  return "pinterest";
87
93
  }
@@ -100,5 +106,5 @@ export function detect(url: string): Platform {
100
106
  if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
101
107
  return "twitter";
102
108
  }
103
- throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
109
+ throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
104
110
  }
package/src/tiktok.ts CHANGED
@@ -14,27 +14,26 @@ import {
14
14
  type PostfetchResult,
15
15
  type MediaItem,
16
16
  } from "./internal";
17
- import { browserUserAgent, firefoxUserAgent } from "./fingerprint";
17
+ import {
18
+ browserUserAgent,
19
+ firefoxNavigationHeaders,
20
+ navigationHeaders,
21
+ } from "./fingerprint";
18
22
 
19
23
  const marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
24
+ const pageAttemptCount = 12;
20
25
 
21
26
  export async function resolveTiktok(input: ResolveContext): Promise<PostfetchResult> {
22
- const userAgent = browserUserAgent();
23
- const pageUrl = await followShortlink(input.net, userAgent, input.url);
27
+ const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
24
28
  const id = videoId(pageUrl);
25
29
  if (!id) {
26
30
  throw new Error("TikTok video id not found");
27
31
  }
28
- const page = await fetchVideoPage(input.net, id, userAgent).catch((error: unknown) => {
29
- if (!recoverablePageError(error)) {
30
- throw error;
31
- }
32
- return fetchVideoPage(input.net, id, firefoxUserAgent());
33
- });
32
+ const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
34
33
  const user = author(page.item) ?? username(pageUrl) ?? "i";
35
34
  const headers: Record<string, string> = {
36
35
  referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
37
- "user-agent": userAgent,
36
+ "user-agent": page.userAgent,
38
37
  };
39
38
  if (page.cookie) {
40
39
  headers.cookie = page.cookie;
@@ -46,6 +45,41 @@ export async function resolveTiktok(input: ResolveContext): Promise<PostfetchRes
46
45
  return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
47
46
  }
48
47
 
48
+ async function videoPage(
49
+ net: Net,
50
+ id: string,
51
+ photo: boolean,
52
+ ): Promise<{ item: Json; cookie: string | null; userAgent: string }> {
53
+ for (let attempt = 0; attempt < pageAttemptCount; attempt += 1) {
54
+ const baseHeaders = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
55
+ const userAgent = baseHeaders["user-agent"];
56
+ const headers = {
57
+ ...baseHeaders,
58
+ referer: "https://www.tiktok.com/",
59
+ "sec-fetch-site": "same-origin",
60
+ };
61
+ try {
62
+ return { ...(await fetchVideoPage(net, id, headers)), userAgent };
63
+ } catch (error) {
64
+ if (!recoverablePageError(error)) {
65
+ throw error;
66
+ }
67
+ }
68
+ }
69
+
70
+ if (!photo) {
71
+ throw new Error("TikTok hydration not found");
72
+ }
73
+ const baseHeaders = navigationHeaders();
74
+ const userAgent = baseHeaders["user-agent"];
75
+ const headers = {
76
+ ...baseHeaders,
77
+ referer: "https://www.tiktok.com/",
78
+ "sec-fetch-site": "same-origin",
79
+ };
80
+ return { ...(await fetchEmbedPage(net, id, headers)), userAgent };
81
+ }
82
+
49
83
  export function tiktokMetadata(item: Json): PostMetadata & { extra?: TiktokExtra } {
50
84
  const user = object(item.author) ? item.author : null;
51
85
  const stats = object(item.stats) ? item.stats : null;
@@ -78,18 +112,25 @@ export function tiktokMetadata(item: Json): PostMetadata & { extra?: TiktokExtra
78
112
  async function fetchVideoPage(
79
113
  net: Net,
80
114
  id: string,
81
- userAgent: string,
115
+ headers: Record<string, string>,
82
116
  ): Promise<{ item: Json; cookie: string | null }> {
83
117
  const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
84
- headers: {
85
- accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
86
- referer: "https://www.tiktok.com/",
87
- "user-agent": userAgent,
88
- },
118
+ headers,
89
119
  });
90
120
  return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
91
121
  }
92
122
 
123
+ async function fetchEmbedPage(
124
+ net: Net,
125
+ id: string,
126
+ headers: Record<string, string>,
127
+ ): Promise<{ item: Json; cookie: string | null }> {
128
+ const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
129
+ headers,
130
+ });
131
+ return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
132
+ }
133
+
93
134
  export function isShortlinkHost(hostname: string): boolean {
94
135
  return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
95
136
  }
@@ -134,11 +175,75 @@ function itemStruct(html: string): Json {
134
175
  return item;
135
176
  }
136
177
 
178
+ function embedItemStruct(html: string, id: string): Json {
179
+ const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
180
+ if (!raw) {
181
+ throw new Error("TikTok embed state not found");
182
+ }
183
+ const parsed: unknown = JSON.parse(raw);
184
+ const source = object(parsed) && object(parsed.source) ? parsed.source : null;
185
+ const data = source && object(source.data) ? source.data : null;
186
+ const rawRoute = data ? data[`/embed/v2/${id}`] : null;
187
+ const route = object(rawRoute) ? rawRoute : null;
188
+ const videoData = route && object(route.videoData) ? route.videoData : null;
189
+ const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
190
+ if (!videoData || !item) {
191
+ throw new Error("TikTok embed item not found");
192
+ }
193
+
194
+ const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
195
+ const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
196
+ const video = object(item.video) ? item.video : {};
197
+ const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
198
+ const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
199
+ const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
200
+ if (displayImages.length === 0) {
201
+ throw new Error("TikTok video hydration not found");
202
+ }
203
+
204
+ return {
205
+ ...item,
206
+ author: {
207
+ nickname: string(authorInfo.nickName),
208
+ uniqueId: string(authorInfo.uniqueId),
209
+ verified: bool(authorInfo.verified),
210
+ },
211
+ desc: string(item.text),
212
+ imagePost: {
213
+ images: displayImages.map((image) => ({
214
+ imageURL: {
215
+ urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url): url is string => Boolean(url)) : [],
216
+ },
217
+ })),
218
+ },
219
+ music: {
220
+ authorName: string(musicInfo.authorName),
221
+ playUrl: firstString(musicInfo.playUrl),
222
+ title: string(musicInfo.musicName),
223
+ },
224
+ stats: {
225
+ commentCount: item.commentCount,
226
+ diggCount: item.diggCount,
227
+ playCount: item.playCount,
228
+ shareCount: item.shareCount,
229
+ },
230
+ video: {
231
+ duration: videoMeta.duration,
232
+ },
233
+ };
234
+ }
235
+
137
236
  function recoverablePageError(error: unknown): boolean {
138
237
  const message = error instanceof Error ? error.message : "";
139
238
  return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
140
239
  }
141
240
 
241
+ function firstString(value: unknown): string | null {
242
+ return Array.isArray(value)
243
+ ? value.map(string).find((candidate): candidate is string => candidate !== null) ?? null
244
+ : string(value);
245
+ }
246
+
142
247
  function downloadUrl(item: Json): string | null {
143
248
  const video = object(item.video) ? item.video : null;
144
249
  return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;