@postfetch/core 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/fingerprint.d.ts +5 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +377 -43
- package/dist/internal.d.ts +2 -2
- package/dist/linkedin.d.ts +3 -0
- package/dist/postfetch.d.ts +1 -1
- package/package.json +3 -2
- package/src/fingerprint.ts +21 -7
- package/src/index.ts +1 -1
- package/src/instagram.ts +54 -22
- package/src/internal.ts +2 -1
- package/src/linkedin.ts +243 -0
- package/src/postfetch.ts +8 -2
- package/src/tiktok.ts +121 -16
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @postfetch/core
|
|
2
2
|
|
|
3
|
-
Zero-dependency typed core that turns Instagram, TikTok and YouTube post URLs into media files. Part of [postfetch](https://github.com/chelokot/postfetch).
|
|
3
|
+
Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files. Part of [postfetch](https://github.com/chelokot/postfetch).
|
|
4
4
|
|
|
5
5
|
```ts
|
|
6
6
|
import { postfetch, download, archive, toResponse, PostfetchError } from "@postfetch/core";
|
package/dist/fingerprint.d.ts
CHANGED
|
@@ -4,8 +4,12 @@ export type BrowserFingerprint = {
|
|
|
4
4
|
secChUaPlatform: string;
|
|
5
5
|
userAgent: string;
|
|
6
6
|
};
|
|
7
|
+
export type NavigationHeaders = Record<string, string> & {
|
|
8
|
+
"user-agent": string;
|
|
9
|
+
};
|
|
7
10
|
export declare function browserFingerprint(): BrowserFingerprint;
|
|
8
|
-
export declare function navigationHeaders():
|
|
11
|
+
export declare function navigationHeaders(): NavigationHeaders;
|
|
9
12
|
export declare function browserUserAgent(): string;
|
|
10
13
|
export declare function firefoxUserAgent(): string;
|
|
14
|
+
export declare function firefoxNavigationHeaders(userAgent?: string): NavigationHeaders;
|
|
11
15
|
export declare function instagramAppUserAgent(): string;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Turn a public social post URL into its media.
|
|
3
3
|
*
|
|
4
|
-
* {@link postfetch} resolves Instagram, TikTok, YouTube, Facebook and X (Twitter)
|
|
4
|
+
* {@link postfetch} resolves Instagram, LinkedIn, TikTok, YouTube, Facebook and X (Twitter)
|
|
5
5
|
* URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
|
|
6
6
|
* (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
|
|
7
7
|
* Cloudflare Workers and browsers alike. It only resolves the media; the
|
package/dist/index.js
CHANGED
|
@@ -124,20 +124,25 @@ function browserFingerprint() {
|
|
|
124
124
|
userAgent: `Mozilla/5.0 (${platform.uaToken}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${version}.0.0.0 Safari/537.36`
|
|
125
125
|
};
|
|
126
126
|
}
|
|
127
|
-
function
|
|
128
|
-
const fingerprint = browserFingerprint();
|
|
127
|
+
function baseNavigationHeaders(userAgent, acceptLanguage) {
|
|
129
128
|
return {
|
|
130
129
|
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
131
|
-
"accept-language":
|
|
132
|
-
"sec-ch-ua": fingerprint.secChUa,
|
|
133
|
-
"sec-ch-ua-mobile": "?0",
|
|
134
|
-
"sec-ch-ua-platform": fingerprint.secChUaPlatform,
|
|
130
|
+
"accept-language": acceptLanguage,
|
|
135
131
|
"sec-fetch-dest": "document",
|
|
136
132
|
"sec-fetch-mode": "navigate",
|
|
137
133
|
"sec-fetch-site": "none",
|
|
138
134
|
"sec-fetch-user": "?1",
|
|
139
135
|
"upgrade-insecure-requests": "1",
|
|
140
|
-
"user-agent":
|
|
136
|
+
"user-agent": userAgent
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
function navigationHeaders() {
|
|
140
|
+
const fingerprint = browserFingerprint();
|
|
141
|
+
return {
|
|
142
|
+
...baseNavigationHeaders(fingerprint.userAgent, fingerprint.acceptLanguage),
|
|
143
|
+
"sec-ch-ua": fingerprint.secChUa,
|
|
144
|
+
"sec-ch-ua-mobile": "?0",
|
|
145
|
+
"sec-ch-ua-platform": fingerprint.secChUaPlatform
|
|
141
146
|
};
|
|
142
147
|
}
|
|
143
148
|
function browserUserAgent() {
|
|
@@ -148,6 +153,9 @@ function firefoxUserAgent() {
|
|
|
148
153
|
const version = pick(firefoxVersions);
|
|
149
154
|
return `Mozilla/5.0 (${platform.uaToken}; rv:${version}.0) Gecko/20100101 Firefox/${version}.0`;
|
|
150
155
|
}
|
|
156
|
+
function firefoxNavigationHeaders(userAgent = firefoxUserAgent()) {
|
|
157
|
+
return baseNavigationHeaders(userAgent, pick(acceptLanguages));
|
|
158
|
+
}
|
|
151
159
|
function instagramAppUserAgent() {
|
|
152
160
|
return pick(instagramAppUserAgents);
|
|
153
161
|
}
|
|
@@ -271,7 +279,21 @@ function embedHeaders() {
|
|
|
271
279
|
}
|
|
272
280
|
async function resolveInstagram(input) {
|
|
273
281
|
const code = shortcode(input.url);
|
|
274
|
-
const
|
|
282
|
+
const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
|
|
283
|
+
const resolvers = [
|
|
284
|
+
() => pageMedia(input.net, code, input.preferredWidth),
|
|
285
|
+
() => mobileMedia(input.net, code, input.preferredWidth),
|
|
286
|
+
() => embedMedia(input.net, code),
|
|
287
|
+
() => graphqlMedia(input.net, code)
|
|
288
|
+
];
|
|
289
|
+
let media = null;
|
|
290
|
+
for (const resolve of resolvers) {
|
|
291
|
+
const candidate = await resolve();
|
|
292
|
+
if (candidate && (!videoRequired || mediaItems(candidate, code, input.preferredWidth).some((item) => item.kind === "video"))) {
|
|
293
|
+
media = candidate;
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
275
297
|
if (!media) {
|
|
276
298
|
throw await instagramUnavailable(input.net, code);
|
|
277
299
|
}
|
|
@@ -481,23 +503,20 @@ async function graphqlMedia(net, code) {
|
|
|
481
503
|
}
|
|
482
504
|
const body = new URLSearchParams({
|
|
483
505
|
...params.body,
|
|
484
|
-
|
|
506
|
+
av: "0",
|
|
507
|
+
doc_id: "26130443479876713",
|
|
485
508
|
fb_api_caller_class: "RelayModern",
|
|
486
|
-
fb_api_req_friendly_name: "
|
|
509
|
+
fb_api_req_friendly_name: "PolarisPostRootQuery",
|
|
487
510
|
server_timestamps: "true",
|
|
488
|
-
variables: JSON.stringify({
|
|
489
|
-
fetch_tagged_user_count: null,
|
|
490
|
-
hoisted_comment_id: null,
|
|
491
|
-
hoisted_reply_id: null,
|
|
492
|
-
shortcode: code
|
|
493
|
-
})
|
|
511
|
+
variables: JSON.stringify({ shortcode: code })
|
|
494
512
|
});
|
|
495
513
|
const response = await net("https://www.instagram.com/graphql/query", {
|
|
496
514
|
body,
|
|
497
515
|
headers: {
|
|
498
516
|
...embedHeaders(),
|
|
499
517
|
...params.headers,
|
|
500
|
-
"X-FB-Friendly-Name": "
|
|
518
|
+
"X-FB-Friendly-Name": "PolarisPostRootQuery",
|
|
519
|
+
"X-Requested-With": "XMLHttpRequest",
|
|
501
520
|
"content-type": "application/x-www-form-urlencoded"
|
|
502
521
|
},
|
|
503
522
|
method: "POST"
|
|
@@ -523,14 +542,14 @@ async function graphqlParams(net, code) {
|
|
|
523
542
|
const push = entryObject("InstagramWebPushInfo", html);
|
|
524
543
|
const lsd = entryObject("LSD", html)?.token ?? randomToken();
|
|
525
544
|
const csrf = entryObject("InstagramSecurityConfig", html)?.csrf_token;
|
|
526
|
-
const cookie =
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
polaris?.
|
|
530
|
-
"
|
|
531
|
-
|
|
532
|
-
"
|
|
533
|
-
|
|
545
|
+
const cookie = instagramCookies(response.headers, {
|
|
546
|
+
csrftoken: string(csrf),
|
|
547
|
+
dpr: "2",
|
|
548
|
+
ig_did: string(polaris?.device_id),
|
|
549
|
+
ig_nrcb: "1",
|
|
550
|
+
mid: string(polaris?.machine_id),
|
|
551
|
+
wd: "1280x720"
|
|
552
|
+
});
|
|
534
553
|
return {
|
|
535
554
|
headers: {
|
|
536
555
|
"X-CSRFToken": string(csrf) ?? "",
|
|
@@ -564,6 +583,12 @@ async function graphqlParams(net, code) {
|
|
|
564
583
|
};
|
|
565
584
|
}
|
|
566
585
|
function gqlShortcodeMedia(data) {
|
|
586
|
+
const webInfo = object(data.xdt_api__v1__media__shortcode__web_info) ? data.xdt_api__v1__media__shortcode__web_info : null;
|
|
587
|
+
const webItems = webInfo && Array.isArray(webInfo.items) ? webInfo.items : [];
|
|
588
|
+
const webMedia = webItems[0];
|
|
589
|
+
if (object(webMedia)) {
|
|
590
|
+
return webMedia;
|
|
591
|
+
}
|
|
567
592
|
const media = data.gql_data && object(data.gql_data) ? data.gql_data.shortcode_media ?? data.gql_data.xdt_shortcode_media : data.shortcode_media ?? data.xdt_shortcode_media;
|
|
568
593
|
return object(media) ? media : null;
|
|
569
594
|
}
|
|
@@ -583,6 +608,24 @@ function mediaItems(media, code, preferredWidth) {
|
|
|
583
608
|
}
|
|
584
609
|
return instagramItem(media, code, null, preferredWidth);
|
|
585
610
|
}
|
|
611
|
+
function instagramCookies(headers, fallback) {
|
|
612
|
+
const getter = headers.getSetCookie;
|
|
613
|
+
const raw = typeof getter === "function" ? getter.call(headers) : headers.get("set-cookie")?.split(/,(?=[^;]+?=)/) ?? [];
|
|
614
|
+
const cookies = new Map;
|
|
615
|
+
for (const value of raw) {
|
|
616
|
+
const pair = value.split(";", 1)[0]?.trim();
|
|
617
|
+
const separator = pair?.indexOf("=") ?? -1;
|
|
618
|
+
if (pair && separator > 0) {
|
|
619
|
+
cookies.set(pair.slice(0, separator), pair.slice(separator + 1));
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
for (const [name, value] of Object.entries(fallback)) {
|
|
623
|
+
if (value && !cookies.has(name)) {
|
|
624
|
+
cookies.set(name, value);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
return [...cookies].map(([name, value]) => `${name}=${value}`).join("; ");
|
|
628
|
+
}
|
|
586
629
|
function instagramItem(media, code, index, preferredWidth) {
|
|
587
630
|
const video = selectVersion(media.video_versions, preferredWidth) ?? string(media.video_url);
|
|
588
631
|
const suffix = index === null ? "" : `_${index}`;
|
|
@@ -646,6 +689,208 @@ function randomToken(length = 8) {
|
|
|
646
689
|
return crypto.getRandomValues(new Uint8Array(length)).reduce((value, byte) => value + (byte % 36).toString(36), "");
|
|
647
690
|
}
|
|
648
691
|
|
|
692
|
+
// src/linkedin.ts
|
|
693
|
+
async function resolveLinkedin(input) {
|
|
694
|
+
const reference = postReference(input.url);
|
|
695
|
+
const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
|
|
696
|
+
const response = await input.net(pageUrl, { headers: navigationHeaders() });
|
|
697
|
+
if (response.status === 404) {
|
|
698
|
+
throw new PostfetchError(404, "LinkedIn post not found", "notFound");
|
|
699
|
+
}
|
|
700
|
+
if (response.status === 429 || response.status === 999) {
|
|
701
|
+
throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
|
|
702
|
+
}
|
|
703
|
+
if (!response.ok) {
|
|
704
|
+
throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
|
|
705
|
+
}
|
|
706
|
+
if (response.url.includes("/signup/") || response.url.includes("/login")) {
|
|
707
|
+
throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
|
|
708
|
+
}
|
|
709
|
+
const html = await response.text();
|
|
710
|
+
const post = linkedinPost(html);
|
|
711
|
+
if (!post) {
|
|
712
|
+
throw new PostfetchError(404, "LinkedIn post unavailable", "unavailable");
|
|
713
|
+
}
|
|
714
|
+
const canonical = string(post["@id"]);
|
|
715
|
+
const canonicalReference = canonical ? postReferenceOrNull(canonical) : null;
|
|
716
|
+
const id = canonicalReference?.id ?? reference.id;
|
|
717
|
+
const items = linkedinItems(post, html, id, pageUrl);
|
|
718
|
+
if (items.length === 0) {
|
|
719
|
+
throw new Error("LinkedIn media url not found");
|
|
720
|
+
}
|
|
721
|
+
return {
|
|
722
|
+
archiveFilename: filename(`linkedin_${id}.zip`),
|
|
723
|
+
id,
|
|
724
|
+
items,
|
|
725
|
+
metadata: linkedinMetadata(post),
|
|
726
|
+
platform: "linkedin"
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
function linkedinMetadata(post) {
|
|
730
|
+
const author = linkedinAuthor(post);
|
|
731
|
+
return {
|
|
732
|
+
title: string(post.text) ?? string(post.name) ?? string(post.headline) ?? undefined,
|
|
733
|
+
text: string(post.description) ?? string(post.articleBody) ?? string(post.text) ?? undefined,
|
|
734
|
+
author: author ? {
|
|
735
|
+
handle: linkedinHandle(string(author.url)) ?? undefined,
|
|
736
|
+
name: string(author.name) ?? undefined
|
|
737
|
+
} : undefined,
|
|
738
|
+
createdAt: isoFromDateString(post.datePublished ?? post.uploadDate),
|
|
739
|
+
likeCount: interactionCount(post.interactionStatistic, "LikeAction"),
|
|
740
|
+
commentCount: count(post.commentCount) ?? interactionCount(post.interactionStatistic, "CommentAction")
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
function postReference(input) {
|
|
744
|
+
const reference = postReferenceOrNull(input);
|
|
745
|
+
if (!reference) {
|
|
746
|
+
throw new PostfetchError(400, "LinkedIn post id not found");
|
|
747
|
+
}
|
|
748
|
+
return reference;
|
|
749
|
+
}
|
|
750
|
+
function postReferenceOrNull(input) {
|
|
751
|
+
let pathname;
|
|
752
|
+
try {
|
|
753
|
+
pathname = new URL(input).pathname.replace(/%3A/gi, ":");
|
|
754
|
+
} catch {
|
|
755
|
+
return null;
|
|
756
|
+
}
|
|
757
|
+
const urn = pathname.match(/\/(?:embed\/)?feed\/update\/urn:li:(activity|share|ugcpost):(\d+)\/?$/i);
|
|
758
|
+
const post = pathname.match(/\/posts\/[^/]*-(activity|share|ugcpost)-(\d+)(?:-[^/]*)?\/?$/i);
|
|
759
|
+
const match = urn ?? post;
|
|
760
|
+
if (!match) {
|
|
761
|
+
return null;
|
|
762
|
+
}
|
|
763
|
+
switch (match[1].toLowerCase()) {
|
|
764
|
+
case "activity":
|
|
765
|
+
return { id: match[2], kind: "activity" };
|
|
766
|
+
case "share":
|
|
767
|
+
return { id: match[2], kind: "share" };
|
|
768
|
+
case "ugcpost":
|
|
769
|
+
return { id: match[2], kind: "ugcPost" };
|
|
770
|
+
default:
|
|
771
|
+
return null;
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
function linkedinAuthor(post) {
|
|
775
|
+
if (object(post.creator)) {
|
|
776
|
+
return post.creator;
|
|
777
|
+
}
|
|
778
|
+
if (object(post.author)) {
|
|
779
|
+
return post.author;
|
|
780
|
+
}
|
|
781
|
+
return null;
|
|
782
|
+
}
|
|
783
|
+
function linkedinPost(html) {
|
|
784
|
+
for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
|
|
785
|
+
if (!/\btype=["']application\/ld\+json["']/i.test(match[1])) {
|
|
786
|
+
continue;
|
|
787
|
+
}
|
|
788
|
+
const parsed = parseJson(match[2]);
|
|
789
|
+
for (const entry of jsonLdEntries(parsed)) {
|
|
790
|
+
if (hasSchemaType(entry, "VideoObject") || hasSchemaType(entry, "SocialMediaPosting")) {
|
|
791
|
+
return entry;
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
return null;
|
|
796
|
+
}
|
|
797
|
+
function parseJson(value) {
|
|
798
|
+
try {
|
|
799
|
+
return JSON.parse(value);
|
|
800
|
+
} catch {
|
|
801
|
+
try {
|
|
802
|
+
return JSON.parse(decodeHtml(value));
|
|
803
|
+
} catch {
|
|
804
|
+
return null;
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
function jsonLdEntries(value) {
|
|
809
|
+
if (Array.isArray(value)) {
|
|
810
|
+
return value.filter(object);
|
|
811
|
+
}
|
|
812
|
+
if (!object(value)) {
|
|
813
|
+
return [];
|
|
814
|
+
}
|
|
815
|
+
return Array.isArray(value["@graph"]) ? value["@graph"].filter(object) : [value];
|
|
816
|
+
}
|
|
817
|
+
function hasSchemaType(value, expected) {
|
|
818
|
+
const type = value["@type"];
|
|
819
|
+
return type === expected || Array.isArray(type) && type.includes(expected);
|
|
820
|
+
}
|
|
821
|
+
function linkedinItems(post, html, id, referer) {
|
|
822
|
+
const headers = { referer, "user-agent": browserUserAgent() };
|
|
823
|
+
if (hasSchemaType(post, "VideoObject")) {
|
|
824
|
+
const url2 = bestVideoUrl(html) ?? string(post.contentUrl);
|
|
825
|
+
return url2 ? [{ filename: filename(`linkedin_${id}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "linkedin", url: url2 }] : [];
|
|
826
|
+
}
|
|
827
|
+
const url = imageUrl(post.image);
|
|
828
|
+
return url ? [{ filename: filename(`linkedin_${id}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "linkedin", url }] : [];
|
|
829
|
+
}
|
|
830
|
+
function bestVideoUrl(html) {
|
|
831
|
+
const encoded = html.match(/<video\b[^>]*\bdata-sources=(["'])(.*?)\1[^>]*>/is)?.[2];
|
|
832
|
+
if (!encoded) {
|
|
833
|
+
return null;
|
|
834
|
+
}
|
|
835
|
+
let parsed;
|
|
836
|
+
try {
|
|
837
|
+
parsed = JSON.parse(decodeHtml(encoded));
|
|
838
|
+
} catch {
|
|
839
|
+
return null;
|
|
840
|
+
}
|
|
841
|
+
const sources = Array.isArray(parsed) ? parsed.filter(object) : [];
|
|
842
|
+
const videos = sources.filter((source2) => string(source2.type) === "video/mp4" && string(source2.src));
|
|
843
|
+
const selected = videos.reduce((best, candidate) => {
|
|
844
|
+
const bitrate = count(candidate["data-bitrate"]) ?? 0;
|
|
845
|
+
const bestBitrate = best ? count(best["data-bitrate"]) ?? 0 : -1;
|
|
846
|
+
return bitrate > bestBitrate ? candidate : best;
|
|
847
|
+
}, null);
|
|
848
|
+
return selected ? string(selected.src) : null;
|
|
849
|
+
}
|
|
850
|
+
function imageUrl(value) {
|
|
851
|
+
if (typeof value === "string") {
|
|
852
|
+
return string(value);
|
|
853
|
+
}
|
|
854
|
+
if (Array.isArray(value)) {
|
|
855
|
+
return value.map(imageUrl).find((url) => Boolean(url)) ?? null;
|
|
856
|
+
}
|
|
857
|
+
return object(value) ? string(value.url) : null;
|
|
858
|
+
}
|
|
859
|
+
function interactionCount(value, action) {
|
|
860
|
+
const statistics = Array.isArray(value) ? value.filter(object) : object(value) ? [value] : [];
|
|
861
|
+
const statistic = statistics.find((entry) => string(entry.interactionType)?.endsWith(`/${action}`));
|
|
862
|
+
return statistic ? count(statistic.userInteractionCount) : undefined;
|
|
863
|
+
}
|
|
864
|
+
function linkedinHandle(value) {
|
|
865
|
+
if (!value) {
|
|
866
|
+
return null;
|
|
867
|
+
}
|
|
868
|
+
try {
|
|
869
|
+
return new URL(value).pathname.match(/^\/(?:company|in|showcase)\/([^/]+)/)?.[1] ?? null;
|
|
870
|
+
} catch {
|
|
871
|
+
return null;
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
function decodeHtml(value) {
|
|
875
|
+
return value.replace(/&(quot|amp|lt|gt|#39|#x27|#(\d+)|#x([0-9a-f]+));/gi, (_entity, name, decimal, hex) => {
|
|
876
|
+
switch (name.toLowerCase()) {
|
|
877
|
+
case "quot":
|
|
878
|
+
return '"';
|
|
879
|
+
case "amp":
|
|
880
|
+
return "&";
|
|
881
|
+
case "lt":
|
|
882
|
+
return "<";
|
|
883
|
+
case "gt":
|
|
884
|
+
return ">";
|
|
885
|
+
case "#39":
|
|
886
|
+
case "#x27":
|
|
887
|
+
return "'";
|
|
888
|
+
default:
|
|
889
|
+
return String.fromCodePoint(decimal ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
|
|
890
|
+
}
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
|
|
649
894
|
// src/isobmff.ts
|
|
650
895
|
function parseBoxes(data, start = 0, end = data.length) {
|
|
651
896
|
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
@@ -1407,23 +1652,18 @@ function trackId(track) {
|
|
|
1407
1652
|
|
|
1408
1653
|
// src/tiktok.ts
|
|
1409
1654
|
var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
|
|
1655
|
+
var pageAttemptCount = 12;
|
|
1410
1656
|
async function resolveTiktok(input) {
|
|
1411
|
-
const
|
|
1412
|
-
const pageUrl = await followShortlink(input.net, userAgent, input.url);
|
|
1657
|
+
const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
|
|
1413
1658
|
const id = videoId(pageUrl);
|
|
1414
1659
|
if (!id) {
|
|
1415
1660
|
throw new Error("TikTok video id not found");
|
|
1416
1661
|
}
|
|
1417
|
-
const page = await
|
|
1418
|
-
if (!recoverablePageError(error)) {
|
|
1419
|
-
throw error;
|
|
1420
|
-
}
|
|
1421
|
-
return fetchVideoPage(input.net, id, firefoxUserAgent());
|
|
1422
|
-
});
|
|
1662
|
+
const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
|
|
1423
1663
|
const user = author(page.item) ?? username(pageUrl) ?? "i";
|
|
1424
1664
|
const headers = {
|
|
1425
1665
|
referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
|
|
1426
|
-
"user-agent": userAgent
|
|
1666
|
+
"user-agent": page.userAgent
|
|
1427
1667
|
};
|
|
1428
1668
|
if (page.cookie) {
|
|
1429
1669
|
headers.cookie = page.cookie;
|
|
@@ -1434,6 +1674,35 @@ async function resolveTiktok(input) {
|
|
|
1434
1674
|
}
|
|
1435
1675
|
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
1436
1676
|
}
|
|
1677
|
+
async function videoPage(net, id, photo) {
|
|
1678
|
+
for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
|
|
1679
|
+
const baseHeaders2 = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
|
|
1680
|
+
const userAgent2 = baseHeaders2["user-agent"];
|
|
1681
|
+
const headers2 = {
|
|
1682
|
+
...baseHeaders2,
|
|
1683
|
+
referer: "https://www.tiktok.com/",
|
|
1684
|
+
"sec-fetch-site": "same-origin"
|
|
1685
|
+
};
|
|
1686
|
+
try {
|
|
1687
|
+
return { ...await fetchVideoPage(net, id, headers2), userAgent: userAgent2 };
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
if (!recoverablePageError(error)) {
|
|
1690
|
+
throw error;
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
}
|
|
1694
|
+
if (!photo) {
|
|
1695
|
+
throw new Error("TikTok hydration not found");
|
|
1696
|
+
}
|
|
1697
|
+
const baseHeaders = navigationHeaders();
|
|
1698
|
+
const userAgent = baseHeaders["user-agent"];
|
|
1699
|
+
const headers = {
|
|
1700
|
+
...baseHeaders,
|
|
1701
|
+
referer: "https://www.tiktok.com/",
|
|
1702
|
+
"sec-fetch-site": "same-origin"
|
|
1703
|
+
};
|
|
1704
|
+
return { ...await fetchEmbedPage(net, id, headers), userAgent };
|
|
1705
|
+
}
|
|
1437
1706
|
function tiktokMetadata(item) {
|
|
1438
1707
|
const user = object(item.author) ? item.author : null;
|
|
1439
1708
|
const stats = object(item.stats) ? item.stats : null;
|
|
@@ -1460,16 +1729,18 @@ function tiktokMetadata(item) {
|
|
|
1460
1729
|
}
|
|
1461
1730
|
};
|
|
1462
1731
|
}
|
|
1463
|
-
async function fetchVideoPage(net, id,
|
|
1732
|
+
async function fetchVideoPage(net, id, headers) {
|
|
1464
1733
|
const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
|
|
1465
|
-
headers
|
|
1466
|
-
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
1467
|
-
referer: "https://www.tiktok.com/",
|
|
1468
|
-
"user-agent": userAgent
|
|
1469
|
-
}
|
|
1734
|
+
headers
|
|
1470
1735
|
});
|
|
1471
1736
|
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
1472
1737
|
}
|
|
1738
|
+
async function fetchEmbedPage(net, id, headers) {
|
|
1739
|
+
const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
|
|
1740
|
+
headers
|
|
1741
|
+
});
|
|
1742
|
+
return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
|
|
1743
|
+
}
|
|
1473
1744
|
function isShortlinkHost(hostname) {
|
|
1474
1745
|
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
1475
1746
|
}
|
|
@@ -1509,10 +1780,68 @@ function itemStruct(html) {
|
|
|
1509
1780
|
}
|
|
1510
1781
|
return item;
|
|
1511
1782
|
}
|
|
1783
|
+
function embedItemStruct(html, id) {
|
|
1784
|
+
const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
1785
|
+
if (!raw) {
|
|
1786
|
+
throw new Error("TikTok embed state not found");
|
|
1787
|
+
}
|
|
1788
|
+
const parsed = JSON.parse(raw);
|
|
1789
|
+
const source2 = object(parsed) && object(parsed.source) ? parsed.source : null;
|
|
1790
|
+
const data = source2 && object(source2.data) ? source2.data : null;
|
|
1791
|
+
const rawRoute = data ? data[`/embed/v2/${id}`] : null;
|
|
1792
|
+
const route = object(rawRoute) ? rawRoute : null;
|
|
1793
|
+
const videoData = route && object(route.videoData) ? route.videoData : null;
|
|
1794
|
+
const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
|
|
1795
|
+
if (!videoData || !item) {
|
|
1796
|
+
throw new Error("TikTok embed item not found");
|
|
1797
|
+
}
|
|
1798
|
+
const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
|
|
1799
|
+
const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
|
|
1800
|
+
const video = object(item.video) ? item.video : {};
|
|
1801
|
+
const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
|
|
1802
|
+
const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
|
|
1803
|
+
const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
|
|
1804
|
+
if (displayImages.length === 0) {
|
|
1805
|
+
throw new Error("TikTok video hydration not found");
|
|
1806
|
+
}
|
|
1807
|
+
return {
|
|
1808
|
+
...item,
|
|
1809
|
+
author: {
|
|
1810
|
+
nickname: string(authorInfo.nickName),
|
|
1811
|
+
uniqueId: string(authorInfo.uniqueId),
|
|
1812
|
+
verified: bool(authorInfo.verified)
|
|
1813
|
+
},
|
|
1814
|
+
desc: string(item.text),
|
|
1815
|
+
imagePost: {
|
|
1816
|
+
images: displayImages.map((image) => ({
|
|
1817
|
+
imageURL: {
|
|
1818
|
+
urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
|
|
1819
|
+
}
|
|
1820
|
+
}))
|
|
1821
|
+
},
|
|
1822
|
+
music: {
|
|
1823
|
+
authorName: string(musicInfo.authorName),
|
|
1824
|
+
playUrl: firstString(musicInfo.playUrl),
|
|
1825
|
+
title: string(musicInfo.musicName)
|
|
1826
|
+
},
|
|
1827
|
+
stats: {
|
|
1828
|
+
commentCount: item.commentCount,
|
|
1829
|
+
diggCount: item.diggCount,
|
|
1830
|
+
playCount: item.playCount,
|
|
1831
|
+
shareCount: item.shareCount
|
|
1832
|
+
},
|
|
1833
|
+
video: {
|
|
1834
|
+
duration: videoMeta.duration
|
|
1835
|
+
}
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1512
1838
|
function recoverablePageError(error) {
|
|
1513
1839
|
const message = error instanceof Error ? error.message : "";
|
|
1514
1840
|
return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
|
|
1515
1841
|
}
|
|
1842
|
+
function firstString(value) {
|
|
1843
|
+
return Array.isArray(value) ? value.map(string).find((candidate) => candidate !== null) ?? null : string(value);
|
|
1844
|
+
}
|
|
1516
1845
|
function downloadUrl(item) {
|
|
1517
1846
|
const video = object(item.video) ? item.video : null;
|
|
1518
1847
|
return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
|
|
@@ -1538,8 +1867,8 @@ function imageItems3(item, user, id, headers) {
|
|
|
1538
1867
|
const imagePost = object(item.imagePost) ? item.imagePost : null;
|
|
1539
1868
|
const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
|
|
1540
1869
|
return images.flatMap((image, index) => {
|
|
1541
|
-
const
|
|
1542
|
-
const list =
|
|
1870
|
+
const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
|
|
1871
|
+
const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
|
|
1543
1872
|
const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
|
|
1544
1873
|
return url ? [{
|
|
1545
1874
|
filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
|
|
@@ -1896,6 +2225,8 @@ async function postfetch(url, options = {}) {
|
|
|
1896
2225
|
return resolveFacebook(context);
|
|
1897
2226
|
case "instagram":
|
|
1898
2227
|
return resolveInstagram(context);
|
|
2228
|
+
case "linkedin":
|
|
2229
|
+
return resolveLinkedin(context);
|
|
1899
2230
|
case "pinterest":
|
|
1900
2231
|
return resolvePinterest(context);
|
|
1901
2232
|
case "reddit":
|
|
@@ -1918,6 +2249,9 @@ function detect(url) {
|
|
|
1918
2249
|
if (host.includes("instagram.com")) {
|
|
1919
2250
|
return "instagram";
|
|
1920
2251
|
}
|
|
2252
|
+
if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
|
|
2253
|
+
return "linkedin";
|
|
2254
|
+
}
|
|
1921
2255
|
if (host.includes("pinterest.") || host === "pin.it") {
|
|
1922
2256
|
return "pinterest";
|
|
1923
2257
|
}
|
|
@@ -1936,7 +2270,7 @@ function detect(url) {
|
|
|
1936
2270
|
if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
|
|
1937
2271
|
return "twitter";
|
|
1938
2272
|
}
|
|
1939
|
-
throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
2273
|
+
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
1940
2274
|
}
|
|
1941
2275
|
// src/remux.ts
|
|
1942
2276
|
function mergeAudioVideo(video, audio) {
|
package/dist/internal.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** A platform that {@link postfetch} can resolve. */
|
|
2
|
-
export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
2
|
+
export type Platform = "facebook" | "instagram" | "linkedin" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
3
3
|
/** The kind of a {@link MediaItem}. */
|
|
4
4
|
export type MediaKind = "audio" | "image" | "video";
|
|
5
5
|
/** A single resolved media file: a direct URL plus everything needed to fetch and name it. */
|
|
@@ -137,7 +137,7 @@ export type PinterestExtra = {
|
|
|
137
137
|
outboundLink?: string;
|
|
138
138
|
};
|
|
139
139
|
/** The result of resolving a post: its platform, id, media items and metadata. */
|
|
140
|
-
export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
|
|
140
|
+
export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"linkedin", never> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
|
|
141
141
|
/** One platform's variant of a {@link PostfetchResult}, carrying that platform's `extra` metadata shape. */
|
|
142
142
|
export type PlatformResult<P extends Platform, Extra> = {
|
|
143
143
|
/** Suggested filename when zipping every item. */
|
package/dist/postfetch.d.ts
CHANGED
|
@@ -14,7 +14,7 @@ export type PostfetchOptions = {
|
|
|
14
14
|
* performs no side effects beyond the lookup; use {@link download}, {@link archive}
|
|
15
15
|
* or {@link toResponse} to turn the result into bytes or a `Response`.
|
|
16
16
|
*
|
|
17
|
-
* @param url A post URL from
|
|
17
|
+
* @param url A post URL from a supported platform.
|
|
18
18
|
* @param options See {@link PostfetchOptions}.
|
|
19
19
|
* @returns The resolved post and its media items.
|
|
20
20
|
* @throws {PostfetchError} If the URL is empty or its host is unsupported.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Zero-dependency typed core that turns Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
|
|
3
|
+
"version": "0.6.0",
|
|
4
|
+
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"exports": {
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"downloader",
|
|
31
31
|
"facebook",
|
|
32
32
|
"instagram",
|
|
33
|
+
"linkedin",
|
|
33
34
|
"media",
|
|
34
35
|
"pinterest",
|
|
35
36
|
"reddit",
|
package/src/fingerprint.ts
CHANGED
|
@@ -32,6 +32,10 @@ export type BrowserFingerprint = {
|
|
|
32
32
|
userAgent: string;
|
|
33
33
|
};
|
|
34
34
|
|
|
35
|
+
export type NavigationHeaders = Record<string, string> & {
|
|
36
|
+
"user-agent": string;
|
|
37
|
+
};
|
|
38
|
+
|
|
35
39
|
export function browserFingerprint(): BrowserFingerprint {
|
|
36
40
|
const version = pick(chromeVersions);
|
|
37
41
|
const platform = pick(desktopPlatforms);
|
|
@@ -43,20 +47,26 @@ export function browserFingerprint(): BrowserFingerprint {
|
|
|
43
47
|
};
|
|
44
48
|
}
|
|
45
49
|
|
|
46
|
-
|
|
47
|
-
const fingerprint = browserFingerprint();
|
|
50
|
+
function baseNavigationHeaders(userAgent: string, acceptLanguage: string): NavigationHeaders {
|
|
48
51
|
return {
|
|
49
52
|
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
50
|
-
"accept-language":
|
|
51
|
-
"sec-ch-ua": fingerprint.secChUa,
|
|
52
|
-
"sec-ch-ua-mobile": "?0",
|
|
53
|
-
"sec-ch-ua-platform": fingerprint.secChUaPlatform,
|
|
53
|
+
"accept-language": acceptLanguage,
|
|
54
54
|
"sec-fetch-dest": "document",
|
|
55
55
|
"sec-fetch-mode": "navigate",
|
|
56
56
|
"sec-fetch-site": "none",
|
|
57
57
|
"sec-fetch-user": "?1",
|
|
58
58
|
"upgrade-insecure-requests": "1",
|
|
59
|
-
"user-agent":
|
|
59
|
+
"user-agent": userAgent,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function navigationHeaders(): NavigationHeaders {
|
|
64
|
+
const fingerprint = browserFingerprint();
|
|
65
|
+
return {
|
|
66
|
+
...baseNavigationHeaders(fingerprint.userAgent, fingerprint.acceptLanguage),
|
|
67
|
+
"sec-ch-ua": fingerprint.secChUa,
|
|
68
|
+
"sec-ch-ua-mobile": "?0",
|
|
69
|
+
"sec-ch-ua-platform": fingerprint.secChUaPlatform,
|
|
60
70
|
};
|
|
61
71
|
}
|
|
62
72
|
|
|
@@ -70,6 +80,10 @@ export function firefoxUserAgent(): string {
|
|
|
70
80
|
return `Mozilla/5.0 (${platform.uaToken}; rv:${version}.0) Gecko/20100101 Firefox/${version}.0`;
|
|
71
81
|
}
|
|
72
82
|
|
|
83
|
+
export function firefoxNavigationHeaders(userAgent = firefoxUserAgent()): NavigationHeaders {
|
|
84
|
+
return baseNavigationHeaders(userAgent, pick(acceptLanguages));
|
|
85
|
+
}
|
|
86
|
+
|
|
73
87
|
export function instagramAppUserAgent(): string {
|
|
74
88
|
return pick(instagramAppUserAgents);
|
|
75
89
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Turn a public social post URL into its media.
|
|
3
3
|
*
|
|
4
|
-
* {@link postfetch} resolves Instagram, TikTok, YouTube, Facebook and X (Twitter)
|
|
4
|
+
* {@link postfetch} resolves Instagram, LinkedIn, TikTok, YouTube, Facebook and X (Twitter)
|
|
5
5
|
* URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
|
|
6
6
|
* (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
|
|
7
7
|
* Cloudflare Workers and browsers alike. It only resolves the media; the
|
package/src/instagram.ts
CHANGED
|
@@ -58,11 +58,21 @@ function embedHeaders(): Record<string, string> {
|
|
|
58
58
|
|
|
59
59
|
export async function resolveInstagram(input: ResolveContext): Promise<PostfetchResult> {
|
|
60
60
|
const code = shortcode(input.url);
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
(
|
|
64
|
-
(
|
|
65
|
-
(
|
|
61
|
+
const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
|
|
62
|
+
const resolvers: Array<() => Promise<Json | null>> = [
|
|
63
|
+
() => pageMedia(input.net, code, input.preferredWidth),
|
|
64
|
+
() => mobileMedia(input.net, code, input.preferredWidth),
|
|
65
|
+
() => embedMedia(input.net, code),
|
|
66
|
+
() => graphqlMedia(input.net, code),
|
|
67
|
+
];
|
|
68
|
+
let media: Json | null = null;
|
|
69
|
+
for (const resolve of resolvers) {
|
|
70
|
+
const candidate = await resolve();
|
|
71
|
+
if (candidate && (!videoRequired || mediaItems(candidate, code, input.preferredWidth).some((item) => item.kind === "video"))) {
|
|
72
|
+
media = candidate;
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
66
76
|
if (!media) {
|
|
67
77
|
throw await instagramUnavailable(input.net, code);
|
|
68
78
|
}
|
|
@@ -301,23 +311,20 @@ async function graphqlMedia(net: Net, code: string): Promise<Json | null> {
|
|
|
301
311
|
}
|
|
302
312
|
const body = new URLSearchParams({
|
|
303
313
|
...params.body,
|
|
304
|
-
|
|
314
|
+
av: "0",
|
|
315
|
+
doc_id: "26130443479876713",
|
|
305
316
|
fb_api_caller_class: "RelayModern",
|
|
306
|
-
fb_api_req_friendly_name: "
|
|
317
|
+
fb_api_req_friendly_name: "PolarisPostRootQuery",
|
|
307
318
|
server_timestamps: "true",
|
|
308
|
-
variables: JSON.stringify({
|
|
309
|
-
fetch_tagged_user_count: null,
|
|
310
|
-
hoisted_comment_id: null,
|
|
311
|
-
hoisted_reply_id: null,
|
|
312
|
-
shortcode: code,
|
|
313
|
-
}),
|
|
319
|
+
variables: JSON.stringify({ shortcode: code }),
|
|
314
320
|
});
|
|
315
321
|
const response = await net("https://www.instagram.com/graphql/query", {
|
|
316
322
|
body,
|
|
317
323
|
headers: {
|
|
318
324
|
...embedHeaders(),
|
|
319
325
|
...params.headers,
|
|
320
|
-
"X-FB-Friendly-Name": "
|
|
326
|
+
"X-FB-Friendly-Name": "PolarisPostRootQuery",
|
|
327
|
+
"X-Requested-With": "XMLHttpRequest",
|
|
321
328
|
"content-type": "application/x-www-form-urlencoded",
|
|
322
329
|
},
|
|
323
330
|
method: "POST",
|
|
@@ -344,14 +351,14 @@ async function graphqlParams(net: Net, code: string): Promise<{ body: Record<str
|
|
|
344
351
|
const push = entryObject("InstagramWebPushInfo", html);
|
|
345
352
|
const lsd = entryObject("LSD", html)?.token ?? randomToken();
|
|
346
353
|
const csrf = entryObject("InstagramSecurityConfig", html)?.csrf_token;
|
|
347
|
-
const cookie =
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
polaris?.
|
|
351
|
-
"
|
|
352
|
-
|
|
353
|
-
"
|
|
354
|
-
|
|
354
|
+
const cookie = instagramCookies(response.headers, {
|
|
355
|
+
csrftoken: string(csrf),
|
|
356
|
+
dpr: "2",
|
|
357
|
+
ig_did: string(polaris?.device_id),
|
|
358
|
+
ig_nrcb: "1",
|
|
359
|
+
mid: string(polaris?.machine_id),
|
|
360
|
+
wd: "1280x720",
|
|
361
|
+
});
|
|
355
362
|
return {
|
|
356
363
|
headers: {
|
|
357
364
|
"X-CSRFToken": string(csrf) ?? "",
|
|
@@ -386,6 +393,12 @@ async function graphqlParams(net: Net, code: string): Promise<{ body: Record<str
|
|
|
386
393
|
}
|
|
387
394
|
|
|
388
395
|
function gqlShortcodeMedia(data: Json): Json | null {
|
|
396
|
+
const webInfo = object(data.xdt_api__v1__media__shortcode__web_info) ? data.xdt_api__v1__media__shortcode__web_info : null;
|
|
397
|
+
const webItems = webInfo && Array.isArray(webInfo.items) ? webInfo.items : [];
|
|
398
|
+
const webMedia = webItems[0];
|
|
399
|
+
if (object(webMedia)) {
|
|
400
|
+
return webMedia;
|
|
401
|
+
}
|
|
389
402
|
const media = data.gql_data && object(data.gql_data)
|
|
390
403
|
? data.gql_data.shortcode_media ?? data.gql_data.xdt_shortcode_media
|
|
391
404
|
: data.shortcode_media ?? data.xdt_shortcode_media;
|
|
@@ -411,6 +424,25 @@ function mediaItems(media: Json, code: string, preferredWidth: number): MediaIte
|
|
|
411
424
|
return instagramItem(media, code, null, preferredWidth);
|
|
412
425
|
}
|
|
413
426
|
|
|
427
|
+
function instagramCookies(headers: Headers, fallback: Record<string, string | null>): string {
|
|
428
|
+
const getter = (headers as Headers & { getSetCookie?: () => string[] }).getSetCookie;
|
|
429
|
+
const raw = typeof getter === "function" ? getter.call(headers) : headers.get("set-cookie")?.split(/,(?=[^;]+?=)/) ?? [];
|
|
430
|
+
const cookies = new Map<string, string>();
|
|
431
|
+
for (const value of raw) {
|
|
432
|
+
const pair = value.split(";", 1)[0]?.trim();
|
|
433
|
+
const separator = pair?.indexOf("=") ?? -1;
|
|
434
|
+
if (pair && separator > 0) {
|
|
435
|
+
cookies.set(pair.slice(0, separator), pair.slice(separator + 1));
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
for (const [name, value] of Object.entries(fallback)) {
|
|
439
|
+
if (value && !cookies.has(name)) {
|
|
440
|
+
cookies.set(name, value);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
return [...cookies].map(([name, value]) => `${name}=${value}`).join("; ");
|
|
444
|
+
}
|
|
445
|
+
|
|
414
446
|
function instagramItem(media: Json, code: string, index: number | null, preferredWidth: number): MediaItem[] {
|
|
415
447
|
const video = selectVersion(media.video_versions, preferredWidth) ?? string(media.video_url);
|
|
416
448
|
const suffix = index === null ? "" : `_${index}`;
|
package/src/internal.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** A platform that {@link postfetch} can resolve. */
|
|
2
|
-
export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
2
|
+
export type Platform = "facebook" | "instagram" | "linkedin" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
3
3
|
|
|
4
4
|
/** The kind of a {@link MediaItem}. */
|
|
5
5
|
export type MediaKind = "audio" | "image" | "video";
|
|
@@ -149,6 +149,7 @@ export type PinterestExtra = {
|
|
|
149
149
|
export type PostfetchResult =
|
|
150
150
|
| PlatformResult<"facebook", never>
|
|
151
151
|
| PlatformResult<"instagram", InstagramExtra>
|
|
152
|
+
| PlatformResult<"linkedin", never>
|
|
152
153
|
| PlatformResult<"pinterest", PinterestExtra>
|
|
153
154
|
| PlatformResult<"reddit", RedditExtra>
|
|
154
155
|
| PlatformResult<"soundcloud", SoundcloudExtra>
|
package/src/linkedin.ts
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import {
|
|
2
|
+
count,
|
|
3
|
+
filename,
|
|
4
|
+
isoFromDateString,
|
|
5
|
+
object,
|
|
6
|
+
PostfetchError,
|
|
7
|
+
string,
|
|
8
|
+
type Json,
|
|
9
|
+
type MediaItem,
|
|
10
|
+
type PostMetadata,
|
|
11
|
+
type PostfetchResult,
|
|
12
|
+
type ResolveContext,
|
|
13
|
+
} from "./internal";
|
|
14
|
+
import { browserUserAgent, navigationHeaders } from "./fingerprint";
|
|
15
|
+
|
|
16
|
+
type LinkedinPostKind = "activity" | "share" | "ugcPost";
|
|
17
|
+
|
|
18
|
+
type LinkedinPostReference = {
|
|
19
|
+
id: string;
|
|
20
|
+
kind: LinkedinPostKind;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export async function resolveLinkedin(input: ResolveContext): Promise<PostfetchResult> {
|
|
24
|
+
const reference = postReference(input.url);
|
|
25
|
+
const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
|
|
26
|
+
const response = await input.net(pageUrl, { headers: navigationHeaders() });
|
|
27
|
+
if (response.status === 404) {
|
|
28
|
+
throw new PostfetchError(404, "LinkedIn post not found", "notFound");
|
|
29
|
+
}
|
|
30
|
+
if (response.status === 429 || response.status === 999) {
|
|
31
|
+
throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
|
|
32
|
+
}
|
|
33
|
+
if (!response.ok) {
|
|
34
|
+
throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
|
|
35
|
+
}
|
|
36
|
+
if (response.url.includes("/signup/") || response.url.includes("/login")) {
|
|
37
|
+
throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const html = await response.text();
|
|
41
|
+
const post = linkedinPost(html);
|
|
42
|
+
if (!post) {
|
|
43
|
+
throw new PostfetchError(404, "LinkedIn post unavailable", "unavailable");
|
|
44
|
+
}
|
|
45
|
+
const canonical = string(post["@id"]);
|
|
46
|
+
const canonicalReference = canonical ? postReferenceOrNull(canonical) : null;
|
|
47
|
+
const id = canonicalReference?.id ?? reference.id;
|
|
48
|
+
const items = linkedinItems(post, html, id, pageUrl);
|
|
49
|
+
if (items.length === 0) {
|
|
50
|
+
throw new Error("LinkedIn media url not found");
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
archiveFilename: filename(`linkedin_${id}.zip`),
|
|
54
|
+
id,
|
|
55
|
+
items,
|
|
56
|
+
metadata: linkedinMetadata(post),
|
|
57
|
+
platform: "linkedin",
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function linkedinMetadata(post: Json): PostMetadata {
|
|
62
|
+
const author = linkedinAuthor(post);
|
|
63
|
+
return {
|
|
64
|
+
title: string(post.text) ?? string(post.name) ?? string(post.headline) ?? undefined,
|
|
65
|
+
text: string(post.description) ?? string(post.articleBody) ?? string(post.text) ?? undefined,
|
|
66
|
+
author: author
|
|
67
|
+
? {
|
|
68
|
+
handle: linkedinHandle(string(author.url)) ?? undefined,
|
|
69
|
+
name: string(author.name) ?? undefined,
|
|
70
|
+
}
|
|
71
|
+
: undefined,
|
|
72
|
+
createdAt: isoFromDateString(post.datePublished ?? post.uploadDate),
|
|
73
|
+
likeCount: interactionCount(post.interactionStatistic, "LikeAction"),
|
|
74
|
+
commentCount: count(post.commentCount) ?? interactionCount(post.interactionStatistic, "CommentAction"),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function postReference(input: string): LinkedinPostReference {
|
|
79
|
+
const reference = postReferenceOrNull(input);
|
|
80
|
+
if (!reference) {
|
|
81
|
+
throw new PostfetchError(400, "LinkedIn post id not found");
|
|
82
|
+
}
|
|
83
|
+
return reference;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function postReferenceOrNull(input: string): LinkedinPostReference | null {
|
|
87
|
+
let pathname: string;
|
|
88
|
+
try {
|
|
89
|
+
pathname = new URL(input).pathname.replace(/%3A/gi, ":");
|
|
90
|
+
} catch {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
const urn = pathname.match(/\/(?:embed\/)?feed\/update\/urn:li:(activity|share|ugcpost):(\d+)\/?$/i);
|
|
94
|
+
const post = pathname.match(/\/posts\/[^/]*-(activity|share|ugcpost)-(\d+)(?:-[^/]*)?\/?$/i);
|
|
95
|
+
const match = urn ?? post;
|
|
96
|
+
if (!match) {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
switch (match[1].toLowerCase()) {
|
|
100
|
+
case "activity":
|
|
101
|
+
return { id: match[2], kind: "activity" };
|
|
102
|
+
case "share":
|
|
103
|
+
return { id: match[2], kind: "share" };
|
|
104
|
+
case "ugcpost":
|
|
105
|
+
return { id: match[2], kind: "ugcPost" };
|
|
106
|
+
default:
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function linkedinAuthor(post: Json): Json | null {
|
|
112
|
+
if (object(post.creator)) {
|
|
113
|
+
return post.creator;
|
|
114
|
+
}
|
|
115
|
+
if (object(post.author)) {
|
|
116
|
+
return post.author;
|
|
117
|
+
}
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function linkedinPost(html: string): Json | null {
|
|
122
|
+
for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
|
|
123
|
+
if (!/\btype=["']application\/ld\+json["']/i.test(match[1])) {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
const parsed = parseJson(match[2]);
|
|
127
|
+
for (const entry of jsonLdEntries(parsed)) {
|
|
128
|
+
if (hasSchemaType(entry, "VideoObject") || hasSchemaType(entry, "SocialMediaPosting")) {
|
|
129
|
+
return entry;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function parseJson(value: string): unknown {
|
|
137
|
+
try {
|
|
138
|
+
return JSON.parse(value);
|
|
139
|
+
} catch {
|
|
140
|
+
try {
|
|
141
|
+
return JSON.parse(decodeHtml(value));
|
|
142
|
+
} catch {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function jsonLdEntries(value: unknown): Json[] {
|
|
149
|
+
if (Array.isArray(value)) {
|
|
150
|
+
return value.filter(object);
|
|
151
|
+
}
|
|
152
|
+
if (!object(value)) {
|
|
153
|
+
return [];
|
|
154
|
+
}
|
|
155
|
+
return Array.isArray(value["@graph"]) ? value["@graph"].filter(object) : [value];
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function hasSchemaType(value: Json, expected: string): boolean {
|
|
159
|
+
const type = value["@type"];
|
|
160
|
+
return type === expected || (Array.isArray(type) && type.includes(expected));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function linkedinItems(post: Json, html: string, id: string, referer: string): MediaItem[] {
|
|
164
|
+
const headers = { referer, "user-agent": browserUserAgent() };
|
|
165
|
+
if (hasSchemaType(post, "VideoObject")) {
|
|
166
|
+
const url = bestVideoUrl(html) ?? string(post.contentUrl);
|
|
167
|
+
return url
|
|
168
|
+
? [{ filename: filename(`linkedin_${id}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "linkedin", url }]
|
|
169
|
+
: [];
|
|
170
|
+
}
|
|
171
|
+
const url = imageUrl(post.image);
|
|
172
|
+
return url
|
|
173
|
+
? [{ filename: filename(`linkedin_${id}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "linkedin", url }]
|
|
174
|
+
: [];
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function bestVideoUrl(html: string): string | null {
|
|
178
|
+
const encoded = html.match(/<video\b[^>]*\bdata-sources=(["'])(.*?)\1[^>]*>/is)?.[2];
|
|
179
|
+
if (!encoded) {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
let parsed: unknown;
|
|
183
|
+
try {
|
|
184
|
+
parsed = JSON.parse(decodeHtml(encoded));
|
|
185
|
+
} catch {
|
|
186
|
+
return null;
|
|
187
|
+
}
|
|
188
|
+
const sources = Array.isArray(parsed) ? parsed.filter(object) : [];
|
|
189
|
+
const videos = sources.filter((source) => string(source.type) === "video/mp4" && string(source.src));
|
|
190
|
+
const selected = videos.reduce<Json | null>((best, candidate) => {
|
|
191
|
+
const bitrate = count(candidate["data-bitrate"]) ?? 0;
|
|
192
|
+
const bestBitrate = best ? count(best["data-bitrate"]) ?? 0 : -1;
|
|
193
|
+
return bitrate > bestBitrate ? candidate : best;
|
|
194
|
+
}, null);
|
|
195
|
+
return selected ? string(selected.src) : null;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function imageUrl(value: unknown): string | null {
|
|
199
|
+
if (typeof value === "string") {
|
|
200
|
+
return string(value);
|
|
201
|
+
}
|
|
202
|
+
if (Array.isArray(value)) {
|
|
203
|
+
return value.map(imageUrl).find((url): url is string => Boolean(url)) ?? null;
|
|
204
|
+
}
|
|
205
|
+
return object(value) ? string(value.url) : null;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function interactionCount(value: unknown, action: string): number | undefined {
|
|
209
|
+
const statistics = Array.isArray(value) ? value.filter(object) : object(value) ? [value] : [];
|
|
210
|
+
const statistic = statistics.find((entry) => string(entry.interactionType)?.endsWith(`/${action}`));
|
|
211
|
+
return statistic ? count(statistic.userInteractionCount) : undefined;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function linkedinHandle(value: string | null): string | null {
|
|
215
|
+
if (!value) {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
try {
|
|
219
|
+
return new URL(value).pathname.match(/^\/(?:company|in|showcase)\/([^/]+)/)?.[1] ?? null;
|
|
220
|
+
} catch {
|
|
221
|
+
return null;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function decodeHtml(value: string): string {
|
|
226
|
+
return value.replace(/&(quot|amp|lt|gt|#39|#x27|#(\d+)|#x([0-9a-f]+));/gi, (_entity, name: string, decimal: string, hex: string) => {
|
|
227
|
+
switch (name.toLowerCase()) {
|
|
228
|
+
case "quot":
|
|
229
|
+
return '"';
|
|
230
|
+
case "amp":
|
|
231
|
+
return "&";
|
|
232
|
+
case "lt":
|
|
233
|
+
return "<";
|
|
234
|
+
case "gt":
|
|
235
|
+
return ">";
|
|
236
|
+
case "#39":
|
|
237
|
+
case "#x27":
|
|
238
|
+
return "'";
|
|
239
|
+
default:
|
|
240
|
+
return String.fromCodePoint(decimal ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
}
|
package/src/postfetch.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { asUrl, createNet, PostfetchError, type Platform, type PostfetchResult } from "./internal";
|
|
2
2
|
import { resolveFacebook } from "./facebook";
|
|
3
3
|
import { resolveInstagram } from "./instagram";
|
|
4
|
+
import { resolveLinkedin } from "./linkedin";
|
|
4
5
|
import { resolvePinterest } from "./pinterest";
|
|
5
6
|
import { resolveReddit } from "./reddit";
|
|
6
7
|
import { resolveSoundcloud } from "./soundcloud";
|
|
@@ -24,7 +25,7 @@ export type PostfetchOptions = {
|
|
|
24
25
|
* performs no side effects beyond the lookup; use {@link download}, {@link archive}
|
|
25
26
|
* or {@link toResponse} to turn the result into bytes or a `Response`.
|
|
26
27
|
*
|
|
27
|
-
* @param url A post URL from
|
|
28
|
+
* @param url A post URL from a supported platform.
|
|
28
29
|
* @param options See {@link PostfetchOptions}.
|
|
29
30
|
* @returns The resolved post and its media items.
|
|
30
31
|
* @throws {PostfetchError} If the URL is empty or its host is unsupported.
|
|
@@ -52,6 +53,8 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
|
|
|
52
53
|
return resolveFacebook(context);
|
|
53
54
|
case "instagram":
|
|
54
55
|
return resolveInstagram(context);
|
|
56
|
+
case "linkedin":
|
|
57
|
+
return resolveLinkedin(context);
|
|
55
58
|
case "pinterest":
|
|
56
59
|
return resolvePinterest(context);
|
|
57
60
|
case "reddit":
|
|
@@ -82,6 +85,9 @@ export function detect(url: string): Platform {
|
|
|
82
85
|
if (host.includes("instagram.com")) {
|
|
83
86
|
return "instagram";
|
|
84
87
|
}
|
|
88
|
+
if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
|
|
89
|
+
return "linkedin";
|
|
90
|
+
}
|
|
85
91
|
if (host.includes("pinterest.") || host === "pin.it") {
|
|
86
92
|
return "pinterest";
|
|
87
93
|
}
|
|
@@ -100,5 +106,5 @@ export function detect(url: string): Platform {
|
|
|
100
106
|
if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
|
|
101
107
|
return "twitter";
|
|
102
108
|
}
|
|
103
|
-
throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
109
|
+
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
104
110
|
}
|
package/src/tiktok.ts
CHANGED
|
@@ -14,27 +14,26 @@ import {
|
|
|
14
14
|
type PostfetchResult,
|
|
15
15
|
type MediaItem,
|
|
16
16
|
} from "./internal";
|
|
17
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
browserUserAgent,
|
|
19
|
+
firefoxNavigationHeaders,
|
|
20
|
+
navigationHeaders,
|
|
21
|
+
} from "./fingerprint";
|
|
18
22
|
|
|
19
23
|
const marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
|
|
24
|
+
const pageAttemptCount = 12;
|
|
20
25
|
|
|
21
26
|
export async function resolveTiktok(input: ResolveContext): Promise<PostfetchResult> {
|
|
22
|
-
const
|
|
23
|
-
const pageUrl = await followShortlink(input.net, userAgent, input.url);
|
|
27
|
+
const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
|
|
24
28
|
const id = videoId(pageUrl);
|
|
25
29
|
if (!id) {
|
|
26
30
|
throw new Error("TikTok video id not found");
|
|
27
31
|
}
|
|
28
|
-
const page = await
|
|
29
|
-
if (!recoverablePageError(error)) {
|
|
30
|
-
throw error;
|
|
31
|
-
}
|
|
32
|
-
return fetchVideoPage(input.net, id, firefoxUserAgent());
|
|
33
|
-
});
|
|
32
|
+
const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
|
|
34
33
|
const user = author(page.item) ?? username(pageUrl) ?? "i";
|
|
35
34
|
const headers: Record<string, string> = {
|
|
36
35
|
referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
|
|
37
|
-
"user-agent": userAgent,
|
|
36
|
+
"user-agent": page.userAgent,
|
|
38
37
|
};
|
|
39
38
|
if (page.cookie) {
|
|
40
39
|
headers.cookie = page.cookie;
|
|
@@ -46,6 +45,41 @@ export async function resolveTiktok(input: ResolveContext): Promise<PostfetchRes
|
|
|
46
45
|
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
47
46
|
}
|
|
48
47
|
|
|
48
|
+
async function videoPage(
|
|
49
|
+
net: Net,
|
|
50
|
+
id: string,
|
|
51
|
+
photo: boolean,
|
|
52
|
+
): Promise<{ item: Json; cookie: string | null; userAgent: string }> {
|
|
53
|
+
for (let attempt = 0; attempt < pageAttemptCount; attempt += 1) {
|
|
54
|
+
const baseHeaders = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
|
|
55
|
+
const userAgent = baseHeaders["user-agent"];
|
|
56
|
+
const headers = {
|
|
57
|
+
...baseHeaders,
|
|
58
|
+
referer: "https://www.tiktok.com/",
|
|
59
|
+
"sec-fetch-site": "same-origin",
|
|
60
|
+
};
|
|
61
|
+
try {
|
|
62
|
+
return { ...(await fetchVideoPage(net, id, headers)), userAgent };
|
|
63
|
+
} catch (error) {
|
|
64
|
+
if (!recoverablePageError(error)) {
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!photo) {
|
|
71
|
+
throw new Error("TikTok hydration not found");
|
|
72
|
+
}
|
|
73
|
+
const baseHeaders = navigationHeaders();
|
|
74
|
+
const userAgent = baseHeaders["user-agent"];
|
|
75
|
+
const headers = {
|
|
76
|
+
...baseHeaders,
|
|
77
|
+
referer: "https://www.tiktok.com/",
|
|
78
|
+
"sec-fetch-site": "same-origin",
|
|
79
|
+
};
|
|
80
|
+
return { ...(await fetchEmbedPage(net, id, headers)), userAgent };
|
|
81
|
+
}
|
|
82
|
+
|
|
49
83
|
export function tiktokMetadata(item: Json): PostMetadata & { extra?: TiktokExtra } {
|
|
50
84
|
const user = object(item.author) ? item.author : null;
|
|
51
85
|
const stats = object(item.stats) ? item.stats : null;
|
|
@@ -78,18 +112,25 @@ export function tiktokMetadata(item: Json): PostMetadata & { extra?: TiktokExtra
|
|
|
78
112
|
async function fetchVideoPage(
|
|
79
113
|
net: Net,
|
|
80
114
|
id: string,
|
|
81
|
-
|
|
115
|
+
headers: Record<string, string>,
|
|
82
116
|
): Promise<{ item: Json; cookie: string | null }> {
|
|
83
117
|
const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
|
|
84
|
-
headers
|
|
85
|
-
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
86
|
-
referer: "https://www.tiktok.com/",
|
|
87
|
-
"user-agent": userAgent,
|
|
88
|
-
},
|
|
118
|
+
headers,
|
|
89
119
|
});
|
|
90
120
|
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
91
121
|
}
|
|
92
122
|
|
|
123
|
+
async function fetchEmbedPage(
|
|
124
|
+
net: Net,
|
|
125
|
+
id: string,
|
|
126
|
+
headers: Record<string, string>,
|
|
127
|
+
): Promise<{ item: Json; cookie: string | null }> {
|
|
128
|
+
const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
|
|
129
|
+
headers,
|
|
130
|
+
});
|
|
131
|
+
return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
|
|
132
|
+
}
|
|
133
|
+
|
|
93
134
|
export function isShortlinkHost(hostname: string): boolean {
|
|
94
135
|
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
95
136
|
}
|
|
@@ -134,11 +175,75 @@ function itemStruct(html: string): Json {
|
|
|
134
175
|
return item;
|
|
135
176
|
}
|
|
136
177
|
|
|
178
|
+
function embedItemStruct(html: string, id: string): Json {
|
|
179
|
+
const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
180
|
+
if (!raw) {
|
|
181
|
+
throw new Error("TikTok embed state not found");
|
|
182
|
+
}
|
|
183
|
+
const parsed: unknown = JSON.parse(raw);
|
|
184
|
+
const source = object(parsed) && object(parsed.source) ? parsed.source : null;
|
|
185
|
+
const data = source && object(source.data) ? source.data : null;
|
|
186
|
+
const rawRoute = data ? data[`/embed/v2/${id}`] : null;
|
|
187
|
+
const route = object(rawRoute) ? rawRoute : null;
|
|
188
|
+
const videoData = route && object(route.videoData) ? route.videoData : null;
|
|
189
|
+
const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
|
|
190
|
+
if (!videoData || !item) {
|
|
191
|
+
throw new Error("TikTok embed item not found");
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
|
|
195
|
+
const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
|
|
196
|
+
const video = object(item.video) ? item.video : {};
|
|
197
|
+
const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
|
|
198
|
+
const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
|
|
199
|
+
const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
|
|
200
|
+
if (displayImages.length === 0) {
|
|
201
|
+
throw new Error("TikTok video hydration not found");
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
return {
|
|
205
|
+
...item,
|
|
206
|
+
author: {
|
|
207
|
+
nickname: string(authorInfo.nickName),
|
|
208
|
+
uniqueId: string(authorInfo.uniqueId),
|
|
209
|
+
verified: bool(authorInfo.verified),
|
|
210
|
+
},
|
|
211
|
+
desc: string(item.text),
|
|
212
|
+
imagePost: {
|
|
213
|
+
images: displayImages.map((image) => ({
|
|
214
|
+
imageURL: {
|
|
215
|
+
urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url): url is string => Boolean(url)) : [],
|
|
216
|
+
},
|
|
217
|
+
})),
|
|
218
|
+
},
|
|
219
|
+
music: {
|
|
220
|
+
authorName: string(musicInfo.authorName),
|
|
221
|
+
playUrl: firstString(musicInfo.playUrl),
|
|
222
|
+
title: string(musicInfo.musicName),
|
|
223
|
+
},
|
|
224
|
+
stats: {
|
|
225
|
+
commentCount: item.commentCount,
|
|
226
|
+
diggCount: item.diggCount,
|
|
227
|
+
playCount: item.playCount,
|
|
228
|
+
shareCount: item.shareCount,
|
|
229
|
+
},
|
|
230
|
+
video: {
|
|
231
|
+
duration: videoMeta.duration,
|
|
232
|
+
},
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
|
|
137
236
|
function recoverablePageError(error: unknown): boolean {
|
|
138
237
|
const message = error instanceof Error ? error.message : "";
|
|
139
238
|
return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
|
|
140
239
|
}
|
|
141
240
|
|
|
241
|
+
function firstString(value: unknown): string | null {
|
|
242
|
+
return Array.isArray(value)
|
|
243
|
+
? value.map(string).find((candidate): candidate is string => candidate !== null) ?? null
|
|
244
|
+
: string(value);
|
|
245
|
+
}
|
|
246
|
+
|
|
142
247
|
function downloadUrl(item: Json): string | null {
|
|
143
248
|
const video = object(item.video) ? item.video : null;
|
|
144
249
|
return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
|