@postfetch/core 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/fingerprint.d.ts +5 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +400 -45
- package/dist/internal.d.ts +2 -2
- package/dist/linkedin.d.ts +3 -0
- package/dist/postfetch.d.ts +1 -1
- package/package.json +3 -2
- package/src/facebook.ts +40 -3
- package/src/fingerprint.ts +21 -7
- package/src/index.ts +1 -1
- package/src/instagram.ts +54 -22
- package/src/internal.ts +2 -1
- package/src/linkedin.ts +243 -0
- package/src/postfetch.ts +8 -2
- package/src/tiktok.ts +121 -16
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @postfetch/core
|
|
2
2
|
|
|
3
|
-
Zero-dependency typed core that turns Instagram, TikTok and YouTube post URLs into media files. Part of [postfetch](https://github.com/chelokot/postfetch).
|
|
3
|
+
Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files. Part of [postfetch](https://github.com/chelokot/postfetch).
|
|
4
4
|
|
|
5
5
|
```ts
|
|
6
6
|
import { postfetch, download, archive, toResponse, PostfetchError } from "@postfetch/core";
|
package/dist/fingerprint.d.ts
CHANGED
|
@@ -4,8 +4,12 @@ export type BrowserFingerprint = {
|
|
|
4
4
|
secChUaPlatform: string;
|
|
5
5
|
userAgent: string;
|
|
6
6
|
};
|
|
7
|
+
export type NavigationHeaders = Record<string, string> & {
|
|
8
|
+
"user-agent": string;
|
|
9
|
+
};
|
|
7
10
|
export declare function browserFingerprint(): BrowserFingerprint;
|
|
8
|
-
export declare function navigationHeaders():
|
|
11
|
+
export declare function navigationHeaders(): NavigationHeaders;
|
|
9
12
|
export declare function browserUserAgent(): string;
|
|
10
13
|
export declare function firefoxUserAgent(): string;
|
|
14
|
+
export declare function firefoxNavigationHeaders(userAgent?: string): NavigationHeaders;
|
|
11
15
|
export declare function instagramAppUserAgent(): string;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Turn a public social post URL into its media.
|
|
3
3
|
*
|
|
4
|
-
* {@link postfetch} resolves Instagram, TikTok, YouTube, Facebook and X (Twitter)
|
|
4
|
+
* {@link postfetch} resolves Instagram, LinkedIn, TikTok, YouTube, Facebook and X (Twitter)
|
|
5
5
|
* URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
|
|
6
6
|
* (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
|
|
7
7
|
* Cloudflare Workers and browsers alike. It only resolves the media; the
|
package/dist/index.js
CHANGED
|
@@ -124,20 +124,25 @@ function browserFingerprint() {
|
|
|
124
124
|
userAgent: `Mozilla/5.0 (${platform.uaToken}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${version}.0.0.0 Safari/537.36`
|
|
125
125
|
};
|
|
126
126
|
}
|
|
127
|
-
function
|
|
128
|
-
const fingerprint = browserFingerprint();
|
|
127
|
+
function baseNavigationHeaders(userAgent, acceptLanguage) {
|
|
129
128
|
return {
|
|
130
129
|
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
131
|
-
"accept-language":
|
|
132
|
-
"sec-ch-ua": fingerprint.secChUa,
|
|
133
|
-
"sec-ch-ua-mobile": "?0",
|
|
134
|
-
"sec-ch-ua-platform": fingerprint.secChUaPlatform,
|
|
130
|
+
"accept-language": acceptLanguage,
|
|
135
131
|
"sec-fetch-dest": "document",
|
|
136
132
|
"sec-fetch-mode": "navigate",
|
|
137
133
|
"sec-fetch-site": "none",
|
|
138
134
|
"sec-fetch-user": "?1",
|
|
139
135
|
"upgrade-insecure-requests": "1",
|
|
140
|
-
"user-agent":
|
|
136
|
+
"user-agent": userAgent
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
function navigationHeaders() {
|
|
140
|
+
const fingerprint = browserFingerprint();
|
|
141
|
+
return {
|
|
142
|
+
...baseNavigationHeaders(fingerprint.userAgent, fingerprint.acceptLanguage),
|
|
143
|
+
"sec-ch-ua": fingerprint.secChUa,
|
|
144
|
+
"sec-ch-ua-mobile": "?0",
|
|
145
|
+
"sec-ch-ua-platform": fingerprint.secChUaPlatform
|
|
141
146
|
};
|
|
142
147
|
}
|
|
143
148
|
function browserUserAgent() {
|
|
@@ -148,6 +153,9 @@ function firefoxUserAgent() {
|
|
|
148
153
|
const version = pick(firefoxVersions);
|
|
149
154
|
return `Mozilla/5.0 (${platform.uaToken}; rv:${version}.0) Gecko/20100101 Firefox/${version}.0`;
|
|
150
155
|
}
|
|
156
|
+
function firefoxNavigationHeaders(userAgent = firefoxUserAgent()) {
|
|
157
|
+
return baseNavigationHeaders(userAgent, pick(acceptLanguages));
|
|
158
|
+
}
|
|
151
159
|
function instagramAppUserAgent() {
|
|
152
160
|
return pick(instagramAppUserAgents);
|
|
153
161
|
}
|
|
@@ -156,7 +164,9 @@ function instagramAppUserAgent() {
|
|
|
156
164
|
async function resolveFacebook(input) {
|
|
157
165
|
const canonical = await canonicalUrl(input.net, input.url);
|
|
158
166
|
const id = facebookId(canonical) ?? facebookId(input.url) ?? "video";
|
|
159
|
-
const
|
|
167
|
+
const embed = await embedVideo(input.net, canonical);
|
|
168
|
+
const watch = !embed && /^\d+$/.test(id) ? await watchVideo(input.net, id) : null;
|
|
169
|
+
const url = embed ?? watch?.url ?? null;
|
|
160
170
|
if (!url) {
|
|
161
171
|
throw new Error("Facebook video not found");
|
|
162
172
|
}
|
|
@@ -169,7 +179,7 @@ async function resolveFacebook(input) {
|
|
|
169
179
|
platform: "facebook",
|
|
170
180
|
url
|
|
171
181
|
};
|
|
172
|
-
return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [item], platform: "facebook" };
|
|
182
|
+
return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [item], metadata: watch?.metadata, platform: "facebook" };
|
|
173
183
|
}
|
|
174
184
|
function navigationHeaders2() {
|
|
175
185
|
return {
|
|
@@ -192,6 +202,25 @@ async function embedVideo(net, canonical) {
|
|
|
192
202
|
const html = await response.text();
|
|
193
203
|
return source(html, "hd_src") ?? source(html, "sd_src");
|
|
194
204
|
}
|
|
205
|
+
async function watchVideo(net, id) {
|
|
206
|
+
const headers = { ...navigationHeaders2(), accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" };
|
|
207
|
+
const response = await net(`https://www.facebook.com/watch/?v=${id}&_rdr`, { headers });
|
|
208
|
+
if (!response.ok) {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
const html = await response.text();
|
|
212
|
+
const url = source(html, "playable_url_quality_hd") ?? source(html, "browser_native_hd_url") ?? source(html, "playable_url") ?? source(html, "browser_native_sd_url");
|
|
213
|
+
return url ? { url, metadata: watchMetadata(html) } : null;
|
|
214
|
+
}
|
|
215
|
+
function watchMetadata(html) {
|
|
216
|
+
const owner = html.match(/"name":("(?:\\.|[^"\\])*"),"__isVideoOwner"/);
|
|
217
|
+
const name = owner ? JSON.parse(owner[1]) : undefined;
|
|
218
|
+
return {
|
|
219
|
+
author: name ? { name } : undefined,
|
|
220
|
+
createdAt: isoFromEpochSeconds(html.match(/"publish_time":(\d+)/)?.[1]),
|
|
221
|
+
viewCount: count(html.match(/"video_view_count":(\d+)/)?.[1])
|
|
222
|
+
};
|
|
223
|
+
}
|
|
195
224
|
function source(html, key) {
|
|
196
225
|
const match = html.match(new RegExp(`"${key}":("(?:\\\\.|[^"\\\\])*")`));
|
|
197
226
|
if (!match) {
|
|
@@ -250,7 +279,21 @@ function embedHeaders() {
|
|
|
250
279
|
}
|
|
251
280
|
async function resolveInstagram(input) {
|
|
252
281
|
const code = shortcode(input.url);
|
|
253
|
-
const
|
|
282
|
+
const videoRequired = /\/(?:reel|reels|tv)\//.test(asUrl(input.url).pathname);
|
|
283
|
+
const resolvers = [
|
|
284
|
+
() => pageMedia(input.net, code, input.preferredWidth),
|
|
285
|
+
() => mobileMedia(input.net, code, input.preferredWidth),
|
|
286
|
+
() => embedMedia(input.net, code),
|
|
287
|
+
() => graphqlMedia(input.net, code)
|
|
288
|
+
];
|
|
289
|
+
let media = null;
|
|
290
|
+
for (const resolve of resolvers) {
|
|
291
|
+
const candidate = await resolve();
|
|
292
|
+
if (candidate && (!videoRequired || mediaItems(candidate, code, input.preferredWidth).some((item) => item.kind === "video"))) {
|
|
293
|
+
media = candidate;
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
254
297
|
if (!media) {
|
|
255
298
|
throw await instagramUnavailable(input.net, code);
|
|
256
299
|
}
|
|
@@ -460,23 +503,20 @@ async function graphqlMedia(net, code) {
|
|
|
460
503
|
}
|
|
461
504
|
const body = new URLSearchParams({
|
|
462
505
|
...params.body,
|
|
463
|
-
|
|
506
|
+
av: "0",
|
|
507
|
+
doc_id: "26130443479876713",
|
|
464
508
|
fb_api_caller_class: "RelayModern",
|
|
465
|
-
fb_api_req_friendly_name: "
|
|
509
|
+
fb_api_req_friendly_name: "PolarisPostRootQuery",
|
|
466
510
|
server_timestamps: "true",
|
|
467
|
-
variables: JSON.stringify({
|
|
468
|
-
fetch_tagged_user_count: null,
|
|
469
|
-
hoisted_comment_id: null,
|
|
470
|
-
hoisted_reply_id: null,
|
|
471
|
-
shortcode: code
|
|
472
|
-
})
|
|
511
|
+
variables: JSON.stringify({ shortcode: code })
|
|
473
512
|
});
|
|
474
513
|
const response = await net("https://www.instagram.com/graphql/query", {
|
|
475
514
|
body,
|
|
476
515
|
headers: {
|
|
477
516
|
...embedHeaders(),
|
|
478
517
|
...params.headers,
|
|
479
|
-
"X-FB-Friendly-Name": "
|
|
518
|
+
"X-FB-Friendly-Name": "PolarisPostRootQuery",
|
|
519
|
+
"X-Requested-With": "XMLHttpRequest",
|
|
480
520
|
"content-type": "application/x-www-form-urlencoded"
|
|
481
521
|
},
|
|
482
522
|
method: "POST"
|
|
@@ -502,14 +542,14 @@ async function graphqlParams(net, code) {
|
|
|
502
542
|
const push = entryObject("InstagramWebPushInfo", html);
|
|
503
543
|
const lsd = entryObject("LSD", html)?.token ?? randomToken();
|
|
504
544
|
const csrf = entryObject("InstagramSecurityConfig", html)?.csrf_token;
|
|
505
|
-
const cookie =
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
polaris?.
|
|
509
|
-
"
|
|
510
|
-
|
|
511
|
-
"
|
|
512
|
-
|
|
545
|
+
const cookie = instagramCookies(response.headers, {
|
|
546
|
+
csrftoken: string(csrf),
|
|
547
|
+
dpr: "2",
|
|
548
|
+
ig_did: string(polaris?.device_id),
|
|
549
|
+
ig_nrcb: "1",
|
|
550
|
+
mid: string(polaris?.machine_id),
|
|
551
|
+
wd: "1280x720"
|
|
552
|
+
});
|
|
513
553
|
return {
|
|
514
554
|
headers: {
|
|
515
555
|
"X-CSRFToken": string(csrf) ?? "",
|
|
@@ -543,6 +583,12 @@ async function graphqlParams(net, code) {
|
|
|
543
583
|
};
|
|
544
584
|
}
|
|
545
585
|
function gqlShortcodeMedia(data) {
|
|
586
|
+
const webInfo = object(data.xdt_api__v1__media__shortcode__web_info) ? data.xdt_api__v1__media__shortcode__web_info : null;
|
|
587
|
+
const webItems = webInfo && Array.isArray(webInfo.items) ? webInfo.items : [];
|
|
588
|
+
const webMedia = webItems[0];
|
|
589
|
+
if (object(webMedia)) {
|
|
590
|
+
return webMedia;
|
|
591
|
+
}
|
|
546
592
|
const media = data.gql_data && object(data.gql_data) ? data.gql_data.shortcode_media ?? data.gql_data.xdt_shortcode_media : data.shortcode_media ?? data.xdt_shortcode_media;
|
|
547
593
|
return object(media) ? media : null;
|
|
548
594
|
}
|
|
@@ -562,6 +608,24 @@ function mediaItems(media, code, preferredWidth) {
|
|
|
562
608
|
}
|
|
563
609
|
return instagramItem(media, code, null, preferredWidth);
|
|
564
610
|
}
|
|
611
|
+
function instagramCookies(headers, fallback) {
|
|
612
|
+
const getter = headers.getSetCookie;
|
|
613
|
+
const raw = typeof getter === "function" ? getter.call(headers) : headers.get("set-cookie")?.split(/,(?=[^;]+?=)/) ?? [];
|
|
614
|
+
const cookies = new Map;
|
|
615
|
+
for (const value of raw) {
|
|
616
|
+
const pair = value.split(";", 1)[0]?.trim();
|
|
617
|
+
const separator = pair?.indexOf("=") ?? -1;
|
|
618
|
+
if (pair && separator > 0) {
|
|
619
|
+
cookies.set(pair.slice(0, separator), pair.slice(separator + 1));
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
for (const [name, value] of Object.entries(fallback)) {
|
|
623
|
+
if (value && !cookies.has(name)) {
|
|
624
|
+
cookies.set(name, value);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
return [...cookies].map(([name, value]) => `${name}=${value}`).join("; ");
|
|
628
|
+
}
|
|
565
629
|
function instagramItem(media, code, index, preferredWidth) {
|
|
566
630
|
const video = selectVersion(media.video_versions, preferredWidth) ?? string(media.video_url);
|
|
567
631
|
const suffix = index === null ? "" : `_${index}`;
|
|
@@ -625,6 +689,208 @@ function randomToken(length = 8) {
|
|
|
625
689
|
return crypto.getRandomValues(new Uint8Array(length)).reduce((value, byte) => value + (byte % 36).toString(36), "");
|
|
626
690
|
}
|
|
627
691
|
|
|
692
|
+
// src/linkedin.ts
|
|
693
|
+
async function resolveLinkedin(input) {
|
|
694
|
+
const reference = postReference(input.url);
|
|
695
|
+
const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
|
|
696
|
+
const response = await input.net(pageUrl, { headers: navigationHeaders() });
|
|
697
|
+
if (response.status === 404) {
|
|
698
|
+
throw new PostfetchError(404, "LinkedIn post not found", "notFound");
|
|
699
|
+
}
|
|
700
|
+
if (response.status === 429 || response.status === 999) {
|
|
701
|
+
throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
|
|
702
|
+
}
|
|
703
|
+
if (!response.ok) {
|
|
704
|
+
throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
|
|
705
|
+
}
|
|
706
|
+
if (response.url.includes("/signup/") || response.url.includes("/login")) {
|
|
707
|
+
throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
|
|
708
|
+
}
|
|
709
|
+
const html = await response.text();
|
|
710
|
+
const post = linkedinPost(html);
|
|
711
|
+
if (!post) {
|
|
712
|
+
throw new PostfetchError(404, "LinkedIn post unavailable", "unavailable");
|
|
713
|
+
}
|
|
714
|
+
const canonical = string(post["@id"]);
|
|
715
|
+
const canonicalReference = canonical ? postReferenceOrNull(canonical) : null;
|
|
716
|
+
const id = canonicalReference?.id ?? reference.id;
|
|
717
|
+
const items = linkedinItems(post, html, id, pageUrl);
|
|
718
|
+
if (items.length === 0) {
|
|
719
|
+
throw new Error("LinkedIn media url not found");
|
|
720
|
+
}
|
|
721
|
+
return {
|
|
722
|
+
archiveFilename: filename(`linkedin_${id}.zip`),
|
|
723
|
+
id,
|
|
724
|
+
items,
|
|
725
|
+
metadata: linkedinMetadata(post),
|
|
726
|
+
platform: "linkedin"
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
function linkedinMetadata(post) {
|
|
730
|
+
const author = linkedinAuthor(post);
|
|
731
|
+
return {
|
|
732
|
+
title: string(post.text) ?? string(post.name) ?? string(post.headline) ?? undefined,
|
|
733
|
+
text: string(post.description) ?? string(post.articleBody) ?? string(post.text) ?? undefined,
|
|
734
|
+
author: author ? {
|
|
735
|
+
handle: linkedinHandle(string(author.url)) ?? undefined,
|
|
736
|
+
name: string(author.name) ?? undefined
|
|
737
|
+
} : undefined,
|
|
738
|
+
createdAt: isoFromDateString(post.datePublished ?? post.uploadDate),
|
|
739
|
+
likeCount: interactionCount(post.interactionStatistic, "LikeAction"),
|
|
740
|
+
commentCount: count(post.commentCount) ?? interactionCount(post.interactionStatistic, "CommentAction")
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
function postReference(input) {
|
|
744
|
+
const reference = postReferenceOrNull(input);
|
|
745
|
+
if (!reference) {
|
|
746
|
+
throw new PostfetchError(400, "LinkedIn post id not found");
|
|
747
|
+
}
|
|
748
|
+
return reference;
|
|
749
|
+
}
|
|
750
|
+
function postReferenceOrNull(input) {
|
|
751
|
+
let pathname;
|
|
752
|
+
try {
|
|
753
|
+
pathname = new URL(input).pathname.replace(/%3A/gi, ":");
|
|
754
|
+
} catch {
|
|
755
|
+
return null;
|
|
756
|
+
}
|
|
757
|
+
const urn = pathname.match(/\/(?:embed\/)?feed\/update\/urn:li:(activity|share|ugcpost):(\d+)\/?$/i);
|
|
758
|
+
const post = pathname.match(/\/posts\/[^/]*-(activity|share|ugcpost)-(\d+)(?:-[^/]*)?\/?$/i);
|
|
759
|
+
const match = urn ?? post;
|
|
760
|
+
if (!match) {
|
|
761
|
+
return null;
|
|
762
|
+
}
|
|
763
|
+
switch (match[1].toLowerCase()) {
|
|
764
|
+
case "activity":
|
|
765
|
+
return { id: match[2], kind: "activity" };
|
|
766
|
+
case "share":
|
|
767
|
+
return { id: match[2], kind: "share" };
|
|
768
|
+
case "ugcpost":
|
|
769
|
+
return { id: match[2], kind: "ugcPost" };
|
|
770
|
+
default:
|
|
771
|
+
return null;
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
function linkedinAuthor(post) {
|
|
775
|
+
if (object(post.creator)) {
|
|
776
|
+
return post.creator;
|
|
777
|
+
}
|
|
778
|
+
if (object(post.author)) {
|
|
779
|
+
return post.author;
|
|
780
|
+
}
|
|
781
|
+
return null;
|
|
782
|
+
}
|
|
783
|
+
function linkedinPost(html) {
|
|
784
|
+
for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
|
|
785
|
+
if (!/\btype=["']application\/ld\+json["']/i.test(match[1])) {
|
|
786
|
+
continue;
|
|
787
|
+
}
|
|
788
|
+
const parsed = parseJson(match[2]);
|
|
789
|
+
for (const entry of jsonLdEntries(parsed)) {
|
|
790
|
+
if (hasSchemaType(entry, "VideoObject") || hasSchemaType(entry, "SocialMediaPosting")) {
|
|
791
|
+
return entry;
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
return null;
|
|
796
|
+
}
|
|
797
|
+
function parseJson(value) {
|
|
798
|
+
try {
|
|
799
|
+
return JSON.parse(value);
|
|
800
|
+
} catch {
|
|
801
|
+
try {
|
|
802
|
+
return JSON.parse(decodeHtml(value));
|
|
803
|
+
} catch {
|
|
804
|
+
return null;
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
function jsonLdEntries(value) {
|
|
809
|
+
if (Array.isArray(value)) {
|
|
810
|
+
return value.filter(object);
|
|
811
|
+
}
|
|
812
|
+
if (!object(value)) {
|
|
813
|
+
return [];
|
|
814
|
+
}
|
|
815
|
+
return Array.isArray(value["@graph"]) ? value["@graph"].filter(object) : [value];
|
|
816
|
+
}
|
|
817
|
+
function hasSchemaType(value, expected) {
|
|
818
|
+
const type = value["@type"];
|
|
819
|
+
return type === expected || Array.isArray(type) && type.includes(expected);
|
|
820
|
+
}
|
|
821
|
+
function linkedinItems(post, html, id, referer) {
|
|
822
|
+
const headers = { referer, "user-agent": browserUserAgent() };
|
|
823
|
+
if (hasSchemaType(post, "VideoObject")) {
|
|
824
|
+
const url2 = bestVideoUrl(html) ?? string(post.contentUrl);
|
|
825
|
+
return url2 ? [{ filename: filename(`linkedin_${id}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "linkedin", url: url2 }] : [];
|
|
826
|
+
}
|
|
827
|
+
const url = imageUrl(post.image);
|
|
828
|
+
return url ? [{ filename: filename(`linkedin_${id}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "linkedin", url }] : [];
|
|
829
|
+
}
|
|
830
|
+
function bestVideoUrl(html) {
|
|
831
|
+
const encoded = html.match(/<video\b[^>]*\bdata-sources=(["'])(.*?)\1[^>]*>/is)?.[2];
|
|
832
|
+
if (!encoded) {
|
|
833
|
+
return null;
|
|
834
|
+
}
|
|
835
|
+
let parsed;
|
|
836
|
+
try {
|
|
837
|
+
parsed = JSON.parse(decodeHtml(encoded));
|
|
838
|
+
} catch {
|
|
839
|
+
return null;
|
|
840
|
+
}
|
|
841
|
+
const sources = Array.isArray(parsed) ? parsed.filter(object) : [];
|
|
842
|
+
const videos = sources.filter((source2) => string(source2.type) === "video/mp4" && string(source2.src));
|
|
843
|
+
const selected = videos.reduce((best, candidate) => {
|
|
844
|
+
const bitrate = count(candidate["data-bitrate"]) ?? 0;
|
|
845
|
+
const bestBitrate = best ? count(best["data-bitrate"]) ?? 0 : -1;
|
|
846
|
+
return bitrate > bestBitrate ? candidate : best;
|
|
847
|
+
}, null);
|
|
848
|
+
return selected ? string(selected.src) : null;
|
|
849
|
+
}
|
|
850
|
+
function imageUrl(value) {
|
|
851
|
+
if (typeof value === "string") {
|
|
852
|
+
return string(value);
|
|
853
|
+
}
|
|
854
|
+
if (Array.isArray(value)) {
|
|
855
|
+
return value.map(imageUrl).find((url) => Boolean(url)) ?? null;
|
|
856
|
+
}
|
|
857
|
+
return object(value) ? string(value.url) : null;
|
|
858
|
+
}
|
|
859
|
+
function interactionCount(value, action) {
|
|
860
|
+
const statistics = Array.isArray(value) ? value.filter(object) : object(value) ? [value] : [];
|
|
861
|
+
const statistic = statistics.find((entry) => string(entry.interactionType)?.endsWith(`/${action}`));
|
|
862
|
+
return statistic ? count(statistic.userInteractionCount) : undefined;
|
|
863
|
+
}
|
|
864
|
+
function linkedinHandle(value) {
|
|
865
|
+
if (!value) {
|
|
866
|
+
return null;
|
|
867
|
+
}
|
|
868
|
+
try {
|
|
869
|
+
return new URL(value).pathname.match(/^\/(?:company|in|showcase)\/([^/]+)/)?.[1] ?? null;
|
|
870
|
+
} catch {
|
|
871
|
+
return null;
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
function decodeHtml(value) {
|
|
875
|
+
return value.replace(/&(quot|amp|lt|gt|#39|#x27|#(\d+)|#x([0-9a-f]+));/gi, (_entity, name, decimal, hex) => {
|
|
876
|
+
switch (name.toLowerCase()) {
|
|
877
|
+
case "quot":
|
|
878
|
+
return '"';
|
|
879
|
+
case "amp":
|
|
880
|
+
return "&";
|
|
881
|
+
case "lt":
|
|
882
|
+
return "<";
|
|
883
|
+
case "gt":
|
|
884
|
+
return ">";
|
|
885
|
+
case "#39":
|
|
886
|
+
case "#x27":
|
|
887
|
+
return "'";
|
|
888
|
+
default:
|
|
889
|
+
return String.fromCodePoint(decimal ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
|
|
890
|
+
}
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
|
|
628
894
|
// src/isobmff.ts
|
|
629
895
|
function parseBoxes(data, start = 0, end = data.length) {
|
|
630
896
|
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
@@ -1386,23 +1652,18 @@ function trackId(track) {
|
|
|
1386
1652
|
|
|
1387
1653
|
// src/tiktok.ts
|
|
1388
1654
|
var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
|
|
1655
|
+
var pageAttemptCount = 12;
|
|
1389
1656
|
async function resolveTiktok(input) {
|
|
1390
|
-
const
|
|
1391
|
-
const pageUrl = await followShortlink(input.net, userAgent, input.url);
|
|
1657
|
+
const pageUrl = await followShortlink(input.net, browserUserAgent(), input.url);
|
|
1392
1658
|
const id = videoId(pageUrl);
|
|
1393
1659
|
if (!id) {
|
|
1394
1660
|
throw new Error("TikTok video id not found");
|
|
1395
1661
|
}
|
|
1396
|
-
const page = await
|
|
1397
|
-
if (!recoverablePageError(error)) {
|
|
1398
|
-
throw error;
|
|
1399
|
-
}
|
|
1400
|
-
return fetchVideoPage(input.net, id, firefoxUserAgent());
|
|
1401
|
-
});
|
|
1662
|
+
const page = await videoPage(input.net, id, asUrl(pageUrl).pathname.includes("/photo/"));
|
|
1402
1663
|
const user = author(page.item) ?? username(pageUrl) ?? "i";
|
|
1403
1664
|
const headers = {
|
|
1404
1665
|
referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
|
|
1405
|
-
"user-agent": userAgent
|
|
1666
|
+
"user-agent": page.userAgent
|
|
1406
1667
|
};
|
|
1407
1668
|
if (page.cookie) {
|
|
1408
1669
|
headers.cookie = page.cookie;
|
|
@@ -1413,6 +1674,35 @@ async function resolveTiktok(input) {
|
|
|
1413
1674
|
}
|
|
1414
1675
|
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
1415
1676
|
}
|
|
1677
|
+
async function videoPage(net, id, photo) {
|
|
1678
|
+
for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
|
|
1679
|
+
const baseHeaders2 = attempt % 2 === 0 ? navigationHeaders() : firefoxNavigationHeaders();
|
|
1680
|
+
const userAgent2 = baseHeaders2["user-agent"];
|
|
1681
|
+
const headers2 = {
|
|
1682
|
+
...baseHeaders2,
|
|
1683
|
+
referer: "https://www.tiktok.com/",
|
|
1684
|
+
"sec-fetch-site": "same-origin"
|
|
1685
|
+
};
|
|
1686
|
+
try {
|
|
1687
|
+
return { ...await fetchVideoPage(net, id, headers2), userAgent: userAgent2 };
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
if (!recoverablePageError(error)) {
|
|
1690
|
+
throw error;
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
}
|
|
1694
|
+
if (!photo) {
|
|
1695
|
+
throw new Error("TikTok hydration not found");
|
|
1696
|
+
}
|
|
1697
|
+
const baseHeaders = navigationHeaders();
|
|
1698
|
+
const userAgent = baseHeaders["user-agent"];
|
|
1699
|
+
const headers = {
|
|
1700
|
+
...baseHeaders,
|
|
1701
|
+
referer: "https://www.tiktok.com/",
|
|
1702
|
+
"sec-fetch-site": "same-origin"
|
|
1703
|
+
};
|
|
1704
|
+
return { ...await fetchEmbedPage(net, id, headers), userAgent };
|
|
1705
|
+
}
|
|
1416
1706
|
function tiktokMetadata(item) {
|
|
1417
1707
|
const user = object(item.author) ? item.author : null;
|
|
1418
1708
|
const stats = object(item.stats) ? item.stats : null;
|
|
@@ -1439,16 +1729,18 @@ function tiktokMetadata(item) {
|
|
|
1439
1729
|
}
|
|
1440
1730
|
};
|
|
1441
1731
|
}
|
|
1442
|
-
async function fetchVideoPage(net, id,
|
|
1732
|
+
async function fetchVideoPage(net, id, headers) {
|
|
1443
1733
|
const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
|
|
1444
|
-
headers
|
|
1445
|
-
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
1446
|
-
referer: "https://www.tiktok.com/",
|
|
1447
|
-
"user-agent": userAgent
|
|
1448
|
-
}
|
|
1734
|
+
headers
|
|
1449
1735
|
});
|
|
1450
1736
|
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
1451
1737
|
}
|
|
1738
|
+
async function fetchEmbedPage(net, id, headers) {
|
|
1739
|
+
const page = await net(`https://www.tiktok.com/embed/v2/${id}`, {
|
|
1740
|
+
headers
|
|
1741
|
+
});
|
|
1742
|
+
return { cookie: cookieHeader(page.headers), item: embedItemStruct(await page.text(), id) };
|
|
1743
|
+
}
|
|
1452
1744
|
function isShortlinkHost(hostname) {
|
|
1453
1745
|
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
1454
1746
|
}
|
|
@@ -1488,10 +1780,68 @@ function itemStruct(html) {
|
|
|
1488
1780
|
}
|
|
1489
1781
|
return item;
|
|
1490
1782
|
}
|
|
1783
|
+
function embedItemStruct(html, id) {
|
|
1784
|
+
const raw = html.match(/<script\b[^>]*\bid=["']__FRONTITY_CONNECT_STATE__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
1785
|
+
if (!raw) {
|
|
1786
|
+
throw new Error("TikTok embed state not found");
|
|
1787
|
+
}
|
|
1788
|
+
const parsed = JSON.parse(raw);
|
|
1789
|
+
const source2 = object(parsed) && object(parsed.source) ? parsed.source : null;
|
|
1790
|
+
const data = source2 && object(source2.data) ? source2.data : null;
|
|
1791
|
+
const rawRoute = data ? data[`/embed/v2/${id}`] : null;
|
|
1792
|
+
const route = object(rawRoute) ? rawRoute : null;
|
|
1793
|
+
const videoData = route && object(route.videoData) ? route.videoData : null;
|
|
1794
|
+
const item = videoData && object(videoData.itemInfos) ? videoData.itemInfos : null;
|
|
1795
|
+
if (!videoData || !item) {
|
|
1796
|
+
throw new Error("TikTok embed item not found");
|
|
1797
|
+
}
|
|
1798
|
+
const authorInfo = object(videoData.authorInfos) ? videoData.authorInfos : {};
|
|
1799
|
+
const musicInfo = object(videoData.musicInfos) ? videoData.musicInfos : {};
|
|
1800
|
+
const video = object(item.video) ? item.video : {};
|
|
1801
|
+
const videoMeta = object(video.videoMeta) ? video.videoMeta : {};
|
|
1802
|
+
const imagePost = object(videoData.imagePostInfo) ? videoData.imagePostInfo : null;
|
|
1803
|
+
const displayImages = imagePost && Array.isArray(imagePost.displayImages) ? imagePost.displayImages.filter(object) : [];
|
|
1804
|
+
if (displayImages.length === 0) {
|
|
1805
|
+
throw new Error("TikTok video hydration not found");
|
|
1806
|
+
}
|
|
1807
|
+
return {
|
|
1808
|
+
...item,
|
|
1809
|
+
author: {
|
|
1810
|
+
nickname: string(authorInfo.nickName),
|
|
1811
|
+
uniqueId: string(authorInfo.uniqueId),
|
|
1812
|
+
verified: bool(authorInfo.verified)
|
|
1813
|
+
},
|
|
1814
|
+
desc: string(item.text),
|
|
1815
|
+
imagePost: {
|
|
1816
|
+
images: displayImages.map((image) => ({
|
|
1817
|
+
imageURL: {
|
|
1818
|
+
urlList: Array.isArray(image.urlList) ? image.urlList.map(string).filter((url) => Boolean(url)) : []
|
|
1819
|
+
}
|
|
1820
|
+
}))
|
|
1821
|
+
},
|
|
1822
|
+
music: {
|
|
1823
|
+
authorName: string(musicInfo.authorName),
|
|
1824
|
+
playUrl: firstString(musicInfo.playUrl),
|
|
1825
|
+
title: string(musicInfo.musicName)
|
|
1826
|
+
},
|
|
1827
|
+
stats: {
|
|
1828
|
+
commentCount: item.commentCount,
|
|
1829
|
+
diggCount: item.diggCount,
|
|
1830
|
+
playCount: item.playCount,
|
|
1831
|
+
shareCount: item.shareCount
|
|
1832
|
+
},
|
|
1833
|
+
video: {
|
|
1834
|
+
duration: videoMeta.duration
|
|
1835
|
+
}
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1491
1838
|
function recoverablePageError(error) {
|
|
1492
1839
|
const message = error instanceof Error ? error.message : "";
|
|
1493
1840
|
return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
|
|
1494
1841
|
}
|
|
1842
|
+
function firstString(value) {
|
|
1843
|
+
return Array.isArray(value) ? value.map(string).find((candidate) => candidate !== null) ?? null : string(value);
|
|
1844
|
+
}
|
|
1495
1845
|
function downloadUrl(item) {
|
|
1496
1846
|
const video = object(item.video) ? item.video : null;
|
|
1497
1847
|
return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
|
|
@@ -1517,8 +1867,8 @@ function imageItems3(item, user, id, headers) {
|
|
|
1517
1867
|
const imagePost = object(item.imagePost) ? item.imagePost : null;
|
|
1518
1868
|
const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
|
|
1519
1869
|
return images.flatMap((image, index) => {
|
|
1520
|
-
const
|
|
1521
|
-
const list =
|
|
1870
|
+
const imageUrl2 = object(image.imageURL) ? image.imageURL : null;
|
|
1871
|
+
const list = imageUrl2 && Array.isArray(imageUrl2.urlList) ? imageUrl2.urlList : [];
|
|
1522
1872
|
const url = list.map(string).find((candidate) => Boolean(candidate)) ?? null;
|
|
1523
1873
|
return url ? [{
|
|
1524
1874
|
filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
|
|
@@ -1875,6 +2225,8 @@ async function postfetch(url, options = {}) {
|
|
|
1875
2225
|
return resolveFacebook(context);
|
|
1876
2226
|
case "instagram":
|
|
1877
2227
|
return resolveInstagram(context);
|
|
2228
|
+
case "linkedin":
|
|
2229
|
+
return resolveLinkedin(context);
|
|
1878
2230
|
case "pinterest":
|
|
1879
2231
|
return resolvePinterest(context);
|
|
1880
2232
|
case "reddit":
|
|
@@ -1897,6 +2249,9 @@ function detect(url) {
|
|
|
1897
2249
|
if (host.includes("instagram.com")) {
|
|
1898
2250
|
return "instagram";
|
|
1899
2251
|
}
|
|
2252
|
+
if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
|
|
2253
|
+
return "linkedin";
|
|
2254
|
+
}
|
|
1900
2255
|
if (host.includes("pinterest.") || host === "pin.it") {
|
|
1901
2256
|
return "pinterest";
|
|
1902
2257
|
}
|
|
@@ -1915,7 +2270,7 @@ function detect(url) {
|
|
|
1915
2270
|
if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
|
|
1916
2271
|
return "twitter";
|
|
1917
2272
|
}
|
|
1918
|
-
throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
2273
|
+
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
1919
2274
|
}
|
|
1920
2275
|
// src/remux.ts
|
|
1921
2276
|
function mergeAudioVideo(video, audio) {
|
package/dist/internal.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** A platform that {@link postfetch} can resolve. */
|
|
2
|
-
export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
2
|
+
export type Platform = "facebook" | "instagram" | "linkedin" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
3
3
|
/** The kind of a {@link MediaItem}. */
|
|
4
4
|
export type MediaKind = "audio" | "image" | "video";
|
|
5
5
|
/** A single resolved media file: a direct URL plus everything needed to fetch and name it. */
|
|
@@ -137,7 +137,7 @@ export type PinterestExtra = {
|
|
|
137
137
|
outboundLink?: string;
|
|
138
138
|
};
|
|
139
139
|
/** The result of resolving a post: its platform, id, media items and metadata. */
|
|
140
|
-
export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
|
|
140
|
+
export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"linkedin", never> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
|
|
141
141
|
/** One platform's variant of a {@link PostfetchResult}, carrying that platform's `extra` metadata shape. */
|
|
142
142
|
export type PlatformResult<P extends Platform, Extra> = {
|
|
143
143
|
/** Suggested filename when zipping every item. */
|
package/dist/postfetch.d.ts
CHANGED
|
@@ -14,7 +14,7 @@ export type PostfetchOptions = {
|
|
|
14
14
|
* performs no side effects beyond the lookup; use {@link download}, {@link archive}
|
|
15
15
|
* or {@link toResponse} to turn the result into bytes or a `Response`.
|
|
16
16
|
*
|
|
17
|
-
* @param url A post URL from
|
|
17
|
+
* @param url A post URL from a supported platform.
|
|
18
18
|
* @param options See {@link PostfetchOptions}.
|
|
19
19
|
* @returns The resolved post and its media items.
|
|
20
20
|
* @throws {PostfetchError} If the URL is empty or its host is unsupported.
|