@postfetch/core 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/download.d.ts +22 -5
- package/dist/index.d.ts +1 -1
- package/dist/index.js +140 -9
- package/dist/internal.d.ts +2 -2
- package/dist/mp4-remux.d.ts +12 -4
- package/package.json +1 -1
- package/src/download.ts +43 -9
- package/src/index.ts +1 -0
- package/src/internal.ts +2 -2
- package/src/mp4-remux.ts +106 -9
- package/src/twitter.ts +63 -1
package/README.md
CHANGED
|
@@ -16,10 +16,11 @@ for (const item of result.items) {
|
|
|
16
16
|
`downloadBlob(url, options?)` materializes an already-resolved media URL for
|
|
17
17
|
clients that need an uploadable `Blob`. Pass the resolved item's headers for
|
|
18
18
|
protected CDN URLs: `downloadBlob(item.url, { headers: item.headers })`. Set
|
|
19
|
-
`remux: true` to
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
19
|
+
`remux: true` to return `{ blob, thumbnail, width, height, duration }`: a
|
|
20
|
+
normalized MP4, upload thumbnail and calculated presentation metadata. Remuxing
|
|
21
|
+
defaults to off, uses `ffmpeg` and `ffprobe` from `PATH` unless their paths are
|
|
22
|
+
set, and throws if the complete result cannot be produced. The legacy
|
|
23
|
+
`(url, headers?, options?)` overload remains supported for non-remux downloads.
|
|
23
24
|
|
|
24
25
|
- `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
|
|
25
26
|
- Every request rotates a consistent browser/app fingerprint, so a fixed user-agent never gets the whole fleet blocked.
|
package/dist/download.d.ts
CHANGED
|
@@ -8,11 +8,21 @@ export type DownloadOptions = {
|
|
|
8
8
|
export type DownloadBlobOptions = DownloadOptions & {
|
|
9
9
|
/** Command used for opt-in MP4 remuxing. Defaults to `ffmpeg` from `PATH`. */
|
|
10
10
|
ffmpegPath?: string;
|
|
11
|
+
/** Command used to inspect remuxed MP4s. Defaults to `ffprobe` from `PATH`. */
|
|
12
|
+
ffprobePath?: string;
|
|
11
13
|
/** Headers required by the resolved media URL. */
|
|
12
14
|
headers?: HeadersInit;
|
|
13
15
|
/** Normalize an MP4 container with FFmpeg. Defaults to `false`. */
|
|
14
16
|
remux?: boolean;
|
|
15
17
|
};
|
|
18
|
+
/** A normalized video and upload metadata produced by `remux: true`. */
|
|
19
|
+
export type RemuxedVideo = {
|
|
20
|
+
blob: Blob;
|
|
21
|
+
duration: number;
|
|
22
|
+
height: number;
|
|
23
|
+
thumbnail: Blob;
|
|
24
|
+
width: number;
|
|
25
|
+
};
|
|
16
26
|
/** A zip archive produced by {@link archive}. */
|
|
17
27
|
export type Archive = {
|
|
18
28
|
/** The zip bytes (store mode, built in-process). */
|
|
@@ -46,20 +56,27 @@ export declare function download(item: MediaItem, options?: DownloadOptions): Pr
|
|
|
46
56
|
* passing its URL to a third party. When the resolved {@link MediaItem} carries
|
|
47
57
|
* required CDN headers, pass them in the options object. Set `remux: true` to
|
|
48
58
|
* normalize an MP4 with an FFmpeg stream copy; if FFmpeg is unavailable or the
|
|
49
|
-
* remux fails, the
|
|
59
|
+
* remux or metadata extraction fails, the operation throws.
|
|
50
60
|
*
|
|
51
61
|
* @param url A direct media URL returned by a resolver.
|
|
52
62
|
* @param options Headers, fetch implementation and opt-in remux behavior.
|
|
53
|
-
* @returns
|
|
63
|
+
* @returns A Blob, or with `remux: true`, the normalized video and its metadata.
|
|
54
64
|
*
|
|
55
65
|
* @example Upload a video with FormData
|
|
56
66
|
* ```ts
|
|
57
67
|
* const [media] = (await postfetch(sourceUrl)).items;
|
|
58
|
-
* const
|
|
59
|
-
* form.append("video", blob, media.filename);
|
|
68
|
+
* const video = await downloadBlob(media.url, { headers: media.headers, remux: true });
|
|
69
|
+
* form.append("video", video.blob, media.filename);
|
|
70
|
+
* form.append("thumbnail", video.thumbnail, "thumbnail.jpg");
|
|
60
71
|
* ```
|
|
61
72
|
*/
|
|
62
|
-
export declare function downloadBlob(url: string, options
|
|
73
|
+
export declare function downloadBlob(url: string, options: DownloadBlobOptions & {
|
|
74
|
+
remux: true;
|
|
75
|
+
}): Promise<RemuxedVideo>;
|
|
76
|
+
/** Download without remuxing and return the upstream media Blob. */
|
|
77
|
+
export declare function downloadBlob(url: string, options?: DownloadBlobOptions & {
|
|
78
|
+
remux?: false;
|
|
79
|
+
}): Promise<Blob>;
|
|
63
80
|
/** @deprecated Pass headers and fetch in a single {@link DownloadBlobOptions} object. */
|
|
64
81
|
export declare function downloadBlob(url: string, headers?: HeadersInit, options?: DownloadOptions): Promise<Blob>;
|
|
65
82
|
/**
|
package/dist/index.d.ts
CHANGED
|
@@ -20,6 +20,6 @@
|
|
|
20
20
|
* @module
|
|
21
21
|
*/
|
|
22
22
|
export { postfetch, detect, type PostfetchOptions } from "./postfetch";
|
|
23
|
-
export { download, downloadBlob, archive, toResponse, type Archive, type DownloadBlobOptions, type DownloadOptions, } from "./download";
|
|
23
|
+
export { download, downloadBlob, archive, toResponse, type Archive, type DownloadBlobOptions, type DownloadOptions, type RemuxedVideo, } from "./download";
|
|
24
24
|
export { PostfetchError } from "./internal";
|
|
25
25
|
export type { InstagramExtra, MediaItem, MediaKind, PinterestExtra, Platform, PlatformResult, PostAuthor, PostfetchReason, PostfetchResult, PostMetadata, RedditExtra, SoundcloudExtra, TiktokExtra, TwitterExtra, TwitterQuotedTweet, YoutubeExtra, } from "./internal";
|
package/dist/index.js
CHANGED
|
@@ -2009,6 +2009,7 @@ async function resolveTwitter(input) {
|
|
|
2009
2009
|
if (string(tweet.__typename) === "TweetTombstone") {
|
|
2010
2010
|
throw new Error("Tweet is unavailable or age-restricted");
|
|
2011
2011
|
}
|
|
2012
|
+
await expandTwitterText(input.net, tweet, id);
|
|
2012
2013
|
const groups = twitterTweets(tweet, id).flatMap(({ id: postId2, tweet: post }) => {
|
|
2013
2014
|
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
2014
2015
|
return media.flatMap((entry, index) => {
|
|
@@ -2032,7 +2033,7 @@ function twitterMetadata(tweet) {
|
|
|
2032
2033
|
extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
|
|
2033
2034
|
}
|
|
2034
2035
|
return {
|
|
2035
|
-
text:
|
|
2036
|
+
text: twitterText(tweet) ?? undefined,
|
|
2036
2037
|
author: user ? {
|
|
2037
2038
|
handle: string(user.screen_name) ?? undefined,
|
|
2038
2039
|
name: string(user.name) ?? undefined,
|
|
@@ -2047,6 +2048,57 @@ function twitterMetadata(tweet) {
|
|
|
2047
2048
|
extra
|
|
2048
2049
|
};
|
|
2049
2050
|
}
|
|
2051
|
+
function noteText(tweet) {
|
|
2052
|
+
const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
|
|
2053
|
+
const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
|
|
2054
|
+
const result = results && object(results.result) ? results.result : null;
|
|
2055
|
+
return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
|
|
2056
|
+
}
|
|
2057
|
+
function twitterText(tweet) {
|
|
2058
|
+
return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
|
|
2059
|
+
}
|
|
2060
|
+
function needsFullText(tweet) {
|
|
2061
|
+
if (noteText(tweet) || string(tweet.full_text)) {
|
|
2062
|
+
return false;
|
|
2063
|
+
}
|
|
2064
|
+
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
2065
|
+
}
|
|
2066
|
+
async function expandTwitterText(net, tweet, rootId) {
|
|
2067
|
+
const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
|
|
2068
|
+
for (const [id, post] of posts) {
|
|
2069
|
+
if (!needsFullText(post)) {
|
|
2070
|
+
continue;
|
|
2071
|
+
}
|
|
2072
|
+
try {
|
|
2073
|
+
const response = await net(`https://api.fxtwitter.com/status/${id}`, {
|
|
2074
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() }
|
|
2075
|
+
}, 1);
|
|
2076
|
+
if (!response.ok) {
|
|
2077
|
+
continue;
|
|
2078
|
+
}
|
|
2079
|
+
const payload = await response.json();
|
|
2080
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
|
|
2081
|
+
continue;
|
|
2082
|
+
}
|
|
2083
|
+
let current = payload.tweet;
|
|
2084
|
+
const seen = new Set;
|
|
2085
|
+
while (current) {
|
|
2086
|
+
const currentId = string(current.id);
|
|
2087
|
+
if (!currentId || seen.has(currentId)) {
|
|
2088
|
+
break;
|
|
2089
|
+
}
|
|
2090
|
+
seen.add(currentId);
|
|
2091
|
+
const target = posts.get(currentId);
|
|
2092
|
+
const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
|
|
2093
|
+
const text = raw ?? string(current.text);
|
|
2094
|
+
if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
|
|
2095
|
+
target.full_text = text;
|
|
2096
|
+
}
|
|
2097
|
+
current = object(current.quote) ? current.quote : null;
|
|
2098
|
+
}
|
|
2099
|
+
} catch {}
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2050
2102
|
function twitterTweets(tweet, rootId) {
|
|
2051
2103
|
const tweets = [];
|
|
2052
2104
|
const seen = new Set;
|
|
@@ -2514,7 +2566,7 @@ function detect(url) {
|
|
|
2514
2566
|
throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
2515
2567
|
}
|
|
2516
2568
|
// src/mp4-remux.ts
|
|
2517
|
-
async function
|
|
2569
|
+
async function prepareMp4(bytes, ffmpegPath = "ffmpeg", ffprobePath = "ffprobe", runtime) {
|
|
2518
2570
|
let directory;
|
|
2519
2571
|
let io;
|
|
2520
2572
|
try {
|
|
@@ -2549,7 +2601,53 @@ async function remuxMp4(bytes, ffmpegPath = "ffmpeg", runtime) {
|
|
|
2549
2601
|
return null;
|
|
2550
2602
|
}
|
|
2551
2603
|
const output = await io.readFile(outputPath);
|
|
2552
|
-
|
|
2604
|
+
if (output.length === 0) {
|
|
2605
|
+
return null;
|
|
2606
|
+
}
|
|
2607
|
+
const thumbnailPath = `${directory}/thumbnail.jpg`;
|
|
2608
|
+
const thumbnailResult = await io.run(ffmpegPath, [
|
|
2609
|
+
"-nostdin",
|
|
2610
|
+
"-hide_banner",
|
|
2611
|
+
"-loglevel",
|
|
2612
|
+
"error",
|
|
2613
|
+
"-y",
|
|
2614
|
+
"-i",
|
|
2615
|
+
outputPath,
|
|
2616
|
+
"-map",
|
|
2617
|
+
"0:v:0",
|
|
2618
|
+
"-frames:v",
|
|
2619
|
+
"1",
|
|
2620
|
+
"-vf",
|
|
2621
|
+
"scale=320:320:force_original_aspect_ratio=decrease",
|
|
2622
|
+
"-c:v",
|
|
2623
|
+
"mjpeg",
|
|
2624
|
+
"-q:v",
|
|
2625
|
+
"5",
|
|
2626
|
+
thumbnailPath
|
|
2627
|
+
]);
|
|
2628
|
+
if (!thumbnailResult.success) {
|
|
2629
|
+
return null;
|
|
2630
|
+
}
|
|
2631
|
+
const thumbnail = await io.readFile(thumbnailPath);
|
|
2632
|
+
if (!validThumbnail(thumbnail)) {
|
|
2633
|
+
return null;
|
|
2634
|
+
}
|
|
2635
|
+
const probe = await io.run(ffprobePath, [
|
|
2636
|
+
"-v",
|
|
2637
|
+
"error",
|
|
2638
|
+
"-select_streams",
|
|
2639
|
+
"v:0",
|
|
2640
|
+
"-show_entries",
|
|
2641
|
+
"stream=width,height,duration:stream_side_data=rotation:format=duration",
|
|
2642
|
+
"-of",
|
|
2643
|
+
"json",
|
|
2644
|
+
outputPath
|
|
2645
|
+
]);
|
|
2646
|
+
if (!probe.success) {
|
|
2647
|
+
return null;
|
|
2648
|
+
}
|
|
2649
|
+
const metadata = parseVideoMetadata(probe.stdout);
|
|
2650
|
+
return metadata ? { bytes: output, thumbnail, ...metadata } : null;
|
|
2553
2651
|
} catch {
|
|
2554
2652
|
return null;
|
|
2555
2653
|
} finally {
|
|
@@ -2560,6 +2658,26 @@ async function remuxMp4(bytes, ffmpegPath = "ffmpeg", runtime) {
|
|
|
2560
2658
|
}
|
|
2561
2659
|
}
|
|
2562
2660
|
}
|
|
2661
|
+
function validThumbnail(bytes) {
|
|
2662
|
+
return bytes.length >= 4 && bytes.length < 200000 && bytes[0] === 255 && bytes[1] === 216 && bytes[bytes.length - 2] === 255 && bytes[bytes.length - 1] === 217;
|
|
2663
|
+
}
|
|
2664
|
+
function parseVideoMetadata(value) {
|
|
2665
|
+
const parsed = JSON.parse(value);
|
|
2666
|
+
const stream = parsed.streams?.[0];
|
|
2667
|
+
const duration = Number(parsed.format?.duration ?? stream?.duration);
|
|
2668
|
+
const width = stream?.width ?? 0;
|
|
2669
|
+
const height2 = stream?.height ?? 0;
|
|
2670
|
+
if (!stream || !(duration > 0 && Number.isFinite(duration) && width > 0 && height2 > 0)) {
|
|
2671
|
+
return null;
|
|
2672
|
+
}
|
|
2673
|
+
const rotation = stream.side_data_list?.find((item) => item.rotation !== undefined)?.rotation ?? 0;
|
|
2674
|
+
const sideways = Math.abs(rotation) % 180 === 90;
|
|
2675
|
+
return {
|
|
2676
|
+
duration: Math.ceil(duration),
|
|
2677
|
+
height: sideways ? width : height2,
|
|
2678
|
+
width: sideways ? height2 : width
|
|
2679
|
+
};
|
|
2680
|
+
}
|
|
2563
2681
|
async function nodeRuntime() {
|
|
2564
2682
|
const [{ spawn }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join }] = await Promise.all([
|
|
2565
2683
|
import("node:child_process"),
|
|
@@ -2574,15 +2692,19 @@ async function nodeRuntime() {
|
|
|
2574
2692
|
await rm(path, { force: true, recursive: true });
|
|
2575
2693
|
},
|
|
2576
2694
|
run: (command, args) => new Promise((resolve2) => {
|
|
2577
|
-
const child = spawn(command, args, { stdio: ["ignore", "
|
|
2695
|
+
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
2578
2696
|
let stderr = "";
|
|
2697
|
+
let stdout = "";
|
|
2698
|
+
child.stdout?.on("data", (chunk) => {
|
|
2699
|
+
stdout += new TextDecoder().decode(chunk);
|
|
2700
|
+
});
|
|
2579
2701
|
child.stderr?.on("data", (chunk) => {
|
|
2580
2702
|
if (stderr.length < 4096) {
|
|
2581
2703
|
stderr += new TextDecoder().decode(chunk).slice(0, 4096 - stderr.length);
|
|
2582
2704
|
}
|
|
2583
2705
|
});
|
|
2584
|
-
child.once("error", (error) => resolve2({ stderr: error.message, success: false }));
|
|
2585
|
-
child.once("close", (code) => resolve2({ stderr, success: code === 0 }));
|
|
2706
|
+
child.once("error", (error) => resolve2({ stderr: error.message, stdout, success: false }));
|
|
2707
|
+
child.once("close", (code) => resolve2({ stderr, stdout, success: code === 0 }));
|
|
2586
2708
|
}),
|
|
2587
2709
|
writeFile: async (path, value) => {
|
|
2588
2710
|
await writeFile(path, value);
|
|
@@ -2838,15 +2960,24 @@ async function downloadBlob(url, optionsOrHeaders = {}, legacyOptions = {}) {
|
|
|
2838
2960
|
if (!modern || optionsOrHeaders.remux !== true) {
|
|
2839
2961
|
return blob;
|
|
2840
2962
|
}
|
|
2841
|
-
const
|
|
2842
|
-
|
|
2963
|
+
const prepared = await prepareMp4(new Uint8Array(await blob.arrayBuffer()), optionsOrHeaders.ffmpegPath, optionsOrHeaders.ffprobePath);
|
|
2964
|
+
if (!prepared) {
|
|
2965
|
+
throw new PostfetchError(500, "MP4 remux or metadata extraction failed");
|
|
2966
|
+
}
|
|
2967
|
+
return {
|
|
2968
|
+
blob: new Blob([toArrayBuffer(prepared.bytes)], { type: blob.type || "video/mp4" }),
|
|
2969
|
+
duration: prepared.duration,
|
|
2970
|
+
height: prepared.height,
|
|
2971
|
+
thumbnail: new Blob([toArrayBuffer(prepared.thumbnail)], { type: "image/jpeg" }),
|
|
2972
|
+
width: prepared.width
|
|
2973
|
+
};
|
|
2843
2974
|
}
|
|
2844
2975
|
function isDownloadBlobOptions(value) {
|
|
2845
2976
|
if (value instanceof Headers || Array.isArray(value)) {
|
|
2846
2977
|
return false;
|
|
2847
2978
|
}
|
|
2848
2979
|
const candidate = value;
|
|
2849
|
-
return "headers" in candidate && typeof candidate.headers !== "string" || typeof candidate.remux === "boolean" || typeof candidate.ffmpegPath === "string" || typeof candidate.fetch === "function";
|
|
2980
|
+
return "headers" in candidate && typeof candidate.headers !== "string" || typeof candidate.remux === "boolean" || typeof candidate.ffmpegPath === "string" || typeof candidate.ffprobePath === "string" || typeof candidate.fetch === "function";
|
|
2850
2981
|
}
|
|
2851
2982
|
async function archive(result, options = {}) {
|
|
2852
2983
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
package/dist/internal.d.ts
CHANGED
|
@@ -47,8 +47,8 @@ export type PostAuthor = {
|
|
|
47
47
|
};
|
|
48
48
|
/**
|
|
49
49
|
* Post metadata normalized across platforms. Every field is optional: an absent
|
|
50
|
-
* field means the
|
|
51
|
-
*
|
|
50
|
+
* field means the sources consulted did not expose it (never a fabricated
|
|
51
|
+
* default — a missing count is `undefined`,
|
|
52
52
|
* not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
|
|
53
53
|
*/
|
|
54
54
|
export type PostMetadata = {
|
package/dist/mp4-remux.d.ts
CHANGED
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
type CommandResult = {
|
|
2
2
|
stderr: string;
|
|
3
|
+
stdout: string;
|
|
3
4
|
success: boolean;
|
|
4
5
|
};
|
|
6
|
+
export type PreparedMp4 = {
|
|
7
|
+
bytes: Uint8Array;
|
|
8
|
+
duration: number;
|
|
9
|
+
height: number;
|
|
10
|
+
thumbnail: Uint8Array;
|
|
11
|
+
width: number;
|
|
12
|
+
};
|
|
5
13
|
export type Mp4RemuxRuntime = {
|
|
6
14
|
makeTempDir(prefix: string): Promise<string>;
|
|
7
15
|
readFile(path: string): Promise<Uint8Array>;
|
|
@@ -10,9 +18,9 @@ export type Mp4RemuxRuntime = {
|
|
|
10
18
|
writeFile(path: string, bytes: Uint8Array): Promise<void>;
|
|
11
19
|
};
|
|
12
20
|
/**
|
|
13
|
-
* Normalize an MP4
|
|
14
|
-
*
|
|
15
|
-
*
|
|
21
|
+
* Normalize an MP4, extract a Telegram-compatible JPEG thumbnail, and calculate
|
|
22
|
+
* the presentation dimensions and duration. Encoded media is left untouched;
|
|
23
|
+
* FFmpeg only rebuilds the container and timing tables.
|
|
16
24
|
*/
|
|
17
|
-
export declare function
|
|
25
|
+
export declare function prepareMp4(bytes: Uint8Array, ffmpegPath?: string, ffprobePath?: string, runtime?: Mp4RemuxRuntime): Promise<PreparedMp4 | null>;
|
|
18
26
|
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.1",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/download.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { assembleHls } from "./hls";
|
|
2
2
|
import { createNet, PostfetchError, type MediaItem, type Net, type PostfetchResult } from "./internal";
|
|
3
|
-
import {
|
|
3
|
+
import { prepareMp4 } from "./mp4-remux";
|
|
4
4
|
import { mergeAudioVideo } from "./remux";
|
|
5
5
|
import { zip } from "./zip";
|
|
6
6
|
|
|
@@ -20,12 +20,23 @@ export type DownloadOptions = {
|
|
|
20
20
|
export type DownloadBlobOptions = DownloadOptions & {
|
|
21
21
|
/** Command used for opt-in MP4 remuxing. Defaults to `ffmpeg` from `PATH`. */
|
|
22
22
|
ffmpegPath?: string;
|
|
23
|
+
/** Command used to inspect remuxed MP4s. Defaults to `ffprobe` from `PATH`. */
|
|
24
|
+
ffprobePath?: string;
|
|
23
25
|
/** Headers required by the resolved media URL. */
|
|
24
26
|
headers?: HeadersInit;
|
|
25
27
|
/** Normalize an MP4 container with FFmpeg. Defaults to `false`. */
|
|
26
28
|
remux?: boolean;
|
|
27
29
|
};
|
|
28
30
|
|
|
31
|
+
/** A normalized video and upload metadata produced by `remux: true`. */
|
|
32
|
+
export type RemuxedVideo = {
|
|
33
|
+
blob: Blob;
|
|
34
|
+
duration: number;
|
|
35
|
+
height: number;
|
|
36
|
+
thumbnail: Blob;
|
|
37
|
+
width: number;
|
|
38
|
+
};
|
|
39
|
+
|
|
29
40
|
/** A zip archive produced by {@link archive}. */
|
|
30
41
|
export type Archive = {
|
|
31
42
|
/** The zip bytes (store mode, built in-process). */
|
|
@@ -72,27 +83,36 @@ export async function download(item: MediaItem, options: DownloadOptions = {}):
|
|
|
72
83
|
* passing its URL to a third party. When the resolved {@link MediaItem} carries
|
|
73
84
|
* required CDN headers, pass them in the options object. Set `remux: true` to
|
|
74
85
|
* normalize an MP4 with an FFmpeg stream copy; if FFmpeg is unavailable or the
|
|
75
|
-
* remux fails, the
|
|
86
|
+
* remux or metadata extraction fails, the operation throws.
|
|
76
87
|
*
|
|
77
88
|
* @param url A direct media URL returned by a resolver.
|
|
78
89
|
* @param options Headers, fetch implementation and opt-in remux behavior.
|
|
79
|
-
* @returns
|
|
90
|
+
* @returns A Blob, or with `remux: true`, the normalized video and its metadata.
|
|
80
91
|
*
|
|
81
92
|
* @example Upload a video with FormData
|
|
82
93
|
* ```ts
|
|
83
94
|
* const [media] = (await postfetch(sourceUrl)).items;
|
|
84
|
-
* const
|
|
85
|
-
* form.append("video", blob, media.filename);
|
|
95
|
+
* const video = await downloadBlob(media.url, { headers: media.headers, remux: true });
|
|
96
|
+
* form.append("video", video.blob, media.filename);
|
|
97
|
+
* form.append("thumbnail", video.thumbnail, "thumbnail.jpg");
|
|
86
98
|
* ```
|
|
87
99
|
*/
|
|
88
|
-
export async function downloadBlob(
|
|
100
|
+
export async function downloadBlob(
|
|
101
|
+
url: string,
|
|
102
|
+
options: DownloadBlobOptions & { remux: true },
|
|
103
|
+
): Promise<RemuxedVideo>;
|
|
104
|
+
/** Download without remuxing and return the upstream media Blob. */
|
|
105
|
+
export async function downloadBlob(
|
|
106
|
+
url: string,
|
|
107
|
+
options?: DownloadBlobOptions & { remux?: false },
|
|
108
|
+
): Promise<Blob>;
|
|
89
109
|
/** @deprecated Pass headers and fetch in a single {@link DownloadBlobOptions} object. */
|
|
90
110
|
export async function downloadBlob(url: string, headers?: HeadersInit, options?: DownloadOptions): Promise<Blob>;
|
|
91
111
|
export async function downloadBlob(
|
|
92
112
|
url: string,
|
|
93
113
|
optionsOrHeaders: DownloadBlobOptions | HeadersInit = {},
|
|
94
114
|
legacyOptions: DownloadOptions = {},
|
|
95
|
-
): Promise<Blob> {
|
|
115
|
+
): Promise<Blob | RemuxedVideo> {
|
|
96
116
|
const modern = isDownloadBlobOptions(optionsOrHeaders);
|
|
97
117
|
const options = modern ? optionsOrHeaders : legacyOptions;
|
|
98
118
|
const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
|
|
@@ -105,8 +125,21 @@ export async function downloadBlob(
|
|
|
105
125
|
if (!modern || optionsOrHeaders.remux !== true) {
|
|
106
126
|
return blob;
|
|
107
127
|
}
|
|
108
|
-
const
|
|
109
|
-
|
|
128
|
+
const prepared = await prepareMp4(
|
|
129
|
+
new Uint8Array(await blob.arrayBuffer()),
|
|
130
|
+
optionsOrHeaders.ffmpegPath,
|
|
131
|
+
optionsOrHeaders.ffprobePath,
|
|
132
|
+
);
|
|
133
|
+
if (!prepared) {
|
|
134
|
+
throw new PostfetchError(500, "MP4 remux or metadata extraction failed");
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
blob: new Blob([toArrayBuffer(prepared.bytes)], { type: blob.type || "video/mp4" }),
|
|
138
|
+
duration: prepared.duration,
|
|
139
|
+
height: prepared.height,
|
|
140
|
+
thumbnail: new Blob([toArrayBuffer(prepared.thumbnail)], { type: "image/jpeg" }),
|
|
141
|
+
width: prepared.width,
|
|
142
|
+
};
|
|
110
143
|
}
|
|
111
144
|
|
|
112
145
|
function isDownloadBlobOptions(value: DownloadBlobOptions | HeadersInit): value is DownloadBlobOptions {
|
|
@@ -118,6 +151,7 @@ function isDownloadBlobOptions(value: DownloadBlobOptions | HeadersInit): value
|
|
|
118
151
|
("headers" in candidate && typeof candidate.headers !== "string") ||
|
|
119
152
|
typeof candidate.remux === "boolean" ||
|
|
120
153
|
typeof candidate.ffmpegPath === "string" ||
|
|
154
|
+
typeof candidate.ffprobePath === "string" ||
|
|
121
155
|
typeof candidate.fetch === "function"
|
|
122
156
|
);
|
|
123
157
|
}
|
package/src/index.ts
CHANGED
package/src/internal.ts
CHANGED
|
@@ -48,8 +48,8 @@ export type PostAuthor = {
|
|
|
48
48
|
|
|
49
49
|
/**
|
|
50
50
|
* Post metadata normalized across platforms. Every field is optional: an absent
|
|
51
|
-
* field means the
|
|
52
|
-
*
|
|
51
|
+
* field means the sources consulted did not expose it (never a fabricated
|
|
52
|
+
* default — a missing count is `undefined`,
|
|
53
53
|
* not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
|
|
54
54
|
*/
|
|
55
55
|
export type PostMetadata = {
|
package/src/mp4-remux.ts
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
1
|
type CommandResult = {
|
|
2
2
|
stderr: string;
|
|
3
|
+
stdout: string;
|
|
3
4
|
success: boolean;
|
|
4
5
|
};
|
|
5
6
|
|
|
7
|
+
export type PreparedMp4 = {
|
|
8
|
+
bytes: Uint8Array;
|
|
9
|
+
duration: number;
|
|
10
|
+
height: number;
|
|
11
|
+
thumbnail: Uint8Array;
|
|
12
|
+
width: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
6
15
|
export type Mp4RemuxRuntime = {
|
|
7
16
|
makeTempDir(prefix: string): Promise<string>;
|
|
8
17
|
readFile(path: string): Promise<Uint8Array>;
|
|
@@ -12,15 +21,16 @@ export type Mp4RemuxRuntime = {
|
|
|
12
21
|
};
|
|
13
22
|
|
|
14
23
|
/**
|
|
15
|
-
* Normalize an MP4
|
|
16
|
-
*
|
|
17
|
-
*
|
|
24
|
+
* Normalize an MP4, extract a Telegram-compatible JPEG thumbnail, and calculate
|
|
25
|
+
* the presentation dimensions and duration. Encoded media is left untouched;
|
|
26
|
+
* FFmpeg only rebuilds the container and timing tables.
|
|
18
27
|
*/
|
|
19
|
-
export async function
|
|
28
|
+
export async function prepareMp4(
|
|
20
29
|
bytes: Uint8Array,
|
|
21
30
|
ffmpegPath = "ffmpeg",
|
|
31
|
+
ffprobePath = "ffprobe",
|
|
22
32
|
runtime?: Mp4RemuxRuntime,
|
|
23
|
-
): Promise<
|
|
33
|
+
): Promise<PreparedMp4 | null> {
|
|
24
34
|
let directory: string | undefined;
|
|
25
35
|
let io: Mp4RemuxRuntime | undefined;
|
|
26
36
|
try {
|
|
@@ -55,7 +65,53 @@ export async function remuxMp4(
|
|
|
55
65
|
return null;
|
|
56
66
|
}
|
|
57
67
|
const output = await io.readFile(outputPath);
|
|
58
|
-
|
|
68
|
+
if (output.length === 0) {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
const thumbnailPath = `${directory}/thumbnail.jpg`;
|
|
72
|
+
const thumbnailResult = await io.run(ffmpegPath, [
|
|
73
|
+
"-nostdin",
|
|
74
|
+
"-hide_banner",
|
|
75
|
+
"-loglevel",
|
|
76
|
+
"error",
|
|
77
|
+
"-y",
|
|
78
|
+
"-i",
|
|
79
|
+
outputPath,
|
|
80
|
+
"-map",
|
|
81
|
+
"0:v:0",
|
|
82
|
+
"-frames:v",
|
|
83
|
+
"1",
|
|
84
|
+
"-vf",
|
|
85
|
+
"scale=320:320:force_original_aspect_ratio=decrease",
|
|
86
|
+
"-c:v",
|
|
87
|
+
"mjpeg",
|
|
88
|
+
"-q:v",
|
|
89
|
+
"5",
|
|
90
|
+
thumbnailPath,
|
|
91
|
+
]);
|
|
92
|
+
if (!thumbnailResult.success) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
const thumbnail = await io.readFile(thumbnailPath);
|
|
96
|
+
if (!validThumbnail(thumbnail)) {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
const probe = await io.run(ffprobePath, [
|
|
100
|
+
"-v",
|
|
101
|
+
"error",
|
|
102
|
+
"-select_streams",
|
|
103
|
+
"v:0",
|
|
104
|
+
"-show_entries",
|
|
105
|
+
"stream=width,height,duration:stream_side_data=rotation:format=duration",
|
|
106
|
+
"-of",
|
|
107
|
+
"json",
|
|
108
|
+
outputPath,
|
|
109
|
+
]);
|
|
110
|
+
if (!probe.success) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
const metadata = parseVideoMetadata(probe.stdout);
|
|
114
|
+
return metadata ? { bytes: output, thumbnail, ...metadata } : null;
|
|
59
115
|
} catch {
|
|
60
116
|
return null;
|
|
61
117
|
} finally {
|
|
@@ -65,6 +121,43 @@ export async function remuxMp4(
|
|
|
65
121
|
}
|
|
66
122
|
}
|
|
67
123
|
|
|
124
|
+
function validThumbnail(bytes: Uint8Array): boolean {
|
|
125
|
+
return (
|
|
126
|
+
bytes.length >= 4 &&
|
|
127
|
+
bytes.length < 200_000 &&
|
|
128
|
+
bytes[0] === 0xff &&
|
|
129
|
+
bytes[1] === 0xd8 &&
|
|
130
|
+
bytes[bytes.length - 2] === 0xff &&
|
|
131
|
+
bytes[bytes.length - 1] === 0xd9
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function parseVideoMetadata(value: string): Pick<PreparedMp4, "duration" | "height" | "width"> | null {
|
|
136
|
+
const parsed = JSON.parse(value) as {
|
|
137
|
+
format?: { duration?: string };
|
|
138
|
+
streams?: Array<{
|
|
139
|
+
duration?: string;
|
|
140
|
+
height?: number;
|
|
141
|
+
side_data_list?: Array<{ rotation?: number }>;
|
|
142
|
+
width?: number;
|
|
143
|
+
}>;
|
|
144
|
+
};
|
|
145
|
+
const stream = parsed.streams?.[0];
|
|
146
|
+
const duration = Number(parsed.format?.duration ?? stream?.duration);
|
|
147
|
+
const width = stream?.width ?? 0;
|
|
148
|
+
const height = stream?.height ?? 0;
|
|
149
|
+
if (!stream || !(duration > 0 && Number.isFinite(duration) && width > 0 && height > 0)) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
const rotation = stream.side_data_list?.find((item) => item.rotation !== undefined)?.rotation ?? 0;
|
|
153
|
+
const sideways = Math.abs(rotation) % 180 === 90;
|
|
154
|
+
return {
|
|
155
|
+
duration: Math.ceil(duration),
|
|
156
|
+
height: sideways ? width : height,
|
|
157
|
+
width: sideways ? height : width,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
68
161
|
// Deno, Bun and Node all expose the Node compatibility modules used here. They
|
|
69
162
|
// stay behind dynamic imports so browser/edge consumers that leave remux off do
|
|
70
163
|
// not load or execute any process/filesystem code.
|
|
@@ -83,15 +176,19 @@ async function nodeRuntime(): Promise<Mp4RemuxRuntime> {
|
|
|
83
176
|
},
|
|
84
177
|
run: (command, args) =>
|
|
85
178
|
new Promise((resolve) => {
|
|
86
|
-
const child = spawn(command, args, { stdio: ["ignore", "
|
|
179
|
+
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
87
180
|
let stderr = "";
|
|
181
|
+
let stdout = "";
|
|
182
|
+
child.stdout?.on("data", (chunk: Uint8Array) => {
|
|
183
|
+
stdout += new TextDecoder().decode(chunk);
|
|
184
|
+
});
|
|
88
185
|
child.stderr?.on("data", (chunk: Uint8Array) => {
|
|
89
186
|
if (stderr.length < 4096) {
|
|
90
187
|
stderr += new TextDecoder().decode(chunk).slice(0, 4096 - stderr.length);
|
|
91
188
|
}
|
|
92
189
|
});
|
|
93
|
-
child.once("error", (error) => resolve({ stderr: error.message, success: false }));
|
|
94
|
-
child.once("close", (code) => resolve({ stderr, success: code === 0 }));
|
|
190
|
+
child.once("error", (error) => resolve({ stderr: error.message, stdout, success: false }));
|
|
191
|
+
child.once("close", (code) => resolve({ stderr, stdout, success: code === 0 }));
|
|
95
192
|
}),
|
|
96
193
|
writeFile: async (path, value) => {
|
|
97
194
|
await writeFile(path, value);
|
package/src/twitter.ts
CHANGED
|
@@ -31,6 +31,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
|
|
|
31
31
|
if (string(tweet.__typename) === "TweetTombstone") {
|
|
32
32
|
throw new Error("Tweet is unavailable or age-restricted");
|
|
33
33
|
}
|
|
34
|
+
await expandTwitterText(input.net, tweet, id);
|
|
34
35
|
const groups = twitterTweets(tweet, id).flatMap(({ id: postId, tweet: post }) => {
|
|
35
36
|
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
36
37
|
return media.flatMap((entry, index) => {
|
|
@@ -58,7 +59,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
|
|
|
58
59
|
extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
|
|
59
60
|
}
|
|
60
61
|
return {
|
|
61
|
-
text: (
|
|
62
|
+
text: twitterText(tweet) ?? undefined,
|
|
62
63
|
author: user
|
|
63
64
|
? {
|
|
64
65
|
handle: string(user.screen_name) ?? undefined,
|
|
@@ -76,6 +77,67 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
|
|
|
76
77
|
};
|
|
77
78
|
}
|
|
78
79
|
|
|
80
|
+
function noteText(tweet: Json): string | null {
|
|
81
|
+
const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
|
|
82
|
+
const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
|
|
83
|
+
const result = results && object(results.result) ? results.result : null;
|
|
84
|
+
return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function twitterText(tweet: Json): string | null {
|
|
88
|
+
return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function needsFullText(tweet: Json): boolean {
|
|
92
|
+
if (noteText(tweet) || string(tweet.full_text)) {
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
// Embedded quotes can omit note_tweet entirely. Near-limit text is only a
|
|
96
|
+
// hint: ordinary long tweets may also incur a lookup, but are never shortened.
|
|
97
|
+
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
|
|
101
|
+
const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
|
|
102
|
+
for (const [id, post] of posts) {
|
|
103
|
+
if (!needsFullText(post)) {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
try {
|
|
107
|
+
// FxTwitter's public v1 API includes full note text and embedded quotes.
|
|
108
|
+
// Keep media and other metadata from X, and don't retry an optional lookup.
|
|
109
|
+
const response = await net(`https://api.fxtwitter.com/status/${id}`, {
|
|
110
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() },
|
|
111
|
+
}, 1);
|
|
112
|
+
if (!response.ok) {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const payload: unknown = await response.json();
|
|
116
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
let current: Json | null = payload.tweet;
|
|
120
|
+
const seen = new Set<string>();
|
|
121
|
+
while (current) {
|
|
122
|
+
const currentId = string(current.id);
|
|
123
|
+
if (!currentId || seen.has(currentId)) {
|
|
124
|
+
break;
|
|
125
|
+
}
|
|
126
|
+
seen.add(currentId);
|
|
127
|
+
const target = posts.get(currentId);
|
|
128
|
+
const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
|
|
129
|
+
const text = raw ?? string(current.text);
|
|
130
|
+
if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
|
|
131
|
+
target.full_text = text;
|
|
132
|
+
}
|
|
133
|
+
current = object(current.quote) ? current.quote : null;
|
|
134
|
+
}
|
|
135
|
+
} catch {
|
|
136
|
+
// A text enrichment outage must not discard a valid syndication result.
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
79
141
|
function twitterTweets(tweet: Json, rootId: string): Array<{ id: string; tweet: Json }> {
|
|
80
142
|
const tweets: Array<{ id: string; tweet: Json }> = [];
|
|
81
143
|
const seen = new Set<string>();
|