@postfetch/core 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/download.d.ts +39 -0
- package/dist/hls.d.ts +15 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.js +958 -21
- package/dist/internal.d.ts +44 -2
- package/dist/isobmff.d.ts +20 -0
- package/dist/pinterest.d.ts +2 -0
- package/dist/postfetch.d.ts +31 -0
- package/dist/reddit.d.ts +2 -0
- package/dist/remux.d.ts +8 -0
- package/dist/soundcloud.d.ts +2 -0
- package/package.json +5 -2
- package/src/download.ts +104 -10
- package/src/hls.ts +120 -0
- package/src/index.ts +21 -0
- package/src/internal.ts +38 -1
- package/src/isobmff.ts +85 -0
- package/src/pinterest.ts +237 -0
- package/src/postfetch.ts +50 -1
- package/src/reddit.ts +293 -0
- package/src/remux.ts +172 -0
- package/src/soundcloud.ts +127 -0
- package/src/youtube.ts +54 -5
package/dist/internal.d.ts
CHANGED
|
@@ -1,18 +1,50 @@
|
|
|
1
|
-
|
|
1
|
+
/** A platform that {@link postfetch} can resolve. */
|
|
2
|
+
export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
3
|
+
/** The kind of a {@link MediaItem}. */
|
|
2
4
|
export type MediaKind = "audio" | "image" | "video";
|
|
5
|
+
/** A single resolved media file: a direct URL plus everything needed to fetch and name it. */
|
|
3
6
|
export type MediaItem = {
|
|
7
|
+
/**
|
|
8
|
+
* When present, this is a video whose audio is a separate stream: {@link MediaItem.url}
|
|
9
|
+
* carries the video track and `audio.url` the audio track (DASH/CMAF hands them out
|
|
10
|
+
* apart). {@link download}, {@link archive} and {@link toResponse} merge the two with
|
|
11
|
+
* the bundled remuxer; callers that read `url` directly get the silent video.
|
|
12
|
+
*/
|
|
13
|
+
audio?: {
|
|
14
|
+
headers: HeadersInit;
|
|
15
|
+
url: string;
|
|
16
|
+
};
|
|
17
|
+
/** Suggested download filename. */
|
|
4
18
|
filename: string;
|
|
19
|
+
/** Headers required to fetch {@link MediaItem.url} from the CDN. */
|
|
5
20
|
headers: HeadersInit;
|
|
21
|
+
/**
|
|
22
|
+
* When `true`, {@link MediaItem.url} (and `audio.url`, if set) are HLS media
|
|
23
|
+
* playlists rather than direct files: {@link download} assembles their segments
|
|
24
|
+
* into a fragmented MP4 before serving — and merges video with audio when both
|
|
25
|
+
* are present. Callers that read `url` directly get the playlist, not the media.
|
|
26
|
+
*/
|
|
27
|
+
hls?: boolean;
|
|
28
|
+
/** Platform media id. */
|
|
6
29
|
id: string;
|
|
30
|
+
/** Whether this item is a video, image or audio file. */
|
|
7
31
|
kind: MediaKind;
|
|
32
|
+
/** MIME type of the media. */
|
|
8
33
|
mime: string;
|
|
34
|
+
/** The platform this item came from. */
|
|
9
35
|
platform: Platform;
|
|
36
|
+
/** Direct CDN URL of the media. */
|
|
10
37
|
url: string;
|
|
11
38
|
};
|
|
39
|
+
/** The result of resolving a post: its platform, id and one or more media items. */
|
|
12
40
|
export type PostfetchResult = {
|
|
41
|
+
/** Suggested filename when zipping every item. */
|
|
13
42
|
archiveFilename: string;
|
|
43
|
+
/** Platform post/media id. */
|
|
14
44
|
id: string;
|
|
45
|
+
/** The post's media, in order. */
|
|
15
46
|
items: MediaItem[];
|
|
47
|
+
/** The platform the post came from. */
|
|
16
48
|
platform: Platform;
|
|
17
49
|
};
|
|
18
50
|
export type Net = (url: string, init?: RequestInit, attempts?: number) => Promise<Response>;
|
|
@@ -21,9 +53,19 @@ export type ResolveContext = {
|
|
|
21
53
|
preferredWidth: number;
|
|
22
54
|
url: string;
|
|
23
55
|
};
|
|
56
|
+
/** An error carrying an HTTP status, so adapters can map failures onto responses. */
|
|
24
57
|
export declare class PostfetchError extends Error {
|
|
58
|
+
/** HTTP status that best describes the failure. */
|
|
25
59
|
readonly status: number;
|
|
26
|
-
|
|
60
|
+
/**
|
|
61
|
+
* Create an error tagged with an HTTP status.
|
|
62
|
+
*
|
|
63
|
+
* @param status HTTP status that best describes the failure.
|
|
64
|
+
* @param message Human-readable message.
|
|
65
|
+
*/
|
|
66
|
+
constructor(
|
|
67
|
+
/** HTTP status that best describes the failure. */
|
|
68
|
+
status: number, message: string);
|
|
27
69
|
}
|
|
28
70
|
export type Json = Record<string, unknown>;
|
|
29
71
|
export declare function createNet(baseFetch?: typeof fetch): Net;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export type Box = {
|
|
2
|
+
type: string;
|
|
3
|
+
/** Offset of the box header (its size field). */
|
|
4
|
+
start: number;
|
|
5
|
+
/** Offset just past the box. */
|
|
6
|
+
end: number;
|
|
7
|
+
/** Offset of the box payload (after the size/type, and any 64-bit largesize). */
|
|
8
|
+
dataStart: number;
|
|
9
|
+
};
|
|
10
|
+
export declare function parseBoxes(data: Uint8Array, start?: number, end?: number): Box[];
|
|
11
|
+
export declare function findBox(boxes: Box[], type: string): Box | undefined;
|
|
12
|
+
export declare function childBoxes(data: Uint8Array, box: Box): Box[];
|
|
13
|
+
/** A standalone copy of a box's bytes, header included. */
|
|
14
|
+
export declare function sliceBox(data: Uint8Array, box: Box): Uint8Array;
|
|
15
|
+
export declare function buildBox(type: string, payload: Uint8Array): Uint8Array;
|
|
16
|
+
export declare function concat(parts: Uint8Array[]): Uint8Array;
|
|
17
|
+
export declare function readU32(data: Uint8Array, offset: number): number;
|
|
18
|
+
export declare function writeU32(data: Uint8Array, offset: number, value: number): void;
|
|
19
|
+
/** The version byte of a full box (the first byte of its payload). */
|
|
20
|
+
export declare function boxVersion(data: Uint8Array, box: Box): number;
|
package/dist/postfetch.d.ts
CHANGED
|
@@ -1,7 +1,38 @@
|
|
|
1
1
|
import { type Platform, type PostfetchResult } from "./internal";
|
|
2
|
+
/** Options for {@link postfetch}. */
|
|
2
3
|
export type PostfetchOptions = {
|
|
4
|
+
/** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
|
|
3
5
|
fetch?: typeof fetch;
|
|
6
|
+
/** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
|
|
4
7
|
preferredWidth?: number;
|
|
5
8
|
};
|
|
9
|
+
/**
|
|
10
|
+
* Resolve a public social post URL to its media.
|
|
11
|
+
*
|
|
12
|
+
* Detects the platform from the URL host, then returns the post's media as typed
|
|
13
|
+
* {@link MediaItem}s — direct CDN URLs plus the headers needed to fetch them. It
|
|
14
|
+
* performs no side effects beyond the lookup; use {@link download}, {@link archive}
|
|
15
|
+
* or {@link toResponse} to turn the result into bytes or a `Response`.
|
|
16
|
+
*
|
|
17
|
+
* @param url A post URL from Instagram, TikTok, YouTube, Facebook or X.
|
|
18
|
+
* @param options See {@link PostfetchOptions}.
|
|
19
|
+
* @returns The resolved post and its media items.
|
|
20
|
+
* @throws {PostfetchError} If the URL is empty or its host is unsupported.
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* ```ts
|
|
24
|
+
* import { postfetch } from "@postfetch/core";
|
|
25
|
+
*
|
|
26
|
+
* const result = await postfetch("https://vt.tiktok.com/ZSxpHvCUM/");
|
|
27
|
+
* for (const item of result.items) console.log(item.kind, item.url);
|
|
28
|
+
* ```
|
|
29
|
+
*/
|
|
6
30
|
export declare function postfetch(url: string, options?: PostfetchOptions): Promise<PostfetchResult>;
|
|
31
|
+
/**
|
|
32
|
+
* Detect which platform a URL belongs to, from its host.
|
|
33
|
+
*
|
|
34
|
+
* @param url Any URL.
|
|
35
|
+
* @returns The matched {@link Platform}.
|
|
36
|
+
* @throws {PostfetchError} If the host is not a supported platform.
|
|
37
|
+
*/
|
|
7
38
|
export declare function detect(url: string): Platform;
|
package/dist/reddit.d.ts
ADDED
package/dist/remux.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Merge a video-only and an audio-only fragmented MP4 into one file with both
|
|
3
|
+
* tracks. The DASH/CMAF streams reddit and YouTube hand out keep video and audio
|
|
4
|
+
* apart; this recombines them without re-encoding — every `moof`/`mdat` pair is
|
|
5
|
+
* copied verbatim (so each `trun` data offset stays valid) and only the track id
|
|
6
|
+
* and fragment sequence number are patched.
|
|
7
|
+
*/
|
|
8
|
+
export declare function mergeAudioVideo(video: Uint8Array, audio: Uint8Array): Uint8Array;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Zero-dependency typed core that turns Facebook, Instagram, TikTok, X and YouTube post URLs into media files.",
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Zero-dependency typed core that turns Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"exports": {
|
|
@@ -31,6 +31,9 @@
|
|
|
31
31
|
"facebook",
|
|
32
32
|
"instagram",
|
|
33
33
|
"media",
|
|
34
|
+
"pinterest",
|
|
35
|
+
"reddit",
|
|
36
|
+
"soundcloud",
|
|
34
37
|
"tiktok",
|
|
35
38
|
"twitter",
|
|
36
39
|
"youtube",
|
package/src/download.ts
CHANGED
|
@@ -1,18 +1,52 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { assembleHls } from "./hls";
|
|
2
|
+
import { createNet, PostfetchError, type MediaItem, type Net, type PostfetchResult } from "./internal";
|
|
3
|
+
import { mergeAudioVideo } from "./remux";
|
|
2
4
|
import { zip } from "./zip";
|
|
3
5
|
|
|
6
|
+
// A muxed item carries a separate audio stream; an HLS item carries playlists.
|
|
7
|
+
// Either way the bytes must be assembled in-process rather than streamed.
|
|
8
|
+
function buffered(item: MediaItem): boolean {
|
|
9
|
+
return Boolean(item.audio) || item.hls === true;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Options for {@link download}, {@link archive} and {@link toResponse}. */
|
|
4
13
|
export type DownloadOptions = {
|
|
14
|
+
/** Custom `fetch` implementation. Defaults to the global `fetch`. */
|
|
5
15
|
fetch?: typeof fetch;
|
|
6
16
|
};
|
|
7
17
|
|
|
18
|
+
/** A zip archive produced by {@link archive}. */
|
|
8
19
|
export type Archive = {
|
|
20
|
+
/** The zip bytes (store mode, built in-process). */
|
|
9
21
|
bytes: Uint8Array;
|
|
22
|
+
/** Suggested download filename. */
|
|
10
23
|
filename: string;
|
|
24
|
+
/** Always `"application/zip"`. */
|
|
11
25
|
mime: "application/zip";
|
|
12
26
|
};
|
|
13
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Fetch a single media item, ready to stream or buffer. A plain item streams
|
|
30
|
+
* straight from its CDN; a muxed item (one with {@link MediaItem.audio}) is
|
|
31
|
+
* downloaded as two streams and merged in-process, so its body is buffered.
|
|
32
|
+
*
|
|
33
|
+
* @param item A {@link MediaItem} from a {@link PostfetchResult}.
|
|
34
|
+
* @param options See {@link DownloadOptions}.
|
|
35
|
+
* @returns The upstream `Response`, or a buffered `Response` of the merged file.
|
|
36
|
+
* @throws {PostfetchError} If the CDN responds with a non-OK status.
|
|
37
|
+
*
|
|
38
|
+
* @example
|
|
39
|
+
* ```ts
|
|
40
|
+
* const result = await postfetch(url);
|
|
41
|
+
* await Bun.write(result.items[0].filename, await download(result.items[0]));
|
|
42
|
+
* ```
|
|
43
|
+
*/
|
|
14
44
|
export async function download(item: MediaItem, options: DownloadOptions = {}): Promise<Response> {
|
|
15
45
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
46
|
+
if (buffered(item)) {
|
|
47
|
+
const bytes = await itemBytes(net, item);
|
|
48
|
+
return new Response(toArrayBuffer(bytes), { headers: { "content-length": String(bytes.length), "content-type": item.mime } });
|
|
49
|
+
}
|
|
16
50
|
const response = await net(item.url, { headers: item.headers });
|
|
17
51
|
if (!response.ok || !response.body) {
|
|
18
52
|
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
@@ -20,20 +54,31 @@ export async function download(item: MediaItem, options: DownloadOptions = {}):
|
|
|
20
54
|
return response;
|
|
21
55
|
}
|
|
22
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Download every item of a result and zip them in-process (store mode).
|
|
59
|
+
*
|
|
60
|
+
* @param result A {@link PostfetchResult}.
|
|
61
|
+
* @param options See {@link DownloadOptions}.
|
|
62
|
+
* @returns The zip bytes and a suggested filename.
|
|
63
|
+
*/
|
|
23
64
|
export async function archive(result: PostfetchResult, options: DownloadOptions = {}): Promise<Archive> {
|
|
24
65
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
25
66
|
const files = await Promise.all(
|
|
26
|
-
result.items.map(async (item) => {
|
|
27
|
-
const response = await net(item.url, { headers: item.headers });
|
|
28
|
-
if (!response.ok) {
|
|
29
|
-
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
30
|
-
}
|
|
31
|
-
return { data: new Uint8Array(await response.arrayBuffer()), name: item.filename };
|
|
32
|
-
}),
|
|
67
|
+
result.items.map(async (item) => ({ data: await itemBytes(net, item), name: item.filename })),
|
|
33
68
|
);
|
|
34
69
|
return { bytes: zip(files), filename: result.archiveFilename, mime: "application/zip" };
|
|
35
70
|
}
|
|
36
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Turn a result into a ready-to-serve `Response`: a single item is streamed as
|
|
74
|
+
* its file (or, when muxed, merged and buffered), multiple items become a zip.
|
|
75
|
+
* Sets `content-disposition` and the `x-media-*` headers. This is what the
|
|
76
|
+
* showcase server and templates return.
|
|
77
|
+
*
|
|
78
|
+
* @param result A {@link PostfetchResult}.
|
|
79
|
+
* @param options See {@link DownloadOptions}.
|
|
80
|
+
* @returns A `Response` a server or edge function can return directly.
|
|
81
|
+
*/
|
|
37
82
|
export async function toResponse(result: PostfetchResult, options: DownloadOptions = {}): Promise<Response> {
|
|
38
83
|
if (result.items.length === 1) {
|
|
39
84
|
return singleResponse(result.items[0], options);
|
|
@@ -42,6 +87,21 @@ export async function toResponse(result: PostfetchResult, options: DownloadOptio
|
|
|
42
87
|
}
|
|
43
88
|
|
|
44
89
|
async function singleResponse(item: MediaItem, options: DownloadOptions): Promise<Response> {
|
|
90
|
+
if (buffered(item)) {
|
|
91
|
+
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
92
|
+
const bytes = await itemBytes(net, item);
|
|
93
|
+
return new Response(toArrayBuffer(bytes), {
|
|
94
|
+
headers: {
|
|
95
|
+
"content-disposition": `attachment; filename="${item.filename}"`,
|
|
96
|
+
"content-length": String(bytes.length),
|
|
97
|
+
"content-type": item.mime,
|
|
98
|
+
"x-media-count": "1",
|
|
99
|
+
"x-media-id": item.id,
|
|
100
|
+
"x-media-kind": item.kind,
|
|
101
|
+
"x-media-platform": item.platform,
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
}
|
|
45
105
|
const media = await download(item, options);
|
|
46
106
|
const headers = new Headers({
|
|
47
107
|
"content-disposition": `attachment; filename="${item.filename}"`,
|
|
@@ -60,8 +120,7 @@ async function singleResponse(item: MediaItem, options: DownloadOptions): Promis
|
|
|
60
120
|
|
|
61
121
|
async function archiveResponse(result: PostfetchResult, options: DownloadOptions): Promise<Response> {
|
|
62
122
|
const { bytes, filename } = await archive(result, options);
|
|
63
|
-
|
|
64
|
-
return new Response(body, {
|
|
123
|
+
return new Response(toArrayBuffer(bytes), {
|
|
65
124
|
headers: {
|
|
66
125
|
"content-disposition": `attachment; filename="${filename}"`,
|
|
67
126
|
"content-length": String(bytes.length),
|
|
@@ -72,3 +131,38 @@ async function archiveResponse(result: PostfetchResult, options: DownloadOptions
|
|
|
72
131
|
},
|
|
73
132
|
});
|
|
74
133
|
}
|
|
134
|
+
|
|
135
|
+
async function itemBytes(net: Net, item: MediaItem): Promise<Uint8Array> {
|
|
136
|
+
if (item.hls) {
|
|
137
|
+
const video = await assembleHls(net, item.url, item.headers);
|
|
138
|
+
if (item.audio) {
|
|
139
|
+
return mergeAudioVideo(video, await assembleHls(net, item.audio.url, item.audio.headers));
|
|
140
|
+
}
|
|
141
|
+
return video;
|
|
142
|
+
}
|
|
143
|
+
if (item.audio) {
|
|
144
|
+
return mergedBytes(net, item, item.audio);
|
|
145
|
+
}
|
|
146
|
+
return fetchBytes(net, item.url, item.headers);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// A muxed item's two streams are fetched in parallel and merged in a single pass.
|
|
150
|
+
async function mergedBytes(net: Net, item: MediaItem, audio: NonNullable<MediaItem["audio"]>): Promise<Uint8Array> {
|
|
151
|
+
const [video, audioBytes] = await Promise.all([
|
|
152
|
+
fetchBytes(net, item.url, item.headers),
|
|
153
|
+
fetchBytes(net, audio.url, audio.headers),
|
|
154
|
+
]);
|
|
155
|
+
return mergeAudioVideo(video, audioBytes);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function fetchBytes(net: Net, url: string, headers: HeadersInit): Promise<Uint8Array> {
|
|
159
|
+
const response = await net(url, { headers });
|
|
160
|
+
if (!response.ok) {
|
|
161
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
162
|
+
}
|
|
163
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
|
|
167
|
+
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
|
168
|
+
}
|
package/src/hls.ts
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { concat } from "./isobmff";
|
|
2
|
+
import type { Net } from "./internal";
|
|
3
|
+
|
|
4
|
+
// HLS support for the fMP4/CMAF playlists reddit-style muxing already covers:
|
|
5
|
+
// a media playlist is an EXT-X-MAP init plus media segments (often byte ranges of
|
|
6
|
+
// one CMAF file), so assembling it is just fetching those byte ranges in order and
|
|
7
|
+
// concatenating them into a fragmented MP4. No MPEG-TS demuxing is involved.
|
|
8
|
+
|
|
9
|
+
type Segment = { url: string; range: { length: number; offset: number } | null };
|
|
10
|
+
|
|
11
|
+
export type HlsVariant = { width: number; height: number; bandwidth: number; url: string; audioGroup: string | null };
|
|
12
|
+
export type HlsMaster = { variants: HlsVariant[]; audio: Record<string, string> };
|
|
13
|
+
|
|
14
|
+
export function isMasterPlaylist(text: string): boolean {
|
|
15
|
+
return text.includes("#EXT-X-STREAM-INF");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function parseMaster(text: string, baseUrl: string): HlsMaster {
|
|
19
|
+
const lines = text.split("\n").map((line) => line.trim());
|
|
20
|
+
const audio: Record<string, string> = {};
|
|
21
|
+
const variants: HlsVariant[] = [];
|
|
22
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
23
|
+
const line = lines[index];
|
|
24
|
+
if (line.startsWith("#EXT-X-MEDIA:") && /TYPE=AUDIO/.test(line)) {
|
|
25
|
+
const group = line.match(/GROUP-ID="([^"]+)"/)?.[1];
|
|
26
|
+
const uri = line.match(/URI="([^"]+)"/)?.[1];
|
|
27
|
+
if (group && uri) {
|
|
28
|
+
audio[group] = new URL(uri, baseUrl).href;
|
|
29
|
+
}
|
|
30
|
+
} else if (line.startsWith("#EXT-X-STREAM-INF:")) {
|
|
31
|
+
const target = lines[index + 1];
|
|
32
|
+
if (target && !target.startsWith("#")) {
|
|
33
|
+
const resolution = line.match(/RESOLUTION=(\d+)x(\d+)/);
|
|
34
|
+
variants.push({
|
|
35
|
+
width: resolution ? Number(resolution[1]) : 0,
|
|
36
|
+
height: resolution ? Number(resolution[2]) : 0,
|
|
37
|
+
bandwidth: Number(line.match(/BANDWIDTH=(\d+)/)?.[1] ?? 0),
|
|
38
|
+
url: new URL(target, baseUrl).href,
|
|
39
|
+
audioGroup: line.match(/AUDIO="([^"]+)"/)?.[1] ?? null,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return { variants, audio };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function assembleHls(net: Net, playlistUrl: string, headers: HeadersInit): Promise<Uint8Array> {
|
|
48
|
+
const response = await net(playlistUrl, { headers });
|
|
49
|
+
if (!response.ok) {
|
|
50
|
+
throw new Error(`HLS playlist failed: ${response.status}`);
|
|
51
|
+
}
|
|
52
|
+
const segments = mediaSegments(await response.text(), playlistUrl);
|
|
53
|
+
if (segments.length === 0) {
|
|
54
|
+
throw new Error("HLS playlist has no segments");
|
|
55
|
+
}
|
|
56
|
+
return concat(await fetchSegments(net, segments, headers));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function mediaSegments(text: string, baseUrl: string): Segment[] {
|
|
60
|
+
const segments: Segment[] = [];
|
|
61
|
+
let pendingRange: { length: number; offset: number } | null = null;
|
|
62
|
+
let nextOffset = 0;
|
|
63
|
+
for (const raw of text.split("\n")) {
|
|
64
|
+
const line = raw.trim();
|
|
65
|
+
if (line.startsWith("#EXT-X-MAP:")) {
|
|
66
|
+
const uri = line.match(/URI="([^"]+)"/)?.[1];
|
|
67
|
+
if (uri) {
|
|
68
|
+
const range = byteRange(line.match(/BYTERANGE="([^"]+)"/)?.[1], nextOffset);
|
|
69
|
+
segments.push({ url: new URL(uri, baseUrl).href, range });
|
|
70
|
+
nextOffset = range ? range.offset + range.length : nextOffset;
|
|
71
|
+
}
|
|
72
|
+
} else if (line.startsWith("#EXT-X-BYTERANGE:")) {
|
|
73
|
+
pendingRange = byteRange(line.slice("#EXT-X-BYTERANGE:".length), nextOffset);
|
|
74
|
+
nextOffset = pendingRange ? pendingRange.offset + pendingRange.length : nextOffset;
|
|
75
|
+
} else if (line.length > 0 && !line.startsWith("#")) {
|
|
76
|
+
segments.push({ url: new URL(line, baseUrl).href, range: pendingRange });
|
|
77
|
+
pendingRange = null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return segments;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// EXT-X-BYTERANGE is `length[@offset]`; a missing offset continues from the end of
|
|
84
|
+
// the previous sub-range of the same resource.
|
|
85
|
+
function byteRange(spec: string | undefined, runningOffset: number): { length: number; offset: number } | null {
|
|
86
|
+
const match = spec?.trim().match(/^(\d+)(?:@(\d+))?$/);
|
|
87
|
+
if (!match) {
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
return { length: Number(match[1]), offset: match[2] === undefined ? runningOffset : Number(match[2]) };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Segments are fetched with a small concurrency window: parallel enough to be fast,
|
|
94
|
+
// bounded so a long playlist does not open hundreds of sockets at once.
|
|
95
|
+
async function fetchSegments(net: Net, segments: Segment[], headers: HeadersInit): Promise<Uint8Array[]> {
|
|
96
|
+
const results = new Array<Uint8Array>(segments.length);
|
|
97
|
+
let cursor = 0;
|
|
98
|
+
async function worker(): Promise<void> {
|
|
99
|
+
while (cursor < segments.length) {
|
|
100
|
+
const index = cursor;
|
|
101
|
+
cursor += 1;
|
|
102
|
+
results[index] = await fetchSegment(net, segments[index], headers);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const workers = Array.from({ length: Math.min(8, segments.length) }, worker);
|
|
106
|
+
await Promise.all(workers);
|
|
107
|
+
return results;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function fetchSegment(net: Net, segment: Segment, headers: HeadersInit): Promise<Uint8Array> {
|
|
111
|
+
const requestHeaders = new Headers(headers);
|
|
112
|
+
if (segment.range) {
|
|
113
|
+
requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
|
|
114
|
+
}
|
|
115
|
+
const response = await net(segment.url, { headers: requestHeaders });
|
|
116
|
+
if (!response.ok) {
|
|
117
|
+
throw new Error(`HLS segment failed: ${response.status}`);
|
|
118
|
+
}
|
|
119
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
120
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,3 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turn a public social post URL into its media.
|
|
3
|
+
*
|
|
4
|
+
* {@link postfetch} resolves Instagram, TikTok, YouTube, Facebook and X (Twitter)
|
|
5
|
+
* URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
|
|
6
|
+
* (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
|
|
7
|
+
* Cloudflare Workers and browsers alike. It only resolves the media; the
|
|
8
|
+
* {@link download}, {@link archive} and {@link toResponse} helpers turn a result
|
|
9
|
+
* into bytes or a `Response`.
|
|
10
|
+
*
|
|
11
|
+
* @example Resolve a reel and save the video
|
|
12
|
+
* ```ts
|
|
13
|
+
* import { postfetch, download } from "@postfetch/core";
|
|
14
|
+
*
|
|
15
|
+
* const result = await postfetch("https://www.instagram.com/reel/DZ0ixNxtvYq/");
|
|
16
|
+
* const [item] = result.items; // { kind: "video", url, headers, filename, ... }
|
|
17
|
+
* const response = await download(item);
|
|
18
|
+
* ```
|
|
19
|
+
*
|
|
20
|
+
* @module
|
|
21
|
+
*/
|
|
1
22
|
export { postfetch, detect, type PostfetchOptions } from "./postfetch";
|
|
2
23
|
export { download, archive, toResponse, type Archive, type DownloadOptions } from "./download";
|
|
3
24
|
export { PostfetchError } from "./internal";
|
package/src/internal.ts
CHANGED
|
@@ -1,21 +1,50 @@
|
|
|
1
|
-
|
|
1
|
+
/** A platform that {@link postfetch} can resolve. */
|
|
2
|
+
export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
|
|
2
3
|
|
|
4
|
+
/** The kind of a {@link MediaItem}. */
|
|
3
5
|
export type MediaKind = "audio" | "image" | "video";
|
|
4
6
|
|
|
7
|
+
/** A single resolved media file: a direct URL plus everything needed to fetch and name it. */
|
|
5
8
|
export type MediaItem = {
|
|
9
|
+
/**
|
|
10
|
+
* When present, this is a video whose audio is a separate stream: {@link MediaItem.url}
|
|
11
|
+
* carries the video track and `audio.url` the audio track (DASH/CMAF hands them out
|
|
12
|
+
* apart). {@link download}, {@link archive} and {@link toResponse} merge the two with
|
|
13
|
+
* the bundled remuxer; callers that read `url` directly get the silent video.
|
|
14
|
+
*/
|
|
15
|
+
audio?: { headers: HeadersInit; url: string };
|
|
16
|
+
/** Suggested download filename. */
|
|
6
17
|
filename: string;
|
|
18
|
+
/** Headers required to fetch {@link MediaItem.url} from the CDN. */
|
|
7
19
|
headers: HeadersInit;
|
|
20
|
+
/**
|
|
21
|
+
* When `true`, {@link MediaItem.url} (and `audio.url`, if set) are HLS media
|
|
22
|
+
* playlists rather than direct files: {@link download} assembles their segments
|
|
23
|
+
* into a fragmented MP4 before serving — and merges video with audio when both
|
|
24
|
+
* are present. Callers that read `url` directly get the playlist, not the media.
|
|
25
|
+
*/
|
|
26
|
+
hls?: boolean;
|
|
27
|
+
/** Platform media id. */
|
|
8
28
|
id: string;
|
|
29
|
+
/** Whether this item is a video, image or audio file. */
|
|
9
30
|
kind: MediaKind;
|
|
31
|
+
/** MIME type of the media. */
|
|
10
32
|
mime: string;
|
|
33
|
+
/** The platform this item came from. */
|
|
11
34
|
platform: Platform;
|
|
35
|
+
/** Direct CDN URL of the media. */
|
|
12
36
|
url: string;
|
|
13
37
|
};
|
|
14
38
|
|
|
39
|
+
/** The result of resolving a post: its platform, id and one or more media items. */
|
|
15
40
|
export type PostfetchResult = {
|
|
41
|
+
/** Suggested filename when zipping every item. */
|
|
16
42
|
archiveFilename: string;
|
|
43
|
+
/** Platform post/media id. */
|
|
17
44
|
id: string;
|
|
45
|
+
/** The post's media, in order. */
|
|
18
46
|
items: MediaItem[];
|
|
47
|
+
/** The platform the post came from. */
|
|
19
48
|
platform: Platform;
|
|
20
49
|
};
|
|
21
50
|
|
|
@@ -27,8 +56,16 @@ export type ResolveContext = {
|
|
|
27
56
|
url: string;
|
|
28
57
|
};
|
|
29
58
|
|
|
59
|
+
/** An error carrying an HTTP status, so adapters can map failures onto responses. */
|
|
30
60
|
export class PostfetchError extends Error {
|
|
61
|
+
/**
|
|
62
|
+
* Create an error tagged with an HTTP status.
|
|
63
|
+
*
|
|
64
|
+
* @param status HTTP status that best describes the failure.
|
|
65
|
+
* @param message Human-readable message.
|
|
66
|
+
*/
|
|
31
67
|
constructor(
|
|
68
|
+
/** HTTP status that best describes the failure. */
|
|
32
69
|
readonly status: number,
|
|
33
70
|
message: string,
|
|
34
71
|
) {
|
package/src/isobmff.ts
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// A minimal ISO Base Media File Format reader/writer — just enough to remux
|
|
2
|
+
// fragmented MP4 (CMAF/DASH) by recombining whole boxes. It never decodes media,
|
|
3
|
+
// only walks the box tree and patches a handful of integer fields in place.
|
|
4
|
+
|
|
5
|
+
export type Box = {
|
|
6
|
+
type: string;
|
|
7
|
+
/** Offset of the box header (its size field). */
|
|
8
|
+
start: number;
|
|
9
|
+
/** Offset just past the box. */
|
|
10
|
+
end: number;
|
|
11
|
+
/** Offset of the box payload (after the size/type, and any 64-bit largesize). */
|
|
12
|
+
dataStart: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export function parseBoxes(data: Uint8Array, start = 0, end = data.length): Box[] {
|
|
16
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
17
|
+
const boxes: Box[] = [];
|
|
18
|
+
let offset = start;
|
|
19
|
+
while (offset + 8 <= end) {
|
|
20
|
+
let size = view.getUint32(offset);
|
|
21
|
+
const type = String.fromCharCode(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]);
|
|
22
|
+
let dataStart = offset + 8;
|
|
23
|
+
if (size === 1) {
|
|
24
|
+
size = Number(view.getBigUint64(offset + 8));
|
|
25
|
+
dataStart = offset + 16;
|
|
26
|
+
} else if (size === 0) {
|
|
27
|
+
size = end - offset;
|
|
28
|
+
}
|
|
29
|
+
if (size < 8 || offset + size > end) {
|
|
30
|
+
break;
|
|
31
|
+
}
|
|
32
|
+
boxes.push({ type, start: offset, end: offset + size, dataStart });
|
|
33
|
+
offset += size;
|
|
34
|
+
}
|
|
35
|
+
return boxes;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function findBox(boxes: Box[], type: string): Box | undefined {
|
|
39
|
+
return boxes.find((box) => box.type === type);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function childBoxes(data: Uint8Array, box: Box): Box[] {
|
|
43
|
+
return parseBoxes(data, box.dataStart, box.end);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** A standalone copy of a box's bytes, header included. */
|
|
47
|
+
export function sliceBox(data: Uint8Array, box: Box): Uint8Array {
|
|
48
|
+
return data.slice(box.start, box.end);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function buildBox(type: string, payload: Uint8Array): Uint8Array {
|
|
52
|
+
const out = new Uint8Array(8 + payload.length);
|
|
53
|
+
const view = new DataView(out.buffer);
|
|
54
|
+
view.setUint32(0, out.length);
|
|
55
|
+
out[4] = type.charCodeAt(0);
|
|
56
|
+
out[5] = type.charCodeAt(1);
|
|
57
|
+
out[6] = type.charCodeAt(2);
|
|
58
|
+
out[7] = type.charCodeAt(3);
|
|
59
|
+
out.set(payload, 8);
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function concat(parts: Uint8Array[]): Uint8Array {
|
|
64
|
+
const total = parts.reduce((sum, part) => sum + part.length, 0);
|
|
65
|
+
const out = new Uint8Array(total);
|
|
66
|
+
let offset = 0;
|
|
67
|
+
for (const part of parts) {
|
|
68
|
+
out.set(part, offset);
|
|
69
|
+
offset += part.length;
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function readU32(data: Uint8Array, offset: number): number {
|
|
75
|
+
return new DataView(data.buffer, data.byteOffset, data.byteLength).getUint32(offset);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function writeU32(data: Uint8Array, offset: number, value: number): void {
|
|
79
|
+
new DataView(data.buffer, data.byteOffset, data.byteLength).setUint32(offset, value);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The version byte of a full box (the first byte of its payload). */
|
|
83
|
+
export function boxVersion(data: Uint8Array, box: Box): number {
|
|
84
|
+
return data[box.dataStart];
|
|
85
|
+
}
|