@postfetch/core 0.2.4 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/download.d.ts +7 -4
- package/dist/hls.d.ts +15 -0
- package/dist/index.js +962 -22
- package/dist/internal.d.ts +18 -1
- package/dist/isobmff.d.ts +20 -0
- package/dist/pinterest.d.ts +2 -0
- package/dist/reddit.d.ts +2 -0
- package/dist/remux.d.ts +8 -0
- package/dist/soundcloud.d.ts +2 -0
- package/dist/tiktok.d.ts +1 -0
- package/dist/twitter.d.ts +1 -0
- package/package.json +5 -2
- package/src/download.ts +72 -14
- package/src/hls.ts +120 -0
- package/src/internal.ts +15 -1
- package/src/isobmff.ts +85 -0
- package/src/pinterest.ts +237 -0
- package/src/postfetch.ts +19 -1
- package/src/reddit.ts +293 -0
- package/src/remux.ts +172 -0
- package/src/soundcloud.ts +127 -0
- package/src/tiktok.ts +5 -1
- package/src/twitter.ts +1 -1
- package/src/youtube.ts +54 -5
package/dist/index.js
CHANGED
|
@@ -520,6 +520,714 @@ function randomToken(length = 8) {
|
|
|
520
520
|
return crypto.getRandomValues(new Uint8Array(length)).reduce((value, byte) => value + (byte % 36).toString(36), "");
|
|
521
521
|
}
|
|
522
522
|
|
|
523
|
+
// src/isobmff.ts
|
|
524
|
+
function parseBoxes(data, start = 0, end = data.length) {
|
|
525
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
526
|
+
const boxes = [];
|
|
527
|
+
let offset = start;
|
|
528
|
+
while (offset + 8 <= end) {
|
|
529
|
+
let size = view.getUint32(offset);
|
|
530
|
+
const type = String.fromCharCode(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]);
|
|
531
|
+
let dataStart = offset + 8;
|
|
532
|
+
if (size === 1) {
|
|
533
|
+
size = Number(view.getBigUint64(offset + 8));
|
|
534
|
+
dataStart = offset + 16;
|
|
535
|
+
} else if (size === 0) {
|
|
536
|
+
size = end - offset;
|
|
537
|
+
}
|
|
538
|
+
if (size < 8 || offset + size > end) {
|
|
539
|
+
break;
|
|
540
|
+
}
|
|
541
|
+
boxes.push({ type, start: offset, end: offset + size, dataStart });
|
|
542
|
+
offset += size;
|
|
543
|
+
}
|
|
544
|
+
return boxes;
|
|
545
|
+
}
|
|
546
|
+
function findBox(boxes, type) {
|
|
547
|
+
return boxes.find((box) => box.type === type);
|
|
548
|
+
}
|
|
549
|
+
function childBoxes(data, box) {
|
|
550
|
+
return parseBoxes(data, box.dataStart, box.end);
|
|
551
|
+
}
|
|
552
|
+
function sliceBox(data, box) {
|
|
553
|
+
return data.slice(box.start, box.end);
|
|
554
|
+
}
|
|
555
|
+
function buildBox(type, payload) {
|
|
556
|
+
const out = new Uint8Array(8 + payload.length);
|
|
557
|
+
const view = new DataView(out.buffer);
|
|
558
|
+
view.setUint32(0, out.length);
|
|
559
|
+
out[4] = type.charCodeAt(0);
|
|
560
|
+
out[5] = type.charCodeAt(1);
|
|
561
|
+
out[6] = type.charCodeAt(2);
|
|
562
|
+
out[7] = type.charCodeAt(3);
|
|
563
|
+
out.set(payload, 8);
|
|
564
|
+
return out;
|
|
565
|
+
}
|
|
566
|
+
function concat(parts) {
|
|
567
|
+
const total = parts.reduce((sum, part) => sum + part.length, 0);
|
|
568
|
+
const out = new Uint8Array(total);
|
|
569
|
+
let offset = 0;
|
|
570
|
+
for (const part of parts) {
|
|
571
|
+
out.set(part, offset);
|
|
572
|
+
offset += part.length;
|
|
573
|
+
}
|
|
574
|
+
return out;
|
|
575
|
+
}
|
|
576
|
+
function readU32(data, offset) {
|
|
577
|
+
return new DataView(data.buffer, data.byteOffset, data.byteLength).getUint32(offset);
|
|
578
|
+
}
|
|
579
|
+
function writeU32(data, offset, value) {
|
|
580
|
+
new DataView(data.buffer, data.byteOffset, data.byteLength).setUint32(offset, value);
|
|
581
|
+
}
|
|
582
|
+
function boxVersion(data, box) {
|
|
583
|
+
return data[box.dataStart];
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// src/hls.ts
|
|
587
|
+
function isMasterPlaylist(text) {
|
|
588
|
+
return text.includes("#EXT-X-STREAM-INF");
|
|
589
|
+
}
|
|
590
|
+
function parseMaster(text, baseUrl) {
|
|
591
|
+
const lines = text.split(`
|
|
592
|
+
`).map((line) => line.trim());
|
|
593
|
+
const audio = {};
|
|
594
|
+
const variants = [];
|
|
595
|
+
for (let index = 0;index < lines.length; index += 1) {
|
|
596
|
+
const line = lines[index];
|
|
597
|
+
if (line.startsWith("#EXT-X-MEDIA:") && /TYPE=AUDIO/.test(line)) {
|
|
598
|
+
const group = line.match(/GROUP-ID="([^"]+)"/)?.[1];
|
|
599
|
+
const uri = line.match(/URI="([^"]+)"/)?.[1];
|
|
600
|
+
if (group && uri) {
|
|
601
|
+
audio[group] = new URL(uri, baseUrl).href;
|
|
602
|
+
}
|
|
603
|
+
} else if (line.startsWith("#EXT-X-STREAM-INF:")) {
|
|
604
|
+
const target = lines[index + 1];
|
|
605
|
+
if (target && !target.startsWith("#")) {
|
|
606
|
+
const resolution = line.match(/RESOLUTION=(\d+)x(\d+)/);
|
|
607
|
+
variants.push({
|
|
608
|
+
width: resolution ? Number(resolution[1]) : 0,
|
|
609
|
+
height: resolution ? Number(resolution[2]) : 0,
|
|
610
|
+
bandwidth: Number(line.match(/BANDWIDTH=(\d+)/)?.[1] ?? 0),
|
|
611
|
+
url: new URL(target, baseUrl).href,
|
|
612
|
+
audioGroup: line.match(/AUDIO="([^"]+)"/)?.[1] ?? null
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
return { variants, audio };
|
|
618
|
+
}
|
|
619
|
+
async function assembleHls(net, playlistUrl, headers) {
|
|
620
|
+
const response = await net(playlistUrl, { headers });
|
|
621
|
+
if (!response.ok) {
|
|
622
|
+
throw new Error(`HLS playlist failed: ${response.status}`);
|
|
623
|
+
}
|
|
624
|
+
const segments = mediaSegments(await response.text(), playlistUrl);
|
|
625
|
+
if (segments.length === 0) {
|
|
626
|
+
throw new Error("HLS playlist has no segments");
|
|
627
|
+
}
|
|
628
|
+
return concat(await fetchSegments(net, segments, headers));
|
|
629
|
+
}
|
|
630
|
+
function mediaSegments(text, baseUrl) {
|
|
631
|
+
const segments = [];
|
|
632
|
+
let pendingRange = null;
|
|
633
|
+
let nextOffset = 0;
|
|
634
|
+
for (const raw of text.split(`
|
|
635
|
+
`)) {
|
|
636
|
+
const line = raw.trim();
|
|
637
|
+
if (line.startsWith("#EXT-X-MAP:")) {
|
|
638
|
+
const uri = line.match(/URI="([^"]+)"/)?.[1];
|
|
639
|
+
if (uri) {
|
|
640
|
+
const range = byteRange(line.match(/BYTERANGE="([^"]+)"/)?.[1], nextOffset);
|
|
641
|
+
segments.push({ url: new URL(uri, baseUrl).href, range });
|
|
642
|
+
nextOffset = range ? range.offset + range.length : nextOffset;
|
|
643
|
+
}
|
|
644
|
+
} else if (line.startsWith("#EXT-X-BYTERANGE:")) {
|
|
645
|
+
pendingRange = byteRange(line.slice("#EXT-X-BYTERANGE:".length), nextOffset);
|
|
646
|
+
nextOffset = pendingRange ? pendingRange.offset + pendingRange.length : nextOffset;
|
|
647
|
+
} else if (line.length > 0 && !line.startsWith("#")) {
|
|
648
|
+
segments.push({ url: new URL(line, baseUrl).href, range: pendingRange });
|
|
649
|
+
pendingRange = null;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
return segments;
|
|
653
|
+
}
|
|
654
|
+
function byteRange(spec, runningOffset) {
|
|
655
|
+
const match = spec?.trim().match(/^(\d+)(?:@(\d+))?$/);
|
|
656
|
+
if (!match) {
|
|
657
|
+
return null;
|
|
658
|
+
}
|
|
659
|
+
return { length: Number(match[1]), offset: match[2] === undefined ? runningOffset : Number(match[2]) };
|
|
660
|
+
}
|
|
661
|
+
async function fetchSegments(net, segments, headers) {
|
|
662
|
+
const results = new Array(segments.length);
|
|
663
|
+
let cursor = 0;
|
|
664
|
+
async function worker() {
|
|
665
|
+
while (cursor < segments.length) {
|
|
666
|
+
const index = cursor;
|
|
667
|
+
cursor += 1;
|
|
668
|
+
results[index] = await fetchSegment(net, segments[index], headers);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
const workers = Array.from({ length: Math.min(8, segments.length) }, worker);
|
|
672
|
+
await Promise.all(workers);
|
|
673
|
+
return results;
|
|
674
|
+
}
|
|
675
|
+
async function fetchSegment(net, segment, headers) {
|
|
676
|
+
const requestHeaders = new Headers(headers);
|
|
677
|
+
if (segment.range) {
|
|
678
|
+
requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
|
|
679
|
+
}
|
|
680
|
+
const response = await net(segment.url, { headers: requestHeaders });
|
|
681
|
+
if (!response.ok) {
|
|
682
|
+
throw new Error(`HLS segment failed: ${response.status}`);
|
|
683
|
+
}
|
|
684
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// src/pinterest.ts
|
|
688
|
+
async function resolvePinterest(input) {
|
|
689
|
+
const id = await pinId(input.net, input.url);
|
|
690
|
+
const pin = await pinResource(input.net, id);
|
|
691
|
+
const items = await mediaItems2(input, pin, id);
|
|
692
|
+
if (items.length === 0) {
|
|
693
|
+
throw new Error("Pinterest media not found");
|
|
694
|
+
}
|
|
695
|
+
return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, platform: "pinterest" };
|
|
696
|
+
}
|
|
697
|
+
async function pinId(net, input) {
|
|
698
|
+
const direct = parsePin(input);
|
|
699
|
+
if (direct) {
|
|
700
|
+
return direct;
|
|
701
|
+
}
|
|
702
|
+
const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
|
|
703
|
+
const resolved = parsePin(response.url);
|
|
704
|
+
if (!resolved) {
|
|
705
|
+
throw new Error("Pinterest pin id not found");
|
|
706
|
+
}
|
|
707
|
+
return resolved;
|
|
708
|
+
}
|
|
709
|
+
function parsePin(input) {
|
|
710
|
+
return asUrl(input).pathname.match(/\/pin\/(\d+)/)?.[1] ?? null;
|
|
711
|
+
}
|
|
712
|
+
async function pinResource(net, id) {
|
|
713
|
+
const data = JSON.stringify({
|
|
714
|
+
options: { id, field_set_key: "unauth_react_main_pin", fetch_visual_search_objects: false },
|
|
715
|
+
context: {}
|
|
716
|
+
});
|
|
717
|
+
const url = new URL("https://www.pinterest.com/resource/PinResource/get/");
|
|
718
|
+
url.searchParams.set("source_url", `/pin/${id}/`);
|
|
719
|
+
url.searchParams.set("data", data);
|
|
720
|
+
const response = await net(url.href, {
|
|
721
|
+
headers: {
|
|
722
|
+
accept: "application/json",
|
|
723
|
+
"user-agent": browserUserAgent(),
|
|
724
|
+
"x-pinterest-pws-handler": "www/pin/[id].js"
|
|
725
|
+
}
|
|
726
|
+
});
|
|
727
|
+
if (!response.ok) {
|
|
728
|
+
throw new Error(`Pinterest resource failed: ${response.status}`);
|
|
729
|
+
}
|
|
730
|
+
const payload = await response.json().catch(() => null);
|
|
731
|
+
const wrapper = object(payload) && object(payload.resource_response) ? payload.resource_response : null;
|
|
732
|
+
const pin = wrapper && object(wrapper.data) ? wrapper.data : null;
|
|
733
|
+
if (!pin) {
|
|
734
|
+
throw new Error("Pinterest pin not found");
|
|
735
|
+
}
|
|
736
|
+
return pin;
|
|
737
|
+
}
|
|
738
|
+
async function mediaItems2(input, pin, id) {
|
|
739
|
+
const video = await standardVideo(input, pin, id);
|
|
740
|
+
if (video.length > 0) {
|
|
741
|
+
return video;
|
|
742
|
+
}
|
|
743
|
+
const story = await storyItems(input, pin, id);
|
|
744
|
+
if (story.length > 0) {
|
|
745
|
+
return story;
|
|
746
|
+
}
|
|
747
|
+
return imageItems(pin, id);
|
|
748
|
+
}
|
|
749
|
+
async function standardVideo(input, pin, id) {
|
|
750
|
+
const item = await videoFromList(input, id, null, videoList(object(pin.videos) ? pin.videos : null));
|
|
751
|
+
return item ? [item] : [];
|
|
752
|
+
}
|
|
753
|
+
async function storyItems(input, pin, id) {
|
|
754
|
+
const story = object(pin.story_pin_data) ? pin.story_pin_data : null;
|
|
755
|
+
if (!story) {
|
|
756
|
+
return [];
|
|
757
|
+
}
|
|
758
|
+
const pages = Array.isArray(story.pages) ? story.pages.filter(object) : [];
|
|
759
|
+
const blocks = pages.flatMap((page) => Array.isArray(page.blocks) ? page.blocks.filter(object) : []);
|
|
760
|
+
const videoBlocks = blocks.filter((block) => object(block.video));
|
|
761
|
+
const videos = [];
|
|
762
|
+
for (const [index, block] of videoBlocks.entries()) {
|
|
763
|
+
const item = await videoFromList(input, id, index + 1, videoList(object(block.video) ? block.video : null));
|
|
764
|
+
if (item) {
|
|
765
|
+
videos.push(item);
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
if (videos.length > 0) {
|
|
769
|
+
return videos;
|
|
770
|
+
}
|
|
771
|
+
if (videoBlocks.length > 0) {
|
|
772
|
+
throw new Error("Pinterest idea pin video has no downloadable rendition");
|
|
773
|
+
}
|
|
774
|
+
return imageItems(pin, id);
|
|
775
|
+
}
|
|
776
|
+
async function videoFromList(input, id, index, renditions) {
|
|
777
|
+
const progressive = progressiveMp4(renditions);
|
|
778
|
+
if (progressive) {
|
|
779
|
+
return videoItem(id, index, progressive);
|
|
780
|
+
}
|
|
781
|
+
const master = hlsUrl(renditions);
|
|
782
|
+
return master ? hlsVideoItem(input.net, id, index, master, input.preferredWidth) : null;
|
|
783
|
+
}
|
|
784
|
+
async function hlsVideoItem(net, id, index, masterUrl, preferredWidth) {
|
|
785
|
+
const headers = { "user-agent": browserUserAgent() };
|
|
786
|
+
const suffix = index === null ? "" : `_${index}`;
|
|
787
|
+
const base = {
|
|
788
|
+
filename: filename(`pinterest_${id}${suffix}.mp4`),
|
|
789
|
+
headers,
|
|
790
|
+
hls: true,
|
|
791
|
+
id,
|
|
792
|
+
kind: "video",
|
|
793
|
+
mime: "video/mp4",
|
|
794
|
+
platform: "pinterest",
|
|
795
|
+
url: masterUrl
|
|
796
|
+
};
|
|
797
|
+
const response = await net(masterUrl, { headers });
|
|
798
|
+
if (!response.ok) {
|
|
799
|
+
throw new Error(`Pinterest HLS failed: ${response.status}`);
|
|
800
|
+
}
|
|
801
|
+
const playlist = await response.text();
|
|
802
|
+
if (!isMasterPlaylist(playlist)) {
|
|
803
|
+
return base;
|
|
804
|
+
}
|
|
805
|
+
const master = parseMaster(playlist, masterUrl);
|
|
806
|
+
const variant = pickVariant(master.variants, preferredWidth);
|
|
807
|
+
if (!variant) {
|
|
808
|
+
throw new Error("Pinterest HLS variant not found");
|
|
809
|
+
}
|
|
810
|
+
const audioUrl = variant.audioGroup ? master.audio[variant.audioGroup] : undefined;
|
|
811
|
+
return { ...base, url: variant.url, ...audioUrl ? { audio: { headers, url: audioUrl } } : {} };
|
|
812
|
+
}
|
|
813
|
+
function pickVariant(variants, preferredWidth) {
|
|
814
|
+
return variants.reduce((current, variant) => {
|
|
815
|
+
if (!current) {
|
|
816
|
+
return variant;
|
|
817
|
+
}
|
|
818
|
+
return Math.abs(variant.width - preferredWidth) < Math.abs(current.width - preferredWidth) ? variant : current;
|
|
819
|
+
}, null);
|
|
820
|
+
}
|
|
821
|
+
function hlsUrl(renditions) {
|
|
822
|
+
return renditions.map((rendition) => string(rendition.url)).find((url) => url !== null && url.toLowerCase().includes(".m3u8")) ?? null;
|
|
823
|
+
}
|
|
824
|
+
function imageItems(pin, id) {
|
|
825
|
+
const images = object(pin.images) ? pin.images : null;
|
|
826
|
+
const orig = images && object(images.orig) ? images.orig : null;
|
|
827
|
+
const url = orig ? string(orig.url) : null;
|
|
828
|
+
if (!url) {
|
|
829
|
+
return [];
|
|
830
|
+
}
|
|
831
|
+
return [
|
|
832
|
+
{
|
|
833
|
+
filename: filename(`pinterest_${id}.${imageExtension(url)}`),
|
|
834
|
+
headers: { "user-agent": browserUserAgent() },
|
|
835
|
+
id,
|
|
836
|
+
kind: "image",
|
|
837
|
+
mime: imageMime(url),
|
|
838
|
+
platform: "pinterest",
|
|
839
|
+
url
|
|
840
|
+
}
|
|
841
|
+
];
|
|
842
|
+
}
|
|
843
|
+
function videoList(videos) {
|
|
844
|
+
const list = videos && object(videos.video_list) ? videos.video_list : null;
|
|
845
|
+
return list ? Object.values(list).filter(object) : [];
|
|
846
|
+
}
|
|
847
|
+
function progressiveMp4(renditions) {
|
|
848
|
+
const mp4s = renditions.filter((rendition) => string(rendition.url)?.toLowerCase().endsWith(".mp4"));
|
|
849
|
+
const best = mp4s.reduce((current, rendition) => {
|
|
850
|
+
const width = number(rendition.width) ?? 0;
|
|
851
|
+
const currentWidth = current ? number(current.width) ?? 0 : -1;
|
|
852
|
+
return width > currentWidth ? rendition : current;
|
|
853
|
+
}, null);
|
|
854
|
+
return best ? string(best.url) : null;
|
|
855
|
+
}
|
|
856
|
+
function videoItem(id, index, url) {
|
|
857
|
+
const suffix = index === null ? "" : `_${index}`;
|
|
858
|
+
return {
|
|
859
|
+
filename: filename(`pinterest_${id}${suffix}.mp4`),
|
|
860
|
+
headers: { "user-agent": browserUserAgent() },
|
|
861
|
+
id,
|
|
862
|
+
kind: "video",
|
|
863
|
+
mime: "video/mp4",
|
|
864
|
+
platform: "pinterest",
|
|
865
|
+
url
|
|
866
|
+
};
|
|
867
|
+
}
|
|
868
|
+
function imageExtension(url) {
|
|
869
|
+
const extension = asUrl(url).pathname.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase();
|
|
870
|
+
return extension === "png" || extension === "webp" || extension === "gif" ? extension : "jpg";
|
|
871
|
+
}
|
|
872
|
+
function imageMime(url) {
|
|
873
|
+
const extension = imageExtension(url);
|
|
874
|
+
if (extension === "png") {
|
|
875
|
+
return "image/png";
|
|
876
|
+
}
|
|
877
|
+
if (extension === "webp") {
|
|
878
|
+
return "image/webp";
|
|
879
|
+
}
|
|
880
|
+
if (extension === "gif") {
|
|
881
|
+
return "image/gif";
|
|
882
|
+
}
|
|
883
|
+
return "image/jpeg";
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
// src/reddit.ts
|
|
887
|
+
var installedClientId = "ohXpoqrZYub1kg";
|
|
888
|
+
async function resolveReddit(input) {
|
|
889
|
+
const id = await postId(input.net, input.url);
|
|
890
|
+
const token = await appToken(input.net);
|
|
891
|
+
const post = await fetchPost(input.net, token, id);
|
|
892
|
+
const items = await mediaItems3(input, post, id);
|
|
893
|
+
if (items.length === 0) {
|
|
894
|
+
throw new Error("Reddit media not found");
|
|
895
|
+
}
|
|
896
|
+
return { archiveFilename: filename(`reddit_${id}.zip`), id, items, platform: "reddit" };
|
|
897
|
+
}
|
|
898
|
+
async function postId(net, input) {
|
|
899
|
+
const direct = parseId(input);
|
|
900
|
+
if (direct) {
|
|
901
|
+
return direct;
|
|
902
|
+
}
|
|
903
|
+
const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
|
|
904
|
+
const resolved = parseId(response.url);
|
|
905
|
+
if (!resolved) {
|
|
906
|
+
throw new Error("Reddit post id not found");
|
|
907
|
+
}
|
|
908
|
+
return resolved;
|
|
909
|
+
}
|
|
910
|
+
function parseId(input) {
|
|
911
|
+
const url = asUrl(input);
|
|
912
|
+
if (url.hostname === "redd.it") {
|
|
913
|
+
const slug = url.pathname.split("/").filter(Boolean)[0];
|
|
914
|
+
return slug && /^[a-z0-9]+$/i.test(slug) ? slug : null;
|
|
915
|
+
}
|
|
916
|
+
return url.pathname.match(/\/comments\/([a-z0-9]+)/i)?.[1] ?? null;
|
|
917
|
+
}
|
|
918
|
+
async function appToken(net) {
|
|
919
|
+
const response = await net("https://www.reddit.com/api/v1/access_token", {
|
|
920
|
+
body: "grant_type=https://oauth.reddit.com/grants/installed_client&device_id=DO_NOT_TRACK_THIS_DEVICE",
|
|
921
|
+
headers: {
|
|
922
|
+
authorization: `Basic ${btoa(`${installedClientId}:`)}`,
|
|
923
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
924
|
+
"user-agent": browserUserAgent()
|
|
925
|
+
},
|
|
926
|
+
method: "POST"
|
|
927
|
+
});
|
|
928
|
+
if (!response.ok) {
|
|
929
|
+
throw new Error(`Reddit auth failed: ${response.status}`);
|
|
930
|
+
}
|
|
931
|
+
const payload = await response.json().catch(() => null);
|
|
932
|
+
const token = object(payload) ? string(payload.access_token) : null;
|
|
933
|
+
if (!token) {
|
|
934
|
+
throw new Error("Reddit token not found");
|
|
935
|
+
}
|
|
936
|
+
return token;
|
|
937
|
+
}
|
|
938
|
+
async function fetchPost(net, token, id) {
|
|
939
|
+
const response = await net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=1`, {
|
|
940
|
+
headers: { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() }
|
|
941
|
+
});
|
|
942
|
+
if (!response.ok) {
|
|
943
|
+
throw new Error(`Reddit post failed: ${response.status}`);
|
|
944
|
+
}
|
|
945
|
+
const payload = await response.json().catch(() => null);
|
|
946
|
+
const listing = Array.isArray(payload) ? payload[0] : null;
|
|
947
|
+
const children = object(listing) && object(listing.data) && Array.isArray(listing.data.children) ? listing.data.children : [];
|
|
948
|
+
const first = children[0];
|
|
949
|
+
const post = object(first) && object(first.data) ? first.data : null;
|
|
950
|
+
if (!post) {
|
|
951
|
+
throw new Error("Reddit post not found");
|
|
952
|
+
}
|
|
953
|
+
return post;
|
|
954
|
+
}
|
|
955
|
+
async function mediaItems3(input, post, id) {
|
|
956
|
+
const own = await extract(input, post, id);
|
|
957
|
+
if (own.length > 0) {
|
|
958
|
+
return own;
|
|
959
|
+
}
|
|
960
|
+
const parents = Array.isArray(post.crosspost_parent_list) ? post.crosspost_parent_list.filter(object) : [];
|
|
961
|
+
return parents[0] ? extract(input, parents[0], id) : [];
|
|
962
|
+
}
|
|
963
|
+
async function extract(input, post, id) {
|
|
964
|
+
const gallery = galleryItems(post, id);
|
|
965
|
+
if (gallery.length > 0) {
|
|
966
|
+
return gallery;
|
|
967
|
+
}
|
|
968
|
+
const video = await videoItems(input, post, id);
|
|
969
|
+
if (video.length > 0) {
|
|
970
|
+
return video;
|
|
971
|
+
}
|
|
972
|
+
return imageItems2(post, id);
|
|
973
|
+
}
|
|
974
|
+
function galleryItems(post, id) {
|
|
975
|
+
const data = object(post.gallery_data) ? post.gallery_data : null;
|
|
976
|
+
const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
|
|
977
|
+
const metadata = object(post.media_metadata) ? post.media_metadata : {};
|
|
978
|
+
return order.flatMap((entry, index) => {
|
|
979
|
+
const mediaId2 = string(entry.media_id);
|
|
980
|
+
const meta = mediaId2 && object(metadata[mediaId2]) ? metadata[mediaId2] : null;
|
|
981
|
+
const best = meta && object(meta.s) ? meta.s : null;
|
|
982
|
+
if (!best) {
|
|
983
|
+
return [];
|
|
984
|
+
}
|
|
985
|
+
const video = string(best.mp4);
|
|
986
|
+
if (video) {
|
|
987
|
+
return [mediaItem(id, index + 1, "video", "video/mp4", video)];
|
|
988
|
+
}
|
|
989
|
+
const image = string(best.u) ?? string(best.gif);
|
|
990
|
+
return image ? [mediaItem(id, index + 1, "image", mimeOf(string(meta?.m), image), image)] : [];
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
async function videoItems(input, post, id) {
|
|
994
|
+
const media = redditVideo(post);
|
|
995
|
+
const fallback = media ? string(media.fallback_url) : null;
|
|
996
|
+
if (!media || !fallback) {
|
|
997
|
+
return [];
|
|
998
|
+
}
|
|
999
|
+
if (media.has_audio !== true) {
|
|
1000
|
+
return [mediaItem(id, null, "video", "video/mp4", fallback)];
|
|
1001
|
+
}
|
|
1002
|
+
const dashUrl = string(media.dash_url);
|
|
1003
|
+
const streams = dashUrl ? await dashStreams(input.net, dashUrl, input.preferredWidth) : null;
|
|
1004
|
+
if (!streams) {
|
|
1005
|
+
throw new Error("Reddit DASH manifest unavailable for muxing");
|
|
1006
|
+
}
|
|
1007
|
+
return [muxedVideoItem(id, streams.video, streams.audio)];
|
|
1008
|
+
}
|
|
1009
|
+
async function dashStreams(net, dashUrl, preferredWidth) {
|
|
1010
|
+
const response = await net(dashUrl, { headers: { "user-agent": browserUserAgent() } });
|
|
1011
|
+
if (!response.ok) {
|
|
1012
|
+
return null;
|
|
1013
|
+
}
|
|
1014
|
+
const manifest = await response.text();
|
|
1015
|
+
const base = dashUrl.slice(0, dashUrl.lastIndexOf("/") + 1);
|
|
1016
|
+
const query = dashUrl.includes("?") ? dashUrl.slice(dashUrl.indexOf("?")) : "";
|
|
1017
|
+
const video = videoRendition(manifest, preferredWidth);
|
|
1018
|
+
const audio = audioRendition(manifest);
|
|
1019
|
+
if (!video || !audio) {
|
|
1020
|
+
return null;
|
|
1021
|
+
}
|
|
1022
|
+
return { video: `${base}${video}${query}`, audio: `${base}${audio}${query}` };
|
|
1023
|
+
}
|
|
1024
|
+
function videoRendition(manifest, preferredWidth) {
|
|
1025
|
+
const renditions = renditionsOf(manifest, false);
|
|
1026
|
+
const sized = renditions.filter((rendition) => rendition.width !== null);
|
|
1027
|
+
const pool = sized.length > 0 ? sized : renditions;
|
|
1028
|
+
const best = pool.reduce((current, rendition) => {
|
|
1029
|
+
if (!current) {
|
|
1030
|
+
return rendition;
|
|
1031
|
+
}
|
|
1032
|
+
if (rendition.width !== null && current.width !== null) {
|
|
1033
|
+
return Math.abs(rendition.width - preferredWidth) < Math.abs(current.width - preferredWidth) ? rendition : current;
|
|
1034
|
+
}
|
|
1035
|
+
return rendition.bandwidth > current.bandwidth ? rendition : current;
|
|
1036
|
+
}, null);
|
|
1037
|
+
return best ? best.base : null;
|
|
1038
|
+
}
|
|
1039
|
+
function audioRendition(manifest) {
|
|
1040
|
+
const best = renditionsOf(manifest, true).reduce((current, rendition) => !current || rendition.bandwidth > current.bandwidth ? rendition : current, null);
|
|
1041
|
+
return best ? best.base : null;
|
|
1042
|
+
}
|
|
1043
|
+
function renditionsOf(manifest, audio) {
|
|
1044
|
+
return manifest.split("<AdaptationSet").slice(1).filter((chunk) => isAudioSet(chunk) === audio).flatMap((chunk) => [...chunk.matchAll(/<Representation\b([^>]*)>([\s\S]*?)<\/Representation>/g)].flatMap((match) => {
|
|
1045
|
+
const segment = match[2].match(/<BaseURL>([^<]+)<\/BaseURL>/)?.[1];
|
|
1046
|
+
if (!segment) {
|
|
1047
|
+
return [];
|
|
1048
|
+
}
|
|
1049
|
+
const width = match[1].match(/(?:^|[^a-zA-Z])width="(\d+)"/)?.[1];
|
|
1050
|
+
return [{ width: width ? Number(width) : null, bandwidth: Number(match[1].match(/bandwidth="(\d+)"/)?.[1] ?? 0), base: segment }];
|
|
1051
|
+
}));
|
|
1052
|
+
}
|
|
1053
|
+
function isAudioSet(chunk) {
|
|
1054
|
+
const head = chunk.slice(0, chunk.indexOf(">"));
|
|
1055
|
+
return /contentType="audio"/.test(head) || /mimeType="audio/.test(head);
|
|
1056
|
+
}
|
|
1057
|
+
function muxedVideoItem(id, videoUrl, audioUrl) {
|
|
1058
|
+
return {
|
|
1059
|
+
audio: { headers: { "user-agent": browserUserAgent() }, url: audioUrl },
|
|
1060
|
+
filename: filename(`reddit_${id}.mp4`),
|
|
1061
|
+
headers: { "user-agent": browserUserAgent() },
|
|
1062
|
+
id,
|
|
1063
|
+
kind: "video",
|
|
1064
|
+
mime: "video/mp4",
|
|
1065
|
+
platform: "reddit",
|
|
1066
|
+
url: videoUrl
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
function redditVideo(post) {
|
|
1070
|
+
const secure = object(post.secure_media) ? post.secure_media : null;
|
|
1071
|
+
const media = object(post.media) ? post.media : null;
|
|
1072
|
+
const fromSecure = secure && object(secure.reddit_video) ? secure.reddit_video : null;
|
|
1073
|
+
const fromMedia = media && object(media.reddit_video) ? media.reddit_video : null;
|
|
1074
|
+
return fromSecure ?? fromMedia;
|
|
1075
|
+
}
|
|
1076
|
+
function imageItems2(post, id) {
|
|
1077
|
+
const url = string(post.url_overridden_by_dest) ?? string(post.url);
|
|
1078
|
+
if (!url) {
|
|
1079
|
+
return [];
|
|
1080
|
+
}
|
|
1081
|
+
const path = asUrl(url).pathname;
|
|
1082
|
+
const isImage = string(post.post_hint) === "image" || asUrl(url).hostname === "i.redd.it" || /\.(jpe?g|png|webp|gif)$/i.test(path);
|
|
1083
|
+
return isImage ? [mediaItem(id, null, "image", mimeOf(null, url), url)] : [];
|
|
1084
|
+
}
|
|
1085
|
+
function mediaItem(id, index, kind, mime, url) {
|
|
1086
|
+
const suffix = index === null ? "" : `_${index}`;
|
|
1087
|
+
return {
|
|
1088
|
+
filename: filename(`reddit_${id}${suffix}.${extensionOf(mime)}`),
|
|
1089
|
+
headers: { "user-agent": browserUserAgent() },
|
|
1090
|
+
id,
|
|
1091
|
+
kind,
|
|
1092
|
+
mime,
|
|
1093
|
+
platform: "reddit",
|
|
1094
|
+
url
|
|
1095
|
+
};
|
|
1096
|
+
}
|
|
1097
|
+
function mimeOf(declared, url) {
|
|
1098
|
+
if (declared && declared.includes("/")) {
|
|
1099
|
+
return declared.replace("image/jpg", "image/jpeg");
|
|
1100
|
+
}
|
|
1101
|
+
const extension = asUrl(url).pathname.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase();
|
|
1102
|
+
if (extension === "png") {
|
|
1103
|
+
return "image/png";
|
|
1104
|
+
}
|
|
1105
|
+
if (extension === "webp") {
|
|
1106
|
+
return "image/webp";
|
|
1107
|
+
}
|
|
1108
|
+
if (extension === "gif") {
|
|
1109
|
+
return "image/gif";
|
|
1110
|
+
}
|
|
1111
|
+
return "image/jpeg";
|
|
1112
|
+
}
|
|
1113
|
+
function extensionOf(mime) {
|
|
1114
|
+
if (mime === "video/mp4") {
|
|
1115
|
+
return "mp4";
|
|
1116
|
+
}
|
|
1117
|
+
if (mime === "image/png") {
|
|
1118
|
+
return "png";
|
|
1119
|
+
}
|
|
1120
|
+
if (mime === "image/webp") {
|
|
1121
|
+
return "webp";
|
|
1122
|
+
}
|
|
1123
|
+
if (mime === "image/gif") {
|
|
1124
|
+
return "gif";
|
|
1125
|
+
}
|
|
1126
|
+
return "jpg";
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
// src/soundcloud.ts
|
|
1130
|
+
async function resolveSoundcloud(input) {
|
|
1131
|
+
const url = await canonicalUrl2(input.net, input.url);
|
|
1132
|
+
const clientId = await appClientId(input.net);
|
|
1133
|
+
const track = await resolveTrack(input.net, url, clientId);
|
|
1134
|
+
const id = trackId(track);
|
|
1135
|
+
const item = await audioItem(input.net, track, id, clientId);
|
|
1136
|
+
return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], platform: "soundcloud" };
|
|
1137
|
+
}
|
|
1138
|
+
async function canonicalUrl2(net, input) {
|
|
1139
|
+
const host = asUrl(input).hostname;
|
|
1140
|
+
if (host !== "on.soundcloud.com" && host !== "snd.sc") {
|
|
1141
|
+
return input;
|
|
1142
|
+
}
|
|
1143
|
+
const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
|
|
1144
|
+
return response.url;
|
|
1145
|
+
}
|
|
1146
|
+
async function appClientId(net) {
|
|
1147
|
+
const home = await net("https://soundcloud.com/", { headers: { "user-agent": browserUserAgent() } });
|
|
1148
|
+
if (!home.ok) {
|
|
1149
|
+
throw new Error(`SoundCloud home failed: ${home.status}`);
|
|
1150
|
+
}
|
|
1151
|
+
const html = await home.text();
|
|
1152
|
+
const scripts = [...html.matchAll(/<script[^>]+src="(https:\/\/a-v2\.sndcdn\.com\/assets\/[^"]+\.js)"/g)].map((match) => match[1]);
|
|
1153
|
+
for (const src of scripts.reverse()) {
|
|
1154
|
+
const response = await net(src, { headers: { "user-agent": browserUserAgent() } }, 1);
|
|
1155
|
+
if (!response.ok) {
|
|
1156
|
+
continue;
|
|
1157
|
+
}
|
|
1158
|
+
const code = await response.text();
|
|
1159
|
+
const id = code.match(/client_id\s*[:=]\s*"([A-Za-z0-9]{20,})"/)?.[1];
|
|
1160
|
+
if (id) {
|
|
1161
|
+
return id;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
throw new Error("SoundCloud client id not found");
|
|
1165
|
+
}
|
|
1166
|
+
async function resolveTrack(net, url, clientId) {
|
|
1167
|
+
const api = new URL("https://api-v2.soundcloud.com/resolve");
|
|
1168
|
+
api.searchParams.set("url", url);
|
|
1169
|
+
api.searchParams.set("client_id", clientId);
|
|
1170
|
+
const response = await net(api.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
|
|
1171
|
+
if (!response.ok) {
|
|
1172
|
+
throw new Error(`SoundCloud resolve failed: ${response.status}`);
|
|
1173
|
+
}
|
|
1174
|
+
const payload = await response.json().catch(() => null);
|
|
1175
|
+
if (!object(payload) || string(payload.kind) !== "track") {
|
|
1176
|
+
throw new Error("SoundCloud track not found");
|
|
1177
|
+
}
|
|
1178
|
+
return payload;
|
|
1179
|
+
}
|
|
1180
|
+
async function audioItem(net, track, id, clientId) {
|
|
1181
|
+
const media = object(track.media) ? track.media : null;
|
|
1182
|
+
const transcodings = media && Array.isArray(media.transcodings) ? media.transcodings.filter(object) : [];
|
|
1183
|
+
const progressive = transcodings.find((transcoding) => protocol(transcoding) === "progressive");
|
|
1184
|
+
const hls = transcodings.find((transcoding) => protocol(transcoding) === "hls" && mimeType(transcoding)?.includes("mp4") === true);
|
|
1185
|
+
const chosen = progressive ?? hls;
|
|
1186
|
+
if (!chosen) {
|
|
1187
|
+
throw new Error("SoundCloud track has no downloadable stream");
|
|
1188
|
+
}
|
|
1189
|
+
const stream = await streamUrl(net, chosen, clientId);
|
|
1190
|
+
const title = string(track.title) ?? id;
|
|
1191
|
+
const headers = { "user-agent": browserUserAgent() };
|
|
1192
|
+
if (chosen === progressive) {
|
|
1193
|
+
return { filename: filename(`soundcloud_${title}.mp3`), headers, id, kind: "audio", mime: "audio/mpeg", platform: "soundcloud", url: stream };
|
|
1194
|
+
}
|
|
1195
|
+
return { filename: filename(`soundcloud_${title}.m4a`), headers, hls: true, id, kind: "audio", mime: "audio/mp4", platform: "soundcloud", url: stream };
|
|
1196
|
+
}
|
|
1197
|
+
async function streamUrl(net, transcoding, clientId) {
|
|
1198
|
+
const endpoint = string(transcoding.url);
|
|
1199
|
+
if (!endpoint) {
|
|
1200
|
+
throw new Error("SoundCloud stream endpoint not found");
|
|
1201
|
+
}
|
|
1202
|
+
const url = new URL(endpoint);
|
|
1203
|
+
url.searchParams.set("client_id", clientId);
|
|
1204
|
+
const response = await net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
|
|
1205
|
+
if (!response.ok) {
|
|
1206
|
+
throw new Error(`SoundCloud stream failed: ${response.status}`);
|
|
1207
|
+
}
|
|
1208
|
+
const payload = await response.json().catch(() => null);
|
|
1209
|
+
const stream = object(payload) ? string(payload.url) : null;
|
|
1210
|
+
if (!stream) {
|
|
1211
|
+
throw new Error("SoundCloud stream url not found");
|
|
1212
|
+
}
|
|
1213
|
+
return stream;
|
|
1214
|
+
}
|
|
1215
|
+
function protocol(transcoding) {
|
|
1216
|
+
const format = object(transcoding.format) ? transcoding.format : null;
|
|
1217
|
+
return format ? string(format.protocol) : null;
|
|
1218
|
+
}
|
|
1219
|
+
function mimeType(transcoding) {
|
|
1220
|
+
const format = object(transcoding.format) ? transcoding.format : null;
|
|
1221
|
+
return format ? string(format.mime_type) : null;
|
|
1222
|
+
}
|
|
1223
|
+
function trackId(track) {
|
|
1224
|
+
const numeric = number(track.id);
|
|
1225
|
+
if (numeric !== null) {
|
|
1226
|
+
return String(numeric);
|
|
1227
|
+
}
|
|
1228
|
+
return string(track.permalink) ?? "track";
|
|
1229
|
+
}
|
|
1230
|
+
|
|
523
1231
|
// src/tiktok.ts
|
|
524
1232
|
var marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
|
|
525
1233
|
async function resolveTiktok(input) {
|
|
@@ -543,7 +1251,7 @@ async function resolveTiktok(input) {
|
|
|
543
1251
|
if (page.cookie) {
|
|
544
1252
|
headers.cookie = page.cookie;
|
|
545
1253
|
}
|
|
546
|
-
const items =
|
|
1254
|
+
const items = mediaItems4(page.item, user, id, headers);
|
|
547
1255
|
if (items.length === 0) {
|
|
548
1256
|
throw new Error("TikTok media not found");
|
|
549
1257
|
}
|
|
@@ -559,9 +1267,12 @@ async function fetchVideoPage(net, id, userAgent) {
|
|
|
559
1267
|
});
|
|
560
1268
|
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
561
1269
|
}
|
|
1270
|
+
function isShortlinkHost(hostname) {
|
|
1271
|
+
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
1272
|
+
}
|
|
562
1273
|
async function followShortlink(net, userAgent, input) {
|
|
563
1274
|
const url = asUrl(input);
|
|
564
|
-
if (!url.hostname
|
|
1275
|
+
if (!isShortlinkHost(url.hostname)) {
|
|
565
1276
|
return input;
|
|
566
1277
|
}
|
|
567
1278
|
const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
|
|
@@ -603,10 +1314,10 @@ function downloadUrl(item) {
|
|
|
603
1314
|
const video = object(item.video) ? item.video : null;
|
|
604
1315
|
return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
|
|
605
1316
|
}
|
|
606
|
-
function
|
|
607
|
-
const images =
|
|
1317
|
+
function mediaItems4(item, user, id, headers) {
|
|
1318
|
+
const images = imageItems3(item, user, id, headers);
|
|
608
1319
|
if (images.length > 0) {
|
|
609
|
-
const audio =
|
|
1320
|
+
const audio = audioItem2(item, user, id, headers);
|
|
610
1321
|
return audio ? [...images, audio] : images;
|
|
611
1322
|
}
|
|
612
1323
|
const url = downloadUrl(item);
|
|
@@ -620,7 +1331,7 @@ function mediaItems2(item, user, id, headers) {
|
|
|
620
1331
|
url
|
|
621
1332
|
}] : [];
|
|
622
1333
|
}
|
|
623
|
-
function
|
|
1334
|
+
function imageItems3(item, user, id, headers) {
|
|
624
1335
|
const imagePost = object(item.imagePost) ? item.imagePost : null;
|
|
625
1336
|
const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
|
|
626
1337
|
return images.flatMap((image, index) => {
|
|
@@ -638,7 +1349,7 @@ function imageItems(item, user, id, headers) {
|
|
|
638
1349
|
}] : [];
|
|
639
1350
|
});
|
|
640
1351
|
}
|
|
641
|
-
function
|
|
1352
|
+
function audioItem2(item, user, id, headers) {
|
|
642
1353
|
const video = object(item.video) ? item.video : null;
|
|
643
1354
|
const music = object(item.music) ? item.music : null;
|
|
644
1355
|
const url = video ? string(video.playAddr) : null;
|
|
@@ -756,22 +1467,64 @@ async function resolveYoutube(input) {
|
|
|
756
1467
|
const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
|
|
757
1468
|
throw new Error(reason ?? "YouTube video unavailable");
|
|
758
1469
|
}
|
|
759
|
-
const
|
|
760
|
-
if (!
|
|
761
|
-
throw new Error("YouTube
|
|
1470
|
+
const streams = selectStreams(payload, input.preferredWidth);
|
|
1471
|
+
if (!streams) {
|
|
1472
|
+
throw new Error("YouTube mp4 stream not found");
|
|
762
1473
|
}
|
|
763
1474
|
const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
|
|
1475
|
+
const headers = { "user-agent": androidVrClient.userAgent };
|
|
764
1476
|
const media = {
|
|
765
1477
|
filename: filename(`youtube_${title ?? id}_${id}.mp4`),
|
|
766
|
-
headers
|
|
1478
|
+
headers,
|
|
767
1479
|
id,
|
|
768
1480
|
kind: "video",
|
|
769
1481
|
mime: "video/mp4",
|
|
770
1482
|
platform: "youtube",
|
|
771
|
-
url:
|
|
1483
|
+
url: streams.video,
|
|
1484
|
+
...streams.audio ? { audio: { headers, url: streams.audio } } : {}
|
|
772
1485
|
};
|
|
773
1486
|
return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], platform: "youtube" };
|
|
774
1487
|
}
|
|
1488
|
+
function selectStreams(payload, preferredWidth) {
|
|
1489
|
+
const adaptive = adaptiveFormats(payload);
|
|
1490
|
+
const video = bestVideo(adaptive, preferredWidth);
|
|
1491
|
+
const audio = bestAudio(adaptive);
|
|
1492
|
+
if (video && audio) {
|
|
1493
|
+
return { video, audio };
|
|
1494
|
+
}
|
|
1495
|
+
const progressive = selectFormat(payload);
|
|
1496
|
+
return progressive ? { video: progressive, audio: null } : null;
|
|
1497
|
+
}
|
|
1498
|
+
function adaptiveFormats(payload) {
|
|
1499
|
+
const root = object(payload) ? payload : null;
|
|
1500
|
+
const streaming = root && object(root.streamingData) ? root.streamingData : null;
|
|
1501
|
+
return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
|
|
1502
|
+
}
|
|
1503
|
+
function bestVideo(formats, preferredWidth) {
|
|
1504
|
+
const candidates = formats.filter((format) => {
|
|
1505
|
+
const mimeType2 = string(format.mimeType);
|
|
1506
|
+
return Boolean(string(format.url)) && mimeType2?.startsWith("video/mp4") === true && mimeType2.includes("avc1");
|
|
1507
|
+
});
|
|
1508
|
+
const best = candidates.reduce((current, format) => {
|
|
1509
|
+
if (!current) {
|
|
1510
|
+
return format;
|
|
1511
|
+
}
|
|
1512
|
+
const width = number(format.width) ?? 0;
|
|
1513
|
+
const currentWidth = number(current.width) ?? 0;
|
|
1514
|
+
return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
|
|
1515
|
+
}, null);
|
|
1516
|
+
return best ? string(best.url) : null;
|
|
1517
|
+
}
|
|
1518
|
+
function bestAudio(formats) {
|
|
1519
|
+
const candidates = formats.filter((format) => Boolean(string(format.url)) && string(format.mimeType)?.startsWith("audio/mp4") === true);
|
|
1520
|
+
const best = candidates.reduce((current, format) => {
|
|
1521
|
+
if (!current) {
|
|
1522
|
+
return format;
|
|
1523
|
+
}
|
|
1524
|
+
return (number(format.bitrate) ?? 0) > (number(current.bitrate) ?? 0) ? format : current;
|
|
1525
|
+
}, null);
|
|
1526
|
+
return best ? string(best.url) : null;
|
|
1527
|
+
}
|
|
775
1528
|
function playerBody(id, session) {
|
|
776
1529
|
return {
|
|
777
1530
|
contentCheckOk: true,
|
|
@@ -904,6 +1657,12 @@ async function postfetch(url, options = {}) {
|
|
|
904
1657
|
return resolveFacebook(context);
|
|
905
1658
|
case "instagram":
|
|
906
1659
|
return resolveInstagram(context);
|
|
1660
|
+
case "pinterest":
|
|
1661
|
+
return resolvePinterest(context);
|
|
1662
|
+
case "reddit":
|
|
1663
|
+
return resolveReddit(context);
|
|
1664
|
+
case "soundcloud":
|
|
1665
|
+
return resolveSoundcloud(context);
|
|
907
1666
|
case "tiktok":
|
|
908
1667
|
return resolveTiktok(context);
|
|
909
1668
|
case "twitter":
|
|
@@ -920,6 +1679,15 @@ function detect(url) {
|
|
|
920
1679
|
if (host.includes("instagram.com")) {
|
|
921
1680
|
return "instagram";
|
|
922
1681
|
}
|
|
1682
|
+
if (host.includes("pinterest.") || host === "pin.it") {
|
|
1683
|
+
return "pinterest";
|
|
1684
|
+
}
|
|
1685
|
+
if (host === "reddit.com" || host.endsWith(".reddit.com") || host === "redd.it") {
|
|
1686
|
+
return "reddit";
|
|
1687
|
+
}
|
|
1688
|
+
if (host.includes("soundcloud.com") || host === "snd.sc") {
|
|
1689
|
+
return "soundcloud";
|
|
1690
|
+
}
|
|
923
1691
|
if (host.includes("youtube.com") || host === "youtu.be") {
|
|
924
1692
|
return "youtube";
|
|
925
1693
|
}
|
|
@@ -929,8 +1697,135 @@ function detect(url) {
|
|
|
929
1697
|
if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
|
|
930
1698
|
return "twitter";
|
|
931
1699
|
}
|
|
932
|
-
throw new PostfetchError(400, "only Facebook, Instagram, TikTok, X and YouTube URLs are supported");
|
|
1700
|
+
throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
|
|
1701
|
+
}
|
|
1702
|
+
// src/remux.ts
|
|
1703
|
+
function mergeAudioVideo(video, audio) {
|
|
1704
|
+
const videoSource = analyze(video);
|
|
1705
|
+
const audioSource = analyze(audio);
|
|
1706
|
+
const audioTrackId = videoSource.trackId === 1 ? 2 : 1;
|
|
1707
|
+
const moov = buildMoov(videoSource, audioSource, audioTrackId);
|
|
1708
|
+
const parts = [sliceBox(video, videoSource.ftyp), moov];
|
|
1709
|
+
let sequence = 1;
|
|
1710
|
+
const fragmentCount = Math.max(videoSource.fragments.length, audioSource.fragments.length);
|
|
1711
|
+
for (let index = 0;index < fragmentCount; index += 1) {
|
|
1712
|
+
const videoFragment = videoSource.fragments[index];
|
|
1713
|
+
if (videoFragment) {
|
|
1714
|
+
parts.push(rewriteMoof(video, videoFragment.moof, null, sequence));
|
|
1715
|
+
parts.push(video.subarray(videoFragment.mdat.start, videoFragment.mdat.end));
|
|
1716
|
+
sequence += 1;
|
|
1717
|
+
}
|
|
1718
|
+
const audioFragment = audioSource.fragments[index];
|
|
1719
|
+
if (audioFragment) {
|
|
1720
|
+
parts.push(rewriteMoof(audio, audioFragment.moof, audioTrackId, sequence));
|
|
1721
|
+
parts.push(audio.subarray(audioFragment.mdat.start, audioFragment.mdat.end));
|
|
1722
|
+
sequence += 1;
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
return concat(parts);
|
|
1726
|
+
}
|
|
1727
|
+
function analyze(data) {
|
|
1728
|
+
const top = parseBoxes(data);
|
|
1729
|
+
const ftyp = findBox(top, "ftyp");
|
|
1730
|
+
const moov = findBox(top, "moov");
|
|
1731
|
+
if (!ftyp || !moov) {
|
|
1732
|
+
throw new Error("not a fragmented MP4");
|
|
1733
|
+
}
|
|
1734
|
+
const moovChildren = childBoxes(data, moov);
|
|
1735
|
+
const trak = findBox(moovChildren, "trak");
|
|
1736
|
+
const mvex = findBox(moovChildren, "mvex");
|
|
1737
|
+
if (!trak || !mvex) {
|
|
1738
|
+
throw new Error("MP4 is not fragmented (no trak/mvex)");
|
|
1739
|
+
}
|
|
1740
|
+
const tkhd = findBox(childBoxes(data, trak), "tkhd");
|
|
1741
|
+
if (!tkhd) {
|
|
1742
|
+
throw new Error("track header not found");
|
|
1743
|
+
}
|
|
1744
|
+
const fragments = [];
|
|
1745
|
+
for (let index = 0;index < top.length; index += 1) {
|
|
1746
|
+
if (top[index].type === "moof") {
|
|
1747
|
+
const mdat = top[index + 1];
|
|
1748
|
+
if (!mdat || mdat.type !== "mdat") {
|
|
1749
|
+
throw new Error("fragment without mdat");
|
|
1750
|
+
}
|
|
1751
|
+
fragments.push({ moof: top[index], mdat });
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
if (fragments.length === 0) {
|
|
1755
|
+
throw new Error("MP4 has no fragments");
|
|
1756
|
+
}
|
|
1757
|
+
return { data, ftyp, moov, trak, mvex, trackId: trackHeaderId(data, tkhd), fragments };
|
|
1758
|
+
}
|
|
1759
|
+
function trackHeaderId(data, tkhd) {
|
|
1760
|
+
return readU32(data, tkhd.dataStart + (boxVersion(data, tkhd) === 1 ? 20 : 12));
|
|
1761
|
+
}
|
|
1762
|
+
function buildMoov(video, audio, audioTrackId) {
|
|
1763
|
+
const moovChildren = childBoxes(video.data, video.moov);
|
|
1764
|
+
const mvhd = patchedMvhd(video, audioTrackId);
|
|
1765
|
+
const meta = findBox(moovChildren, "meta");
|
|
1766
|
+
const videoTrak = sliceBox(video.data, video.trak);
|
|
1767
|
+
const audioTrak = retrackTrak(sliceBox(audio.data, audio.trak), audioTrackId);
|
|
1768
|
+
const mvex = buildMvex(video, audio, audioTrackId);
|
|
1769
|
+
const children = [mvhd];
|
|
1770
|
+
if (meta) {
|
|
1771
|
+
children.push(sliceBox(video.data, meta));
|
|
1772
|
+
}
|
|
1773
|
+
children.push(videoTrak, audioTrak, mvex);
|
|
1774
|
+
return buildBox("moov", concat(children));
|
|
1775
|
+
}
|
|
1776
|
+
function patchedMvhd(video, audioTrackId) {
|
|
1777
|
+
const mvhd = findBox(childBoxes(video.data, video.moov), "mvhd");
|
|
1778
|
+
if (!mvhd) {
|
|
1779
|
+
throw new Error("movie header not found");
|
|
1780
|
+
}
|
|
1781
|
+
const copy = sliceBox(video.data, mvhd);
|
|
1782
|
+
writeU32(copy, copy.length - 4, Math.max(video.trackId, audioTrackId) + 1);
|
|
1783
|
+
return copy;
|
|
1784
|
+
}
|
|
1785
|
+
function buildMvex(video, audio, audioTrackId) {
|
|
1786
|
+
const videoMvex = childBoxes(video.data, video.mvex);
|
|
1787
|
+
const mehd = findBox(videoMvex, "mehd");
|
|
1788
|
+
const videoTrex = findBox(videoMvex, "trex");
|
|
1789
|
+
const audioTrex = findBox(childBoxes(audio.data, audio.mvex), "trex");
|
|
1790
|
+
if (!videoTrex || !audioTrex) {
|
|
1791
|
+
throw new Error("track extends box not found");
|
|
1792
|
+
}
|
|
1793
|
+
const audioTrexCopy = sliceBox(audio.data, audioTrex);
|
|
1794
|
+
writeU32(audioTrexCopy, audioTrex.dataStart - audioTrex.start + 4, audioTrackId);
|
|
1795
|
+
const children = [];
|
|
1796
|
+
if (mehd) {
|
|
1797
|
+
children.push(sliceBox(video.data, mehd));
|
|
1798
|
+
}
|
|
1799
|
+
children.push(sliceBox(video.data, videoTrex), audioTrexCopy);
|
|
1800
|
+
return buildBox("mvex", concat(children));
|
|
1801
|
+
}
|
|
1802
|
+
function retrackTrak(trak, trackId2) {
|
|
1803
|
+
const tkhd = findBox(parseBoxes(trak, 8, trak.length), "tkhd");
|
|
1804
|
+
if (!tkhd) {
|
|
1805
|
+
throw new Error("track header not found");
|
|
1806
|
+
}
|
|
1807
|
+
writeU32(trak, tkhd.dataStart + (boxVersion(trak, tkhd) === 1 ? 20 : 12), trackId2);
|
|
1808
|
+
return trak;
|
|
1809
|
+
}
|
|
1810
|
+
function rewriteMoof(data, moofBox, trackId2, sequence) {
|
|
1811
|
+
const moof = sliceBox(data, moofBox);
|
|
1812
|
+
const moofChildren = parseBoxes(moof, 8, moof.length);
|
|
1813
|
+
const mfhd = findBox(moofChildren, "mfhd");
|
|
1814
|
+
if (!mfhd) {
|
|
1815
|
+
throw new Error("movie fragment header not found");
|
|
1816
|
+
}
|
|
1817
|
+
writeU32(moof, mfhd.dataStart + 4, sequence);
|
|
1818
|
+
if (trackId2 !== null) {
|
|
1819
|
+
const traf = findBox(moofChildren, "traf");
|
|
1820
|
+
const tfhd = traf ? findBox(parseBoxes(moof, traf.dataStart, traf.end), "tfhd") : undefined;
|
|
1821
|
+
if (!tfhd) {
|
|
1822
|
+
throw new Error("track fragment header not found");
|
|
1823
|
+
}
|
|
1824
|
+
writeU32(moof, tfhd.dataStart + 4, trackId2);
|
|
1825
|
+
}
|
|
1826
|
+
return moof;
|
|
933
1827
|
}
|
|
1828
|
+
|
|
934
1829
|
// src/zip.ts
|
|
935
1830
|
var encoder = new TextEncoder;
|
|
936
1831
|
var table = new Uint32Array(256);
|
|
@@ -1024,8 +1919,15 @@ function crc32(data) {
|
|
|
1024
1919
|
}
|
|
1025
1920
|
|
|
1026
1921
|
// src/download.ts
|
|
1922
|
+
function buffered(item) {
|
|
1923
|
+
return Boolean(item.audio) || item.hls === true;
|
|
1924
|
+
}
|
|
1027
1925
|
async function download(item, options = {}) {
|
|
1028
1926
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
1927
|
+
if (buffered(item)) {
|
|
1928
|
+
const bytes = await itemBytes(net, item);
|
|
1929
|
+
return new Response(toArrayBuffer(bytes), { headers: { "content-length": String(bytes.length), "content-type": item.mime } });
|
|
1930
|
+
}
|
|
1029
1931
|
const response = await net(item.url, { headers: item.headers });
|
|
1030
1932
|
if (!response.ok || !response.body) {
|
|
1031
1933
|
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
@@ -1034,13 +1936,7 @@ async function download(item, options = {}) {
|
|
|
1034
1936
|
}
|
|
1035
1937
|
async function archive(result, options = {}) {
|
|
1036
1938
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
1037
|
-
const files = await Promise.all(result.items.map(async (item) => {
|
|
1038
|
-
const response = await net(item.url, { headers: item.headers });
|
|
1039
|
-
if (!response.ok) {
|
|
1040
|
-
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
1041
|
-
}
|
|
1042
|
-
return { data: new Uint8Array(await response.arrayBuffer()), name: item.filename };
|
|
1043
|
-
}));
|
|
1939
|
+
const files = await Promise.all(result.items.map(async (item) => ({ data: await itemBytes(net, item), name: item.filename })));
|
|
1044
1940
|
return { bytes: zip(files), filename: result.archiveFilename, mime: "application/zip" };
|
|
1045
1941
|
}
|
|
1046
1942
|
async function toResponse(result, options = {}) {
|
|
@@ -1050,6 +1946,21 @@ async function toResponse(result, options = {}) {
|
|
|
1050
1946
|
return archiveResponse(result, options);
|
|
1051
1947
|
}
|
|
1052
1948
|
async function singleResponse(item, options) {
|
|
1949
|
+
if (buffered(item)) {
|
|
1950
|
+
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
1951
|
+
const bytes = await itemBytes(net, item);
|
|
1952
|
+
return new Response(toArrayBuffer(bytes), {
|
|
1953
|
+
headers: {
|
|
1954
|
+
"content-disposition": `attachment; filename="${item.filename}"`,
|
|
1955
|
+
"content-length": String(bytes.length),
|
|
1956
|
+
"content-type": item.mime,
|
|
1957
|
+
"x-media-count": "1",
|
|
1958
|
+
"x-media-id": item.id,
|
|
1959
|
+
"x-media-kind": item.kind,
|
|
1960
|
+
"x-media-platform": item.platform
|
|
1961
|
+
}
|
|
1962
|
+
});
|
|
1963
|
+
}
|
|
1053
1964
|
const media = await download(item, options);
|
|
1054
1965
|
const headers = new Headers({
|
|
1055
1966
|
"content-disposition": `attachment; filename="${item.filename}"`,
|
|
@@ -1067,8 +1978,7 @@ async function singleResponse(item, options) {
|
|
|
1067
1978
|
}
|
|
1068
1979
|
async function archiveResponse(result, options) {
|
|
1069
1980
|
const { bytes, filename: filename2 } = await archive(result, options);
|
|
1070
|
-
|
|
1071
|
-
return new Response(body, {
|
|
1981
|
+
return new Response(toArrayBuffer(bytes), {
|
|
1072
1982
|
headers: {
|
|
1073
1983
|
"content-disposition": `attachment; filename="${filename2}"`,
|
|
1074
1984
|
"content-length": String(bytes.length),
|
|
@@ -1079,6 +1989,36 @@ async function archiveResponse(result, options) {
|
|
|
1079
1989
|
}
|
|
1080
1990
|
});
|
|
1081
1991
|
}
|
|
1992
|
+
async function itemBytes(net, item) {
|
|
1993
|
+
if (item.hls) {
|
|
1994
|
+
const video = await assembleHls(net, item.url, item.headers);
|
|
1995
|
+
if (item.audio) {
|
|
1996
|
+
return mergeAudioVideo(video, await assembleHls(net, item.audio.url, item.audio.headers));
|
|
1997
|
+
}
|
|
1998
|
+
return video;
|
|
1999
|
+
}
|
|
2000
|
+
if (item.audio) {
|
|
2001
|
+
return mergedBytes(net, item, item.audio);
|
|
2002
|
+
}
|
|
2003
|
+
return fetchBytes(net, item.url, item.headers);
|
|
2004
|
+
}
|
|
2005
|
+
async function mergedBytes(net, item, audio) {
|
|
2006
|
+
const [video, audioBytes] = await Promise.all([
|
|
2007
|
+
fetchBytes(net, item.url, item.headers),
|
|
2008
|
+
fetchBytes(net, audio.url, audio.headers)
|
|
2009
|
+
]);
|
|
2010
|
+
return mergeAudioVideo(video, audioBytes);
|
|
2011
|
+
}
|
|
2012
|
+
async function fetchBytes(net, url, headers) {
|
|
2013
|
+
const response = await net(url, { headers });
|
|
2014
|
+
if (!response.ok) {
|
|
2015
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
2016
|
+
}
|
|
2017
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
2018
|
+
}
|
|
2019
|
+
function toArrayBuffer(bytes) {
|
|
2020
|
+
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
2021
|
+
}
|
|
1082
2022
|
export {
|
|
1083
2023
|
toResponse,
|
|
1084
2024
|
postfetch,
|