@postfetch/core 0.13.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hls.d.ts +1 -1
- package/dist/index.js +171 -47
- package/dist/internal.d.ts +5 -1
- package/package.json +1 -1
- package/src/download.ts +17 -13
- package/src/hls.ts +14 -11
- package/src/internal.ts +24 -0
- package/src/spotify.ts +131 -30
package/dist/hls.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -37,6 +37,21 @@ function createNet(baseFetch = globalThis.fetch) {
|
|
|
37
37
|
throw lastError instanceof Error ? lastError : new Error("request failed");
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
+
async function readBody(net, url, init, read) {
|
|
41
|
+
for (let attempt = 1;; attempt += 1) {
|
|
42
|
+
const response = await net(url, init);
|
|
43
|
+
if (!response.ok || !response.body)
|
|
44
|
+
return read(response);
|
|
45
|
+
try {
|
|
46
|
+
return await read(response);
|
|
47
|
+
} catch (error) {
|
|
48
|
+
if (attempt === 3)
|
|
49
|
+
throw error;
|
|
50
|
+
await response.body?.cancel().catch(() => {});
|
|
51
|
+
await sleep(500 * 2 ** (attempt - 1));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
40
55
|
function object(value) {
|
|
41
56
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
42
57
|
}
|
|
@@ -1256,11 +1271,13 @@ function parseMaster(text, baseUrl, options = {}) {
|
|
|
1256
1271
|
return { variants, audio };
|
|
1257
1272
|
}
|
|
1258
1273
|
async function assembleHls(net, playlistUrl, headers) {
|
|
1259
|
-
const
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1274
|
+
const text = await readBody(net, playlistUrl, { headers }, async (response) => {
|
|
1275
|
+
if (!response.ok) {
|
|
1276
|
+
throw new Error(`HLS playlist failed: ${response.status}`);
|
|
1277
|
+
}
|
|
1278
|
+
return response.text();
|
|
1279
|
+
});
|
|
1280
|
+
const segments = mediaSegments(text, playlistUrl);
|
|
1264
1281
|
if (segments.length === 0) {
|
|
1265
1282
|
throw new Error("HLS playlist has no segments");
|
|
1266
1283
|
}
|
|
@@ -1317,11 +1334,12 @@ async function fetchSegment(net, segment, headers) {
|
|
|
1317
1334
|
if (segment.range) {
|
|
1318
1335
|
requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
|
|
1319
1336
|
}
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1337
|
+
return readBody(net, segment.url, { headers: requestHeaders }, async (response) => {
|
|
1338
|
+
if (!response.ok) {
|
|
1339
|
+
throw new Error(`HLS segment failed: ${response.status}`);
|
|
1340
|
+
}
|
|
1341
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
1342
|
+
});
|
|
1325
1343
|
}
|
|
1326
1344
|
|
|
1327
1345
|
// src/pinterest.ts
|
|
@@ -2327,8 +2345,7 @@ async function resolveSpotify(input) {
|
|
|
2327
2345
|
throw new PostfetchError(400, "Spotify requires an open.spotify.com track URL");
|
|
2328
2346
|
}
|
|
2329
2347
|
const track = await spotifyTrack(input, id);
|
|
2330
|
-
const youtubeUrl = await
|
|
2331
|
-
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2348
|
+
const { youtube, youtubeUrl, liveFallback } = await matchingYoutube(input, track);
|
|
2332
2349
|
return {
|
|
2333
2350
|
archiveFilename: filename(`spotify_${id}.zip`),
|
|
2334
2351
|
comments: [],
|
|
@@ -2338,7 +2355,12 @@ async function resolveSpotify(input) {
|
|
|
2338
2355
|
metadata: {
|
|
2339
2356
|
title: track.title,
|
|
2340
2357
|
author: { name: track.artists.join(", ") },
|
|
2341
|
-
extra: {
|
|
2358
|
+
extra: {
|
|
2359
|
+
artists: track.artists,
|
|
2360
|
+
durationSeconds: track.durationSeconds,
|
|
2361
|
+
youtubeUrl,
|
|
2362
|
+
...liveFallback ? { youtubeLiveFallback: true } : {}
|
|
2363
|
+
}
|
|
2342
2364
|
}
|
|
2343
2365
|
};
|
|
2344
2366
|
}
|
|
@@ -2365,9 +2387,73 @@ async function spotifyTrack(input, id) {
|
|
|
2365
2387
|
const duration = number(entity.duration);
|
|
2366
2388
|
return { title, artists, durationSeconds: duration !== null && duration > 0 ? duration / 1000 : undefined };
|
|
2367
2389
|
}
|
|
2368
|
-
async function
|
|
2390
|
+
async function matchingYoutube(input, track) {
|
|
2391
|
+
const quotedTitle = `"${track.title.replace(/"/g, "")}"`;
|
|
2392
|
+
const primaryQuery = `${track.artists.join(" ")} ${track.title} audio`;
|
|
2393
|
+
const queries = new Set([
|
|
2394
|
+
primaryQuery,
|
|
2395
|
+
`${track.artists[0]} ${quotedTitle}`,
|
|
2396
|
+
track.title,
|
|
2397
|
+
`${track.artists[0]} ${track.title.replace(/\s+/g, "")}`
|
|
2398
|
+
]);
|
|
2399
|
+
const attempted = new Set;
|
|
2400
|
+
const live = new Map;
|
|
2401
|
+
let resolutionError;
|
|
2402
|
+
let searchError;
|
|
2403
|
+
for (const query of queries) {
|
|
2404
|
+
let candidates;
|
|
2405
|
+
try {
|
|
2406
|
+
candidates = await searchYoutube(input, query);
|
|
2407
|
+
} catch (error) {
|
|
2408
|
+
if (query === primaryQuery)
|
|
2409
|
+
throw error;
|
|
2410
|
+
searchError = error;
|
|
2411
|
+
continue;
|
|
2412
|
+
}
|
|
2413
|
+
const originals = candidates.map((candidate, index) => ({
|
|
2414
|
+
candidate,
|
|
2415
|
+
score: matchScore(track, candidate) - index / 1000
|
|
2416
|
+
})).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
2417
|
+
for (const candidate of candidates) {
|
|
2418
|
+
if (Number.isFinite(matchScore(track, candidate)))
|
|
2419
|
+
continue;
|
|
2420
|
+
const score = matchScore(track, candidate, true);
|
|
2421
|
+
if (Number.isFinite(score)) {
|
|
2422
|
+
const url = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
2423
|
+
live.set(url, Math.max(live.get(url) ?? -Infinity, score));
|
|
2424
|
+
}
|
|
2425
|
+
}
|
|
2426
|
+
for (const { candidate } of originals) {
|
|
2427
|
+
const youtubeUrl = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
2428
|
+
if (attempted.has(youtubeUrl) || attempted.size >= 3)
|
|
2429
|
+
continue;
|
|
2430
|
+
attempted.add(youtubeUrl);
|
|
2431
|
+
try {
|
|
2432
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2433
|
+
return { youtube, youtubeUrl, liveFallback: false };
|
|
2434
|
+
} catch (error) {
|
|
2435
|
+
resolutionError = error;
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
const performances = [...live].sort((left, right) => right[1] - left[1]).slice(0, 2);
|
|
2440
|
+
for (const [youtubeUrl] of performances) {
|
|
2441
|
+
try {
|
|
2442
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2443
|
+
return { youtube, youtubeUrl, liveFallback: true };
|
|
2444
|
+
} catch (error) {
|
|
2445
|
+
resolutionError = error;
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
if (resolutionError)
|
|
2449
|
+
throw resolutionError;
|
|
2450
|
+
if (searchError)
|
|
2451
|
+
throw searchError;
|
|
2452
|
+
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "noMatch");
|
|
2453
|
+
}
|
|
2454
|
+
async function searchYoutube(input, query) {
|
|
2369
2455
|
const url = new URL("https://www.youtube.com/results");
|
|
2370
|
-
url.searchParams.set("search_query",
|
|
2456
|
+
url.searchParams.set("search_query", query);
|
|
2371
2457
|
url.searchParams.set("hl", "en");
|
|
2372
2458
|
const response = await input.net(url.href, {
|
|
2373
2459
|
headers: { "accept-language": "en-US,en;q=0.9", cookie: "SOCS=CAI", "user-agent": browserUserAgent() }
|
|
@@ -2377,17 +2463,12 @@ async function matchYoutube(input, track) {
|
|
|
2377
2463
|
}
|
|
2378
2464
|
const html = await response.text();
|
|
2379
2465
|
const raw = html.match(/(?:\bytInitialData|window\[["']ytInitialData["']\])\s*=\s*(\{[\s\S]*?)<\/script>/)?.[1];
|
|
2380
|
-
const
|
|
2466
|
+
const jsonScript = html.match(/<script\b[^>]*\bid=["']yt-initial-data["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
2467
|
+
const payload = parseJson2(jsonScript) ?? parseAssignment(raw);
|
|
2381
2468
|
if (!payload || !object(payload.contents)) {
|
|
2382
2469
|
throw new PostfetchError(502, "YouTube search results not found");
|
|
2383
2470
|
}
|
|
2384
|
-
|
|
2385
|
-
const ranked = candidates.map((candidate, index) => ({ candidate, score: matchScore(track, candidate) - index / 1000 })).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
2386
|
-
const id = ranked[0]?.candidate.videoId;
|
|
2387
|
-
if (typeof id !== "string") {
|
|
2388
|
-
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "notFound");
|
|
2389
|
-
}
|
|
2390
|
-
return `https://www.youtube.com/watch?v=${id}`;
|
|
2471
|
+
return videoRenderers(payload.contents);
|
|
2391
2472
|
}
|
|
2392
2473
|
function videoRenderers(value) {
|
|
2393
2474
|
if (Array.isArray(value))
|
|
@@ -2409,28 +2490,65 @@ function normalize(value) {
|
|
|
2409
2490
|
function contains(haystack, needle) {
|
|
2410
2491
|
return needle.length > 0 && ` ${haystack} `.includes(` ${needle} `);
|
|
2411
2492
|
}
|
|
2493
|
+
function artistMatches(value, artist) {
|
|
2494
|
+
if (contains(value, artist))
|
|
2495
|
+
return true;
|
|
2496
|
+
return /^[bcdfghjklmnpqrstvwxyz]{6,}$/.test(artist) && value.split(" ").some((word) => /^[a-z]+$/.test(word) && word.replace(/[aeiou]/g, "") === artist);
|
|
2497
|
+
}
|
|
2412
2498
|
function durationSeconds(value) {
|
|
2413
2499
|
if (!/^\d+(?::\d{2}){1,2}$/.test(value))
|
|
2414
2500
|
return null;
|
|
2415
2501
|
return value.split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0);
|
|
2416
2502
|
}
|
|
2417
|
-
function
|
|
2503
|
+
function topicArtistMatch(author, artist) {
|
|
2504
|
+
if (!author.endsWith(" topic"))
|
|
2505
|
+
return false;
|
|
2506
|
+
const actual = author.slice(0, -6).trim().split(" ");
|
|
2507
|
+
const expected = artist.split(" ");
|
|
2508
|
+
if (actual.length !== expected.length)
|
|
2509
|
+
return false;
|
|
2510
|
+
let typos = 0;
|
|
2511
|
+
return expected.every((word, index) => {
|
|
2512
|
+
const candidate = actual[index];
|
|
2513
|
+
if (candidate === word)
|
|
2514
|
+
return true;
|
|
2515
|
+
if (word.length < 6 || candidate.length !== word.length || ++typos > 1)
|
|
2516
|
+
return false;
|
|
2517
|
+
for (let offset = 0;offset < word.length - 1; offset += 1) {
|
|
2518
|
+
const swapped = word.slice(0, offset) + word[offset + 1] + word[offset] + word.slice(offset + 2);
|
|
2519
|
+
if (candidate === swapped)
|
|
2520
|
+
return true;
|
|
2521
|
+
}
|
|
2522
|
+
return false;
|
|
2523
|
+
});
|
|
2524
|
+
}
|
|
2525
|
+
function matchScore(track, candidate, allowLive = false) {
|
|
2418
2526
|
if (typeof candidate.videoId !== "string" || !/^[A-Za-z0-9_-]{11}$/.test(candidate.videoId))
|
|
2419
2527
|
return -Infinity;
|
|
2420
2528
|
const title = normalize(text(candidate.title));
|
|
2421
2529
|
const author = normalize(text(candidate.ownerText) || text(candidate.longBylineText) || text(candidate.shortBylineText));
|
|
2422
|
-
const
|
|
2530
|
+
const trackTitle = track.title.replace(/\s*[([](?:feat\.?|ft\.?).*?[)\]]/gi, "");
|
|
2531
|
+
const expected = normalize(trackTitle);
|
|
2532
|
+
const artists = track.artists.map(normalize);
|
|
2533
|
+
const artistUpload = artistMatches(author, artists[0]) || topicArtistMatch(author, artists[0]);
|
|
2534
|
+
const credited = artists.map((artist, index) => artistMatches(`${title} ${author}`, artist) || index === 0 && artistUpload);
|
|
2535
|
+
if (!credited[0])
|
|
2536
|
+
return -Infinity;
|
|
2423
2537
|
const tokens = expected.split(" ").filter(Boolean);
|
|
2424
2538
|
if (tokens.length === 0)
|
|
2425
2539
|
return -Infinity;
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
if (
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2540
|
+
let coverage = tokens.filter((token) => contains(title, token)).length / tokens.length;
|
|
2541
|
+
const joinedTitle = expected.replace(/ /g, "");
|
|
2542
|
+
const joinedMatch = tokens.length > 1 && joinedTitle.length >= 6 && contains(title, joinedTitle);
|
|
2543
|
+
if (joinedMatch)
|
|
2544
|
+
coverage = 1;
|
|
2545
|
+
if (coverage < (tokens.length <= 3 ? 1 : 0.8) || tokens.length <= 3 && !contains(title, expected) && !joinedMatch) {
|
|
2546
|
+
const parts = trackTitle.split(/\s+[-–—|]\s+/);
|
|
2547
|
+
if (parts.length < 3 || !parts[0].includes(":") || artists.length < 2 || !credited.every(Boolean) || !contains(title, normalize(parts.slice(1).join(" ")))) {
|
|
2548
|
+
return -Infinity;
|
|
2549
|
+
}
|
|
2550
|
+
coverage = 1;
|
|
2551
|
+
}
|
|
2434
2552
|
const duration = durationSeconds(text(candidate.lengthText));
|
|
2435
2553
|
if (duration === null || duration <= 0)
|
|
2436
2554
|
return -Infinity;
|
|
@@ -2441,13 +2559,16 @@ function matchScore(track, candidate) {
|
|
|
2441
2559
|
let recordingTitle = title;
|
|
2442
2560
|
for (const artist of artists)
|
|
2443
2561
|
recordingTitle = recordingTitle.replace(artist, "");
|
|
2444
|
-
|
|
2562
|
+
const isLive = /\blive\b/.test(recordingTitle);
|
|
2563
|
+
const requestedLive = /\blive\b/.test(expected);
|
|
2564
|
+
if (isLive !== requestedLive && !(allowLive && isLive && !requestedLive))
|
|
2565
|
+
return -Infinity;
|
|
2566
|
+
for (const version of [/\bremix\b/, /\bcover\b/, /\bkaraoke\b/, /\binstrumental\b/, /\bslowed\b/, /\b(?:sped|speed) up\b/, /\bnightcore\b/, /\bacoustic\b/]) {
|
|
2445
2567
|
if (version.test(recordingTitle) !== version.test(expected))
|
|
2446
2568
|
return -Infinity;
|
|
2447
2569
|
}
|
|
2448
|
-
const
|
|
2449
|
-
|
|
2450
|
-
return coverage * 10 + (artistUpload ? 8 : 0) + credited * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
2570
|
+
const creditCoverage = credited.filter(Boolean).length / artists.length;
|
|
2571
|
+
return coverage * 10 + (artistUpload ? 8 : 0) + creditCoverage * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
2451
2572
|
}
|
|
2452
2573
|
function parseJson2(value) {
|
|
2453
2574
|
try {
|
|
@@ -3640,11 +3761,13 @@ async function downloadBlob(input, optionsOrHeaders = {}, legacyOptions = {}) {
|
|
|
3640
3761
|
const options = modern ? optionsOrHeaders : legacyOptions;
|
|
3641
3762
|
const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
|
|
3642
3763
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
3643
|
-
const
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3764
|
+
const item = typeof input === "string" ? null : { ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers };
|
|
3765
|
+
const blob = item && buffered(item) ? new Blob([toArrayBuffer(await itemBytes2(net, item))], { type: item.mime }) : await readBody(net, typeof input === "string" ? input : input.url, { headers: item?.headers ?? headers }, async (response) => {
|
|
3766
|
+
if (!response.ok || !response.body) {
|
|
3767
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
3768
|
+
}
|
|
3769
|
+
return response.blob();
|
|
3770
|
+
});
|
|
3648
3771
|
if (!modern || optionsOrHeaders.remux !== true) {
|
|
3649
3772
|
return blob;
|
|
3650
3773
|
}
|
|
@@ -3748,11 +3871,12 @@ async function mergedBytes(net, item, audio) {
|
|
|
3748
3871
|
return mergeAudioVideo(video, audioBytes);
|
|
3749
3872
|
}
|
|
3750
3873
|
async function fetchBytes(net, url, headers) {
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3874
|
+
return readBody(net, url, { headers }, async (response) => {
|
|
3875
|
+
if (!response.ok) {
|
|
3876
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
3877
|
+
}
|
|
3878
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
3879
|
+
});
|
|
3756
3880
|
}
|
|
3757
3881
|
function toArrayBuffer(bytes) {
|
|
3758
3882
|
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
package/dist/internal.d.ts
CHANGED
|
@@ -152,6 +152,8 @@ export type SpotifyExtra = {
|
|
|
152
152
|
durationSeconds?: number;
|
|
153
153
|
/** Public watch URL passed to the existing YouTube resolver. */
|
|
154
154
|
youtubeUrl: string;
|
|
155
|
+
/** A live performance was selected after original-recording searches failed. */
|
|
156
|
+
youtubeLiveFallback?: boolean;
|
|
155
157
|
};
|
|
156
158
|
/** Pinterest-specific metadata. */
|
|
157
159
|
export type PinterestExtra = {
|
|
@@ -207,7 +209,7 @@ export type ResolveContext = {
|
|
|
207
209
|
* Lets callers tell a user-facing limitation (age-gated, private, removed) apart
|
|
208
210
|
* from a transient/technical failure.
|
|
209
211
|
*/
|
|
210
|
-
export type PostfetchReason = "ageRestricted" | "private" | "loginRequired" | "deleted" | "notFound" | "rateLimited" | "unavailable";
|
|
212
|
+
export type PostfetchReason = "ageRestricted" | "private" | "loginRequired" | "deleted" | "notFound" | "noMatch" | "rateLimited" | "unavailable";
|
|
211
213
|
/** An error carrying an HTTP status, so adapters can map failures onto responses. */
|
|
212
214
|
export declare class PostfetchError extends Error {
|
|
213
215
|
/** HTTP status that best describes the failure. */
|
|
@@ -229,6 +231,8 @@ export declare class PostfetchError extends Error {
|
|
|
229
231
|
}
|
|
230
232
|
export type Json = Record<string, unknown>;
|
|
231
233
|
export declare function createNet(baseFetch?: typeof fetch): Net;
|
|
234
|
+
/** Retry interrupted body reads; request/status retries remain owned by net. */
|
|
235
|
+
export declare function readBody<T>(net: Net, url: string, init: RequestInit, read: (response: Response) => Promise<T>): Promise<T>;
|
|
232
236
|
export declare function object(value: unknown): value is Json;
|
|
233
237
|
export declare function string(value: unknown): string | null;
|
|
234
238
|
export declare function number(value: unknown): number | null;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.1",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/download.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { assembleHls } from "./hls";
|
|
2
|
-
import { createNet, PostfetchError, type MediaItem, type Net, type PostfetchResult } from "./internal";
|
|
2
|
+
import { createNet, PostfetchError, readBody, type MediaItem, type Net, type PostfetchResult } from "./internal";
|
|
3
3
|
import { prepareMp4 } from "./mp4-remux";
|
|
4
4
|
import { mergeAudioVideo } from "./remux";
|
|
5
5
|
import { zip } from "./zip";
|
|
@@ -118,13 +118,16 @@ export async function downloadBlob(
|
|
|
118
118
|
const options = modern ? optionsOrHeaders : legacyOptions;
|
|
119
119
|
const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
|
|
120
120
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
121
|
-
const
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
121
|
+
const item = typeof input === "string" ? null
|
|
122
|
+
: { ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers };
|
|
123
|
+
const blob = item && buffered(item)
|
|
124
|
+
? new Blob([toArrayBuffer(await itemBytes(net, item))], { type: item.mime })
|
|
125
|
+
: await readBody(net, typeof input === "string" ? input : input.url, { headers: item?.headers ?? headers }, async (response) => {
|
|
126
|
+
if (!response.ok || !response.body) {
|
|
127
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
128
|
+
}
|
|
129
|
+
return response.blob();
|
|
130
|
+
});
|
|
128
131
|
if (!modern || optionsOrHeaders.remux !== true) {
|
|
129
132
|
return blob;
|
|
130
133
|
}
|
|
@@ -267,11 +270,12 @@ async function mergedBytes(net: Net, item: MediaItem, audio: NonNullable<MediaIt
|
|
|
267
270
|
}
|
|
268
271
|
|
|
269
272
|
async function fetchBytes(net: Net, url: string, headers: HeadersInit): Promise<Uint8Array> {
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
273
|
+
return readBody(net, url, { headers }, async (response) => {
|
|
274
|
+
if (!response.ok) {
|
|
275
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
276
|
+
}
|
|
277
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
278
|
+
});
|
|
275
279
|
}
|
|
276
280
|
|
|
277
281
|
function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
|
package/src/hls.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { concat } from "./isobmff";
|
|
2
|
-
import type
|
|
2
|
+
import { readBody, type Net } from "./internal";
|
|
3
3
|
import { packedAacToMp4 } from "./packed-aac";
|
|
4
4
|
|
|
5
5
|
// HLS support for the fMP4/CMAF playlists reddit-style muxing already covers:
|
|
@@ -54,11 +54,13 @@ export function parseMaster(text: string, baseUrl: string, options: { preferOrig
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
export async function assembleHls(net: Net, playlistUrl: string, headers: HeadersInit): Promise<Uint8Array> {
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
57
|
+
const text = await readBody(net, playlistUrl, { headers }, async (response) => {
|
|
58
|
+
if (!response.ok) {
|
|
59
|
+
throw new Error(`HLS playlist failed: ${response.status}`);
|
|
60
|
+
}
|
|
61
|
+
return response.text();
|
|
62
|
+
});
|
|
63
|
+
const segments = mediaSegments(text, playlistUrl);
|
|
62
64
|
if (segments.length === 0) {
|
|
63
65
|
throw new Error("HLS playlist has no segments");
|
|
64
66
|
}
|
|
@@ -122,9 +124,10 @@ async function fetchSegment(net: Net, segment: Segment, headers: HeadersInit): P
|
|
|
122
124
|
if (segment.range) {
|
|
123
125
|
requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
|
|
124
126
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
127
|
+
return readBody(net, segment.url, { headers: requestHeaders }, async (response) => {
|
|
128
|
+
if (!response.ok) {
|
|
129
|
+
throw new Error(`HLS segment failed: ${response.status}`);
|
|
130
|
+
}
|
|
131
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
132
|
+
});
|
|
130
133
|
}
|
package/src/internal.ts
CHANGED
|
@@ -159,6 +159,8 @@ export type SpotifyExtra = {
|
|
|
159
159
|
durationSeconds?: number;
|
|
160
160
|
/** Public watch URL passed to the existing YouTube resolver. */
|
|
161
161
|
youtubeUrl: string;
|
|
162
|
+
/** A live performance was selected after original-recording searches failed. */
|
|
163
|
+
youtubeLiveFallback?: boolean;
|
|
162
164
|
};
|
|
163
165
|
|
|
164
166
|
/** Pinterest-specific metadata. */
|
|
@@ -237,6 +239,7 @@ export type PostfetchReason =
|
|
|
237
239
|
| "loginRequired"
|
|
238
240
|
| "deleted"
|
|
239
241
|
| "notFound"
|
|
242
|
+
| "noMatch"
|
|
240
243
|
| "rateLimited"
|
|
241
244
|
| "unavailable";
|
|
242
245
|
|
|
@@ -289,6 +292,27 @@ export function createNet(baseFetch: typeof fetch = globalThis.fetch): Net {
|
|
|
289
292
|
};
|
|
290
293
|
}
|
|
291
294
|
|
|
295
|
+
/** Retry interrupted body reads; request/status retries remain owned by net. */
|
|
296
|
+
export async function readBody<T>(
|
|
297
|
+
net: Net,
|
|
298
|
+
url: string,
|
|
299
|
+
init: RequestInit,
|
|
300
|
+
read: (response: Response) => Promise<T>,
|
|
301
|
+
): Promise<T> {
|
|
302
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
303
|
+
const response = await net(url, init);
|
|
304
|
+
// Let callers report HTTP/empty-body failures without restarting request retries.
|
|
305
|
+
if (!response.ok || !response.body) return read(response);
|
|
306
|
+
try {
|
|
307
|
+
return await read(response);
|
|
308
|
+
} catch (error) {
|
|
309
|
+
if (attempt === 3) throw error;
|
|
310
|
+
await response.body?.cancel().catch(() => {});
|
|
311
|
+
await sleep(500 * 2 ** (attempt - 1));
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
292
316
|
export function object(value: unknown): value is Json {
|
|
293
317
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
294
318
|
}
|
package/src/spotify.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { browserUserAgent } from "./fingerprint";
|
|
2
|
-
import { asUrl, filename, number, object, PostfetchError, string, type Json, type PlatformResult, type ResolveContext, type SpotifyExtra } from "./internal";
|
|
2
|
+
import { asUrl, filename, number, object, PostfetchError, string, type Json, type PlatformResult, type PostfetchResult, type ResolveContext, type SpotifyExtra } from "./internal";
|
|
3
3
|
import { resolveYoutube } from "./youtube";
|
|
4
4
|
|
|
5
5
|
type Track = { title: string; artists: string[]; durationSeconds?: number };
|
|
@@ -12,9 +12,7 @@ export async function resolveSpotify(input: ResolveContext): Promise<PlatformRes
|
|
|
12
12
|
throw new PostfetchError(400, "Spotify requires an open.spotify.com track URL");
|
|
13
13
|
}
|
|
14
14
|
const track = await spotifyTrack(input, id);
|
|
15
|
-
const youtubeUrl = await
|
|
16
|
-
// Keep YouTube's media, filenames, separate audio and client fallback intact.
|
|
17
|
-
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
15
|
+
const { youtube, youtubeUrl, liveFallback } = await matchingYoutube(input, track);
|
|
18
16
|
return {
|
|
19
17
|
archiveFilename: filename(`spotify_${id}.zip`),
|
|
20
18
|
comments: [],
|
|
@@ -24,7 +22,10 @@ export async function resolveSpotify(input: ResolveContext): Promise<PlatformRes
|
|
|
24
22
|
metadata: {
|
|
25
23
|
title: track.title,
|
|
26
24
|
author: { name: track.artists.join(", ") },
|
|
27
|
-
extra: {
|
|
25
|
+
extra: {
|
|
26
|
+
artists: track.artists, durationSeconds: track.durationSeconds, youtubeUrl,
|
|
27
|
+
...(liveFallback ? { youtubeLiveFallback: true } : {}),
|
|
28
|
+
},
|
|
28
29
|
},
|
|
29
30
|
};
|
|
30
31
|
}
|
|
@@ -55,9 +56,71 @@ async function spotifyTrack(input: ResolveContext, id: string): Promise<Track> {
|
|
|
55
56
|
return { title, artists, durationSeconds: duration !== null && duration > 0 ? duration / 1000 : undefined };
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
async function
|
|
59
|
+
async function matchingYoutube(input: ResolveContext, track: Track): Promise<{ youtube: PostfetchResult; youtubeUrl: string; liveFallback: boolean }> {
|
|
60
|
+
const quotedTitle = `"${track.title.replace(/"/g, "")}"`;
|
|
61
|
+
const primaryQuery = `${track.artists.join(" ")} ${track.title} audio`;
|
|
62
|
+
const queries = new Set([
|
|
63
|
+
primaryQuery,
|
|
64
|
+
`${track.artists[0]} ${quotedTitle}`,
|
|
65
|
+
track.title,
|
|
66
|
+
`${track.artists[0]} ${track.title.replace(/\s+/g, "")}`,
|
|
67
|
+
]);
|
|
68
|
+
const attempted = new Set<string>();
|
|
69
|
+
const live = new Map<string, number>();
|
|
70
|
+
let resolutionError: unknown;
|
|
71
|
+
let searchError: unknown;
|
|
72
|
+
for (const query of queries) {
|
|
73
|
+
let candidates: Json[];
|
|
74
|
+
try {
|
|
75
|
+
candidates = await searchYoutube(input, query);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
// A failed primary search is a technical failure, not a missing match.
|
|
78
|
+
if (query === primaryQuery) throw error;
|
|
79
|
+
searchError = error;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const originals = candidates.map((candidate, index) => ({
|
|
83
|
+
candidate, score: matchScore(track, candidate) - index / 1000,
|
|
84
|
+
})).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
85
|
+
for (const candidate of candidates) {
|
|
86
|
+
if (Number.isFinite(matchScore(track, candidate))) continue;
|
|
87
|
+
const score = matchScore(track, candidate, true);
|
|
88
|
+
if (Number.isFinite(score)) {
|
|
89
|
+
const url = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
90
|
+
live.set(url, Math.max(live.get(url) ?? -Infinity, score));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
for (const { candidate } of originals) {
|
|
94
|
+
const youtubeUrl = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
95
|
+
if (attempted.has(youtubeUrl) || attempted.size >= 3) continue;
|
|
96
|
+
attempted.add(youtubeUrl);
|
|
97
|
+
try {
|
|
98
|
+
// Keep YouTube's separate streams and client fallback intact for each upload.
|
|
99
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
100
|
+
return { youtube, youtubeUrl, liveFallback: false };
|
|
101
|
+
} catch (error) {
|
|
102
|
+
resolutionError = error;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Try at most two live performances, after all searches for the original.
|
|
107
|
+
const performances = [...live].sort((left, right) => right[1] - left[1]).slice(0, 2);
|
|
108
|
+
for (const [youtubeUrl] of performances) {
|
|
109
|
+
try {
|
|
110
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
111
|
+
return { youtube, youtubeUrl, liveFallback: true };
|
|
112
|
+
} catch (error) {
|
|
113
|
+
resolutionError = error;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (resolutionError) throw resolutionError;
|
|
117
|
+
if (searchError) throw searchError;
|
|
118
|
+
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "noMatch");
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function searchYoutube(input: ResolveContext, query: string): Promise<Json[]> {
|
|
59
122
|
const url = new URL("https://www.youtube.com/results");
|
|
60
|
-
url.searchParams.set("search_query",
|
|
123
|
+
url.searchParams.set("search_query", query);
|
|
61
124
|
url.searchParams.set("hl", "en");
|
|
62
125
|
const response = await input.net(url.href, {
|
|
63
126
|
headers: { "accept-language": "en-US,en;q=0.9", cookie: "SOCS=CAI", "user-agent": browserUserAgent() },
|
|
@@ -66,22 +129,15 @@ async function matchYoutube(input: ResolveContext, track: Track): Promise<string
|
|
|
66
129
|
throw new PostfetchError(502, `YouTube search failed: ${response.status}`);
|
|
67
130
|
}
|
|
68
131
|
const html = await response.text();
|
|
69
|
-
// Read the JSON
|
|
70
|
-
//
|
|
132
|
+
// Read the JSON script or legacy assignment without evaluating JavaScript.
|
|
133
|
+
// Assignment parsing balances the object and ignores trailing statements.
|
|
71
134
|
const raw = html.match(/(?:\bytInitialData|window\[["']ytInitialData["']\])\s*=\s*(\{[\s\S]*?)<\/script>/)?.[1];
|
|
72
|
-
const
|
|
135
|
+
const jsonScript = html.match(/<script\b[^>]*\bid=["']yt-initial-data["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
136
|
+
const payload = parseJson(jsonScript) ?? parseAssignment(raw);
|
|
73
137
|
if (!payload || !object(payload.contents)) {
|
|
74
138
|
throw new PostfetchError(502, "YouTube search results not found");
|
|
75
139
|
}
|
|
76
|
-
|
|
77
|
-
const ranked = candidates.map((candidate, index) => ({ candidate, score: matchScore(track, candidate) - index / 1000 }))
|
|
78
|
-
.filter(({ score }) => Number.isFinite(score))
|
|
79
|
-
.sort((left, right) => right.score - left.score);
|
|
80
|
-
const id = ranked[0]?.candidate.videoId;
|
|
81
|
-
if (typeof id !== "string") {
|
|
82
|
-
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "notFound");
|
|
83
|
-
}
|
|
84
|
-
return `https://www.youtube.com/watch?v=${id}`;
|
|
140
|
+
return videoRenderers(payload.contents);
|
|
85
141
|
}
|
|
86
142
|
|
|
87
143
|
function videoRenderers(value: unknown): Json[] {
|
|
@@ -106,23 +162,66 @@ function contains(haystack: string, needle: string): boolean {
|
|
|
106
162
|
return needle.length > 0 && ` ${haystack} `.includes(` ${needle} `);
|
|
107
163
|
}
|
|
108
164
|
|
|
165
|
+
function artistMatches(value: string, artist: string): boolean {
|
|
166
|
+
if (contains(value, artist)) return true;
|
|
167
|
+
// Recognize long, vowel-free abbreviations such as Chkbns / Cheekbones.
|
|
168
|
+
// Match whole words, never a short/common acronym or arbitrary edit distance.
|
|
169
|
+
return /^[bcdfghjklmnpqrstvwxyz]{6,}$/.test(artist) && value.split(" ").some((word) =>
|
|
170
|
+
/^[a-z]+$/.test(word) && word.replace(/[aeiou]/g, "") === artist
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
109
174
|
function durationSeconds(value: string): number | null {
|
|
110
175
|
if (!/^\d+(?::\d{2}){1,2}$/.test(value)) return null;
|
|
111
176
|
return value.split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0);
|
|
112
177
|
}
|
|
113
178
|
|
|
114
|
-
|
|
179
|
+
// Some auto-generated Topic channels transpose adjacent letters in an artist's
|
|
180
|
+
// name. Allow one such typo in a long word only, with every other word exact.
|
|
181
|
+
function topicArtistMatch(author: string, artist: string): boolean {
|
|
182
|
+
if (!author.endsWith(" topic")) return false;
|
|
183
|
+
const actual = author.slice(0, -6).trim().split(" ");
|
|
184
|
+
const expected = artist.split(" ");
|
|
185
|
+
if (actual.length !== expected.length) return false;
|
|
186
|
+
let typos = 0;
|
|
187
|
+
return expected.every((word, index) => {
|
|
188
|
+
const candidate = actual[index];
|
|
189
|
+
if (candidate === word) return true;
|
|
190
|
+
if (word.length < 6 || candidate.length !== word.length || ++typos > 1) return false;
|
|
191
|
+
for (let offset = 0; offset < word.length - 1; offset += 1) {
|
|
192
|
+
const swapped = word.slice(0, offset) + word[offset + 1] + word[offset] + word.slice(offset + 2);
|
|
193
|
+
if (candidate === swapped) return true;
|
|
194
|
+
}
|
|
195
|
+
return false;
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function matchScore(track: Track, candidate: Json, allowLive = false): number {
|
|
115
200
|
if (typeof candidate.videoId !== "string" || !/^[A-Za-z0-9_-]{11}$/.test(candidate.videoId)) return -Infinity;
|
|
116
201
|
const title = normalize(text(candidate.title));
|
|
117
202
|
const author = normalize(text(candidate.ownerText) || text(candidate.longBylineText) || text(candidate.shortBylineText));
|
|
118
|
-
const
|
|
203
|
+
const trackTitle = track.title.replace(/\s*[([](?:feat\.?|ft\.?).*?[)\]]/gi, "");
|
|
204
|
+
const expected = normalize(trackTitle);
|
|
205
|
+
const artists = track.artists.map(normalize);
|
|
206
|
+
const artistUpload = artistMatches(author, artists[0]) || topicArtistMatch(author, artists[0]);
|
|
207
|
+
const credited = artists.map((artist, index) => artistMatches(`${title} ${author}`, artist) || (index === 0 && artistUpload));
|
|
208
|
+
if (!credited[0]) return -Infinity;
|
|
119
209
|
const tokens = expected.split(" ").filter(Boolean);
|
|
120
210
|
if (tokens.length === 0) return -Infinity;
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (!contains(
|
|
211
|
+
let coverage = tokens.filter((token) => contains(title, token)).length / tokens.length;
|
|
212
|
+
const joinedTitle = expected.replace(/ /g, "");
|
|
213
|
+
const joinedMatch = tokens.length > 1 && joinedTitle.length >= 6 && contains(title, joinedTitle);
|
|
214
|
+
if (joinedMatch) coverage = 1;
|
|
215
|
+
if (coverage < (tokens.length <= 3 ? 1 : 0.8) || (tokens.length <= 3 && !contains(title, expected) && !joinedMatch)) {
|
|
216
|
+
// A soundtrack title may prefix the song and its show/version with project
|
|
217
|
+
// context, e.g. "Project: Subtitle - Song - Show". Require the complete
|
|
218
|
+
// remaining phrase and every credited artist before accepting its omission.
|
|
219
|
+
const parts = trackTitle.split(/\s+[-–—|]\s+/);
|
|
220
|
+
if (parts.length < 3 || !parts[0].includes(":") || artists.length < 2 || !credited.every(Boolean) || !contains(title, normalize(parts.slice(1).join(" ")))) {
|
|
221
|
+
return -Infinity;
|
|
222
|
+
}
|
|
223
|
+
coverage = 1;
|
|
224
|
+
}
|
|
126
225
|
const duration = durationSeconds(text(candidate.lengthText));
|
|
127
226
|
if (duration === null || duration <= 0) return -Infinity; // Excludes live streams and upcoming videos.
|
|
128
227
|
const tolerance = Math.min(30, Math.max(12, (track.durationSeconds ?? duration) * 0.08));
|
|
@@ -132,12 +231,14 @@ function matchScore(track: Track, candidate: Json): number {
|
|
|
132
231
|
// are removed first so e.g. a band called "Live" is not treated as a live edit.
|
|
133
232
|
let recordingTitle = title;
|
|
134
233
|
for (const artist of artists) recordingTitle = recordingTitle.replace(artist, "");
|
|
135
|
-
|
|
234
|
+
const isLive = /\blive\b/.test(recordingTitle);
|
|
235
|
+
const requestedLive = /\blive\b/.test(expected);
|
|
236
|
+
if (isLive !== requestedLive && !(allowLive && isLive && !requestedLive)) return -Infinity;
|
|
237
|
+
for (const version of [/\bremix\b/, /\bcover\b/, /\bkaraoke\b/, /\binstrumental\b/, /\bslowed\b/, /\b(?:sped|speed) up\b/, /\bnightcore\b/, /\bacoustic\b/]) {
|
|
136
238
|
if (version.test(recordingTitle) !== version.test(expected)) return -Infinity;
|
|
137
239
|
}
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
return coverage * 10 + (artistUpload ? 8 : 0) + credited * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
240
|
+
const creditCoverage = credited.filter(Boolean).length / artists.length;
|
|
241
|
+
return coverage * 10 + (artistUpload ? 8 : 0) + creditCoverage * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
141
242
|
}
|
|
142
243
|
|
|
143
244
|
function parseJson(value: string | undefined): Json | null {
|