@postfetch/core 0.13.0 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/hls.d.ts +1 -1
- package/dist/index.js +204 -64
- package/dist/internal.d.ts +5 -1
- package/package.json +1 -1
- package/src/download.ts +17 -13
- package/src/hls.ts +14 -11
- package/src/internal.ts +24 -0
- package/src/linkedin.ts +34 -16
- package/src/postfetch.ts +1 -1
- package/src/spotify.ts +131 -30
package/README.md
CHANGED
|
@@ -13,6 +13,8 @@ for (const item of result.items) {
|
|
|
13
13
|
|
|
14
14
|
`postfetch(url, options?)` detects the platform and resolves its media into a typed `PostfetchResult` (URLs plus the headers needed to fetch them). It performs no side effects beyond the lookup — `download`, `downloadBlob`, `archive` and `toResponse` materialize the result.
|
|
15
15
|
|
|
16
|
+
LinkedIn public posts also accept `lnkd.in` shortlinks, including `https://lnkd.in/p/…`.
|
|
17
|
+
|
|
16
18
|
Spotify track URLs are matched to YouTube by artist, title and duration, preferring
|
|
17
19
|
artist uploads and rejecting recognizable alternate versions unless requested.
|
|
18
20
|
Results have `platform: "spotify"`, Spotify metadata and
|
package/dist/hls.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -37,6 +37,21 @@ function createNet(baseFetch = globalThis.fetch) {
|
|
|
37
37
|
throw lastError instanceof Error ? lastError : new Error("request failed");
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
+
async function readBody(net, url, init, read) {
|
|
41
|
+
for (let attempt = 1;; attempt += 1) {
|
|
42
|
+
const response = await net(url, init);
|
|
43
|
+
if (!response.ok || !response.body)
|
|
44
|
+
return read(response);
|
|
45
|
+
try {
|
|
46
|
+
return await read(response);
|
|
47
|
+
} catch (error) {
|
|
48
|
+
if (attempt === 3)
|
|
49
|
+
throw error;
|
|
50
|
+
await response.body?.cancel().catch(() => {});
|
|
51
|
+
await sleep(500 * 2 ** (attempt - 1));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
40
55
|
function object(value) {
|
|
41
56
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
42
57
|
}
|
|
@@ -781,21 +796,10 @@ function randomToken(length = 8) {
|
|
|
781
796
|
|
|
782
797
|
// src/linkedin.ts
|
|
783
798
|
async function resolveLinkedin(input) {
|
|
784
|
-
const reference = postReference(input
|
|
799
|
+
const reference = await postReference(input);
|
|
785
800
|
const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
|
|
786
801
|
const response = await input.net(pageUrl, { headers: navigationHeaders() });
|
|
787
|
-
|
|
788
|
-
throw new PostfetchError(404, "LinkedIn post not found", "notFound");
|
|
789
|
-
}
|
|
790
|
-
if (response.status === 429 || response.status === 999) {
|
|
791
|
-
throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
|
|
792
|
-
}
|
|
793
|
-
if (!response.ok) {
|
|
794
|
-
throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
|
|
795
|
-
}
|
|
796
|
-
if (response.url.includes("/signup/") || response.url.includes("/login")) {
|
|
797
|
-
throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
|
|
798
|
-
}
|
|
802
|
+
checkResponse(response);
|
|
799
803
|
const html = await response.text();
|
|
800
804
|
const post = linkedinPost(html);
|
|
801
805
|
if (!post) {
|
|
@@ -831,8 +835,31 @@ function linkedinMetadata(post) {
|
|
|
831
835
|
commentCount: count(post.commentCount) ?? interactionCount(post.interactionStatistic, "CommentAction")
|
|
832
836
|
};
|
|
833
837
|
}
|
|
834
|
-
function
|
|
835
|
-
|
|
838
|
+
function checkResponse(response) {
|
|
839
|
+
if (response.status === 404) {
|
|
840
|
+
throw new PostfetchError(404, "LinkedIn post not found", "notFound");
|
|
841
|
+
}
|
|
842
|
+
if (response.status === 429 || response.status === 999) {
|
|
843
|
+
throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
|
|
844
|
+
}
|
|
845
|
+
if (!response.ok) {
|
|
846
|
+
throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
|
|
847
|
+
}
|
|
848
|
+
if (response.url.includes("/signup/") || response.url.includes("/login")) {
|
|
849
|
+
throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
async function postReference(input) {
|
|
853
|
+
let url = input.url;
|
|
854
|
+
if (new URL(url).hostname === "lnkd.in") {
|
|
855
|
+
const response = await input.net(url, { headers: navigationHeaders(), redirect: "manual" });
|
|
856
|
+
await response.body?.cancel();
|
|
857
|
+
if (![301, 302, 303, 307, 308].includes(response.status)) {
|
|
858
|
+
checkResponse(response);
|
|
859
|
+
}
|
|
860
|
+
url = response.headers.get("location") ?? "";
|
|
861
|
+
}
|
|
862
|
+
const reference = postReferenceOrNull(url);
|
|
836
863
|
if (!reference) {
|
|
837
864
|
throw new PostfetchError(400, "LinkedIn post id not found");
|
|
838
865
|
}
|
|
@@ -841,7 +868,11 @@ function postReference(input) {
|
|
|
841
868
|
function postReferenceOrNull(input) {
|
|
842
869
|
let pathname;
|
|
843
870
|
try {
|
|
844
|
-
|
|
871
|
+
const url = new URL(input);
|
|
872
|
+
if (url.hostname !== "linkedin.com" && !url.hostname.endsWith(".linkedin.com")) {
|
|
873
|
+
return null;
|
|
874
|
+
}
|
|
875
|
+
pathname = url.pathname.replace(/%3A/gi, ":");
|
|
845
876
|
} catch {
|
|
846
877
|
return null;
|
|
847
878
|
}
|
|
@@ -1256,11 +1287,13 @@ function parseMaster(text, baseUrl, options = {}) {
|
|
|
1256
1287
|
return { variants, audio };
|
|
1257
1288
|
}
|
|
1258
1289
|
async function assembleHls(net, playlistUrl, headers) {
|
|
1259
|
-
const
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1290
|
+
const text = await readBody(net, playlistUrl, { headers }, async (response) => {
|
|
1291
|
+
if (!response.ok) {
|
|
1292
|
+
throw new Error(`HLS playlist failed: ${response.status}`);
|
|
1293
|
+
}
|
|
1294
|
+
return response.text();
|
|
1295
|
+
});
|
|
1296
|
+
const segments = mediaSegments(text, playlistUrl);
|
|
1264
1297
|
if (segments.length === 0) {
|
|
1265
1298
|
throw new Error("HLS playlist has no segments");
|
|
1266
1299
|
}
|
|
@@ -1317,11 +1350,12 @@ async function fetchSegment(net, segment, headers) {
|
|
|
1317
1350
|
if (segment.range) {
|
|
1318
1351
|
requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
|
|
1319
1352
|
}
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1353
|
+
return readBody(net, segment.url, { headers: requestHeaders }, async (response) => {
|
|
1354
|
+
if (!response.ok) {
|
|
1355
|
+
throw new Error(`HLS segment failed: ${response.status}`);
|
|
1356
|
+
}
|
|
1357
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
1358
|
+
});
|
|
1325
1359
|
}
|
|
1326
1360
|
|
|
1327
1361
|
// src/pinterest.ts
|
|
@@ -2327,8 +2361,7 @@ async function resolveSpotify(input) {
|
|
|
2327
2361
|
throw new PostfetchError(400, "Spotify requires an open.spotify.com track URL");
|
|
2328
2362
|
}
|
|
2329
2363
|
const track = await spotifyTrack(input, id);
|
|
2330
|
-
const youtubeUrl = await
|
|
2331
|
-
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2364
|
+
const { youtube, youtubeUrl, liveFallback } = await matchingYoutube(input, track);
|
|
2332
2365
|
return {
|
|
2333
2366
|
archiveFilename: filename(`spotify_${id}.zip`),
|
|
2334
2367
|
comments: [],
|
|
@@ -2338,7 +2371,12 @@ async function resolveSpotify(input) {
|
|
|
2338
2371
|
metadata: {
|
|
2339
2372
|
title: track.title,
|
|
2340
2373
|
author: { name: track.artists.join(", ") },
|
|
2341
|
-
extra: {
|
|
2374
|
+
extra: {
|
|
2375
|
+
artists: track.artists,
|
|
2376
|
+
durationSeconds: track.durationSeconds,
|
|
2377
|
+
youtubeUrl,
|
|
2378
|
+
...liveFallback ? { youtubeLiveFallback: true } : {}
|
|
2379
|
+
}
|
|
2342
2380
|
}
|
|
2343
2381
|
};
|
|
2344
2382
|
}
|
|
@@ -2365,9 +2403,73 @@ async function spotifyTrack(input, id) {
|
|
|
2365
2403
|
const duration = number(entity.duration);
|
|
2366
2404
|
return { title, artists, durationSeconds: duration !== null && duration > 0 ? duration / 1000 : undefined };
|
|
2367
2405
|
}
|
|
2368
|
-
async function
|
|
2406
|
+
async function matchingYoutube(input, track) {
|
|
2407
|
+
const quotedTitle = `"${track.title.replace(/"/g, "")}"`;
|
|
2408
|
+
const primaryQuery = `${track.artists.join(" ")} ${track.title} audio`;
|
|
2409
|
+
const queries = new Set([
|
|
2410
|
+
primaryQuery,
|
|
2411
|
+
`${track.artists[0]} ${quotedTitle}`,
|
|
2412
|
+
track.title,
|
|
2413
|
+
`${track.artists[0]} ${track.title.replace(/\s+/g, "")}`
|
|
2414
|
+
]);
|
|
2415
|
+
const attempted = new Set;
|
|
2416
|
+
const live = new Map;
|
|
2417
|
+
let resolutionError;
|
|
2418
|
+
let searchError;
|
|
2419
|
+
for (const query of queries) {
|
|
2420
|
+
let candidates;
|
|
2421
|
+
try {
|
|
2422
|
+
candidates = await searchYoutube(input, query);
|
|
2423
|
+
} catch (error) {
|
|
2424
|
+
if (query === primaryQuery)
|
|
2425
|
+
throw error;
|
|
2426
|
+
searchError = error;
|
|
2427
|
+
continue;
|
|
2428
|
+
}
|
|
2429
|
+
const originals = candidates.map((candidate, index) => ({
|
|
2430
|
+
candidate,
|
|
2431
|
+
score: matchScore(track, candidate) - index / 1000
|
|
2432
|
+
})).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
2433
|
+
for (const candidate of candidates) {
|
|
2434
|
+
if (Number.isFinite(matchScore(track, candidate)))
|
|
2435
|
+
continue;
|
|
2436
|
+
const score = matchScore(track, candidate, true);
|
|
2437
|
+
if (Number.isFinite(score)) {
|
|
2438
|
+
const url = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
2439
|
+
live.set(url, Math.max(live.get(url) ?? -Infinity, score));
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
for (const { candidate } of originals) {
|
|
2443
|
+
const youtubeUrl = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
2444
|
+
if (attempted.has(youtubeUrl) || attempted.size >= 3)
|
|
2445
|
+
continue;
|
|
2446
|
+
attempted.add(youtubeUrl);
|
|
2447
|
+
try {
|
|
2448
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2449
|
+
return { youtube, youtubeUrl, liveFallback: false };
|
|
2450
|
+
} catch (error) {
|
|
2451
|
+
resolutionError = error;
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
const performances = [...live].sort((left, right) => right[1] - left[1]).slice(0, 2);
|
|
2456
|
+
for (const [youtubeUrl] of performances) {
|
|
2457
|
+
try {
|
|
2458
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
2459
|
+
return { youtube, youtubeUrl, liveFallback: true };
|
|
2460
|
+
} catch (error) {
|
|
2461
|
+
resolutionError = error;
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
if (resolutionError)
|
|
2465
|
+
throw resolutionError;
|
|
2466
|
+
if (searchError)
|
|
2467
|
+
throw searchError;
|
|
2468
|
+
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "noMatch");
|
|
2469
|
+
}
|
|
2470
|
+
async function searchYoutube(input, query) {
|
|
2369
2471
|
const url = new URL("https://www.youtube.com/results");
|
|
2370
|
-
url.searchParams.set("search_query",
|
|
2472
|
+
url.searchParams.set("search_query", query);
|
|
2371
2473
|
url.searchParams.set("hl", "en");
|
|
2372
2474
|
const response = await input.net(url.href, {
|
|
2373
2475
|
headers: { "accept-language": "en-US,en;q=0.9", cookie: "SOCS=CAI", "user-agent": browserUserAgent() }
|
|
@@ -2377,17 +2479,12 @@ async function matchYoutube(input, track) {
|
|
|
2377
2479
|
}
|
|
2378
2480
|
const html = await response.text();
|
|
2379
2481
|
const raw = html.match(/(?:\bytInitialData|window\[["']ytInitialData["']\])\s*=\s*(\{[\s\S]*?)<\/script>/)?.[1];
|
|
2380
|
-
const
|
|
2482
|
+
const jsonScript = html.match(/<script\b[^>]*\bid=["']yt-initial-data["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
2483
|
+
const payload = parseJson2(jsonScript) ?? parseAssignment(raw);
|
|
2381
2484
|
if (!payload || !object(payload.contents)) {
|
|
2382
2485
|
throw new PostfetchError(502, "YouTube search results not found");
|
|
2383
2486
|
}
|
|
2384
|
-
|
|
2385
|
-
const ranked = candidates.map((candidate, index) => ({ candidate, score: matchScore(track, candidate) - index / 1000 })).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
2386
|
-
const id = ranked[0]?.candidate.videoId;
|
|
2387
|
-
if (typeof id !== "string") {
|
|
2388
|
-
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "notFound");
|
|
2389
|
-
}
|
|
2390
|
-
return `https://www.youtube.com/watch?v=${id}`;
|
|
2487
|
+
return videoRenderers(payload.contents);
|
|
2391
2488
|
}
|
|
2392
2489
|
function videoRenderers(value) {
|
|
2393
2490
|
if (Array.isArray(value))
|
|
@@ -2409,28 +2506,65 @@ function normalize(value) {
|
|
|
2409
2506
|
function contains(haystack, needle) {
|
|
2410
2507
|
return needle.length > 0 && ` ${haystack} `.includes(` ${needle} `);
|
|
2411
2508
|
}
|
|
2509
|
+
function artistMatches(value, artist) {
|
|
2510
|
+
if (contains(value, artist))
|
|
2511
|
+
return true;
|
|
2512
|
+
return /^[bcdfghjklmnpqrstvwxyz]{6,}$/.test(artist) && value.split(" ").some((word) => /^[a-z]+$/.test(word) && word.replace(/[aeiou]/g, "") === artist);
|
|
2513
|
+
}
|
|
2412
2514
|
function durationSeconds(value) {
|
|
2413
2515
|
if (!/^\d+(?::\d{2}){1,2}$/.test(value))
|
|
2414
2516
|
return null;
|
|
2415
2517
|
return value.split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0);
|
|
2416
2518
|
}
|
|
2417
|
-
function
|
|
2519
|
+
function topicArtistMatch(author, artist) {
|
|
2520
|
+
if (!author.endsWith(" topic"))
|
|
2521
|
+
return false;
|
|
2522
|
+
const actual = author.slice(0, -6).trim().split(" ");
|
|
2523
|
+
const expected = artist.split(" ");
|
|
2524
|
+
if (actual.length !== expected.length)
|
|
2525
|
+
return false;
|
|
2526
|
+
let typos = 0;
|
|
2527
|
+
return expected.every((word, index) => {
|
|
2528
|
+
const candidate = actual[index];
|
|
2529
|
+
if (candidate === word)
|
|
2530
|
+
return true;
|
|
2531
|
+
if (word.length < 6 || candidate.length !== word.length || ++typos > 1)
|
|
2532
|
+
return false;
|
|
2533
|
+
for (let offset = 0;offset < word.length - 1; offset += 1) {
|
|
2534
|
+
const swapped = word.slice(0, offset) + word[offset + 1] + word[offset] + word.slice(offset + 2);
|
|
2535
|
+
if (candidate === swapped)
|
|
2536
|
+
return true;
|
|
2537
|
+
}
|
|
2538
|
+
return false;
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
2541
|
+
function matchScore(track, candidate, allowLive = false) {
|
|
2418
2542
|
if (typeof candidate.videoId !== "string" || !/^[A-Za-z0-9_-]{11}$/.test(candidate.videoId))
|
|
2419
2543
|
return -Infinity;
|
|
2420
2544
|
const title = normalize(text(candidate.title));
|
|
2421
2545
|
const author = normalize(text(candidate.ownerText) || text(candidate.longBylineText) || text(candidate.shortBylineText));
|
|
2422
|
-
const
|
|
2546
|
+
const trackTitle = track.title.replace(/\s*[([](?:feat\.?|ft\.?).*?[)\]]/gi, "");
|
|
2547
|
+
const expected = normalize(trackTitle);
|
|
2548
|
+
const artists = track.artists.map(normalize);
|
|
2549
|
+
const artistUpload = artistMatches(author, artists[0]) || topicArtistMatch(author, artists[0]);
|
|
2550
|
+
const credited = artists.map((artist, index) => artistMatches(`${title} ${author}`, artist) || index === 0 && artistUpload);
|
|
2551
|
+
if (!credited[0])
|
|
2552
|
+
return -Infinity;
|
|
2423
2553
|
const tokens = expected.split(" ").filter(Boolean);
|
|
2424
2554
|
if (tokens.length === 0)
|
|
2425
2555
|
return -Infinity;
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
if (
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2556
|
+
let coverage = tokens.filter((token) => contains(title, token)).length / tokens.length;
|
|
2557
|
+
const joinedTitle = expected.replace(/ /g, "");
|
|
2558
|
+
const joinedMatch = tokens.length > 1 && joinedTitle.length >= 6 && contains(title, joinedTitle);
|
|
2559
|
+
if (joinedMatch)
|
|
2560
|
+
coverage = 1;
|
|
2561
|
+
if (coverage < (tokens.length <= 3 ? 1 : 0.8) || tokens.length <= 3 && !contains(title, expected) && !joinedMatch) {
|
|
2562
|
+
const parts = trackTitle.split(/\s+[-–—|]\s+/);
|
|
2563
|
+
if (parts.length < 3 || !parts[0].includes(":") || artists.length < 2 || !credited.every(Boolean) || !contains(title, normalize(parts.slice(1).join(" ")))) {
|
|
2564
|
+
return -Infinity;
|
|
2565
|
+
}
|
|
2566
|
+
coverage = 1;
|
|
2567
|
+
}
|
|
2434
2568
|
const duration = durationSeconds(text(candidate.lengthText));
|
|
2435
2569
|
if (duration === null || duration <= 0)
|
|
2436
2570
|
return -Infinity;
|
|
@@ -2441,13 +2575,16 @@ function matchScore(track, candidate) {
|
|
|
2441
2575
|
let recordingTitle = title;
|
|
2442
2576
|
for (const artist of artists)
|
|
2443
2577
|
recordingTitle = recordingTitle.replace(artist, "");
|
|
2444
|
-
|
|
2578
|
+
const isLive = /\blive\b/.test(recordingTitle);
|
|
2579
|
+
const requestedLive = /\blive\b/.test(expected);
|
|
2580
|
+
if (isLive !== requestedLive && !(allowLive && isLive && !requestedLive))
|
|
2581
|
+
return -Infinity;
|
|
2582
|
+
for (const version of [/\bremix\b/, /\bcover\b/, /\bkaraoke\b/, /\binstrumental\b/, /\bslowed\b/, /\b(?:sped|speed) up\b/, /\bnightcore\b/, /\bacoustic\b/]) {
|
|
2445
2583
|
if (version.test(recordingTitle) !== version.test(expected))
|
|
2446
2584
|
return -Infinity;
|
|
2447
2585
|
}
|
|
2448
|
-
const
|
|
2449
|
-
|
|
2450
|
-
return coverage * 10 + (artistUpload ? 8 : 0) + credited * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
2586
|
+
const creditCoverage = credited.filter(Boolean).length / artists.length;
|
|
2587
|
+
return coverage * 10 + (artistUpload ? 8 : 0) + creditCoverage * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
2451
2588
|
}
|
|
2452
2589
|
function parseJson2(value) {
|
|
2453
2590
|
try {
|
|
@@ -3227,7 +3364,7 @@ function detect(url) {
|
|
|
3227
3364
|
if (host.includes("instagram.com")) {
|
|
3228
3365
|
return "instagram";
|
|
3229
3366
|
}
|
|
3230
|
-
if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
|
|
3367
|
+
if (host === "linkedin.com" || host.endsWith(".linkedin.com") || host === "lnkd.in") {
|
|
3231
3368
|
return "linkedin";
|
|
3232
3369
|
}
|
|
3233
3370
|
if (host.includes("pinterest.") || host === "pin.it") {
|
|
@@ -3640,11 +3777,13 @@ async function downloadBlob(input, optionsOrHeaders = {}, legacyOptions = {}) {
|
|
|
3640
3777
|
const options = modern ? optionsOrHeaders : legacyOptions;
|
|
3641
3778
|
const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
|
|
3642
3779
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
3643
|
-
const
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3780
|
+
const item = typeof input === "string" ? null : { ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers };
|
|
3781
|
+
const blob = item && buffered(item) ? new Blob([toArrayBuffer(await itemBytes2(net, item))], { type: item.mime }) : await readBody(net, typeof input === "string" ? input : input.url, { headers: item?.headers ?? headers }, async (response) => {
|
|
3782
|
+
if (!response.ok || !response.body) {
|
|
3783
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
3784
|
+
}
|
|
3785
|
+
return response.blob();
|
|
3786
|
+
});
|
|
3648
3787
|
if (!modern || optionsOrHeaders.remux !== true) {
|
|
3649
3788
|
return blob;
|
|
3650
3789
|
}
|
|
@@ -3748,11 +3887,12 @@ async function mergedBytes(net, item, audio) {
|
|
|
3748
3887
|
return mergeAudioVideo(video, audioBytes);
|
|
3749
3888
|
}
|
|
3750
3889
|
async function fetchBytes(net, url, headers) {
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3890
|
+
return readBody(net, url, { headers }, async (response) => {
|
|
3891
|
+
if (!response.ok) {
|
|
3892
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
3893
|
+
}
|
|
3894
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
3895
|
+
});
|
|
3756
3896
|
}
|
|
3757
3897
|
function toArrayBuffer(bytes) {
|
|
3758
3898
|
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
package/dist/internal.d.ts
CHANGED
|
@@ -152,6 +152,8 @@ export type SpotifyExtra = {
|
|
|
152
152
|
durationSeconds?: number;
|
|
153
153
|
/** Public watch URL passed to the existing YouTube resolver. */
|
|
154
154
|
youtubeUrl: string;
|
|
155
|
+
/** A live performance was selected after original-recording searches failed. */
|
|
156
|
+
youtubeLiveFallback?: boolean;
|
|
155
157
|
};
|
|
156
158
|
/** Pinterest-specific metadata. */
|
|
157
159
|
export type PinterestExtra = {
|
|
@@ -207,7 +209,7 @@ export type ResolveContext = {
|
|
|
207
209
|
* Lets callers tell a user-facing limitation (age-gated, private, removed) apart
|
|
208
210
|
* from a transient/technical failure.
|
|
209
211
|
*/
|
|
210
|
-
export type PostfetchReason = "ageRestricted" | "private" | "loginRequired" | "deleted" | "notFound" | "rateLimited" | "unavailable";
|
|
212
|
+
export type PostfetchReason = "ageRestricted" | "private" | "loginRequired" | "deleted" | "notFound" | "noMatch" | "rateLimited" | "unavailable";
|
|
211
213
|
/** An error carrying an HTTP status, so adapters can map failures onto responses. */
|
|
212
214
|
export declare class PostfetchError extends Error {
|
|
213
215
|
/** HTTP status that best describes the failure. */
|
|
@@ -229,6 +231,8 @@ export declare class PostfetchError extends Error {
|
|
|
229
231
|
}
|
|
230
232
|
export type Json = Record<string, unknown>;
|
|
231
233
|
export declare function createNet(baseFetch?: typeof fetch): Net;
|
|
234
|
+
/** Retry interrupted body reads; request/status retries remain owned by net. */
|
|
235
|
+
export declare function readBody<T>(net: Net, url: string, init: RequestInit, read: (response: Response) => Promise<T>): Promise<T>;
|
|
232
236
|
export declare function object(value: unknown): value is Json;
|
|
233
237
|
export declare function string(value: unknown): string | null;
|
|
234
238
|
export declare function number(value: unknown): number | null;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.2",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/download.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { assembleHls } from "./hls";
|
|
2
|
-
import { createNet, PostfetchError, type MediaItem, type Net, type PostfetchResult } from "./internal";
|
|
2
|
+
import { createNet, PostfetchError, readBody, type MediaItem, type Net, type PostfetchResult } from "./internal";
|
|
3
3
|
import { prepareMp4 } from "./mp4-remux";
|
|
4
4
|
import { mergeAudioVideo } from "./remux";
|
|
5
5
|
import { zip } from "./zip";
|
|
@@ -118,13 +118,16 @@ export async function downloadBlob(
|
|
|
118
118
|
const options = modern ? optionsOrHeaders : legacyOptions;
|
|
119
119
|
const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
|
|
120
120
|
const net = createNet(options.fetch ?? globalThis.fetch);
|
|
121
|
-
const
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
121
|
+
const item = typeof input === "string" ? null
|
|
122
|
+
: { ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers };
|
|
123
|
+
const blob = item && buffered(item)
|
|
124
|
+
? new Blob([toArrayBuffer(await itemBytes(net, item))], { type: item.mime })
|
|
125
|
+
: await readBody(net, typeof input === "string" ? input : input.url, { headers: item?.headers ?? headers }, async (response) => {
|
|
126
|
+
if (!response.ok || !response.body) {
|
|
127
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
128
|
+
}
|
|
129
|
+
return response.blob();
|
|
130
|
+
});
|
|
128
131
|
if (!modern || optionsOrHeaders.remux !== true) {
|
|
129
132
|
return blob;
|
|
130
133
|
}
|
|
@@ -267,11 +270,12 @@ async function mergedBytes(net: Net, item: MediaItem, audio: NonNullable<MediaIt
|
|
|
267
270
|
}
|
|
268
271
|
|
|
269
272
|
async function fetchBytes(net: Net, url: string, headers: HeadersInit): Promise<Uint8Array> {
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
273
|
+
return readBody(net, url, { headers }, async (response) => {
|
|
274
|
+
if (!response.ok) {
|
|
275
|
+
throw new PostfetchError(502, `download failed: ${response.status}`);
|
|
276
|
+
}
|
|
277
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
278
|
+
});
|
|
275
279
|
}
|
|
276
280
|
|
|
277
281
|
function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
|
package/src/hls.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { concat } from "./isobmff";
|
|
2
|
-
import type
|
|
2
|
+
import { readBody, type Net } from "./internal";
|
|
3
3
|
import { packedAacToMp4 } from "./packed-aac";
|
|
4
4
|
|
|
5
5
|
// HLS support for the fMP4/CMAF playlists reddit-style muxing already covers:
|
|
@@ -54,11 +54,13 @@ export function parseMaster(text: string, baseUrl: string, options: { preferOrig
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
export async function assembleHls(net: Net, playlistUrl: string, headers: HeadersInit): Promise<Uint8Array> {
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
57
|
+
const text = await readBody(net, playlistUrl, { headers }, async (response) => {
|
|
58
|
+
if (!response.ok) {
|
|
59
|
+
throw new Error(`HLS playlist failed: ${response.status}`);
|
|
60
|
+
}
|
|
61
|
+
return response.text();
|
|
62
|
+
});
|
|
63
|
+
const segments = mediaSegments(text, playlistUrl);
|
|
62
64
|
if (segments.length === 0) {
|
|
63
65
|
throw new Error("HLS playlist has no segments");
|
|
64
66
|
}
|
|
@@ -122,9 +124,10 @@ async function fetchSegment(net: Net, segment: Segment, headers: HeadersInit): P
|
|
|
122
124
|
if (segment.range) {
|
|
123
125
|
requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
|
|
124
126
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
127
|
+
return readBody(net, segment.url, { headers: requestHeaders }, async (response) => {
|
|
128
|
+
if (!response.ok) {
|
|
129
|
+
throw new Error(`HLS segment failed: ${response.status}`);
|
|
130
|
+
}
|
|
131
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
132
|
+
});
|
|
130
133
|
}
|
package/src/internal.ts
CHANGED
|
@@ -159,6 +159,8 @@ export type SpotifyExtra = {
|
|
|
159
159
|
durationSeconds?: number;
|
|
160
160
|
/** Public watch URL passed to the existing YouTube resolver. */
|
|
161
161
|
youtubeUrl: string;
|
|
162
|
+
/** A live performance was selected after original-recording searches failed. */
|
|
163
|
+
youtubeLiveFallback?: boolean;
|
|
162
164
|
};
|
|
163
165
|
|
|
164
166
|
/** Pinterest-specific metadata. */
|
|
@@ -237,6 +239,7 @@ export type PostfetchReason =
|
|
|
237
239
|
| "loginRequired"
|
|
238
240
|
| "deleted"
|
|
239
241
|
| "notFound"
|
|
242
|
+
| "noMatch"
|
|
240
243
|
| "rateLimited"
|
|
241
244
|
| "unavailable";
|
|
242
245
|
|
|
@@ -289,6 +292,27 @@ export function createNet(baseFetch: typeof fetch = globalThis.fetch): Net {
|
|
|
289
292
|
};
|
|
290
293
|
}
|
|
291
294
|
|
|
295
|
+
/** Retry interrupted body reads; request/status retries remain owned by net. */
|
|
296
|
+
export async function readBody<T>(
|
|
297
|
+
net: Net,
|
|
298
|
+
url: string,
|
|
299
|
+
init: RequestInit,
|
|
300
|
+
read: (response: Response) => Promise<T>,
|
|
301
|
+
): Promise<T> {
|
|
302
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
303
|
+
const response = await net(url, init);
|
|
304
|
+
// Let callers report HTTP/empty-body failures without restarting request retries.
|
|
305
|
+
if (!response.ok || !response.body) return read(response);
|
|
306
|
+
try {
|
|
307
|
+
return await read(response);
|
|
308
|
+
} catch (error) {
|
|
309
|
+
if (attempt === 3) throw error;
|
|
310
|
+
await response.body?.cancel().catch(() => {});
|
|
311
|
+
await sleep(500 * 2 ** (attempt - 1));
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
292
316
|
export function object(value: unknown): value is Json {
|
|
293
317
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
294
318
|
}
|
package/src/linkedin.ts
CHANGED
|
@@ -21,21 +21,10 @@ type LinkedinPostReference = {
|
|
|
21
21
|
};
|
|
22
22
|
|
|
23
23
|
export async function resolveLinkedin(input: ResolveContext): Promise<PostfetchResult> {
|
|
24
|
-
const reference = postReference(input
|
|
24
|
+
const reference = await postReference(input);
|
|
25
25
|
const pageUrl = `https://www.linkedin.com/feed/update/urn:li:${reference.kind}:${reference.id}`;
|
|
26
26
|
const response = await input.net(pageUrl, { headers: navigationHeaders() });
|
|
27
|
-
|
|
28
|
-
throw new PostfetchError(404, "LinkedIn post not found", "notFound");
|
|
29
|
-
}
|
|
30
|
-
if (response.status === 429 || response.status === 999) {
|
|
31
|
-
throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
|
|
32
|
-
}
|
|
33
|
-
if (!response.ok) {
|
|
34
|
-
throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
|
|
35
|
-
}
|
|
36
|
-
if (response.url.includes("/signup/") || response.url.includes("/login")) {
|
|
37
|
-
throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
|
|
38
|
-
}
|
|
27
|
+
checkResponse(response);
|
|
39
28
|
|
|
40
29
|
const html = await response.text();
|
|
41
30
|
const post = linkedinPost(html);
|
|
@@ -76,8 +65,33 @@ export function linkedinMetadata(post: Json): PostMetadata {
|
|
|
76
65
|
};
|
|
77
66
|
}
|
|
78
67
|
|
|
79
|
-
function
|
|
80
|
-
|
|
68
|
+
function checkResponse(response: Response): void {
|
|
69
|
+
if (response.status === 404) {
|
|
70
|
+
throw new PostfetchError(404, "LinkedIn post not found", "notFound");
|
|
71
|
+
}
|
|
72
|
+
if (response.status === 429 || response.status === 999) {
|
|
73
|
+
throw new PostfetchError(429, "LinkedIn rate limit reached", "rateLimited");
|
|
74
|
+
}
|
|
75
|
+
if (!response.ok) {
|
|
76
|
+
throw new PostfetchError(response.status, "LinkedIn post unavailable", "unavailable");
|
|
77
|
+
}
|
|
78
|
+
if (response.url.includes("/signup/") || response.url.includes("/login")) {
|
|
79
|
+
throw new PostfetchError(401, "LinkedIn login required", "loginRequired");
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function postReference(input: ResolveContext): Promise<LinkedinPostReference> {
|
|
84
|
+
let url = input.url;
|
|
85
|
+
if (new URL(url).hostname === "lnkd.in") {
|
|
86
|
+
// Read the destination without fetching the post twice or following outbound links.
|
|
87
|
+
const response = await input.net(url, { headers: navigationHeaders(), redirect: "manual" });
|
|
88
|
+
await response.body?.cancel();
|
|
89
|
+
if (![301, 302, 303, 307, 308].includes(response.status)) {
|
|
90
|
+
checkResponse(response);
|
|
91
|
+
}
|
|
92
|
+
url = response.headers.get("location") ?? "";
|
|
93
|
+
}
|
|
94
|
+
const reference = postReferenceOrNull(url);
|
|
81
95
|
if (!reference) {
|
|
82
96
|
throw new PostfetchError(400, "LinkedIn post id not found");
|
|
83
97
|
}
|
|
@@ -87,7 +101,11 @@ function postReference(input: string): LinkedinPostReference {
|
|
|
87
101
|
function postReferenceOrNull(input: string): LinkedinPostReference | null {
|
|
88
102
|
let pathname: string;
|
|
89
103
|
try {
|
|
90
|
-
|
|
104
|
+
const url = new URL(input);
|
|
105
|
+
if (url.hostname !== "linkedin.com" && !url.hostname.endsWith(".linkedin.com")) {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
pathname = url.pathname.replace(/%3A/gi, ":");
|
|
91
109
|
} catch {
|
|
92
110
|
return null;
|
|
93
111
|
}
|
package/src/postfetch.ts
CHANGED
|
@@ -171,7 +171,7 @@ export function detect(url: string): Platform {
|
|
|
171
171
|
if (host.includes("instagram.com")) {
|
|
172
172
|
return "instagram";
|
|
173
173
|
}
|
|
174
|
-
if (host === "linkedin.com" || host.endsWith(".linkedin.com")) {
|
|
174
|
+
if (host === "linkedin.com" || host.endsWith(".linkedin.com") || host === "lnkd.in") {
|
|
175
175
|
return "linkedin";
|
|
176
176
|
}
|
|
177
177
|
if (host.includes("pinterest.") || host === "pin.it") {
|
package/src/spotify.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { browserUserAgent } from "./fingerprint";
|
|
2
|
-
import { asUrl, filename, number, object, PostfetchError, string, type Json, type PlatformResult, type ResolveContext, type SpotifyExtra } from "./internal";
|
|
2
|
+
import { asUrl, filename, number, object, PostfetchError, string, type Json, type PlatformResult, type PostfetchResult, type ResolveContext, type SpotifyExtra } from "./internal";
|
|
3
3
|
import { resolveYoutube } from "./youtube";
|
|
4
4
|
|
|
5
5
|
type Track = { title: string; artists: string[]; durationSeconds?: number };
|
|
@@ -12,9 +12,7 @@ export async function resolveSpotify(input: ResolveContext): Promise<PlatformRes
|
|
|
12
12
|
throw new PostfetchError(400, "Spotify requires an open.spotify.com track URL");
|
|
13
13
|
}
|
|
14
14
|
const track = await spotifyTrack(input, id);
|
|
15
|
-
const youtubeUrl = await
|
|
16
|
-
// Keep YouTube's media, filenames, separate audio and client fallback intact.
|
|
17
|
-
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
15
|
+
const { youtube, youtubeUrl, liveFallback } = await matchingYoutube(input, track);
|
|
18
16
|
return {
|
|
19
17
|
archiveFilename: filename(`spotify_${id}.zip`),
|
|
20
18
|
comments: [],
|
|
@@ -24,7 +22,10 @@ export async function resolveSpotify(input: ResolveContext): Promise<PlatformRes
|
|
|
24
22
|
metadata: {
|
|
25
23
|
title: track.title,
|
|
26
24
|
author: { name: track.artists.join(", ") },
|
|
27
|
-
extra: {
|
|
25
|
+
extra: {
|
|
26
|
+
artists: track.artists, durationSeconds: track.durationSeconds, youtubeUrl,
|
|
27
|
+
...(liveFallback ? { youtubeLiveFallback: true } : {}),
|
|
28
|
+
},
|
|
28
29
|
},
|
|
29
30
|
};
|
|
30
31
|
}
|
|
@@ -55,9 +56,71 @@ async function spotifyTrack(input: ResolveContext, id: string): Promise<Track> {
|
|
|
55
56
|
return { title, artists, durationSeconds: duration !== null && duration > 0 ? duration / 1000 : undefined };
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
async function
|
|
59
|
+
async function matchingYoutube(input: ResolveContext, track: Track): Promise<{ youtube: PostfetchResult; youtubeUrl: string; liveFallback: boolean }> {
|
|
60
|
+
const quotedTitle = `"${track.title.replace(/"/g, "")}"`;
|
|
61
|
+
const primaryQuery = `${track.artists.join(" ")} ${track.title} audio`;
|
|
62
|
+
const queries = new Set([
|
|
63
|
+
primaryQuery,
|
|
64
|
+
`${track.artists[0]} ${quotedTitle}`,
|
|
65
|
+
track.title,
|
|
66
|
+
`${track.artists[0]} ${track.title.replace(/\s+/g, "")}`,
|
|
67
|
+
]);
|
|
68
|
+
const attempted = new Set<string>();
|
|
69
|
+
const live = new Map<string, number>();
|
|
70
|
+
let resolutionError: unknown;
|
|
71
|
+
let searchError: unknown;
|
|
72
|
+
for (const query of queries) {
|
|
73
|
+
let candidates: Json[];
|
|
74
|
+
try {
|
|
75
|
+
candidates = await searchYoutube(input, query);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
// A failed primary search is a technical failure, not a missing match.
|
|
78
|
+
if (query === primaryQuery) throw error;
|
|
79
|
+
searchError = error;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const originals = candidates.map((candidate, index) => ({
|
|
83
|
+
candidate, score: matchScore(track, candidate) - index / 1000,
|
|
84
|
+
})).filter(({ score }) => Number.isFinite(score)).sort((left, right) => right.score - left.score);
|
|
85
|
+
for (const candidate of candidates) {
|
|
86
|
+
if (Number.isFinite(matchScore(track, candidate))) continue;
|
|
87
|
+
const score = matchScore(track, candidate, true);
|
|
88
|
+
if (Number.isFinite(score)) {
|
|
89
|
+
const url = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
90
|
+
live.set(url, Math.max(live.get(url) ?? -Infinity, score));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
for (const { candidate } of originals) {
|
|
94
|
+
const youtubeUrl = `https://www.youtube.com/watch?v=${candidate.videoId}`;
|
|
95
|
+
if (attempted.has(youtubeUrl) || attempted.size >= 3) continue;
|
|
96
|
+
attempted.add(youtubeUrl);
|
|
97
|
+
try {
|
|
98
|
+
// Keep YouTube's separate streams and client fallback intact for each upload.
|
|
99
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
100
|
+
return { youtube, youtubeUrl, liveFallback: false };
|
|
101
|
+
} catch (error) {
|
|
102
|
+
resolutionError = error;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Try at most two live performances, after all searches for the original.
|
|
107
|
+
const performances = [...live].sort((left, right) => right[1] - left[1]).slice(0, 2);
|
|
108
|
+
for (const [youtubeUrl] of performances) {
|
|
109
|
+
try {
|
|
110
|
+
const youtube = await resolveYoutube({ ...input, url: youtubeUrl });
|
|
111
|
+
return { youtube, youtubeUrl, liveFallback: true };
|
|
112
|
+
} catch (error) {
|
|
113
|
+
resolutionError = error;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (resolutionError) throw resolutionError;
|
|
117
|
+
if (searchError) throw searchError;
|
|
118
|
+
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "noMatch");
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function searchYoutube(input: ResolveContext, query: string): Promise<Json[]> {
|
|
59
122
|
const url = new URL("https://www.youtube.com/results");
|
|
60
|
-
url.searchParams.set("search_query",
|
|
123
|
+
url.searchParams.set("search_query", query);
|
|
61
124
|
url.searchParams.set("hl", "en");
|
|
62
125
|
const response = await input.net(url.href, {
|
|
63
126
|
headers: { "accept-language": "en-US,en;q=0.9", cookie: "SOCS=CAI", "user-agent": browserUserAgent() },
|
|
@@ -66,22 +129,15 @@ async function matchYoutube(input: ResolveContext, track: Track): Promise<string
|
|
|
66
129
|
throw new PostfetchError(502, `YouTube search failed: ${response.status}`);
|
|
67
130
|
}
|
|
68
131
|
const html = await response.text();
|
|
69
|
-
// Read the JSON
|
|
70
|
-
//
|
|
132
|
+
// Read the JSON script or legacy assignment without evaluating JavaScript.
|
|
133
|
+
// Assignment parsing balances the object and ignores trailing statements.
|
|
71
134
|
const raw = html.match(/(?:\bytInitialData|window\[["']ytInitialData["']\])\s*=\s*(\{[\s\S]*?)<\/script>/)?.[1];
|
|
72
|
-
const
|
|
135
|
+
const jsonScript = html.match(/<script\b[^>]*\bid=["']yt-initial-data["'][^>]*>([\s\S]*?)<\/script>/i)?.[1];
|
|
136
|
+
const payload = parseJson(jsonScript) ?? parseAssignment(raw);
|
|
73
137
|
if (!payload || !object(payload.contents)) {
|
|
74
138
|
throw new PostfetchError(502, "YouTube search results not found");
|
|
75
139
|
}
|
|
76
|
-
|
|
77
|
-
const ranked = candidates.map((candidate, index) => ({ candidate, score: matchScore(track, candidate) - index / 1000 }))
|
|
78
|
-
.filter(({ score }) => Number.isFinite(score))
|
|
79
|
-
.sort((left, right) => right.score - left.score);
|
|
80
|
-
const id = ranked[0]?.candidate.videoId;
|
|
81
|
-
if (typeof id !== "string") {
|
|
82
|
-
throw new PostfetchError(404, "No matching YouTube video found for this Spotify track", "notFound");
|
|
83
|
-
}
|
|
84
|
-
return `https://www.youtube.com/watch?v=${id}`;
|
|
140
|
+
return videoRenderers(payload.contents);
|
|
85
141
|
}
|
|
86
142
|
|
|
87
143
|
function videoRenderers(value: unknown): Json[] {
|
|
@@ -106,23 +162,66 @@ function contains(haystack: string, needle: string): boolean {
|
|
|
106
162
|
return needle.length > 0 && ` ${haystack} `.includes(` ${needle} `);
|
|
107
163
|
}
|
|
108
164
|
|
|
165
|
+
function artistMatches(value: string, artist: string): boolean {
|
|
166
|
+
if (contains(value, artist)) return true;
|
|
167
|
+
// Recognize long, vowel-free abbreviations such as Chkbns / Cheekbones.
|
|
168
|
+
// Match whole words, never a short/common acronym or arbitrary edit distance.
|
|
169
|
+
return /^[bcdfghjklmnpqrstvwxyz]{6,}$/.test(artist) && value.split(" ").some((word) =>
|
|
170
|
+
/^[a-z]+$/.test(word) && word.replace(/[aeiou]/g, "") === artist
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
109
174
|
function durationSeconds(value: string): number | null {
|
|
110
175
|
if (!/^\d+(?::\d{2}){1,2}$/.test(value)) return null;
|
|
111
176
|
return value.split(":").reduce((seconds, part) => seconds * 60 + Number(part), 0);
|
|
112
177
|
}
|
|
113
178
|
|
|
114
|
-
|
|
179
|
+
// Some auto-generated Topic channels transpose adjacent letters in an artist's
|
|
180
|
+
// name. Allow one such typo in a long word only, with every other word exact.
|
|
181
|
+
function topicArtistMatch(author: string, artist: string): boolean {
|
|
182
|
+
if (!author.endsWith(" topic")) return false;
|
|
183
|
+
const actual = author.slice(0, -6).trim().split(" ");
|
|
184
|
+
const expected = artist.split(" ");
|
|
185
|
+
if (actual.length !== expected.length) return false;
|
|
186
|
+
let typos = 0;
|
|
187
|
+
return expected.every((word, index) => {
|
|
188
|
+
const candidate = actual[index];
|
|
189
|
+
if (candidate === word) return true;
|
|
190
|
+
if (word.length < 6 || candidate.length !== word.length || ++typos > 1) return false;
|
|
191
|
+
for (let offset = 0; offset < word.length - 1; offset += 1) {
|
|
192
|
+
const swapped = word.slice(0, offset) + word[offset + 1] + word[offset] + word.slice(offset + 2);
|
|
193
|
+
if (candidate === swapped) return true;
|
|
194
|
+
}
|
|
195
|
+
return false;
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function matchScore(track: Track, candidate: Json, allowLive = false): number {
|
|
115
200
|
if (typeof candidate.videoId !== "string" || !/^[A-Za-z0-9_-]{11}$/.test(candidate.videoId)) return -Infinity;
|
|
116
201
|
const title = normalize(text(candidate.title));
|
|
117
202
|
const author = normalize(text(candidate.ownerText) || text(candidate.longBylineText) || text(candidate.shortBylineText));
|
|
118
|
-
const
|
|
203
|
+
const trackTitle = track.title.replace(/\s*[([](?:feat\.?|ft\.?).*?[)\]]/gi, "");
|
|
204
|
+
const expected = normalize(trackTitle);
|
|
205
|
+
const artists = track.artists.map(normalize);
|
|
206
|
+
const artistUpload = artistMatches(author, artists[0]) || topicArtistMatch(author, artists[0]);
|
|
207
|
+
const credited = artists.map((artist, index) => artistMatches(`${title} ${author}`, artist) || (index === 0 && artistUpload));
|
|
208
|
+
if (!credited[0]) return -Infinity;
|
|
119
209
|
const tokens = expected.split(" ").filter(Boolean);
|
|
120
210
|
if (tokens.length === 0) return -Infinity;
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (!contains(
|
|
211
|
+
let coverage = tokens.filter((token) => contains(title, token)).length / tokens.length;
|
|
212
|
+
const joinedTitle = expected.replace(/ /g, "");
|
|
213
|
+
const joinedMatch = tokens.length > 1 && joinedTitle.length >= 6 && contains(title, joinedTitle);
|
|
214
|
+
if (joinedMatch) coverage = 1;
|
|
215
|
+
if (coverage < (tokens.length <= 3 ? 1 : 0.8) || (tokens.length <= 3 && !contains(title, expected) && !joinedMatch)) {
|
|
216
|
+
// A soundtrack title may prefix the song and its show/version with project
|
|
217
|
+
// context, e.g. "Project: Subtitle - Song - Show". Require the complete
|
|
218
|
+
// remaining phrase and every credited artist before accepting its omission.
|
|
219
|
+
const parts = trackTitle.split(/\s+[-–—|]\s+/);
|
|
220
|
+
if (parts.length < 3 || !parts[0].includes(":") || artists.length < 2 || !credited.every(Boolean) || !contains(title, normalize(parts.slice(1).join(" ")))) {
|
|
221
|
+
return -Infinity;
|
|
222
|
+
}
|
|
223
|
+
coverage = 1;
|
|
224
|
+
}
|
|
126
225
|
const duration = durationSeconds(text(candidate.lengthText));
|
|
127
226
|
if (duration === null || duration <= 0) return -Infinity; // Excludes live streams and upcoming videos.
|
|
128
227
|
const tolerance = Math.min(30, Math.max(12, (track.durationSeconds ?? duration) * 0.08));
|
|
@@ -132,12 +231,14 @@ function matchScore(track: Track, candidate: Json): number {
|
|
|
132
231
|
// are removed first so e.g. a band called "Live" is not treated as a live edit.
|
|
133
232
|
let recordingTitle = title;
|
|
134
233
|
for (const artist of artists) recordingTitle = recordingTitle.replace(artist, "");
|
|
135
|
-
|
|
234
|
+
const isLive = /\blive\b/.test(recordingTitle);
|
|
235
|
+
const requestedLive = /\blive\b/.test(expected);
|
|
236
|
+
if (isLive !== requestedLive && !(allowLive && isLive && !requestedLive)) return -Infinity;
|
|
237
|
+
for (const version of [/\bremix\b/, /\bcover\b/, /\bkaraoke\b/, /\binstrumental\b/, /\bslowed\b/, /\b(?:sped|speed) up\b/, /\bnightcore\b/, /\bacoustic\b/]) {
|
|
136
238
|
if (version.test(recordingTitle) !== version.test(expected)) return -Infinity;
|
|
137
239
|
}
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
return coverage * 10 + (artistUpload ? 8 : 0) + credited * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
240
|
+
const creditCoverage = credited.filter(Boolean).length / artists.length;
|
|
241
|
+
return coverage * 10 + (artistUpload ? 8 : 0) + creditCoverage * 2 + (contains(title, "official") ? 2 : 0) + 3 * (1 - delta / tolerance);
|
|
141
242
|
}
|
|
142
243
|
|
|
143
244
|
function parseJson(value: string | undefined): Json | null {
|