@letsmoe/otter-sdk 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +185 -2
- package/package.json +40 -4
- package/src/aes-gcm.ts +189 -0
- package/src/browser.ts +39 -0
- package/src/client.ts +23 -0
- package/src/context.ts +87 -0
- package/src/definition.ts +104 -0
- package/src/errors.ts +85 -0
- package/src/extract.ts +134 -0
- package/src/generated/index.ts +79 -0
- package/src/generated/runtime/batcher.ts +275 -0
- package/src/generated/runtime/createClient.ts +68 -0
- package/src/generated/runtime/error.ts +29 -0
- package/src/generated/runtime/fetcher.ts +97 -0
- package/src/generated/runtime/generateGraphqlOperation.ts +225 -0
- package/src/generated/runtime/index.ts +13 -0
- package/src/generated/runtime/linkTypeMap.ts +156 -0
- package/src/generated/runtime/typeSelection.ts +95 -0
- package/src/generated/runtime/types.ts +69 -0
- package/src/generated/schema.graphql +838 -0
- package/src/generated/schema.ts +1611 -0
- package/src/generated/types.ts +1744 -0
- package/src/hosters/dead-link.ts +55 -0
- package/src/hosters/dood.ts +69 -0
- package/src/hosters/filemoon.ts +152 -0
- package/src/hosters/generic.ts +63 -0
- package/src/hosters/index.ts +14 -0
- package/src/hosters/recaptcha.ts +61 -0
- package/src/hosters/registry.ts +74 -0
- package/src/hosters/streamtape.ts +63 -0
- package/src/hosters/vidara.ts +48 -0
- package/src/hosters/vinovo.ts +55 -0
- package/src/hosters/voe-decoding.ts +131 -0
- package/src/hosters/voe-domains.ts +39 -0
- package/src/hosters/voe.ts +95 -0
- package/src/http-client.ts +295 -0
- package/src/http.ts +97 -0
- package/src/index.ts +10 -0
- package/src/languages.ts +73 -0
- package/src/match.ts +120 -0
- package/src/model.ts +110 -0
- package/src/packer.ts +106 -0
- package/src/plugin.ts +4 -0
- package/src/run.ts +88 -0
- package/src/runtime.ts +274 -0
- package/src/testing.ts +123 -0
- package/src/timing.ts +11 -0
- package/src/types.ts +148 -0
- package/src/url.ts +60 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// VOE hides its stream sources in obfuscated page data. These are the
|
|
2
|
+
// reversible steps it combines; each variant of the page uses another order.
|
|
3
|
+
|
|
4
|
+
import type { SourceInput } from "../model.ts";
|
|
5
|
+
import { base64, mimeType } from "../extract.ts";
|
|
6
|
+
|
|
7
|
+
/** Separator strings VOE sprinkles into the encoded data. */
|
|
8
|
+
export const DEFAULT_MARKERS = ["@$", "^^", "~@", "%?", "*~", "!!", "#&"];
|
|
9
|
+
|
|
10
|
+
export function rot13(text: string): string {
|
|
11
|
+
return text.replace(/[A-Za-z]/g, (letter) => {
|
|
12
|
+
const base = letter <= "Z" ? 65 : 97;
|
|
13
|
+
return String.fromCharCode(((letter.charCodeAt(0) - base + 13) % 26) + base);
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function shiftBack(text: string, shift: number): string {
|
|
18
|
+
return Array.from(text, (letter) => String.fromCharCode(letter.charCodeAt(0) - shift)).join("");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function reverse(text: string): string {
|
|
22
|
+
return Array.from(text).reverse().join("");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function stripMarkers(text: string, markers: string[]): string {
|
|
26
|
+
return markers.reduce((stripped, marker) => stripped.split(marker).join(""), text);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** rot13, strip markers, base64, shift by -3, reverse, base64: the format of the current player pages. */
|
|
30
|
+
export function decodeMarkedPayload(payload: string, markers: string[]): string {
|
|
31
|
+
const unmarked = stripMarkers(rot13(payload), markers);
|
|
32
|
+
return base64(reverse(shiftBack(base64(unmarked), 3)));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function sourceFromMap(map: Record<string, unknown>): SourceInput | undefined {
|
|
36
|
+
const candidates: [string, boolean][] = [
|
|
37
|
+
["direct_access_url", true],
|
|
38
|
+
["mp4", true],
|
|
39
|
+
["source", false],
|
|
40
|
+
["hls", false],
|
|
41
|
+
["file", false],
|
|
42
|
+
];
|
|
43
|
+
for (const [key, isDirectFile] of candidates) {
|
|
44
|
+
const url = map[key];
|
|
45
|
+
if (typeof url !== "string" || url === "") continue;
|
|
46
|
+
if (isDirectFile) return { url, mime_type: "video/mp4" };
|
|
47
|
+
return { url, mime_type: mimeType(url, "application/x-mpegURL") };
|
|
48
|
+
}
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function sourceFromText(text: string): SourceInput | undefined {
|
|
53
|
+
const hls = text.match(/https?:\/\/[^\s"']+\.m3u8[^\s"']*/i);
|
|
54
|
+
if (hls) return { url: hls[0], mime_type: "application/x-mpegURL" };
|
|
55
|
+
const mp4 = text.match(/https?:\/\/[^\s"']+\.mp4[^\s"']*/i);
|
|
56
|
+
if (mp4) return { url: mp4[0], mime_type: "video/mp4" };
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** A decoded payload is either a JSON map of sources or free text holding a URL. */
|
|
61
|
+
export function sourceFromDecoded(decoded: string): SourceInput | undefined {
|
|
62
|
+
try {
|
|
63
|
+
return sourceFromMap(JSON.parse(decoded));
|
|
64
|
+
} catch {
|
|
65
|
+
return sourceFromText(decoded);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function safely<Result>(decode: () => Result | undefined): Result | undefined {
|
|
70
|
+
try {
|
|
71
|
+
return decode();
|
|
72
|
+
} catch {
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Older pages: MKGMa="..." holds rot13, base64, shift, reverse, base64. */
|
|
78
|
+
export function fromMkgmaVariable(html: string): SourceInput | undefined {
|
|
79
|
+
const found = html.match(/MKGMa="([^"]+)"/);
|
|
80
|
+
if (!found) return undefined;
|
|
81
|
+
return safely(() => {
|
|
82
|
+
const unscored = rot13(found[1]).replace(/_/g, "");
|
|
83
|
+
return sourceFromDecoded(base64(reverse(shiftBack(base64(unscored), 3))));
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Older pages: a168c='...' holds base64 of the reversed JSON. */
|
|
88
|
+
export function fromA168cVariable(html: string): SourceInput | undefined {
|
|
89
|
+
const found = html.match(/a168c\s*=\s*'([^']+)'/);
|
|
90
|
+
if (!found) return undefined;
|
|
91
|
+
return safely(() => sourceFromDecoded(reverse(base64(found[1].replace(/\\/g, "")))));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Current pages: `<script type="application/json">["payload"]</script>`. */
|
|
95
|
+
export function fromJsonScript(html: string, markers: string[]): SourceInput | undefined {
|
|
96
|
+
const scripts = html.matchAll(/<script[^>]+type=["']application\/json["'][^>]*>([\s\S]*?)<\/script>/gi);
|
|
97
|
+
for (const script of scripts) {
|
|
98
|
+
const source = safely(() => {
|
|
99
|
+
const payload = JSON.parse(script[1].trim());
|
|
100
|
+
if (!Array.isArray(payload) || typeof payload[0] !== "string") return undefined;
|
|
101
|
+
return sourceFromDecoded(decodeMarkedPayload(payload[0], markers));
|
|
102
|
+
});
|
|
103
|
+
if (source) return source;
|
|
104
|
+
}
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** The markers a page's companion script defines: `['@$','^^',...]`. */
|
|
109
|
+
export function markersFromScript(script: string): string[] | undefined {
|
|
110
|
+
const list = script.match(/\[(?:'\W{2}'[,\]]){1,9}/);
|
|
111
|
+
if (!list) return undefined;
|
|
112
|
+
return list[0].slice(2, -2).split("','");
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const PLAIN_PATTERNS = [
|
|
116
|
+
/"hls"\s*:\s*"(https?:[^"]+\.m3u8[^"]*)"/i,
|
|
117
|
+
/'hls'\s*:\s*'(https?:[^']+\.m3u8[^']*)'/i,
|
|
118
|
+
/"file"\s*:\s*"(https?:[^"]+\.m3u8[^"]*)"/i,
|
|
119
|
+
/src\s*:\s*["'](https?:[^"']+\.m3u8[^"']*)["']/i,
|
|
120
|
+
/"file"\s*:\s*"(https?:[^"]+\.mp4[^"]*)"/i,
|
|
121
|
+
/'file'\s*:\s*'(https?:[^']+\.mp4[^']*)'/i,
|
|
122
|
+
/mp4["']:\s*["']([^"']+)["']/i,
|
|
123
|
+
];
|
|
124
|
+
|
|
125
|
+
export function fromPlainPatterns(html: string): SourceInput | undefined {
|
|
126
|
+
for (const pattern of PLAIN_PATTERNS) {
|
|
127
|
+
const found = pattern.exec(html);
|
|
128
|
+
if (found) return { url: found[1], mime_type: mimeType(found[1]) };
|
|
129
|
+
}
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Mirror domains VOE has used, from the ResolveURL project's list. VOE rotates
|
|
2
|
+
// them often; a link on a missing mirror falls back to the generic resolver.
|
|
3
|
+
export const VOE_DOMAINS: string[] = [
|
|
4
|
+
"voe.sx", "voe-unblock.com", "voe-unblock.net", "voeunblock.com",
|
|
5
|
+
"un-block-voe.net", "voeunbl0ck.com", "voeunblck.com", "voeunblk.com",
|
|
6
|
+
"voe-un-block.com", "jonathansociallike.com", "voeun-block.net", "v-o-e-unblock.com",
|
|
7
|
+
"edwardarriveoften.com", "nathanfromsubject.com", "audaciousdefaulthouse.com", "launchreliantcleaverriver.com",
|
|
8
|
+
"kennethofficialitem.com", "reputationsheriffkennethsand.com", "fittingcentermondaysunday.com", "lukecomparetwo.com",
|
|
9
|
+
"housecardsummerbutton.com", "fraudclatterflyingcar.com", "wolfdyslectic.com", "bigclatterhomesguideservice.com",
|
|
10
|
+
"uptodatefinishconferenceroom.com", "jayservicestuff.com", "realfinanceblogcenter.com", "tinycat-voe-fashion.com",
|
|
11
|
+
"35volitantplimsoles5.com", "20demidistance9elongations.com", "telyn610zoanthropy.com", "toxitabellaeatrebates306.com",
|
|
12
|
+
"greaseball6eventual20.com", "745mingiestblissfully.com", "19turanosephantasia.com", "30sensualizeexpression.com",
|
|
13
|
+
"321naturelikefurfuroid.com", "449unceremoniousnasoseptal.com", "guidon40hyporadius9.com", "cyamidpulverulence530.com",
|
|
14
|
+
"boonlessbestselling244.com", "antecoxalbobbing1010.com", "matriculant401merited.com", "scatch176duplicities.com",
|
|
15
|
+
"availedsmallest.com", "counterclockwisejacky.com", "simpulumlamerop.com", "paulkitchendark.com",
|
|
16
|
+
"metagnathtuggers.com", "gamoneinterrupted.com", "chromotypic.com", "crownmakermacaronicism.com",
|
|
17
|
+
"generatesnitrosate.com", "yodelswartlike.com", "figeterpiazine.com", "strawberriesporail.com",
|
|
18
|
+
"valeronevijao.com", "timberwoodanotia.com", "apinchcaseation.com", "nectareousoverelate.com",
|
|
19
|
+
"nonesnanking.com", "kathleenmemberhistory.com", "stevenimaginelittle.com", "jamiesamewalk.com",
|
|
20
|
+
"bradleyviewdoctor.com", "sandrataxeight.com", "graceaddresscommunity.com", "shannonpersonalcost.com",
|
|
21
|
+
"cindyeyefinal.com", "michaelapplysome.com", "sethniceletter.com", "brucevotewithin.com",
|
|
22
|
+
"rebeccaneverbase.com", "loriwithinfamily.com", "roberteachfinal.com", "erikcoldperson.com",
|
|
23
|
+
"jasminetesttry.com", "heatherdiscussionwhen.com", "robertplacespace.com", "alleneconomicmatter.com",
|
|
24
|
+
"josephseveralconcern.com", "donaldlineelse.com", "lisatrialidea.com", "toddpartneranimal.com",
|
|
25
|
+
"jamessoundcost.com", "brittneystandardwestern.com", "sandratableother.com", "robertordercharacter.com",
|
|
26
|
+
"maxfinishseveral.com", "chuckle-tube.com", "kristiesoundsimply.com", "adrianmissionminute.com",
|
|
27
|
+
"richardsignfish.com", "jennifercertaindevelopment.com", "diananatureforeign.com", "goofy-banana.com",
|
|
28
|
+
"mariatheserepublican.com", "johnalwayssame.com", "kellywhatcould.com", "jilliandescribecompany.com",
|
|
29
|
+
"lukesitturn.com", "mikaylaarealike.com", "christopheruntilpoint.com", "walterprettytheir.com",
|
|
30
|
+
"crystaltreatmenteast.com", "lauradaydo.com", "smoki.cc", "lancewhosedifficult.com",
|
|
31
|
+
"ogladaj.me", "dianaavoidthey.com", "jefferycontrolmodel.com", "marissasharecareer.com",
|
|
32
|
+
"charlestoughrace.com", "ianrequireadult.com", "timmaybealready.com", "jessicayeahcatch.com",
|
|
33
|
+
"johnbeyondnation.com", "jeanprofessorcentral.com", "juliewomanwish.com",
|
|
34
|
+
"garylargeavailable.com", "jennifereconomicgive.com", "pamelachangemission.com", "ellenpoliticalfollow.com",
|
|
35
|
+
"caseyimpactstation.com", "matthewhotelscience.com", "jessicachoosemake.com", "stevenfamilyedge.com",
|
|
36
|
+
"tracylocalschool.com", "eugenemakedraw.com", "johnfullwonder.com", "katherineschoolphone.com",
|
|
37
|
+
"jamesbornmain.com", "jeremyparticipantanything.com", "teresapoliticallearn.com", "megankiddifference.com",
|
|
38
|
+
...Array.from({ length: 10 }, (_, index) => `voeunblock${index + 1}.com`),
|
|
39
|
+
];
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { PluginBlockedError, PluginSourceError } from "../errors.ts";
|
|
2
|
+
import type { PluginContext } from "../definition.ts";
|
|
3
|
+
import type { SourceInput } from "../model.ts";
|
|
4
|
+
import {
|
|
5
|
+
DEFAULT_MARKERS,
|
|
6
|
+
fromA168cVariable,
|
|
7
|
+
fromJsonScript,
|
|
8
|
+
fromMkgmaVariable,
|
|
9
|
+
fromPlainPatterns,
|
|
10
|
+
markersFromScript,
|
|
11
|
+
} from "./voe-decoding.ts";
|
|
12
|
+
import { originOf, resolveAgainst } from "../url.ts";
|
|
13
|
+
import { getLinkPage } from "./dead-link.ts";
|
|
14
|
+
import { defineHoster } from "./registry.ts";
|
|
15
|
+
import { VOE_DOMAINS } from "./voe-domains.ts";
|
|
16
|
+
|
|
17
|
+
const MAX_REDIRECT_PAGES = 3;
|
|
18
|
+
const REDIRECT_PAGE_MAX_LENGTH = 3000;
|
|
19
|
+
|
|
20
|
+
interface PlayerPage {
|
|
21
|
+
url: string;
|
|
22
|
+
html: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Small pages that only forward the browser: either to a mirror (the else
|
|
27
|
+
* branch of the permanentToken check) or via `window.location.href`.
|
|
28
|
+
*/
|
|
29
|
+
function redirectTarget(html: string): string | undefined {
|
|
30
|
+
if (html.length > REDIRECT_PAGE_MAX_LENGTH) return undefined;
|
|
31
|
+
if (!/permanentToken|const currentUrl/.test(html)) return undefined;
|
|
32
|
+
const elseBranch = html.match(/}\s*else\s*\{[^}]*window\.location\.href\s*=\s*['"]([^'"]+)['"]/);
|
|
33
|
+
if (elseBranch) return elseBranch[1];
|
|
34
|
+
const direct = html.match(/window\.location\.href\s*=\s*['"]([^'"]+)['"]/);
|
|
35
|
+
if (direct) return direct[1];
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function loadPlayerPage(url: string, ctx: PluginContext): Promise<PlayerPage> {
|
|
40
|
+
let current = url;
|
|
41
|
+
for (let hop = 0; hop < MAX_REDIRECT_PAGES; hop++) {
|
|
42
|
+
const response = await getLinkPage(current, ctx);
|
|
43
|
+
const html = await response.text();
|
|
44
|
+
const target = redirectTarget(html);
|
|
45
|
+
if (!target) return { url: current, html };
|
|
46
|
+
current = resolveAgainst(current, target);
|
|
47
|
+
}
|
|
48
|
+
throw new PluginSourceError("extraction_failed", `VOE: too many redirect pages from ${url}`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Current pages keep their marker list in a script next to the payload. */
|
|
52
|
+
async function markersOf(html: string, pageUrl: string, ctx: PluginContext): Promise<string[]> {
|
|
53
|
+
const scriptSource = html.match(/json">\["[^"]+"\]<\/script>\s*<script\s*src="([^"]+)"/);
|
|
54
|
+
if (!scriptSource) return DEFAULT_MARKERS;
|
|
55
|
+
try {
|
|
56
|
+
const script = await ctx.http.get(resolveAgainst(pageUrl, scriptSource[1]), { referer: pageUrl });
|
|
57
|
+
return markersFromScript(await script.text()) || DEFAULT_MARKERS;
|
|
58
|
+
} catch {
|
|
59
|
+
return DEFAULT_MARKERS;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function extractSource(page: PlayerPage, ctx: PluginContext): Promise<SourceInput | undefined> {
|
|
64
|
+
const markers = await markersOf(page.html, page.url, ctx);
|
|
65
|
+
return (
|
|
66
|
+
fromJsonScript(page.html, markers) ||
|
|
67
|
+
fromMkgmaVariable(page.html) ||
|
|
68
|
+
fromA168cVariable(page.html) ||
|
|
69
|
+
fromPlainPatterns(page.html)
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// What every VOE player page carries in some form: the encoded payload, its older variables, or the plain HLS link.
|
|
74
|
+
const PLAYER_PAYLOAD = /<script[^>]+type=["']application\/json["']|\b(?:mkgma|a168c)\b|["']hls["']\s*:|\.m3u8/i;
|
|
75
|
+
|
|
76
|
+
async function retrieveVoe(url: string, ctx: PluginContext): Promise<SourceInput> {
|
|
77
|
+
const page = await loadPlayerPage(url, ctx);
|
|
78
|
+
const source = await extractSource(page, ctx);
|
|
79
|
+
if (source) {
|
|
80
|
+
return { ...source, headers: { Referer: originOf(page.url) + "/" } };
|
|
81
|
+
}
|
|
82
|
+
// A page without any player payload is a challenge (VOE domains sit behind DDoS-Guard) or an interstitial, not a changed player.
|
|
83
|
+
if (!PLAYER_PAYLOAD.test(page.html)) {
|
|
84
|
+
throw new PluginBlockedError(`VOE: ${page.url} served "${pageTitleOf(page.html)}" instead of the player`);
|
|
85
|
+
}
|
|
86
|
+
throw new PluginSourceError("extraction_failed", `VOE: no stream found on ${page.url}`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function pageTitleOf(html: string): string {
|
|
90
|
+
const title = html.match(/<title[^>]*>([^<]*)<\/title>/i);
|
|
91
|
+
if (!title) return "a page without a title";
|
|
92
|
+
return title[1].trim();
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export const voe = defineHoster({ name: "VOE", domains: VOE_DOMAINS, retrieve: retrieveVoe });
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
import { parse, type HTMLElement } from "node-html-parser";
|
|
2
|
+
import { HttpError, PluginBlockedError, PluginRateLimitError, PluginSourceError } from "./errors.ts";
|
|
3
|
+
import { resolveAgainst } from "./url.ts";
|
|
4
|
+
import { BROWSER_HEADERS, fetchViaSolver, hasSolver } from "./http.ts";
|
|
5
|
+
|
|
6
|
+
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
7
|
+
const DEFAULT_RETRIES = 2;
|
|
8
|
+
const RETRY_BACKOFF_MS = 250;
|
|
9
|
+
|
|
10
|
+
const CLOUDFLARE_MARKERS = [
|
|
11
|
+
"cf-chl-",
|
|
12
|
+
"_cf_chl_opt",
|
|
13
|
+
"challenge-platform",
|
|
14
|
+
"cf-browser-verification",
|
|
15
|
+
"challenges.cloudflare.com",
|
|
16
|
+
"attention required! | cloudflare",
|
|
17
|
+
];
|
|
18
|
+
|
|
19
|
+
export interface RequestOptions {
|
|
20
|
+
headers?: Record<string, string>;
|
|
21
|
+
referer?: string;
|
|
22
|
+
/** Default 15000. */
|
|
23
|
+
timeout_ms?: number;
|
|
24
|
+
/** Extra attempts after a network error or 5xx. Default 2. */
|
|
25
|
+
retries?: number;
|
|
26
|
+
/** Return 4xx/5xx responses instead of throwing; a Cloudflare block still throws. */
|
|
27
|
+
allow_error_status?: boolean;
|
|
28
|
+
/** Default true. When false, a redirect is returned as is and its target is in the Location header. */
|
|
29
|
+
follow_redirects?: boolean;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface PostOptions extends RequestOptions {
|
|
33
|
+
/** A raw body, or an object sent as application/x-www-form-urlencoded. */
|
|
34
|
+
body?: string | Record<string, string>;
|
|
35
|
+
/** Sent as application/json; wins over `body`. */
|
|
36
|
+
json?: unknown;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** A completed request. The body is read once and cached, so the readers can be mixed freely. */
|
|
40
|
+
export interface HttpResponse {
|
|
41
|
+
readonly status: number;
|
|
42
|
+
/** Final URL after redirects. */
|
|
43
|
+
readonly url: string;
|
|
44
|
+
text(): Promise<string>;
|
|
45
|
+
json<Data = unknown>(): Promise<Data>;
|
|
46
|
+
html(): Promise<HTMLElement>;
|
|
47
|
+
response(): Response;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface HttpClient {
|
|
51
|
+
get(url: string, options?: RequestOptions): Promise<HttpResponse>;
|
|
52
|
+
post(url: string, options?: PostOptions): Promise<HttpResponse>;
|
|
53
|
+
/** Follows redirects and returns the destination; returns `url` itself when the request fails. */
|
|
54
|
+
finalUrl(url: string, options?: RequestOptions): Promise<string>;
|
|
55
|
+
/** Resolves a path or URL against the plugin's (possibly overridden) base_url. */
|
|
56
|
+
absolute(path: string): string;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface HttpClientConfig {
|
|
60
|
+
baseUrl?: string;
|
|
61
|
+
languages?: string[];
|
|
62
|
+
fetch?: typeof fetch;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
interface PreparedRequest {
|
|
66
|
+
url: string;
|
|
67
|
+
init: RequestInit;
|
|
68
|
+
retries: number;
|
|
69
|
+
timeoutMs: number;
|
|
70
|
+
allowErrorStatus: boolean;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** A response whose body may already have been read. */
|
|
74
|
+
interface RawResult {
|
|
75
|
+
response: Response;
|
|
76
|
+
bodyText?: string;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
class FetchedResponse implements HttpResponse {
|
|
80
|
+
private bodyText?: string;
|
|
81
|
+
|
|
82
|
+
constructor(
|
|
83
|
+
private readonly rawResponse: Response,
|
|
84
|
+
preloadedText?: string,
|
|
85
|
+
) {
|
|
86
|
+
this.bodyText = preloadedText;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
get status(): number {
|
|
90
|
+
return this.rawResponse.status;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
get url(): string {
|
|
94
|
+
return this.rawResponse.url;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async text(): Promise<string> {
|
|
98
|
+
if (this.bodyText === undefined) {
|
|
99
|
+
this.bodyText = await this.rawResponse.text();
|
|
100
|
+
}
|
|
101
|
+
return this.bodyText;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async json<Data = unknown>(): Promise<Data> {
|
|
105
|
+
return JSON.parse(await this.text()) as Data;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async html(): Promise<HTMLElement> {
|
|
109
|
+
return parse(await this.text());
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
response(): Response {
|
|
113
|
+
return this.rawResponse;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Accept-Language for the plugin's first language, falling back to English. */
|
|
118
|
+
export function acceptLanguage(languages: string[] | undefined): string {
|
|
119
|
+
const english = "en-US,en;q=0.9";
|
|
120
|
+
if (!languages || languages.length === 0) return english;
|
|
121
|
+
const language = languages[0];
|
|
122
|
+
if (language === "en") return english;
|
|
123
|
+
const base = language.split("-")[0];
|
|
124
|
+
if (language === base) return `${language}-${language.toUpperCase()},${language};q=0.9,en;q=0.8`;
|
|
125
|
+
return `${language},${base};q=0.9,en;q=0.8`;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function isCloudflareChallenge(status: number, headers: Headers, bodyText: string): boolean {
|
|
129
|
+
if (status !== 403 && status !== 503) return false;
|
|
130
|
+
if (headers.get("cf-mitigated") === "challenge") return true;
|
|
131
|
+
const lowered = bodyText.toLowerCase();
|
|
132
|
+
if (lowered.includes("just a moment") && lowered.includes("cloudflare")) return true;
|
|
133
|
+
return CLOUDFLARE_MARKERS.some((marker) => lowered.includes(marker));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The error a response status stands for when the caller did not ask for error responses. */
|
|
137
|
+
export function errorForStatus(url: string, status: number): Error {
|
|
138
|
+
const message = `HTTP ${status} fetching ${url}`;
|
|
139
|
+
if (status === 429) return new PluginRateLimitError(message, status);
|
|
140
|
+
if (status >= 500) return new PluginSourceError("source_offline", message, status);
|
|
141
|
+
return new HttpError(message, status);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function sleep(milliseconds: number): Promise<void> {
|
|
145
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function formEncode(fields: Record<string, string>): string {
|
|
149
|
+
return new URLSearchParams(fields).toString();
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export function createHttpClient(config: HttpClientConfig): HttpClient {
|
|
153
|
+
return new PluginHttpClient(config);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
class PluginHttpClient implements HttpClient {
|
|
157
|
+
private readonly fetchImplementation: typeof fetch;
|
|
158
|
+
|
|
159
|
+
constructor(private readonly config: HttpClientConfig) {
|
|
160
|
+
this.fetchImplementation = config.fetch ?? fetch;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
get(url: string, options: RequestOptions = {}): Promise<HttpResponse> {
|
|
164
|
+
return this.send(this.prepare("GET", url, options));
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
post(url: string, options: PostOptions = {}): Promise<HttpResponse> {
|
|
168
|
+
const request = this.prepare("POST", url, options);
|
|
169
|
+
this.attachBody(request.init, options);
|
|
170
|
+
return this.send(request);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async finalUrl(url: string, options: RequestOptions = {}): Promise<string> {
|
|
174
|
+
const request = this.prepare("GET", url, options);
|
|
175
|
+
try {
|
|
176
|
+
const response = await this.fetchImplementation(request.url, this.withTimeout(request));
|
|
177
|
+
return response.url || request.url;
|
|
178
|
+
} catch {
|
|
179
|
+
return request.url;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
absolute(path: string): string {
|
|
184
|
+
if (/^https?:\/\//i.test(path)) return path;
|
|
185
|
+
if (path.startsWith("//")) return "https:" + path;
|
|
186
|
+
if (!this.config.baseUrl) {
|
|
187
|
+
throw new Error(`cannot resolve "${path}": the plugin declares no base_url`);
|
|
188
|
+
}
|
|
189
|
+
return resolveAgainst(this.config.baseUrl + "/", path);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
private prepare(method: string, path: string, options: RequestOptions): PreparedRequest {
|
|
193
|
+
const headers: Record<string, string> = {
|
|
194
|
+
"User-Agent": BROWSER_HEADERS["User-Agent"],
|
|
195
|
+
Accept: BROWSER_HEADERS.Accept,
|
|
196
|
+
"Accept-Language": acceptLanguage(this.config.languages),
|
|
197
|
+
};
|
|
198
|
+
if (options.referer) headers.Referer = options.referer;
|
|
199
|
+
Object.assign(headers, options.headers);
|
|
200
|
+
|
|
201
|
+
let retries = DEFAULT_RETRIES;
|
|
202
|
+
if (options.retries !== undefined) retries = options.retries;
|
|
203
|
+
const init: RequestInit = { method, headers };
|
|
204
|
+
if (options.follow_redirects === false) init.redirect = "manual";
|
|
205
|
+
return {
|
|
206
|
+
url: this.absolute(path),
|
|
207
|
+
init,
|
|
208
|
+
retries,
|
|
209
|
+
timeoutMs: options.timeout_ms || DEFAULT_TIMEOUT_MS,
|
|
210
|
+
allowErrorStatus: options.allow_error_status === true,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
private withTimeout(request: PreparedRequest): RequestInit {
|
|
215
|
+
return { ...request.init, signal: AbortSignal.timeout(request.timeoutMs) };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
private attachBody(init: RequestInit, options: PostOptions): void {
|
|
219
|
+
const headers = init.headers as Record<string, string>;
|
|
220
|
+
if (options.json !== undefined) {
|
|
221
|
+
headers["Content-Type"] = "application/json";
|
|
222
|
+
init.body = JSON.stringify(options.json);
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
if (typeof options.body === "string") {
|
|
226
|
+
init.body = options.body;
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
if (options.body) {
|
|
230
|
+
headers["Content-Type"] = "application/x-www-form-urlencoded";
|
|
231
|
+
init.body = formEncode(options.body);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private async send(request: PreparedRequest): Promise<HttpResponse> {
|
|
236
|
+
const raw = await this.fetchWithRetries(request);
|
|
237
|
+
const status = raw.response.status;
|
|
238
|
+
if (status >= 200 && status < 400) return new FetchedResponse(raw.response, raw.bodyText);
|
|
239
|
+
|
|
240
|
+
const bodyText = await this.readBody(raw);
|
|
241
|
+
if (isCloudflareChallenge(status, raw.response.headers, bodyText)) {
|
|
242
|
+
return this.solveChallenge(request, raw.response);
|
|
243
|
+
}
|
|
244
|
+
if (request.allowErrorStatus) return new FetchedResponse(raw.response, bodyText);
|
|
245
|
+
throw errorForStatus(request.url, status);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
private async readBody(raw: RawResult): Promise<string> {
|
|
249
|
+
if (raw.bodyText !== undefined) return raw.bodyText;
|
|
250
|
+
return raw.response.text();
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
private async solveChallenge(request: PreparedRequest, response: Response): Promise<HttpResponse> {
|
|
254
|
+
const message = `${request.url} is behind a Cloudflare challenge`;
|
|
255
|
+
if (request.init.method !== "GET" || !hasSolver()) {
|
|
256
|
+
throw new PluginBlockedError(message, response.status);
|
|
257
|
+
}
|
|
258
|
+
try {
|
|
259
|
+
const solvedHtml = await fetchViaSolver(request.url);
|
|
260
|
+
return new FetchedResponse(response, solvedHtml);
|
|
261
|
+
} catch (error) {
|
|
262
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
263
|
+
throw new PluginBlockedError(`${message} and the solver failed: ${reason}`, response.status);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Retries network errors and 5xx; returns the last response once attempts run out. */
|
|
268
|
+
private async fetchWithRetries(request: PreparedRequest): Promise<RawResult> {
|
|
269
|
+
let lastFailure = "";
|
|
270
|
+
for (let attempt = 0; attempt <= request.retries; attempt++) {
|
|
271
|
+
if (attempt > 0) await sleep(RETRY_BACKOFF_MS * attempt);
|
|
272
|
+
try {
|
|
273
|
+
const raw = await this.fetchOnce(request);
|
|
274
|
+
if (!this.shouldRetry(raw) || attempt === request.retries) return raw;
|
|
275
|
+
} catch (error) {
|
|
276
|
+
lastFailure = error instanceof Error ? error.message : String(error);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
throw new PluginSourceError("source_offline", `${request.url} is unreachable: ${lastFailure}`);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** A Cloudflare challenge answers 503 too, and retrying it cannot help. */
|
|
283
|
+
private shouldRetry(raw: RawResult): boolean {
|
|
284
|
+
const status = raw.response.status;
|
|
285
|
+
if (status < 500) return false;
|
|
286
|
+
return !isCloudflareChallenge(status, raw.response.headers, raw.bodyText || "");
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** Reads the body of a 503 up front, because that is where a challenge page shows. */
|
|
290
|
+
private async fetchOnce(request: PreparedRequest): Promise<RawResult> {
|
|
291
|
+
const response = await this.fetchImplementation(request.url, this.withTimeout(request));
|
|
292
|
+
if (response.status !== 503) return { response };
|
|
293
|
+
return { response, bodyText: await response.text() };
|
|
294
|
+
}
|
|
295
|
+
}
|
package/src/http.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
export const BROWSER_HEADERS = {
|
|
2
|
+
"User-Agent":
|
|
3
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
4
|
+
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
5
|
+
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
|
6
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export function resolveUrl(base: string, path: string): string {
|
|
10
|
+
if (path.startsWith("http")) return path;
|
|
11
|
+
if (path.startsWith("//")) return "https:" + path;
|
|
12
|
+
const origin = new URL(base).origin;
|
|
13
|
+
return origin + (path.startsWith("/") ? path : "/" + path);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export async function fetchHtml(
|
|
17
|
+
url: string,
|
|
18
|
+
extraHeaders?: Record<string, string>
|
|
19
|
+
): Promise<string> {
|
|
20
|
+
const res = await fetch(url, {
|
|
21
|
+
headers: { ...BROWSER_HEADERS, ...extraHeaders },
|
|
22
|
+
});
|
|
23
|
+
if (!res.ok) throw new Error(`HTTP ${res.status} fetching ${url}`);
|
|
24
|
+
return res.text();
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// FlareSolverr endpoint (e.g. http://127.0.0.1:8191/v1), injected via env by the
|
|
28
|
+
// backend. Absent = no solver configured.
|
|
29
|
+
function solverEndpoint(): string | undefined {
|
|
30
|
+
const url = process.env.FLARESOLVERR_URL;
|
|
31
|
+
return url && url.trim() ? url.trim() : undefined;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function hasSolver(): boolean {
|
|
35
|
+
return solverEndpoint() !== undefined;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Fetches a URL through FlareSolverr, which drives a headless browser to clear
|
|
39
|
+
// Cloudflare's challenge and returns the fully rendered HTML. Throws when no
|
|
40
|
+
// solver is configured or the solver reports a non-ok status.
|
|
41
|
+
export async function fetchViaSolver(url: string, maxTimeoutMs = 60_000): Promise<string> {
|
|
42
|
+
const endpoint = solverEndpoint();
|
|
43
|
+
if (!endpoint) throw new Error("no FLARESOLVERR_URL configured");
|
|
44
|
+
|
|
45
|
+
const res = await fetch(endpoint, {
|
|
46
|
+
method: "POST",
|
|
47
|
+
headers: { "Content-Type": "application/json" },
|
|
48
|
+
body: JSON.stringify({ cmd: "request.get", url, maxTimeout: maxTimeoutMs }),
|
|
49
|
+
signal: AbortSignal.timeout(maxTimeoutMs + 5_000),
|
|
50
|
+
});
|
|
51
|
+
if (!res.ok) throw new Error(`solver HTTP ${res.status} fetching ${url}`);
|
|
52
|
+
|
|
53
|
+
const body = (await res.json()) as {
|
|
54
|
+
status?: string;
|
|
55
|
+
message?: string;
|
|
56
|
+
solution?: { response?: string };
|
|
57
|
+
};
|
|
58
|
+
if (body.status !== "ok" || !body.solution?.response) {
|
|
59
|
+
throw new Error(`solver failed for ${url}: ${body.message ?? "unknown"}`);
|
|
60
|
+
}
|
|
61
|
+
return body.solution.response;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Fetches HTML with a plain request, transparently retrying through the solver
|
|
65
|
+
// when the origin answers 403 (Cloudflare block) and a solver is available.
|
|
66
|
+
export async function fetchHtmlSmart(
|
|
67
|
+
url: string,
|
|
68
|
+
extraHeaders?: Record<string, string>
|
|
69
|
+
): Promise<string> {
|
|
70
|
+
try {
|
|
71
|
+
return await fetchHtml(url, extraHeaders);
|
|
72
|
+
} catch (error) {
|
|
73
|
+
const blocked = error instanceof Error && error.message.includes("403");
|
|
74
|
+
if (blocked && hasSolver()) return fetchViaSolver(url);
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export async function followRedirect(url: string): Promise<string> {
|
|
80
|
+
try {
|
|
81
|
+
const res = await fetch(url, {
|
|
82
|
+
headers: BROWSER_HEADERS,
|
|
83
|
+
redirect: "follow",
|
|
84
|
+
});
|
|
85
|
+
return res.url || url;
|
|
86
|
+
} catch {
|
|
87
|
+
return url;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function domainOf(url: string): string {
|
|
92
|
+
try {
|
|
93
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
94
|
+
} catch {
|
|
95
|
+
return "";
|
|
96
|
+
}
|
|
97
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export * from "./types.ts";
|
|
2
|
+
export * from "./http.ts";
|
|
3
|
+
export * from "./plugin.ts";
|
|
4
|
+
export * from "./model.ts";
|
|
5
|
+
export * from "./errors.ts";
|
|
6
|
+
export { match, type Matcher } from "./match.ts";
|
|
7
|
+
export { extract, type Extractor, type ExtractedMedia } from "./extract.ts";
|
|
8
|
+
export type { HttpClient, HttpResponse, PostOptions, RequestOptions } from "./http-client.ts";
|
|
9
|
+
export type { Browser } from "./browser.ts";
|
|
10
|
+
export { defineHoster, hosters, HosterRegistry, type HosterDefinition } from "./hosters/index.ts";
|