@serifpersia/dango 3.6.7 → 3.6.9

package/server/dist/providers/remote-loader.js

@@ -1,614 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import crypto from 'crypto';
-import { execFileSync } from 'node:child_process';
-import { pathToFileURL } from 'url';
-import * as cheerio from 'cheerio';
-import { gotScraping } from 'got-scraping';
-import logger from '../logger.js';
-import { CONFIG } from '../config.js';
-import { requestContext } from '../utils/request-context.js';
-import { sanitizeCfClearance, buildCfClearanceCookie } from '../utils/cookie.utils.js';
-import { decodeMaybeGzip, parseJsonBody } from '../utils/http.utils.js';
-import { buildQueryVariants, pickBestMatch } from './title-matching.js';
-import { anilistRequest, parseMalId, searchAnilistByTitle } from '../lib/anilist.js';
-import { kitsuMetaByAnilistId } from '../lib/kitsu.js';
-import { getTmdbKey, TMDB_BASE, TMDB_IMAGE } from '../lib/tmdb.js';
-import { parseRegistry, } from './remote-types.js';
-const FETCH_TIMEOUT_MS = 15000;
-const GENERIC_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
-function withTimeout(p, ms, label) {
-    let timer;
-    const timeout = new Promise((_, reject) => {
-        timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
-    });
-    return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
-}
-function isHttp(url) {
-    return /^https?:\/\//i.test(url);
-}
-function stripFilePrefix(p) {
-    if (p.startsWith('file://')) {
-        const u = new URL(p);
-        return u.pathname;
-    }
-    return p;
-}
-function looksLikePath(p) {
-    if (isHttp(p) || p.startsWith('file://'))
-        return false;
-    return p.includes('/') || p.includes('\\') || p.endsWith('.json');
-}
-async function readManifestBytes(registryUrl) {
-    if (isHttp(registryUrl)) {
-        const res = await withTimeout(fetch(registryUrl, { headers: { 'User-Agent': GENERIC_UA, Accept: 'application/json' } }), FETCH_TIMEOUT_MS, 'registry fetch');
-        if (!res.ok)
-            throw new Error(`registry HTTP ${res.status}: ${registryUrl}`);
-        return decodeMaybeGzip(Buffer.from(await res.arrayBuffer()));
-    }
-    const filePath = path.resolve(stripFilePrefix(registryUrl));
-    return await fs.promises.readFile(filePath);
-}
-async function readModuleBytes(entryUrl) {
-    if (isHttp(entryUrl)) {
-        const res = await withTimeout(fetch(entryUrl, { headers: { 'User-Agent': GENERIC_UA, Accept: 'text/javascript,*/*' } }), FETCH_TIMEOUT_MS, 'provider fetch');
-        if (!res.ok)
-            throw new Error(`provider HTTP ${res.status}: ${entryUrl}`);
-        return decodeMaybeGzip(Buffer.from(await res.arrayBuffer()));
-    }
-    const filePath = path.resolve(stripFilePrefix(entryUrl));
-    return await fs.promises.readFile(filePath);
-}
-function resolveEntryUrl(entry, registryUrl) {
-    const e = (entry ?? '').trim();
-    if (isHttp(e) || e.startsWith('file://'))
-        return e;
-    if (isHttp(registryUrl))
-        return new URL(e, registryUrl).href;
-    const baseDir = path.dirname(path.resolve(stripFilePrefix(registryUrl)));
-    return path.resolve(baseDir, e);
-}
-function sha256Hex(data) {
-    return crypto.createHash('sha256').update(data).digest('hex');
-}
-function aes256CbcDecryptJson(b64url, keyUtf8, ivUtf8) {
-    let encrypted;
-    try {
-        encrypted = Buffer.from(b64url, 'base64url');
-    }
-    catch {
-        return null;
-    }
-    if (!encrypted.length || encrypted.length % 16 !== 0)
-        return null;
-    try {
-        const key = Buffer.alloc(32);
-        Buffer.from(keyUtf8).copy(key);
-        const decipher = crypto.createDecipheriv('aes-256-cbc', key, Buffer.from(ivUtf8));
-        const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
-        return JSON.parse(decrypted.toString('utf8'));
-    }
-    catch {
-        return null;
-    }
-}
-function curlRaw(url, headers) {
-    const args = [
-        '-sSL',
-        '-A',
-        GENERIC_UA,
-        ...Object.entries(headers ?? {}).flatMap(([k, v]) => ['-H', `${k}: ${v}`]),
-        '--max-time',
-        '15',
-        '-w',
-        '\n__HTTP_STATUS__%{http_code}',
-        '--',
-        url,
-    ];
-    let out;
-    try {
-        out = execFileSync('curl', args, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
-    }
-    catch {
-        return null;
-    }
-    const m = out.match(/__HTTP_STATUS__(\d+)\s*$/);
-    const status = m ? parseInt(m[1], 10) : 0;
-    return { status, body: m && m.index !== undefined ? out.slice(0, m.index) : out };
-}
-function createCtx(cache) {
-    const fetchWithTimeout = async (url, init, ms = FETCH_TIMEOUT_MS) => {
-        return withTimeout(fetch(url, {
-            ...init,
-            headers: { 'User-Agent': GENERIC_UA, ...init?.headers },
-        }), ms, `fetch ${url}`);
-    };
-    return {
-        cache: {
-            get: (key) => cache.get(key),
-            set: (key, value, ttlSeconds) => {
-                cache.set(key, value, ttlSeconds);
-            },
-        },
-        logger: {
-            info: (obj, msg) => {
-                logger.info(obj, msg);
-            },
-            warn: (obj, msg) => {
-                logger.warn(obj, msg);
-            },
-            error: (obj, msg) => {
-                logger.error(obj, msg);
-            },
-            debug: (obj, msg) => {
-                logger.debug(obj, msg);
-            },
-        },
-        fetchText: async (url, init) => {
-            try {
-                const res = await fetchWithTimeout(url, init);
-                if (!res.ok)
-                    return null;
-                return await res.text();
-            }
-            catch {
-                return null;
-            }
-        },
-        fetchJson: async (url, init) => {
-            const res = await fetchWithTimeout(url, init);
-            if (!res.ok)
-                throw new Error(`HTTP ${res.status}: ${url}`);
-            return parseJsonBody(res);
-        },
-        proxyUrl: (rawUrl, referer) => {
-            return `/api/proxy?url=${encodeURIComponent(rawUrl)}&referer=${encodeURIComponent(referer)}`;
-        },
-        userAgent: GENERIC_UA,
-        titleMatch: {
-            buildQueryVariants,
-            pickBestMatch,
-        },
-        resolveBestShowId: async (title, romaji, searchFn) => {
-            const targets = [title, romaji].filter((t) => typeof t === 'string' && t.trim().length > 0);
-            if (targets.length === 0)
-                return null;
-            for (const variant of buildQueryVariants(title, romaji)) {
-                let results;
-                try {
-                    results = await searchFn(variant);
-                }
-                catch (e) {
-                    if (e.message === 'AUTH_REQUIRED')
-                        throw e;
-                    continue;
-                }
-                if (!results || results.length === 0)
-                    continue;
-                const match = pickBestMatch(results.filter((r) => r && typeof r.id === 'string' && r.id.length > 0), targets);
-                if (match)
-                    return match.item.id;
-            }
-            return null;
-        },
-        anilist: {
-            request: (query, vars) => anilistRequest(query, vars),
-            parseMalId,
-            searchByTitle: searchAnilistByTitle,
-        },
-        kitsu: {
-            metaByAnilistId: kitsuMetaByAnilistId,
-        },
-        tmdb: {
-            base: TMDB_BASE,
-            image: TMDB_IMAGE,
-            get: async (path) => {
-                try {
-                    const key = await getTmdbKey();
-                    const sep = path.includes('?') ? '&' : '?';
-                    const res = await fetchWithTimeout(`${TMDB_BASE}${path}${sep}api_key=${key}`);
-                    if (!res.ok)
-                        return null;
-                    return parseJsonBody(res);
-                }
-                catch {
-                    return null;
-                }
-            },
-        },
-        request: {
-            get: (key) => requestContext.getStore()?.get(key),
-        },
-        cookies: {
-            sanitizeCfClearance,
-            buildCfClearanceCookie,
-        },
-        crypto: {
-            aes256CbcDecryptJson,
-            hmacSha256Base64Url: (key, message) => crypto.createHmac('sha256', key).update(message).digest('base64url'),
-        },
-        scraping: {
-            fetch: async (urlOrOptions, maybeOptions) => {
-                const opts = (typeof urlOrOptions === 'string'
-                    ? { url: urlOrOptions, ...(maybeOptions ?? {}) }
-                    : urlOrOptions);
-                const { url, ...rest } = opts;
-                const res = await gotScraping({
-                    url: url,
-                    method: 'GET',
-                    responseType: 'text',
-                    throwHttpErrors: false,
-                    ...rest,
-                });
-                return { statusCode: res.statusCode, body: String(res.body ?? ''), headers: res.headers };
-            },
-        },
-        curl: {
-            getJson: async (url, headers) => {
-                try {
-                    const r = curlRaw(url, headers);
-                    if (!r || r.status !== 200)
-                        return null;
-                    return JSON.parse(r.body);
-                }
-                catch {
-                    return null;
-                }
-            },
-            getText: async (url, headers) => {
-                const r = curlRaw(url, headers);
-                if (!r || r.status !== 200)
-                    return null;
-                return r.body;
-            },
-        },
-        cheerio: {
-            load: (html) => cheerio.load(html),
-        },
-    };
-}
-function shapeOf(p) {
-    if (!p || typeof p !== 'object')
-        return null;
-    const o = p;
-    return typeof o.name === 'string' ? o : null;
-}
-function hasFns(o, fns) {
-    return fns.every((f) => typeof o[f] === 'function');
-}
-function isValidVideoProvider(p) {
-    const o = shapeOf(p);
-    return !!o && hasFns(o, ['search', 'getEpisodes', 'getStreamUrls']);
-}
-function isValidMangaProvider(p) {
-    const o = shapeOf(p);
-    return !!o && hasFns(o, ['search', 'getDetail', 'getChapters', 'getPages']);
-}
-function isValidTvProvider(p) {
-    const o = shapeOf(p);
-    return !!o && (typeof o.getSources === 'function' || typeof o.getEmbedUrl === 'function');
-}
-function makeEmbedProvider(id, base) {
-    const root = base.trim().replace(/\/+$/, '');
-    return {
-        name: id,
-        getEmbedUrl: async (media) => {
-            const tmdbId = Number(media.tmdbId);
-            if (!tmdbId)
-                return null;
-            if (media.type === 'movie')
-                return `${root}/movie/${tmdbId}`;
-            return `${root}/tv/${tmdbId}/${media.season || 1}/${media.episode || 1}`;
-        },
-    };
-}
-async function loadEntry(item, registryUrl, cacheDir, ctx, allowUnsigned) {
-    if (item.embedBase) {
-        const provider = makeEmbedProvider(item.id, item.embedBase);
-        instanceCache.set(`${item.kind ?? 'anime'}:${item.id}-${item.version}-embed`, { provider });
-        return { provider };
-    }
-    const entryUrl = resolveEntryUrl(item.entry, registryUrl);
-    await fs.promises.mkdir(cacheDir, { recursive: true });
-    const fileName = `${item.id}-${item.version}.mjs`;
-    const filePath = path.join(cacheDir, fileName);
-    const instanceKey = `${item.kind ?? 'anime'}:${item.id}-${item.version}-${(item.sha256 ?? 'nosha').slice(0, 12)}`;
-    let bytes = null;
-    try {
-        const cached = await fs.promises.readFile(filePath);
-        if (item.sha256 && sha256Hex(cached) === item.sha256) {
-            bytes = cached;
-        }
-        else if (!item.sha256 && allowUnsigned) {
-            bytes = cached;
-        }
-    }
-    catch {
-        // ignore
-    }
-    if (bytes) {
-        const hit = instanceCache.get(instanceKey);
-        if (hit) {
-            logger.debug({ provider: item.id, version: item.version }, 'remote provider reused cached instance');
-            return { provider: hit.provider, facets: hit.facets };
-        }
-    }
-    else {
-        try {
-            bytes = await readModuleBytes(entryUrl);
-        }
-        catch (err) {
-            return {
-                provider: null,
-                error: `download failed: ${err.message}`,
-            };
-        }
-        if (item.sha256) {
-            const actual = sha256Hex(bytes);
-            if (actual !== item.sha256) {
-                return { provider: null, error: `sha256 mismatch for ${item.id}` };
-            }
-        }
-        else if (!allowUnsigned) {
-            return { provider: null, error: `missing sha256 for ${item.id}` };
-        }
-        await fs.promises.writeFile(filePath, bytes);
-    }
-    const cacheBuster = `${item.version}-${(item.sha256 ?? 'nosha').slice(0, 12)}`;
-    const result = await instantiateModule(filePath, cacheBuster, ctx, item);
-    if (result.provider && !result.error) {
-        instanceCache.set(instanceKey, { provider: result.provider, facets: result.facets });
-    }
-    return result;
-}
-const instanceCache = new Map();
-function extractBrowseFacets(mod) {
-    let genres;
-    let orders;
-    for (const [key, value] of Object.entries(mod)) {
-        if (key === 'default' || !Array.isArray(value) || value.length === 0)
-            continue;
-        if (!value.every((v) => typeof v === 'string' && v.length > 0))
-            continue;
-        const list = value.filter((v) => v.length <= 64).slice(0, 500);
-        if (list.length === 0)
-            continue;
-        const name = key.toUpperCase();
-        if (name.includes('GENRE') || name.includes('TAG')) {
-            genres ??= list;
-        }
-        else if (name.includes('ORDER') || name.includes('SORT')) {
-            orders ??= list;
-        }
-    }
-    if (!genres && !orders)
-        return undefined;
-    return { ...(genres ? { genres } : {}), ...(orders ? { orders } : {}) };
-}
-async function instantiateModule(filePath, cacheBuster, ctx, item) {
-    const id = item.id;
-    let mod;
-    try {
-        const importUrl = `${pathToFileURL(filePath).href}?v=${encodeURIComponent(cacheBuster)}`;
-        mod = (await withTimeout(import(importUrl), FETCH_TIMEOUT_MS, `import ${id}`));
-    }
-    catch (err) {
-        return {
-            provider: null,
-            error: `import failed: ${err.message}`,
-        };
-    }
-    const factory = mod.default;
-    if (typeof factory !== 'function') {
-        return { provider: null, error: 'missing default factory export' };
-    }
-    let provider;
-    try {
-        provider = await withTimeout(Promise.resolve(factory(ctx)), 10000, `factory ${id}`);
-    }
-    catch (err) {
-        return {
-            provider: null,
-            error: `factory failed: ${err.message}`,
-        };
-    }
-    if (item.kind === 'manga'
-        ? !isValidMangaProvider(provider)
-        : item.kind === 'tv'
-            ? !isValidTvProvider(provider)
-            : !isValidVideoProvider(provider)) {
-        return { provider: null, error: 'invalid provider shape' };
-    }
-    return { provider, facets: extractBrowseFacets(mod) };
-}
-async function loadCachedFallback(item, prev, cacheDir, ctx) {
-    if (!prev.sha256)
-        return { provider: null, error: 'no verified fallback' };
-    const filePath = path.join(cacheDir, `${prev.id}-${prev.version}.mjs`);
-    let bytes;
-    try {
-        bytes = await fs.promises.readFile(filePath);
-    }
-    catch {
-        return { provider: null, error: 'no cached fallback file' };
-    }
-    if (sha256Hex(bytes) !== prev.sha256) {
-        return { provider: null, error: 'cached fallback failed verification' };
-    }
-    const cacheBuster = `${prev.version}-${prev.sha256.slice(0, 12)}`;
-    return instantiateModule(filePath, cacheBuster, ctx, { ...item, version: prev.version });
-}
-export async function loadRemoteProviders(opts) {
-    const registryUrl = (opts.registryUrl ?? CONFIG.PROVIDER_REPO_URL ?? '').trim();
-    const out = {
-        providers: {},
-        mangaProviders: {},
-        tvProviders: {},
-        catalog: [],
-        proxyRules: [],
-    };
-    if (!registryUrl)
-        return out;
-    const cacheDir = path.join(CONFIG.ROOT, 'remote-providers');
-    const cachedManifestPath = path.join(cacheDir, 'registry.json');
-    let prevRegistry = null;
-    try {
-        prevRegistry = parseRegistry(JSON.parse((await fs.promises.readFile(cachedManifestPath)).toString('utf8')));
-    }
-    catch {
-        // ignore
-    }
-    const prevById = new Map((prevRegistry?.providers ?? []).map((p) => [p.id.toLowerCase(), p]));
-    let raw;
-    try {
-        const bytes = await readManifestBytes(registryUrl);
-        raw = JSON.parse(bytes.toString('utf8'));
-        await fs.promises.mkdir(cacheDir, { recursive: true });
-        await fs.promises.writeFile(cachedManifestPath, JSON.stringify(raw));
-    }
-    catch (err) {
-        logger.error({ err, registryUrl }, 'remote providers: manifest fetch failed, trying disk cache');
-        try {
-            raw = JSON.parse((await fs.promises.readFile(cachedManifestPath)).toString('utf8'));
-        }
-        catch {
-            return out;
-        }
-    }
-    let registry;
-    try {
-        registry = parseRegistry(raw);
-    }
-    catch (err) {
-        logger.error({ err, registryUrl }, 'remote providers: manifest parse failed');
-        return out;
-    }
-    const enabledSet = new Set((opts.enabledProviders ?? []).map((s) => s.toLowerCase()));
-    const explicitFilter = enabledSet.size > 0;
-    const ctx = createCtx(opts.cache);
-    const allowUnsigned = opts.allowUnsigned ??
-        (process.env.DANGO_ALLOW_UNSIGNED_PROVIDERS === '1' || looksLikePath(registryUrl));
-    for (const item of registry.providers) {
-        const key = item.id.toLowerCase();
-        if (item.enabledByDefault === false && (!explicitFilter || !enabledSet.has(key))) {
-            out.catalog.push(toCatalog(item, false, 'disabled by default'));
-            continue;
-        }
-        if (explicitFilter && !enabledSet.has(key)) {
-            out.catalog.push(toCatalog(item, false, 'not in enabledProviders'));
-            continue;
-        }
-        const { provider, facets, error } = await loadEntry(item, registryUrl, cacheDir, ctx, allowUnsigned);
-        if (provider && !error) {
-            if (item.kind === 'manga' && isValidMangaProvider(provider)) {
-                out.mangaProviders[key] = provider;
-            }
-            else if (item.kind === 'tv' && isValidTvProvider(provider)) {
-                out.tvProviders[key] = provider;
-            }
-            else if (item.kind !== 'manga' && item.kind !== 'tv' && isValidVideoProvider(provider)) {
-                out.providers[key] = provider;
-            }
-            else {
-                out.catalog.push(toCatalog(item, false, 'invalid provider shape for kind'));
-                logger.warn({ provider: item.id }, 'remote provider kind mismatch, skipped');
-                continue;
-            }
-            out.catalog.push(withFacets(facets, withServers(item, provider, toCatalog(item, true))));
-            if (item.proxy)
-                out.proxyRules.push(item.proxy);
-            logger.info({ provider: item.id, version: item.version }, 'remote provider loaded');
-        }
-        else {
-            const prev = prevById.get(key);
-            if (prev && prev.version !== item.version) {
-                const fb = await loadCachedFallback(item, prev, cacheDir, ctx);
-                if (fb.provider && !fb.error) {
-                    if (item.kind === 'manga' && isValidMangaProvider(fb.provider)) {
-                        out.mangaProviders[key] = fb.provider;
-                    }
-                    else if (item.kind === 'tv' && isValidTvProvider(fb.provider)) {
-                        out.tvProviders[key] = fb.provider;
-                    }
-                    else if (item.kind !== 'manga' &&
-                        item.kind !== 'tv' &&
-                        isValidVideoProvider(fb.provider)) {
-                        out.providers[key] = fb.provider;
-                    }
-                    else {
-                        fb.error = 'cached fallback failed validation';
-                    }
-                }
-                if (fb.provider && !fb.error) {
-                    out.catalog.push({
-                        ...withFacets(fb.facets, withServers(item, fb.provider, toCatalog({ ...item, version: prev.version }, true))),
-                        error: `serving cached ${prev.version}: ${error}`,
-                    });
-                    if (item.proxy)
-                        out.proxyRules.push(item.proxy);
-                    logger.warn({ provider: item.id, error, fallback: prev.version }, 'remote provider update failed, serving cached version');
-                    continue;
-                }
-            }
-            out.catalog.push(toCatalog(item, false, error));
-            logger.warn({ provider: item.id, error }, 'remote provider skipped');
-        }
-    }
-    const failedIds = new Set(out.catalog.filter((c) => !c.loaded).map((c) => `${c.id}-`));
-    try {
-        const expected = new Set(registry.providers.map((item) => `${item.id}-${item.version}.mjs`.toLowerCase()));
-        const expectedInstances = new Set(registry.providers.map((item) => `${item.kind ?? 'anime'}:${item.id}-${item.version}-${(item.sha256 ?? 'nosha').slice(0, 12)}`.toLowerCase()));
-        for (const key of instanceCache.keys()) {
-            if (!expectedInstances.has(key.toLowerCase()))
-                instanceCache.delete(key);
-        }
-        const files = await fs.promises.readdir(cacheDir);
-        for (const file of files) {
-            if (!file.toLowerCase().endsWith('.mjs'))
-                continue;
-            if (!expected.has(file.toLowerCase())) {
-                if ([...failedIds].some((prefix) => file.toLowerCase().startsWith(prefix)))
-                    continue;
-                await fs.promises.unlink(path.join(cacheDir, file));
-                logger.info({ file }, 'remote providers: removed superseded module');
-            }
-        }
-    }
-    catch (err) {
-        logger.warn({ err }, 'remote providers: superseded module cleanup failed');
-    }
-    return out;
-}
-function withServers(item, provider, cat) {
-    if (item.kind !== 'tv' || !provider || typeof provider !== 'object')
-        return cat;
-    const servers = provider.servers;
-    if (Array.isArray(servers) && servers.every((s) => typeof s === 'string') && servers.length > 0) {
-        return { ...cat, servers: [...servers] };
-    }
-    return cat;
-}
-function withFacets(facets, cat) {
-    const genres = (facets?.genres ?? []).length > 0 ? facets?.genres : undefined;
-    const orders = (facets?.orders ?? []).length > 0 ? facets?.orders : undefined;
-    if (!genres && !orders)
-        return cat;
-    return { ...cat, facets: { ...(genres ? { genres } : {}), ...(orders ? { orders } : {}) } };
-}
-function toCatalog(item, loaded, error) {
-    return {
-        id: item.id.toLowerCase(),
-        label: item.label,
-        version: item.version,
-        mature: item.mature,
-        kind: item.kind ?? 'anime',
-        sub: item.sub,
-        tier: item.tier,
-        modes: item.modes ?? (item.kind === 'tv' ? [] : ['sub', 'dub']),
-        browse: item.browse,
-        enabledByDefault: item.enabledByDefault !== false,
-        loaded,
-        remote: true,
-        error,
-    };
-}