@serifpersia/dango 3.3.4

package/server/dist/lib/offline-db.js

@@ -0,0 +1,401 @@
+import crypto from 'crypto';
+import logger from '../logger.js';
+import { SettingsRepository } from '../repositories/settings.repository.js';
+import { decodeMaybeGzip } from '../utils/http.utils.js';
+const log = logger.child({ module: 'OfflineDb' });
+const OFFLINE_DB_URLS = [
+    'https://github.com/cedya77/anime-offline-database/releases/latest/download/anime-offline-database-minified.json',
+    'https://raw.githubusercontent.com/cedya77/anime-offline-database/master/anime-offline-database-minified.json',
+    'https://github.com/manami-project/anime-offline-database/releases/latest/download/anime-offline-database-minified.json',
+    'https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database-minified.json',
+];
+const FETCH_IDLE_TIMEOUT_MS = 60000;
+const FETCH_OVERALL_TIMEOUT_MS = 30 * 60 * 1000;
+const MAX_BODY_BYTES = 100 * 1024 * 1024;
+const CHUNK_SIZE = 1000;
+const INTERRUPTED_RETRY_COOLDOWN_MS = 60 * 60 * 1000;
+const CANONICAL_GENRES = [
+    'Action',
+    'Adventure',
+    'Comedy',
+    'Drama',
+    'Fantasy',
+    'Horror',
+    'Mahou Shoujo',
+    'Mecha',
+    'Music',
+    'Mystery',
+    'Psychological',
+    'Romance',
+    'Sci-Fi',
+    'Slice of Life',
+    'Sports',
+    'Supernatural',
+    'Thriller',
+];
+const canonicalGenreMap = new Map(CANONICAL_GENRES.map((g) => [g.toLowerCase(), g]));
+const SCHEMA_VERSION = 1;
+const TABLE = 'anime_id_map';
+const STAGING_TABLE = 'anime_id_map_new';
+function ensureTable(db, table = TABLE) {
+    const row = db.get("SELECT name FROM sqlite_master WHERE type='table' AND name=?", [table]);
+    if (!row) {
+        db.run(`CREATE TABLE ${table} (id INTEGER PRIMARY KEY AUTOINCREMENT, anilist_id INTEGER, mal_id INTEGER, title TEXT, thumbnail TEXT, type TEXT, genres TEXT)`);
+    }
+    else {
+        const columns = db.all(`PRAGMA table_info(${table})`);
+        const names = new Set(columns.map((c) => c.name));
+        if (!names.has('mal_id'))
+            db.run(`ALTER TABLE ${table} ADD COLUMN mal_id INTEGER`);
+        if (!names.has('type'))
+            db.run(`ALTER TABLE ${table} ADD COLUMN type TEXT`);
+        if (!names.has('genres'))
+            db.run(`ALTER TABLE ${table} ADD COLUMN genres TEXT`);
+    }
+    db.run(`CREATE INDEX IF NOT EXISTS idx_anime_id_map_anilist ON ${table}(anilist_id)`);
+    db.run(`CREATE INDEX IF NOT EXISTS idx_anime_id_map_mal ON ${table}(mal_id)`);
+}
+function latestSaturdayUtc() {
+    const now = new Date();
+    const daysSinceSaturday = (now.getUTCDay() + 1) % 7;
+    return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() - daysSinceSaturday, 0, 0, 0, 0));
+}
+async function fetchText(url) {
+    const controller = new AbortController();
+    const overall = setTimeout(() => controller.abort(), FETCH_OVERALL_TIMEOUT_MS);
+    let idle;
+    const touchIdle = () => {
+        if (idle)
+            clearTimeout(idle);
+        idle = setTimeout(() => controller.abort(), FETCH_IDLE_TIMEOUT_MS);
+    };
+    try {
+        const res = await fetch(url, {
+            signal: controller.signal,
+            headers: { 'User-Agent': 'Dango-Media-Client/1.0', Accept: 'application/json' },
+        });
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status}`);
+        const length = res.headers.get('content-length');
+        if (length && Number(length) > MAX_BODY_BYTES)
+            throw new Error('Response too large');
+        if (!res.body)
+            throw new Error('Empty response body');
+        touchIdle();
+        const chunks = [];
+        let total = 0;
+        const reader = res.body.getReader();
+        try {
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                touchIdle();
+                total += value.byteLength;
+                if (total > MAX_BODY_BYTES)
+                    throw new Error('Response too large');
+                chunks.push(Buffer.from(value));
+            }
+        }
+        finally {
+            reader.releaseLock();
+        }
+        const text = decodeMaybeGzip(Buffer.concat(chunks)).toString('utf8');
+        if (text.length > MAX_BODY_BYTES)
+            throw new Error('Response too large');
+        return text;
+    }
+    finally {
+        if (idle)
+            clearTimeout(idle);
+        clearTimeout(overall);
+    }
+}
+class OfflineDb {
+    malToEntry = new Map();
+    anilistToMal = new Map();
+    anilistToEntry = new Map();
+    isInitialized = false;
+    isRefreshing = false;
+    async init(db) {
+        try {
+            ensureTable(db);
+            this.loadCache(db);
+            log.info(`OfflineDb loaded ${this.malToEntry.size} MAL mappings from SQLite`);
+            const wasInterrupted = SettingsRepository.getByKey(db, 'offlineDbLastStatus')?.value === 'updating';
+            const autoUpdateSetting = SettingsRepository.getByKey(db, 'offlineDbAutoUpdateEnabled');
+            const autoUpdateEnabled = !autoUpdateSetting || autoUpdateSetting.value !== 'false';
+            this.backfillShowsMetaGenres(db);
+            if (this.malToEntry.size === 0) {
+                log.info('OfflineDb cache is empty. Starting background download...');
+                this.refreshDatabase(db).catch((err) => {
+                    log.warn({ err: err?.message }, 'Background offline database download failed');
+                });
+            }
+            else if (wasInterrupted && autoUpdateEnabled && this.interruptedRetryDue(db)) {
+                log.info('Previous offline database refresh was interrupted. Retrying in background...');
+                SettingsRepository.upsert(db, 'offlineDbLastStatus', 'idle');
+                SettingsRepository.upsert(db, 'offlineDbLastMessage', 'Previous refresh was interrupted by a restart. Retrying automatically...');
+                this.refreshDatabase(db).catch((err) => {
+                    log.warn({ err: err?.message }, 'Interrupted offline database retry failed');
+                });
+            }
+            else if (wasInterrupted) {
+                SettingsRepository.upsert(db, 'offlineDbLastStatus', 'idle');
+                SettingsRepository.upsert(db, 'offlineDbLastMessage', 'Previous refresh was interrupted by a restart. Trigger an update to retry.');
+            }
+            this.isInitialized = true;
+        }
+        catch (e) {
+            log.error({ err: e }, 'Failed to initialize OfflineDb');
+        }
+    }
+    backfillShowsMetaGenres(db) {
+        try {
+            ensureTable(db);
+            db.run(`
+        UPDATE shows_meta
+        SET genres = (
+          SELECT genres FROM ${TABLE}
+          WHERE anilist_id = CAST(shows_meta.id AS INTEGER)
+            AND genres IS NOT NULL AND genres != '[]'
+          LIMIT 1
+        )
+        WHERE (genres IS NULL OR genres = '' OR genres = '[]')
+          AND EXISTS (
+            SELECT 1 FROM ${TABLE}
+            WHERE anilist_id = CAST(shows_meta.id AS INTEGER)
+              AND genres IS NOT NULL AND genres != '[]'
+          )
+      `);
+            return 1;
+        }
+        catch (err) {
+            log.warn({ err }, 'Failed to backfill shows_meta genres from offline db');
+            return 0;
+        }
+    }
+    loadCache(db) {
+        try {
+            ensureTable(db);
+            const rows = db.all(`SELECT anilist_id, mal_id, title, thumbnail, type, genres FROM ${TABLE} WHERE anilist_id IS NOT NULL AND mal_id IS NOT NULL`);
+            this.malToEntry.clear();
+            this.anilistToMal.clear();
+            this.anilistToEntry.clear();
+            for (const row of rows) {
+                if (!row.anilist_id || !row.mal_id)
+                    continue;
+                const entry = {
+                    anilistId: row.anilist_id,
+                    malId: row.mal_id,
+                    title: row.title || '',
+                    thumbnail: row.thumbnail || undefined,
+                    type: row.type || undefined,
+                    genres: row.genres || undefined,
+                };
+                this.malToEntry.set(row.mal_id, entry);
+                this.anilistToMal.set(row.anilist_id, row.mal_id);
+                this.anilistToEntry.set(row.anilist_id, entry);
+            }
+        }
+        catch (err) {
+            log.error({ err }, 'Failed to load offline mappings into memory');
+        }
+    }
+    getByMalId(malId) {
+        return this.malToEntry.get(malId) ?? null;
+    }
+    getByAnilistId(anilistId) {
+        return this.anilistToEntry.get(anilistId) ?? null;
+    }
+    getAnilistIdByMal(malId) {
+        return this.malToEntry.get(malId)?.anilistId ?? null;
+    }
+    getMalIdByAnilist(anilistId) {
+        return this.anilistToMal.get(anilistId) ?? null;
+    }
+    getOfflineDbInfo(db) {
+        const get = (key) => SettingsRepository.getByKey(db, key)?.value ?? null;
+        const autoUpdate = get('offlineDbAutoUpdateEnabled');
+        const lastStatus = get('offlineDbLastStatus');
+        const entryCount = Number(get('offlineDbEntryCount') ?? this.malToEntry.size) || 0;
+        return {
+            totalMapped: this.malToEntry.size,
+            totalMalMapped: this.malToEntry.size,
+            isInitialized: this.isInitialized,
+            isRefreshing: this.isRefreshing,
+            autoUpdateEnabled: autoUpdate ? autoUpdate !== 'false' : true,
+            lastCheckedAt: get('offlineDbLastCheckedAt'),
+            lastUpdatedAt: get('offlineDbLastUpdatedAt'),
+            lastStatus: lastStatus || (this.isRefreshing ? 'updating' : 'idle'),
+            lastMessage: get('offlineDbLastMessage'),
+            sourceUrl: get('offlineDbSourceUrl'),
+            releaseTag: get('offlineDbReleaseTag'),
+            contentSha256: get('offlineDbContentSha256'),
+            entryCount,
+            schemaVersion: SCHEMA_VERSION,
+            nextDueAt: latestSaturdayUtc().toISOString(),
+        };
+    }
+    checkWeeklyUpdateDue(db) {
+        const autoUpdate = SettingsRepository.getByKey(db, 'offlineDbAutoUpdateEnabled');
+        if (autoUpdate && autoUpdate.value === 'false')
+            return false;
+        const lastChecked = SettingsRepository.getByKey(db, 'offlineDbLastCheckedAt');
+        if (!lastChecked?.value)
+            return true;
+        const lastCheckedTime = new Date(lastChecked.value).getTime();
+        if (Number.isNaN(lastCheckedTime))
+            return true;
+        return lastCheckedTime < latestSaturdayUtc().getTime();
+    }
+    interruptedRetryDue(db) {
+        const lastChecked = SettingsRepository.getByKey(db, 'offlineDbLastCheckedAt');
+        if (!lastChecked?.value)
+            return true;
+        const lastCheckedTime = new Date(lastChecked.value).getTime();
+        if (Number.isNaN(lastCheckedTime))
+            return true;
+        return Date.now() - lastCheckedTime > INTERRUPTED_RETRY_COOLDOWN_MS;
+    }
+    async executeScheduledUpdate(db) {
+        if (this.isRefreshing) {
+            log.info('Scheduled offline database update skipped: refresh already in progress');
+            return;
+        }
+        log.info('Starting scheduled weekly offline database update...');
+        const result = await this.refreshDatabase(db);
+        if (result.success)
+            log.info(`Scheduled offline database update succeeded: ${result.count} entries`);
+        else
+            log.warn(`Scheduled offline database update failed: ${result.error}`);
+    }
+    async refreshDatabase(db) {
+        if (this.isRefreshing) {
+            return { success: false, count: this.malToEntry.size, error: 'Refresh already in progress' };
+        }
+        this.isRefreshing = true;
+        const startedAt = Date.now();
+        SettingsRepository.upsert(db, 'offlineDbLastStatus', 'updating');
+        SettingsRepository.upsert(db, 'offlineDbLastCheckedAt', new Date().toISOString());
+        SettingsRepository.upsert(db, 'offlineDbLastMessage', 'Downloading and parsing database...');
+        try {
+            let text = null;
+            let sourceUrl = null;
+            let lastErr = null;
+            for (const url of OFFLINE_DB_URLS) {
+                try {
+                    log.info(`Fetching anime-offline-database from ${url}...`);
+                    text = await fetchText(url);
+                    const parsed = JSON.parse(text);
+                    if (parsed && Array.isArray(parsed.data)) {
+                        sourceUrl = url;
+                        break;
+                    }
+                    text = null;
+                }
+                catch (err) {
+                    lastErr = err;
+                    log.warn(`Failed downloading from ${url}: ${err.message}`);
+                }
+            }
+            if (!text)
+                throw lastErr || new Error('Could not download anime-offline-database from any source');
+            const sha256 = crypto.createHash('sha256').update(text).digest('hex');
+            const storedSha = SettingsRepository.getByKey(db, 'offlineDbContentSha256')?.value;
+            const data = JSON.parse(text).data;
+            log.info(`Parsing ${data.length} entries from anime-offline-database...`);
+            const anilistRegex = /anilist\.co\/anime\/(\d+)/i;
+            const malRegex = /myanimelist\.net\/anime\/(\d+)/i;
+            const parsedEntries = [];
+            for (const item of data) {
+                const rec = item;
+                if (!Array.isArray(rec.sources))
+                    continue;
+                let anilistId = null;
+                let malId = null;
+                for (const src of rec.sources) {
+                    if (typeof src !== 'string')
+                        continue;
+                    const anilistMatch = src.match(anilistRegex);
+                    if (anilistMatch)
+                        anilistId = parseInt(anilistMatch[1], 10);
+                    const malMatch = src.match(malRegex);
+                    if (malMatch)
+                        malId = parseInt(malMatch[1], 10);
+                }
+                if (!anilistId || !malId)
+                    continue;
+                let thumbnail = typeof rec.thumbnail === 'string'
+                    ? rec.thumbnail
+                    : typeof rec.picture === 'string'
+                        ? rec.picture
+                        : '';
+                if (thumbnail.includes('cdn.myanimelist.net/images/anime/')) {
+                    thumbnail = thumbnail.replace(/t\.(jpe?g|png|webp)$/i, 'l.$1');
+                }
+                let genres;
+                if (Array.isArray(rec.tags)) {
+                    const matched = new Set();
+                    for (const tag of rec.tags) {
+                        if (typeof tag !== 'string')
+                            continue;
+                        const canonical = canonicalGenreMap.get(tag.trim().toLowerCase());
+                        if (canonical)
+                            matched.add(canonical);
+                    }
+                    if (matched.size > 0)
+                        genres = JSON.stringify([...matched]);
+                }
+                parsedEntries.push({
+                    anilistId,
+                    malId,
+                    title: typeof rec.title === 'string' ? rec.title : '',
+                    thumbnail: thumbnail || undefined,
+                    type: typeof rec.type === 'string' ? rec.type : undefined,
+                    genres,
+                });
+            }
+            log.info(`Saving ${parsedEntries.length} mapped entries to SQLite...`);
+            db.run(`DROP TABLE IF EXISTS ${STAGING_TABLE}`);
+            ensureTable(db, STAGING_TABLE);
+            const stmt = db.prepare(`INSERT INTO ${STAGING_TABLE} (anilist_id, mal_id, title, thumbnail, type, genres) VALUES (?, ?, ?, ?, ?, ?)`);
+            for (let i = 0; i < parsedEntries.length; i += CHUNK_SIZE) {
+                const chunk = parsedEntries.slice(i, i + CHUNK_SIZE);
+                db.serialize(() => {
+                    for (const entry of chunk) {
+                        stmt.run(entry.anilistId, entry.malId, entry.title, entry.thumbnail ?? null, entry.type ?? null, entry.genres ?? null);
+                    }
+                });
+                await new Promise((resolve) => setImmediate(resolve));
+            }
+            db.run(`DROP TABLE IF EXISTS ${TABLE}`);
+            db.run(`ALTER TABLE ${STAGING_TABLE} RENAME TO ${TABLE}`);
+            ensureTable(db);
+            this.loadCache(db);
+            this.backfillShowsMetaGenres(db);
+            const releaseTag = sourceUrl?.includes('/releases/') ? 'latest' : 'raw';
+            SettingsRepository.upsert(db, 'offlineDbLastStatus', 'success');
+            SettingsRepository.upsert(db, 'offlineDbLastUpdatedAt', new Date().toISOString());
+            SettingsRepository.upsert(db, 'offlineDbSourceUrl', sourceUrl ?? '');
+            SettingsRepository.upsert(db, 'offlineDbReleaseTag', releaseTag);
+            SettingsRepository.upsert(db, 'offlineDbContentSha256', sha256);
+            SettingsRepository.upsert(db, 'offlineDbEntryCount', String(parsedEntries.length));
+            const skipped = storedSha === sha256;
+            SettingsRepository.upsert(db, 'offlineDbLastMessage', `Updated ${parsedEntries.length} entries in ${Math.round((Date.now() - startedAt) / 1000)}s${skipped ? ' (unchanged upstream hash)' : ''}.`);
+            log.info(`OfflineDb refresh completed: ${parsedEntries.length} entries from ${sourceUrl}`);
+            return { success: true, count: parsedEntries.length };
+        }
+        catch (err) {
+            const errMsg = err.message || 'Failed to refresh offline database';
+            log.error({ err }, 'Failed to refresh anime-offline-database');
+            SettingsRepository.upsert(db, 'offlineDbLastStatus', 'failed');
+            SettingsRepository.upsert(db, 'offlineDbLastMessage', errMsg);
+            return { success: false, count: this.malToEntry.size, error: errMsg };
+        }
+        finally {
+            this.isRefreshing = false;
+        }
+    }
+}
+export const offlineDb = new OfflineDb();

package/server/dist/lib/temp-ids.js

@@ -0,0 +1,3 @@
+export const TEMP_SHOW_ID_PREFIX = 'dango-mt-';
+export const isTempShowId = (id) => !!id && /^dango-mt-\d+$/.test(id);
+export const isTempSyncRow = (row) => Object.values(row).some((v) => typeof v === 'string' && v.startsWith(TEMP_SHOW_ID_PREFIX));

package/server/dist/lib/tmdb.js

@@ -0,0 +1,72 @@
+import { parseJsonBody } from '../utils/http.utils.js';
+const TMDB_BASE = 'https://api.themoviedb.org/3';
+const TMDB_IMAGE = 'https://image.tmdb.org/t/p';
+const FALLBACK_TMDB_KEY = '9e7096a7575623aa30c66e9cc987e411';
+const TMDB_KEYS = [
+    'fb7bb23f03b6994dafc674c074d01761',
+    'e55425032d3d0f371fc776f302e7c09b',
+    '8301a21598f8b45668d5711a814f01f6',
+    '8cf43ad9c085135b9479ad5cf6bbcbda',
+    'da63548086e399ffc910fbc08526df05',
+    '13e53ff644a8bd4ba37b3e1044ad24f3',
+    '269890f657dddf4635473cf4cf456576',
+    'a2f888b27315e62e471b2d587048f32e',
+    '8476a7ab80ad76f0936744df0430e67c',
+    '5622cafbfe8f8cfe358a29c53e19bba0',
+    'ae4bd1b6fce2a5648671bfc171d15ba4',
+    '257654f35e3dff105574f97fb4b97035',
+    '2f4038e83265214a0dcd6ec2eb3276f5',
+    '9e43f45f94705cc8e1d5a0400d19a7b7',
+    'af6887753365e14160254ac7f4345dd2',
+    '06f10fc8741a672af455421c239a1ffc',
+    '09ad8ace66eec34302943272db0e8d2c',
+];
+let cachedKey = null;
+export { TMDB_BASE, TMDB_IMAGE };
+export async function getTmdbKey() {
+    if (!cachedKey) {
+        cachedKey =
+            process.env.TMDB_API_KEY ||
+                TMDB_KEYS[Math.floor(Math.random() * TMDB_KEYS.length)] ||
+                FALLBACK_TMDB_KEY;
+    }
+    return cachedKey;
+}
+export async function tmdbSearch(query) {
+    const key = await getTmdbKey();
+    const url = `${TMDB_BASE}/search/multi?api_key=${key}&query=${encodeURIComponent(query)}&include_adult=false`;
+    const res = await fetch(url);
+    if (!res.ok)
+        return null;
+    const json = await parseJsonBody(res);
+    return json.results ?? null;
+}
+export async function tmdbTvDetails(tmdbId) {
+    const key = await getTmdbKey();
+    const url = `${TMDB_BASE}/tv/${tmdbId}?api_key=${key}`;
+    const res = await fetch(url);
+    if (!res.ok)
+        return null;
+    const json = await parseJsonBody(res);
+    return json;
+}
+export async function findTmdbDefaultBackdrop(titleParts) {
+    const searchNames = [titleParts.english, titleParts.romaji, titleParts.native].filter(Boolean);
+    for (const name of searchNames) {
+        const results = await tmdbSearch(name);
+        if (!results)
+            continue;
+        const tvResults = results.filter((r) => r.media_type === 'tv');
+        if (tvResults.length === 0)
+            continue;
+        const bestMatch = tvResults.sort((a, b) => (b.vote_count || 0) - (a.vote_count || 0))[0];
+        const details = await tmdbTvDetails(bestMatch.id);
+        if (!details?.backdrop_path)
+            continue;
+        return {
+            backdrop: `${TMDB_IMAGE}/original${details.backdrop_path}`,
+            overview: details.overview || bestMatch.overview || undefined,
+        };
+    }
+    return null;
+}