cross-seed 7.0.0-21 → 7.0.0-23

package/dist/libraryInventory.js

@@ -0,0 +1,701 @@
+import { db } from "./db.js";
+import { inBatches } from "./utils.js";
+export var LibraryItemSource;
+(function (LibraryItemSource) {
+    LibraryItemSource["CLIENT"] = "torrentClient";
+    LibraryItemSource["TORRENT"] = "torrentFile";
+    LibraryItemSource["DATA"] = "dataDir";
+})(LibraryItemSource || (LibraryItemSource = {}));
+export var LibraryScanStatus;
+(function (LibraryScanStatus) {
+    LibraryScanStatus["RUNNING"] = "running";
+    LibraryScanStatus["SUCCEEDED"] = "succeeded";
+    LibraryScanStatus["FAILED"] = "failed";
+})(LibraryScanStatus || (LibraryScanStatus = {}));
+const libraryItemSortColumns = {
+    lastSeenAt: "last_seen_at",
+    title: "title",
+    length: "length",
+    source: "source",
+};
+function deserializeScan(row) {
+    return {
+        id: row.id,
+        source: row.source,
+        sourceInstance: row.source_instance,
+        status: row.status,
+        startedAt: row.started_at,
+        finishedAt: row.finished_at,
+        error: row.error,
+        itemsSeen: row.items_seen,
+        itemsUpserted: row.items_upserted,
+        itemsRemoved: row.items_removed,
+    };
+}
+function timestampToISOString(timestamp) {
+    return new Date(timestamp).toISOString();
+}
+function parseJsonArray(value) {
+    if (!value)
+        return null;
+    return JSON.parse(value);
+}
+function deserializeInventoryItem(row, files) {
+    return {
+        id: row.id,
+        source: row.source,
+        sourceInstance: row.source_instance,
+        sourceKey: row.source_key,
+        clientHost: row.client_host,
+        infoHash: row.info_hash,
+        path: row.path,
+        name: row.name,
+        title: row.title,
+        length: row.length,
+        savePath: row.save_path,
+        category: row.category,
+        tags: parseJsonArray(row.tags),
+        trackers: parseJsonArray(row.trackers),
+        contentFingerprint: row.content_fingerprint,
+        firstSeenAt: timestampToISOString(row.first_seen_at),
+        lastSeenAt: timestampToISOString(row.last_seen_at),
+        lastIndexedAt: timestampToISOString(row.last_indexed_at),
+        updatedAt: timestampToISOString(row.updated_at),
+        fileCount: Number(row.file_count ?? 0),
+        ...(files ? { files } : {}),
+    };
+}
+function encodeLibraryItemsCursor(row, sortBy, sortDirection) {
+    const cursor = {
+        sortBy,
+        sortDirection,
+        value: row[libraryItemSortColumns[sortBy]],
+        sourceKey: row.source_key,
+    };
+    return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
+}
+function decodeLibraryItemsCursor(cursor) {
+    if (!cursor)
+        return null;
+    const parsed = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
+    if ((parsed.sortBy !== "lastSeenAt" &&
+        parsed.sortBy !== "title" &&
+        parsed.sortBy !== "length" &&
+        parsed.sortBy !== "source") ||
+        (parsed.sortDirection !== "asc" && parsed.sortDirection !== "desc") ||
+        (typeof parsed.value !== "string" &&
+            typeof parsed.value !== "number") ||
+        typeof parsed.sourceKey !== "string") {
+        throw new Error("Invalid library items cursor");
+    }
+    return parsed;
+}
+export function createClientLibrarySourceKey(input) {
+    return `${LibraryItemSource.CLIENT}:${input.clientHost}:${input.infoHash}`;
+}
+export function createTorrentLibrarySourceKey(filePath) {
+    return `${LibraryItemSource.TORRENT}:${filePath}`;
+}
+export function createDataLibrarySourceKey(path) {
+    return `${LibraryItemSource.DATA}:${path}`;
+}
+function createContentFingerprint(input) {
+    return JSON.stringify({
+        infoHash: input.infoHash ?? null,
+        path: input.path ?? null,
+        name: input.name,
+        title: input.title,
+        files: input.files,
+        length: input.length,
+        savePath: input.savePath ?? null,
+        category: input.category ?? null,
+        tags: input.tags ?? null,
+        trackers: input.trackers ?? null,
+    });
+}
+export function createClientLibraryItemInput(searchee) {
+    return {
+        source: LibraryItemSource.CLIENT,
+        sourceInstance: searchee.clientHost,
+        sourceKey: createClientLibrarySourceKey({
+            clientHost: searchee.clientHost,
+            infoHash: searchee.infoHash,
+        }),
+        clientHost: searchee.clientHost,
+        infoHash: searchee.infoHash,
+        name: searchee.name,
+        title: searchee.title,
+        files: searchee.files,
+        length: searchee.length,
+        savePath: searchee.savePath,
+        category: searchee.category ?? null,
+        tags: searchee.tags ?? null,
+        trackers: searchee.trackers,
+        contentFingerprint: createContentFingerprint({
+            infoHash: searchee.infoHash,
+            name: searchee.name,
+            title: searchee.title,
+            files: searchee.files,
+            length: searchee.length,
+            savePath: searchee.savePath,
+            category: searchee.category ?? null,
+            tags: searchee.tags ?? null,
+            trackers: searchee.trackers,
+        }),
+    };
+}
+export function createTorrentLibraryItemInput(searchee, filePath, sourceInstance, options = {}) {
+    return {
+        source: LibraryItemSource.TORRENT,
+        sourceInstance,
+        sourceKey: createTorrentLibrarySourceKey(filePath),
+        infoHash: searchee.infoHash,
+        path: filePath,
+        name: searchee.name,
+        title: searchee.title,
+        files: searchee.files,
+        length: searchee.length,
+        category: searchee.category ?? null,
+        tags: searchee.tags ?? null,
+        trackers: searchee.trackers ?? null,
+        contentFingerprint: options.contentFingerprint ??
+            createContentFingerprint({
+                infoHash: searchee.infoHash,
+                path: filePath,
+                name: searchee.name,
+                title: searchee.title,
+                files: searchee.files,
+                length: searchee.length,
+                category: searchee.category ?? null,
+                tags: searchee.tags ?? null,
+                trackers: searchee.trackers ?? null,
+            }),
+    };
+}
+export function createDataLibraryItemInput(searchee, sourceInstance) {
+    return {
+        source: LibraryItemSource.DATA,
+        sourceInstance,
+        sourceKey: createDataLibrarySourceKey(searchee.path),
+        path: searchee.path,
+        name: searchee.name,
+        title: searchee.title,
+        files: searchee.files,
+        length: searchee.length,
+        contentFingerprint: createContentFingerprint({
+            path: searchee.path,
+            name: searchee.name,
+            title: searchee.title,
+            files: searchee.files,
+            length: searchee.length,
+        }),
+    };
+}
+export function createLibraryFileInputs(item) {
+    return item.files.map((file) => ({
+        source: item.source,
+        sourceInstance: item.sourceInstance ?? null,
+        itemSourceKey: item.sourceKey,
+        path: file.path,
+        name: file.name,
+        length: file.length,
+    }));
+}
+export async function beginLibraryScan(input) {
+    const now = input.now ?? Date.now();
+    const [scan] = await db("library_scan")
+        .insert({
+        source: input.source,
+        source_instance: input.sourceInstance ?? null,
+        status: LibraryScanStatus.RUNNING,
+        started_at: now,
+    })
+        .returning("*");
+    return deserializeScan(scan);
+}
+export async function getLibraryScan(id) {
+    const scan = await db("library_scan").where({ id }).first();
+    return scan ? deserializeScan(scan) : null;
+}
+export async function listLibraryItems(options) {
+    const limit = options.limit ?? 50;
+    const offset = options.offset ?? 0;
+    const sortBy = options.sortBy ?? "lastSeenAt";
+    const sortDirection = options.sortDirection ?? "desc";
+    const cursor = decodeLibraryItemsCursor(options.cursor);
+    const createQuery = (filterOptions) => db("library_item")
+        .whereNull("missing_since")
+        .modify((queryBuilder) => {
+        if (filterOptions.includeSource && options.source) {
+            queryBuilder.where("source", options.source);
+        }
+        if (options.sourceInstance !== undefined) {
+            if (options.sourceInstance === null) {
+                queryBuilder.whereNull("source_instance");
+            }
+            else {
+                queryBuilder.where("source_instance", options.sourceInstance);
+            }
+        }
+        if (options.search) {
+            queryBuilder.where((searchBuilder) => {
+                searchBuilder
+                    .where("title", "like", `%${options.search}%`)
+                    .orWhere("name", "like", `%${options.search}%`)
+                    .orWhere("path", "like", `%${options.search}%`)
+                    .orWhere("info_hash", "like", `%${options.search}%`);
+            });
+        }
+    });
+    const query = createQuery({ includeSource: true });
+    const sourceCountsQuery = createQuery({
+        includeSource: false,
+    });
+    const sortColumn = libraryItemSortColumns[sortBy];
+    const countRows = (await query
+        .clone()
+        .count({ count: "*" }));
+    const sourceCountRows = (await sourceCountsQuery
+        .clone()
+        .select("source")
+        .count({ count: "*" })
+        .groupBy("source")
+        .orderBy("source", "asc"));
+    const rows = (await query
+        .clone()
+        .select("library_item.*")
+        .select(db("library_file")
+        .count("*")
+        .whereRaw("library_file.item_source_key = library_item.source_key")
+        .whereNull("library_file.missing_since")
+        .as("file_count"))
+        .modify((queryBuilder) => {
+        if (!cursor)
+            return;
+        if (cursor.sortBy !== sortBy ||
+            cursor.sortDirection !== sortDirection) {
+            throw new Error("Library items cursor sort does not match query");
+        }
+        queryBuilder.where((cursorBuilder) => {
+            cursorBuilder
+                .where(sortColumn, sortDirection === "asc" ? ">" : "<", cursor.value)
+                .orWhere((tieBuilder) => {
+                tieBuilder
+                    .where(sortColumn, cursor.value)
+                    .where("source_key", ">", cursor.sourceKey);
+            });
+        });
+    })
+        .orderBy(sortColumn, sortDirection)
+        .orderBy("source_key", "asc")
+        .limit(limit + 1)
+        .offset(cursor ? 0 : offset));
+    const nextRow = rows.length > limit ? rows[limit] : null;
+    const pageRows = rows.slice(0, limit);
+    const filesBySourceKey = new Map();
+    if (options.includeFiles && pageRows.length) {
+        const fileRows = (await db("library_file")
+            .whereIn("item_source_key", pageRows.map((row) => row.source_key))
+            .whereNull("missing_since")
+            .select("item_source_key", "path", "name", "length")
+            .orderBy("path", "asc"));
+        for (const file of fileRows) {
+            const files = filesBySourceKey.get(file.item_source_key) ?? [];
+            files.push({
+                path: file.path,
+                name: file.name,
+                length: file.length,
+            });
+            filesBySourceKey.set(file.item_source_key, files);
+        }
+    }
+    return {
+        items: pageRows.map((row) => deserializeInventoryItem(row, options.includeFiles
+            ? (filesBySourceKey.get(row.source_key) ?? [])
+            : undefined)),
+        total: Number(countRows[0]?.count ?? 0),
+        limit,
+        offset,
+        nextCursor: nextRow
+            ? encodeLibraryItemsCursor(pageRows[pageRows.length - 1], sortBy, sortDirection)
+            : null,
+        sourceCounts: sourceCountRows.map((row) => ({
+            source: row.source,
+            count: Number(row.count),
+        })),
+    };
+}
+export async function getLibraryScanStatuses(options = {}) {
+    const scans = (await db("library_scan")
+        .modify((builder) => {
+        if (options.source)
+            builder.where("source", options.source);
+        if (options.sourceInstance !== undefined) {
+            if (options.sourceInstance === null) {
+                builder.whereNull("source_instance");
+            }
+            else {
+                builder.where("source_instance", options.sourceInstance);
+            }
+        }
+    })
+        .orderBy("id", "desc"));
+    const latestScans = new Map();
+    for (const scan of scans) {
+        const key = `${scan.source}\0${scan.source_instance ?? ""}`;
+        if (!latestScans.has(key))
+            latestScans.set(key, scan);
+    }
+    return Promise.all(Array.from(latestScans.values()).map(async (scan) => {
+        const scope = { source: scan.source };
+        const itemCountQuery = db("library_item")
+            .where(scope)
+            .whereNull("missing_since")
+            .modify((builder) => {
+            if (scan.source_instance === null) {
+                builder.whereNull("source_instance");
+            }
+            else {
+                builder.where("source_instance", scan.source_instance);
+            }
+        });
+        const fileCountQuery = db("library_file")
+            .where(scope)
+            .whereNull("missing_since")
+            .modify((builder) => {
+            if (scan.source_instance === null) {
+                builder.whereNull("source_instance");
+            }
+            else {
+                builder.where("source_instance", scan.source_instance);
+            }
+        });
+        const [itemCount, fileCount] = (await Promise.all([
+            itemCountQuery.count({
+                count: "*",
+            }),
+            fileCountQuery.count({
+                count: "*",
+            }),
+        ]));
+        return {
+            source: scan.source,
+            sourceInstance: scan.source_instance,
+            status: scan.status,
+            startedAt: timestampToISOString(scan.started_at),
+            finishedAt: scan.finished_at !== null
+                ? timestampToISOString(scan.finished_at)
+                : null,
+            error: scan.error,
+            itemsSeen: scan.items_seen,
+            itemsUpserted: scan.items_upserted,
+            itemsRemoved: scan.items_removed,
+            activeItems: Number(itemCount[0]?.count ?? 0),
+            activeFiles: Number(fileCount[0]?.count ?? 0),
+        };
+    }));
+}
+async function incrementScanCounts(scanId, counts) {
+    await db("library_scan")
+        .where({ id: scanId })
+        .increment("items_seen", counts.itemsSeen ?? 0)
+        .increment("items_upserted", counts.itemsUpserted ?? 0)
+        .increment("items_removed", counts.itemsRemoved ?? 0);
+}
+export async function upsertLibraryItems(scanId, items, options = {}) {
+    if (!items.length)
+        return;
+    const now = options.now ?? Date.now();
+    const existingRows = await db("library_item")
+        .whereIn("source_key", items.map((item) => item.sourceKey))
+        .select("source_key", "content_fingerprint", "missing_since");
+    const existingBySourceKey = new Map(existingRows.map((row) => [row.source_key, row]));
+    const unchangedSourceKeys = [];
+    const changedItems = items.filter((item) => {
+        const existing = existingBySourceKey.get(item.sourceKey);
+        if (existing &&
+            existing.missing_since === null &&
+            existing.content_fingerprint === (item.contentFingerprint ?? null)) {
+            unchangedSourceKeys.push(item.sourceKey);
+            return false;
+        }
+        return true;
+    });
+    await markLibraryItemsSeen(scanId, unchangedSourceKeys, { now });
+    if (!changedItems.length)
+        return;
+    const rows = changedItems.map((item) => ({
+        source: item.source,
+        source_instance: item.sourceInstance ?? null,
+        source_key: item.sourceKey,
+        client_host: item.clientHost ?? null,
+        info_hash: item.infoHash ?? null,
+        path: item.path ?? null,
+        name: item.name,
+        title: item.title,
+        files: JSON.stringify(item.files),
+        length: item.length,
+        save_path: item.savePath ?? null,
+        category: item.category ?? null,
+        tags: item.tags ? JSON.stringify(item.tags) : null,
+        trackers: item.trackers ? JSON.stringify(item.trackers) : null,
+        content_fingerprint: item.contentFingerprint ?? null,
+        first_seen_at: now,
+        last_seen_at: now,
+        last_indexed_at: now,
+        missing_since: null,
+        last_seen_scan_id: scanId,
+        updated_at: now,
+    }));
+    await inBatches(rows, async (batch) => {
+        await db("library_item")
+            .insert(batch)
+            .onConflict("source_key")
+            .merge([
+            "source",
+            "source_instance",
+            "client_host",
+            "info_hash",
+            "path",
+            "name",
+            "title",
+            "files",
+            "length",
+            "save_path",
+            "category",
+            "tags",
+            "trackers",
+            "content_fingerprint",
+            "last_seen_at",
+            "last_indexed_at",
+            "missing_since",
+            "last_seen_scan_id",
+            "updated_at",
+        ]);
+    });
+    await incrementScanCounts(scanId, {
+        itemsSeen: changedItems.length,
+        itemsUpserted: changedItems.length,
+    });
+}
+export async function markLibraryItemsSeen(scanId, sourceKeys, options = {}) {
+    if (!sourceKeys.length)
+        return;
+    const now = options.now ?? Date.now();
+    await inBatches(sourceKeys, async (batch) => {
+        await db("library_item").whereIn("source_key", batch).update({
+            last_seen_at: now,
+            last_seen_scan_id: scanId,
+            missing_since: null,
+        });
+    });
+    await incrementScanCounts(scanId, { itemsSeen: sourceKeys.length });
+}
+export async function upsertLibraryFiles(scanId, files, options = {}) {
+    if (!files.length)
+        return;
+    const now = options.now ?? Date.now();
+    const rows = files.map((file) => ({
+        source: file.source,
+        source_instance: file.sourceInstance ?? null,
+        item_source_key: file.itemSourceKey,
+        path: file.path,
+        name: file.name,
+        length: file.length,
+        first_seen_at: now,
+        last_seen_at: now,
+        missing_since: null,
+        last_seen_scan_id: scanId,
+        updated_at: now,
+    }));
+    await inBatches(rows, async (batch) => {
+        await db("library_file")
+            .insert(batch)
+            .onConflict(["item_source_key", "path"])
+            .merge([
+            "source",
+            "source_instance",
+            "name",
+            "length",
+            "last_seen_at",
+            "missing_since",
+            "last_seen_scan_id",
+            "updated_at",
+        ]);
+    });
+}
+export async function pruneMissingLibraryFiles(scanId, options = {}) {
+    const scan = await getLibraryScan(scanId);
+    if (!scan)
+        throw new Error(`Library scan ${scanId} not found`);
+    if (scan.status !== LibraryScanStatus.SUCCEEDED) {
+        throw new Error(`Cannot prune library files for ${scan.status} scan ${scanId}`);
+    }
+    const now = options.now ?? Date.now();
+    const query = db("library_file")
+        .where({
+        source: scan.source,
+        missing_since: null,
+    })
+        .where((builder) => {
+        if (scan.sourceInstance === null) {
+            builder.whereNull("source_instance");
+        }
+        else {
+            builder.where("source_instance", scan.sourceInstance);
+        }
+    })
+        .where((builder) => {
+        builder
+            .whereNull("last_seen_scan_id")
+            .orWhere("last_seen_scan_id", "!=", scanId);
+    });
+    return query.update({
+        missing_since: now,
+        updated_at: now,
+    });
+}
+export async function pruneMissingLibraryFilesForItems(scanId, itemSourceKeys, options = {}) {
+    if (!itemSourceKeys.length)
+        return 0;
+    const now = options.now ?? Date.now();
+    let filesRemoved = 0;
+    await inBatches(itemSourceKeys, async (batch) => {
+        filesRemoved += await db("library_file")
+            .whereIn("item_source_key", batch)
+            .whereNull("missing_since")
+            .where((builder) => {
+            builder
+                .whereNull("last_seen_scan_id")
+                .orWhere("last_seen_scan_id", "!=", scanId);
+        })
+            .update({
+            missing_since: now,
+            updated_at: now,
+        });
+    });
+    return filesRemoved;
+}
+export async function completeLibraryScan(scanId, options = {}) {
+    await db("library_scan")
+        .where({ id: scanId })
+        .update({
+        status: LibraryScanStatus.SUCCEEDED,
+        finished_at: options.now ?? Date.now(),
+        error: null,
+    });
+    const scan = await getLibraryScan(scanId);
+    if (!scan)
+        throw new Error(`Library scan ${scanId} not found`);
+    return scan;
+}
+export async function failLibraryScan(scanId, error, options = {}) {
+    await db("library_scan")
+        .where({ id: scanId })
+        .update({
+        status: LibraryScanStatus.FAILED,
+        finished_at: options.now ?? Date.now(),
+        error: error instanceof Error ? error.message : String(error),
+    });
+    const scan = await getLibraryScan(scanId);
+    if (!scan)
+        throw new Error(`Library scan ${scanId} not found`);
+    return scan;
+}
+export async function pruneMissingLibraryItems(scanId, options = {}) {
+    const scan = await getLibraryScan(scanId);
+    if (!scan)
+        throw new Error(`Library scan ${scanId} not found`);
+    if (scan.status !== LibraryScanStatus.SUCCEEDED) {
+        throw new Error(`Cannot prune library items for ${scan.status} scan ${scanId}`);
+    }
+    const now = options.now ?? Date.now();
+    const query = db("library_item")
+        .where({
+        source: scan.source,
+        missing_since: null,
+    })
+        .where((builder) => {
+        if (scan.sourceInstance === null) {
+            builder.whereNull("source_instance");
+        }
+        else {
+            builder.where("source_instance", scan.sourceInstance);
+        }
+    })
+        .where((builder) => {
+        builder
+            .whereNull("last_seen_scan_id")
+            .orWhere("last_seen_scan_id", "!=", scanId);
+    });
+    const itemsRemoved = await query.update({
+        missing_since: now,
+        updated_at: now,
+    });
+    await incrementScanCounts(scanId, { itemsRemoved });
+    return itemsRemoved;
+}
+export async function markLibraryItemsMissingBySourceKeys(sourceKeys, options = {}) {
+    if (!sourceKeys.length)
+        return 0;
+    let itemsRemoved = 0;
+    const now = options.now ?? Date.now();
+    await inBatches(sourceKeys, async (batch) => {
+        itemsRemoved += await db("library_item")
+            .whereIn("source_key", batch)
+            .whereNull("missing_since")
+            .update({
+            missing_since: now,
+            updated_at: now,
+        });
+        await db("library_file")
+            .whereIn("item_source_key", batch)
+            .whereNull("missing_since")
+            .update({
+            missing_since: now,
+            updated_at: now,
+        });
+    });
+    return itemsRemoved;
+}
+export async function syncLibraryItems(options) {
+    const scan = options.scanId === undefined
+        ? await beginLibraryScan({
+            source: options.source,
+            sourceInstance: options.sourceInstance ?? null,
+            now: options.now,
+        })
+        : await getLibraryScan(options.scanId);
+    if (!scan ||
+        scan.status !== LibraryScanStatus.RUNNING ||
+        scan.source !== options.source ||
+        scan.sourceInstance !== (options.sourceInstance ?? null)) {
+        throw new Error("Invalid library scan for source sync");
+    }
+    try {
+        await upsertLibraryItems(scan.id, options.itemsToUpsert ?? [], {
+            now: options.now,
+        });
+        await markLibraryItemsSeen(scan.id, options.sourceKeysSeen ?? [], {
+            now: options.now,
+        });
+        await upsertLibraryFiles(scan.id, options.filesToUpsert ?? [], {
+            now: options.now,
+        });
+        const completedScan = await completeLibraryScan(scan.id, {
+            now: options.now,
+        });
+        await pruneMissingLibraryFilesForItems(scan.id, Array.from(new Set((options.filesToUpsert ?? []).map((file) => file.itemSourceKey))), { now: options.now });
+        if (options.pruneMissing) {
+            await pruneMissingLibraryItems(scan.id, { now: options.now });
+            await pruneMissingLibraryFiles(scan.id, { now: options.now });
+            return (await getLibraryScan(scan.id)) ?? completedScan;
+        }
+        return completedScan;
+    }
+    catch (error) {
+        await failLibraryScan(scan.id, error, { now: options.now });
+        throw error;
+    }
+}
+//# sourceMappingURL=libraryInventory.js.map