@phosart/common 0.4.22

package/src/lib/server/character.ts

@@ -0,0 +1,113 @@
+import { $DATA } from './directories.ts';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import * as yaml from 'yaml';
+import { FullCharacter, RawCharacter } from './models/Character.ts';
+import type { z } from 'zod';
+import { processImageFastcache } from './imageprocess.ts';
+import { cacheVersion, getCache, fullPath, relPath } from './util.ts';
+import { flushFastCache, readFastCache, type FastCache } from './fastcache.ts';
+import { galleries } from './gallery.ts';
+import { normalizeCharacter, type NormalizedCharacter } from '../util/art.ts';
+
+export type CharacterCache = Record<string, z.infer<typeof FullCharacter>>;
+export type RawCharacterCache = Record<string, z.infer<typeof RawCharacter>>;
+
+const CHARACTER_PATH = path.join('characters', 'characters.yaml');
+
+async function doResolveImage(
+	filename: string,
+	character: z.infer<typeof RawCharacter>,
+	fc: FastCache
+): Promise<z.infer<typeof FullCharacter>> {
+	const full = processImageFastcache(
+		fc,
+		fullPath(filename, character.picture.image),
+		relPath(filename, character.picture.image)
+	);
+	const thumb = character.thumbnail
+		? processImageFastcache(
+				fc,
+				fullPath(filename, character.thumbnail.image),
+				relPath(filename, character.thumbnail.image)
+			)
+		: undefined;
+
+	return {
+		...character,
+		picture: { ...(await full), alt: character.picture.alt },
+		thumbnail: thumb
+			? {
+					...(await thumb),
+					alt: character.thumbnail!.alt
+				}
+			: undefined
+	};
+}
+
+export async function rawCharacters(): Promise<RawCharacterCache> {
+	const cached = getCache().rawCharacterCache;
+	const nextVersion = await cacheVersion();
+	if (cached.cache && cached.version === nextVersion) {
+		return cached.cache;
+	}
+	const document = yaml.parse(
+		await fs.readFile(path.join($DATA(), CHARACTER_PATH), { encoding: 'utf-8' })
+	);
+
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	const characters: Record<string, z.infer<typeof RawCharacter>> = (document as any[])
+		.flatMap((obj, idx) => {
+			const ch = RawCharacter.parse({ ...obj, index: idx++ });
+			return ch;
+		})
+		.reduce((acc, cur) => ({ ...acc, [cur.name]: cur }), {});
+
+	getCache().rawCharacterCache.cache = characters;
+	getCache().rawCharacterCache.version = nextVersion;
+	return characters;
+}
+
+export async function characters(): Promise<CharacterCache> {
+	const cached = getCache().characterCache;
+	const nextVersion = await cacheVersion();
+	if (cached.cache && cached.version === nextVersion) {
+		return cached.cache;
+	}
+
+	const characters = await rawCharacters();
+	const fc = await readFastCache();
+
+	const resolved = Object.fromEntries<z.infer<typeof FullCharacter>>(
+		await Promise.all(
+			Object.entries(characters).map<Promise<[string, z.infer<typeof FullCharacter>]>>(
+				async ([k, v]) => {
+					const resolved = await doResolveImage(CHARACTER_PATH, v, fc);
+					return [k, resolved];
+				}
+			)
+		)
+	);
+
+	await flushFastCache(fc);
+	getCache().characterCache.cache = resolved;
+	getCache().characterCache.version = nextVersion;
+	return resolved;
+}
+
+export async function getAllCharacters(): Promise<Array<NormalizedCharacter>> {
+	const all = await characters();
+	const mapped: NormalizedCharacter[] = Object.values(await galleries())
+		.flatMap((g) => g.pieces)
+		.flatMap((p) => p.characters)
+		.flatMap((ch) => normalizeCharacter(ch, all))
+		.concat(Object.values(all).map((ch) => normalizeCharacter(ch.name, all)));
+
+	// Collapse by name, from pair
+	const record: Record<string, NormalizedCharacter> = {};
+	for (const m of mapped) {
+		record[`${m.name}\x00${m.from}`] = m;
+	}
+
+	return Object.values(record);
+}

package/src/lib/server/directories.ts

@@ -0,0 +1,45 @@
+import { join, dirname } from 'node:path';
+import * as fs from 'node:fs';
+
+function tryFindRoot(start: string) {
+	let cur = dirname(start);
+	while (cur.length > 1) {
+		if (cur.includes('node_modules')) {
+			cur = join(cur, '..');
+			continue;
+		}
+
+		try {
+			// If package.json parses correctly...
+			JSON.parse(fs.readFileSync(join(cur, 'package.json'), { encoding: 'utf-8' }));
+			return cur;
+		} catch {
+			cur = join(cur, '..');
+		}
+	}
+	return null;
+}
+
+let cachedRoot: string | null = null;
+
+function findRoot() {
+	if (cachedRoot) {
+		return cachedRoot;
+	}
+	let root: string | null = null;
+	if (process.env.PROJECT_ROOT) {
+		root = tryFindRoot(process.env.PROJECT_ROOT);
+	}
+	root = root ?? tryFindRoot(process.argv[1]) ?? tryFindRoot(process.cwd());
+
+	if (!root) {
+		throw new Error('Failed to find root');
+	}
+
+	cachedRoot = root;
+	return root;
+}
+
+export const $ROOT = () => join(findRoot(), 'src');
+export const $DATA = () => join($ROOT(), 'data');
+export const $PUBLIC = () => join($ROOT(), '..', 'static', '_');

package/src/lib/server/fastcache.ts

@@ -0,0 +1,66 @@
+import { stat } from 'node:fs/promises';
+import { $DATA } from './directories.ts';
+import path from 'node:path';
+import { Logger } from 'tslog';
+import { getLogLevel } from './util.ts';
+import { readPack, writePack } from './pack.ts';
+const FastcacheLogger = new Logger({ minLevel: getLogLevel() });
+
+const $FASTCACHEPATH = () => path.join($DATA(), '.fastcache.pack.gz');
+
+export interface FastCache {
+	[relpath: string]: {
+		mtime: number;
+		hash: string;
+	};
+}
+
+export async function getFastHash(
+	fc: FastCache,
+	fullpath: string,
+	relpath: string
+): Promise<[string | null, number | null]> {
+	const cacheinfo = fc[relpath];
+	if (!cacheinfo) return [null, null];
+
+	const fstat = await stat(fullpath);
+	const mtime = fstat.mtimeMs;
+
+	if (mtime !== cacheinfo.mtime) {
+		return [null, mtime];
+	}
+
+	return [cacheinfo.hash, mtime];
+}
+
+export async function updateFastCache(
+	fc: FastCache,
+	fullpath: string,
+	relpath: string,
+	hash: string,
+	premtime?: number | null
+) {
+	const mtime = premtime ?? (await stat(fullpath)).mtimeMs;
+
+	fc[relpath] = {
+		mtime,
+		hash
+	};
+}
+
+export async function flushFastCache(fc: FastCache) {
+	try {
+		await writePack($FASTCACHEPATH(), fc);
+	} catch (err) {
+		FastcacheLogger.warn('Failed to write fastcache', err);
+	}
+}
+
+export async function readFastCache(): Promise<FastCache> {
+	try {
+		return await readPack($FASTCACHEPATH());
+	} catch (err) {
+		FastcacheLogger.warn('Failed to read fastcache', err);
+		return {};
+	}
+}

package/src/lib/server/filter.ts

@@ -0,0 +1,71 @@
+import {
+	normalizeArtist,
+	normalizeCharacter,
+	type ArtPiece,
+	type ResourceRef
+} from '../util/art.ts';
+
+function filterOne(piece: ArtPiece, by: ResourceRef, negated: boolean): boolean {
+	if (!by.type || !by.resource) {
+		return false;
+	}
+
+	switch (by.type) {
+		case 'artist': {
+			const artists = normalizeArtist(piece.artist);
+			let hasArtist = artists.some(
+				(artist) => artist.name.toLowerCase() === by.resource.name.toLowerCase()
+			);
+			if (negated) {
+				hasArtist = !hasArtist;
+			}
+			return hasArtist;
+		}
+		case 'character': {
+			const characters = normalizeCharacter(piece.characters);
+			let hasCharacter = characters.some(
+				(ch) =>
+					ch.name.toLowerCase() === by.resource.name.toLowerCase() &&
+					(ch.from?.toLowerCase() === by.resource.from?.toLowerCase() ||
+						(!ch.from && !by.resource.from))
+			);
+			if (negated) {
+				hasCharacter = !hasCharacter;
+			}
+			return hasCharacter;
+		}
+		case 'piece': {
+			let isPiece = by.resource.slug === piece.slug;
+			if (negated) {
+				isPiece = !isPiece;
+			}
+			return isPiece;
+		}
+		case 'tag': {
+			let hasTag = piece.tags.map((t) => t.toLowerCase()).includes(by.resource.toLowerCase());
+			if (negated) {
+				hasTag = !hasTag;
+			}
+			return hasTag;
+		}
+	}
+}
+
+export function filter(
+	gallery: ArtPiece[] | Record<string, ArtPiece>,
+	by?: ResourceRef,
+	{ negated, sorted }: { negated?: boolean; sorted?: boolean } = {}
+): ArtPiece[] {
+	if (!Array.isArray(gallery)) gallery = Object.values(gallery);
+
+	const results = by
+		? gallery.filter((piece) => {
+				return filterOne(piece, by, negated ?? false);
+			})
+		: gallery;
+
+	if (!sorted) {
+		return results;
+	}
+	return results.sort((a, b) => b.date.getTime() - a.date.getTime());
+}

package/src/lib/server/gallery.ts

@@ -0,0 +1,259 @@
+import { glob } from 'glob';
+import * as yaml from 'yaml';
+import * as path from 'node:path';
+import * as fs from 'node:fs/promises';
+import { z } from 'zod';
+import { Logger } from 'tslog';
+import {
+	BaseArtPiece,
+	BaseGallery,
+	FullArtPiece,
+	FullGallery,
+	RawGallery
+} from './models/Gallery.ts';
+import { $DATA } from './directories.ts';
+import { processImageFastcache, processVideoFastcache } from './imageprocess.ts';
+import {
+	cacheVersion,
+	clearCache,
+	getCache,
+	hash,
+	fullPath,
+	relPath,
+	getLogLevel
+} from './util.ts';
+import { flushFastCache, readFastCache, type FastCache } from './fastcache.ts';
+import type { ArtPiece } from '../util/art.ts';
+import { asRecord } from '../util/util.ts';
+const GalleryLogger = new Logger({ minLevel: getLogLevel() });
+
+type FileName = string;
+type BaseGalleryT = z.infer<typeof BaseGallery>;
+type RawGalleryT = z.infer<typeof RawGallery>;
+type FullGalleryT = z.infer<typeof FullGallery>;
+type GalleryListing<T> = Record<FileName, T>;
+
+export type GalleryCache = GalleryListing<FullGalleryT>;
+export type RawGalleryCache = GalleryListing<RawGalleryT>;
+
+type BaseArtPieceT = z.infer<typeof BaseArtPiece>;
+type FullArtPieceT = z.infer<typeof FullArtPiece>;
+
+async function resolveExtendsRec(
+	fname: string,
+	gallery: RawGalleryT,
+	rec: Record<FileName, RawGalleryT>,
+	fc: FastCache
+): Promise<FullGalleryT> {
+	if (!('$extends' in gallery)) {
+		return await resolveReferences(gallery, fname, fc);
+	}
+	const full: FullGalleryT = { pieces: [] };
+
+	for (const relative of gallery.$extends) {
+		const base = path.dirname(fname);
+		const target = path.join(base, relative);
+		if (rec[target]) {
+			const result = await resolveExtendsRec(target, rec[target], rec, fc);
+			if ('$extends' in result) {
+				throw new Error('Impossible');
+			}
+			full.pieces.push(...result.pieces);
+		} else {
+			GalleryLogger.warn('[GALLERY] Could not find target for', target, 'when parsing', fname);
+		}
+	}
+
+	return full;
+}
+async function resolveExtends(
+	rec: GalleryListing<RawGalleryT>,
+	fc: FastCache
+): Promise<GalleryListing<FullGalleryT>> {
+	const listing: GalleryListing<FullGalleryT> = {};
+	for (const [fname, gallery] of Object.entries(rec)) {
+		if ('$extends' in gallery) {
+			listing[fname] = await resolveExtendsRec(fname, gallery, rec, fc);
+		} else {
+			listing[fname] = await resolveReferences(gallery, fname, fc);
+		}
+	}
+	return listing;
+}
+
+async function resolvePieceReferences(
+	piece: BaseArtPieceT,
+	filename: FileName,
+	fc: FastCache
+): Promise<FullArtPieceT> {
+	const relpath = relPath(filename, piece.image);
+	const fullpath = fullPath(filename, piece.image);
+	const image = await processImageFastcache(fc, fullpath, relpath, { position: piece.position });
+
+	const alts = await Promise.all(
+		piece.alts?.map(async (alt) => ({
+			...alt,
+			image: await processImageFastcache(
+				fc,
+				fullPath(filename, alt.image),
+				relPath(filename, alt.image),
+				{ position: piece.position }
+			),
+			video: alt.video
+				? {
+						full: await processVideoFastcache(
+							fc,
+							fullPath(filename, alt.video.full),
+							relPath(filename, alt.video.full),
+							'full'
+						),
+						thumb: await processVideoFastcache(
+							fc,
+							fullPath(filename, alt.video.thumb),
+							relPath(filename, alt.video.thumb),
+							'thumb'
+						)
+					}
+				: undefined
+		})) ?? []
+	);
+	const video = piece.video
+		? {
+				full: await processVideoFastcache(
+					fc,
+					fullPath(filename, piece.video.full),
+					relPath(filename, piece.video.full),
+					'full'
+				),
+				thumb: await processVideoFastcache(
+					fc,
+					fullPath(filename, piece.video.thumb),
+					relPath(filename, piece.video.thumb),
+					'thumb'
+				)
+			}
+		: undefined;
+
+	return { ...piece, image, alts, video };
+}
+
+async function resolveReferences(
+	gallery: BaseGalleryT,
+	fname: FileName,
+	fc: FastCache
+): Promise<FullGalleryT> {
+	return {
+		pieces: await Promise.all(
+			gallery.pieces.map((piece) => resolvePieceReferences(piece, fname, fc))
+		)
+	};
+}
+
+export async function rawGalleries(): Promise<RawGalleryCache> {
+	const cached = getCache().rawGalleryCache;
+	const nextVersion = await cacheVersion();
+	if (cached.cache && cached.version === nextVersion) {
+		return cached.cache;
+	}
+	const allGalleries = await glob('./**/*.gallery', { cwd: $DATA() });
+	const documents = (
+		await Promise.all(
+			allGalleries.map((fn) =>
+				fs
+					.readFile(path.join($DATA(), fn), { encoding: 'utf-8' })
+					.then((text) => ({ filename: fn, text }))
+			)
+		)
+	)
+		.map(({ filename, text }) => ({ filename, obj: yaml.parse(text) }))
+		.map(({ filename, obj }) => {
+			GalleryLogger.silly('[GALLERY] Parsing', filename);
+			obj.pieces = obj.pieces
+				? obj.pieces.map((p: BaseArtPieceT) => ({ ...p, slug: p.id ?? hash(p) }))
+				: undefined;
+			return { filename, obj: RawGallery.parse(obj) };
+		})
+		.reduce<Record<FileName, RawGalleryT>>(
+			(rec, { filename, obj }) => ({ ...rec, [filename.replaceAll(path.sep, '/')]: obj }),
+			{}
+		);
+
+	getCache().rawGalleryCache.cache = documents;
+	getCache().rawGalleryCache.version = nextVersion;
+	return documents;
+}
+
+export async function galleries(doRetry: boolean = true) {
+	const cached = getCache().galleryCache;
+	const nextVersion = await cacheVersion();
+	if (cached.cache && cached.version === nextVersion) {
+		return cached.cache;
+	}
+
+	const documents = Object.fromEntries(
+		Object.entries(await rawGalleries())
+			.map(([path, g]) => {
+				if ('pieces' in g) {
+					g = {
+						pieces: g.pieces
+							.map((p) => {
+								if (p.deindexed) {
+									return null;
+								}
+								if (p.alts) {
+									p = { ...p, alts: p.alts.filter((a) => !a.deindexed) };
+								}
+								return p;
+							})
+							.filter(truthy)
+					};
+					if (g.pieces.length === 0) {
+						return null;
+					}
+					return [path, g];
+				} else {
+					if (g.$extends.length === 0) {
+						return null;
+					}
+					return [path, g];
+				}
+			})
+			.filter(truthy)
+	);
+
+	try {
+		const fc = await readFastCache();
+		const out = await resolveExtends(documents, fc);
+		await flushFastCache(fc);
+		getCache().galleryCache.cache = out;
+		getCache().galleryCache.version = nextVersion;
+		return out;
+	} catch (e) {
+		if (doRetry) {
+			GalleryLogger.warn('Encountered error, clearing cache and trying again', e);
+			clearCache();
+			return await galleries(/* doRetry = */ false);
+		} else {
+			throw e;
+		}
+	}
+}
+
+function truthy<T>(t: T | null | undefined): t is T {
+	return !!t;
+}
+
+export async function allPieces(): Promise<Record<string, ArtPiece>> {
+	return asRecord(
+		Object.values(await galleries()).flatMap((p) => p.pieces),
+		(p) => p.slug
+	);
+}
+
+export async function getPieceBySlug(slug: string): Promise<ArtPiece | null> {
+	return (await allPieces())[slug] ?? null;
+}
+
+export function hasNsfw(gallery: ArtPiece[]): boolean {
+	return gallery.some((p) => p.nsfw);
+}