@phosart/common 0.4.22

package/dist/server/imageprocess.js

@@ -0,0 +1,243 @@
+import sharp, {} from 'sharp';
+import { Logger } from 'tslog';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { getLogLevel, hashUrl } from "./util.js";
+import { $PUBLIC } from "./directories.js";
+import { getFastHash, updateFastCache } from "./fastcache.js";
+import sharpPhash from 'sharp-phash';
+import { Sema } from 'async-sema';
+import os from 'node:os';
+const ImageProcessLog = new Logger({ minLevel: getLogLevel() });
+const sema = new Sema(os.cpus().length, { capacity: 1000 });
+const phash = sharpPhash;
+let processedHashes = new Set();
+const LQIP_WIDTH = 64;
+const FORMATS = ['avif', 'webp'];
+const WIDTHS = [640, 960, 1280, 1920, 3840];
+const THUMBS = [160, 320];
+export function getProcessedHashes() {
+    return processedHashes;
+}
+export function clearProcessedHashes() {
+    processedHashes = new Set();
+}
+export async function getUnusedHashes() {
+    const eligible = new Set();
+    const pubdir = await fs.readdir($PUBLIC(), { withFileTypes: true, recursive: false });
+    await Promise.all(pubdir.map(async (dirent) => {
+        if (!dirent.isDirectory()) {
+            return;
+        }
+        const ents = await fs.readdir(path.resolve(path.join($PUBLIC(), dirent.name)));
+        if (!ents.includes('.keep.mark')) {
+            eligible.add(dirent.name);
+        }
+    }));
+    return eligible.difference(processedHashes);
+}
+export async function deleteHashes(unused) {
+    if (unused.size === 0) {
+        return;
+    }
+    ImageProcessLog.warn('Cleaning up generated directory:', unused.size, 'unused folders found...');
+    const respub = path.resolve($PUBLIC());
+    for (const hash of unused) {
+        const resolved = path.resolve(path.join($PUBLIC(), hash));
+        if (!resolved.startsWith(respub)) {
+            ImageProcessLog.warn('Tried to clean hash at path', resolved, 'which is not a subdirectory of public', respub, '??');
+        }
+        else {
+            ImageProcessLog.debug('Deleting', resolved, '...');
+            await fs.rm(resolved, { recursive: true, force: true });
+            ImageProcessLog.info('Deleted unused generated directory', resolved, '...');
+        }
+    }
+}
+export async function cleanUnusedHashes() {
+    await deleteHashes(await getUnusedHashes());
+}
+async function getPictureDetails(h) {
+    try {
+        return JSON.parse(await fs.readFile(path.join($PUBLIC(), h, 'details.json'), { encoding: 'utf-8' }));
+    }
+    catch {
+        return null;
+    }
+}
+export async function processVideoFastcache(fc, fullpath, relpath, name, processOptions = {}) {
+    const [prehash, mtime] = await getFastHash(fc, fullpath, relpath);
+    const [path, hash] = await doProcessVideo(fullpath, name, prehash, processOptions);
+    if (!prehash) {
+        await updateFastCache(fc, fullpath, relpath, hash, mtime);
+    }
+    return path;
+}
+export async function processVideo(url, name) {
+    return (await doProcessVideo(url, name, null))[0];
+}
+async function doProcessVideo(url, name, prehash, processOptions = {}) {
+    const h = prehash ?? (await hashUrl(url));
+    processedHashes.add(h);
+    name = name + path.extname(url);
+    const outputDir = path.join($PUBLIC(), h);
+    await fs.mkdir(outputDir, { recursive: true });
+    await fs.copyFile(url, path.join(outputDir, name));
+    if (processOptions?.forceKeep) {
+        await fs.writeFile(path.join(outputDir, '.keep.mark'), '');
+    }
+    return [`/_/${h}/${name}`, h];
+}
+export async function processImageFastcache(fc, fullpath, relpath, processOptions = {}) {
+    const [prehash, mtime] = await getFastHash(fc, fullpath, relpath);
+    const [image, hash] = await doProcessImage(fullpath, prehash, processOptions);
+    if (!prehash) {
+        await updateFastCache(fc, fullpath, relpath, hash, mtime);
+    }
+    return image;
+}
+export async function processImage(url, processOptions = {}) {
+    return (await doProcessImage(url, null, processOptions))[0];
+}
+async function doProcessImage(url, prehash, processOptions = {}) {
+    const h = prehash ?? (await hashUrl(url));
+    processedHashes.add(h);
+    ImageProcessLog.silly('[IMAGE] Got image with hash', h);
+    const cached = await getPictureDetails(h);
+    if (cached) {
+        ImageProcessLog.debug('[IMAGE] Found cached details for', url, h);
+        return [cached, h];
+    }
+    const tok = await sema.acquire();
+    try {
+        ImageProcessLog.debug('[IMAGE] Starting to process', url);
+        const image = sharp(url, { animated: true });
+        const details = await _doProcessImage(url, image, h, processOptions);
+        ImageProcessLog.info('[IMAGE] Finished processing', url, details);
+        return [details, h];
+    }
+    finally {
+        sema.release(tok);
+    }
+}
+function removeDuplicates(images) {
+    for (let i = images.length - 1; i >= 0; i--) {
+        const me = images[i];
+        if (images.findIndex((v) => v.format === me.format && v.h === me.h && v.w === me.w && v.src === me.src) !== i) {
+            images.splice(i, 1);
+        }
+    }
+    return images;
+}
+async function _doProcessImage(url, image, hash, processOptions = {}) {
+    const meta = await image.metadata();
+    const fullTransformations = meta.format === 'gif'
+        ? [
+            { format: 'gif', width: meta.width },
+            { format: 'webp', width: meta.width }
+        ]
+        : FORMATS.flatMap((format) => WIDTHS.flatMap((width) => ({ format, width, position: processOptions?.position })));
+    const thumbnailTransformations = (meta.format === 'gif' ? ['gif', 'webp'] : FORMATS).flatMap((format) => THUMBS.flatMap((width) => ({
+        format,
+        width,
+        height: width,
+        position: processOptions?.position
+    })));
+    const fulls = Promise.all(fullTransformations.map((tf) => doSaveImage(url, hash, doTransformImage(image, tf), tf, meta.pages ?? 1))).then(removeDuplicates);
+    const thumbs = Promise.all(thumbnailTransformations.map((tf) => doSaveImage(url, hash, doTransformImage(image, tf), tf, meta.pages ?? 1))).then(removeDuplicates);
+    const phPromise = phash(url);
+    const fullLqip = doLQIP(image, {
+        format: 'webp',
+        width: LQIP_WIDTH,
+        position: processOptions?.position
+    });
+    const thumbLqip = doLQIP(image, {
+        format: 'webp',
+        width: LQIP_WIDTH,
+        height: LQIP_WIDTH,
+        position: processOptions?.position
+    });
+    const data = {
+        phash: await phPromise,
+        full: {
+            sha256: hash,
+            lqip: await fullLqip,
+            sources: (await fulls).reduce((all, cur) => ({
+                ...all,
+                [cur.format]: [...(all[cur.format] ?? []), cur]
+            }), {}),
+            fallback: (await fulls).reduce((best, cur) => (cur.format === 'webp' && cur.w > best.w ? cur : best), (await fulls)[0])
+        },
+        thumbnail: {
+            sha256: hash,
+            lqip: await thumbLqip,
+            sources: (await thumbs).reduce((all, cur) => ({
+                ...all,
+                [cur.format]: [...(all[cur.format] ?? []), cur]
+            }), {}),
+            fallback: (await thumbs).reduce((best, cur) => (cur.format === 'webp' && cur.w > best.w ? cur : best), (await thumbs)[0])
+        }
+    };
+    await fs.writeFile(path.join($PUBLIC(), hash, 'details.json'), JSON.stringify(data, null, 4), {
+        encoding: 'utf-8'
+    });
+    if (processOptions?.forceKeep) {
+        await fs.writeFile(path.join($PUBLIC(), hash, '.keep.mark'), '');
+    }
+    ImageProcessLog.info('[IMAGE] Processed image ', hash);
+    return data;
+}
+async function doSaveImage(url, hash, image, tf, pages) {
+    const base = path.join($PUBLIC(), hash);
+    await fs.mkdir(base, { recursive: true });
+    ImageProcessLog.silly('[IMAGE] Starting transformation of', url, 'to', tf);
+    return await new Promise((resolve, reject) => image.toBuffer(async (err, buf, info) => {
+        if (err)
+            return void reject(err);
+        if (info.format === 'heif')
+            info.format = 'avif';
+        const name = `${info.width}x${info.height / pages}.${info.format}`;
+        const output = path.join(base, name);
+        await fs.writeFile(output, buf);
+        ImageProcessLog.debug('[IMAGE] Saved new image', output, 'for transformation of', url, tf);
+        resolve({
+            src: `/_/${hash}/${name}`,
+            w: info.width,
+            h: info.height / pages,
+            format: info.format
+        });
+    }));
+}
+function doTransformImage(image, { format, width, height = -1, position = 'north' }) {
+    return image
+        .clone()
+        .resize({
+        width: width,
+        height: height === -1 ? undefined : height,
+        position,
+        withoutEnlargement: true
+    })
+        .toFormat(format);
+}
+async function doLQIP(image, { width, height = -1, position = 'north' }) {
+    return await new Promise((resolve, reject) => image
+        .clone()
+        .resize({
+        width: width,
+        height: height === -1 ? undefined : height,
+        position,
+        withoutEnlargement: true
+    })
+        .toFormat('webp', { quality: 20 })
+        .toBuffer((err, buffer, info) => {
+        if (err)
+            return void reject(err);
+        const dataUrl = `data:image/webp;base64,${buffer.toString('base64')}`;
+        resolve({
+            format: 'webp',
+            w: info.width,
+            h: info.height,
+            src: dataUrl
+        });
+    }));
+}

package/dist/server/index.d.ts

@@ -0,0 +1,15 @@
+export { type ArtistCache, artists, getAllArtists } from './artist.ts';
+export { type CharacterCache, characters, type RawCharacterCache, rawCharacters, getAllCharacters } from './character.ts';
+export { $DATA as $ART, $PUBLIC, $ROOT } from './directories.ts';
+export { filter } from './filter.ts';
+export { type GalleryCache, type RawGalleryCache, galleries, rawGalleries, allPieces, getPieceBySlug } from './gallery.ts';
+export { processImage, processVideo, processImageFastcache, processVideoFastcache, clearProcessedHashes, cleanUnusedHashes } from './imageprocess.ts';
+export { Artist } from './models/Artist.ts';
+export { FullCharacter, RawCharacter } from './models/Character.ts';
+export { BaseArtPiece, BaseGallery, FullArtPiece, FullGallery, RawGallery, Video, BaseArtist, ExtendedGallery } from './models/Gallery.ts';
+export { Image, Picture, Source } from './models/image.ts';
+export { clearCache } from './util.ts';
+export { getFastHash, readFastCache, flushFastCache, updateFastCache } from './fastcache.ts';
+export { readThemeConfig, readThemeSchema, type BuiltinSettings, type SettingsFor, type ThemeSettingsSchema, ZThemeSettingsSchema, builtinSettings, validateSchema, writeThemeConfig } from './theme/schema.ts';
+export { readPack, writePack } from './pack.ts';
+//# sourceMappingURL=index.d.ts.map

package/dist/server/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lib/server/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EACN,KAAK,cAAc,EACnB,UAAU,EACV,KAAK,iBAAiB,EACtB,aAAa,EACb,gBAAgB,EAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EACN,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,SAAS,EACT,YAAY,EACZ,SAAS,EACT,cAAc,EACd,MAAM,cAAc,CAAC;AACtB,OAAO,EACN,YAAY,EACZ,YAAY,EACZ,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EACN,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,EACV,eAAe,EACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAE7F,OAAO,EACN,eAAe,EACf,eAAe,EACf,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC"}

package/dist/server/index.js

@@ -0,0 +1,15 @@
+// Reexport server-only modules here
+export { artists, getAllArtists } from "./artist.js";
+export { characters, rawCharacters, getAllCharacters } from "./character.js";
+export { $DATA as $ART, $PUBLIC, $ROOT } from "./directories.js";
+export { filter } from "./filter.js";
+export { galleries, rawGalleries, allPieces, getPieceBySlug } from "./gallery.js";
+export { processImage, processVideo, processImageFastcache, processVideoFastcache, clearProcessedHashes, cleanUnusedHashes } from "./imageprocess.js";
+export { Artist } from "./models/Artist.js";
+export { FullCharacter, RawCharacter } from "./models/Character.js";
+export { BaseArtPiece, BaseGallery, FullArtPiece, FullGallery, RawGallery, Video, BaseArtist, ExtendedGallery } from "./models/Gallery.js";
+export { Image, Picture, Source } from "./models/image.js";
+export { clearCache } from "./util.js";
+export { getFastHash, readFastCache, flushFastCache, updateFastCache } from "./fastcache.js";
+export { readThemeConfig, readThemeSchema, ZThemeSettingsSchema, builtinSettings, validateSchema, writeThemeConfig } from "./theme/schema.js";
+export { readPack, writePack } from "./pack.js";

package/dist/server/models/Artist.d.ts

@@ -0,0 +1,7 @@
+import { z } from 'zod';
+export declare const Artist: z.ZodObject<{
+    name: z.ZodString;
+    links: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodLiteral<"twitter">, z.ZodLiteral<"facebook">, z.ZodLiteral<"instagram">, z.ZodLiteral<"tumblr">, z.ZodLiteral<"toyhouse">, z.ZodLiteral<"website">, z.ZodLiteral<"linktree">, z.ZodString]>>;
+    type: z.ZodDefault<z.ZodLiteral<"Artist">>;
+}, z.core.$strip>;
+//# sourceMappingURL=Artist.d.ts.map

package/dist/server/models/Artist.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Artist.d.ts","sourceRoot":"","sources":["../../../src/lib/server/models/Artist.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,MAAM;;;;iBAgBjB,CAAC"}

package/dist/server/models/Artist.js

@@ -0,0 +1,15 @@
+import { z } from 'zod';
+export const Artist = z.object({
+    name: z.string(),
+    links: z.record(z.string(), z.union([
+        z.literal('twitter'),
+        z.literal('facebook'),
+        z.literal('instagram'),
+        z.literal('tumblr'),
+        z.literal('toyhouse'),
+        z.literal('website'),
+        z.literal('linktree'),
+        z.string()
+    ])),
+    type: z.literal('Artist').default('Artist')
+});

package/dist/server/models/Character.d.ts

@@ -0,0 +1,108 @@
+import { z } from 'zod';
+export declare const RawCharacter: z.ZodObject<{
+    name: z.ZodString;
+    pronouns: z.ZodString;
+    thumbnail: z.ZodOptional<z.ZodObject<{
+        image: z.ZodString;
+        alt: z.ZodString;
+    }, z.core.$strip>>;
+    picture: z.ZodObject<{
+        image: z.ZodString;
+        alt: z.ZodString;
+    }, z.core.$strip>;
+    description: z.ZodString;
+    short_description: z.ZodOptional<z.ZodString>;
+    index: z.ZodNumber;
+    type: z.ZodDefault<z.ZodLiteral<"Character">>;
+}, z.core.$strip>;
+export declare const FullCharacter: z.ZodObject<{
+    name: z.ZodString;
+    pronouns: z.ZodString;
+    thumbnail: z.ZodOptional<z.ZodIntersection<z.ZodObject<{
+        full: z.ZodObject<{
+            sources: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>>>;
+            fallback: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            lqip: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            sha256: z.ZodString;
+        }, z.core.$strip>;
+        thumbnail: z.ZodObject<{
+            sources: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>>>;
+            fallback: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            lqip: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            sha256: z.ZodString;
+        }, z.core.$strip>;
+        phash: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        alt: z.ZodString;
+    }, z.core.$strip>>>;
+    picture: z.ZodIntersection<z.ZodObject<{
+        full: z.ZodObject<{
+            sources: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>>>;
+            fallback: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            lqip: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            sha256: z.ZodString;
+        }, z.core.$strip>;
+        thumbnail: z.ZodObject<{
+            sources: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>>>;
+            fallback: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            lqip: z.ZodObject<{
+                src: z.ZodString;
+                w: z.ZodNumber;
+                h: z.ZodNumber;
+            }, z.core.$strip>;
+            sha256: z.ZodString;
+        }, z.core.$strip>;
+        phash: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        alt: z.ZodString;
+    }, z.core.$strip>>;
+    description: z.ZodString;
+    short_description: z.ZodOptional<z.ZodString>;
+    index: z.ZodNumber;
+    type: z.ZodDefault<z.ZodLiteral<"Character">>;
+}, z.core.$strip>;
+//# sourceMappingURL=Character.d.ts.map

package/dist/server/models/Character.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Character.d.ts","sourceRoot":"","sources":["../../../src/lib/server/models/Character.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAsBxB,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;iBAA4B,CAAC;AACtD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAwB,CAAC"}

package/dist/server/models/Character.js

@@ -0,0 +1,21 @@
+import { z } from 'zod';
+import { Picture } from "./image.js";
+const AltPicture = Picture.and(z.object({ alt: z.string() }));
+const BaseAltPicture = z.object({
+    image: z.string(),
+    alt: z.string()
+});
+function Character(imageType) {
+    return z.object({
+        name: z.string(),
+        pronouns: z.string(),
+        thumbnail: imageType.optional(),
+        picture: imageType,
+        description: z.string(),
+        short_description: z.string().optional(),
+        index: z.number(),
+        type: z.literal('Character').default('Character')
+    });
+}
+export const RawCharacter = Character(BaseAltPicture);
+export const FullCharacter = Character(AltPicture);