@coywolf/emdash 0.0.0-stage → 0.35.13

package/src/files/site.ts

@@ -0,0 +1,137 @@
+/**
+ * Site-side reads and writes for File Downloads, outside the plugin context
+ * (the download middleware and the Astro download card). They go straight to
+ * D1: plugin settings live in `options` (read with the feature switches),
+ * large uploads and download counts in `_plugin_storage`, media library files
+ * in `media`. Reads are cached per isolate; download counts are batched and written after the response.
+ */
+import { PLUGIN_ID, invalidateFeatures, readSiteSetting, registerSiteSetting } from "../core/features.js";
+import { createDownloadCounter } from "./counts.js";
+import { isUploadId, normalizeBase, safeColor } from "./format.js";
+
+export const COLLECTIONS = {
+	uploads: "files_uploads",
+	usage: "files_usage",
+	counts: "files_counts",
+} as const;
+
+/** Site settings the card and downloads need (never the S3 credentials). */
+export interface SiteSettings {
+	base: string;
+	publicBaseUrl: string;
+	scheme: "auto" | "light" | "dark";
+	accent: string;
+}
+
+/** A downloadable file, from either store. */
+export interface FileRecord {
+	id: string;
+	source: "upload" | "media";
+	name: string;
+	type: string;
+	size: number;
+	uploadedAt: string;
+	key: string;
+}
+
+/** Large-upload metadata in plugin storage. */
+export interface UploadDoc {
+	id: string;
+	key: string;
+	name: string;
+	size: number;
+	type: string;
+	uploadedAt: string;
+	status: "uploading" | "ready";
+	uploadId?: string;
+	partSize?: number;
+}
+
+const SETTING_KEYS = ["filesBase", "filesPublicBaseUrl", "filesScheme", "filesAccent"] as const;
+for (const key of SETTING_KEYS) registerSiteSetting(key);
+const FILE_TTL = 60_000;
+const FILE_CACHE_MAX = 500;
+
+/** The last stored values and the settings made from them, so a cache hit doesn't rebuild them. */
+let settingsMemo: { raw: unknown[]; value: SiteSettings } | null = null;
+const fileCache = new Map<string, { value: FileRecord | null; at: number }>();
+
+/** Forget cached settings and files (after admin changes in this isolate). */
+export function invalidateSiteCache(id?: string): void {
+	if (id) fileCache.delete(id);
+	else {
+		invalidateFeatures();
+		fileCache.clear();
+	}
+}
+
+export function settingsFrom(raw: Record<string, unknown>): SiteSettings {
+	const scheme = raw.filesScheme === "light" || raw.filesScheme === "dark" ? raw.filesScheme : "auto";
+	const publicBase = typeof raw.filesPublicBaseUrl === "string" ? raw.filesPublicBaseUrl.trim().replace(/\/+$/, "") : "";
+	return {
+		base: normalizeBase(raw.filesBase),
+		publicBaseUrl: /^https:\/\/[^\s"'<>]+$/i.test(publicBase) ? publicBase : "",
+		scheme,
+		accent: safeColor(raw.filesAccent),
+	};
+}
+
+/** The settings from D1, in the feature switches' query and per-isolate cache (defaults if it can't be read). */
+export async function readSiteSettings(db: D1Database): Promise<SiteSettings> {
+	const reads = await Promise.all(SETTING_KEYS.map((key) => readSiteSetting(key, db)));
+	if (reads.some((r) => !r)) console.error("coywolf-pack files: could not read settings");
+	const raw = reads.map((r) => r?.value ?? null);
+	const memo = settingsMemo;
+	if (memo && raw.every((v, i) => v === memo.raw[i])) return memo.value;
+	const value = settingsFrom(Object.fromEntries(SETTING_KEYS.map((key, i) => [key, raw[i] ?? undefined])));
+	settingsMemo = { raw, value };
+	return value;
+}
+
+export function uploadToRecord(doc: UploadDoc): FileRecord {
+	return { id: doc.id, source: "upload", name: doc.name, type: doc.type, size: doc.size, uploadedAt: doc.uploadedAt, key: doc.key };
+}
+
+/** Look up a file by id: a ready large upload, or a ready media library item. Null when missing. */
+export async function resolveFile(db: D1Database, id: string): Promise<FileRecord | null> {
+	const hit = fileCache.get(id);
+	if (hit && Date.now() - hit.at < FILE_TTL) return hit.value;
+	let value: FileRecord | null = null;
+	if (isUploadId(id)) {
+		const row = await db
+			.prepare("SELECT data FROM _plugin_storage WHERE plugin_id = ? AND collection = ? AND id = ?")
+			.bind(PLUGIN_ID, COLLECTIONS.uploads, id)
+			.first<{ data: string }>();
+		const doc = row ? (JSON.parse(row.data) as UploadDoc) : null;
+		value = doc && doc.status === "ready" ? uploadToRecord(doc) : null;
+	} else {
+		const row = await db
+			.prepare("SELECT id, filename, mime_type, size, storage_key, created_at, status FROM media WHERE id = ?")
+			.bind(id)
+			.first<{ id: string; filename: string; mime_type: string; size: number | null; storage_key: string; created_at: string; status: string | null }>();
+		value =
+			row && (row.status ?? "ready") === "ready"
+				? {
+						id: row.id,
+						source: "media",
+						name: row.filename,
+						type: row.mime_type,
+						size: row.size ?? 0,
+						uploadedAt: row.created_at,
+						key: row.storage_key,
+					}
+				: null;
+	}
+	if (fileCache.size >= FILE_CACHE_MAX) fileCache.delete(fileCache.keys().next().value as string);
+	fileCache.set(id, { value, at: Date.now() });
+	return value;
+}
+
+// ── Download counts ──────────────────────────────────────────────
+
+const counter = createDownloadCounter(PLUGIN_ID, COLLECTIONS.counts);
+
+/** Count a download; counts within a second in this isolate are written together after the response. */
+export function countDownload(db: D1Database, id: string, waitUntil: (p: Promise<unknown>) => void): void {
+	counter.count(db, id, waitUntil);
+}

package/src/files/walker.ts

@@ -0,0 +1,58 @@
+/**
+ * Find coywolf-file blocks in saved content. Portable Text can sit in any
+ * field and nest (columns, sections, repeaters), so the walk is generic:
+ * every object with `_type: "coywolf-file"` and a string `id` counts.
+ * No imports, so `node --test` can load this file directly.
+ */
+
+export const BLOCK_TYPE = "coywolf-file";
+
+export interface FileBlockRef {
+	/** File id (large-upload id or media library id). */
+	id: string;
+	/** Top-level field the block was found in. */
+	field: string;
+}
+
+const MAX_DEPTH = 64;
+
+/** Every coywolf-file block in a content record, in document order. */
+export function findFileBlocks(content: unknown): FileBlockRef[] {
+	const out: FileBlockRef[] = [];
+	const seen = new WeakSet<object>();
+	// The field is the first key below the record (or below `data`, where EmDash keeps fields).
+	const visit = (value: unknown, path: string[], depth: number) => {
+		if (depth > MAX_DEPTH || value === null || typeof value !== "object") return;
+		if (seen.has(value)) return;
+		seen.add(value);
+		if (Array.isArray(value)) {
+			for (const item of value) visit(item, path, depth + 1);
+			return;
+		}
+		const record = value as Record<string, unknown>;
+		if (record._type === BLOCK_TYPE && typeof record.id === "string" && record.id.trim()) {
+			out.push({ id: record.id.trim(), field: (path[0] === "data" ? path[1] : path[0]) ?? "" });
+		}
+		for (const [key, child] of Object.entries(record)) {
+			if (child && typeof child === "object") visit(child, path.length < 2 ? [...path, key] : path, depth + 1);
+		}
+	};
+	visit(content, [], 0);
+	return out;
+}
+
+/** Distinct file ids referenced by a content record. */
+export function fileIdsIn(content: unknown): string[] {
+	return [...new Set(findFileBlocks(content).map((b) => b.id))];
+}
+
+/** A readable title for a content record (EmDash puts fields under `data`). */
+export function entryTitle(content: Record<string, unknown>): string {
+	const data = (content.data && typeof content.data === "object" ? content.data : content) as Record<string, unknown>;
+	for (const key of ["title", "name", "headline"]) {
+		const value = data[key] ?? content[key];
+		if (typeof value === "string" && value.trim()) return value.trim().slice(0, 200);
+	}
+	if (typeof content.slug === "string" && content.slug) return content.slug;
+	return String(content.id ?? "Untitled");
+}

package/src/formUploads/lib.ts

@@ -0,0 +1,266 @@
+/**
+ * Private form uploads: the pure parts (no EmDash or Cloudflare imports, so
+ * tests run them directly). See ./wrap.ts for how the Forms plugin is wrapped.
+ */
+
+/** Prefix of the media id the Forms plugin stores for a file kept in the private bucket. */
+export const PRIVATE_PREFIX = "coywolf-private:";
+/** Forms plugin KV: one entry per private file (the index the admin page lists). */
+export const ENTRY_PREFIX = "coywolf-private-upload:";
+/** Forms plugin KV: which forms keep their files private, and for how long. */
+export const CONFIG_KEY = "coywolf-private-uploads:config";
+/** Cron task (on the Forms plugin) that tidies files whose submission is gone, and applies the retention setting. */
+export const SWEEP_TASK = "coywolf-private-uploads";
+/** Route names added to the Forms plugin, under /_emdash/api/plugins/emdash-forms/. */
+export const ROUTE_PREFIX = "coywolf-private-uploads/";
+/** The Forms plugin's own cap for one file; enforced again here in case a future version raises it. */
+export const MAX_FILE_BYTES = 10 * 1024 * 1024;
+/** Download chunk: plugin route responses are capped at 8 MB. */
+export const CHUNK_BYTES = 4 * 1024 * 1024;
+/** A file with no submission after this long belongs to a submission that failed to save. */
+export const ORPHAN_AFTER_MS = 60 * 60_000;
+
+export interface FormUploadsConfig {
+	/** "all": every form's files are private. "selected": only the forms in `forms` (form ids). */
+	scope: "all" | "selected";
+	forms: string[];
+	/** Delete private files this many days after upload (the submission stays). 0 keeps them until the submission is deleted. */
+	retentionDays: number;
+}
+
+export const DEFAULT_CONFIG: FormUploadsConfig = { scope: "all", forms: [], retentionDays: 0 };
+
+export function normalizeConfig(value: unknown): FormUploadsConfig {
+	const v = (value && typeof value === "object" ? value : {}) as Partial<FormUploadsConfig>;
+	const forms = Array.isArray(v.forms) ? [...new Set(v.forms.filter((f): f is string => typeof f === "string" && f.length > 0 && f.length <= 200))].slice(0, 500) : [];
+	const days = Number(v.retentionDays);
+	return {
+		scope: v.scope === "selected" ? "selected" : "all",
+		forms,
+		retentionDays: Number.isInteger(days) && days > 0 ? Math.min(days, 3650) : 0,
+	};
+}
+
+/** Whether a form's files go to the private bucket. Matches the form's id or slug. */
+export function formSelected(config: FormUploadsConfig, form: { id: string; slug?: string }): boolean {
+	if (config.scope === "all") return true;
+	return config.forms.includes(form.id) || (!!form.slug && config.forms.includes(form.slug));
+}
+
+/** One private file, as kept in the Forms plugin's KV. */
+export interface UploadEntry {
+	id: string;
+	/** Object key in the private bucket. */
+	key: string;
+	formId: string;
+	formSlug: string;
+	formName: string;
+	fieldName: string;
+	fieldLabel: string;
+	/** Sanitized file name (what the download is saved as). */
+	filename: string;
+	/** Type the visitor's browser reported (shown to admins; never served as-is). */
+	contentType: string;
+	size: number;
+	uploadedAt: string;
+	/** The Forms submission it belongs to; null until the submission is saved. */
+	submissionId: string | null;
+	/** "media-library" when moved by the one-time migration. */
+	source?: "form" | "media-library";
+}
+
+export const privateMediaId = (id: string) => `${PRIVATE_PREFIX}${id}`;
+
+/** The upload id in a private media id, or null for an ordinary media-library id. */
+export function privateIdOf(mediaId: unknown): string | null {
+	if (typeof mediaId !== "string" || !mediaId.startsWith(PRIVATE_PREFIX)) return null;
+	const id = mediaId.slice(PRIVATE_PREFIX.length);
+	return UPLOAD_ID.test(id) ? id : null;
+}
+
+export const UPLOAD_ID = /^[a-f0-9]{32}$/;
+
+export function newUploadId(): string {
+	return [...crypto.getRandomValues(new Uint8Array(16))].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+export const objectKey = (id: string) => `uploads/${id}`;
+
+const WINDOWS_RESERVED = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
+const MAX_NAME = 120;
+
+/**
+ * A file name that's safe to show, store and save: the last path segment,
+ * without control characters, characters Windows or macOS refuse
+ * (\ / : * ? " < > |), leading dots (hidden files) or trailing dots and
+ * spaces, at most 120 characters with the extension kept. Falls back to
+ * "upload".
+ */
+export function sanitizeFilename(name: unknown): string {
+	let base = String(name ?? "")
+		.normalize("NFC")
+		.split(/[\\/]/)
+		.pop()!
+		// biome-ignore lint/suspicious/noControlCharactersInRegex: stripping control characters is the point.
+		.replace(/[\u0000-\u001f\u007f-\u009f​-‏‪-‮⁦-⁩]/g, "")
+		.replace(/[:*?"<>|]/g, "_")
+		.replace(/\s+/g, " ")
+		.trim()
+		.replace(/^[.\s]+/, "")
+		.replace(/[.\s]+$/, "");
+	const dot = base.lastIndexOf(".");
+	let stem = dot > 0 ? base.slice(0, dot) : base;
+	let ext = dot > 0 ? base.slice(dot + 1) : "";
+	if (ext.length > 16 || !/^[\p{L}\p{N}_-]+$/u.test(ext)) {
+		stem = base;
+		ext = "";
+	}
+	if (WINDOWS_RESERVED.test(stem)) stem = `_${stem}`;
+	const room = MAX_NAME - (ext ? ext.length + 1 : 0);
+	if ([...stem].length > room) stem = [...stem].slice(0, room).join("").trimEnd();
+	base = ext ? `${stem}.${ext}` : stem;
+	return base && base !== "." ? base : "upload";
+}
+
+/**
+ * Types a download may be labeled with. Anything else (HTML, SVG, XML,
+ * JavaScript, CSS, unknown types) goes out as application/octet-stream, so
+ * a browser never renders an uploaded file as a page or runs it.
+ */
+const SAFE_TYPES = new Set([
+	"application/pdf",
+	"application/zip",
+	"application/gzip",
+	"application/json",
+	"application/msword",
+	"application/rtf",
+	"application/vnd.ms-excel",
+	"application/vnd.ms-powerpoint",
+	"application/vnd.oasis.opendocument.text",
+	"application/vnd.oasis.opendocument.spreadsheet",
+	"application/vnd.oasis.opendocument.presentation",
+	"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+	"application/vnd.openxmlformats-officedocument.presentationml.presentation",
+	"image/png",
+	"image/jpeg",
+	"image/gif",
+	"image/webp",
+	"image/avif",
+	"image/heic",
+	"image/heif",
+	"image/bmp",
+	"image/tiff",
+	"text/plain",
+	"text/csv",
+	"audio/mpeg",
+	"audio/mp4",
+	"audio/ogg",
+	"audio/wav",
+	"audio/webm",
+	"video/mp4",
+	"video/quicktime",
+	"video/webm",
+	"video/ogg",
+]);
+
+export function safeContentType(type: unknown): string {
+	const mime = String(type ?? "")
+		.split(";", 1)[0]!
+		.trim()
+		.toLowerCase();
+	return SAFE_TYPES.has(mime) ? mime : "application/octet-stream";
+}
+
+/**
+ * Types a form may store: the same as EmDash's media library accepts from a
+ * plugin upload (images, video, audio, PDF), so keeping files private never
+ * lets a form take files it couldn't take before. The type is the one the
+ * visitor's browser reported; the field's own `accept` list is checked by
+ * the Forms plugin first.
+ */
+const UPLOAD_TYPES = ["image/png", "image/jpeg", "image/gif", "image/webp", "image/avif", "image/jxl", "application/pdf"];
+const UPLOAD_PREFIXES = ["video/", "audio/"];
+const MIME = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/;
+
+export function uploadTypeAllowed(type: unknown): boolean {
+	const mime = String(type ?? "")
+		.split(";", 1)[0]!
+		.trim()
+		.toLowerCase();
+	if (!MIME.test(mime)) return false;
+	return UPLOAD_TYPES.includes(mime) || UPLOAD_PREFIXES.some((p) => mime.startsWith(p));
+}
+
+/** "1.2 MB" */
+export function formatBytes(bytes: number): string {
+	if (!Number.isFinite(bytes) || bytes < 1024) return `${Math.max(0, Math.round(bytes || 0))} B`;
+	const units = ["KB", "MB", "GB"];
+	let value = bytes / 1024;
+	let unit = 0;
+	while (value >= 1024 && unit < units.length - 1) {
+		value /= 1024;
+		unit++;
+	}
+	return `${value >= 10 ? Math.round(value) : value.toFixed(1)} ${units[unit]}`;
+}
+
+/** Byte range of download chunk `part` of a file of `size` bytes, or null past the end. */
+export function chunkRange(size: number, part: number): { start: number; end: number } | null {
+	if (!Number.isInteger(part) || part < 0) return null;
+	const start = part * CHUNK_BYTES;
+	if (start >= size && !(size === 0 && part === 0)) return null;
+	return { start, end: Math.min(size, start + CHUNK_BYTES) - 1 };
+}
+
+/** Total size from a "bytes 0-99/1234" Content-Range header. */
+export function totalFromContentRange(header: string | null): number | null {
+	const match = /\/(\d+)\s*$/.exec(header ?? "");
+	return match ? Number(match[1]) : null;
+}
+
+/**
+ * Which of the submitted files a Forms upload call is for. The Forms plugin
+ * uploads files one by one with (filename, type, bytes) and no field name,
+ * so match on name and size, skipping fields already matched.
+ */
+export function matchField(
+	files: Record<string, { filename: string; contentType?: string; bytes: { byteLength: number } }> | undefined,
+	filename: string,
+	size: number,
+	taken: Set<string>,
+): string {
+	for (const [field, file] of Object.entries(files ?? {})) {
+		if (taken.has(field)) continue;
+		if (file.filename === filename && file.bytes.byteLength === size) {
+			taken.add(field);
+			return field;
+		}
+	}
+	return "";
+}
+
+/** A short line from a submission's answers, so admins can tell submissions apart. */
+export function submissionPreview(data: Record<string, unknown> | undefined, max = 140): string {
+	const parts: string[] = [];
+	for (const [key, value] of Object.entries(data ?? {})) {
+		if (key.startsWith("_") || key === "cf-turnstile-response") continue;
+		const text = Array.isArray(value) ? value.join(", ") : value === null || value === undefined ? "" : String(value);
+		const clean = text.replace(/\s+/g, " ").trim();
+		if (clean) parts.push(clean);
+		if (parts.join(" · ").length >= max) break;
+	}
+	const line = parts.join(" · ");
+	return line.length > max ? `${line.slice(0, max - 1).trimEnd()}…` : line;
+}
+
+/** Entries the daily sweep deletes: past retention, orphaned, or whose submission is gone. */
+export function sweepDecision(
+	entry: UploadEntry,
+	opts: { now: number; retentionDays: number; submissionExists: boolean | null },
+): "keep" | "retention" | "orphan" | "relink" {
+	const age = opts.now - Date.parse(entry.uploadedAt);
+	if (opts.retentionDays > 0 && age > opts.retentionDays * 86_400_000) return "retention";
+	if (entry.submissionId) return opts.submissionExists === false ? "orphan" : "keep";
+	return age > ORPHAN_AFTER_MS ? "relink" : "keep";
+}

package/src/formUploads/pack.ts

@@ -0,0 +1,55 @@
+/**
+ * Private form uploads module: the feature switch and the Form uploads admin
+ * page. The work happens in the Forms plugin itself, wrapped by
+ * privateFormUploads() (see ./wrap.ts), so the page talks to routes on the
+ * Forms plugin (/_emdash/api/plugins/emdash-forms/coywolf-private-uploads/*).
+ */
+import type { PluginDescriptor } from "emdash";
+
+import { registerFeatures } from "../core/features.js";
+import type { PackModule } from "../core/module.js";
+import { FEATURE, WRAP_OPTION, type PrivateFormUploadsOptions } from "./wrap.js";
+
+export type { PrivateFormUploadsOptions } from "./wrap.js";
+
+export const FEATURES = [
+	{
+		id: FEATURE,
+		label: "Private form uploads",
+		description:
+			"Files sent through Forms plugin forms go to a private storage bucket that only admins can open, instead of the public media library.",
+		default: false,
+	},
+];
+registerFeatures(FEATURES);
+
+export function formUploadsPack(): PackModule {
+	return {
+		id: "formUploads",
+		label: "Private form uploads",
+		features: FEATURES,
+		adminPages: [{ path: "/form-uploads", label: "Form uploads", icon: "lock" }],
+	};
+}
+
+/**
+ * Wrap the Forms plugin's descriptor so its file uploads can be kept private:
+ *
+ * ```js
+ * import { coywolfPlugin, privateFormUploads } from "@coywolf/emdash";
+ * import { formsPlugin } from "@emdash-cms/plugin-forms";
+ * plugins: [coywolfPlugin({...}), privateFormUploads(formsPlugin())]
+ * ```
+ *
+ * The plugin keeps its id, storage, admin pages and settings; only its code
+ * entry changes to "@coywolf/emdash/forms", which runs the Forms plugin with
+ * private uploads. Nothing changes until the feature is turned on.
+ */
+export function privateFormUploads<T extends object>(forms: PluginDescriptor<T>, options: PrivateFormUploadsOptions = {}): PluginDescriptor<T> {
+	if (forms.id !== "emdash-forms") throw new Error(`privateFormUploads() wraps the EmDash Forms plugin (emdash-forms), not "${forms.id}".`);
+	return {
+		...forms,
+		entrypoint: "@coywolf/emdash/forms",
+		options: { ...(forms.options ?? {}), [WRAP_OPTION]: options } as T,
+	};
+}