emdash 1.0.1 → 1.2.0

package/src/storage/local.ts

@@ -16,10 +16,12 @@ function isNodeError(error: unknown): error is NodeJS.ErrnoException {
 	return error instanceof Error && "code" in error;
 }
 
+import { resolveByteRange } from "./range.js";
 import type {
 	Storage,
 	LocalStorageConfig,
 	UploadResult,
+	DownloadOptions,
 	DownloadResult,
 	ListResult,
 	ListOptions,
@@ -108,7 +110,7 @@ export class LocalStorage implements Storage {
 		}
 	}
 
-	async download(key: string): Promise<DownloadResult> {
+	async download(key: string, options: DownloadOptions = {}): Promise<DownloadResult> {
 		try {
 			const filePath = this.getFilePath(key);
 
@@ -117,7 +119,11 @@ export class LocalStorage implements Storage {
 			}
 
 			const stat = await fs.stat(filePath);
-			const nodeStream = createReadStream(filePath);
+			const range = options.range && resolveByteRange(options.range, stat.size);
+			const nodeStream = createReadStream(
+				filePath,
+				range ? { start: range.offset, end: range.offset + range.length - 1 } : undefined,
+			);
 
 			// Convert Node.js stream to web ReadableStream
 			// Readable.toWeb returns ReadableStream (which is ReadableStream<unknown>),
@@ -135,6 +141,8 @@ export class LocalStorage implements Storage {
 				body: webStream,
 				contentType,
 				size: stat.size,
+				lastModified: stat.mtime,
+				...(range && { range }),
 			};
 		} catch (error) {
 			if (error instanceof EmDashStorageError) throw error;

package/src/storage/range.ts

@@ -0,0 +1,54 @@
+/**
+ * Byte ranges for partial downloads (HTTP `Range` requests)
+ */
+
+import type { ByteRange } from "./types.js";
+
+const BYTES_UNIT_PATTERN = /^bytes=/i;
+const RANGE_SPEC_PATTERN = /^(\d*)-(\d*)$/;
+
+/**
+ * Parse a `Range` header that asks for a single byte range.
+ *
+ * Returns null when the header should be ignored and the whole file served:
+ * it's missing or malformed, uses another unit, or asks for several ranges.
+ */
+export function parseRangeHeader(header: string | null): ByteRange | null {
+	if (!header || !BYTES_UNIT_PATTERN.test(header)) return null;
+	const [spec, ...otherSpecs] = header
+		.slice("bytes=".length)
+		.split(",")
+		.map((part) => part.trim())
+		.filter((part) => part !== "");
+	const match =
+		spec !== undefined && otherSpecs.length === 0 ? RANGE_SPEC_PATTERN.exec(spec) : null;
+	if (!match) return null;
+
+	const [, first = "", last = ""] = match;
+	if (first === "") {
+		const suffix = Number(last);
+		return last !== "" && Number.isSafeInteger(suffix) ? { suffix } : null;
+	}
+	const offset = Number(first);
+	if (!Number.isSafeInteger(offset)) return null;
+	if (last === "") return { offset };
+	const end = Number(last);
+	if (!Number.isSafeInteger(end) || end < offset) return null;
+	return { offset, length: end - offset + 1 };
+}
+
+/**
+ * The bytes a range selects from a file of `size` bytes, or null when it
+ * selects none.
+ */
+export function resolveByteRange(
+	range: ByteRange,
+	size: number,
+): { offset: number; length: number } | null {
+	if ("suffix" in range) {
+		const length = Math.min(range.suffix, size);
+		return length > 0 ? { offset: size - length, length } : null;
+	}
+	const length = Math.min(range.length ?? size, size - range.offset);
+	return range.offset >= 0 && length > 0 ? { offset: range.offset, length } : null;
+}

package/src/storage/s3.ts

@@ -22,6 +22,8 @@ import type {
 	Storage,
 	S3StorageConfig,
 	UploadResult,
+	ByteRange,
+	DownloadOptions,
 	DownloadResult,
 	ListResult,
 	ListOptions,
@@ -112,12 +114,28 @@ export function resolveS3Config(partial: Record<string, unknown>): S3StorageConf
 }
 
 const TRAILING_SLASH_PATTERN = /\/$/;
+const CONTENT_RANGE_PATTERN = /^bytes (\d+)-(\d+)\/(\d+)$/;
 
 /** Type guard for AWS SDK errors (have a `name` property) */
 function hasErrorName(error: unknown): error is Error & { name: string } {
 	return error instanceof Error && typeof error.name === "string";
 }
 
+function formatRangeHeader(range: ByteRange): string {
+	if ("suffix" in range) return `bytes=-${range.suffix}`;
+	const end = range.length === undefined ? "" : range.offset + range.length - 1;
+	return `bytes=${range.offset}-${end}`;
+}
+
+function parseContentRange(
+	header: string,
+): { offset: number; length: number; size: number } | undefined {
+	const match = CONTENT_RANGE_PATTERN.exec(header);
+	if (!match) return undefined;
+	const offset = Number(match[1]);
+	return { offset, length: Number(match[2]) - offset + 1, size: Number(match[3]) };
+}
+
 /**
  * S3-compatible storage implementation
  */
@@ -195,14 +213,9 @@ export class S3Storage implements Storage {
 		}
 	}
 
-	async download(key: string): Promise<DownloadResult> {
+	async download(key: string, options: DownloadOptions = {}): Promise<DownloadResult> {
 		try {
-			const response = await this.client.send(
-				new GetObjectCommand({
-					Bucket: this.bucket,
-					Key: key,
-				}),
-			);
+			const { response, served } = await this.getObject(key, options.range);
 
 			if (!response.Body) {
 				throw new EmDashStorageError(`File not found: ${key}`, "NOT_FOUND");
@@ -210,11 +223,23 @@ export class S3Storage implements Storage {
 
 			// Convert SDK stream to web ReadableStream
 			const body = response.Body.transformToWebStream();
+			const contentType = response.ContentType || "application/octet-stream";
+
+			if (served) {
+				return {
+					body,
+					contentType,
+					size: served.size,
+					lastModified: response.LastModified,
+					range: { offset: served.offset, length: served.length },
+				};
+			}
 
 			return {
 				body,
-				contentType: response.ContentType || "application/octet-stream",
+				contentType,
 				size: response.ContentLength || 0,
+				lastModified: response.LastModified,
 			};
 		} catch (error) {
 			if (
@@ -227,6 +252,25 @@ export class S3Storage implements Storage {
 		}
 	}
 
+	/** Get the object, or all of it when the service can't serve `range` */
+	private async getObject(key: string, range: ByteRange | undefined) {
+		const get = (rangeHeader?: string) =>
+			this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key, Range: rangeHeader }));
+		if (range) {
+			try {
+				const response = await get(formatRangeHeader(range));
+				// A service that ignores `Range` sends the whole object without `Content-Range`.
+				if (!response.ContentRange) return { response };
+				const served = parseContentRange(response.ContentRange);
+				if (served) return { response, served };
+				await response.Body?.transformToWebStream().cancel();
+			} catch (error) {
+				if (!hasErrorName(error) || error.name !== "InvalidRange") throw error;
+			}
+		}
+		return { response: await get() };
+	}
+
 	async delete(key: string): Promise<void> {
 		try {
 			await this.client.send(

package/src/storage/types.ts

@@ -73,6 +73,26 @@ export interface UploadResult {
 	size: number;
 }
 
+/**
+ * A single byte range of a file, as requested by an HTTP `Range` header
+ */
+export type ByteRange =
+	/** `length` bytes from `offset`, or every byte from `offset` when `length` is omitted */
+	| { offset: number; length?: number }
+	/** The last `suffix` bytes */
+	| { suffix: number };
+
+/**
+ * Options for downloading a file
+ */
+export interface DownloadOptions {
+	/**
+	 * Download only this part of the file. Adapters that can't serve it,
+	 * including when it starts past the end of the file, return the whole file.
+	 */
+	range?: ByteRange;
+}
+
 /**
  * Download result
  */
@@ -81,8 +101,15 @@ export interface DownloadResult {
 	body: ReadableStream<Uint8Array>;
 	/** MIME type */
 	contentType: string;
-	/** File size in bytes */
+	/** Size of the whole file in bytes, even when `body` holds only part of it */
 	size: number;
+	/**
+	 * The part of the file `body` holds, when the adapter served
+	 * `DownloadOptions.range`. Absent when `body` is the whole file.
+	 */
+	range?: { offset: number; length: number };
+	/** Last modification time of the stored object, when the backend can provide it. */
+	lastModified?: Date;
 }
 
 /**
@@ -167,9 +194,9 @@ export interface Storage {
 	}): Promise<UploadResult>;
 
 	/**
-	 * Download a file from storage
+	 * Download a file from storage, or part of it with `options.range`
 	 */
-	download(key: string): Promise<DownloadResult>;
+	download(key: string, options?: DownloadOptions): Promise<DownloadResult>;
 
 	/**
 	 * Delete a file from storage

package/src/transfer/format/columns.ts

@@ -544,6 +544,9 @@ export const NON_PORTABLE_TABLES: Readonly<Record<string, string>> = Object.free
 	_emdash_entry_locks: "runtime",
 	_emdash_cron_tasks: "runtime",
 	_emdash_redirect_write_lock: "runtime",
+	_emdash_redirect_state: "derived",
+	_emdash_redirect_artifacts: "derived",
+	_emdash_redirect_generation_artifacts: "derived",
 	_emdash_404_log: "runtime",
 	_emdash_revision_prune_queue: "runtime",
 	_emdash_media_upload_attempts: "runtime",

package/src/transfer/import/rebuild.ts

@@ -29,6 +29,7 @@ import { FTSManager } from "../../search/fts-manager.js";
 import type { SearchTokenizer } from "../../search/types.js";
 import { SEARCH_TOKENIZERS } from "../../search/types.js";
 import { invalidateSiteSettingsCache } from "../../settings/index.js";
+import { isMissingTableError } from "../../utils/db-errors.js";
 import { TransferError } from "../errors.js";
 import { compareIds, type CollectionRecord } from "../format/kinds.js";
 import { POST_IMPORT_OPTION_RESETS } from "../format/settings.js";
@@ -43,8 +44,8 @@ const FTS_POPULATE_STATEMENTS = 4;
 const STALE_STATEMENTS = 4;
 const OPTION_STATEMENTS = 4;
 const TAXONOMY_STATEMENTS = 4;
-/** Cache invalidation runs no queries; its checkpoint does. */
-const CACHE_STATEMENTS = 1;
+/** Unpublishing the redirect generation, plus the checkpoint. */
+const CACHE_STATEMENTS = 2;
 
 export interface RebuildPosition {
 	step: RebuildStep;
@@ -280,6 +281,7 @@ async function taxonomyUnit(
 }
 
 async function invalidateCaches(context: ImportContext): Promise<void> {
+	await unpublishRedirectArtifacts(context);
 	for (const namespace of Object.values(CacheNamespace)) invalidateObjectCache(namespace);
 	for (const collection of await collections(context)) {
 		invalidateCollectionCache(collection.slug);
@@ -298,3 +300,16 @@ async function invalidateCaches(context: ImportContext): Promise<void> {
 	invalidateRedirectCache();
 	invalidateUrlPatternCache();
 }
+
+/** Point redirect readers at the imported rules table until a repair republishes. */
+async function unpublishRedirectArtifacts(context: ImportContext): Promise<void> {
+	try {
+		await context.db
+			.updateTable("_emdash_redirect_state")
+			.set({ generation: null, generation_revision: -1 })
+			.where("id", "=", 1)
+			.execute();
+	} catch (error) {
+		if (!isMissingTableError(error)) throw error;
+	}
+}

package/src/transfer/import/scaffold.ts

@@ -8,6 +8,7 @@
 import { sql } from "kysely";
 
 import { processDueMediaUsageCollectionDeletions } from "../../media/usage/collection-deletion-processor.js";
+import { invalidateMenuObjectCache } from "../../object-cache/index.js";
 import { SchemaError, SchemaRegistry } from "../../schema/registry.js";
 import { TransferError } from "../errors.js";
 import type { ImportTransformation } from "../format/transformations.js";
@@ -44,16 +45,22 @@ export async function clearScaffold(
 ): Promise<ScaffoldProgress> {
 	const items = declaredScaffold(context);
 	let next = step;
-	while (next < items.length) {
-		const item = items[next];
-		if (!item) break;
-		if (!context.budget.canStart({ queries: ITEM_STATEMENTS })) {
-			return { state: "continue", step: next };
+	let removedMenus = false;
+	try {
+		while (next < items.length) {
+			const item = items[next];
+			if (!item) break;
+			if (!context.budget.canStart({ queries: ITEM_STATEMENTS })) {
+				return { state: "continue", step: next };
+			}
+			context.budget.start();
+			await removeItem(context, item);
+			if (item.type === "menu" || item.type === "menu_item") removedMenus = true;
+			next++;
+			await checkpoint(next);
 		}
-		context.budget.start();
-		await removeItem(context, item);
-		next++;
-		await checkpoint(next);
+	} finally {
+		if (removedMenus) invalidateMenuObjectCache();
 	}
 
 	const collectionIds = items.flatMap((item) => (item.type === "collection" ? [item.id] : []));

package/src/ui.ts

@@ -78,6 +78,8 @@ export {
 	Columns,
 	Break,
 	HtmlBlock,
+	Iframe,
+	Video,
 	// Marks
 	Superscript,
 	Subscript,

package/src/utils/sanitize.ts

@@ -1,20 +1,25 @@
+import {
+	SITE_HTML_ALLOWED_ATTRIBUTES,
+	SITE_HTML_ALLOWED_SCHEMES,
+	SITE_HTML_ALLOWED_TAGS,
+	SITE_HTML_IFRAME_HOSTS,
+} from "@emdash-cms/admin/html-block";
 import sanitizeHtml from "sanitize-html";
 
 /**
  * Sanitize HTML content to prevent XSS attacks.
  *
  * Allows standard formatting tags, images, iframes (from specific providers),
- * and basic attributes.
+ * and basic attributes. The admin's inline HTML block preview uses the same
+ * allowlist.
  */
 export function sanitizeContent(html: string): string {
 	return sanitizeHtml(html, {
-		allowedTags: [...sanitizeHtml.defaults.allowedTags, "img", "span", "iframe"],
-		allowedAttributes: {
-			...sanitizeHtml.defaults.allowedAttributes,
-			"*": ["class", "id", "data-*"],
-			iframe: ["src", "width", "height", "frameborder", "allow", "allowfullscreen"],
-			img: ["src", "srcset", "alt", "title", "width", "height", "loading"],
-		},
-		allowedIframeHostnames: ["www.youtube.com", "player.vimeo.com"],
+		allowedTags: [...SITE_HTML_ALLOWED_TAGS],
+		allowedAttributes: Object.fromEntries(
+			Object.entries(SITE_HTML_ALLOWED_ATTRIBUTES).map(([tag, names]) => [tag, [...names]]),
+		),
+		allowedSchemes: [...SITE_HTML_ALLOWED_SCHEMES],
+		allowedIframeHostnames: [...SITE_HTML_IFRAME_HOSTS],
 	});
 }

package/src/utils/slugify.ts

@@ -8,5 +8,10 @@ export { slugify } from "@emdash-cms/admin/slugify";
  * Call this on `Astro.params.slug` before using it in database lookups.
  */
 export function decodeSlug(raw: string | undefined): string | undefined {
-	return raw ? decodeURIComponent(raw) : undefined;
+	if (!raw) return undefined;
+	try {
+		return decodeURIComponent(raw);
+	} catch {
+		return undefined;
+	}
 }

package/src/virtual-modules.d.ts

@@ -29,6 +29,8 @@ declare module "virtual:emdash/config" {
 		siteUrl?: string;
 		astroCspEnabled?: boolean;
 		trailingSlash?: "always" | "never" | "ignore";
+		/** `routePattern` of the image endpoint EmDash installed; absent when it installed none. */
+		imageEndpointRoute?: string;
 	}
 
 	const config: VirtualConfig;
@@ -81,6 +83,7 @@ declare module "virtual:emdash/dialect" {
 		lastContentWriteAt?: number;
 	}
 	export interface RequestScopedDb {
+		/** A per-request handle on the configured database, never a different one. */
 		db: Kysely<unknown>;
 		commit: () => void;
 		/**