emdash 1.1.0 → 1.3.0

package/src/schema/slug.ts

@@ -0,0 +1,19 @@
+/**
+ * The collection and field slug rule.
+ */
+
+/** Lowercase letter first, then lowercase letters, digits, and underscores. */
+export const SCHEMA_SLUG_PATTERN = /^[a-z][a-z0-9_]*$/;
+
+/** Maximum length of a collection or field slug. */
+export const MAX_SCHEMA_SLUG_LENGTH = 63;
+
+/** Whether `value` is a valid collection or field slug. */
+export function isValidSchemaSlug(value: unknown): value is string {
+	return (
+		typeof value === "string" &&
+		value.length > 0 &&
+		value.length <= MAX_SCHEMA_SLUG_LENGTH &&
+		SCHEMA_SLUG_PATTERN.test(value)
+	);
+}

package/src/search/query.ts

@@ -63,7 +63,9 @@ function decodeSearchOffset(cursor: string): number {
 
 /** Pattern to split on whitespace for query term extraction */
 const WHITESPACE_SPLIT_PATTERN = /\s+/;
-const FTS_OPERATORS_PATTERN = /\b(AND|OR|NOT|NEAR)\b/i;
+// FTS5 operators are uppercase-only; a case-insensitive match would treat
+// ordinary words as syntax and skip the safe quoting path below.
+const FTS_OPERATORS_PATTERN = /\b(AND|OR|NOT|NEAR)\b/;
 const DOUBLE_QUOTE_PATTERN = /"/g;
 
 /**

package/src/search/text-extraction.ts

@@ -8,6 +8,7 @@
 import { toPlainText } from "@portabletext/toolkit";
 
 import type { PortableTextBlock } from "../content/converters/types.js";
+import { isPortableTextVideoBlock } from "../content/converters/video.js";
 
 /**
  * Validate that a value looks like a Portable Text block array.
@@ -44,6 +45,10 @@ function extractCustomBlockText(block: PortableTextBlock): string {
 		return parts.join(" ");
 	}
 
+	if (isPortableTextVideoBlock(block)) {
+		return block.caption ?? "";
+	}
+
 	return "";
 }
 
@@ -51,7 +56,7 @@ function extractCustomBlockText(block: PortableTextBlock): string {
  * Extract plain text from Portable Text blocks
  *
  * Uses @portabletext/toolkit's toPlainText for standard blocks,
- * plus extracts text from custom block types (code, images with alt/caption).
+ * plus extracts text from custom block types (code, images with alt/caption, video captions).
  *
  * @param blocks - Array of Portable Text blocks (or a JSON string)
  * @returns Plain text content

package/src/seed/apply.ts

@@ -50,6 +50,7 @@ import { invalidateSiteSettingsCache, setSiteSettings } from "../settings/index.
 import type { SiteSettings } from "../settings/types.js";
 import type { Storage } from "../storage/types.js";
 import { chunks, SQL_BATCH_SIZE } from "../utils/chunks.js";
+import { computeContentHash } from "../utils/hash.js";
 import type {
 	SeedFile,
 	SeedField,
@@ -206,8 +207,9 @@ export interface SeedApplyProgress {
  * the platform's. A call that stops returns `complete: false`; applying the
  * same seed again continues with the first item not yet created. A call
  * creates at least one item before it stops, so repeated calls finish. A
- * `$media` URL used by entries in different calls is downloaded and stored
- * again, as a new media row, in each of those calls. Requires
+ * `$media` URL used by entries in different calls is still downloaded each time
+ * so its content hash can be computed, but the resulting media row is reused
+ * when an existing ready row has the same hash. Requires
  * `onConflict: "skip"`.
  */
 export async function applySeedWithinBudget(
@@ -2324,7 +2326,7 @@ async function resolveMedia(
 			filename: filename ?? undefined,
 		};
 		ctx.mediaCache.set(url, mediaValue);
-		result.media.created++;
+		result.media.skipped++;
 		return mediaValue;
 	}
 
@@ -2359,15 +2361,37 @@ async function resolveMedia(
 		const contentType = response.headers.get("content-type") || "application/octet-stream";
 		const ext = getExtensionFromContentType(contentType) || getExtensionFromUrl(url) || ".bin";
 
+		// Get the body as buffer
+		const arrayBuffer = await response.arrayBuffer();
+		const body = new Uint8Array(arrayBuffer);
+
+		// Deduplicate by content hash so re-running a seed does not mint a
+		// fresh media row for a file that already exists in the library.
+		const contentHash = await computeContentHash(body);
+		const mediaRepo = new MediaRepository(ctx.db);
+		const existing = await mediaRepo.findByContentHash(contentHash);
+		if (existing) {
+			const mediaValue: MediaValue = {
+				provider: "local",
+				id: existing.id,
+				alt: alt ?? existing.alt ?? undefined,
+				width: existing.width ?? undefined,
+				height: existing.height ?? undefined,
+				mimeType: existing.mimeType,
+				filename: existing.filename,
+				meta: { storageKey: existing.storageKey },
+			};
+			ctx.mediaCache.set(url, mediaValue);
+			result.media.skipped++;
+			console.log(`  ✅ Reused: ${existing.filename}`);
+			return mediaValue;
+		}
+
 		// Generate filename and storage key
 		const storageId = ulid();
 		const finalFilename = filename || generateFilename(url, ext);
 		const storageKey = `${storageId}${ext}`;
 
-		// Get the body as buffer
-		const arrayBuffer = await response.arrayBuffer();
-		const body = new Uint8Array(arrayBuffer);
-
 		// Get image dimensions if it's an image
 		let width: number | undefined;
 		let height: number | undefined;
@@ -2385,7 +2409,6 @@ async function resolveMedia(
 		});
 
 		// Create media record
-		const mediaRepo = new MediaRepository(ctx.db);
 		const media = await mediaRepo.create({
 			filename: finalFilename,
 			mimeType: contentType,
@@ -2395,6 +2418,7 @@ async function resolveMedia(
 			alt,
 			caption,
 			storageKey,
+			contentHash,
 			status: "ready",
 		});
 

package/src/seed/validate.ts

@@ -14,6 +14,8 @@ import {
 	MAX_COLLECTION_GROUP_LENGTH,
 	MAX_COLLECTION_ICON_LENGTH,
 	MAX_COLLECTION_LIST_COLUMNS,
+	RESERVED_COLLECTION_SLUGS,
+	RESERVED_FIELD_SLUGS,
 } from "../schema/types.js";
 import { compileUrlPattern } from "../schema/url-pattern.js";
 import type { SeedFile, SeedMenuItem, SeedTaxonomy, ValidationResult } from "./types.js";
@@ -187,6 +189,11 @@ export function validateSeed(data: unknown): ValidationResult {
 						);
 					}
 
+					// Reserved slugs pass the format check but are rejected by the schema registry on apply
+					if (RESERVED_COLLECTION_SLUGS.includes(collection.slug)) {
+						errors.push(`${prefix}.slug: collection slug "${collection.slug}" is reserved`);
+					}
+
 					// Check for duplicate slugs
 					if (collectionSlugs.has(collection.slug)) {
 						errors.push(`${prefix}.slug: duplicate collection slug "${collection.slug}"`);
@@ -294,6 +301,11 @@ export function validateSeed(data: unknown): ValidationResult {
 								);
 							}
 
+							// Reserved slugs pass the format check but are rejected by the schema registry on apply
+							if (RESERVED_FIELD_SLUGS.includes(field.slug)) {
+								errors.push(`${fieldPrefix}.slug: field slug "${field.slug}" is reserved`);
+							}
+
 							// Check for duplicate field slugs
 							if (fieldSlugs.has(field.slug)) {
 								errors.push(
@@ -333,6 +345,19 @@ export function validateSeed(data: unknown): ValidationResult {
 								`${fieldPrefix}.indexed: a reference field with a targetCollection stores no column to index`,
 							);
 						}
+
+						if (field.type === "repeater") {
+							const subFields = field.validation?.subFields;
+							if ("fields" in field) {
+								warnings.push(
+									`${fieldPrefix}.fields: repeater sub-fields must be defined in validation.subFields; these fields are ignored`,
+								);
+							} else if (!Array.isArray(subFields) || subFields.length === 0) {
+								warnings.push(
+									`${fieldPrefix}.validation.subFields: repeater needs a non-empty array of sub-fields, so its rows have nothing to edit`,
+								);
+							}
+						}
 					}
 				}
 			}
@@ -716,6 +741,12 @@ export function validateSeed(data: unknown): ValidationResult {
 						if (widget.type === "component" && !widget.componentId) {
 							errors.push(`${widgetPrefix}: componentId is required for component widgets`);
 						}
+
+						if ("settings" in widget) {
+							warnings.push(
+								`${widgetPrefix}.settings: not applied; widget options belong in "props"`,
+							);
+						}
 					}
 				}
 			}

package/src/seo/sitemap-name.ts

@@ -0,0 +1,22 @@
+/**
+ * The `{name}` in a per-collection sitemap path, `/sitemap-{name}.xml`.
+ */
+
+import { isValidSchemaSlug } from "../schema/slug.js";
+
+/** `{collection}` for page 1, `{collection}-{n}` for page n from 2 up. */
+const SITEMAP_NAME_RE = /^([a-z][a-z0-9_]*)(?:-([2-9]|[1-9]\d{1,5}))?$/;
+
+/**
+ * The collection and page a sitemap name refers to, or null when the name
+ * can't belong to any collection (its collection part must be a valid
+ * collection slug).
+ */
+export function parseCollectionSitemapName(
+	name: string,
+): { collection: string; page: number } | null {
+	const match = SITEMAP_NAME_RE.exec(name);
+	const collection = match?.[1];
+	if (!match || !isValidSchemaSlug(collection)) return null;
+	return { collection, page: match[2] ? Number(match[2]) : 1 };
+}

package/src/settings/index.ts

@@ -24,6 +24,8 @@ import {
 } from "../plugins/settings.js";
 import type { CacheHint } from "../query.js";
 import { peekRequestCache, requestCached } from "../request-cache.js";
+import { bumpRunningExportWriteEpochs } from "../transfer/fence.js";
+import { isMissingTableError } from "../utils/db-errors.js";
 
 /** Object-cache namespace for site settings. */
 const SETTINGS_CACHE_NAMESPACE = "settings";
@@ -375,6 +377,10 @@ export async function setSiteSettings(
 		else updates[`${SETTINGS_PREFIX}${key}`] = value;
 	}
 
+	if (Object.keys(updates).length === 0 && deletions.length === 0 && nestedPatches.length === 0) {
+		return;
+	}
+
 	try {
 		await withTransaction(db, async (trx) => {
 			const transactionOptions = new OptionsRepository(trx);
@@ -391,6 +397,20 @@ export async function setSiteSettings(
 				if (Object.keys(next).length === 0) await transactionOptions.delete(optionName);
 				else await transactionOptions.set(optionName, next);
 			}
+
+			// A running export detects writes to tables without updated_at via
+			// the operation's write_epoch. Record that bump in the same
+			// transaction so the settings update is not accepted unless the
+			// export fence can see it. When the transfer system has not been
+			// installed yet (pre-migration, or a minimal seed fixture), there
+			// is no running export to notify, so the absence of its table is
+			// not a reason to reject the settings update.
+			try {
+				await bumpRunningExportWriteEpochs(trx);
+			} catch (error) {
+				if (isMissingTableError(error)) return;
+				throw error;
+			}
 		});
 	} finally {
 		invalidateSiteSettingsCache();

package/src/settings/types.ts

@@ -44,6 +44,8 @@ export interface SeoSettings {
 	defaultOgImage?: MediaReference;
 	/** Custom robots.txt content. If unset, a default is generated. */
 	robotsTxt?: string;
+	/** Add rules to robots.txt that ask AI crawlers not to use the site for training. */
+	disallowAiTraining?: boolean;
 	/** Google Search Console verification meta tag content */
 	googleVerification?: string;
 	/** Bing Webmaster Tools verification meta tag content */

package/src/storage/index.ts

@@ -13,6 +13,8 @@ export type {
 	LocalStorageConfig,
 	UploadResult,
 	DownloadResult,
+	DownloadOptions,
+	ByteRange,
 	ListResult,
 	ListOptions,
 	FileInfo,

package/src/storage/local.ts

@@ -16,10 +16,12 @@ function isNodeError(error: unknown): error is NodeJS.ErrnoException {
 	return error instanceof Error && "code" in error;
 }
 
+import { resolveByteRange } from "./range.js";
 import type {
 	Storage,
 	LocalStorageConfig,
 	UploadResult,
+	DownloadOptions,
 	DownloadResult,
 	ListResult,
 	ListOptions,
@@ -108,7 +110,7 @@ export class LocalStorage implements Storage {
 		}
 	}
 
-	async download(key: string): Promise<DownloadResult> {
+	async download(key: string, options: DownloadOptions = {}): Promise<DownloadResult> {
 		try {
 			const filePath = this.getFilePath(key);
 
@@ -117,7 +119,11 @@ export class LocalStorage implements Storage {
 			}
 
 			const stat = await fs.stat(filePath);
-			const nodeStream = createReadStream(filePath);
+			const range = options.range && resolveByteRange(options.range, stat.size);
+			const nodeStream = createReadStream(
+				filePath,
+				range ? { start: range.offset, end: range.offset + range.length - 1 } : undefined,
+			);
 
 			// Convert Node.js stream to web ReadableStream
 			// Readable.toWeb returns ReadableStream (which is ReadableStream<unknown>),
@@ -135,6 +141,8 @@ export class LocalStorage implements Storage {
 				body: webStream,
 				contentType,
 				size: stat.size,
+				lastModified: stat.mtime,
+				...(range && { range }),
 			};
 		} catch (error) {
 			if (error instanceof EmDashStorageError) throw error;

package/src/storage/range.ts

@@ -0,0 +1,54 @@
+/**
+ * Byte ranges for partial downloads (HTTP `Range` requests)
+ */
+
+import type { ByteRange } from "./types.js";
+
+const BYTES_UNIT_PATTERN = /^bytes=/i;
+const RANGE_SPEC_PATTERN = /^(\d*)-(\d*)$/;
+
+/**
+ * Parse a `Range` header that asks for a single byte range.
+ *
+ * Returns null when the header should be ignored and the whole file served:
+ * it's missing or malformed, uses another unit, or asks for several ranges.
+ */
+export function parseRangeHeader(header: string | null): ByteRange | null {
+	if (!header || !BYTES_UNIT_PATTERN.test(header)) return null;
+	const [spec, ...otherSpecs] = header
+		.slice("bytes=".length)
+		.split(",")
+		.map((part) => part.trim())
+		.filter((part) => part !== "");
+	const match =
+		spec !== undefined && otherSpecs.length === 0 ? RANGE_SPEC_PATTERN.exec(spec) : null;
+	if (!match) return null;
+
+	const [, first = "", last = ""] = match;
+	if (first === "") {
+		const suffix = Number(last);
+		return last !== "" && Number.isSafeInteger(suffix) ? { suffix } : null;
+	}
+	const offset = Number(first);
+	if (!Number.isSafeInteger(offset)) return null;
+	if (last === "") return { offset };
+	const end = Number(last);
+	if (!Number.isSafeInteger(end) || end < offset) return null;
+	return { offset, length: end - offset + 1 };
+}
+
+/**
+ * The bytes a range selects from a file of `size` bytes, or null when it
+ * selects none.
+ */
+export function resolveByteRange(
+	range: ByteRange,
+	size: number,
+): { offset: number; length: number } | null {
+	if ("suffix" in range) {
+		const length = Math.min(range.suffix, size);
+		return length > 0 ? { offset: size - length, length } : null;
+	}
+	const length = Math.min(range.length ?? size, size - range.offset);
+	return range.offset >= 0 && length > 0 ? { offset: range.offset, length } : null;
+}

package/src/storage/s3.ts

@@ -22,6 +22,8 @@ import type {
 	Storage,
 	S3StorageConfig,
 	UploadResult,
+	ByteRange,
+	DownloadOptions,
 	DownloadResult,
 	ListResult,
 	ListOptions,
@@ -112,12 +114,28 @@ export function resolveS3Config(partial: Record<string, unknown>): S3StorageConf
 }
 
 const TRAILING_SLASH_PATTERN = /\/$/;
+const CONTENT_RANGE_PATTERN = /^bytes (\d+)-(\d+)\/(\d+)$/;
 
 /** Type guard for AWS SDK errors (have a `name` property) */
 function hasErrorName(error: unknown): error is Error & { name: string } {
 	return error instanceof Error && typeof error.name === "string";
 }
 
+function formatRangeHeader(range: ByteRange): string {
+	if ("suffix" in range) return `bytes=-${range.suffix}`;
+	const end = range.length === undefined ? "" : range.offset + range.length - 1;
+	return `bytes=${range.offset}-${end}`;
+}
+
+function parseContentRange(
+	header: string,
+): { offset: number; length: number; size: number } | undefined {
+	const match = CONTENT_RANGE_PATTERN.exec(header);
+	if (!match) return undefined;
+	const offset = Number(match[1]);
+	return { offset, length: Number(match[2]) - offset + 1, size: Number(match[3]) };
+}
+
 /**
  * S3-compatible storage implementation
  */
@@ -195,14 +213,9 @@ export class S3Storage implements Storage {
 		}
 	}
 
-	async download(key: string): Promise<DownloadResult> {
+	async download(key: string, options: DownloadOptions = {}): Promise<DownloadResult> {
 		try {
-			const response = await this.client.send(
-				new GetObjectCommand({
-					Bucket: this.bucket,
-					Key: key,
-				}),
-			);
+			const { response, served } = await this.getObject(key, options.range);
 
 			if (!response.Body) {
 				throw new EmDashStorageError(`File not found: ${key}`, "NOT_FOUND");
@@ -210,11 +223,23 @@ export class S3Storage implements Storage {
 
 			// Convert SDK stream to web ReadableStream
 			const body = response.Body.transformToWebStream();
+			const contentType = response.ContentType || "application/octet-stream";
+
+			if (served) {
+				return {
+					body,
+					contentType,
+					size: served.size,
+					lastModified: response.LastModified,
+					range: { offset: served.offset, length: served.length },
+				};
+			}
 
 			return {
 				body,
-				contentType: response.ContentType || "application/octet-stream",
+				contentType,
 				size: response.ContentLength || 0,
+				lastModified: response.LastModified,
 			};
 		} catch (error) {
 			if (
@@ -227,6 +252,25 @@ export class S3Storage implements Storage {
 		}
 	}
 
+	/** Get the object, or all of it when the service can't serve `range` */
+	private async getObject(key: string, range: ByteRange | undefined) {
+		const get = (rangeHeader?: string) =>
+			this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key, Range: rangeHeader }));
+		if (range) {
+			try {
+				const response = await get(formatRangeHeader(range));
+				// A service that ignores `Range` sends the whole object without `Content-Range`.
+				if (!response.ContentRange) return { response };
+				const served = parseContentRange(response.ContentRange);
+				if (served) return { response, served };
+				await response.Body?.transformToWebStream().cancel();
+			} catch (error) {
+				if (!hasErrorName(error) || error.name !== "InvalidRange") throw error;
+			}
+		}
+		return { response: await get() };
+	}
+
 	async delete(key: string): Promise<void> {
 		try {
 			await this.client.send(

package/src/storage/types.ts

@@ -73,6 +73,26 @@ export interface UploadResult {
 	size: number;
 }
 
+/**
+ * A single byte range of a file, as requested by an HTTP `Range` header
+ */
+export type ByteRange =
+	/** `length` bytes from `offset`, or every byte from `offset` when `length` is omitted */
+	| { offset: number; length?: number }
+	/** The last `suffix` bytes */
+	| { suffix: number };
+
+/**
+ * Options for downloading a file
+ */
+export interface DownloadOptions {
+	/**
+	 * Download only this part of the file. Adapters that can't serve it,
+	 * including when it starts past the end of the file, return the whole file.
+	 */
+	range?: ByteRange;
+}
+
 /**
  * Download result
  */
@@ -81,8 +101,15 @@ export interface DownloadResult {
 	body: ReadableStream<Uint8Array>;
 	/** MIME type */
 	contentType: string;
-	/** File size in bytes */
+	/** Size of the whole file in bytes, even when `body` holds only part of it */
 	size: number;
+	/**
+	 * The part of the file `body` holds, when the adapter served
+	 * `DownloadOptions.range`. Absent when `body` is the whole file.
+	 */
+	range?: { offset: number; length: number };
+	/** Last modification time of the stored object, when the backend can provide it. */
+	lastModified?: Date;
 }
 
 /**
@@ -167,9 +194,9 @@ export interface Storage {
 	}): Promise<UploadResult>;
 
 	/**
-	 * Download a file from storage
+	 * Download a file from storage, or part of it with `options.range`
 	 */
-	download(key: string): Promise<DownloadResult>;
+	download(key: string, options?: DownloadOptions): Promise<DownloadResult>;
 
 	/**
 	 * Delete a file from storage

package/src/ui.ts

@@ -79,6 +79,7 @@ export {
 	Break,
 	HtmlBlock,
 	Iframe,
+	Video,
 	// Marks
 	Superscript,
 	Subscript,

package/src/utils/slugify.ts

@@ -8,5 +8,10 @@ export { slugify } from "@emdash-cms/admin/slugify";
  * Call this on `Astro.params.slug` before using it in database lookups.
  */
 export function decodeSlug(raw: string | undefined): string | undefined {
-	return raw ? decodeURIComponent(raw) : undefined;
+	if (!raw) return undefined;
+	try {
+		return decodeURIComponent(raw);
+	} catch {
+		return undefined;
+	}
 }