emdash 0.39.0 → 0.40.0

package/src/transfer/fence.ts

@@ -0,0 +1,193 @@
+/**
+ * Unified site write fence: media-usage activation and transfer imports.
+ *
+ * One query answers both questions, so a fenced write path pays a single
+ * round trip. A transfer import fences writes while it executes and, after a
+ * failure or cancellation once writing started, until it is abandoned. The
+ * same query reports whether an export is running, so a fenced write can
+ * bump the export's write epoch without an extra query when none is.
+ */
+
+import { sql, type Kysely } from "kysely";
+
+import { apiError } from "../api/error.js";
+import { findMediaUsageActivationWriteFenceError } from "../api/media-usage-write-fence.js";
+import type { Database } from "../database/types.js";
+import { isMissingTableError } from "../utils/db-errors.js";
+
+export type SiteWriteFenceCode =
+	| "MEDIA_USAGE_ACTIVATION_IN_PROGRESS"
+	| "MEDIA_USAGE_ACTIVATION_CHECK_FAILED"
+	| "TRANSFER_IMPORT_IN_PROGRESS"
+	| "TRANSFER_FENCE_CHECK_FAILED";
+
+export interface SiteWriteFenceError {
+	code: SiteWriteFenceCode;
+	message: string;
+	status: 503;
+	/** The fencing import, when the fence is a transfer import. */
+	operationId?: string;
+}
+
+export class SiteWriteBlockedError extends Error {
+	readonly code: SiteWriteFenceCode;
+	readonly status: 503;
+	readonly operationId: string | undefined;
+
+	constructor(error: SiteWriteFenceError) {
+		super(error.message);
+		this.name = error.code;
+		this.code = error.code;
+		this.status = error.status;
+		this.operationId = error.operationId;
+	}
+}
+
+/** Which fences apply to a write. Both apply unless one is turned off. */
+export interface SiteWriteFenceScope {
+	mediaUsage?: boolean;
+	transfer?: boolean;
+}
+
+export interface SiteWriteFenceStatus {
+	error: SiteWriteFenceError | null;
+	/** Whether an export was running when the fence was read. */
+	exportRunning: boolean;
+}
+
+const CHECK_FAILED: SiteWriteFenceError = {
+	code: "TRANSFER_FENCE_CHECK_FAILED",
+	message: "Unable to verify whether site writes are allowed",
+	status: 503,
+};
+
+/** Read both fences and the running-export flag with one query. */
+export async function readSiteWriteFence(
+	db: Kysely<Database>,
+	scope: SiteWriteFenceScope = {},
+): Promise<SiteWriteFenceStatus> {
+	const mediaUsage = scope.mediaUsage !== false;
+	const transfer = scope.transfer !== false;
+	let row:
+		| { media_state: string | null; import_id: string | null; export_id: string | null }
+		| undefined;
+	try {
+		const result = await sql<{
+			media_state: string | null;
+			import_id: string | null;
+			export_id: string | null;
+		}>`
+			SELECT
+				(
+					SELECT state FROM _emdash_media_usage_activation
+					WHERE task_key = 'incremental_capture'
+				) AS media_state,
+				(
+					SELECT id FROM _emdash_transfer_operations
+					WHERE kind = 'import'
+						AND (
+							state IN ('running', 'verifying')
+							OR (state IN ('failed', 'cancelled') AND mutation_started_at IS NOT NULL)
+						)
+					LIMIT 1
+				) AS import_id,
+				(
+					SELECT id FROM _emdash_transfer_operations
+					WHERE kind = 'export' AND state = 'running'
+					LIMIT 1
+				) AS export_id
+		`.execute(db);
+		row = result.rows[0];
+	} catch (error) {
+		if (isMissingTableError(error)) {
+			return {
+				error: mediaUsage ? await findMediaUsageActivationWriteFenceError(db) : null,
+				exportRunning: false,
+			};
+		}
+		console.error("[transfer] Failed to check the site write fence:", error);
+		return { error: CHECK_FAILED, exportRunning: false };
+	}
+
+	const exportRunning = Boolean(row?.export_id);
+	if (transfer && row?.import_id) {
+		return {
+			error: {
+				code: "TRANSFER_IMPORT_IN_PROGRESS",
+				message: "A site import is in progress or incomplete; writes are disabled",
+				status: 503,
+				operationId: row.import_id,
+			},
+			exportRunning,
+		};
+	}
+	if (mediaUsage && row?.media_state === "activating") {
+		return {
+			error: {
+				code: "MEDIA_USAGE_ACTIVATION_IN_PROGRESS",
+				message: "Media usage activation is in progress",
+				status: 503,
+			},
+			exportRunning,
+		};
+	}
+	return { error: null, exportRunning };
+}
+
+export async function findSiteWriteFenceError(
+	db: Kysely<Database>,
+): Promise<SiteWriteFenceError | null> {
+	return (await readSiteWriteFence(db)).error;
+}
+
+/**
+ * Tell running exports that a fenced write happened, so their consistency
+ * fence restarts them. Failures are logged, not thrown: the write itself is
+ * allowed either way.
+ */
+export async function recordSiteWrite(db: Kysely<Database>): Promise<void> {
+	try {
+		await bumpRunningExportWriteEpochs(db);
+	} catch (error) {
+		console.error("[transfer] Failed to record a write for running exports:", error);
+	}
+}
+
+/** Increments the write epoch of every running export; returns how many there were. */
+export async function bumpRunningExportWriteEpochs(db: Kysely<Database>): Promise<number> {
+	const result = await db
+		.updateTable("_emdash_transfer_operations")
+		.set({ write_epoch: sql<number>`write_epoch + 1` })
+		.where("kind", "=", "export")
+		.where("state", "=", "running")
+		.executeTakeFirst();
+	return Number(result.numUpdatedRows ?? 0n);
+}
+
+/** Records a completed write for running exports; a no-op when none was running. */
+export type RecordSiteWrite = () => Promise<void>;
+
+const NOTHING_TO_RECORD: RecordSiteWrite = async () => {};
+
+/**
+ * Throw {@link SiteWriteBlockedError} when a fence blocks writes. Call it
+ * immediately before the write, and call the returned function once the
+ * write has succeeded: recording it afterwards means an export whose fence
+ * was captured while the write ran still sees it.
+ */
+export async function assertSiteWriteAllowed(db: Kysely<Database>): Promise<RecordSiteWrite> {
+	const status = await readSiteWriteFence(db);
+	if (status.error) throw new SiteWriteBlockedError(status.error);
+	return status.exportRunning ? () => recordSiteWrite(db) : NOTHING_TO_RECORD;
+}
+
+/**
+ * The fence as an API error response, or null when writes are allowed.
+ * Records nothing: for the plugin lifecycle routes that call it, the fence
+ * middleware records a successful response's write.
+ */
+export async function checkSiteWriteFence(db: Kysely<Database>): Promise<Response | null> {
+	const status = await readSiteWriteFence(db);
+	if (status.error) return apiError(status.error.code, status.error.message, status.error.status);
+	return null;
+}

package/src/transfer/format/canonical.ts

@@ -0,0 +1,151 @@
+/**
+ * Canonical JSON serialization for site packages.
+ *
+ * Rules (normative for formatVersion "1"):
+ * - object keys are sorted by UTF-16 code unit order ({@link compareUtf16});
+ * - no insignificant whitespace;
+ * - strings and numbers are encoded exactly as `JSON.stringify` encodes them;
+ * - object properties whose value is `undefined` are omitted;
+ * - `undefined` inside an array, non-finite numbers, and any value that is not
+ *   a plain object, array, string, finite number, boolean, or `null` are
+ *   rejected;
+ * - nesting depth is at most {@link TRANSFER_LIMITS.jsonDepth} containers
+ *   (the root object or array is depth 1).
+ *
+ * The manifest, every NDJSON line, the plan, the receipt, and logical digests
+ * all use this one implementation.
+ */
+
+import { TRANSFER_LIMITS } from "./limits.js";
+
+export class CanonicalJsonError extends Error {
+	readonly path: string;
+
+	constructor(message: string, path: string) {
+		super(`${message} at ${path}`);
+		this.name = "CanonicalJsonError";
+		this.path = path;
+	}
+}
+
+export type JsonPrimitive = string | number | boolean | null;
+export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue };
+
+export function compareUtf16(a: string, b: string): number {
+	if (a === b) return 0;
+	return a < b ? -1 : 1;
+}
+
+export function canonicalJson(value: unknown, options: { maxDepth?: number } = {}): string {
+	const maxDepth = options.maxDepth ?? TRANSFER_LIMITS.jsonDepth;
+	const parts: string[] = [];
+	write(value, parts, 0, maxDepth, "$");
+	return parts.join("");
+}
+
+function write(value: unknown, out: string[], depth: number, maxDepth: number, path: string): void {
+	if (value === null) {
+		out.push("null");
+		return;
+	}
+	switch (typeof value) {
+		case "string":
+			out.push(JSON.stringify(value));
+			return;
+		case "number":
+			if (!Number.isFinite(value)) throw new CanonicalJsonError("Non-finite number", path);
+			out.push(JSON.stringify(value));
+			return;
+		case "boolean":
+			out.push(value ? "true" : "false");
+			return;
+		case "object":
+			break;
+		default:
+			throw new CanonicalJsonError(`Unsupported ${typeof value} value`, path);
+	}
+
+	if (depth + 1 > maxDepth) {
+		throw new CanonicalJsonError(`Nesting deeper than ${maxDepth}`, path);
+	}
+
+	if (Array.isArray(value)) {
+		out.push("[");
+		for (let index = 0; index < value.length; index++) {
+			if (index > 0) out.push(",");
+			const item: unknown = value[index];
+			if (item === undefined)
+				throw new CanonicalJsonError("Undefined array item", `${path}[${index}]`);
+			write(item, out, depth + 1, maxDepth, `${path}[${index}]`);
+		}
+		out.push("]");
+		return;
+	}
+
+	const prototype: unknown = Object.getPrototypeOf(value);
+	if (prototype !== Object.prototype && prototype !== null) {
+		throw new CanonicalJsonError("Non-plain object", path);
+	}
+	const entries: Array<[string, unknown]> = Object.entries(value);
+	const present = entries
+		.filter(([, item]) => item !== undefined)
+		.toSorted(([a], [b]) => compareUtf16(a, b));
+	out.push("{");
+	present.forEach(([key, item], index) => {
+		if (index > 0) out.push(",");
+		out.push(JSON.stringify(key), ":");
+		write(item, out, depth + 1, maxDepth, `${path}.${key}`);
+	});
+	out.push("}");
+}
+
+/**
+ * Container nesting depth of a JSON text, measured without parsing it so an
+ * adversarial document cannot exhaust the parser's stack first. Returns
+ * `Infinity` as soon as `limit` is exceeded.
+ */
+export function measureJsonDepth(text: string, limit: number = TRANSFER_LIMITS.jsonDepth): number {
+	let depth = 0;
+	let max = 0;
+	let inString = false;
+	for (let index = 0; index < text.length; index++) {
+		const code = text.charCodeAt(index);
+		if (inString) {
+			if (code === 0x5c) index++;
+			else if (code === 0x22) inString = false;
+			continue;
+		}
+		if (code === 0x22) inString = true;
+		else if (code === 0x7b || code === 0x5b) {
+			depth++;
+			if (depth > max) {
+				max = depth;
+				if (max > limit) return Number.POSITIVE_INFINITY;
+			}
+		} else if (code === 0x7d || code === 0x5d) depth--;
+	}
+	return max;
+}
+
+/**
+ * Parse a JSON text that must already be in canonical form. Throws
+ * {@link CanonicalJsonError} when the depth limit is exceeded, the text is not
+ * valid JSON, or re-serializing the parsed value does not reproduce the text
+ * byte for byte.
+ */
+export function parseCanonicalJson(text: string, options: { maxDepth?: number } = {}): unknown {
+	const maxDepth = options.maxDepth ?? TRANSFER_LIMITS.jsonDepth;
+	if (measureJsonDepth(text, maxDepth) > maxDepth) {
+		throw new CanonicalJsonError(`Nesting deeper than ${maxDepth}`, "$");
+	}
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(text);
+	} catch {
+		throw new CanonicalJsonError("Invalid JSON", "$");
+	}
+	if (canonicalJson(parsed, { maxDepth }) !== text) {
+		throw new CanonicalJsonError("JSON is not in canonical form", "$");
+	}
+	return parsed;
+}

package/src/transfer/format/column-codec.ts

@@ -0,0 +1,275 @@
+/**
+ * The single decoding and encoding path between database column values and
+ * record property values. Exporter readers, verification, and importer
+ * writers must all go through these functions so a value read on one dialect
+ * and written on another round-trips to the same record value.
+ *
+ * Rules:
+ * - SQL `NULL` decodes to `undefined` (the property is omitted) and
+ *   `undefined`/`null` encode to `NULL`.
+ * - `boolean`: 0/1 integers ↔ booleans (any non-zero number decodes to true).
+ * - `integer`: numbers, and Postgres `bigint` strings, decode to numbers when
+ *   they are safe integers. Any other value (non-numeric text a SQLite column
+ *   may hold) decodes to itself.
+ * - `real`: numbers. On Postgres, `real` is float4 and node-pg renders it
+ *   with float4 precision (`0.1`), which differs from the float4 value itself
+ *   (`Math.fround(0.1)`). Select REAL columns with {@link selectColumn}, which
+ *   casts to float8 on Postgres, so the decoded number is exactly the stored
+ *   float4 value.
+ * - `json` (JSON held in a TEXT column): the parsed value; text that is not
+ *   valid JSON decodes to the text itself. Encoding always writes
+ *   `JSON.stringify`, so raw text that is not JSON is stored as a JSON
+ *   string: EmDash parses these columns as JSON when it reads them.
+ * - `nativeJson` (a column declared `json`: `ec_*` JSON fields): on
+ *   Postgres, node-pg returns the parsed value; on SQLite the column has
+ *   NUMERIC affinity, so numbers come back as numbers and everything else as
+ *   text decoded like `json`. Encoding on Postgres is always
+ *   `JSON.stringify`; on SQLite numbers are written as numbers and the rest
+ *   like `json`.
+ * - `text`: strings verbatim.
+ * - A JSON `null` decodes like SQL `NULL` (absent): node-pg cannot tell them
+ *   apart in a `json` column, so neither dialect does. A NOT NULL JSON
+ *   column can only hold JSON `null` for an absent value, so encoding for
+ *   one (`notNull`) writes JSON `null`.
+ *
+ * Invariant: for every value `v` produced by `decodeColumn`,
+ * `decodeColumn(encodeColumn(v))` on either dialect yields `v`.
+ */
+
+import { sql, type AliasedRawBuilder, type Kysely } from "kysely";
+
+import { detectDialect, type DatabaseDialectType } from "../../database/dialect-helpers.js";
+import { TransferError } from "../errors.js";
+import type { ColumnCodec, ColumnSpec } from "./columns.js";
+import { CONTENT_TABLE_COLUMNS } from "./columns.js";
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any Kysely instance
+type DialectSource = DatabaseDialectType | Kysely<any>;
+
+const SAFE_INTEGER_TEXT = /^-?\d+$/;
+
+function dialectOf(source: DialectSource): DatabaseDialectType {
+	return typeof source === "string" ? source : detectDialect(source);
+}
+
+function tryParseJson(text: string): { ok: true; value: unknown } | { ok: false } {
+	try {
+		return { ok: true, value: JSON.parse(text) };
+	} catch {
+		return { ok: false };
+	}
+}
+
+function invalid(codec: ColumnCodec, reason: string): TransferError {
+	return new TransferError("TRANSFER_RECORD_INVALID", "Value cannot be stored in its column", {
+		detail: { codec, reason },
+	});
+}
+
+/** Decode one raw column value. Returns `undefined` for SQL `NULL`. */
+export function decodeColumn(source: DialectSource, codec: ColumnCodec, raw: unknown): unknown {
+	if (raw === null || raw === undefined) return undefined;
+	const dialect = dialectOf(source);
+	switch (codec) {
+		case "text":
+			return raw;
+		case "boolean":
+			if (typeof raw === "number") return raw !== 0;
+			if (typeof raw === "bigint") return raw !== 0n;
+			if (typeof raw === "string" && SAFE_INTEGER_TEXT.test(raw)) return Number(raw) !== 0;
+			if (typeof raw === "boolean") return raw;
+			return raw;
+		case "integer":
+			if (typeof raw === "bigint")
+				return Number.isSafeInteger(Number(raw)) ? Number(raw) : String(raw);
+			if (typeof raw === "string" && SAFE_INTEGER_TEXT.test(raw)) {
+				const value = Number(raw);
+				return Number.isSafeInteger(value) ? value : raw;
+			}
+			return raw;
+		case "real":
+			if (typeof raw === "string" && dialect === "postgres") return Number(raw);
+			return raw;
+		case "json":
+			if (typeof raw !== "string") return raw;
+			return decodeJsonText(raw);
+		case "nativeJson":
+			if (dialect === "postgres") return raw;
+			if (typeof raw !== "string") return raw;
+			return decodeJsonText(raw);
+	}
+}
+
+function decodeJsonText(text: string): unknown {
+	const parsed = tryParseJson(text);
+	if (!parsed.ok) return text;
+	return parsed.value === null ? undefined : parsed.value;
+}
+
+/**
+ * Encode one record value for a column. Returns `null` for an absent value,
+ * or JSON `null` when `notNull` is set for a JSON column.
+ */
+export function encodeColumn(
+	source: DialectSource,
+	codec: ColumnCodec,
+	value: unknown,
+	options: { notNull?: boolean } = {},
+): unknown {
+	if (value === undefined || value === null) {
+		return options.notNull && (codec === "json" || codec === "nativeJson") ? "null" : null;
+	}
+	const dialect = dialectOf(source);
+	switch (codec) {
+		case "text":
+			if (typeof value !== "string") throw invalid(codec, "expected a string");
+			return value;
+		case "boolean":
+			if (typeof value !== "boolean") throw invalid(codec, "expected a boolean");
+			return value ? 1 : 0;
+		case "integer":
+			if (typeof value === "number") {
+				if (!Number.isSafeInteger(value)) throw invalid(codec, "expected an integer");
+				return value;
+			}
+			if (typeof value === "string" && dialect === "sqlite") return value;
+			throw invalid(codec, "expected an integer");
+		case "real":
+			if (typeof value === "number") {
+				if (!Number.isFinite(value)) throw invalid(codec, "expected a finite number");
+				return value;
+			}
+			if (typeof value === "string" && dialect === "sqlite") return value;
+			throw invalid(codec, "expected a number");
+		case "json":
+			return encodeJsonText(value);
+		case "nativeJson":
+			if (dialect === "postgres") return JSON.stringify(value);
+			if (typeof value === "number") return value;
+			return encodeJsonText(value);
+	}
+}
+
+function encodeJsonText(value: unknown): string {
+	const text = JSON.stringify(value);
+	if (text === undefined) throw invalid("json", "value is not JSON");
+	return text;
+}
+
+/**
+ * Select expression for a column, aliased to the column name. REAL columns
+ * are cast to float8 on Postgres (see the module comment).
+ */
+export function selectColumn(
+	source: DialectSource,
+	column: string,
+	codec: ColumnCodec,
+): AliasedRawBuilder<unknown, string> {
+	if (codec === "real" && dialectOf(source) === "postgres") {
+		return sql`${sql.ref(column)}::float8`.as(column);
+	}
+	return sql`${sql.ref(column)}`.as(column);
+}
+
+function propertyOf(spec: ColumnSpec): { property: string; codec: ColumnCodec } | null {
+	return "property" in spec ? { property: spec.property, codec: spec.codec } : null;
+}
+
+/**
+ * Select expressions for every exported column of a table (columns whose
+ * class has a property), in `specs` order.
+ */
+export function selectExportedColumns(
+	source: DialectSource,
+	specs: Readonly<Record<string, ColumnSpec>>,
+): AliasedRawBuilder<unknown, string>[] {
+	const expressions: AliasedRawBuilder<unknown, string>[] = [];
+	for (const [column, spec] of Object.entries(specs)) {
+		const mapped = propertyOf(spec);
+		if (mapped) expressions.push(selectColumn(source, column, mapped.codec));
+	}
+	return expressions;
+}
+
+/** Decode a row into record properties; NULL columns are omitted. */
+export function decodeRow(
+	source: DialectSource,
+	specs: Readonly<Record<string, ColumnSpec>>,
+	row: Readonly<Record<string, unknown>>,
+): Record<string, unknown> {
+	const dialect = dialectOf(source);
+	const properties: Record<string, unknown> = {};
+	for (const [column, spec] of Object.entries(specs)) {
+		const mapped = propertyOf(spec);
+		if (!mapped || !Object.hasOwn(row, column)) continue;
+		const value = decodeColumn(dialect, mapped.codec, row[column]);
+		if (value !== undefined) properties[mapped.property] = value;
+	}
+	return properties;
+}
+
+/**
+ * Encode record properties into column values for every exported column of
+ * `specs`. Absent properties become `NULL`. Columns of other classes are not
+ * included; writers fill those themselves.
+ */
+export function encodeRow(
+	source: DialectSource,
+	specs: Readonly<Record<string, ColumnSpec>>,
+	properties: Readonly<Record<string, unknown>>,
+): Record<string, unknown> {
+	const dialect = dialectOf(source);
+	const row: Record<string, unknown> = {};
+	for (const [column, spec] of Object.entries(specs)) {
+		const mapped = propertyOf(spec);
+		if (!mapped) continue;
+		row[column] = encodeColumn(
+			dialect,
+			mapped.codec,
+			Object.hasOwn(properties, mapped.property) ? properties[mapped.property] : undefined,
+		);
+	}
+	return row;
+}
+
+/** Whether an `ec_*` column is a field column rather than a standard column. */
+export function isContentFieldColumn(column: string): boolean {
+	return !Object.hasOwn(CONTENT_TABLE_COLUMNS, column);
+}
+
+/**
+ * Decode an `ec_*` row into the standard entry properties and the `fields`
+ * map (field slug → value). `specs` comes from `getColumnSpecs`.
+ */
+export function decodeContentRow(
+	source: DialectSource,
+	specs: Readonly<Record<string, ColumnSpec>>,
+	row: Readonly<Record<string, unknown>>,
+): { properties: Record<string, unknown>; fields: Record<string, unknown> } {
+	const standard: Record<string, ColumnSpec> = {};
+	const fieldSpecs: Record<string, ColumnSpec> = {};
+	for (const [column, spec] of Object.entries(specs)) {
+		if (isContentFieldColumn(column)) fieldSpecs[column] = spec;
+		else standard[column] = spec;
+	}
+	return {
+		properties: decodeRow(source, standard, row),
+		fields: decodeRow(source, fieldSpecs, row),
+	};
+}
+
+/** Inverse of {@link decodeContentRow}. */
+export function encodeContentRow(
+	source: DialectSource,
+	specs: Readonly<Record<string, ColumnSpec>>,
+	properties: Readonly<Record<string, unknown>>,
+	fields: Readonly<Record<string, unknown>>,
+): Record<string, unknown> {
+	const standard: Record<string, ColumnSpec> = {};
+	const fieldSpecs: Record<string, ColumnSpec> = {};
+	for (const [column, spec] of Object.entries(specs)) {
+		if (isContentFieldColumn(column)) fieldSpecs[column] = spec;
+		else standard[column] = spec;
+	}
+	return { ...encodeRow(source, standard, properties), ...encodeRow(source, fieldSpecs, fields) };
+}