emdash 0.11.1 → 0.13.0

package/src/registry/plugin-id.ts

@@ -0,0 +1,116 @@
+/**
+ * Plugin identifier helpers for the experimental decentralized plugin
+ * registry.
+ *
+ * Registry plugins are addressed by `(publisher_did, slug)`, but the
+ * EmDash runtime threads a single `pluginId: string` through every
+ * install primitive (R2 storage keys, `PluginStateRepository`,
+ * `syncMarketplacePlugins`, sandbox cache keys). Rather than refactor
+ * everything to carry a composite identifier, we normalize the registry
+ * tuple to an opaque content-addressed id that satisfies the existing
+ * `validatePluginIdentifier` shape (`/^[a-z][a-z0-9_-]*$/`).
+ *
+ * The normalized id is:
+ *
+ *   `r_` + base32-encoded SHA-256(publisher_did + "\n" + slug), truncated.
+ *
+ * Properties:
+ *
+ *   - Deterministic. The same `(publisher, slug)` always produces the
+ *     same id, so re-resolving an installed plugin's metadata against
+ *     the aggregator is a straightforward lookup keyed by the columns
+ *     stored alongside `plugin_id` in `plugin_states`.
+ *   - Collision-resistant. 80 bits of truncated hash; a 50% birthday
+ *     collision happens around 2^40 distinct plugins, well beyond what
+ *     this registry will ever index.
+ *   - R2-safe. Lowercase alphanumerics + underscore (no hyphens), no
+ *     `:` or `/`. Existing sandbox cache keys (`${pluginId}:${version}`)
+ *     keep working because the id contains no `:`.
+ *   - Syntactically distinct from typical marketplace plugin ids: the
+ *     `r_` prefix plus exactly 16 base32 characters is unlikely to be
+ *     chosen as a marketplace id. Not formally guaranteed by the
+ *     validator -- marketplace ids may begin with `r_` and contain
+ *     hyphens -- so the install handler also performs an explicit
+ *     pre-existing-row check at the derived id and rejects any cross-
+ *     source collision (`PLUGIN_ID_COLLISION`).
+ *
+ * Reverse lookup (id → publisher + slug) requires the `plugin_states`
+ * row -- the hash is one-way. That's intentional: any code path that
+ * needs the human-meaningful pair already has the state row in hand.
+ */
+
+/** Length (in base32 characters) of the truncated hash portion of the id. */
+const HASH_LENGTH = 16;
+
+/** Total expected length of a registry plugin id. */
+export const REGISTRY_PLUGIN_ID_LENGTH = 2 /* "r_" */ + HASH_LENGTH;
+
+/**
+ * Regex matching a well-formed registry plugin id. Used by call sites
+ * that need to distinguish registry installs from marketplace installs
+ * without consulting the `source` column on `plugin_states`.
+ *
+ * The base32 alphabet here uses RFC 4648 lowercase without padding,
+ * matching {@link base32Encode}'s output.
+ */
+export const REGISTRY_PLUGIN_ID_PATTERN = /^r_[a-z2-7]{16}$/;
+
+const BASE32_ALPHABET = "abcdefghijklmnopqrstuvwxyz234567";
+
+/**
+ * RFC 4648 base32 encoding without padding, lowercase. Implemented inline
+ * rather than depending on a multibase library because (a) we only need
+ * lowercase base32 here, (b) we need it to run identically in workerd,
+ * Node, and the browser, and (c) the implementation is fewer lines than
+ * the import statement would be.
+ */
+function base32Encode(bytes: Uint8Array): string {
+	let bits = 0;
+	let value = 0;
+	let out = "";
+	for (const byte of bytes) {
+		value = (value << 8) | byte;
+		bits += 8;
+		while (bits >= 5) {
+			bits -= 5;
+			out += BASE32_ALPHABET[(value >>> bits) & 0x1f];
+		}
+	}
+	if (bits > 0) {
+		out += BASE32_ALPHABET[(value << (5 - bits)) & 0x1f];
+	}
+	return out;
+}
+
+/**
+ * Derive the normalized plugin id for a registry-published plugin.
+ *
+ * Throws if either input is empty or whitespace-only -- a missing DID
+ * or slug is always a programming error in the install path, not a
+ * recoverable runtime condition.
+ */
+export async function makeRegistryPluginId(publisherDid: string, slug: string): Promise<string> {
+	const did = publisherDid.trim();
+	const s = slug.trim();
+	if (!did) throw new Error("makeRegistryPluginId: publisherDid is required");
+	if (!s) throw new Error("makeRegistryPluginId: slug is required");
+
+	// `\n` separator avoids ambiguity: no canonical did:plc / did:web form
+	// contains a literal newline, so `("a", "b\nc")` cannot hash to the
+	// same bytes as `("a\nb", "c")`.
+	const input = `${did}\n${s}`;
+	const hashBuffer = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(input));
+	const encoded = base32Encode(new Uint8Array(hashBuffer));
+	return `r_${encoded.slice(0, HASH_LENGTH)}`;
+}
+
+/**
+ * Return whether `pluginId` is a well-formed registry plugin id.
+ *
+ * This is a syntactic check, not a database lookup -- it answers
+ * "could this id have come from `makeRegistryPluginId`?", not "is this
+ * plugin installed?".
+ */
+export function isRegistryPluginId(pluginId: string): boolean {
+	return REGISTRY_PLUGIN_ID_PATTERN.test(pluginId);
+}

package/src/registry/types.ts

@@ -0,0 +1,206 @@
+/**
+ * Public types for the experimental plugin registry.
+ *
+ * Kept in their own module so they don't get re-bundled into the
+ * `astro/integration/runtime.ts` chunk's dist output. tsdown / rolldown
+ * are sensitive to which top-level types live alongside `definePlugin`'s
+ * overloads, and pulling these types into the integration module
+ * affected downstream `definePlugin()` overload resolution for trusted
+ * plugins built against core's dist (see commit history for the
+ * detailed write-up).
+ */
+
+/**
+ * Experimental plugin registry configuration.
+ *
+ * See {@link ExperimentalConfig.registry}.
+ */
+export interface RegistryConfig {
+	/**
+	 * Base URL of the registry aggregator (an atproto AppView that indexes
+	 * the firehose for `pm.fair.package.*` and `com.emdashcms.*` records).
+	 *
+	 * Must be the origin where the aggregator's XRPC endpoints are mounted,
+	 * such that `${aggregatorUrl}/xrpc/<nsid>` resolves to a valid endpoint.
+	 *
+	 * Must be HTTPS in production; `http://localhost` or `http://127.0.0.1`
+	 * are accepted in dev.
+	 */
+	aggregatorUrl: string;
+
+	/**
+	 * Optional comma-separated list of labeller DIDs forwarded as the
+	 * `atproto-accept-labelers` header on every aggregator request.
+	 *
+	 * Format follows the atproto convention:
+	 * `did:plc:abc;redact, did:plc:def`
+	 *
+	 * When unset, the aggregator applies its operator-default labeller set
+	 * (typically the EmDash publisher-verification labeller and any
+	 * additional trusted labellers the aggregator operator configured).
+	 */
+	acceptLabelers?: string;
+
+	/**
+	 * Site-level policy applied to the latest-release selection filter.
+	 *
+	 * These filters operate over the signed records the aggregator returns;
+	 * they are not protocol-level constraints. See the RFC's
+	 * "Update Discovery and Takedowns" section for the integration point.
+	 */
+	policy?: {
+		/**
+		 * Hold back releases newer than this when computing the recommended
+		 * install or update version. Mitigates "compromised publisher
+		 * account pushes a malicious release of an established plugin" by
+		 * giving the takedown labeller a detection window.
+		 *
+		 * Accepts a duration string (`"24h"`, `"48h"`, `"72h"`, `"7d"`) or a
+		 * number of seconds.
+		 *
+		 * Currently applies uniformly to all releases. A future addition
+		 * may exempt brand-new packages (those with no prior release
+		 * history) so the holdback doesn't block first-time publishing,
+		 * but that exemption is not implemented yet; use
+		 * {@link minimumReleaseAgeExclude} to allowlist trusted publishers
+		 * whose packages should install immediately.
+		 *
+		 * Defaults to `undefined` (no holdback). A future trust/moderation
+		 * RFC will specify the recommended default.
+		 */
+		minimumReleaseAge?: string | number;
+
+		/**
+		 * Packages exempt from the {@link minimumReleaseAge} holdback. Use
+		 * for publishers whose release tempo you've explicitly accepted --
+		 * your own first-party plugins, a trusted partner, etc.
+		 *
+		 * Each entry is either:
+		 *   - A bare publisher DID (e.g. `"did:plc:abc123"`) -- every
+		 *     package from that publisher is exempt.
+		 *   - A `<did>/<slug>` pair (e.g.
+		 *     `"did:plc:abc123/hotfix-plugin"`) -- only that specific
+		 *     package is exempt.
+		 *
+		 * Whole-publisher exemptions are the common case: trust is
+		 * naturally a property of the publisher, not of each individual
+		 * package. Per-package exemptions exist for cases where a publisher
+		 * has one plugin you want fast-track installs for and others you'd
+		 * rather hold back.
+		 *
+		 * Only DIDs are accepted -- not handles. Handles are mutable
+		 * aggregator-supplied envelope data, and accepting them as a
+		 * trust input would let a compromised aggregator bypass the
+		 * holdback by claiming any handle for any package. DIDs are
+		 * tied to the AT URI of the package record itself, so even a
+		 * compromised aggregator cannot lie about which DID published
+		 * a release.
+		 *
+		 * Mirrors pnpm's `minimumReleaseAgeExclude`.
+		 *
+		 * @example
+		 * ```ts
+		 * minimumReleaseAgeExclude: [
+		 *   "did:plc:emdashfirstparty",     // every package from this publisher
+		 *   "did:plc:abc123/hotfix-plugin", // just this one package
+		 * ]
+		 * ```
+		 */
+		minimumReleaseAgeExclude?: readonly string[];
+	};
+}
+
+/**
+ * Shorthand: pass a bare aggregator URL string in place of a full
+ * `RegistryConfig` object when you don't need `acceptLabelers` or
+ * `policy`. The normalizer expands the string into
+ * `{ aggregatorUrl: <string> }` before any downstream code sees it.
+ *
+ * @example
+ * ```ts
+ * experimental: {
+ *   registry: "https://registry.emdashcms.com",
+ * }
+ * ```
+ *
+ * Equivalent to:
+ * ```ts
+ * experimental: {
+ *   registry: { aggregatorUrl: "https://registry.emdashcms.com" },
+ * }
+ * ```
+ */
+export type RegistryConfigInput = string | RegistryConfig;
+
+/**
+ * Experimental EmDash features. See `EmDashConfig.experimental`.
+ *
+ * Each field is independently opt-in. Fields may be promoted out of
+ * `experimental` (becoming top-level `EmDashConfig` options) or removed
+ * in minor releases; check the changelog when upgrading.
+ */
+export interface ExperimentalConfig {
+	/**
+	 * Decentralized plugin registry.
+	 *
+	 * When set, replaces the centralized `marketplace` for the admin UI's
+	 * browse and install flows. The registry is an atproto-backed
+	 * federation: package metadata lives in each publisher's PDS and
+	 * an aggregator (the `aggregatorUrl`) indexes the firehose and
+	 * exposes read-only XRPC endpoints for discovery.
+	 *
+	 * See [RFC 0001](https://github.com/emdash-cms/emdash/pull/694) for
+	 * the protocol design.
+	 *
+	 * **Trust model (v1, experimental).** Today EmDash trusts the
+	 * configured aggregator with these claims, per package and per
+	 * release:
+	 *
+	 *   - The publisher DID associated with a `(did, slug)` pair.
+	 *   - The artifact `url`, the artifact `checksum`, and any mirror
+	 *     URLs returned for a release.
+	 *   - The published handle for a DID (used for display only;
+	 *     EmDash separately verifies the DID->handle round-trip in the
+	 *     admin UI before treating a handle as confirmed).
+	 *
+	 * What EmDash verifies independently before activating an
+	 * installed plugin:
+	 *
+	 *   - The artifact bytes hash to the checksum the aggregator
+	 *     returned (so a malicious mirror or in-transit tamper can't
+	 *     swap the bundle).
+	 *   - The bundle's `manifest.id` matches the requested slug, and
+	 *     its `manifest.version` matches the release version (so an
+	 *     attacker who controls the aggregator can't trick the
+	 *     sandbox into addressing the wrong plugin id).
+	 *   - The bundle's `manifest.capabilities` matches what the admin
+	 *     acknowledged in the consent dialog (so a publisher can't
+	 *     ship a bundle that requests more permissions than the
+	 *     dialog displayed).
+	 *
+	 * What's NOT yet verified:
+	 *
+	 *   - Full MST proof / publisher signature on the release record.
+	 *     A compromised aggregator can forge a release for any DID
+	 *     and slug, and the install will succeed as long as the
+	 *     bundle matches the (forged) checksum.
+	 *   - Per-release replay / rollback: the aggregator chooses which
+	 *     release version is "latest".
+	 *
+	 * **Recommendation.** Until full signature verification lands,
+	 * point `aggregatorUrl` only at an aggregator you operate
+	 * yourself or one you trust with the same level of authority as
+	 * a centralized plugin source. The `policy.minimumReleaseAge` and
+	 * `acceptLabelers` knobs partially mitigate by widening the
+	 * detection window for takedowns, but they assume the labeller
+	 * system is operating.
+	 *
+	 * Requires `sandboxRunner` to be configured -- registry plugins
+	 * always run sandboxed.
+	 *
+	 * Accepts a bare URL string as shorthand for
+	 * `{ aggregatorUrl: "..." }`. Use the full object form when you
+	 * need `acceptLabelers` or `policy`.
+	 */
+	registry?: RegistryConfigInput;
+}

package/src/search/fts-manager.ts

@@ -81,8 +81,13 @@ export class FTSManager {
 		// id and locale are UNINDEXED (used for joining/filtering, not searched)
 		const columns = ["id UNINDEXED", "locale UNINDEXED", ...searchableFields].join(", ");
 
-		// Create the FTS5 virtual table
-		// Using content= to make it a contentless FTS table (we manage sync ourselves)
+		// Create the FTS5 virtual table.
+		// `content='<table>'` makes this an *external content* FTS5 table:
+		// the inverted index lives in the FTS shadow tables, but the actual
+		// row data lives in the backing content table. The triggers in
+		// `createTriggers` keep the index in sync; they MUST use the
+		// external-content-safe `'delete'` command (see notes there) to
+		// avoid `SQLITE_CORRUPT_VTAB` on UPDATE/DELETE.
 		// tokenize='porter unicode61' enables stemming (run matches running, ran, etc.)
 		await sql
 			.raw(`
@@ -106,13 +111,51 @@ export class FTSManager {
 	 * `deleted_at IS NULL`. This keeps soft-deleted content out of the
 	 * search index and ensures the FTS row count matches the non-deleted
 	 * content count (which `verifyAndRepairIndex` relies on).
+	 *
+	 * IMPORTANT: The FTS5 virtual table is created with `content='ec_<slug>'`
+	 * which makes it an *external content* FTS5 table. For external-content
+	 * tables, removing a row must use the documented `'delete'` command and
+	 * supply the OLD column values explicitly, e.g.:
+	 *
+	 *     INSERT INTO fts(fts, rowid, col1, col2)
+	 *     VALUES('delete', OLD.rowid, OLD.col1, OLD.col2);
+	 *
+	 * Using `DELETE FROM fts WHERE rowid = OLD.rowid` is the correct form
+	 * for *contentless* tables but is unsafe for external-content tables:
+	 * FTS5 then reads column values from the backing content table, which
+	 * in an AFTER UPDATE trigger already holds the NEW values. The wrong
+	 * tokens get removed and the inverted index drifts out of sync until
+	 * SQLite raises `SQLITE_CORRUPT_VTAB` on the next mutation. See
+	 * https://www.sqlite.org/fts5.html#external_content_tables.
+	 *
+	 * The UPDATE and DELETE triggers gate the `'delete'` on
+	 * `OLD.deleted_at IS NULL` because the INSERT trigger never indexed
+	 * rows that were already soft-deleted. Issuing `'delete'` for a rowid
+	 * that was never inserted into the FTS index is itself a corruption
+	 * trigger -- FTS5's `'delete'` is not a no-op on missing rowids and
+	 * raises `SQLITE_CORRUPT_VTAB`. Affected paths include restore-from-
+	 * trash (UPDATE where `OLD.deleted_at IS NOT NULL`), permanent-delete
+	 * from trash (DELETE on a soft-deleted row), and any edit on a row
+	 * that's currently in the trash.
 	 */
 	private async createTriggers(collectionSlug: string, searchableFields: string[]): Promise<void> {
 		this.validateInputs(collectionSlug, searchableFields);
+		if (searchableFields.length === 0) {
+			throw new Error(
+				`Cannot create FTS triggers for collection "${collectionSlug}": no searchable fields. ` +
+					`Mark at least one field as searchable before enabling search.`,
+			);
+		}
 		const ftsTable = this.getFtsTableName(collectionSlug);
 		const contentTable = this.getContentTableName(collectionSlug);
 		const fieldList = searchableFields.join(", ");
 		const newFieldList = searchableFields.map((f) => `NEW.${f}`).join(", ");
+		// `'delete'` takes the FTS5 virtual table name as the first column,
+		// then the rowid being removed, then the OLD value of every column
+		// declared on the FTS5 table (in declaration order: id, locale,
+		// then each searchable field).
+		const oldFieldList = searchableFields.map((f) => `OLD.${f}`).join(", ");
+
 		// Insert trigger - only index non-deleted content
 		await sql
 			.raw(`
@@ -126,15 +169,24 @@ export class FTSManager {
 		`)
 			.execute(this.db);
 
-		// Update trigger - always remove the old FTS row, only re-insert
-		// if the row is not soft-deleted. This handles both content edits
-		// and soft-delete operations (UPDATE SET deleted_at = ...).
+		// Update trigger - remove the old row from the FTS index using the
+		// external-content-safe `'delete'` command (which uses OLD column
+		// values, captured before the row was modified), then re-insert
+		// the new values when the row is still visible.
+		//
+		// `'delete'` is gated on `OLD.deleted_at IS NULL` because rows that
+		// were soft-deleted are not in the FTS index (the INSERT trigger
+		// skips them). Issuing `'delete'` for a missing rowid raises
+		// `SQLITE_CORRUPT_VTAB`, which would break restore-from-trash and
+		// edits to soft-deleted rows.
 		await sql
 			.raw(`
 			CREATE TRIGGER IF NOT EXISTS "${ftsTable}_update" 
 			AFTER UPDATE ON "${contentTable}" 
 			BEGIN
-				DELETE FROM "${ftsTable}" WHERE rowid = OLD.rowid;
+				INSERT INTO "${ftsTable}"("${ftsTable}", rowid, id, locale, ${fieldList})
+				SELECT 'delete', OLD.rowid, OLD.id, OLD.locale, ${oldFieldList}
+				WHERE OLD.deleted_at IS NULL;
 				INSERT INTO "${ftsTable}"(rowid, id, locale, ${fieldList})
 				SELECT NEW.rowid, NEW.id, NEW.locale, ${newFieldList}
 				WHERE NEW.deleted_at IS NULL;
@@ -142,13 +194,18 @@ export class FTSManager {
 		`)
 			.execute(this.db);
 
-		// Delete trigger
+		// Delete trigger - same external-content-safe `'delete'` form,
+		// gated on `OLD.deleted_at IS NULL` for the same reason as the
+		// UPDATE trigger: permanent-delete from trash hits a row whose
+		// `deleted_at` is already set and which was never indexed.
 		await sql
 			.raw(`
 			CREATE TRIGGER IF NOT EXISTS "${ftsTable}_delete" 
 			AFTER DELETE ON "${contentTable}" 
 			BEGIN
-				DELETE FROM "${ftsTable}" WHERE rowid = OLD.rowid;
+				INSERT INTO "${ftsTable}"("${ftsTable}", rowid, id, locale, ${fieldList})
+				SELECT 'delete', OLD.rowid, OLD.id, OLD.locale, ${oldFieldList}
+				WHERE OLD.deleted_at IS NULL;
 			END
 		`)
 			.execute(this.db);
@@ -372,9 +429,15 @@ export class FTSManager {
 	}
 
 	/**
-	 * Verify FTS index integrity and rebuild if corrupted.
+	 * Verify FTS index integrity and rebuild if drift is detected.
+	 *
+	 * Cheap belt-and-braces check, run lazily on the first search request
+	 * per isolate. The expensive cases (corrupted indexes from pre-fix
+	 * EmDash versions, broken legacy triggers) are handled at boot time by
+	 * migration `039_fix_fts5_triggers`, not here. This routine sticks to:
 	 *
-	 * Checks for row count mismatch between content table and FTS table.
+	 *   1. FTS table missing while config says search is enabled -> rebuild.
+	 *   2. Row count mismatch between content table and FTS docsize -> rebuild.
 	 *
 	 * Returns true if the index was rebuilt, false if it was healthy.
 	 */
@@ -397,16 +460,15 @@ export class FTSManager {
 			return true;
 		}
 
-		// Check 1: Row count mismatch
+		// Row count parity check. For external-content FTS tables, COUNT(*)
+		// on the virtual table is answered from the backing content table
+		// (including soft-deleted rows), so we use the docsize shadow table
+		// which tracks rows actually present in the full-text index.
 		const contentCount = await sql<{ count: number }>`
 			SELECT COUNT(*) as count FROM ${sql.ref(contentTable)}
 			WHERE deleted_at IS NULL
 		`.execute(this.db);
 
-		// For external-content FTS tables, COUNT(*) on the virtual table is
-		// answered from the backing content table, including soft-deleted rows.
-		// The docsize shadow table tracks the rows actually present in the
-		// full-text index, which is what we need for repair decisions.
 		const ftsCount = await sql<{ count: number }>`
 			SELECT COUNT(*) as count FROM "${sql.raw(ftsDocsizeTable)}"
 		`.execute(this.db);