@agent-native/core-corpus 0.216.0 → 0.217.0

package/corpus/templates/content/server/db/schema.ts

@@ -20,6 +20,8 @@ export const documents = table("documents", {
   bodyRevision: integer("body_revision").notNull().default(0),
   collabBodyRevision: integer("collab_body_revision"),
   liveBodySwitchedAt: text("live_body_switched_at"),
+  // `_collab_docs.version` the stored body was last derived from.
+  liveCopyVersion: integer("live_copy_version"),
   description: text("description").notNull().default(""),
   icon: text("icon"),
   position: integer("position").notNull().default(0),
@@ -802,6 +804,24 @@ export const contentDatabaseSourceExecutions = table(
   },
 );
 
+export const contentDatabaseBuilderBlobCleanup = table(
+  "content_database_builder_blob_cleanup",
+  {
+    blobReference: text("blob_reference").primaryKey(),
+    createdAt: text("created_at").notNull().default(now()),
+    attemptCount: integer("attempt_count").notNull().default(0),
+    nextAttemptAt: text("next_attempt_at")
+      .notNull()
+      .default("1970-01-01T00:00:00.000Z"),
+  },
+  (cleanup) => [
+    index("content_database_builder_blob_cleanup_due_idx").on(
+      cleanup.nextAttemptAt,
+      cleanup.createdAt,
+    ),
+  ],
+);
+
 export const contentDatabaseSourceExecutionClaims = table(
   "content_database_source_execution_claims",
   {

package/corpus/templates/content/server/lib/content-trash-purge.ts

@@ -26,6 +26,7 @@ import {
 } from "drizzle-orm";
 import { alias } from "drizzle-orm/pg-core";
 
+import { cleanupBuilderPrivatePayload } from "../../actions/_builder-cms-blob-custody.js";
 import { contentTrashPredicates } from "../../actions/_content-trash-query.js";
 import {
   deleteTrashedDocumentSubtree,
@@ -305,6 +306,7 @@ export async function processContentTrashPurge(operationId: string) {
           }
 
           const units = unitToDelete ? [unitToDelete] : [];
+          const deletedBlobReferences = new Set<string>();
           let deletedDelta = 0;
           for (const unit of units) {
             const frozen = allPending.filter(
@@ -316,6 +318,7 @@ export async function processContentTrashPurge(operationId: string) {
                 transactionDb,
                 unit.rootDocumentId,
                 unit.ownerEmail,
+                deletedBlobReferences,
                 frozen,
                 frozen[0]?.expectedScopeFingerprint,
               );
@@ -372,12 +375,23 @@ export async function processContentTrashPurge(operationId: string) {
               });
             if (!countReceipt) throw new ContentTrashPurgeLeaseLostError();
           }
-          return { blockedDelta, deletedDelta, unitCount: units.length };
+          return {
+            blockedDelta,
+            deletedDelta,
+            deletedBlobReferences,
+            unitCount: units.length,
+          };
         },
         { isolationLevel: "serializable" },
       );
       claimed.blockedCount += batchResult.blockedDelta;
       claimed.deletedCount += batchResult.deletedDelta;
+      for (const reference of batchResult.deletedBlobReferences) {
+        await cleanupBuilderPrivatePayload(
+          reference,
+          "deleted document source",
+        );
+      }
       if (batchResult.unitCount === 0) break;
     }
   } catch (error) {

package/corpus/templates/content/server/lib/live-body-writer.ts

@@ -3,7 +3,7 @@ import {
   evaluateFeatureFlagRules,
   getFeatureFlagRulesStrict,
 } from "@agent-native/core/feature-flags";
-import { eq } from "drizzle-orm";
+import { eq, sql } from "drizzle-orm";
 
 import {
   LIVE_BODY_SHADOW_FLAG,
@@ -19,11 +19,14 @@ import { schema } from "../db/index.js";
 // The flags are evaluated for the page, not the caller. Two collaborators with
 // different flags would otherwise write one body down two paths at once.
 //
-// `collab_body_revision == body_revision` is what makes the kill switch safe.
-// While the flags are off, a switched page takes legacy SQL writes that its
-// live copy never sees; once the flags come back, deriving from that stale
-// copy would overwrite them. A legacy write bumps only `body_revision`, so the
-// page stays on the legacy path until it is converted again.
+// Two markers make the kill switch safe, one per side. While the flags are
+// off, a switched page takes legacy SQL writes its live copy never sees, and
+// editor updates that commit to Yjs on main's path, where nothing derives the
+// stored body. A legacy write bumps only `body_revision`, so
+// `collab_body_revision` falls behind it; a Yjs commit bumps only
+// `_collab_docs.version`, so `live_copy_version` falls behind that. Either
+// keeps the page on the legacy path until it is converted again; deriving
+// from a copy out of step with the stored body would overwrite one side.
 
 export type LiveBodyWritePath = "live_copy" | "legacy";
 
@@ -31,6 +34,7 @@ export type LiveBodyLegacyReason =
   | "no_page"
   | "not_switched"
   | "live_copy_behind"
+  | "stored_body_behind"
   | "flag_off";
 
 export type LiveBodyWriterGate =
@@ -39,55 +43,144 @@ export type LiveBodyWriterGate =
 
 export type LiveBodyWriterPageRow = Pick<
   typeof schema.documents.$inferSelect,
+  | "id"
   | "liveBodySwitchedAt"
   | "bodyRevision"
   | "collabBodyRevision"
+  | "liveCopyVersion"
   | "ownerEmail"
   | "orgId"
 >;
 
 export const liveBodyWriterPageColumns = {
+  id: schema.documents.id,
   liveBodySwitchedAt: schema.documents.liveBodySwitchedAt,
   bodyRevision: schema.documents.bodyRevision,
   collabBodyRevision: schema.documents.collabBodyRevision,
+  liveCopyVersion: schema.documents.liveCopyVersion,
   ownerEmail: schema.documents.ownerEmail,
   orgId: schema.documents.orgId,
 };
 
 /**
- * Decides the write path for a page row the caller already read, for example
- * under the row lock its write holds; pass that transaction so the flags are
- * read in it rather than from the request's settings snapshot. Unreadable or
- * malformed flag state throws: reading it as "off" would send a switched
- * page's writes down the legacy path.
+ * Decides the write path for a page row the caller read under its row lock;
+ * pass that transaction so the flags and the stored live copy's version are
+ * read in it rather than from the request's settings snapshot. A caller
+ * without the lock uses `liveBodyWriterGate`. Unreadable or malformed flag
+ * state throws: reading it as "off" would send a switched page's writes down
+ * the legacy path.
  */
 export async function liveBodyWriterGateForRow(
   row: LiveBodyWriterPageRow | null | undefined,
   options: { transaction?: DbExec } = {},
+): Promise<LiveBodyWriterGate> {
+  return gateForMarkers(
+    row,
+    async (documentId) => {
+      const { rows } = await (options.transaction ?? getDbExec()).execute({
+        sql: "SELECT version FROM _collab_docs WHERE doc_id = ?",
+        args: [documentId],
+      });
+      return (rows[0] as { version?: unknown } | undefined)?.version;
+    },
+    options,
+  );
+}
+
+async function gateForMarkers(
+  row: LiveBodyWriterPageRow | null | undefined,
+  readCollabVersion: (documentId: string) => Promise<unknown>,
+  options: { transaction?: DbExec },
 ): Promise<LiveBodyWriterGate> {
   if (!row) return { writePath: "legacy", reason: "no_page" };
   if (!row.liveBodySwitchedAt)
     return { writePath: "legacy", reason: "not_switched" };
   if (row.collabBodyRevision !== row.bodyRevision)
     return { writePath: "legacy", reason: "live_copy_behind" };
-  const scope = { userEmail: row.ownerEmail, orgId: row.orgId };
-  const [writer, shadow] = await Promise.all(
-    [LIVE_BODY_WRITER_FLAG, LIVE_BODY_SHADOW_FLAG].map(async (flag) =>
-      evaluateFeatureFlagRules(
-        flag.key,
-        await getFeatureFlagRulesStrict(flag.key, {
-          ...scope,
-          transaction: options.transaction,
-        }),
-        scope,
-      ),
-    ),
+  // The stored version, not the one an edit was built on: a peer's commit
+  // moves both markers, and core's fence retries the overtaken edit.
+  const collabVersion = parseCollabVersion(
+    row.id,
+    await readCollabVersion(row.id),
   );
-  return writer && shadow
+  if (collabVersion === null || row.liveCopyVersion !== collabVersion)
+    return { writePath: "legacy", reason: "stored_body_behind" };
+  return (await liveBodyWriterFlagsOn(row, options))
     ? { writePath: "live_copy" }
     : { writePath: "legacy", reason: "flag_off" };
 }
 
+// Core's version for the page's collab doc; null when it has none.
+function parseCollabVersion(documentId: string, version: unknown) {
+  if (version === undefined || version === null) return null;
+  const parsed = Number(version);
+  if (!Number.isSafeInteger(parsed)) {
+    throw new Error(
+      `Page ${documentId}'s collab doc has an unreadable version (${String(version)}).`,
+    );
+  }
+  return parsed;
+}
+
+/** Both flags, for the page's owner and org; throws like the gate. */
+export async function liveBodyWriterFlagsOn(
+  row: Pick<LiveBodyWriterPageRow, "ownerEmail" | "orgId">,
+  options: { transaction?: DbExec } = {},
+): Promise<boolean> {
+  const [writer, shadow] = await Promise.all(
+    [LIVE_BODY_WRITER_FLAG, LIVE_BODY_SHADOW_FLAG].map((flag) =>
+      flagOnForPage(flag.key, row, options.transaction),
+    ),
+  );
+  return writer && shadow;
+}
+
+async function flagOnForPage(
+  key: string,
+  row: Pick<LiveBodyWriterPageRow, "ownerEmail" | "orgId">,
+  transaction: DbExec | undefined,
+): Promise<boolean> {
+  const scope = { userEmail: row.ownerEmail, orgId: row.orgId };
+  const rules = await getFeatureFlagRulesStrict(key, { ...scope, transaction });
+  if (evaluateFeatureFlagRules(key, rules, scope)) return true;
+  if (row.orgId) return false;
+  // `enable-for-current-user` stores an enablement in the org its manager had
+  // active, but a personal page has no org, so look for the owner's own
+  // listing in each of their orgs. Only that listing counts: an org's rollout
+  // covers the org's pages, not its members' personal ones.
+  const owner = row.ownerEmail.trim().toLowerCase();
+  for (const orgId of await ownerOrgIds(owner, transaction)) {
+    const orgRules = await getFeatureFlagRulesStrict(key, {
+      orgId,
+      transaction,
+    });
+    if (orgRules.mode !== "off" && orgRules.emails.includes(owner)) return true;
+  }
+  return false;
+}
+
+// Read here rather than through core's membership cache, which answers for the
+// request's caller and turns an unreadable table into "no orgs".
+async function ownerOrgIds(
+  owner: string,
+  transaction: DbExec | undefined,
+): Promise<string[]> {
+  const exec = transaction ?? getDbExec();
+  const relation = await exec.execute({
+    sql: "SELECT to_regclass('org_members') AS relation",
+    args: [],
+  });
+  // A deployment without organizations has no memberships.
+  if (!relation.rows[0]?.relation) return [];
+  const { rows } = await exec.execute({
+    sql: `SELECT org_id FROM org_members
+          WHERE LOWER(email) = ? AND federation_removal_pending_at IS NULL
+          ORDER BY org_id`,
+    args: [owner],
+  });
+  return rows.map((row) => String((row as { org_id: unknown }).org_id));
+}
+
 // Request reads retain settings initialization and the request snapshot. A
 // transaction context is explicit: an executor's shape does not identify a
 // transaction. Ordinary database adapters enter through core's withDbExec.
@@ -173,6 +266,7 @@ function readContext(options: LiveBodyWriterReadOptions): {
   throw new LiveBodyWriterContextError();
 }
 
+/** The write path for a page, read without its row lock. */
 export async function liveBodyWriterGate(
   docId: string,
   options: LiveBodyWriterReadOptions = {},
@@ -184,8 +278,26 @@ export async function liveBodyWriterGate(
     .from(schema.documents)
     .where(eq(schema.documents.id, docId))
     .limit(1);
-  return liveBodyWriterGateForRow(
-    row,
+  // Most pages stop here, before `_collab_docs`, which a deployment creates
+  // with its first collab doc.
+  if (!row) return { writePath: "legacy", reason: "no_page" };
+  if (!row.liveBodySwitchedAt)
+    return { writePath: "legacy", reason: "not_switched" };
+  // A live-copy commit moves the page's markers and its collab doc's version
+  // together, so read them in one statement: two reads could straddle a
+  // commit and see the page behind its own live copy.
+  // guard:allow-unscoped — callers check access to the page first, and the gate returns only its write path.
+  const [markers] = await db
+    .select({
+      ...liveBodyWriterPageColumns,
+      collabVersion: sql<unknown>`(SELECT collab.version FROM _collab_docs AS collab WHERE collab.doc_id = documents.id)`,
+    })
+    .from(schema.documents)
+    .where(eq(schema.documents.id, docId))
+    .limit(1);
+  return gateForMarkers(
+    markers,
+    async () => markers?.collabVersion,
     context.kind === "transaction" ? { transaction: context.executor } : {},
   );
 }

package/corpus/templates/content/server/live-copy/binding.ts

@@ -0,0 +1,31 @@
+import {
+  createBoundedTaskRunner,
+  defineLiveCopy,
+} from "@agent-native/core/collab";
+
+import { LIVE_BODY_FIELD } from "../../shared/live-body.js";
+import { liveBodyWriterGate } from "../lib/live-body-writer.js";
+import { persistContentLiveCopy } from "./persist.js";
+import source from "./tasks.ts?bounded-task";
+import { liveBodyConversionFor } from "./write-cause.js";
+
+/** Edits `tasks.ts` defines, by name. */
+export const CONTENT_LIVE_COPY_EDITS = {
+  replaceBody: "replace-body",
+} as const;
+
+/**
+ * Content's page bodies as live copies. A page is covered while the D2 gate
+ * says `live_copy`, and while it is being switched. Eligibility uses the
+ * request's flag snapshot; persist rechecks flags in its locked transaction.
+ */
+export const contentLiveCopy = defineLiveCopy({
+  name: "content-body",
+  appliesTo: async (documentId) => {
+    if (liveBodyConversionFor(documentId) !== undefined) return true;
+    return (await liveBodyWriterGate(documentId)).writePath === "live_copy";
+  },
+  runner: createBoundedTaskRunner({ source }),
+  field: LIVE_BODY_FIELD,
+  persist: persistContentLiveCopy,
+});

package/corpus/templates/content/server/live-copy/bounded-task.d.ts

@@ -0,0 +1,6 @@
+// `import source from "./tasks.ts?bounded-task"` is bundled at build time by
+// core's bounded-task plugin into a source the worker runner evaluates.
+declare module "*?bounded-task" {
+  const source: import("@agent-native/core/collab").BoundedTaskSource;
+  export default source;
+}

package/corpus/templates/content/server/live-copy/exec-db.ts

@@ -0,0 +1 @@
+export { drizzleOverExec } from "../db/exec.js";

package/corpus/templates/content/server/live-copy/holds.ts

@@ -0,0 +1,59 @@
+import {
+  BoundedTaskError,
+  createBoundedTaskRunner,
+  type BoundedTaskErrorCode,
+} from "@agent-native/core/collab";
+
+import { CONTENT_HOLDS_BODY_TASK, type HoldsBodyResult } from "./task-names.js";
+import source from "./tasks.ts?bounded-task";
+
+// Its own worker: a save's comparison runs under the page lock, and on the
+// live copy's worker a slow one would hold up every page's collab updates.
+const runner = createBoundedTaskRunner({ source });
+
+// Limits a body can hit. Anything else the worker reports is a bug here.
+const UNCHECKED: ReadonlySet<BoundedTaskErrorCode> = new Set([
+  "unavailable",
+  "queue_full",
+  "queue_timeout",
+  "input_too_large",
+  "deadline_exceeded",
+  "memory_exceeded",
+  "worker_crashed",
+]);
+
+const RESULTS: ReadonlySet<unknown> = new Set<HoldsBodyResult>([
+  "held",
+  "unheld",
+  "noncanonical",
+]);
+
+/**
+ * Whether a switched page's stored body holds every change a browser save
+ * made to the body it was authored on, with any other changes clear of them
+ * (`bodyHoldsChanges`). The save's text then reached the live copy over
+ * collab, and only a peer's commits differ. `noncanonical` when the save's
+ * body is not its own serialization, and `unchecked` when the comparison hit
+ * the worker's limits.
+ */
+export async function liveCopyHoldsSave(args: {
+  base: string;
+  stored: string;
+  candidate: string;
+}): Promise<HoldsBodyResult | "unchecked"> {
+  let result: unknown;
+  try {
+    result = await runner.run<unknown>(CONTENT_HOLDS_BODY_TASK, args);
+  } catch (error) {
+    if (error instanceof BoundedTaskError && UNCHECKED.has(error.code)) {
+      return "unchecked";
+    }
+    throw error;
+  }
+  if (!RESULTS.has(result)) {
+    throw new TypeError(
+      `${CONTENT_HOLDS_BODY_TASK} answered ${JSON.stringify(result)}`,
+    );
+  }
+  return result as HoldsBodyResult;
+}