@agent-native/core-corpus 0.216.0 → 0.218.0

package/corpus/templates/content/server/live-copy/switch.ts

@@ -0,0 +1,136 @@
+import type { ActionRunContext } from "@agent-native/core/action";
+import {
+  mutateLiveCopy,
+  readLiveCopy,
+  seedLiveCopy,
+} from "@agent-native/core/collab";
+import { eq } from "drizzle-orm";
+
+import { assertSourceAllowsPageEdit } from "../../actions/_document-source.js";
+import { getDb, schema } from "../db/index.js";
+import {
+  liveBodyWriterFlagsOn,
+  liveBodyWriterGate,
+  liveBodyWriterPageColumns,
+  type LiveBodyWriterGate,
+} from "../lib/live-body-writer.js";
+import { CONTENT_LIVE_COPY_EDITS, contentLiveCopy } from "./binding.js";
+import {
+  LiveBodyConversionError,
+  persistLiveBodyConversion,
+} from "./persist.js";
+import { withLiveBodyConversion } from "./write-cause.js";
+
+const CONVERSION_ATTEMPTS = 3;
+
+async function readPage(documentId: string) {
+  // guard:allow-unscoped — set-live-body-switch checks editor access before it switches a page.
+  const [page] = await getDb()
+    .select({
+      ...liveBodyWriterPageColumns,
+      content: schema.documents.content,
+      sourceMode: schema.documents.sourceMode,
+      sourceRootPath: schema.documents.sourceRootPath,
+      sourcePath: schema.documents.sourcePath,
+    })
+    .from(schema.documents)
+    .where(eq(schema.documents.id, documentId))
+    .limit(1);
+  return page;
+}
+
+/**
+ * Switches a page to its live copy: makes the live copy hold the stored body
+ * (seeding it, or editing it to match, so open editors merge rather than
+ * reload), then marks the page switched in the same commit. The stored body
+ * wins over Yjs-only edits, so switch a page before an editor binds to it.
+ * Throws `LiveBodyConversionError` `flag_off` unless both flags are on for
+ * the page's owner, and `base_changed` when the body keeps moving.
+ */
+export async function switchPageToLiveCopy(
+  documentId: string,
+  options: { ctx?: ActionRunContext } = {},
+): Promise<LiveBodyWriterGate> {
+  for (let attempt = 0; attempt < CONVERSION_ATTEMPTS; attempt += 1) {
+    const page = await readPage(documentId);
+    if (!page) return { writePath: "legacy", reason: "no_page" };
+    assertSourceAllowsPageEdit(page);
+    const gate = await liveBodyWriterGate(documentId);
+    if (gate.writePath === "live_copy") return gate;
+    if (!(await liveBodyWriterFlagsOn(page))) {
+      throw new LiveBodyConversionError(
+        "flag_off",
+        documentId,
+        `The live-copy writer is off for page ${documentId}'s owner.`,
+      );
+    }
+    const conversion = { documentId, baseRevision: page.bodyRevision };
+    try {
+      await withLiveBodyConversion(
+        conversion,
+        { ctx: options.ctx, operation: "switch-live-body" },
+        async () => {
+          const live = await readLiveCopy(contentLiveCopy, documentId);
+          if (live.kind === "live") {
+            await mutateLiveCopy(contentLiveCopy, documentId, {
+              edit: CONTENT_LIVE_COPY_EDITS.replaceBody,
+              input: { content: page.content },
+              // An edit that changes nothing commits no live copy, so the
+              // binding's persist does not run; switch the page here.
+              persist: async (tx, change) => {
+                if (!change.changed) {
+                  await persistLiveBodyConversion(
+                    tx,
+                    conversion,
+                    change.text,
+                    change.version,
+                  );
+                }
+              },
+            });
+            return;
+          }
+          const seeded = await seedLiveCopy(contentLiveCopy, documentId, {
+            source: page.content,
+          });
+          if (!seeded.seeded) {
+            throw new LiveBodyConversionError(
+              "base_changed",
+              documentId,
+              `Page ${documentId}'s live copy was seeded while it was being switched.`,
+            );
+          }
+        },
+      );
+    } catch (error) {
+      if (
+        error instanceof LiveBodyConversionError &&
+        error.code === "base_changed"
+      ) {
+        continue;
+      }
+      throw error;
+    }
+    return liveBodyWriterGate(documentId);
+  }
+  throw new LiveBodyConversionError(
+    "base_changed",
+    documentId,
+    `Page ${documentId} kept changing while it was being switched to its live copy.`,
+  );
+}
+
+/**
+ * Sends a page's body writes back to the legacy path. Its live copy stays;
+ * the first legacy body write leaves it behind, so switching back converts
+ * the page again.
+ */
+export async function switchPageToLegacy(
+  documentId: string,
+): Promise<LiveBodyWriterGate> {
+  await getDb()
+    .update(schema.documents)
+    .set({ liveBodySwitchedAt: null })
+    .where(eq(schema.documents.id, documentId));
+  return liveBodyWriterGate(documentId);
+}

package/corpus/templates/content/server/live-copy/task-names.ts

@@ -0,0 +1,4 @@
+/** Content's own tasks in the live copy's bounded worker, beside core's. */
+export const CONTENT_HOLDS_BODY_TASK = "content.holdsBody";
+
+export type HoldsBodyResult = "held" | "unheld" | "noncanonical";

package/corpus/templates/content/server/live-copy/tasks.ts

@@ -0,0 +1,183 @@
+import {
+  defineLiveCopyTasks,
+  type LiveCopyTaskRefusal,
+} from "@agent-native/core/collab/live-copy-tasks";
+import type { Node as PMNode, Schema } from "@tiptap/pm/model";
+import {
+  initProseMirrorDoc,
+  prosemirrorToYXmlFragment,
+  updateYFragment,
+} from "@tiptap/y-tiptap";
+import * as Y from "yjs";
+
+import { createContentEditorStructuralSchema } from "../../shared/content-editor-structural-schema.js";
+import { bodyHoldsChanges } from "../../shared/document-intent-merge.js";
+import {
+  LIVE_BODY_FIELD,
+  materializeLiveBody,
+} from "../../shared/live-body.js";
+import { docToNfm, nfmToDoc, parseNfmWithLosses } from "../../shared/nfm.js";
+import { CONTENT_HOLDS_BODY_TASK, type HoldsBodyResult } from "./task-names.js";
+
+// The worker half of Content's live copy: every handler here runs in the
+// bounded task runner, so import only what a worker bundle can carry.
+
+let structuralSchema: Schema | undefined;
+
+function schema(): Schema {
+  structuralSchema ??= createContentEditorStructuralSchema();
+  return structuralSchema;
+}
+
+function refused(reason: string, detail?: string): LiveCopyTaskRefusal {
+  return { kind: "refused", reason, ...(detail ? { detail } : {}) };
+}
+
+function assertBodyField(field: string): void {
+  if (field !== LIVE_BODY_FIELD) {
+    throw new Error(
+      `Content's live copy binds "${LIVE_BODY_FIELD}", not "${field}".`,
+    );
+  }
+}
+
+function deriveBody(doc: Y.Doc, field: string): string | LiveCopyTaskRefusal {
+  assertBodyField(field);
+  // Building the tree deletes what the schema rejects, and derive must not
+  // edit the doc it is given.
+  const copy = new Y.Doc();
+  try {
+    Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
+    const body = materializeLiveBody(copy);
+    if (body.kind === "lossy") {
+      return refused("lossy", "the page schema cannot carry part of the body");
+    }
+    if (body.kind === "uninitialized") return refused("empty");
+    return body.markdown;
+  } finally {
+    copy.destroy();
+  }
+}
+
+/**
+ * The tree a Markdown body becomes in the editor. A body the parser or the
+ * schema cannot keep in full is refused: the live copy would keep the rest
+ * and the page would lose that part on its next commit.
+ */
+function buildBody(markdown: string): PMNode | LiveCopyTaskRefusal {
+  // What the parser dropped is in neither serialization compared below.
+  const { doc: parsed, losses } = parseNfmWithLosses(markdown);
+  if (losses.length > 0) {
+    const kinds = [...new Set(losses.map((loss) => loss.kind))].join(", ");
+    return refused("lossy", `the parser drops part of this body (${kinds})`);
+  }
+  let root: PMNode;
+  try {
+    root = schema().nodeFromJSON(parsed);
+    root.check();
+  } catch (error) {
+    return refused(
+      "unrepresentable",
+      error instanceof Error ? error.message : undefined,
+    );
+  }
+  if (docToNfm(root.toJSON()) !== docToNfm(parsed)) {
+    return refused("lossy", "the page schema drops part of this body");
+  }
+  return root;
+}
+
+function replaceBodyInput(
+  input: unknown,
+): { content: string; expected?: string } | LiveCopyTaskRefusal {
+  if (typeof input !== "object" || input === null) {
+    return refused("invalid_input", "expected { content, expected? }");
+  }
+  const { content, expected } = input as Record<string, unknown>;
+  if (typeof content !== "string") {
+    return refused("invalid_input", "content must be a string");
+  }
+  if (expected !== undefined && typeof expected !== "string") {
+    return refused("invalid_input", "expected must be a string");
+  }
+  return expected === undefined ? { content } : { content, expected };
+}
+
+function isRefusal(value: unknown): value is LiveCopyTaskRefusal {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    (value as { kind?: unknown }).kind === "refused"
+  );
+}
+
+function isCanonical(markdown: string): boolean {
+  try {
+    return docToNfm(nfmToDoc(markdown)) === markdown;
+  } catch {
+    // coercion-ok: a body the parser rejects is not canonical, and its save keeps its draft.
+    return false;
+  }
+}
+
+function holdsBodyInput(input: unknown): {
+  base: string;
+  stored: string;
+  candidate: string;
+} {
+  const { base, stored, candidate } = (input ?? {}) as Record<string, unknown>;
+  if (
+    typeof base !== "string" ||
+    typeof stored !== "string" ||
+    typeof candidate !== "string"
+  ) {
+    throw new TypeError("expected { base, stored, candidate } as strings");
+  }
+  return { base, stored, candidate };
+}
+
+const liveCopyTasks = defineLiveCopyTasks({
+  derive: deriveBody,
+
+  seed(doc, field, source) {
+    assertBodyField(field);
+    const root = buildBody(source);
+    if (isRefusal(root)) return root;
+    prosemirrorToYXmlFragment(root, doc.getXmlFragment(field));
+  },
+
+  edits: {
+    /**
+     * Makes the body `content`, editing only the blocks that differ so open
+     * editors keep their place. With `expected`, refuses `base_changed`
+     * unless the current body is exactly that text.
+     */
+    "replace-body"(doc, field, input) {
+      const parsed = replaceBodyInput(input);
+      if (isRefusal(parsed)) return parsed;
+      if (parsed.expected !== undefined) {
+        const current = deriveBody(doc, field);
+        if (isRefusal(current)) return current;
+        if (current !== parsed.expected) return refused("base_changed");
+      }
+      const root = buildBody(parsed.content);
+      if (isRefusal(root)) return root;
+      const fragment = doc.getXmlFragment(field);
+      const { meta } = initProseMirrorDoc(fragment, schema());
+      updateYFragment(doc, fragment, root, meta);
+    },
+  },
+});
+
+export const tasks = {
+  ...liveCopyTasks,
+  /** Whether the stored body holds every change `candidate` made to `base`. */
+  async [CONTENT_HOLDS_BODY_TASK](input: unknown): Promise<HoldsBodyResult> {
+    const { base, stored, candidate } = holdsBodyInput(input);
+    // The comparison reads the candidate in its parsed form, and a parse can
+    // drop text, such as text directly inside a table. Only an editor's own
+    // serialization says what the save holds.
+    if (!isCanonical(candidate)) return "noncanonical";
+    return bodyHoldsChanges(base, stored, candidate) ? "held" : "unheld";
+  },
+};

package/corpus/templates/content/server/live-copy/write-cause.ts

@@ -0,0 +1,85 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
+import type { ActionRunContext } from "@agent-native/core/action";
+
+import type { DocumentBodyIntent } from "../../shared/document-intent-order.js";
+
+export interface ContentBodyIntentRecord {
+  /**
+   * Without `authoredBaseRevision`, persist records the revision the edit
+   * was applied to: the page's, under its lock.
+   */
+  intent: Omit<DocumentBodyIntent, "authoredBaseRevision"> & {
+    authoredBaseRevision?: number;
+  };
+  /**
+   * For a writer that knows its result only once the edit runs: without
+   * these, persist takes the committed body's hash and the blocks it changed.
+   */
+  candidateHash?: string;
+  metadataHash?: string;
+  changedBlockIndexes?: number[];
+  displacedCheckpointId?: string;
+}
+
+/**
+ * Why a server write changed a page body. A live-copy commit's `persist`
+ * records it: a History transition for every commit, and the body intent once
+ * the commit's revision is known. Server writers pass it to
+ * `editContentLiveCopy`, which also records an unchanged edit's intent; core
+ * runs no binding persist for one. Give each edit its own cause; an intent
+ * recorded twice would claim two revisions.
+ */
+export interface ContentBodyWriteCause {
+  ctx?: ActionRunContext;
+  /** The History operation, such as `"accept-suggestion"`. */
+  operation: string;
+  origin?: string;
+  historySessionId?: string;
+  intent?: ContentBodyIntentRecord;
+}
+
+/** A page being switched to its live copy, from the body at `baseRevision`. */
+export interface LiveBodyConversion {
+  documentId: string;
+  baseRevision: number;
+}
+
+interface WriteScope {
+  cause: ContentBodyWriteCause;
+  conversion?: LiveBodyConversion;
+}
+
+const scopes = new AsyncLocalStorage<WriteScope>();
+
+export function withContentBodyWriteCause<T>(
+  cause: ContentBodyWriteCause,
+  run: () => Promise<T>,
+): Promise<T> {
+  return scopes.run({ cause }, run);
+}
+
+export function currentContentBodyWriteCause():
+  | ContentBodyWriteCause
+  | undefined {
+  return scopes.getStore()?.cause;
+}
+
+/**
+ * Runs a conversion. The page is not switched until its commit lands, so the
+ * binding treats `conversion.documentId` as a live copy only inside `run`.
+ */
+export function withLiveBodyConversion<T>(
+  conversion: LiveBodyConversion,
+  cause: ContentBodyWriteCause,
+  run: () => Promise<T>,
+): Promise<T> {
+  return scopes.run({ cause, conversion }, run);
+}
+
+export function liveBodyConversionFor(
+  documentId: string,
+): LiveBodyConversion | undefined {
+  const conversion = scopes.getStore()?.conversion;
+  return conversion?.documentId === documentId ? conversion : undefined;
+}

package/corpus/templates/content/server/live-copy/write.ts

@@ -0,0 +1,85 @@
+import {
+  mutateLiveCopy,
+  type LiveCopyCallChange,
+  type LiveCopyMutationResult,
+} from "@agent-native/core/collab";
+import type { DbExec } from "@agent-native/core/db";
+
+import { CONTENT_LIVE_COPY_EDITS, contentLiveCopy } from "./binding.js";
+import { persistUnchangedContentBodyEdit } from "./persist.js";
+import {
+  withContentBodyWriteCause,
+  type ContentBodyWriteCause,
+} from "./write-cause.js";
+
+/**
+ * A server writer's replace names the body it was built from. Core retries
+ * an edit a peer's commit overtook, and a replace without `expected` would
+ * silently overwrite the peer's text on that retry.
+ */
+type ContentLiveCopyEditInput<Edit extends string> =
+  typeof CONTENT_LIVE_COPY_EDITS.replaceBody extends Edit
+    ? { content: string; expected: string }
+    : unknown;
+
+function replaceBodyInput(
+  documentId: string,
+  input: unknown,
+): { content: unknown; expected: string } {
+  const { content, expected } = (input ?? {}) as {
+    content?: unknown;
+    expected?: unknown;
+  };
+  if (typeof expected !== "string") {
+    throw new Error(
+      `A server writer's replace of page ${documentId} must name the body it was built from as \`expected\`.`,
+    );
+  }
+  // The worker gets a structured clone, which drops a getter's or a
+  // non-enumerable property's value, so send the values checked here.
+  return { content, expected };
+}
+
+/**
+ * Applies a named edit to a switched page's live copy for a server writer.
+ * A changed edit commits through the binding's persist; an unchanged one
+ * still locks the page in Content's order and records the cause's intent,
+ * which core's binding persist never sees. `persist` runs after either, in
+ * the same transaction, with the page locked.
+ *
+ * Throws `LIVE_BODY_WRITE_PATH_CHANGED` when the page left its live copy
+ * after the writer chose it; nothing commits, so the writer can take its
+ * legacy path. An edit a peer's commit overtook is retried against the
+ * peer's body, so a replace whose `expected` body changed is refused
+ * `base_changed`.
+ */
+export async function editContentLiveCopy<Edit extends string>(
+  documentId: string,
+  options: {
+    cause: ContentBodyWriteCause;
+    edit: Edit;
+    input: ContentLiveCopyEditInput<Edit>;
+    requestSource?: string;
+    persist?: (tx: DbExec, change: LiveCopyCallChange) => Promise<void>;
+  },
+): Promise<LiveCopyMutationResult> {
+  const input =
+    options.edit === CONTENT_LIVE_COPY_EDITS.replaceBody
+      ? replaceBodyInput(documentId, options.input)
+      : options.input;
+  return withContentBodyWriteCause(options.cause, () =>
+    mutateLiveCopy(contentLiveCopy, documentId, {
+      edit: options.edit,
+      input,
+      ...(options.requestSource
+        ? { requestSource: options.requestSource }
+        : {}),
+      persist: async (tx, change) => {
+        if (!change.changed) {
+          await persistUnchangedContentBodyEdit(tx, documentId, options.cause);
+        }
+        await options.persist?.(tx, change);
+      },
+    }),
+  );
+}

package/corpus/templates/content/server/plugins/builder-private-payload-cleanup.ts

@@ -0,0 +1,22 @@
+import {
+  defineNitroPlugin,
+  registerRecurringSweepHandler,
+} from "@agent-native/core/server";
+
+import { drainBuilderPrivatePayloadCleanup } from "../../actions/_builder-cms-blob-custody.js";
+
+export const BUILDER_PRIVATE_PAYLOAD_CLEANUP_SWEEP_ID =
+  "content-builder-private-payload-cleanup";
+
+export function registerBuilderPrivatePayloadCleanupSweep() {
+  return registerRecurringSweepHandler(
+    BUILDER_PRIVATE_PAYLOAD_CLEANUP_SWEEP_ID,
+    async ({ deadlineAt, signal }) => {
+      await drainBuilderPrivatePayloadCleanup({ deadlineAt, signal });
+    },
+  );
+}
+
+export default defineNitroPlugin(() => {
+  registerBuilderPrivatePayloadCleanupSweep();
+});

package/corpus/templates/content/server/plugins/collab.ts

@@ -1,9 +1,12 @@
 import { createCollabPlugin } from "@agent-native/core/server";
 
+import { contentLiveCopy } from "../live-copy/binding.js";
+
 export default createCollabPlugin({
   table: "documents",
   contentColumn: "content",
   idColumn: "id",
   autoSeed: false, // Seeding happens via edit-document action, not on startup
   access: { mode: "resource", resourceType: "document" },
+  liveCopy: contentLiveCopy,
 });

package/corpus/templates/content/server/plugins/db.ts

@@ -1479,6 +1479,34 @@ export const runContentMigrations = runMigrations(
       name: "content-live-body-switch",
       sql: `ALTER TABLE documents ADD COLUMN IF NOT EXISTS live_body_switched_at TEXT`,
     },
+    {
+      version: 125,
+      name: "content-live-copy-version",
+      sql: `ALTER TABLE documents ADD COLUMN IF NOT EXISTS live_copy_version INTEGER`,
+    },
+    {
+      version: 126,
+      name: "builder-private-payload-cleanup-queue",
+      sql: `ALTER TABLE document_comments ADD COLUMN IF NOT EXISTS property_id TEXT;
+        ALTER TABLE document_comments ADD COLUMN IF NOT EXISTS property_name TEXT;
+        ALTER TABLE document_comments ADD COLUMN IF NOT EXISTS property_value TEXT;
+        CREATE INDEX IF NOT EXISTS document_comments_document_idx
+          ON document_comments (document_id, owner_email);
+        CREATE TABLE IF NOT EXISTS content_database_builder_blob_cleanup (
+          blob_reference TEXT PRIMARY KEY,
+          created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+        )`,
+    },
+    {
+      version: 127,
+      name: "builder-private-payload-cleanup-retries",
+      sql: `ALTER TABLE content_database_builder_blob_cleanup
+          ADD COLUMN IF NOT EXISTS attempt_count INTEGER NOT NULL DEFAULT 0;
+        ALTER TABLE content_database_builder_blob_cleanup
+          ADD COLUMN IF NOT EXISTS next_attempt_at TEXT NOT NULL DEFAULT '1970-01-01T00:00:00.000Z';
+        CREATE INDEX IF NOT EXISTS content_database_builder_blob_cleanup_due_idx
+          ON content_database_builder_blob_cleanup (next_attempt_at, created_at)`,
+    },
   ],
   { table: "content_migrations" },
 );

package/corpus/templates/content/shared/api.ts

@@ -161,9 +161,23 @@ export interface DocumentUpdateRequest {
   loadedContentWasEmpty?: boolean;
 }
 
+/**
+ * The body part of a browser save on a page whose body is its live copy. The
+ * body was not written: the editor's updates reach the live copy over collab.
+ * `matchesLiveCopy` says whether every submitted, authored and snapshot body
+ * equals the body derived from the live copy at `collabVersion`.
+ */
+export interface LiveCopyBodySave {
+  status: "live_copy";
+  matchesLiveCopy: boolean;
+  bodyRevision: number;
+  collabVersion: number;
+}
+
 export interface DocumentUpdateResponse extends Document {
   urlPath: string;
   softDeletedDatabaseIds: string[];
+  bodySave?: LiveCopyBodySave;
 }
 
 export interface DocumentMoveRequest {
@@ -461,12 +475,19 @@ export type ContentDatabaseItemLeadingKind =
   | "checkbox"
   | "status";
 export type ContentDatabaseItemTitleWeight = "regular" | "medium" | "bold";
+/** What line 2 shows: a property, or the first lines of each row's page. */
+export type ContentDatabaseItemLine2Source = "property" | "page";
 
 /** Which property fills each part of a list row. */
 export interface ContentDatabaseItemLayout {
   leading: { kind: ContentDatabaseItemLeadingKind; propertyId: string | null };
   titleWeight: ContentDatabaseItemTitleWeight;
-  line2: { propertyId: string | null; lines: 1 | 2 | 3 };
+  /** A saved layout leaves `source` out when line 2 shows a property. */
+  line2: {
+    source?: ContentDatabaseItemLine2Source;
+    propertyId: string | null;
+    lines: 1 | 2 | 3;
+  };
   line3: string[];
   trailing: string[];
 }
@@ -612,6 +633,11 @@ export interface ContentDatabaseItem {
   properties: DocumentProperty[];
   workspaceFilesDatabaseId?: string | null;
   bodyHydration?: ContentDatabaseBodyHydration;
+  /**
+   * The first lines of the row's page as plain text. List reads include it
+   * only when a List view of the collection shows page text in line 2.
+   */
+  excerpt?: string;
   sourceRecord?: ContentDatabaseSourceRow;
   canonicalKey?: string | null;
   sourceOverlays?: ContentDatabaseSourceOverlay[];
@@ -891,6 +917,7 @@ export interface ContentDatabaseSourceExecution {
   idempotencyKey: string;
   summary: string;
   payload: Record<string, unknown>;
+  payloadUnavailable?: true;
   lastError: string | null;
   createdAt: string;
   updatedAt: string;
@@ -977,6 +1004,7 @@ export interface ContentDatabaseSource {
     readMode?: "fixture" | "builder-api" | (string & {}) | null;
     connectionId?: string | null;
     connectionLabel?: string | null;
+    builderSpacePublicKey?: string | null;
     truthPolicy?: ContentDatabaseSourceTruthPolicy;
     syncPolicy?: "manual" | "keep_in_sync";
     liveBridgeEnabled?: boolean;
@@ -986,6 +1014,7 @@ export interface ContentDatabaseSource {
     lastReadMatchedRowCount?: number;
     lastReadLimit?: number;
     lastReadFetchedEntryCount?: number;
+    lastReadCollectionCount?: number;
     lastReadPartial?: boolean;
     lastReadHasMore?: boolean;
     lastReadNextOffset?: number;