@agent-native/core-corpus 0.214.0 → 0.216.0

package/corpus/templates/content/e2e/live-body-gate.ts

@@ -0,0 +1,188 @@
+import { expect, test, type Page } from "@playwright/test";
+
+import { knownLossSpec, type KnownLoss } from "../acceptance/known-loss";
+import { GATE_ARM, type GateArm } from "./gate-arm";
+import { ACTION_HEADERS, postAction, postUiAction } from "./helpers";
+
+export { GATE_ARM, type GateArm };
+
+// Decision D2 of the stored-form rebuild: a page takes the new body paths only
+// when both flags are on for the person and the page itself is switched.
+// Turning the shadow flag on alone measures parity and must not switch anyone.
+export const SHADOW_FLAG = "content.live-body-shadow";
+export const WRITER_FLAG = "content.live-body-writer";
+
+/**
+ * Known-loss details for a check main fails differently on each arm. Only this
+ * run's arm entry tags it, so a lane whose fix flips one arm deletes that entry
+ * and the check turns required there while the other arm keeps reporting.
+ */
+export function knownLossOnArm(losses: Partial<Record<GateArm, KnownLoss>>) {
+  const loss = losses[GATE_ARM];
+  return loss ? knownLossSpec(loss) : {};
+}
+
+/** The flag values the signed-in account gets, keyed by registered flag. */
+async function flagValues(page: Page): Promise<Record<string, boolean>> {
+  const response = await page.request.get(
+    "/_agent-native/actions/get-feature-flags",
+    { headers: ACTION_HEADERS },
+  );
+  const text = await response.text();
+  expect(
+    response.ok(),
+    `get-feature-flags (${response.status()}): ${text}`,
+  ).toBe(true);
+  return JSON.parse(text) as Record<string, boolean>;
+}
+
+/**
+ * Put a page this test just created on its run's side of the gate; pass null
+ * when the test creates its page through the UI. The off arm proves neither
+ * flag reaches the account; the on arm skips, with the reason, until the build
+ * has everything the gate needs, and never passes in its place.
+ */
+export async function enterGateArm(
+  page: Page,
+  documentId: string | null,
+): Promise<void> {
+  const flags = await flagValues(page);
+  if (GATE_ARM === "off") {
+    for (const key of [SHADOW_FLAG, WRITER_FLAG])
+      expect(flags[key], `${key} is on in the gate-off arm`).not.toBe(true);
+    return;
+  }
+  if (!(WRITER_FLAG in flags))
+    test.skip(
+      true,
+      `Gate on: this build registers no ${WRITER_FLAG} flag. L3 adds it with the D2 per-page switch.`,
+    );
+  for (const key of [SHADOW_FLAG, WRITER_FLAG])
+    await postAction(page, "set-feature-flag", {
+      operation: "enable-for-current-user",
+      key,
+    });
+  const enabled = await flagValues(page);
+  for (const key of [SHADOW_FLAG, WRITER_FLAG])
+    expect(enabled[key], `${key} after enabling it for this account`).toBe(
+      true,
+    );
+  await switchPage(page, documentId);
+}
+
+interface FlagRules {
+  mode: "off" | "on" | "rules";
+  emails: string[];
+  orgIds: string[];
+  percentage: number;
+}
+
+/**
+ * Take the signed-in account back off both gate flags. enterGateArm adds the
+ * account's email to each flag's rules in its organization, and that entry
+ * outlives the run, so a later gate-off run signed in as the same account
+ * would find the flags on. Other entries in the rules stay as they are.
+ */
+export async function leaveGateArm(page: Page): Promise<void> {
+  if (GATE_ARM !== "on") return;
+  const session = await page.request.get("/_agent-native/auth/session", {
+    headers: { Accept: "application/json" },
+  });
+  const email = ((await session.json()) as { email?: string }).email
+    ?.trim()
+    .toLowerCase();
+  if (!email)
+    throw new Error(
+      `Leaving the gate-on arm: no signed-in account (session ${session.status()}).`,
+    );
+  const response = await page.request.get(
+    "/_agent-native/actions/list-feature-flags",
+    { headers: ACTION_HEADERS },
+  );
+  const text = await response.text();
+  if (!response.ok())
+    throw new Error(`list-feature-flags (${response.status()}): ${text}`);
+  const listed = JSON.parse(text) as {
+    status: string;
+    flags: { key: string; rules: FlagRules }[];
+  };
+  // set-feature-flag needs the same permission, so an account that cannot
+  // list the flags never enabled them.
+  if (listed.status === "forbidden") return;
+  for (const flag of listed.flags) {
+    if (flag.key !== SHADOW_FLAG && flag.key !== WRITER_FLAG) continue;
+    if (!flag.rules.emails.includes(email)) continue;
+    const { mode, emails, orgIds, percentage } = flag.rules;
+    await postAction(page, "set-feature-flag", {
+      operation: "replace-rules",
+      key: flag.key,
+      rules: {
+        mode,
+        emails: emails.filter((entry) => entry !== email),
+        orgIds,
+        percentage,
+      },
+    });
+  }
+  const left = await flagValues(page);
+  for (const key of [SHADOW_FLAG, WRITER_FLAG])
+    if (left[key] === true)
+      throw new Error(
+        `${key} is still on for ${email} after leaving the gate-on arm, so its rules turn it on for more than this account.`,
+      );
+}
+
+const SWITCH_ACTION = "set-live-body-switch";
+
+/**
+ * Switch the page through L3's action, then require both the switch and
+ * get-document to say the page writes through its live copy. A build that has
+ * the switch before its writers are routed answers "legacy", and a gate-on
+ * scenario run there would pass on the old path, so the arm skips instead.
+ */
+async function switchPage(
+  page: Page,
+  documentId: string | null,
+): Promise<void> {
+  if (documentId === null)
+    test.skip(
+      true,
+      "Gate on: this test creates its page through the UI, and only a page that exists can be switched.",
+    );
+  const response = await postUiAction(page, SWITCH_ACTION, {
+    id: documentId,
+    switched: true,
+  });
+  const text = await response.text();
+  if (response.status() === 404 && text.includes("Cannot find any route"))
+    test.skip(
+      true,
+      `Gate on: this build has no ${SWITCH_ACTION} action. L3 adds it with conversion of the page's live copy.`,
+    );
+  expect(
+    response.ok(),
+    `${SWITCH_ACTION} (${response.status()}): ${text}`,
+  ).toBe(true);
+  const switched = JSON.parse(text) as { writePath?: string; reason?: string };
+  if (switched.writePath !== "live_copy")
+    test.skip(
+      true,
+      `Gate on: ${SWITCH_ACTION} left the page on the ${switched.writePath ?? "unreported"} write path${switched.reason ? ` (${switched.reason})` : ""}.`,
+    );
+  const read = await page.request.get("/_agent-native/actions/get-document", {
+    params: { id: documentId! },
+    headers: ACTION_HEADERS,
+  });
+  const readText = await read.text();
+  expect(read.ok(), `get-document (${read.status()}): ${readText}`).toBe(true);
+  const { writePath } = JSON.parse(readText) as { writePath?: string };
+  if (writePath === undefined)
+    test.skip(
+      true,
+      "Gate on: get-document does not report writePath yet, so the arm cannot confirm the page writes through its live copy.",
+    );
+  expect(
+    writePath,
+    `${SWITCH_ACTION} said live_copy, but get-document says otherwise`,
+  ).toBe("live_copy");
+}

package/corpus/templates/content/e2e/playwright.config.ts

@@ -2,11 +2,16 @@ import path from "node:path";
 
 import { defineConfig, devices } from "@playwright/test";
 
+import { AUTH_FILES } from "./auth-files";
+import { runServerNonce } from "./server-identity";
+
 const BASE_URL = process.env.CONTENT_BASE_URL || "http://127.0.0.1:8090"; // e2e-harness-ignore: specs read the served URL from here
 
 // CI serves the production build it just made on the job's Postgres. Locally,
-// point CONTENT_BASE_URL at a running server instead.
+// point CONTENT_BASE_URL at a running server instead, started with
+// PING_MESSAGE set to the run's CONTENT_E2E_SERVER_NONCE.
 const SERVE_BUILD = process.env.CONTENT_E2E_SERVE_BUILD === "1";
+const SERVER_NONCE = runServerNonce(SERVE_BUILD);
 
 export default defineConfig({
   testDir: ".",
@@ -18,6 +23,7 @@ export default defineConfig({
   expect: { timeout: 20_000 },
   reporter: [["list"], ["json", { outputFile: ".report.json" }]],
   globalSetup: "./global-setup.ts",
+  globalTeardown: "./global-teardown.ts",
   use: {
     baseURL: BASE_URL,
     trace: "retain-on-failure",
@@ -30,7 +36,7 @@ export default defineConfig({
       name: "authed",
       use: {
         ...devices["Desktop Chrome"],
-        storageState: ".auth/state.json",
+        storageState: AUTH_FILES.state,
       },
     },
   ],
@@ -38,7 +44,10 @@ export default defineConfig({
     ? {
         command: "node .output/server/index.mjs",
         cwd: path.join(import.meta.dirname, ".."),
-        env: { PORT: new URL(BASE_URL).port || "80" },
+        env: {
+          PORT: new URL(BASE_URL).port || "80",
+          PING_MESSAGE: SERVER_NONCE!,
+        },
         url: `${BASE_URL}/_agent-native/ping`,
         reuseExistingServer: false,
         timeout: 180_000,

package/corpus/templates/content/e2e/server-identity.ts

@@ -0,0 +1,47 @@
+import { randomUUID } from "node:crypto";
+
+import type { APIRequestContext } from "@playwright/test";
+
+/**
+ * The ping message that names this run's server. A run that starts the build
+ * itself makes one up and hands it to the server as PING_MESSAGE. Otherwise
+ * whoever started the server sets the same value there and here. A server
+ * that failed to start leaves its port to whatever else listens on it, and
+ * without this the run would test that server instead.
+ */
+export function runServerNonce(serveBuild: boolean): string | undefined {
+  // The runner hands its environment to every worker it starts, and each
+  // worker loads the config again, so the value lives in the environment.
+  // The runner never keeps one it inherited: that may be an earlier run's,
+  // still answered by a server that run left behind.
+  if (serveBuild && process.env.TEST_WORKER_INDEX === undefined) {
+    // guard:allow-env-mutation — Playwright config, never a server request path.
+    process.env.CONTENT_E2E_SERVER_NONCE = randomUUID();
+  }
+  return process.env.CONTENT_E2E_SERVER_NONCE;
+}
+
+/** Refuse any server whose ping does not answer with this run's nonce. */
+export async function expectRunServer(
+  request: APIRequestContext,
+  baseURL: string,
+): Promise<void> {
+  const nonce = process.env.CONTENT_E2E_SERVER_NONCE;
+  if (!nonce)
+    throw new Error(
+      `${baseURL} is a server this run did not start, so the run cannot tell it from anything else on that port. Start it with PING_MESSAGE=<nonce> and run with CONTENT_E2E_SERVER_NONCE=<the same nonce>, or set CONTENT_E2E_SERVE_BUILD=1 so the run starts the build itself.`,
+    );
+  // A redirect is refused, not followed: the run would accept the answer of
+  // the server it leads to while testing the one that sent it.
+  const response = await request.get(`${baseURL}/_agent-native/ping`, {
+    maxRedirects: 0,
+  });
+  const text = await response.text();
+  const headers = response.headers();
+  const json = (headers["content-type"] ?? "").includes("json");
+  const message = response.ok() && json ? JSON.parse(text).message : undefined;
+  if (message !== nonce)
+    throw new Error(
+      `The server at ${baseURL} is not the one this run names: its ping answered ${response.status()}${headers.location ? ` redirecting to ${headers.location}` : ""} ${JSON.stringify(text.slice(0, 200))}, not the message ${nonce}. Its own server did not start, or another holds the port.`,
+    );
+}

package/corpus/templates/content/e2e/stored-form/collab-blocks.ts

@@ -0,0 +1,123 @@
+import { expect, type Page } from "@playwright/test";
+import * as Y from "yjs";
+
+/** One top-level block of a page's live copy, as the server holds it. */
+export interface CollabBlock {
+  /** The block's Yjs item id; a block rebuilt from text gets a new one. */
+  itemId: string;
+  type: string;
+  text: string;
+}
+
+const FRAGMENT = "default";
+
+function plainText(node: Y.XmlElement | Y.XmlText | Y.XmlHook): string {
+  if (node instanceof Y.XmlText)
+    return node
+      .toDelta()
+      .map((op: { insert?: unknown }) =>
+        typeof op.insert === "string" ? op.insert : "",
+      )
+      .join("");
+  if (node instanceof Y.XmlElement)
+    return node
+      .toArray()
+      .map((child) => plainText(child))
+      .join("");
+  return "";
+}
+
+/**
+ * The server's live copy of a page, block by block. Reading it seeds a page
+ * no browser has opened, so a case that needs an unseeded page reads it only
+ * after the first tab opens.
+ */
+export async function collabBlocks(
+  page: Page,
+  documentId: string,
+): Promise<CollabBlock[]> {
+  const response = await page.request.get(
+    `/_agent-native/collab/${documentId}/state`,
+  );
+  const text = await response.text();
+  expect(response.ok(), `collab state (${response.status()}): ${text}`).toBe(
+    true,
+  );
+  const { state } = JSON.parse(text) as { state?: unknown };
+  if (typeof state !== "string" || !state)
+    throw new Error(
+      `collab state for ${documentId} carried no update: ${text}`,
+    );
+  const doc = new Y.Doc();
+  try {
+    Y.applyUpdate(doc, Buffer.from(state, "base64"));
+    return doc
+      .getXmlFragment(FRAGMENT)
+      .toArray()
+      .map((node) => {
+        const id = node._item?.id;
+        if (!id) throw new Error(`a ${FRAGMENT} block has no Yjs item`);
+        return {
+          itemId: `${id.client}:${id.clock}`,
+          type: node instanceof Y.XmlElement ? node.nodeName : "text",
+          text: plainText(node),
+        };
+      });
+  } finally {
+    doc.destroy();
+  }
+}
+
+/**
+ * The server's live copy once its text holds every needle. An open tab seeds
+ * the live copy a moment after its editor turns ready, so a snapshot taken
+ * before that reads an empty page.
+ */
+export async function liveCopyContaining(
+  page: Page,
+  documentId: string,
+  needles: readonly string[],
+  timeout = 45_000,
+): Promise<CollabBlock[]> {
+  let blocks: CollabBlock[] = [];
+  await expect
+    .poll(
+      async () => {
+        blocks = await collabBlocks(page, documentId);
+        const text = blocks.map((block) => block.text).join("\n");
+        return needles.filter((needle) => !text.includes(needle));
+      },
+      { message: "text missing from the server's live copy", timeout },
+    )
+    .toEqual([]);
+  return blocks;
+}
+
+/**
+ * Require every block whose text is in `untouched` to keep the Yjs item it
+ * had before. A writer that rebuilds the page from text gives every block a
+ * new item, which drops the cursors, comments and suggestions anchored there.
+ */
+export function expectBlocksKept(
+  before: readonly CollabBlock[],
+  after: readonly CollabBlock[],
+  untouched: readonly string[],
+): void {
+  const evidence = JSON.stringify({ before, after });
+  for (const text of untouched) {
+    const was = before.filter((block) => block.text === text);
+    const now = after.filter((block) => block.text === text);
+    expect(
+      was,
+      `the live copy before had one "${text}": ${evidence}`,
+    ).toHaveLength(1);
+    expect(
+      now,
+      `the live copy after has one "${text}": ${evidence}`,
+    ).toHaveLength(1);
+    expect(
+      now[0].itemId,
+      `"${text}" was rebuilt instead of kept: ${evidence}`,
+    ).toBe(was[0].itemId);
+  }
+}

package/corpus/templates/content/package.json

@@ -8,7 +8,7 @@
     "dev:database": "node scripts/dev-database.mjs",
     "build": "agent-native build",
     "start": "agent-native start",
-    "test": "vitest --run",
+    "test": "vitest --run --tags-filter \"!known-loss\"",
     "test:parity": "vitest --run parity",
     "test:parity-capabilities": "vitest --run parity actions/content-database-lifecycle.db.test.ts actions/bind-content-database-source-field.db.test.ts actions/_local-file-documents.test.ts actions/builder-source-review-gates.db.test.ts",
     "eval:parity": "agent-native eval parity",

package/corpus/templates/content/scripts/_stored-form-census-kinds.ts

@@ -420,6 +420,7 @@ type MdNode = {
  */
 type ReadProfile = {
   tokens: Map<string, number>;
+  pendingText: string[];
   backslashes: number;
   letters: string[];
   marks: Record<(typeof MARKS)[number], number[]>;
@@ -450,6 +451,7 @@ const TEXT_ATTRIBUTES = ["label", "summary", "title"] as const;
 function emptyProfile(): ReadProfile {
   return {
     tokens: new Map(),
+    pendingText: [],
     backslashes: 0,
     letters: [],
     marks: Object.fromEntries(
@@ -478,6 +480,7 @@ const PROFILE_COUNTS = [
 
 /** Adds what `part` read, as text that follows `profile`'s so far. */
 function appendProfile(profile: ReadProfile, part: ReadProfile) {
+  flushText(profile);
   const offset = profile.letters.length;
   for (const letter of part.letters) profile.letters.push(letter);
   for (const mark of MARKS)
@@ -496,10 +499,17 @@ function addTokens(profile: ReadProfile, text: string) {
 }
 
 function addText(profile: ReadProfile, text: string) {
-  addTokens(profile, text);
+  profile.pendingText.push(text);
   profile.backslashes += text.split("\\").length - 1;
 }
 
+function flushText(profile: ReadProfile) {
+  if (!profile.pendingText.length) return;
+  // Marks and links split a visible word into nodes without splitting its text.
+  addTokens(profile, profile.pendingText.join(""));
+  profile.pendingText.length = 0;
+}
+
 const gfmReader = unified().use(remarkParse).use(remarkGfm);
 
 // Content's own syntax, which no agent writes as prose: tags such as
@@ -579,7 +589,10 @@ function withoutNfmSyntax(body: string): string {
       if (NFM_TAG_LINE.test(line))
         return line.replace(NFM_TAG_LINE, "$1<!-- -->");
       return outsideCodeSpans(line, (text) =>
-        text.replace(NFM_INLINE_TAG, " "),
+        // Keeping spans as HTML avoids joining the code delimiters around them.
+        text.replace(NFM_INLINE_TAG, (tag) =>
+          /^<\/?span(?:\s|>)/.test(tag) ? tag : " ",
+        ),
       ).replace(NFM_BLOCK_ATTRIBUTES, "$1");
     })
     .join("\n");
@@ -646,6 +659,16 @@ function gfmProfile(body: string): ReadProfile {
   const markdown = withoutNfmSyntax(body);
   const profile = emptyProfile();
   const walk = (node: MdNode, listDepth: number, marks: ReadonlySet<Mark>) => {
+    const block = [
+      "paragraph",
+      "heading",
+      "tableCell",
+      "code",
+      "break",
+      "image",
+      "imageReference",
+    ].includes(node.type);
+    if (block) flushText(profile);
     let inner = marks;
     switch (node.type) {
       case "strong":
@@ -719,8 +742,10 @@ function gfmProfile(body: string): ReadProfile {
     }
     const childDepth = node.type === "list" ? listDepth + 1 : listDepth;
     for (const child of node.children ?? []) walk(child, childDepth, inner);
+    if (block) flushText(profile);
   };
   walk(gfmReader.parse(markdown) as MdNode, 0, new Set());
+  flushText(profile);
   return profile;
 }
 
@@ -745,6 +770,7 @@ function nfmProfile(document: PMNode): ReadProfile {
     }
   };
   const walk = (node: PMNode, listDepth: number) => {
+    if (node.type !== "text") flushText(profile);
     if (typeof node.text === "string") {
       addText(profile, node.text);
       addMarkedText(
@@ -785,8 +811,10 @@ function nfmProfile(document: PMNode): ReadProfile {
     );
     for (const child of children)
       walk(child, isList ? listDepth + 1 : listDepth);
+    if (node.type !== "text") flushText(profile);
   };
   walk(document, 0);
+  flushText(profile);
   return profile;
 }
 

package/corpus/templates/content/scripts/historical-icon-apply.ts

@@ -16,6 +16,7 @@ import {
   lockPrimaryBlocksFields,
   persistBlocksFieldIdentity,
 } from "../actions/_blocks-field-identity.js";
+import { lockDocumentForSave } from "../actions/_document-save-locks.js";
 import { getDb, schema } from "../server/db/index.js";
 import { bodyRevisionForContent } from "../server/lib/document-body-revision.js";
 import { recordDocumentHistoryTransition } from "../server/lib/document-history.js";
@@ -178,6 +179,13 @@ export async function replaceHistoricalIconIfUnchanged(
         scopedOrg(schema.documents.orgId),
         eq(schema.documents.content, reference.contentRaw!),
       );
+      const transaction = tx as unknown as ReturnType<typeof getDb>;
+      // A body save's lock order, memberships before the page.
+      const { lockedMembershipIds } = await lockDocumentForSave(transaction, {
+        documentId: reference.rowId,
+        ownerEmail: owner,
+        body: true,
+      });
       const [current] = await tx
         .select()
         .from(schema.documents)
@@ -189,10 +197,10 @@ export async function replaceHistoricalIconIfUnchanged(
         throw new Error(
           "Local-file callout migration requires its file authority",
         );
-      const transaction = tx as unknown as ReturnType<typeof getDb>;
       const fields = await lockPrimaryBlocksFields(
         transaction,
         reference.rowId,
+        { heldMembershipIds: lockedMembershipIds },
       );
       const now = nextDocumentUpdatedAt(current.updatedAt);
       const changed = await tx