@taprootio/site-authoring 0.19.0 → 0.20.0

package/README.md

@@ -645,6 +645,24 @@ Other C0/C1 controls, U+2028/U+2029, U+FEFF, and literal `<br>` tags are rejecte
 with the exact fact field path. A label is optional and stays outside the value's link. See
 `taproot-site help page free-form` and `taproot-site help component cta`.
 
+### Forms by id in 0.20.0
+
+A page stores the form it places by the form's id, so renaming a form never
+breaks a page, but a workspace still names a form by its key: `forms/<key>.json`
+and the `formKey` of a `component:form` block are unchanged, and the same files
+still push to a fresh site whose ids all differ. `pages push` looks each key up
+on the site, sends the form's id, and refuses a key the site does not have
+(`pages.form_unknown`) or has archived (`pages.form_archived`) before it writes
+anything; `plan` reports the same. `pages pull` writes the key back. An id the
+site no longer has stays an id in the pulled file, and a push of it is refused
+until you choose a form. Form names are now unique on a site, ignoring case.
+The deploy that introduces this rewrites the stored page bodies, which changes the
+body revision of every page that places a form, so the first `pages pull` after it
+can report `pages.pull_conflict` for such a Markdown page even though its content
+is the same. Run `taproot-site pages push <path>` for that page, or delete the file
+and pull again. `forms validate` and `forms push` also refuse two forms with the
+same name (`forms.name_duplicate`, `forms.name_taken`).
+
 ### Place categories in 0.18.0
 
 A place has one category for every site, taken from Google's types, and a place

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@taprootio/site-authoring",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "Taproot site authoring CLI: sign in once in the browser, then pull, validate, push, preview, approve, and deploy a site from the terminal",
   "type": "module",
   "license": "ISC",

package/src/cli.js

@@ -239,7 +239,9 @@ const VERBS = Object.freeze([
       + "and announced on the human channel. New downloads and compared bodies are excluded. "
       + "A page Taproot generates (a tag, an archive, a place, a folder) is pulled as an update-only '.pm.json' "
       + "(template 'generated'): you edit its custom title, breadcrumb title, custom description and introduction, "
-      + "and the manifest keeps the title and description the site reports.",
+      + "and the manifest keeps the title and description the site reports. "
+      + "A form a page places is written back by its key, the name 'forms/<key>.json' uses, though the site stores "
+      + "the form's id.",
   },
   {
     name: VERB_PAGES_META_SET,
@@ -305,6 +307,8 @@ const VERBS = Object.freeze([
     positionals: "pagePaths",
     note:
       "Positional page paths narrow the push to those pages; with none, every workspace page is validated and sent. "
+      + "A page names a form by key (formKey); the push looks each key up on the site, sends the form's id, and "
+      + "refuses a key the site does not have or has archived (pages.form_unknown, pages.form_archived) before writing. "
       + "A link in a page being sent whose address is only http(s):// plus its own text is reported in linkWarnings "
       + "(content.link_autolinked) without refusing. "
       + "A page is created without an author unless its source names one ('author:' in the front matter, or 'pages "
@@ -394,7 +398,7 @@ const VERBS = Object.freeze([
     capabilities: [CAPABILITY_CONTENT, CAPABILITY_DESIGN],
     tokens: ["plan"],
     summary: "Check the whole workspace against the live site and order what is left to send. Writes nothing.",
-    note: "Runs every check the pushes run — each page against the site's pages, revisions and videos; media that "
+    note: "Runs every check the pushes run — each page against the site's pages, revisions, videos and forms; media that "
       + "pages reference and media upload has not sent; the theme, appearance and footer colors against the site's "
       + "presentation revision; the footer against its draft; navigation against the live page list — and lists "
       + "every problem with its area, file, code and field. It then orders the steps that remain: media upload, "

package/src/content/components.js

@@ -352,11 +352,14 @@ const SPACER = Object.freeze({
 
 /**
  * A form defined under Site → Forms (or with `forms push`), placed by key. The
- * key is checked against the site's forms when the page is published, not here:
- * this table only knows the shape.
+ * workspace names a form by its key so the same files push to any site; the site
+ * stores the form's id, and `pages push` and `pages pull` translate between the
+ * two (TR01216). The key is checked against the site's forms when the page is
+ * pushed, not here: this table only knows the shape.
  */
 const FORM = Object.freeze({
-  formKey: str(),
+  // Required: a placement with no form chosen can never publish, so it is refused here, not at deploy.
+  formKey: str({ required: true, nonWhitespace: true }),
 });
 
 function deepFreeze(value) {
@@ -648,9 +651,9 @@ const COMPONENT_DEFINITIONS = Object.freeze({
     FORM,
     "Form",
     "A form from Site → Forms that visitors fill in and submit.",
-    { formKey: "" },
+    {},
     [
-      "Set formKey to the key of an existing form ('forms push' creates it); an unknown or archived key fails the publish.",
+      "Set formKey to the key of an existing form ('forms push' creates it); 'pages push' refuses an unknown or archived key, and the site stores the form's id, not the key.",
       "The button text and what happens after a response is sent belong to the form, not the page: set submit_label and after_submit in the form's file (see 'help forms').",
       "The form's field labels are its accessible names; write each so it makes sense out of context.",
     ],

package/src/form-references.js

@@ -0,0 +1,103 @@
+import { listSiteForms } from "./forms-api.js";
+import { contentDocuments, parseComponentData } from "./typed-pages.js";
+
+/**
+ * Where the workspace and the site spell a placed form differently (TR01216).
+ *
+ * A workspace names a form by its key, as `forms/<key>.json` and the Markdown
+ * `form` fence do, so the same workspace can be pushed to a fresh site whose
+ * ids all differ. A page stored on the site names the form by its id. `pages
+ * push` turns keys into ids on the way out and `pages pull` turns ids back
+ * into keys, the way a form's `page_path` and `page_resource_id` already
+ * translate. The translation is block-scoped: a `formKey` anywhere but a `form`
+ * component block is author prose.
+ */
+
+const FORM_COMPONENT = "form";
+
+/** Each `form` component block in the document's content documents, with its parsed data. */
+function formBlocks(document_) {
+  const found = [];
+  const visit = (value) => {
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+    } else if (value !== null && typeof value === "object") {
+      if (value.type === "componentBlock" && value.attrs?.componentType === FORM_COMPONENT) {
+        const data = typeof value.attrs.componentData === "string"
+          ? parseComponentData(value.attrs.componentData)
+          : undefined;
+        if (data !== null && typeof data === "object" && !Array.isArray(data)) found.push({ attrs: value.attrs, data });
+        return;
+      }
+      Object.values(value).forEach(visit);
+    }
+  };
+  for (const { doc } of contentDocuments(document_)) visit(doc);
+  return found;
+}
+
+/** The distinct form keys a workspace document places, in document order. */
+export function placedFormKeys(document_) {
+  const keys = [];
+  for (const { data } of formBlocks(document_)) {
+    if (typeof data.formKey === "string" && data.formKey !== "" && !keys.includes(data.formKey)) keys.push(data.formKey);
+  }
+  return keys;
+}
+
+/** Whether a document read from the site places any form by id. */
+export function placesFormIds(document_) {
+  return formBlocks(document_).some(({ data }) => typeof data.formId === "string");
+}
+
+/**
+ * A copy of the document with every placed form named by id, for the wire. A key
+ * the index does not know is left as it is: `placedFormProblems` has already
+ * refused the push for it, so nothing sends it.
+ */
+export function withFormIds(document_, idByKey) {
+  return translate(document_, (data) => {
+    const id = typeof data.formKey === "string" ? idByKey.get(data.formKey) : undefined;
+    if (id === undefined) return undefined;
+    const rest = { ...data };
+    delete rest.formKey;
+    return { ...rest, formId: id };
+  });
+}
+
+/**
+ * A copy of a document read from the site with every placed form named by key,
+ * for the workspace. An id the site no longer has stays an id: pull must not hide
+ * what the site holds, and a push of that file is refused until the author picks a form.
+ */
+export function withFormKeys(document_, keyById) {
+  return translate(document_, (data) => {
+    const key = typeof data.formId === "string" ? keyById.get(data.formId) : undefined;
+    if (key === undefined) return undefined;
+    const rest = { ...data };
+    delete rest.formId;
+    return { ...rest, formKey: key };
+  });
+}
+
+function translate(document_, convert) {
+  const copy = structuredClone(document_);
+  for (const { attrs, data } of formBlocks(copy)) {
+    const converted = convert(data);
+    if (converted !== undefined) attrs.componentData = JSON.stringify(converted);
+  }
+  return copy;
+}
+
+/** The site's forms by key and by id, archived ones marked. One request serves a whole push or pull. */
+export async function readFormIndex(client, siteId) {
+  const idByKey = new Map();
+  const keyById = new Map();
+  const archivedKeys = new Set();
+  for (const form of await listSiteForms(client, siteId)) {
+    keyById.set(form.id, form.key);
+    if (form.archived) archivedKeys.add(form.key);
+    else idByKey.set(form.key, form.id);
+  }
+  return { idByKey, keyById, archivedKeys };
+}

package/src/forms-workspace.js

@@ -49,9 +49,33 @@ export async function readWorkspaceForms(workspaceDir, keys, validationOptions =
     const document_ = await readWorkspaceJson(workspaceDir, file, FORM_LIMITS.fileBytes);
     forms.push({ key, file, document: validateFormDocument(document_, key, validationOptions) });
   }
+  requireDistinctNames(forms);
   return forms;
 }
 
+/** A form name as the site compares it: trimmed, ignoring case (TR01216). */
+export function comparableFormName(name) {
+  return String(name).trim().toLowerCase();
+}
+
+/** Names are unique per site, so two files in one push cannot share one. */
+function requireDistinctNames(forms) {
+  const seen = new Map();
+  for (const form of forms) {
+    const name = comparableFormName(form.document.name);
+    const other = seen.get(name);
+    if (other !== undefined) {
+      throw new SiteAuthoringError(
+        "forms.name_duplicate",
+        `${form.file} and ${other} both name the form '${form.document.name}'. A form name is unique on a site, `
+          + "ignoring case; rename one of them.",
+        { field: `${form.file}:name` },
+      );
+    }
+    seen.set(name, form.file);
+  }
+}
+
 /**
  * The forms baseline in the pull manifest (TR01061).
  *

package/src/reference-help.js

@@ -86,7 +86,7 @@ import {
 
 export { getAppearanceReference, getFooterReference, getThemeReference };
 
-export const REFERENCE_VERSION = 36;
+export const REFERENCE_VERSION = 37;
 
 // The date-limit examples `help forms` prints; tests run them through the form validator.
 export const FORM_DATE_WINDOW_FIELD = Object.freeze({
@@ -775,7 +775,9 @@ const WORKFLOW_REFERENCES = Object.freeze({
       + "version of each form this workspace last saw. 'forms validate' needs only taproot-site.json.",
       `The file is { key, name, sink, retention_days, notify_owner, definition }. key must equal the file's name `
       + `and is lowercase letters, digits, and single hyphens, at most ${FORM_KEY_MAXIMUM_LENGTH} characters. name is `
-      + `1 to ${FORM_LIMITS.nameCharacters} characters. sink is one of ${FORM_SINKS.join(", ")} and defaults to `
+      + `1 to ${FORM_LIMITS.nameCharacters} characters and unique on the site, ignoring case: two files, or a file and another form on the site, with the same name `
+      + "are refused (forms.name_duplicate, forms.name_taken) before anything is written. "
+      + `sink is one of ${FORM_SINKS.join(", ")} and defaults to `
       + "'none' (store submissions only); 'newsletter' requests double-opt-in confirmation for the contact address. "
       + "A newsletter sink requires a required email contact field and at least one consent checkbox; every consent checkbox must be checked. "
       + `retention_days is ${FORM_RETENTION_DAYS.minimum} to ${FORM_RETENTION_DAYS.maximum} (a site's plan may cap it lower): `
@@ -846,8 +848,9 @@ const WORKFLOW_REFERENCES = Object.freeze({
       "Submissions are never pulled or pushed. They are visitors' personal data: read, mark, export and delete "
       + "them on the site's Forms page.",
       "To show a form, place the free-form component 'form' on a page (a component:form block) with formKey set "
-      + "to the form's key ('help component form' lists its properties). Push the form first and deploy: the release "
-      + "freezes the form it was built with.",
+      + "to the form's key ('help component form' lists its properties). Push the form first: 'pages push' turns the "
+      + "key into the form's id for the site and refuses a key the site does not have or has archived, and 'pages pull' "
+      + "turns the id back into the key. Then deploy: the release freezes the form it was built with.",
       "The delegation.content capability carries site.forms.manage, which every forms verb needs: it covers "
       + "definitions, settings and status. Visitors' submissions are in no capability: reading them "
       + "(site.forms.submissions.read) and finding, marking or deleting them (site.forms.submissions.manage) belong "

package/src/typed-pages.js

@@ -769,7 +769,7 @@ export const COVER_IMAGE_UNUSED_MESSAGE = "The selected cover image must be used
  * component JSON, so a string that only parses once decoded is read the way the
  * server reads it.
  */
-function parseComponentData(text) {
+export function parseComponentData(text) {
   let decoded = text;
   for (let round = 0; round < 3 && /&(?:quot|amp|#39|lt|gt);/u.test(decoded); round += 1) {
     decoded = decoded.replace(/&quot;/gu, "\"").replace(/&#39;/gu, "'").replace(/&lt;/gu, "<").replace(/&gt;/gu, ">")

package/src/verbs/forms-push.js

@@ -5,11 +5,12 @@ import {
   createSiteForm,
   getPageIndex,
   getSiteFormByKey,
+  listSiteForms,
   saveSiteFormVersion,
   updateSiteFormSettings,
 } from "../forms-api.js";
 import { afterSubmitPage, formFileName, projectFormForWorkspace, storedDefinition, validateNewsletterDefinition } from "../forms-contract.js";
-import { readWorkspaceForms, recordedForm, siteSettingsBaseline, withRecordedForm } from "../forms-workspace.js";
+import { comparableFormName, readWorkspaceForms, recordedForm, siteSettingsBaseline, withRecordedForm } from "../forms-workspace.js";
 import { openSession, successResult, warnIfExternalWritesPaused } from "../session.js";
 import { ApiError } from "../transport.js";
 import { readManifest, writeManifest, writeWorkspaceJson } from "../workspace.js";
@@ -155,6 +156,26 @@ export async function formsPush(invocation) {
     );
   }
 
+  // A form name is unique on the site, ignoring case, so a name another form holds is refused here with
+  // the rest of the checks, not by the first write that reaches it.
+  const naming = plans.filter((plan) => plan.action === "create" || plan.settingsChanged);
+  if (naming.length > 0) {
+    const siteForms = await withRefusalGuidance(onProgress, "forms push", async () => await listSiteForms(client, siteId));
+    for (const plan of naming) {
+      const name = comparableFormName(plan.desired.name);
+      const holder = siteForms.find((siteForm) => siteForm.key !== plan.form.key && comparableFormName(siteForm.name) === name);
+      if (holder !== undefined) {
+        throw new SiteAuthoringError(
+          "forms.name_taken",
+          `${formFileName(plan.form.key)}: the site's form '${holder.key}'${holder.archived ? " (archived)" : ""} already `
+            + `has the name '${plan.desired.name}'. A form name is unique on a site, ignoring case; rename one of them. `
+            + "Nothing was written.",
+          { field: `${formFileName(plan.form.key)}:name` },
+        );
+      }
+    }
+  }
+
   const results = [];
   for (const plan of plans) {
     let { desired } = plan;

package/src/verbs/pages-push.js

@@ -21,6 +21,7 @@ import {
 } from "../authors-contract.js";
 import { VERB_PAGES_PUSH } from "../constants.js";
 import { autolinkWarnings } from "../content/link-warnings.js";
+import { placedFormKeys, readFormIndex, withFormIds } from "../form-references.js";
 import { ApiError } from "../transport.js";
 import { waitForProcessing } from "./media-upload.js";
 import { SiteAuthoringError } from "../errors.js";
@@ -878,6 +879,46 @@ export async function placedVideoProblems(client, siteId, planned) {
   return problems;
 }
 
+/**
+ * The pages that place a form the site does not have, or has archived, and the
+ * site's forms by key for the wire (TR01216). A page is stored with the form's id,
+ * so a key that names no active form cannot be sent at all. Pages that will not be
+ * sent are not asked about, and a push that places no form reads nothing.
+ */
+export async function placedFormProblems(client, siteId, planned) {
+  const placements = new Map();
+  for (const page of planned) {
+    if (page.unchanged) continue;
+    for (const key of placedFormKeys(page.document)) {
+      if (!placements.has(key)) placements.set(key, page.file);
+    }
+  }
+  if (placements.size === 0) return { problems: [], idByKey: new Map() };
+
+  const index = await readFormIndex(client, siteId);
+  const problems = [];
+  for (const [key, file] of placements) {
+    if (index.idByKey.has(key)) continue;
+    problems.push(...problemsOf(
+      index.archivedKeys.has(key)
+        ? documentError(
+          "pages.form_archived",
+          `'${file}' places the form '${key}', which is archived. Restore the form under Site → Forms or remove it `
+            + "from the page.",
+          file,
+        )
+        : documentError(
+          "pages.form_unknown",
+          `'${file}' places the form '${key}', which this site does not have. Run 'taproot-site forms push' to create `
+            + "it, or fix the key.",
+          file,
+        ),
+      { area: CHECK_AREA.pages, file },
+    ));
+  }
+  return { problems, idByKey: index.idByKey };
+}
+
 /**
  * Gives each `video-embed` block that has no poster the provider's thumbnail,
  * copied into the site's images by the server (TR01109). One request per provider
@@ -1565,6 +1606,9 @@ export async function pagesPush(invocation) {
     // push, before anything is written, so a page cannot be sent that
     // publishing would then refuse.
     plan.problems.push(...await placedVideoProblems(client, siteId, planned));
+    // Likewise a form: pages store its id, which only the site's own forms can supply.
+    const placedForms = await placedFormProblems(client, siteId, planned);
+    plan.problems.push(...placedForms.problems);
 
     // Under apply, the workspace read ledger has already held every source and
     // manifest to the plan. What it cannot see is the site: every page the plan
@@ -1634,7 +1678,7 @@ export async function pagesPush(invocation) {
           path: page.pagePath,
           title: page.title,
           shortDescription: page.description,
-          template: wireTemplate(page.document),
+          template: wireTemplate(withFormIds(page.document, placedForms.idByKey)),
           ...wirePageFields(page.document),
           // Only a name the site has not been told: repeating the author a page
           // already has would let a later change of that person's permissions

package/src/verbs/plan.js

@@ -45,6 +45,7 @@ import { navPush, validateNavigationWorkspaceDocument } from "./nav-push.js";
 import {
   liveAuthorsRefresher,
   pagesPush,
+  placedFormProblems,
   placedVideoProblems,
   planPages,
   plannedPageItem,
@@ -283,6 +284,7 @@ async function preflight(session, invocation) {
     });
     problems.push(...pages.problems);
     problems.push(...await placedVideoProblems(client, siteId, pages.planned));
+    problems.push(...(await placedFormProblems(client, siteId, pages.planned)).problems);
     const sending = pages.planned.filter((page) => !page.unchanged);
 
     // An image a page names must be one the site holds; an image this plan
@@ -433,7 +435,7 @@ function planReport(planned) {
     ready: problems.length === 0,
     steps,
     checked: [
-      "every page source, against the live site's pages, revisions, and video library",
+      "every page source, against the live site's pages, revisions, video library, and the forms it places",
       "media under media/ that pages reference and media upload has not sent",
       "the theme, appearance, and footer colors, against the site's presentation revision",
       "the footer document, against the site's footer draft and the pages it still holds",
@@ -441,7 +443,7 @@ function planReport(planned) {
       "navigation shape and page targets, against the live page list and navigation",
     ],
     notChecked: [
-      "redirects and forms: run 'redirects push' and 'forms push' on their own",
+      "redirects, and the forms files themselves: run 'redirects push' and 'forms push' on their own",
       "rendering: preview pages before approving them",
       "approval, staging, and production, which stay separate commands",
     ],

package/src/verbs/pull.js

@@ -33,6 +33,7 @@ import {
   SETTINGS_TYPE_TAPROOT_STYLES,
 } from "../settings-catalog.js";
 import { ApiError } from "../transport.js";
+import { placesFormIds, readFormIndex, withFormKeys } from "../form-references.js";
 import {
   generatedIdentity,
   isAuthorableTemplateType,
@@ -643,6 +644,15 @@ async function requireWorkspaceBelongsToSite(workspaceDir, siteId) {
 
 export async function pull(invocation) {
   const { client, config, siteId, surface, now, onProgress } = await openSession(invocation);
+  // A page names its forms by id on the site and by key in the workspace (TR01216). The
+  // site's forms are read once, and only when a page body places one.
+  let formKeys;
+  const workspaceBodyFromPage = async (page, listedTemplateType) => {
+    const body = workspaceDocumentFromPage(page, listedTemplateType);
+    if (body === undefined || !placesFormIds(body)) return body;
+    formKeys ??= readFormIndex(client, siteId).then((index) => index.keyById);
+    return withFormKeys(body, await formKeys);
+  };
   await ensureWorkspaceRoot(config);
 
   await requireWorkspaceBelongsToSite(config.workspaceDir, siteId);
@@ -891,7 +901,7 @@ export async function pull(invocation) {
       if (source === undefined) continue;
       onProgress(`Reading page '${summary.path}'.`);
       const page = await getPage(client, summary.pageId, bodyStatusFor(summary));
-      const body = workspaceDocumentFromPage(page, summary.templateType);
+      const body = await workspaceBodyFromPage(page, summary.templateType);
       const description = typeof page.shortDescription === "string" ? page.shortDescription : "";
       if (body === undefined) {
         // The site's body is unreadable, but the workspace file is still this
@@ -1261,7 +1271,7 @@ export async function pull(invocation) {
         }
         onProgress(`Reading page '${summary.path}'.`);
         const page = await getPage(client, summary.pageId, bodyStatusFor(summary));
-        const body = workspaceDocumentFromPage(page, summary.templateType);
+        const body = await workspaceBodyFromPage(page, summary.templateType);
         entry.description = typeof page.shortDescription === "string" ? page.shortDescription : "";
         entry.title = titleFromRead(page, summary);
         entry.path = pathFromRead(page, summary);