@taprootio/site-authoring 0.12.6 → 0.13.0

package/src/reference-help.js

@@ -20,6 +20,27 @@ import {
   SHIPPED_FIXTURE_NAME,
   shippedFixtureDirectory,
 } from "./fixture-contract.js";
+import {
+  DATE_ANCHORS,
+  FIELD_TYPES,
+  MAX_AFTER_SUBMIT_MESSAGE_LENGTH,
+  MAX_CONSENT_LABEL_LENGTH,
+  MAX_DATE_OFFSET_DAYS,
+  MAX_FIELDS,
+  MAX_HINT_LENGTH,
+  MAX_SUBMIT_LABEL_LENGTH,
+  MAX_LABEL_LENGTH,
+  MAX_SUBMISSION_BYTES,
+  RELATIVE_BOUND_GRACE_DAYS,
+} from "./field-validation.js";
+import {
+  FORM_FILE_EXTENSION,
+  FORM_KEY_MAXIMUM_LENGTH,
+  FORM_LIMITS,
+  FORM_RETENTION_DAYS,
+  FORM_SINKS,
+  FORMS_DIRECTORY,
+} from "./forms-contract.js";
 import {
   formatPresentationReference,
   getAppearanceReference,
@@ -50,7 +71,23 @@ import {
 
 export { getAppearanceReference, getFooterReference, getThemeReference };
 
-export const REFERENCE_VERSION = 28;
+export const REFERENCE_VERSION = 32;
+
+// The date-limit examples `help forms` prints; tests run them through the form validator.
+export const FORM_DATE_WINDOW_FIELD = Object.freeze({
+  id: "visit_day",
+  type: "date",
+  label: "Day of your visit",
+  min: Object.freeze({ from: "today" }),
+  max: Object.freeze({ from: "today", offset_days: 30 }),
+});
+export const FORM_NEXT_MONTH_FIELD = Object.freeze({
+  id: "start_day",
+  type: "date",
+  label: "A day next month",
+  min: Object.freeze({ from: "start_of_next_month" }),
+  max: Object.freeze({ from: "end_of_next_month" }),
+});
 export const PAGE_TYPES = Object.freeze(["free-form", ...TYPED_PAGE_TYPES]);
 
 function deepFreeze(value) {
@@ -363,6 +400,11 @@ export const REFERENCE_TOPICS = Object.freeze([
     usage: `${CLI_BINARY_NAME} help redirects`,
     summary: "Author redirects.json: entry shape, normalization, and the refusals.",
   }),
+  Object.freeze({
+    name: "forms",
+    usage: `${CLI_BINARY_NAME} help forms`,
+    summary: "Author a form as forms/<key>.json: file shape, field types, validate, pull, push, and placing it on a page.",
+  }),
   Object.freeze({
     name: "media",
     usage: `${CLI_BINARY_NAME} help media`,
@@ -526,7 +568,10 @@ const WORKFLOW_REFERENCES = Object.freeze({
       `Pull the baseline: ${CLI_BINARY_NAME} pull writes settings/taproot-styles.json with both schemes' complete effective `
       + "themes (the stored theme resolved over the same defaults every consumer renders) beside brand.json, "
       + "site-header.json and site-publishing-preferences.json. Keep that pull as the baseline you edit; never build "
-      + "a theme from memory or from an example. semanticMappings holds only authored pins, listed in explicitMappingTokens.",
+      + "a theme from memory or from an example. semanticMappings holds only authored pins, listed in explicitMappingTokens. "
+      + "site-publishing-preferences.json also carries timeZone, the site's IANA zone (UTC until the owner sets it), "
+      + "which decides what \"today\" means for relative form date bounds; it is read-only here, so ask the owner to set "
+      + "it in the editor under Site settings.",
       "Edit both schemes: settings.lightTheme and settings.darkTheme are separate documents. Declare anchors, then roles "
       + "(help theme lists the role slots and how anchor references resolve), tune typography per scheme, and keep "
       + "contexts and intents paired across schemes. Set appearance scalars (default scheme, header, logos, favicon) "
@@ -681,9 +726,110 @@ const WORKFLOW_REFERENCES = Object.freeze({
       ]),
     }),
   }),
+  forms: Object.freeze({
+    title: "Form workspace contract",
+    summary: `A form is one JSON file, ${FORMS_DIRECTORY}/<key>${FORM_FILE_EXTENSION}. 'forms validate' proves it offline, `
+      + "'forms push' creates or updates it on the site, and 'forms pull' reads the site's copy back.",
+    usage: `${CLI_BINARY_NAME} forms validate [key...] | ${CLI_BINARY_NAME} forms pull | ${CLI_BINARY_NAME} forms push [key...]`,
+    details: Object.freeze([
+      `Run 'pull' once first: 'forms pull' and 'forms push' need the pulled workspace's manifest, which records the `
+      + "version of each form this workspace last saw. 'forms validate' needs only taproot-site.json.",
+      `The file is { key, name, sink, retention_days, definition }. key must equal the file's name `
+      + `and is lowercase letters, digits, and single hyphens, at most ${FORM_KEY_MAXIMUM_LENGTH} characters. name is `
+      + `1 to ${FORM_LIMITS.nameCharacters} characters. sink is one of ${FORM_SINKS.join(", ")} and defaults to `
+      + "'none' (store submissions only); 'newsletter' also subscribes the contact address to the site's newsletter. "
+      + `retention_days is ${FORM_RETENTION_DAYS.minimum} to ${FORM_RETENTION_DAYS.maximum} (a site's plan may cap it lower): `
+      + "submissions are deleted that many days after they arrive. Leave it out and the site chooses on creation (its "
+      + "plan's default, usually 365) and keeps the current value on an update. Shortening it deletes the submissions that fall outside it, "
+      + "so a key cannot do it; the site refuses it, and an owner or admin must make that change.",
+      `definition is { fields, contact_field?, submit_label?, after_submit? } (contact_field only with several email fields). Each field is { id, type, label, hint?, ...constraints }; the closed `
+      + `field types are ${FIELD_TYPES.join(", ")}. hint is optional helper text of up to ${MAX_HINT_LENGTH} `
+      + "characters, drawn under the field on the published form; it is the only author-defined text printed there, so a required "
+      + "field is not marked 'Required' and a visitor learns it from the error on submit. "
+      + `At most ${MAX_FIELDS} fields and ${MAX_SUBMISSION_BYTES} bytes `
+      + "per submission. There are no author-supplied regular expressions: use 'format' from its closed list. "
+      + "The exact constraints per type are in shared/field-schema.json, which 'forms validate' enforces by "
+      + "running the same validator the published form and the site use. Each refusal names the field path and a "
+      + "stable code such as duplicate_id or unknown_constraint.",
+      "A date field's min and max are each a fixed 'YYYY-MM-DD' date or a relative limit, { from, offset_days? }. "
+      + `from is one of ${DATE_ANCHORS.join(", ")}, the site's calendar date today or that day of this or next month; `
+      + `offset_days is a whole number of days from -${MAX_DATE_OFFSET_DAYS} to ${MAX_DATE_OFFSET_DAYS} (default 0; a `
+      + "year either way covers a leap year, and further out is a fixed date). There is no expression language. "
+      + "'Today' is the site's date in the site's time zone, which is UTC until the owner sets one in the editor's "
+      + "Site settings; 'pull' shows it as timeZone in settings/site-publishing-preferences.json, read-only here. "
+      + "The published form sets the date picker's limits when the page loads and checks them again when the visitor "
+      + `submits; the site lets a relative minimum reach back ${RELATIVE_BOUND_GRACE_DAYS} day so a visitor who loaded `
+      + "the page before midnight and submits after it is not refused for a date the page offered (a relative maximum "
+      + `and a fixed date are never loosened). Within the next 30 days: ${JSON.stringify(FORM_DATE_WINDOW_FIELD)}. `
+      + `A day next month: ${JSON.stringify(FORM_NEXT_MONTH_FIELD)}. A fixed and a relative limit may be mixed. 'forms `
+      + "validate' refuses an unknown anchor or an offset out of range as invalid_constraint, and a minimum after the "
+      + "maximum as min_exceeds_max only when both are fixed dates or both start from the same anchor; other pairs "
+      + "depend on the day and are not compared.",
+      `submit_label is the text on the submit button, up to ${MAX_SUBMIT_LABEL_LENGTH} characters; leave it out for "Send". `
+      + "after_submit says what the visitor sees once a response is sent. Leave it out, or write "
+      + `{ "show": "message" }, for Taproot's own confirmation; { "show": "message", "message": "..." } replaces `
+      + `its words (up to ${MAX_AFTER_SUBMIT_MESSAGE_LENGTH} characters, line breaks kept). { "show": "page", `
+      + `"page_path": "/thanks" } sends the visitor to that page on this site instead. The page is written as `
+      + "its path: a leading slash, no trailing slash, '/' for the home page. 'forms push' looks the page up "
+      + "and stores it by its resource id, so the form keeps pointing at the page if it is renamed, and 'forms "
+      + "pull' writes the page's current path back. A page_resource_id is accepted in place of page_path, and a "
+      + "pull keeps it only for a page that no longer exists. A file whose page_path names no page on the site is "
+      + "refused as forms.page_not_found, so push the page first. It need not be published to save the form, but "
+      + "a deploy whose release does not publish that page is refused, naming the form and the page. The path in "
+      + "each release is the one the page has in that release, so the published form never asks the site where "
+      + "to go. The page must be one you created, the home page included: a generated page (folder, tag or "
+      + "archive index), the 404 page, a legal page, a profile home and an integration-managed page are refused as forms.page_not_allowed. The button text and the confirmation belong to the form, not to the page that places it: "
+      + "the 'form' component takes only formKey.",
+      "A form's contact address is the value of one email field, stored with each submission. That is what lets the "
+      + "owner find and delete every row for a visitor who asks. With exactly one email field, that field is the "
+      + "contact address and the file leaves contact_field out (a file that names it anyway is accepted, and a pull "
+      + "writes it without). With two or more email fields, contact_field must name one of them; leaving it out is "
+      + "refused as contact_field_required. A form with no email field has no contact address, and contact_field "
+      + "there is refused as invalid_contact_field.",
+      `A consent field's label is the exact words the visitor agrees to, up to ${MAX_CONSENT_LABEL_LENGTH} characters `
+      + `(every other field's label is at most ${MAX_LABEL_LENGTH}); a form has no separate consent text. `
+      + "Each version freezes the labels, and every submission records the hash of each consent field's label as "
+      + "the visitor saw it, so a response shows what each box agreed to.",
+      "Editing a definition appends a version on the site; versions are never changed. A page keeps the form "
+      + "version it was published with until the next deploy, and a rollback restores the form it shipped with.",
+      "Both verbs write the file in one canonical key order, so a pulled file pushed back unchanged reports "
+      + "'unchanged' and leaves the bytes as they were. Write your own files in that order and a push rewrites "
+      + "nothing you did not change.",
+      "'forms push' validates every file and compares every form with the site before the first write. It refuses "
+      + "forms.concurrent_modification when the site's version moved since this workspace last read it, "
+      + "forms.pull_required when the site has a form with that key this workspace never pulled, and forms.archived "
+      + "when the form is archived on the site. Reconcile with 'forms pull' and retry; nothing was written.",
+      "Submissions are never pulled or pushed. They are visitors' personal data: read, mark, export and delete "
+      + "them on the site's Forms page.",
+      "To show a form, place the free-form component 'form' on a page (a component:form block) with formKey set "
+      + "to the form's key ('help component form' lists its properties). Push the form first and deploy: the release "
+      + "freezes the form it was built with.",
+      "The delegation.content capability carries site.forms.manage, which every forms verb needs: it covers "
+      + "definitions, settings and status. Visitors' submissions are in no capability: reading them "
+      + "(site.forms.submissions.read) and finding, marking or deleting them (site.forms.submissions.manage) belong "
+      + "to the site's owners and admins on the site's Forms page, never to a key. A refusal names the missing "
+      + "permission.",
+    ]),
+    example: Object.freeze({
+      key: "contact",
+      name: "Contact",
+      sink: "none",
+      retention_days: 365,
+      definition: Object.freeze({
+        fields: Object.freeze([
+          Object.freeze({ id: "name", type: "text", label: "Your name", required: true, max_length: 120 }),
+          Object.freeze({ id: "email", type: "email", label: "Email", required: true }),
+          Object.freeze({ id: "message", type: "long_text", label: "Message", required: true, max_length: 2000 }),
+          Object.freeze({ id: "agree", type: "consent", label: "I agree that you may store this message and reply to me." }),
+        ]),
+        submit_label: "Send message",
+        after_submit: Object.freeze({ show: "page", page_path: "/thanks" }),
+      }),
+    }),
+  }),
   media: Object.freeze({
     title: "Media upload contract",
-    summary: "Upload workspace-root-relative raster files and receive component-ready media objects.",
+    summary: "Upload workspace-root-relative raster files and videos and receive component-ready media objects and video ids.",
     usage: `${CLI_BINARY_NAME} media upload [path...]`,
     details: Object.freeze([
       "Paths are relative to the configured workspace root, not the shell's current directory.",
@@ -692,6 +838,8 @@ const WORKFLOW_REFERENCES = Object.freeze({
       "Prepare variants locally with any image tool before uploading: a transparent logo, its @2x copy, a scheme-specific dark logo when one asset does not read on both headers, and a square favicon. The CLI uploads what you give it; it has no recolor, crop, or SVG upload command.",
       "Assign both schemes after uploading: lightLogoId and darkLogoId in settings/taproot-styles.json and brand.faviconId in settings/brand.json take the returned image ids; theme push writes them. See help walkthrough for the full design pass.",
       "Each result item includes media: { imageId, src, urls, width, height, alt }.",
+      "MP4, MOV, and WebM files are uploaded as videos (up to 250 MB each; the site's licence sets the cap, and the container is read from the file's bytes, not its name). The CLI reads each file with mediabunny. A file that is H.264 and AAC but not already an MP4 with its index first is rewritten as one without re-encoding; anything else is sent as it is and judged by the server. When the server refuses a file, its one-line message is printed as media.video_refused. A video is ready to play as soon as the upload is confirmed. The CLI sets no poster, because Node has no video decoder: the Videos page picks one automatically for a video uploaded there, and lets the owner choose another frame, so set a CLI-uploaded video's poster there. The command reports videos.items: { file, videoId, contentType, byteLength, deduplicated, durationMilliseconds, component }, where component.markdown is the component:video fence and component.block the componentBlock node that place it. Needs the site's licence to include video.",
+      "Place an uploaded video on a page with a video component: a component:video fence in Markdown or a componentBlock of type video, with the returned videoId and optional poster, caption, and aspectRatio. See help component video. pages push refuses a video that is not in the site's library; validate checks it against fixture.videoIds.",
       "The same src/urls delivery fields are saved in .taproot-site-media.json for page and component authoring.",
     ]),
     example: Object.freeze({
@@ -746,6 +894,9 @@ const WORKFLOW_REFERENCES = Object.freeze({
       `Required root fields: ${FIXTURE_REQUIRED_ROOT_FIELDS.join(", ")}.`,
       `Optional root fields: ${FIXTURE_OPTIONAL_ROOT_FIELDS.join(", ")}. Appearance and footer metadata, when present, `
       + "must match the local settings. Snapshot times and deployments are not live evidence. Other root fields are refused.",
+      `The optional forms root field is { items: { <key>: { id, version } } }: 'forms pull' also records a settings baseline, which a fixture does not carry. Each key `
+      + `names a ${FORMS_DIRECTORY}/<key>${FORM_FILE_EXTENSION} that must validate as in 'help forms', and no form file `
+      + "may sit beside them unbound.",
       `manifestVersion must be ${MANIFEST_VERSION}; fixture.contractVersion must be ${FIXTURE_CONTRACT_VERSION}.`,
       "siteId, every pageId and resourceId, every settings entityId, and every fixture.imageIds entry is a canonical "
       + "lowercase UUID. They are deterministic fixture identities, not identities from a live site.",
@@ -776,7 +927,9 @@ const WORKFLOW_REFERENCES = Object.freeze({
         SETTINGS_GROUPS.map((group) => `${group.settingsType} to '${SETTINGS_DIRECTORY}/${group.file}'`).join("; ")
       }.`,
       `The fixture block carries ${FIXTURE_METADATA_FIELDS.join(", ")}. Every image a page references by imageId must `
-      + "be listed in fixture.imageIds, and every absolute delivery URL a page uses must sit on a declared origin.",
+      + "be listed in fixture.imageIds, and every absolute delivery URL a page uses must sit on a declared origin. "
+      + "videoIds is optional: list the ready videos a page may place with a video component, and validate refuses a "
+      + "video component whose videoId is not listed. A live push makes the same check against the site's video library.",
       `Each deliveryOrigins entry is an origin-only HTTPS URL (no path, query, fragment, or credentials) on `
       + `${FIXTURE_DELIVERY_ORIGIN_DOMAIN} or a subdomain of it, at most ${FIXTURE_MAXIMUM_DELIVERY_ORIGINS} of them, `
       + "with no duplicates. Fixtures are copied and shipped, so a real delivery host in one would be a live "
@@ -876,6 +1029,12 @@ export function getComponentReference(type) {
     accessibility: definition.accessibility,
     example,
     markdownExample,
+    ...(definition.authoring
+      ? {
+        authoringNotes: definition.authoring.notes,
+        authoringMarkdownExample: definition.authoring.markdownExample,
+      }
+      : {}),
     componentBlockExample: {
       type: "componentBlock",
       attrs: {
@@ -1692,7 +1851,11 @@ export function formatReferenceResult(result) {
         component.accessibility.map((note) => `  - ${note}`).join("\n")
       }\n\nValid componentData object:\n${
         JSON.stringify(component.example, null, 2)
-      }\n\nMarkdown component fence:\n${component.markdownExample}\n\nRemember: componentData stores JSON.stringify(the object), not the object itself. Use --json for the exact nested schema and a complete componentBlock example.\n`;
+      }\n\nMarkdown component fence:\n${component.markdownExample}${
+        component.authoringNotes === undefined
+          ? ""
+          : `\n\nAuthoring by link:\n${component.authoringNotes.map((note) => `  - ${note}`).join("\n")}\n${component.authoringMarkdownExample}`
+      }\n\nRemember: componentData stores JSON.stringify(the object), not the object itself. Use --json for the exact nested schema and a complete componentBlock example.\n`;
     }
     case "workflow": {
       const reference = result.reference;

package/src/session.js

@@ -373,7 +373,28 @@ export async function openSession(invocation = {}) {
     // Deliberately swallowed; see above.
   }
 
-  const client = new SiteApiClient(buildClientOptions(invocation, apiBaseUrl, exchanged.key));
+  // The exchanged credential lasts about an hour. A long operation (a video
+  // upload and its encoding) re-exchanges the same stored sign-in for the same
+  // site and capabilities shortly before it lapses, rather than failing midway
+  // with the work already done.
+  const client = new SiteApiClient({
+    ...buildClientOptions(invocation, apiBaseUrl, exchanged.key),
+    credentialRenewal: {
+      expiresAtMilliseconds: Date.parse(exchanged.expiresAt),
+      renew: async () => {
+        const renewed = await withRefusalGuidance(
+          onProgress,
+          "sign-in token exchange",
+          async () =>
+            await exchangeSiteAuthoringToken(accountClient, {
+              siteId: config.siteId,
+              capabilities: narrowedCapabilities(invocation, config.authoringSurface),
+            }),
+        );
+        return { key: renewed.key, expiresAtMilliseconds: Date.parse(renewed.expiresAt) };
+      },
+    },
+  });
   return {
     config,
     client,

package/src/settings-catalog.js

@@ -116,6 +116,9 @@ export const SETTINGS_GROUPS = Object.freeze([
       // exist is worse than one that omits the field.
       { name: "commentsMode", wireType: "enum", zero: "SITE_COMMENTS_MODE_UNSPECIFIED" },
       ...stringFields("tiptapImagePlacement"),
+      // One IANA zone id (TR01091). The server answers UTC for a site that
+      // never set one, so a pulled snapshot is never empty.
+      ...stringFields("timeZone"),
       { name: "allowSearch", wireType: "boolean" },
       { name: "showFollowButton", wireType: "boolean" },
       { name: "albumSeamless", wireType: "boolean" },

package/src/transport.js

@@ -22,6 +22,8 @@ import {
   REFUSAL_UNCLASSIFIED,
   ROLLOUT_REFUSAL_FIELD,
 } from "./constants.js";
+import { Readable } from "node:stream";
+
 import { validateApiBaseUrl } from "./config.js";
 import { hasControlCharacter, sanitizeDiagnostic, SiteAuthoringError } from "./errors.js";
 
@@ -43,6 +45,9 @@ const ALLOWED_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE"]);
 // `REPLACEABLE_POST_PATHS` below.
 const IDEMPOTENT_METHODS = new Set(["GET"]);
 
+/** An exchanged credential is renewed once it is this close to expiring. */
+const CREDENTIAL_RENEWAL_MARGIN_MILLISECONDS = 5 * 60_000;
+
 const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
 const RETRYABLE_RESPONSE_ERRORS = new Set([
   "transport.invalid_json",
@@ -611,9 +616,20 @@ export class SiteApiClient {
    */
   #authorization;
 
+  /**
+   * How an exchanged credential is replaced before it expires, or `undefined`
+   * for a credential that is not exchanged (a key in the environment). The
+   * exchanged credential lives about an hour, and a video upload plus its
+   * encoding can outlast that.
+   */
+  #renewCredential;
+  #credentialExpiresAtMilliseconds;
+  #renewing;
+
   constructor({
     apiBaseUrl,
     token,
+    credentialRenewal,
     fetch = globalThis.fetch,
     sleep = defaultSleep,
     signal,
@@ -643,6 +659,12 @@ export class SiteApiClient {
     this.apiPathPrefix = new URL(this.apiBaseUrl).pathname;
     this.anonymous = anonymous;
     this.#authorization = anonymous ? undefined : `Bearer ${token}`;
+    // A renewal whose first expiry is not a real time could never say when it
+    // is due, and would exchange on every request, so it is not installed.
+    if (!anonymous && credentialRenewal !== undefined && Number.isFinite(credentialRenewal.expiresAtMilliseconds)) {
+      this.#renewCredential = credentialRenewal.renew;
+      this.#credentialExpiresAtMilliseconds = credentialRenewal.expiresAtMilliseconds;
+    }
     this.fetch = fetch;
     this.sleep = sleep;
     this.signal = signal;
@@ -665,6 +687,42 @@ export class SiteApiClient {
     return await this.#request(path, options, false);
   }
 
+  /**
+   * Replaces an exchanged credential that is within five minutes of expiring,
+   * once however many requests ask at the same time, before the request that
+   * would have carried the old one. A failed renewal is the request's failure:
+   * sending a credential known to be expiring would only be refused later.
+   */
+  async #renewCredentialIfDue() {
+    if (
+      this.#renewCredential === undefined
+      || this.now() < this.#credentialExpiresAtMilliseconds - CREDENTIAL_RENEWAL_MARGIN_MILLISECONDS
+    ) return;
+    this.#renewing ??= (async () => {
+      try {
+        const renewed = await this.#renewCredential();
+        if (!isWellFormedCredential(renewed?.key) || !Number.isFinite(renewed?.expiresAtMilliseconds)) {
+          throw new SiteAuthoringError(
+            "auth.renewal_invalid",
+            "Taproot returned an invalid renewed credential.",
+          );
+        }
+        this.#authorization = `Bearer ${renewed.key}`;
+        this.#credentialExpiresAtMilliseconds = renewed.expiresAtMilliseconds;
+        // A credential that is due again the moment it arrives (a lifetime
+        // shorter than the margin, or a clock far ahead of the server's) would
+        // be exchanged for on every request, each minting a key. It is used as
+        // it is, and the server's own refusal speaks if it has truly lapsed.
+        if (renewed.expiresAtMilliseconds - CREDENTIAL_RENEWAL_MARGIN_MILLISECONDS <= this.now()) {
+          this.#renewCredential = undefined;
+        }
+      } finally {
+        this.#renewing = undefined;
+      }
+    })();
+    await this.#renewing;
+  }
+
   /**
    * Replays only the POST shapes whose server contract answers a replay instead
    * of duplicating work — see `REPLACEABLE_POST_PATHS`. This is deliberately not
@@ -711,6 +769,7 @@ export class SiteApiClient {
     }
     const replayable = IDEMPOTENT_METHODS.has(method) || replaceableMutation;
     const serializedBody = body === undefined ? undefined : JSON.stringify(body);
+    await this.#renewCredentialIfDue();
     // Whether any attempt of a mutation has ended without an authoritative
     // outcome. Accumulated, not read off the final attempt: a claim whose first
     // send committed the mint and died on the reply, and whose retry then failed
@@ -843,10 +902,19 @@ export class SiteApiClient {
    * shape that may be replayed.
    */
   async upload({ url: uploadUrl, requiredHeaders } = {}, bodyBytes) {
+    // A video original is streamed from disk: `{ byteLength, stream, timeoutMilliseconds }`,
+    // where `stream()` returns a fresh readable of exactly `byteLength` bytes so a
+    // retried attempt reads the file again from the start.
+    const streamed = bodyBytes !== null
+      && typeof bodyBytes === "object"
+      && !ArrayBuffer.isView(bodyBytes)
+      && typeof bodyBytes.stream === "function"
+      && Number.isSafeInteger(bodyBytes.byteLength)
+      && bodyBytes.byteLength >= 0;
     if (
       typeof uploadUrl !== "string"
       || uploadUrl.length === 0
-      || !ArrayBuffer.isView(bodyBytes)
+      || !(streamed || ArrayBuffer.isView(bodyBytes))
     ) {
       throw new SiteAuthoringError("upload.contract_invalid", "Taproot returned an invalid upload capability.");
     }
@@ -920,9 +988,15 @@ export class SiteApiClient {
         response = await this.fetch(url, {
           method: "PUT",
           redirect: "error",
-          signal: operationSignal(this.signal, LIMITS.uploadMilliseconds, this.timeoutSignal),
+          signal: operationSignal(
+            this.signal,
+            (streamed ? bodyBytes.timeoutMilliseconds : undefined) ?? LIMITS.uploadMilliseconds,
+            this.timeoutSignal,
+          ),
           headers,
-          body: bodyBytes,
+          ...(streamed
+            ? { body: Readable.toWeb(bodyBytes.stream()), duplex: "half" }
+            : { body: bodyBytes }),
         });
       } catch {
         if (attempt + 1 < LIMITS.uploadAttempts && !this.signal?.aborted) {

package/src/typed-pages.js

@@ -1,4 +1,5 @@
 import { FREE_FORM_TEMPLATE_VERSION, TEMPLATE_TYPE_FREE_FORM } from "./api.js";
+import { isVideoEmbedId } from "./content/video-embed-url.js";
 import { SiteAuthoringError } from "./errors.js";
 
 /**
@@ -596,6 +597,26 @@ export const COVER_IMAGE_UNUSED_MESSAGE = "The selected cover image must be used
  * wrongly thinks unused would be refused for nothing, and the server has the
  * final say.
  */
+/**
+ * The parsed JSON of a `componentData` string, or `undefined` when it is not
+ * JSON. The server decodes HTML entities (up to three rounds) before reading
+ * component JSON, so a string that only parses once decoded is read the way the
+ * server reads it.
+ */
+function parseComponentData(text) {
+  let decoded = text;
+  for (let round = 0; round < 3 && /&(?:quot|amp|#39|lt|gt);/u.test(decoded); round += 1) {
+    decoded = decoded.replace(/&quot;/gu, "\"").replace(/&#39;/gu, "'").replace(/&lt;/gu, "<").replace(/&gt;/gu, ">")
+      .replace(/&amp;/gu, "&");
+  }
+  for (const candidate of [text, decoded]) {
+    try {
+      return JSON.parse(candidate);
+    } catch { /* not JSON: try the decoded form, then nothing to read */ }
+  }
+  return undefined;
+}
+
 export function usedImageIds(document_) {
   const found = new Set();
   const visit = (value) => {
@@ -607,19 +628,8 @@ export function usedImageIds(document_) {
         else if (key === "imageIds" && Array.isArray(child)) {
           child.filter((id) => typeof id === "string").forEach((id) => found.add(id.toLowerCase()));
         } else if (key === "componentData" && typeof child === "string") {
-          // The server decodes HTML entities (up to three rounds) before reading component JSON.
-          let text = child;
-          for (let round = 0; round < 3 && /&(?:quot|amp|#39|lt|gt);/u.test(text); round += 1) {
-            text = text.replace(/&quot;/gu, "\"").replace(/&#39;/gu, "'").replace(/&lt;/gu, "<").replace(/&gt;/gu, ">")
-              .replace(/&amp;/gu, "&");
-          }
-          try {
-            visit(JSON.parse(child));
-          } catch {
-            try {
-              visit(JSON.parse(text));
-            } catch { /* not JSON: nothing to read */ }
-          }
+          const parsed = parseComponentData(child);
+          if (parsed !== undefined) visit(parsed);
         } else visit(child);
       }
     }
@@ -629,6 +639,66 @@ export function usedImageIds(document_) {
   return found;
 }
 
+/**
+ * Every video id the page places: the `videoId` of each `video` component block
+ * in any of its documents. Block-scoped on purpose — a `videoId` anywhere else
+ * is author prose, not a reference the publisher resolves.
+ */
+export function placedVideoIds(document_) {
+  const found = new Set();
+  const visit = (value) => {
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+    } else if (isPlainObject(value)) {
+      if (value.type === "componentBlock" && isPlainObject(value.attrs) && value.attrs.componentType === "video") {
+        const data = typeof value.attrs.componentData === "string"
+          ? parseComponentData(value.attrs.componentData)
+          : undefined;
+        if (isPlainObject(data) && typeof data.videoId === "string") found.add(data.videoId.toLowerCase());
+        return;
+      }
+      Object.values(value).forEach(visit);
+    }
+  };
+  for (const { doc } of contentDocuments(document_)) visit(doc);
+  return found;
+}
+
+/**
+ * The `video-embed` blocks in any of the page's documents that name a video and
+ * hold no poster, each with a way to set one. `pages push` copies the provider's
+ * thumbnail into the site's images for these (TR01109). Block-scoped like
+ * `placedVideoIds`.
+ */
+export function posterlessVideoEmbeds(document_) {
+  const found = [];
+  const visit = (value) => {
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+    } else if (isPlainObject(value)) {
+      if (value.type === "componentBlock" && isPlainObject(value.attrs) && value.attrs.componentType === "video-embed") {
+        const data = typeof value.attrs.componentData === "string"
+          ? parseComponentData(value.attrs.componentData)
+          : undefined;
+        if (isPlainObject(data) && (data.poster === undefined || data.poster === null)
+          && isVideoEmbedId(data.provider, data.videoId)) {
+          found.push({
+            provider: data.provider,
+            videoId: data.videoId,
+            setPoster: (poster) => {
+              value.attrs.componentData = JSON.stringify({ ...data, poster });
+            },
+          });
+        }
+        return;
+      }
+      Object.values(value).forEach(visit);
+    }
+  };
+  for (const { doc } of contentDocuments(document_)) visit(doc);
+  return found;
+}
+
 /** The selected cover image when the page does not use it, else `undefined`. */
 export function unusedCoverImageId(document_) {
   if (documentTemplate(document_) === TEMPLATE_FREE_FORM) return undefined;

package/src/verbs/forms-pull.js

@@ -0,0 +1,67 @@
+import { withRefusalGuidance } from "../api.js";
+import { VERB_FORMS_PULL } from "../constants.js";
+import { getPageIndex, getSiteForm, listSiteForms } from "../forms-api.js";
+import { afterSubmitPage, formFileName, projectFormForWorkspace } from "../forms-contract.js";
+import { formsManifestEntry, siteSettingsBaseline } from "../forms-workspace.js";
+import { boundedList, openSession, successResult } from "../session.js";
+import { readManifest, writeManifest, writeWorkspaceJson } from "../workspace.js";
+
+const MAXIMUM_REPORTED = 200;
+
+/**
+ * `forms pull` — write one file per live form and record the versions read.
+ *
+ * It overwrites a form's file with the site's state, so a local edit that was
+ * never pushed is replaced; that is what pull means everywhere else in this
+ * CLI. An archived form is reported and not written, because `forms push`
+ * refuses an archived form and a file that can never be pushed is noise.
+ */
+export async function formsPull(invocation) {
+  const { client, config, siteId, onProgress } = await openSession(invocation);
+  // The manifest is the pulled workspace this baseline belongs to, the same
+  // precondition every push has.
+  const manifest = await readManifest(config.workspaceDir, siteId);
+
+  const listed = await withRefusalGuidance(onProgress, "forms pull", async () => await listSiteForms(client, siteId));
+  const live = listed.filter((form) => !form.archived);
+  const archived = listed.filter((form) => form.archived).map((form) => form.key).sort();
+
+  const baselines = [];
+  const written = [];
+  // A form that sends visitors to a page is written with the page's path, which needs the page list once.
+  let pages;
+  for (const summary of live) {
+    const { form, version } = await withRefusalGuidance(
+      onProgress,
+      "forms pull",
+      async () => await getSiteForm(client, siteId, summary.id),
+    );
+    if (pages === undefined && afterSubmitPage(version.definition)) {
+      pages = await withRefusalGuidance(onProgress, "forms pull", async () => await getPageIndex(client, siteId, { onProgress }));
+    }
+    const document_ = projectFormForWorkspace({
+      key: form.key,
+      name: form.name,
+      sink: form.sink,
+      retention_days: form.retentionDays,
+      definition: version.definition,
+    }, pages?.pathById);
+    await writeWorkspaceJson(config.workspaceDir, formFileName(form.key), document_);
+    baselines.push([form.key, { id: form.id, version: version.version, settings: siteSettingsBaseline(form) }]);
+    written.push(form.key);
+  }
+
+  manifest.forms = formsManifestEntry(baselines);
+  await writeManifest(config.workspaceDir, manifest);
+  onProgress(`Wrote ${written.length} form file${written.length === 1 ? "" : "s"} and recorded their versions.`);
+
+  const reported = boundedList(written.sort(), MAXIMUM_REPORTED);
+  return successResult(VERB_FORMS_PULL, siteId, {
+    forms: {
+      total: written.length,
+      items: reported.items.map((key) => ({ key, file: formFileName(key) })),
+      ...(reported.truncated ? { itemsTruncated: true } : {}),
+      ...(archived.length > 0 ? { archived } : {}),
+    },
+  });
+}