@hajimism/dek 0.3.3 → 0.4.0

package/src/core/playwright-visual.ts

@@ -1,7 +1,9 @@
 import type { Browser, BrowserContext, Page } from "playwright";
+import { findMisalignments } from "./alignment.ts";
 import { freezeTransition, loadVideoDoc } from "./capture-go.ts";
 import { measureFill } from "./fill.ts";
 import { finishBeat } from "./finish-beat.ts";
+import { findLineBreaks, measureTextFlowsInPage } from "./line-break.ts";
 import {
   crossesEdge,
   findClippedText,
@@ -31,6 +33,7 @@ import type {
   PagesRequest,
   PagesResponse,
   PdfRequest,
+  PptxPictureFile,
   PptxRequest,
   PptxResponse,
   ResponseTo,
@@ -39,8 +42,24 @@ import type {
 } from "./playwright.ts";
 import { captureMotion } from "./playwright-motion.ts";
 import { renderSheets } from "./playwright-sheet.ts";
+import { mapPictures } from "./pptx-package.ts";
+import {
+  describePptxPicturesInPage,
+  hidePptxLiftedInPage,
+  measurePptxPaintInPage,
+  measurePptxThemeInPage,
+  PPTX_PSEUDO_ATTR,
+  pptxFontProbesInPage,
+  pptxPictureEmptyInPage,
+  pptxSvgInPage,
+  SVG_CHECKED,
+  SVG_PAINT,
+  showOnlyPptxNodeInPage,
+} from "./pptx-paint.ts";
+import { planPptxLayers, pptxPageFacts } from "./pptx-plan.ts";
 import { measurePptxTextsInPage, PPTX_OWNER_ATTR } from "./pptx-text.ts";
-import { pseudoTextBoxes } from "./pseudo-text.ts";
+import type { PptxPageTheme } from "./pptx-theme.ts";
+import { pseudoElementBoxes, pseudoTextBoxes } from "./pseudo-text.ts";
 import { motionSheets } from "./sheet.ts";
 import {
   type Box,
@@ -133,7 +152,10 @@ async function printPdf(context: BrowserContext, request: PdfRequest): Promise<D
   return {};
 }
 
-/** Each slide's text measured and taken out, and the slide shot without it, page by page. */
+/**
+ * Each slide's text and paint measured; what is laid over the picture planned, each picture
+ * drawn alone, and the slide shot without what was laid over it, page by page.
+ */
 async function shootPptx(context: BrowserContext, request: PptxRequest): Promise<PptxResponse> {
   const slides = await visitInPages(
     request.pages,
@@ -141,19 +163,86 @@ async function shootPptx(context: BrowserContext, request: PptxRequest): Promise
     async (page, pageReq) => {
       await page.setContent(pageReq.html, { waitUntil: "load" });
       await page.evaluate(() => document.fonts.ready.then(() => undefined));
-      const measured = await page.evaluate(measurePptxTextsInPage);
+      const paint = {
+        ...(await page.evaluate(measurePptxPaintInPage)),
+        described: await page.evaluate(describePptxPicturesInPage),
+      };
+      const texts = await page.evaluate(measurePptxTextsInPage);
       const fonts = await platformFonts(page);
-      await page.screenshot({ path: pageReq.screenshotPath, fullPage: false });
+      const placed = await pseudoElementBoxes(page, PPTX_PSEUDO_ATTR, "border", [
+        "before",
+        "after",
+        "marker",
+      ]);
+      for (const run of [
+        ...texts.boxes.flatMap((box) => box.runs),
+        ...texts.blocks.flatMap((block) => block.runs),
+      ]) {
+        const font = fonts.get(run.owner);
+        if (font !== undefined) {
+          run.font = font;
+        }
+      }
+      const plan = planPptxLayers(pptxPageFacts(paint, texts, placed));
+
+      const files = new Map<number, PptxPictureFile>();
+      for (const [index, node] of plan.lifted.pictures.entries()) {
+        const measured = paint.nodes[node];
+        const rect =
+          (measured?.paint?.kind === "picture" ? measured.paint.extent : undefined) ??
+          measured?.rect;
+        if (!rect) {
+          continue;
+        }
+        const svg = await page.evaluate(pptxSvgInPage, {
+          node,
+          paint: SVG_PAINT,
+          checked: SVG_CHECKED,
+        });
+        const png = pageReq.screenshotPath.replace(/\.png$/, `-${index + 1}.png`);
+        await page.evaluate(showOnlyPptxNodeInPage, node);
+        await page.screenshot({
+          path: png,
+          omitBackground: true,
+          clip: {
+            x: paint.origin.left + rect.left,
+            y: paint.origin.top + rect.top,
+            width: rect.right - rect.left,
+            height: rect.bottom - rect.top,
+          },
+        });
+        await page.evaluate(showOnlyPptxNodeInPage, null);
+        files.set(node, svg === undefined ? { png } : { png, svg });
+      }
+      const layers = mapPictures(plan.layers, ({ node, ...picture }) => ({
+        ...picture,
+        ...(files.get(node) ?? { png: "" }),
+      }));
+      // With the background PowerPoint's, the slide is shot on nothing, and a picture with
+      // nothing left in it is no picture.
+      const { background } = plan;
+      await page.evaluate(hidePptxLiftedInPage, {
+        ...plan.lifted,
+        background: background !== undefined,
+      });
+      const shot = await page.screenshot({
+        path: pageReq.screenshotPath,
+        fullPage: false,
+        ...(background ? { omitBackground: true } : {}),
+      });
+      const empty =
+        background !== undefined &&
+        (await page.evaluate(
+          pptxPictureEmptyInPage,
+          `data:image/png;base64,${shot.toString("base64")}`,
+        ));
       return {
         slug: pageReq.slug,
-        description: measured.description,
-        boxes: measured.boxes.map((box) => ({
-          ...box,
-          runs: box.runs.map(({ owner, ...run }) => {
-            const font = fonts.get(owner);
-            return font === undefined ? run : { ...run, font };
-          }),
-        })),
+        description: plan.description,
+        layers,
+        picture: !empty,
+        ...(background ? { background } : {}),
+        theme: await pageTheme(page),
       };
     },
   );
@@ -161,12 +250,44 @@ async function shootPptx(context: BrowserContext, request: PptxRequest): Promise
 }
 
 /**
- * The font Chromium drew each marked element's text with, by the element's number: the one that
- * drew most of its glyphs, as the DevTools protocol reports it. A theme names a stack, or only
- * `sans-serif`; this is the face that was found.
+ * What the slide says of the deck's theme: its color tokens, and each font token as the
+ * `font-family` it is and the faces Chromium drew Latin and East Asian text in with it. Measured
+ * after the slide's last shot, since it puts text in the page to measure.
  */
+async function pageTheme(page: Page): Promise<PptxPageTheme> {
+  const { colors, families } = await page.evaluate(measurePptxThemeInPage);
+  await page.evaluate(pptxFontProbesInPage, true);
+  const faces = await platformFaces(page, "data-dek-pptx-probe");
+  await page.evaluate(pptxFontProbesInPage, false);
+  const fonts: PptxPageTheme["fonts"] = {};
+  for (const token of ["title", "body"] as const) {
+    const family = families[token];
+    const latin = faces.get(`${token}-latin`);
+    if (family !== undefined && latin !== undefined) {
+      fonts[token] = { family, latin, ea: faces.get(`${token}-ea`) ?? latin };
+    }
+  }
+  return { colors, fonts };
+}
+
+/** The font Chromium drew each marked element's text with, by the element's number. */
 async function platformFonts(page: Page): Promise<Map<number, string>> {
   const fonts = new Map<number, string>();
+  for (const [owner, face] of await platformFaces(page, PPTX_OWNER_ATTR)) {
+    if (Number.isInteger(Number(owner))) {
+      fonts.set(Number(owner), face);
+    }
+  }
+  return fonts;
+}
+
+/**
+ * The font Chromium drew each element marked with `attr` in, by the attribute's value: the one
+ * that drew most of its glyphs, as the DevTools protocol reports it. A theme names a stack, or
+ * only `sans-serif`; this is the face that was found.
+ */
+async function platformFaces(page: Page, attr: string): Promise<Map<string, string>> {
+  const fonts = new Map<string, string>();
   const cdp = await page.context().newCDPSession(page);
   try {
     await cdp.send("DOM.enable");
@@ -176,19 +297,19 @@ async function platformFonts(page: Page): Promise<Map<number, string>> {
     };
     const { nodeIds } = (await cdp.send("DOM.querySelectorAll", {
       nodeId: root.nodeId,
-      selector: `[${PPTX_OWNER_ATTR}]`,
+      selector: `[${attr}]`,
     })) as { nodeIds: number[] };
     for (const nodeId of nodeIds) {
       const { attributes } = (await cdp.send("DOM.getAttributes", { nodeId })) as {
         attributes: string[];
       };
-      const owner = Number(attributes[attributes.indexOf(PPTX_OWNER_ATTR) + 1]);
+      const value = attributes[attributes.indexOf(attr) + 1];
       const { fonts: used } = (await cdp.send("CSS.getPlatformFontsForNode", { nodeId })) as {
         fonts: Array<{ familyName: string; glyphCount: number }>;
       };
       const main = [...used].sort((a, b) => b.glyphCount - a.glyphCount)[0];
-      if (main && Number.isInteger(owner)) {
-        fonts.set(owner, main.familyName);
+      if (main && value !== undefined) {
+        fonts.set(value, main.familyName);
       }
     }
   } finally {
@@ -240,6 +361,8 @@ async function visitPages(context: BrowserContext, request: PagesRequest): Promi
     drawErrors: found.flatMap((page) => page.drawErrors ?? []),
     collisions: found.flatMap((page) => page.collisions ?? []),
     fills: found.flatMap((page) => page.fills ?? []),
+    lineBreaks: found.flatMap((page) => page.lineBreaks ?? []),
+    misalignments: found.flatMap((page) => page.misalignments ?? []),
   };
 }
 
@@ -343,7 +466,14 @@ async function visitPage(
   pageReq: VisualPage,
   { actions, viewport }: Pick<PagesRequest, "actions" | "viewport">,
 ): Promise<PagesResponse> {
-  const response: PagesResponse = { overflows: [], contrasts: [], collisions: [], fills: [] };
+  const response: PagesResponse = {
+    overflows: [],
+    contrasts: [],
+    collisions: [],
+    fills: [],
+    lineBreaks: [],
+    misalignments: [],
+  };
   const failing: Failing[] = [];
   const crossing: Crossing[] = [];
   await page.setContent(pageReq.html, { waitUntil: "load" });
@@ -362,6 +492,36 @@ async function visitPage(
     if (actions.includes("fill") && frame) {
       response.fills?.push({ slug, step, ...measureFill(frame, measured.elements) });
     }
+    if (actions.includes("alignment")) {
+      for (const { element, other, ...miss } of findMisalignments(measured.elements)) {
+        const [one, two] = [measured.elements[element], measured.elements[other]];
+        response.misalignments?.push({
+          slug,
+          step,
+          box: one?.box ?? "",
+          ...(one?.text ? { text: one.text } : {}),
+          other: two?.box ?? "",
+          ...(two?.text ? { otherText: two.text } : {}),
+          ...miss,
+        });
+      }
+    }
+    if (actions.includes("breaks")) {
+      const { wordBreak, flows } = await page.evaluate(measureTextFlowsInPage);
+      const textWraps = new Map(flows.map((flow) => [flow.element, flow.textWrap]));
+      for (const { element, ...found } of findLineBreaks(flows)) {
+        const el = measured.elements[element];
+        response.lineBreaks?.push({
+          slug,
+          step,
+          box: el?.box ?? "",
+          ...(el?.text ? { text: el.text } : {}),
+          ...found,
+          wordBreak,
+          textWrap: textWraps.get(element) ?? "wrap",
+        });
+      }
+    }
     if (actions.includes("overflow") && frame) {
       for (const { element, ...overflow } of [
         ...findOverflows(frame, measured.elements),

package/src/core/playwright.ts

@@ -2,11 +2,14 @@ import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { pathToFileURL } from "node:url";
 import type { Browser } from "playwright";
+import type { AlignedEdge, Misalignment } from "./alignment.ts";
 import { DekError } from "./error.ts";
 import type { Fill } from "./fill.ts";
+import type { LineBreakKind } from "./line-break.ts";
 import { type Collision, EDGES, type Overflow } from "./overflow.ts";
 import { moduleFilePath } from "./path.ts";
-import type { PptxTextBox } from "./pptx-package.ts";
+import type { PptxFill, PptxLayer, PptxTextBox } from "./pptx-package.ts";
+import type { PptxPageTheme } from "./pptx-theme.ts";
 import type { MotionBeat, SheetSpec } from "./sheet.ts";
 import { exitWorker, readWorkerRequest, runJsonWorker, workerCommand } from "./spawn.ts";
 import type { Position } from "./step.ts";
@@ -41,7 +44,7 @@ export type MotionSpec = {
 };
 
 /** What to measure on each page; a screenshot is asked for by the page's `screenshotPath`. */
-export type PageAction = "overflow" | "contrast" | "fill";
+export type PageAction = "overflow" | "contrast" | "fill" | "breaks" | "alignment";
 
 type Viewport = { width: number; height: number };
 
@@ -84,8 +87,9 @@ export type MotionRequest = {
 };
 
 /**
- * Each slide's still page drawn for a PPTX, at twice its size: its text measured line by line,
- * then the page shot without it, to `screenshotPath`.
+ * Each slide's still page drawn for a PPTX, at twice its size: its text, boxes, and pictures
+ * measured, each picture drawn alone beside `screenshotPath`, then the page shot without what
+ * is laid over it, to `screenshotPath`.
  */
 export type PptxRequest = {
   kind: "pptx";
@@ -93,9 +97,22 @@ export type PptxRequest = {
   pages: Array<{ html: string; slug: string; screenshotPath: string }>;
 };
 
-/** Each page's text, in the order of the request's pages, with the font each run was drawn in. */
+/** A picture on its way to the package: the PNG the worker wrote, and the SVG it read. */
+export type PptxPictureFile = { png: string; svg?: string };
+
+/** Each page's layers, in the order of the request's pages, with the font each run was drawn in. */
 export type PptxResponse = {
-  slides: Array<{ slug: string; description: string; boxes: PptxTextBox[] }>;
+  slides: Array<{
+    slug: string;
+    description: string;
+    layers: PptxLayer<PptxPictureFile>[];
+    /** Whether anything is left in the slide's picture; false for nothing. */
+    picture?: boolean;
+    /** The slide's background, when PowerPoint paints it. */
+    background?: PptxFill;
+    /** What the slide says of the deck's theme. */
+    theme?: PptxPageTheme;
+  }>;
 };
 
 /** What the Playwright worker is asked; the kind says which payload comes with it. */
@@ -144,6 +161,30 @@ type CollisionFinding = Collision & { slug: string; step: string };
 /** How much of the frame one page fills, and where. */
 export type FillFinding = Fill & { slug: string; step: string };
 
+/**
+ * A line break that reads wrong on one page, with what decides where the fix goes: the slide's own
+ * `word-break` and the text's `text-wrap`, as the page computes them.
+ */
+type LineBreakFinding = {
+  slug: string;
+  step: string;
+  box: string;
+  text?: string;
+  lines: string[];
+  wordBreak: string;
+  textWrap: string;
+} & LineBreakKind;
+
+/** Two items on one page that nearly line up and do not, each named as the page shows it. */
+type MisalignmentFinding = Omit<Misalignment, "element" | "other"> & {
+  slug: string;
+  step: string;
+  box: string;
+  text?: string;
+  other: string;
+  otherText?: string;
+};
+
 export type PagesResponse = {
   overflows: OverflowFinding[];
   contrasts: ContrastFinding[];
@@ -153,6 +194,10 @@ export type PagesResponse = {
   collisions?: CollisionFinding[];
   /** One for each page, when `fill` was asked for; absent from a runner that has none. */
   fills?: FillFinding[];
+  /** Absent from a runner that has none to report. */
+  lineBreaks?: LineBreakFinding[];
+  /** Absent from a runner that has none to report. */
+  misalignments?: MisalignmentFinding[];
 };
 
 export type MotionResponse = {
@@ -318,13 +363,23 @@ function spawnRunner(
 const RESPONSE_PARSERS: {
   [K in keyof VisualResponses]: (fields: Record<string, unknown>) => VisualResponses[K] | null;
 } = {
-  pages: ({ overflows, contrasts, drawErrors = [], collisions = [], fills = [] }) =>
+  pages: ({
+    overflows,
+    contrasts,
+    drawErrors = [],
+    collisions = [],
+    fills = [],
+    lineBreaks = [],
+    misalignments = [],
+  }) =>
     isArrayOf(overflows, isOverflowFinding) &&
     isArrayOf(contrasts, isContrastFinding) &&
     isArrayOf(drawErrors, isDrawErrorFinding) &&
     isArrayOf(collisions, isCollisionFinding) &&
-    isArrayOf(fills, isFillFinding)
-      ? { overflows, contrasts, drawErrors, collisions, fills }
+    isArrayOf(fills, isFillFinding) &&
+    isArrayOf(lineBreaks, isLineBreakFinding) &&
+    isArrayOf(misalignments, isMisalignmentFinding)
+      ? { overflows, contrasts, drawErrors, collisions, fills, lineBreaks, misalignments }
       : null,
   pdf: () => ({}),
   morph: () => ({}),
@@ -401,6 +456,37 @@ function isFillFinding(value: unknown): value is FillFinding {
   );
 }
 
+function isLineBreakFinding(value: unknown): value is LineBreakFinding {
+  if (!isFinding(value)) {
+    return false;
+  }
+  const { kind, lines, phrased, tail, wordBreak, textWrap } = value;
+  return (
+    isArrayOf(lines, isString) &&
+    isString(wordBreak) &&
+    isString(textWrap) &&
+    ((kind === "split" && isArrayOf(phrased, isString)) || (kind === "short" && isString(tail)))
+  );
+}
+
+const ALIGNED_EDGES: ReadonlySet<unknown> = new Set<AlignedEdge>([
+  "top",
+  "middle",
+  "bottom",
+  "left",
+  "center",
+  "right",
+]);
+
+function isMisalignmentFinding(value: unknown): value is MisalignmentFinding {
+  // Both name two elements by their boxes and their texts, as a collision does.
+  if (!isCollisionFinding(value)) {
+    return false;
+  }
+  const { edge, of, by } = value as Record<string, unknown>;
+  return ALIGNED_EDGES.has(edge) && (of === "box" || of === "text") && isNumber(by);
+}
+
 function isDrawErrorFinding(value: unknown): value is DrawErrorFinding {
   if (!value || typeof value !== "object") {
     return false;
@@ -453,34 +539,125 @@ function isPptxSlide(value: unknown): value is PptxResponse["slides"][number] {
   if (!value || typeof value !== "object") {
     return false;
   }
-  const { slug, description, boxes } = value as Record<string, unknown>;
-  return isString(slug) && isString(description) && isArrayOf(boxes, isPptxBox);
+  const { slug, description, layers, picture, background, theme } = value as Record<
+    string,
+    unknown
+  >;
+  return (
+    isString(slug) &&
+    isString(description) &&
+    isArrayOf(layers, isPptxLayer) &&
+    (picture === undefined || typeof picture === "boolean") &&
+    (background === undefined || isFill(background)) &&
+    (theme === undefined || (Boolean(theme) && typeof theme === "object"))
+  );
+}
+
+function isFill(value: unknown): value is PptxFill {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const fill = value as Record<string, unknown>;
+  return fill.kind === "solid"
+    ? isPaint(fill.paint)
+    : fill.kind === "linear" &&
+        isNumber(fill.angle) &&
+        isArrayOf(
+          fill.stops,
+          (stop: unknown): stop is object =>
+            Boolean(stop) &&
+            typeof stop === "object" &&
+            isNumber((stop as Record<string, unknown>).at) &&
+            isPaint((stop as Record<string, unknown>).paint),
+        );
 }
 
-function isPptxBox(value: unknown): value is PptxTextBox {
+function isPptxLayer(value: unknown): value is PptxLayer<PptxPictureFile> {
   if (!value || typeof value !== "object") {
     return false;
   }
-  const box = value as Record<string, unknown>;
+  const layer = value as Record<string, unknown>;
+  const at = ["x", "y", "width", "height"].every((key) => isNumber(layer[key]));
+  switch (layer.kind) {
+    case "text":
+      return isPptxBox(layer);
+    case "rule":
+      return (
+        ["x1", "y1", "x2", "y2", "width"].every((key) => isNumber(layer[key])) &&
+        isPaint(layer.paint)
+      );
+    case "picture":
+      return (
+        at &&
+        isString(layer.png) &&
+        isString(layer.description) &&
+        (layer.svg === undefined || isString(layer.svg))
+      );
+    case "shape":
+      return (
+        at &&
+        isNumber(layer.radius) &&
+        (layer.fill === undefined || isPaint(layer.fill)) &&
+        isArrayOf(layer.layers, isPptxLayer) &&
+        (layer.text === undefined || isShapeText(layer.text))
+      );
+    default:
+      return false;
+  }
+}
+
+function isPaint(value: unknown): boolean {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const { color, alpha } = value as Record<string, unknown>;
+  return isArrayOf(color, isNumber) && isNumber(alpha);
+}
+
+function isShapeText(value: unknown): boolean {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const { inset, paragraphs } = value as Record<string, unknown>;
   return (
-    ["x", "y", "width", "height", "lineHeight"].every((key) => isNumber(box[key])) &&
-    isArrayOf(box.runs, (run: unknown): run is PptxTextBox["runs"][number] => {
-      if (!run || typeof run !== "object") {
+    Boolean(inset) &&
+    typeof inset === "object" &&
+    isArrayOf(paragraphs, (paragraph: unknown): paragraph is object => {
+      if (!paragraph || typeof paragraph !== "object") {
         return false;
       }
-      const { text, size, alpha, letterSpacing, color, font } = run as Record<string, unknown>;
-      return (
-        isString(text) &&
-        isNumber(size) &&
-        isNumber(alpha) &&
-        isNumber(letterSpacing) &&
-        isArrayOf(color, isNumber) &&
-        (font === undefined || isString(font))
-      );
+      const { runs, lineHeight } = paragraph as Record<string, unknown>;
+      return isNumber(lineHeight) && isArrayOf(runs, isPptxRun);
     })
   );
 }
 
+function isPptxBox(value: unknown): value is PptxTextBox {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const box = value as Record<string, unknown>;
+  return (
+    ["x", "y", "width", "height", "lineHeight"].every((key) => isNumber(box[key])) &&
+    isArrayOf(box.runs, isPptxRun)
+  );
+}
+
+function isPptxRun(run: unknown): run is PptxTextBox["runs"][number] {
+  if (!run || typeof run !== "object") {
+    return false;
+  }
+  const { text, size, alpha, letterSpacing, color, font } = run as Record<string, unknown>;
+  return (
+    isString(text) &&
+    isNumber(size) &&
+    isNumber(alpha) &&
+    isNumber(letterSpacing) &&
+    isArrayOf(color, isNumber) &&
+    (font === undefined || isString(font))
+  );
+}
+
 function isMotionBeat(value: unknown): value is MotionBeat {
   if (!value || typeof value !== "object") {
     return false;