mosaic-headless 1.30.0 → 1.31.0

package/tools/sweep_node_types.py

@@ -7,6 +7,20 @@
 sweep.json needs: base, version, cookie, nonce, themeID. --setup fills in masterID,
 templateID and parentNodeID and writes them back.
 
+## Which page it reads
+
+The template --setup creates is assigned to `archive-post.php` - the post archive -
+so the probes render wherever WordPress serves that archive. That used to be `/`,
+and the sweep read `/`. Then the site got a static front page (`show_on_front=page`),
+`/` became an ordinary page with its own template, and the sweep went on committing
+probes into a master nothing rendered: every type, a plain `div` included, came
+back COMMITTED. Nothing failed; it simply measured the wrong page.
+
+So the page is named rather than assumed: `sweepPath` in the config, default
+`?post_type=post`, which serves the post archive whatever the front page is set to.
+Before a sweep, a plain `div` is committed and looked for on that page; if it does
+not render, the run stops instead of recording a column of false COMMITTEDs.
+
 ## Why one at a time
 
 A batch sweep is worthless here. Mosaic accepts structurally impossible placements at
@@ -79,9 +93,14 @@ class Client:
         )
 
     def page(self, path=""):
-        """Fetch a public page, cache-busted. `path` picks a page other than the home page."""
-        req = urllib.request.Request("%s/%s?sweep=%s" % (self.base, path.strip("/") + "/" if path else "",
-                                                         uuid.uuid4().hex[:8]))
+        """Fetch a public page, cache-busted. `path` picks a page other than the home page:
+        a slug (`my-page`) or a query (`?post_type=post`)."""
+        bust = uuid.uuid4().hex[:8]
+        if path.startswith("?"):
+            url = "%s/%s&sweep=%s" % (self.base, path, bust)
+        else:
+            url = "%s/%s?sweep=%s" % (self.base, path.strip("/") + "/" if path else "", bust)
+        req = urllib.request.Request(url)
         try:
             with urllib.request.urlopen(req, timeout=120) as r:
                 return r.read().decode("utf-8", "replace")
@@ -223,6 +242,34 @@ def unwrap(resp, what):
     return resp["response"]
 
 
+def sweep_path(cfg):
+    """The public URL the sweep master renders on. See "Which page it reads"."""
+    return cfg.get("sweepPath", "?post_type=post")
+
+
+def assert_probe_renders(client, cfg):
+    """Commit a plain div and require it on the sweep page, then remove it.
+
+    A div renders on any healthy page. If it does not show up, the page being read is
+    not the one the sweep master renders on, and every row this run would write is
+    a false COMMITTED - so stop before writing any."""
+    master_id = cfg["masterID"]
+    key = "node/master/%s" % master_id
+    doc = unwrap(client.get("masterDocumentInstance/%s" % master_id), "masterDocumentInstance")
+    baseline_ids = {n["ID"] for n in doc[key]}
+    attr, _nid, recs = probe_records("div", cfg["parentNodeID"], master_id)
+    client.commit("masterDocumentInstance/%s" % master_id, envelopes(doc), {key: recs})
+    try:
+        html = client.page(sweep_path(cfg))
+        if ('id="%s"' % attr) not in html:
+            sys.exit("a plain div committed to the sweep master does not render on /%s - "
+                     "that page is not the one the master renders on, so every row would "
+                     "read COMMITTED. Set sweepPath in the config to a URL served by the "
+                     "post archive (the default is ?post_type=post)." % sweep_path(cfg))
+    finally:
+        delete_subtree(client, master_id, baseline_ids, cfg["parentNodeID"])
+
+
 def setup(client, cfg, config_path):
     """Create the master (which heals into a default node tree) and the template."""
     tei = unwrap(client.get("adminMasterEditorInstance"), "adminMasterEditorInstance")
@@ -282,7 +329,7 @@ def setup(client, cfg, config_path):
         json.dump(cfg, fh, indent=1)
     print("masterID=%s\ntemplateID=%s\nparentNodeID=%s" % (master_id, template_id, parent))
 
-    html = client.page()
+    html = client.page(sweep_path(cfg))
     print("baseline page: %d bytes%s" % (len(html), "" if len(html) >= MIN_HEALTHY_BYTES else "  <-- ALREADY BROKEN"))
 
 
@@ -347,12 +394,14 @@ def delete_subtree(client, master_id, keep_ids, parent_id=None):
 
 
 def sweep(client, cfg, types, out_path):
+    assert_probe_renders(client, cfg)
     master_id = cfg["masterID"]
     doc = unwrap(client.get("masterDocumentInstance/%s" % master_id), "masterDocumentInstance")
     key = "node/master/%s" % master_id
     baseline_ids = {n["ID"] for n in doc[key]}
 
-    baseline_html = client.page()
+    path = sweep_path(cfg)
+    baseline_html = client.page(path)
     if len(baseline_html) < MIN_HEALTHY_BYTES:
         sys.exit("baseline page is already broken (%d bytes); reset the theme first" % len(baseline_html))
     print("baseline: %d bytes, %d nodes\n" % (len(baseline_html), len(baseline_ids)))
@@ -373,7 +422,7 @@ def sweep(client, cfg, types, out_path):
                 err = "PHP fatal / gateway error during commit"
             tag, classes, echoed, size = "", "", "", ""
         else:
-            html = client.page()
+            html = client.page(path)
             size = len(html)
             if size < MIN_HEALTHY_BYTES:
                 outcome, tag, classes, echoed = "BROKE_PAGE", "", "", ""

package/tools/verify_browser.py

@@ -213,7 +213,7 @@ def tidy(text):
     report every hairline on the page as overridden."""
     t = re.sub(r"\s*,\s*", ",", (text or "").strip().lower())
     t = re.sub(r"\s+", " ", t)
-    return re.sub(r"(?<![\d.])\.(\d)", r"0.
", t)
+    return re.sub(r"(?<![\d.])\.(\d)", r"0.\1", t)
 
 
 # The comparison knows properties by their Mosaic key, but customDeclarations hands it raw
@@ -609,7 +609,8 @@ def main():
     ap.add_argument("--config",
                     help="JSON with base/cookie/nonce; omit it and the MOSAIC_SITE_URL / MOSAIC_REST_COOKIE / MOSAIC_REST_NONCE environment variables supply them instead")
     ap.add_argument("--site", required=True)
-    ap.add_argument("--csv", help="the computed-value table")
+    ap.add_argument("--csv", help="a DIRECTORY: the computed-value table is "
+                    "written one file per page, <dir>/<slug>.csv")
     ap.add_argument("--audit", help="the design-audit findings")
     ap.add_argument("--ack", default=os.path.join(
         os.path.dirname(os.path.abspath(__file__)), "..", "data",
@@ -754,13 +755,24 @@ def main():
     # count in the release gate - should not have to find the blank line between them.
     # The audit file is written even when it is empty: a header with no rows says
     # "this ran and found nothing", where a missing file says nothing at all.
+    #
+    # The computed-value table is one file per page, named by its slug, with no url
+    # column: a single file carried the site's address 4,000 times over and passed
+    # the plugin directory's 256 KiB per-file limit. The slug is the filename, and
+    # the address is whatever --config pointed at.
     if a.csv:
-        with open(a.csv, "w", newline="", encoding="utf-8") as fh:
-            w = csv.writer(fh)
-            w.writerow(["url", "breakpoint", "attrID", "property", "declared",
-                        "computed", "status"])
-            w.writerows(rows)
-        print("\nwrote %s (%d rows)" % (a.csv, len(rows)))
+        os.makedirs(a.csv, exist_ok=True)
+        per_page = {}
+        for r in rows:
+            per_page.setdefault(r[0].rstrip("/").rsplit("/", 1)[-1], []).append(r[1:])
+        for slug, page_rows in per_page.items():
+            out = os.path.join(a.csv, slug + ".csv")
+            with open(out, "w", newline="", encoding="utf-8") as fh:
+                w = csv.writer(fh)
+                w.writerow(["breakpoint", "attrID", "property", "declared",
+                            "computed", "status"])
+                w.writerows(page_rows)
+            print("\nwrote %s (%d rows)" % (out, len(page_rows)))
     if a.audit:
         with open(a.audit, "w", newline="", encoding="utf-8") as fh:
             w = csv.writer(fh)

package/tools/verify_conversion.py

@@ -29,6 +29,7 @@ import argparse
 import csv
 import html as htmllib
 import json
+import os
 import re
 import sys
 import time
@@ -129,7 +130,7 @@ def main():
     ap.add_argument("--report", help="the CSV from_elementor.py wrote")
     ap.add_argument("--csv")
     ap.add_argument("--width", type=int, default=1440)
-    ap.add_argument("--browser", help="browser-verification CSV from verify_browser.py")
+    ap.add_argument("--browser", help="browser-verification CSV, or the per-page directory, from verify_browser.py")
     ap.add_argument("--audit", help="design-audit CSV from verify_browser.py")
     ap.add_argument("--rwd", help="rwd-verification CSV from verify_rwd.py")
     ap.add_argument("--prefix", default="el",
@@ -230,7 +231,10 @@ def main():
               "own media query" % len(rr) if not bad else
               "%d of %d not verified" % (len(bad), len(rr)))
     if a.browser:
-        br = list(csv.DictReader(open(a.browser, encoding="utf-8")))
+        # a file, or the per-page directory verify_browser.py writes
+        files = ([os.path.join(a.browser, f) for f in sorted(os.listdir(a.browser))
+                  if f.endswith(".csv")] if os.path.isdir(a.browser) else [a.browser])
+        br = [r for f in files for r in csv.DictReader(open(f, encoding="utf-8"))]
         ok = sum(1 for r in br if r.get("status") == "ok")
         over = [r for r in br if r.get("status") == "OVERRIDDEN"]
         check("COMPUTED", bool(br) and not over,

package/tools/verify_dialog.py

@@ -68,14 +68,15 @@ DIALOG_STATE = """id => {
   const el = document.getElementById(id);
   if (!el) return null;
   const cs = getComputedStyle(el);
-  return {tag: el.tagName.toLowerCase(), open: !!el.open,
+  return {tag: el.tagName.toLowerCase(), open: !!(el.open || el.matches(':popover-open') || el.getAttribute('data-mosaic-dialog-state') === 'open'),
           display: cs.display, visible: cs.display !== 'none',
           role: el.getAttribute('role'), ariaModal: el.getAttribute('aria-modal'),
           ariaLabel: el.getAttribute('aria-label'),
           closedby: el.getAttribute('data-mosaic-modal-closedby'),
           hasOverlay: !!el.querySelector('.m-modal-overlay'),
           hasWindow: !!el.querySelector('.m-modal-window'),
-          inTopLayer: typeof el.matches === 'function' && el.matches(':modal')};
+          inTopLayer: typeof el.matches === 'function'
+              && (el.matches(':modal') || el.matches(':popover-open'))};
 }"""
 
 # An exit-intent trigger listens for the pointer leaving through the top of the

package/tools/verify_dialog_lab.py

@@ -38,10 +38,10 @@ def row(check, result, detail=""):
     print("%-34s %-5s %s" % (check, result, detail))
 
 
-OPEN = "(id) => { const e = document.getElementById(id); return e ? !!e.open : null; }"
+OPEN = "(id) => { const e = document.getElementById(id); return e ? !!(e.open || e.matches(':popover-open') || e.getAttribute('data-mosaic-dialog-state') === 'open') : null; }"
 BOX = """(id) => {
   const e = document.getElementById(id);
-  if (!e || !e.open) return null;
+  if (!e || !(e.open || e.matches(':popover-open') || e.getAttribute('data-mosaic-dialog-state') === 'open')) return null;
   const w = e.querySelector('.m-modal-window');
   const r = w.getBoundingClientRect();
   const hr = e.getBoundingClientRect();
@@ -59,8 +59,8 @@ BOX = """(id) => {
 
 def shut_all(p):
     p.evaluate("""() => document.querySelectorAll('dialog.m-modal').forEach(d => {
-        if (d.open) { try { d.mosaicDialog ? d.mosaicDialog.close() : d.close(); }
-                      catch(e) { d.close(); } } })""")
+        if ((d.open || d.matches(':popover-open') || d.getAttribute('data-mosaic-dialog-state') === 'open')) { try { (d.mosaicDialog ? (d.mosaicDialog.requestClose || d.mosaicDialog.close).call(d.mosaicDialog) : d.matches(':popover-open') ? d.hidePopover() : d.close()); }
+                      catch(e) {} } })""")
     p.wait_for_timeout(250)
 
 
@@ -122,10 +122,10 @@ def main():
         # version of this verifier passed 36/36 on a page where all ten were
         # permanently on screen.
         p.evaluate("""() => document.querySelectorAll('dialog.m-modal')
-            .forEach(d => { if (d.open) { try { d.close(); } catch (e) {} } })""")
+            .forEach(d => { if ((d.open || d.matches(':popover-open') || d.getAttribute('data-mosaic-dialog-state') === 'open')) { try { (d.mosaicDialog ? (d.mosaicDialog.requestClose || d.mosaicDialog.close).call(d.mosaicDialog) : d.matches(':popover-open') ? d.hidePopover() : d.close()); } catch (e) {} } })""")
         p.wait_for_timeout(400)
         shown = p.evaluate("""() => [...document.querySelectorAll('dialog.m-modal')]
-            .filter(d => !d.open)
+            .filter(d => !(d.open || d.matches(':popover-open') || d.getAttribute('data-mosaic-dialog-state') === 'open'))
             .filter(d => { const r = d.getBoundingClientRect();
                            return r.width > 0 && r.height > 0; })
             .map(d => d.id + ' (display:' + getComputedStyle(d).display + ')')""")
@@ -241,7 +241,7 @@ def main():
                 p2.goto(URL + "?v=%s%d" % (order[0], i), wait_until="load")
                 p2.wait_for_timeout(2200)
                 p2.evaluate("""() => document.querySelectorAll('dialog.m-modal')
-                    .forEach(d => { if (d.open) d.close(); })""")
+                    .forEach(d => { if ((d.open || d.matches(':popover-open') || d.getAttribute('data-mosaic-dialog-state') === 'open')) (d.mosaicDialog ? (d.mosaicDialog.requestClose || d.mosaicDialog.close).call(d.mosaicDialog) : d.matches(':popover-open') ? d.hidePopover() : d.close()); })""")
                 steps = []
                 for which in order:
                     p2.click("#dl-open-%s" % which)
@@ -249,7 +249,7 @@ def main():
                     opened = [k for k in ("aba", "abb") if p2.evaluate(OPEN, "dl-" + k)]
                     steps.append("%s>%s" % (which, "+".join(opened) or "none"))
                     p2.evaluate("""() => document.querySelectorAll('dialog.m-modal')
-                        .forEach(d => { if (d.open) d.close(); })""")
+                        .forEach(d => { if ((d.open || d.matches(':popover-open') || d.getAttribute('data-mosaic-dialog-state') === 'open')) (d.mosaicDialog ? (d.mosaicDialog.requestClose || d.mosaicDialog.close).call(d.mosaicDialog) : d.matches(':popover-open') ? d.hidePopover() : d.close()); })""")
                     p2.wait_for_timeout(200)
                 seen[",".join(steps)] += 1
                 if i == 0:
@@ -289,7 +289,7 @@ def main():
         # document underneath measures clean.
         FIT = """(id) => {
           const e = document.getElementById(id);
-          if (!e || !e.open) return null;
+          if (!e || !(e.open || e.matches(':popover-open') || e.getAttribute('data-mosaic-dialog-state') === 'open')) return null;
           const w = e.querySelector('.m-modal-window');
           const r = w.getBoundingClientRect(), h = e.getBoundingClientRect();
           let over = 0, worst = null;
@@ -309,7 +309,7 @@ def main():
             pm.goto(URL, wait_until="load")
             pm.wait_for_timeout(2000)
             pm.evaluate("() => document.querySelectorAll('dialog')"
-                        ".forEach(d => d.open && d.close())")
+                        ".forEach(d => (d.open || d.matches(':popover-open') || d.getAttribute('data-mosaic-dialog-state') === 'open') && (d.mosaicDialog ? (d.mosaicDialog.requestClose || d.mosaicDialog.close).call(d.mosaicDialog) : d.matches(':popover-open') ? d.hidePopover() : d.close()))")
             for key in ("centre", "sheet", "drawer", "takeover", "gate", "aba"):
                 try:
                     pm.click("#dl-open-%s" % key, timeout=5000)
@@ -328,7 +328,7 @@ def main():
                         % (f["scrollW"], f["clientW"], f["over"],
                            (" <%s>" % f["worst"]) if f["over"] > 1 else "", f["doc"]))
                 pm.evaluate("() => document.querySelectorAll('dialog')"
-                            ".forEach(d => d.open && d.close())")
+                            ".forEach(d => (d.open || d.matches(':popover-open') || d.getAttribute('data-mosaic-dialog-state') === 'open') && (d.mosaicDialog ? (d.mosaicDialog.requestClose || d.mosaicDialog.close).call(d.mosaicDialog) : d.matches(':popover-open') ? d.hidePopover() : d.close()))")
                 pm.wait_for_timeout(200)
             cm.close()
 

package/tools/verify_intro.py

@@ -53,8 +53,8 @@ import time
 
 # When to read. Dense through the sequence, then two well past its end - the last
 # two are what turn "it looked right" into "it finished".
-SAMPLES_MS = [120, 700, 1400, 2200, 3000, 3800, 4600, 5400, 6200, 7000,
-              7800, 8300, 8900, 11000, 13000]
+SAMPLES_MS = [120, 400, 700, 1000, 1300, 1600, 1900, 2200, 2500, 2800,
+              3100, 3400, 3800, 5000, 7000]
 
 # What to read at each sample. Anything whose id starts with the veil prefix is
 # treated as part of the intro; everything else is content that must end up visible.
@@ -258,7 +258,9 @@ def main():
                     help="ids under this prefix are the intro, not the content")
     ap.add_argument("--min-ambient", type=int, default=3,
                     help="how many elements must still be moving once settled")
-    ap.add_argument("--deadline-ms", type=int, default=8900,
+    # 3400, not the 8900 it used to be: the entrance was compressed to a 2.15s veil
+    # plus its .9s exit, because nine seconds before the headline was the defect.
+    ap.add_argument("--deadline-ms", type=int, default=3400,
                     help="by this point the intro must be over")
     ap.add_argument("--csv")
     ap.add_argument("--frames")