@bettercms-ai/convert 0.9.0 → 0.10.0

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @bettercms-ai/convert
 
+## 0.10.0
+
+### Minor Changes
+
+- ac3a9ff: Declare the shapes the derive lane mints for every visible body tag: a heading whose styled run is
+  part of its value (`richtext inline`, bound whole rather than wrapped around its text node), a
+  standalone `span`/`time`/`small`/`label`/`option` leaf, a `<pre>` code block (the `<pre>` is the
+  field, not the `<code>` inside it), and a group row whose optional leaf is empty (covered by its
+  own route's templated declaration instead of reported `NO_ORIGINAL`). A heading split by a `<br />`
+  is found again: its value carries no space, because no text node holds one.
+
+  An inline heading the lane cannot own whole — its element holds a component, a `<slot>` or an
+  expression, which no stored string can re-emit — is refused by name (`SUBTREE_NOT_HTML`) instead of
+  being bound over the top of it.
+
+  New `--xray <dist-dir>` mode: how much of each built page's visible copy no binding owns, and the
+  biggest buckets of it, so the loop is build -> measure -> fix -> build with no deploy in it.
+
 ## 0.9.0
 
 ### Minor Changes

package/README.md

@@ -38,6 +38,27 @@ Exit codes: `0` done — the pending list is printed, not hidden; `1` something
 | `--strict` | exit 2 when any path is pending. |
 | `--overwrite-helper` | replace a `bcms-content` helper whose bytes this tool did not write — including one of ours that has since been edited. |
 | `--verbose` | stack traces, and every file considered. |
+| `--xray <dist-dir>` | a mode of its own: measure a LOCAL build instead of converting one. |
+
+### `--xray` — what is not editable
+
+A receipt says what the codemod bound. It cannot say what it was never asked to bind, so a site can
+report full coverage while a quarter of its visible text is unreachable from the editor. `--xray`
+walks a build and counts that, per page, with no deploy in the loop:
+
+```bash
+npm run build && npx @bettercms-ai/convert --xray dist
+```
+
+```
+docs/index.html  visible=3512 unaddressable=701 (20.0%)
+     379ch    4n  code @ main  e.g. "const site = { direction: 'rtl', language: 'ar'," | "npm install"
+     188ch   12n  span @ main>ul  e.g. "Performance" | "100"
+```
+
+Glyphs, ordinals, the platform badge and skip links are excluded from BOTH halves of the ratio —
+counting them would put a floor under every route that no amount of binding could lift. `--strict`
+exits 2 while any page is above 2 %. It takes no `--brief`: it measures, it does not convert.
 
 ## What it actually does
 
@@ -152,6 +173,8 @@ rendering that binding — `<img>`, `<source>`, `<video poster>`, or `<Image>`/`
 different paths share a sentence and position cannot separate them (both are skipped; the rest of
 the file still converts), `SUBSTRING_ONLY`, `IN_SCRIPT_OR_COMMENT`, `KIND_MISMATCH`,
 `REPEATER_AMBIGUOUS` (one array declaration, two loops over it — the row has two homes),
+`SUBTREE_NOT_HTML` (an INLINE richtext heading whose element holds a component, a `<slot>` or an
+expression: its whole subtree is the value and no stored string can re-emit that),
 `IMAGE_ASSET_UNRESOLVED`, `PARSE_ERROR`. `NOT_IN_SOURCE` is reserved for what it says: a path
 `locate` found in a file is `IN_EXPRESSION` when it sits in the module region or an expression
 node, and where `NOT_IN_SOURCE` is still the answer the row names the file it was found in. And

package/dist/cli.js

@@ -2562,6 +2562,7 @@ var IMAGE_SOURCES = {
   video: ["poster"]
 };
 var isImageSource = (site) => site.where === "attr" && (IMAGE_SOURCES[site.tag.toLowerCase()] ?? []).includes(site.attr ?? "");
+var ownsSubtree = (kind, site, inline) => kind === "richtext" && (inline === true ? !site.nonHtml : !(site.mixed && site.textRange !== null));
 var norm = (value) => decodeEntities(value.normalize("NFC")).replace(/\s+/g, " ").trim();
 var BCMS_ATTRS = [
   "data-bcms-field",
@@ -2594,6 +2595,7 @@ function openTagEnd(content, start) {
   }
   return content.length;
 }
+var OPAQUE_TAGS = /* @__PURE__ */ new Set(["pre"]);
 var isBlank = (value) => value.trim() === "";
 function renderedText(node) {
   if (node.type === "text") return node.value;
@@ -2601,6 +2603,11 @@ function renderedText(node) {
   return "";
 }
 var attrsOf = (node) => Object.fromEntries(node.attrs.map((a) => [a.name, a.value]));
+function nonHtmlSubtree(node) {
+  return node.children.some(
+    (c) => c.type === "expr" || c.type === "element" && (!isIntrinsic(c.tag) || c.tag === "slot" || nonHtmlSubtree(c))
+  );
+}
 function soleTextChild(node) {
   const texts = node.children.filter((c) => c.type === "text" && !isBlank(c.value));
   return texts.length === 1 ? texts[0] : null;
@@ -2637,10 +2644,12 @@ function collectSites(roots, literals) {
     if (node.type !== "element") return matched;
     const attrs = attrsOf(node);
     let index = 0;
-    for (const child of node.children) {
-      const childWalk = { ancestors: [...walk.ancestors, node.tag], siblingIndex: index };
-      if (child.type === "element") index++;
-      for (const hit of visit(child, childWalk)) matched.add(hit);
+    if (!OPAQUE_TAGS.has(node.tag)) {
+      for (const child of node.children) {
+        const childWalk = { ancestors: [...walk.ancestors, node.tag], siblingIndex: index };
+        if (child.type === "element") index++;
+        for (const hit of visit(child, childWalk)) matched.add(hit);
+      }
     }
     if (node.raw) {
       const text2 = norm(renderedTextRaw(node));
@@ -2666,6 +2675,7 @@ function collectSites(roots, literals) {
         textRange: soleTextChild(node)?.range ?? null,
         literal,
         partial: false,
+        ...nonHtmlSubtree(node) ? { nonHtml: true } : {},
         ancestors: walk.ancestors,
         siblingIndex: walk.siblingIndex
       });
@@ -2691,6 +2701,7 @@ function collectSites(roots, literals) {
           textRange: own.range,
           literal,
           partial: false,
+          ...nonHtmlSubtree(node) ? { nonHtml: true } : {},
           ancestors: walk.ancestors,
           siblingIndex: walk.siblingIndex
         });
@@ -2741,10 +2752,12 @@ function partials(node, literals, walk, out) {
   if (node.type !== "element" || node.raw) return false;
   let deeper = false;
   let index = 0;
-  for (const child of node.children) {
-    const childWalk = { ancestors: [...walk.ancestors, node.tag], siblingIndex: index };
-    if (child.type === "element") index++;
-    if (partials(child, literals, childWalk, out)) deeper = true;
+  if (!OPAQUE_TAGS.has(node.tag)) {
+    for (const child of node.children) {
+      const childWalk = { ancestors: [...walk.ancestors, node.tag], siblingIndex: index };
+      if (child.type === "element") index++;
+      if (partials(child, literals, childWalk, out)) deeper = true;
+    }
   }
   if (deeper) return true;
   const text = norm(renderedText(node));
@@ -4384,14 +4397,23 @@ function isKnownHelper(content, dialect) {
 
 // src/locate.ts
 var flat = (value) => value.replace(/\s+/g, " ").trim();
-var stripTags = (value) => value.replace(/<br\s*\/?>/gi, " ").replace(/<[^>]*>/g, "");
+var stripTags = (value, br = " ") => value.replace(/<br\s*\/?>/gi, br).replace(/<[^>]*>/g, "");
 function locate(brief, sources, options) {
   const flattened = sources.map((s) => {
     const readable = unwrapJsxStrings(decodeEntities(s.content));
     return {
       path: s.path,
       content: flat(readable),
-      text: options?.stripTags ? flat(stripTags(readable)) : null
+      text: options?.stripTags ? flat(stripTags(readable)) : null,
+      /**
+       * 🔴 A `<br>` IS A SPACE TO A READER AND NOTHING TO `textContent`, and the derive lane reads
+       * the DOM. `<h1>Launch your site<br /><span>fast.</span></h1>` is one INLINE richtext field
+       * whose value is `Launch your sitefast.` — no space, because no text node holds one — while
+       * this prefilter's only reading inserted one, so the file was never opened and every split
+       * heading on every imported site came back NOT_IN_SOURCE. Both readings are searched: this
+       * decides which FILES to open, and `collectSites` still has to match an element exactly.
+       */
+      tight: options?.stripTags ? flat(stripTags(readable, "")) : null
     };
   });
   const located = [];
@@ -4403,7 +4425,9 @@ function locate(brief, sources, options) {
         unlocated.push({ route: page.route, scope: p.scope, path: p.path, reason: "no-original" });
         continue;
       }
-      const files = flattened.filter((f) => f.content.includes(original) || f.text !== null && f.text.includes(original)).map((f) => f.path);
+      const files = flattened.filter(
+        (f) => f.content.includes(original) || f.text !== null && f.text.includes(original) || f.tight !== null && f.tight.includes(original)
+      ).map((f) => f.path);
       if (files.length === 0) {
         unlocated.push({ route: page.route, scope: p.scope, path: p.path, reason: "not-in-source" });
         continue;
@@ -5102,7 +5126,7 @@ function rewriteFile(content, dialect, rewrites, imports, extra = []) {
       addDeclaration(site.attrInsertAt, declaration(rewrite));
       continue;
     }
-    if (kind === "richtext" && !(site.mixed && site.textRange)) {
+    if (ownsSubtree(kind, site, rewrite.inline)) {
       if (!isIntrinsic(site.tag)) continue;
       if (rule.richtext && site.inner) {
         source.remove(site.inner.start, site.inner.end);
@@ -6376,6 +6400,66 @@ function isSourceCandidate(path) {
   return lower.split("/").slice(0, -1).some((segment) => TEMPLATE_DIRS.has(segment));
 }
 
+// src/xray.ts
+var SKIP_TAGS = /* @__PURE__ */ new Set(["svg", "iframe", "head", "title"]);
+var CONTEXT_TAGS = /* @__PURE__ */ new Set([
+  "nav",
+  "header",
+  "footer",
+  "main",
+  "aside",
+  "form",
+  "button",
+  "a",
+  "ul",
+  "ol",
+  "table",
+  "dialog",
+  "details",
+  "figure",
+  "blockquote"
+]);
+var BOUND_ATTRS = ["data-bcms-field", "data-bcms-layout-field", "data-bcms-path", "data-bcms-block"];
+var isDecorativeGlyph = (text) => text.length <= 2 || /^\d{1,2}\s*[/.)\-–]?$/.test(text) || !new RegExp("\\p{L}", "u").test(text) || /^made with\b/i.test(text) || /^skip to\b/i.test(text);
+var MAX_BUCKETS = 12;
+var flat2 = (value) => value.replace(/\s+/g, " ").trim();
+function measureUnaddressable(html) {
+  const parsed = parse2(html);
+  let visible = 0;
+  let unaddressable = 0;
+  const buckets = /* @__PURE__ */ new Map();
+  const visit = (node, tag, context, bound) => {
+    if (node.type === "text") {
+      const text = flat2(node.value);
+      if (!text || isDecorativeGlyph(text)) return;
+      visible += text.length;
+      if (bound) return;
+      unaddressable += text.length;
+      const key = `${tag} @ ${context}`;
+      const held = buckets.get(key) ?? { tag, context, chars: 0, nodes: 0, samples: [] };
+      held.chars += text.length;
+      held.nodes += 1;
+      if (held.samples.length < 2) held.samples.push(text.slice(0, 50));
+      buckets.set(key, held);
+      return;
+    }
+    if (node.type !== "element" || node.raw || SKIP_TAGS.has(node.tag)) return;
+    const attrs = new Map(node.attrs.map((a) => [a.name, a.value]));
+    if (attrs.has("hidden") || attrs.get("aria-hidden") === "true" || /display\s*:\s*none/.test(attrs.get("style") ?? "")) {
+      return;
+    }
+    const owned = bound || BOUND_ATTRS.some((name) => attrs.has(name));
+    const chain = CONTEXT_TAGS.has(node.tag) ? context === "body" ? node.tag : `${context}>${node.tag}` : context;
+    for (const child of node.children) visit(child, node.tag, chain, owned);
+  };
+  for (const root of parsed.roots) visit(root, "body", "body", false);
+  return {
+    visible,
+    unaddressable,
+    buckets: [...buckets.values()].sort((a, b) => b.chars - a.chars).slice(0, MAX_BUCKETS)
+  };
+}
+
 // src/carry.ts
 var GLOBALS = /* @__PURE__ */ new Set([
   "Astro",
@@ -8256,7 +8340,8 @@ async function convertSources(briefIn, sources, options) {
         kind: p.kind,
         fallback: p.original ?? "",
         literal: norm(p.original ?? ""),
-        locator: p.locator
+        locator: p.locator,
+        ...p.inline ? { inline: true } : {}
       });
     }
   }
@@ -8481,6 +8566,16 @@ async function convertSources(briefIn, sources, options) {
       declaredRow(target, witness !== null, witness ?? byHand);
     } else rewritten++;
   }
+  const templated = dynamicBindings.filter((d) => d.prop === null);
+  for (let i = pending.length - 1; i >= 0; i--) {
+    const row = pending[i];
+    const target = targets.get(keyOf(row.route, row.scope, row.path));
+    if (row.reason !== "NO_ORIGINAL" || !target || target.fallback !== "") continue;
+    const shape = shapeOfPath(target.path);
+    if (!templated.some((d) => d.path === shape && attributable(target, d.file))) continue;
+    pending.splice(i, 1);
+    rewritten++;
+  }
   for (const [file, splices] of [...componentEdits].sort(([a], [b]) => a.localeCompare(b))) {
     const source = new MagicString6(contents.get(file));
     const seen = /* @__PURE__ */ new Set();
@@ -8793,6 +8888,10 @@ async function plan(file, content, dialect, targets, sites, importAt, outcomes,
       }
       let refusal = null;
       for (const site of fresh) {
+        if (target.inline && site.where === "text" && !ownsSubtree(target.kind, site, true)) {
+          refusal = "SUBTREE_NOT_HTML";
+          continue;
+        }
         let bindings;
         let bindingsMerge;
         if (site.where === "prop" && site.attr) {
@@ -8827,7 +8926,8 @@ async function plan(file, content, dialect, targets, sites, importAt, outcomes,
           // 🔴 AND ONLY WHEN THE ELEMENT'S SUBTREE IS THE VALUE. A value that is one TEXT NODE of
           // a mixed element gets a wrapper of its own, so taking the element's inner range as its
           // fallback swallowed the `<em>` beside it — the wrapper then rendered the sibling twice.
-          fallback: target.kind === "richtext" && site.inner && !(site.mixed && site.textRange) ? content.slice(site.inner.start, site.inner.end).trim() : target.fallback,
+          ...target.inline ? { inline: true } : {},
+          fallback: site.inner && ownsSubtree(target.kind, site, target.inline) ? content.slice(site.inner.start, site.inner.end).trim() : target.fallback,
           snapshot: snapshotName(target.slug),
           slug: target.slug,
           scope: target.scope,
@@ -9188,7 +9288,7 @@ function emitStubs2(dialects, slugs, contents, out) {
 // package.json
 var package_default = {
   name: "@bettercms-ai/convert",
-  version: "0.9.0",
+  version: "0.10.0",
   type: "module",
   description: "Deterministic codemod that turns an imported site's text-matched bindings into declared ones \u2014 parser offsets in, one magic-string pass out, plus a receipt of everything it could not do.",
   bin: {
@@ -9249,6 +9349,7 @@ function parseArgs(argv) {
     brief: void 0,
     componentize: false,
     forms: false,
+    xray: void 0,
     plan: void 0,
     copy: "bind",
     root: ".",
@@ -9264,6 +9365,7 @@ function parseArgs(argv) {
     if (arg === "--brief") args.brief = argv[++i];
     else if (arg === "--componentize") args.componentize = true;
     else if (arg === "--forms") args.forms = true;
+    else if (arg === "--xray") args.xray = argv[++i];
     else if (arg === "--plan") args.plan = argv[++i];
     else if (arg === "--copy") {
       const mode = argv[++i];
@@ -9291,6 +9393,7 @@ var HELP = `bettercms-convert ${VERSION}
   bettercms-convert --brief <json> --root . [options]
   bettercms-convert --componentize --brief <json> --plan <json> --root . [options]
   bettercms-convert --forms --brief <json> --root . [options]
+  bettercms-convert --xray <dist-dir>
 
   --brief <file>       The conversion brief, as get_conversion_brief returns it. Required.
   --componentize       Lift each planned SECTION into a component the CMS can reorder.
@@ -9298,6 +9401,11 @@ var HELP = `bettercms-convert ${VERSION}
                        endpoint, the id, a marker per field and the submit script. A form
                        the brief lists as a DRAFT is wired and receipted: publish it in the
                        Forms tab, or it rejects every submission with a 403.
+  --xray <dist-dir>    Measure a LOCAL build instead of converting: how much of each page's
+                       visible copy no binding owns, and the biggest buckets of it. Run it
+                       after a build to see what the next conversion still has to reach,
+                       without deploying. Takes no --brief. Exit 2 with --strict when any
+                       page is above 2 %.
   --plan <file>        The componentize plan, as get_componentize_plan returns it.
                        Required with --componentize.
   --copy bind|instance Which copy model the plan is expected to describe. Default: bind
@@ -9418,12 +9526,53 @@ ${extracted} sections extracted (${registered} registered, ${inlineOnly} inline-
     );
   }
 }
+async function xray(dir, strict) {
+  const pages = [];
+  const walk = async (at) => {
+    for (const entry of (await readdir(at, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
+      const full = join2(at, entry.name);
+      if (entry.isDirectory()) await walk(full);
+      else if (/\.html?$/i.test(entry.name)) pages.push(full);
+    }
+  };
+  try {
+    await walk(dir);
+  } catch (error) {
+    throw new CliError(`${dir} could not be read (${error.message}). Build the site first.`);
+  }
+  if (pages.length === 0) throw new CliError(`No .html files under ${dir}. Is that the build output?`);
+  let over = 0;
+  for (const page of pages) {
+    const { visible, unaddressable, buckets } = measureUnaddressable(await readFile(page, "utf-8"));
+    const share = visible === 0 ? 0 : 100 * unaddressable / visible;
+    if (share > 2) over++;
+    process.stdout.write(
+      `${relative(dir, page) || page}  visible=${visible} unaddressable=${unaddressable} (${share.toFixed(1)}%)
+`
+    );
+    for (const bucket of buckets) {
+      const samples = bucket.samples.map((s) => JSON.stringify(s)).join(" | ");
+      process.stdout.write(
+        `  ${String(bucket.chars).padStart(6)}ch ${String(bucket.nodes).padStart(4)}n  ${bucket.tag} @ ${bucket.context}  e.g. ${samples}
+`
+      );
+    }
+  }
+  process.stdout.write(`
+${pages.length - over} of ${pages.length} pages at or under 2% unaddressable
+`);
+  return strict && over > 0 ? 2 : 0;
+}
 async function main() {
   const args = parseArgs(process.argv.slice(2));
   if (args.help) {
     process.stdout.write(HELP);
     return 0;
   }
+  if (args.xray) {
+    if (args.brief) throw new CliError("--xray measures a build; it reads no brief. See --help.");
+    return await xray(resolve(args.xray), args.strict);
+  }
   if (!args.brief) throw new CliError("--brief <file> is required. See --help.");
   if (args.plan && !args.componentize) {
     throw new CliError("--plan is only read with --componentize. See --help.");