@turntrout/subfont 1.0.0

package/lib/extractReferencedCustomPropertyNames.js

@@ -0,0 +1,20 @@
+const postcssValueParser = require('postcss-value-parser');
+
+function extractReferencedCustomPropertyNames(cssValue) {
+  const rootNode = postcssValueParser(cssValue);
+  const customPropertyNames = new Set();
+  for (const node of rootNode.nodes) {
+    if (
+      node.type === 'function' &&
+      node.value === 'var' &&
+      node.nodes.length >= 1 &&
+      node.nodes[0].type === 'word' &&
+      /^--/.test(node.nodes[0].value)
+    ) {
+      customPropertyNames.add(node.nodes[0].value);
+    }
+  }
+  return customPropertyNames;
+}
+
+module.exports = extractReferencedCustomPropertyNames;

package/lib/extractVisibleText.js

@@ -0,0 +1,64 @@
+const parse5 = require('parse5');
+
+const INVISIBLE_ELEMENTS = new Set(['script', 'style', 'svg', 'template']);
+const TEXT_ATTRIBUTES = new Set([
+  'alt',
+  'title',
+  'placeholder',
+  'value',
+  'aria-label',
+]);
+
+/**
+ * Fast extraction of visible text content from HTML source.
+ * Used as a lightweight alternative to full font-tracer for pages
+ * that share the same CSS configuration as an already-traced page.
+ *
+ * Walks the parse5 tree collecting text nodes and content attributes
+ * (alt, title, placeholder, value, aria-label), skipping invisible
+ * elements (script, style, svg, template).
+ */
+function extractVisibleText(html) {
+  const document = parse5.parse(html);
+  const parts = [];
+
+  function walk(node) {
+    if (node.nodeName && INVISIBLE_ELEMENTS.has(node.nodeName)) {
+      return;
+    }
+
+    // Collect relevant attribute values
+    if (node.attrs) {
+      const isHiddenInput =
+        node.nodeName === 'input' &&
+        node.attrs.some(
+          (a) => a.name === 'type' && a.value.toLowerCase() === 'hidden'
+        );
+      for (const attr of node.attrs) {
+        if (TEXT_ATTRIBUTES.has(attr.name) && attr.value) {
+          if (attr.name === 'value' && isHiddenInput) {
+            continue;
+          }
+          parts.push(attr.value);
+        }
+      }
+    }
+
+    // Collect text content
+    if (node.nodeName === '#text' && node.value) {
+      parts.push(node.value);
+    }
+
+    // Recurse into child nodes
+    if (node.childNodes) {
+      for (const child of node.childNodes) {
+        walk(child);
+      }
+    }
+  }
+
+  walk(document);
+  return parts.join(' ');
+}
+
+module.exports = extractVisibleText;

package/lib/findCustomPropertyDefinitions.js

@@ -0,0 +1,54 @@
+const extractReferencedCustomPropertyNames = require('./extractReferencedCustomPropertyNames');
+
+// Find all custom property definitions grouped by the custom properties they contribute to
+function findCustomPropertyDefinitions(cssAssets) {
+  const definitionsByProp = {};
+  const incomingReferencesByProp = {};
+  for (const cssAsset of cssAssets) {
+    cssAsset.eachRuleInParseTree((cssRule) => {
+      if (
+        cssRule.parent.type === 'rule' &&
+        cssRule.type === 'decl' &&
+        /^--/.test(cssRule.prop)
+      ) {
+        (definitionsByProp[cssRule.prop] =
+          definitionsByProp[cssRule.prop] || new Set()).add(cssRule);
+        for (const customPropertyName of extractReferencedCustomPropertyNames(
+          cssRule.value
+        )) {
+          (incomingReferencesByProp[cssRule.prop] =
+            incomingReferencesByProp[cssRule.prop] || new Set()).add(
+            customPropertyName
+          );
+        }
+      }
+    });
+  }
+  const expandedDefinitionsByProp = {};
+  for (const prop of Object.keys(definitionsByProp)) {
+    expandedDefinitionsByProp[prop] = new Set();
+    const seenProps = new Set();
+    const queue = [prop];
+    while (queue.length > 0) {
+      const referencedProp = queue.shift();
+      if (!seenProps.has(referencedProp)) {
+        seenProps.add(referencedProp);
+        if (definitionsByProp[referencedProp]) {
+          for (const cssRule of definitionsByProp[referencedProp]) {
+            expandedDefinitionsByProp[prop].add(cssRule);
+          }
+        }
+        const incomingReferences = incomingReferencesByProp[referencedProp];
+        if (incomingReferences) {
+          for (const incomingReference of incomingReferences) {
+            queue.push(incomingReference);
+          }
+        }
+      }
+    }
+  }
+
+  return expandedDefinitionsByProp;
+}
+
+module.exports = findCustomPropertyDefinitions;

package/lib/fontFaceHelpers.js

@@ -0,0 +1,292 @@
+const crypto = require('crypto');
+const stripLocalTokens = require('./stripLocalTokens');
+const unicodeRange = require('./unicodeRange');
+const normalizeFontPropertyValue = require('./normalizeFontPropertyValue');
+
+const contentTypeByFontFormat = {
+  woff: 'font/woff', // https://tools.ietf.org/html/rfc8081#section-4.4.5
+  woff2: 'font/woff2',
+  truetype: 'font/ttf',
+};
+
+function stringifyFontFamily(name) {
+  if (/[^a-z0-9_-]/i.test(name)) {
+    return `"${name.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
+  } else {
+    return name;
+  }
+}
+
+function maybeCssQuote(value) {
+  if (/^\w+$/.test(value)) {
+    return value;
+  } else {
+    return `'${value.replace(/'/g, "\\'")}'`;
+  }
+}
+
+function getPreferredFontUrl(cssFontFaceSrcRelations = []) {
+  const formatOrder = ['woff2', 'woff', 'truetype', 'opentype'];
+
+  const typeOrder = ['Woff2', 'Woff', 'Ttf', 'Otf'];
+
+  for (const format of formatOrder) {
+    const relation = cssFontFaceSrcRelations.find(
+      (r) => r.format && r.format.toLowerCase() === format
+    );
+
+    if (relation) {
+      return relation.to.url;
+    }
+  }
+
+  for (const assetType of typeOrder) {
+    const relation = cssFontFaceSrcRelations.find(
+      (r) => r.to.type === assetType
+    );
+
+    if (relation) {
+      return relation.to.url;
+    }
+  }
+
+  return undefined;
+}
+
+// Temporarily switch all relation hrefs to absolute so that
+// node.toString() emits fully-qualified URLs in the @font-face src.
+function getFontFaceDeclarationText(node, relations) {
+  const originalHrefTypeByRelation = new Map();
+  for (const relation of relations) {
+    originalHrefTypeByRelation.set(relation, relation.hrefType);
+    relation.hrefType = 'absolute';
+  }
+
+  const text = node.toString();
+  // Put the hrefTypes that were set to absolute back to their original state:
+  for (const [
+    relation,
+    originalHrefType,
+  ] of originalHrefTypeByRelation.entries()) {
+    relation.hrefType = originalHrefType;
+  }
+  return text;
+}
+
+const fontOrder = ['woff2', 'woff', 'truetype'];
+
+function getFontFaceForFontUsage(fontUsage) {
+  const subsets = fontOrder
+    .filter((format) => fontUsage.subsets[format])
+    .map((format) => ({
+      format,
+      url: `data:${contentTypeByFontFormat[format]};base64,${fontUsage.subsets[
+        format
+      ].toString('base64')}`,
+    }));
+
+  const resultString = ['@font-face {'];
+
+  resultString.push(
+    ...Object.keys(fontUsage.props)
+      .sort()
+      .map((prop) => {
+        let value = fontUsage.props[prop];
+
+        if (prop === 'font-family') {
+          value = maybeCssQuote(`${value}__subset`);
+        }
+
+        if (prop === 'src') {
+          value = subsets
+            .map((subset) => `url(${subset.url}) format('${subset.format}')`)
+            .join(', ');
+        }
+
+        return `${prop}: ${value};`;
+      })
+      .map((str) => `  ${str}`)
+  );
+
+  // Intersect used codepoints with original (font's character set) so
+  // the unicode-range only advertises characters actually in the subset.
+  // This is essential for unicode-range-split fonts (e.g. CJK) where
+  // the text may contain characters outside this font file's range.
+  let effectiveUsedCodepoints = fontUsage.codepoints.used;
+  if (
+    fontUsage.codepoints.original &&
+    fontUsage.codepoints.original.length > 0
+  ) {
+    const originalSet = new Set(fontUsage.codepoints.original);
+    const filtered = fontUsage.codepoints.used.filter((cp) =>
+      originalSet.has(cp)
+    );
+    if (filtered.length > 0) {
+      effectiveUsedCodepoints = filtered;
+    }
+  }
+  resultString.push(
+    `  unicode-range: ${unicodeRange(effectiveUsedCodepoints)};`
+  );
+
+  resultString.push('}');
+
+  return resultString.join('\n');
+}
+
+function getUnusedVariantsStylesheet(
+  fontUsages,
+  accumulatedFontFaceDeclarations
+) {
+  // Find the available @font-face declarations where the font-family is used
+  // (so there will be subsets created), but the specific variant isn't used.
+  return accumulatedFontFaceDeclarations
+    .filter(
+      (decl) =>
+        fontUsages.some((fontUsage) =>
+          fontUsage.fontFamilies.has(decl['font-family'])
+        ) &&
+        !fontUsages.some(
+          ({ props }) =>
+            props['font-style'] === decl['font-style'] &&
+            props['font-weight'] === decl['font-weight'] &&
+            props['font-stretch'] === decl['font-stretch'] &&
+            props['font-family'].toLowerCase() ===
+              decl['font-family'].toLowerCase()
+        )
+    )
+    .map((props) => {
+      let src = stripLocalTokens(props.src);
+      if (props.relations.length > 0) {
+        const targets = props.relations.map((relation) => relation.to.url);
+        src = src.replace(
+          props.relations[0].tokenRegExp,
+          () => `url('${targets.shift().replace(/'/g, "\\'")}')`
+        );
+      }
+      let rule = `@font-face{font-family:${maybeCssQuote(`${props['font-family']}__subset`)};font-stretch:${props['font-stretch']};font-style:${props['font-style']};font-weight:${props['font-weight']};src:${src}`;
+      if (props['unicode-range']) {
+        rule += `;unicode-range:${props['unicode-range']}`;
+      }
+      // Preserve @font-face metric descriptors used for CLS optimization
+      for (const descriptor of [
+        'size-adjust',
+        'ascent-override',
+        'descent-override',
+        'line-gap-override',
+      ]) {
+        if (props[descriptor]) {
+          rule += `;${descriptor}:${props[descriptor]}`;
+        }
+      }
+      rule += '}';
+      return rule;
+    })
+    .join('');
+}
+
+function getFontUsageStylesheet(fontUsages) {
+  return fontUsages
+    .filter((fontUsage) => fontUsage.subsets)
+    .map((fontUsage) => getFontFaceForFontUsage(fontUsage))
+    .join('');
+}
+
+function getCodepoints(text) {
+  const codepointSet = new Set();
+  for (const char of text) {
+    codepointSet.add(char.codePointAt(0));
+  }
+
+  // Make sure that space is always part of the subset fonts (and that it's announced in unicode-range).
+  // Prevents Chrome from going off and downloading the fallback:
+  // https://gitter.im/assetgraph/assetgraph?at=5f01f6e13a0d3931fad4021b
+  codepointSet.add(32);
+
+  return [...codepointSet];
+}
+
+function cssAssetIsEmpty(cssAsset) {
+  return cssAsset.parseTree.nodes.every(
+    (node) => node.type === 'comment' && !node.text.startsWith('!')
+  );
+}
+
+function parseFontWeightRange(str) {
+  if (typeof str === 'undefined' || str === 'auto') {
+    return [-Infinity, Infinity];
+  }
+  let minFontWeight = 400;
+  let maxFontWeight = 400;
+  const fontWeightTokens = str.split(/\s+/).map((str) => parseFloat(str));
+  if (
+    [1, 2].includes(fontWeightTokens.length) &&
+    !fontWeightTokens.some(isNaN)
+  ) {
+    minFontWeight = maxFontWeight = fontWeightTokens[0];
+    if (fontWeightTokens.length === 2) {
+      maxFontWeight = fontWeightTokens[1];
+    }
+  }
+  return [minFontWeight, maxFontWeight];
+}
+
+function parseFontStretchRange(str) {
+  if (typeof str === 'undefined' || str.toLowerCase() === 'auto') {
+    return [-Infinity, Infinity];
+  }
+  let minFontStretch = 100;
+  let maxFontStretch = 100;
+  const fontStretchTokens = str
+    .split(/\s+/)
+    .map((str) => parseFloat(normalizeFontPropertyValue('font-stretch', str)));
+  if (
+    [1, 2].includes(fontStretchTokens.length) &&
+    !fontStretchTokens.some(isNaN)
+  ) {
+    minFontStretch = maxFontStretch = fontStretchTokens[0];
+    if (fontStretchTokens.length === 2) {
+      maxFontStretch = fontStretchTokens[1];
+    }
+  }
+  return [minFontStretch, maxFontStretch];
+}
+
+function uniqueChars(text) {
+  return [...new Set(text)].sort().join('');
+}
+
+function uniqueCharsFromArray(texts) {
+  const charSet = new Set();
+  for (const text of texts) {
+    for (const char of text) {
+      charSet.add(char);
+    }
+  }
+  return [...charSet].sort().join('');
+}
+
+function hashHexPrefix(stringOrBuffer) {
+  return crypto
+    .createHash('sha256')
+    .update(stringOrBuffer)
+    .digest('hex')
+    .slice(0, 10);
+}
+
+module.exports = {
+  stringifyFontFamily,
+  maybeCssQuote,
+  getPreferredFontUrl,
+  getFontFaceDeclarationText,
+  getFontFaceForFontUsage,
+  getUnusedVariantsStylesheet,
+  getFontUsageStylesheet,
+  getCodepoints,
+  cssAssetIsEmpty,
+  parseFontWeightRange,
+  parseFontStretchRange,
+  uniqueChars,
+  uniqueCharsFromArray,
+  hashHexPrefix,
+};

package/lib/fontTracerWorker.js

@@ -0,0 +1,76 @@
+/**
+ * Worker thread for running fontTracer in parallel.
+ *
+ * Receives: { type: 'trace', taskId, htmlText, stylesheetsWithPredicates }
+ * Returns:  { type: 'result', taskId, textByProps: [{text, props}] }
+ *
+ * Re-parses HTML with jsdom inside the worker since DOM objects
+ * cannot be transferred via structured clone.
+ */
+
+const { parentPort } = require('worker_threads');
+const { JSDOM } = require('jsdom');
+const postcss = require('postcss');
+const memoizeSync = require('memoizesync');
+const fontTracer = require('font-tracer');
+const getCssRulesByProperty = require('./getCssRulesByProperty');
+
+// Each worker gets its own memoized getCssRulesByProperty instance.
+// Since pages on the same site typically share stylesheets, the
+// memoization is effective even within a single worker processing
+// multiple pages sequentially.
+const memoizedGetCssRulesByProperty = memoizeSync(getCssRulesByProperty);
+
+parentPort.on('message', (msg) => {
+  if (msg.type === 'init') {
+    parentPort.postMessage({ type: 'ready' });
+    return;
+  }
+
+  if (msg.type === 'trace') {
+    try {
+      const { taskId, htmlText, stylesheetsWithPredicates: serialized } = msg;
+
+      // Re-parse HTML with jsdom to get a DOM document
+      const dom = new JSDOM(htmlText);
+      const document = dom.window.document;
+
+      // Re-parse CSS from serialized text — asset objects with PostCSS
+      // trees can't cross the structured clone boundary.
+      const stylesheetsWithPredicates = serialized.map((entry) => ({
+        asset: { parseTree: postcss.parse(entry.text) },
+        text: entry.text,
+        predicates: entry.predicates,
+      }));
+
+      // Run fontTracer — asset is undefined (skips conditional comments
+      // and noscript traversal, which is acceptable for modern sites)
+      const textByProps = fontTracer(document, {
+        stylesheetsWithPredicates,
+        getCssRulesByProperty: memoizedGetCssRulesByProperty,
+      });
+
+      // Clean up jsdom to free memory
+      dom.window.close();
+
+      // Strip any non-serializable data from results
+      const serializableResults = textByProps.map((entry) => ({
+        text: entry.text,
+        props: { ...entry.props },
+      }));
+
+      parentPort.postMessage({
+        type: 'result',
+        taskId,
+        textByProps: serializableResults,
+      });
+    } catch (err) {
+      parentPort.postMessage({
+        type: 'error',
+        taskId: msg.taskId,
+        error: err.message,
+        stack: err.stack,
+      });
+    }
+  }
+});

package/lib/gatherStylesheetsWithPredicates.js

@@ -0,0 +1,87 @@
+module.exports = function gatherStylesheetsWithPredicates(
+  assetGraph,
+  htmlAsset,
+  relationIndex
+) {
+  const assetStack = [];
+  const incomingMedia = [];
+  const conditionalCommentConditionStack = [];
+  const result = [];
+  (function traverse(asset, isWithinNotIeConditionalComment, isWithinNoscript) {
+    if (assetStack.includes(asset)) {
+      // Cycle detected
+      return;
+    } else if (!asset.isLoaded) {
+      return;
+    }
+    assetStack.push(asset);
+    // Use pre-built index if available, otherwise fall back to findRelations
+    const relations = relationIndex
+      ? relationIndex.get(asset) || []
+      : assetGraph.findRelations({
+          from: asset,
+          type: {
+            $in: [
+              'HtmlStyle',
+              'SvgStyle',
+              'CssImport',
+              'HtmlConditionalComment',
+              'HtmlNoscript',
+            ],
+          },
+        });
+    for (const relation of relations) {
+      if (relation.type === 'HtmlNoscript') {
+        traverse(relation.to, isWithinNotIeConditionalComment, true);
+      } else if (relation.type === 'HtmlConditionalComment') {
+        conditionalCommentConditionStack.push(relation.condition);
+        traverse(
+          relation.to,
+          isWithinNotIeConditionalComment ||
+            (relation.conditionalComments &&
+              relation.conditionalComments.length > 0),
+          isWithinNoscript
+        );
+        conditionalCommentConditionStack.pop();
+      } else {
+        const media = relation.media;
+        if (media) {
+          incomingMedia.push(media);
+        }
+        traverse(
+          relation.to,
+          isWithinNotIeConditionalComment ||
+            (relation.conditionalComments &&
+              relation.conditionalComments.length > 0),
+          isWithinNoscript
+        );
+        if (media) {
+          incomingMedia.pop();
+        }
+      }
+    }
+    assetStack.pop();
+    if (asset.type === 'Css') {
+      const predicates = {};
+      for (const incomingMedium of incomingMedia) {
+        predicates[`mediaQuery:${incomingMedium}`] = true;
+      }
+      for (const conditionalCommentCondition of conditionalCommentConditionStack) {
+        predicates[`conditionalComment:${conditionalCommentCondition}`] = true;
+      }
+      if (isWithinNoscript) {
+        predicates.script = false;
+      }
+      if (isWithinNotIeConditionalComment) {
+        predicates['conditionalComment:IE'] = false;
+      }
+      result.push({
+        asset,
+        text: asset.text,
+        predicates,
+      });
+    }
+  })(htmlAsset);
+
+  return result;
+};