@terrazzo/parser 0.0.18 → 0.1.0

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @terrazzo/parser
 
+## 0.1.0
+
+### Minor Changes
+
+- [#319](https://github.com/terrazzoapp/terrazzo/pull/319) [`e7f272d`](https://github.com/terrazzoapp/terrazzo/commit/e7f272defcd889f5a410fdbd30497cf704671b32) Thanks [@drwpow](https://github.com/drwpow)! - ⚠️ Breaking change: dimension and duration tokens normalize to object syntax in plugins (following upcoming changes in DTCG spec; see https://github.com/design-tokens/community-group/pull/244).
+
+### Patch Changes
+
+- Updated dependencies [[`e7f272d`](https://github.com/terrazzoapp/terrazzo/commit/e7f272defcd889f5a410fdbd30497cf704671b32)]:
+  - @terrazzo/token-tools@0.1.0
+
+## 0.0.19
+
+### Patch Changes
+
+- [#313](https://github.com/terrazzoapp/terrazzo/pull/313) [`1408594`](https://github.com/terrazzoapp/terrazzo/commit/1408594de029f57137c936dc2ff9ab949f039215) Thanks [@drwpow](https://github.com/drwpow)! - Fix bug in gradient position aliasing
+
+- [#313](https://github.com/terrazzoapp/terrazzo/pull/313) [`1408594`](https://github.com/terrazzoapp/terrazzo/commit/1408594de029f57137c936dc2ff9ab949f039215) Thanks [@drwpow](https://github.com/drwpow)! - Improve alias type validation
+
 ## 0.0.18
 
 ### Patch Changes

package/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing to @terrazzo/parser
+
+This document contains developer notes and context for @terrazzo/parser. For general contribution guidelines, see [CONTRIBUTING.md](../../CONTRIBUTING.md) in the root.
+
+## What this package does
+
+- Parses, validates, and normalizes tokens.json
+- Executes plugins and builds output
+- Executes linting
+
+## What this package DOES NOT do
+
+- Write files to disk (that’s a job for Node.js; this can be run in a browser)
+
+## Manual JS and DTS
+
+This package is unique in its manual handling of `.js` and `.d.ts` files (TypeScript definitions). The expected norm is to write `.ts`, and have `.js` and `.d.ts` be generated automatically from source. But in this project, we write `.js` by hand because this specific module has several unique concerns:
+
+1. **It’s dealing with external (possibly invalid) code.** Now, you’d think that `.ts` would be better at handling this, but the reality when dealing with user-submitted code is if you type it too strongly, TS lulls you into a false sense of security and misses necessary assertions. If you type it weakly, then TS requires _too many_ assertions and you end up with boilerplate and noise. Or if you don’t type it at all, then TS doesn’t really provide any value in the first place, and you might as well skip it.
+
+2. **It’s dealing with ASTs.** We’re parsing JSON ASTs using Momoa, and building ad hoc structures on-the-fly. ASTs usually trip TS up because the discrimination isn’t clear (e.g. if `.type == 'foo'`, then `[property]` exists). Most ASTs don’t have perfect types out-of-the-box, and attempting to re-type an AST yourself is overkill. Further, to the previous point, we may be dealing with invalid code, and the AST may not always be in predictable shapes! When faced with these problems, TS at best requires extra boilerplate; at worst guides you into incorrect decisions.
+
+Given both of these, you usually end up with so many `// @ts-ignore`s or `as any`s that TS doesn’t provide benefit.
+
+But this package is unique! Look in any other package in this monorepo, and you’ll see it’s fully written in `.ts` source files. Different concerns require different approaches.

package/logger.js

@@ -25,8 +25,12 @@ export function formatMessage(entry, severity) {
   }
   if (entry.src) {
     const start = entry.node?.loc?.start;
-    // note: strip "file://" protocol, but not href
-    message = `${message}\n\n${entry.filename ? `${entry.filename.href.replace(/^file:\/\//, '')}:${start?.line ?? 0}:${start?.column ?? 0}\n\n` : ''}${codeFrameColumns(entry.src, { start }, { highlightCode: false })}`;
+    //  strip "file://" protocol, but not href
+    const loc = entry.filename
+      ? `${entry.filename?.href.replace(/^file:\/\//, '')}:${start?.line ?? 0}:${start?.column ?? 0}\n\n`
+      : '';
+    const codeFrame = codeFrameColumns(entry.src, { start }, { highlightCode: false });
+    message = `${message}\n\n${loc}${codeFrame}`;
   }
   return message;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@terrazzo/parser",
-  "version": "0.0.18",
+  "version": "0.1.0",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
   "type": "module",
   "author": {
@@ -30,16 +30,16 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@humanwhocodes/momoa": "^3.2.1",
+    "@humanwhocodes/momoa": "^3.3.0",
     "@types/babel__code-frame": "^7.0.6",
     "@types/culori": "^2.1.1",
     "culori": "^4.0.1",
     "is-what": "^4.1.16",
     "merge-anything": "^5.1.7",
-    "picocolors": "^1.1.0",
+    "picocolors": "^1.1.1",
     "wildcard-match": "^5.1.3",
-    "yaml": "^2.5.1",
-    "@terrazzo/token-tools": "^0.0.9"
+    "yaml": "^2.6.0",
+    "@terrazzo/token-tools": "^0.1.0"
   },
   "devDependencies": {
     "esbuild": "^0.23.1",

package/parse/index.js

@@ -131,7 +131,12 @@ export default async function parse(
       if (mode === '.') {
         continue; // skip shadow of root value
       }
-      applyAliases(tokens[id].mode[mode], { tokens, node: tokens[id].mode[mode].source.node, logger });
+      applyAliases(tokens[id].mode[mode], {
+        tokens,
+        node: tokens[id].mode[mode].source.node,
+        logger,
+        src: sources[tokens[id].source.loc]?.src,
+      });
     }
   }
   logger.debug({
@@ -235,10 +240,11 @@ async function parseSingle(input, { filename, logger, config, skipLint, continue
           $typeInheritance[path.join('.') || '.'] = node.value.members.find((m) => m.name.value === '$type');
         }
 
+        const id = path.join('.');
+
         if (members.$value) {
           const extensions = members.$extensions ? getObjMembers(members.$extensions) : undefined;
           const sourceNode = structuredClone(node);
-          const id = path.join('.');
 
           // get parent type by taking the closest-scoped $type (length === closer)
           let parent$type;
@@ -304,7 +310,7 @@ async function parseSingle(input, { filename, logger, config, skipLint, continue
           }
 
           tokens[id] = token;
-        } else if (members.value) {
+        } else if (!id.includes('.$value') && members.value) {
           logger.warn({ message: `Group ${id} has "value". Did you mean "$value"?`, filename, node, src });
         }
       }
@@ -419,10 +425,10 @@ export function maybeJSONString(input) {
 export function resolveAlias(alias, { tokens, logger, filename, src, node, scanned = [] }) {
   const { id } = parseAlias(alias);
   if (!tokens[id]) {
-    logger.error({ message: `Alias "${alias}" not found`, filename, src, node });
+    logger.error({ message: `Alias "${alias}" not found.`, filename, src, node });
   }
   if (scanned.includes(id)) {
-    logger.error({ message: `Circular alias detected from "${alias}"`, filename, src, node });
+    logger.error({ message: `Circular alias detected from "${alias}".`, filename, src, node });
   }
   const token = tokens[id];
   if (!isAlias(token.$value)) {
@@ -431,8 +437,43 @@ export function resolveAlias(alias, { tokens, logger, filename, src, node, scann
   return resolveAlias(token.$value, { tokens, logger, filename, node, src, scanned: [...scanned, id] });
 }
 
+/** Throw error if resolved alias for composite properties doesn’t match expected $type. */
+const COMPOSITE_TYPE_VALUES = {
+  border: {
+    color: ['color'],
+    width: ['dimension'],
+    strokeStyle: ['strokeStyle'],
+  },
+  gradient: {
+    color: ['color'],
+    position: ['number'],
+  },
+  shadow: {
+    color: ['color'],
+    position: ['dimension'],
+  },
+  strokeStyle: {
+    dashArray: ['dimension'],
+  },
+  transition: {
+    duration: ['duration'],
+    delay: ['duration'],
+    timingFunction: ['cubicBezier'],
+  },
+  typography: {
+    fontFamily: ['fontFamily'],
+    fontSize: ['dimension'],
+    fontWeight: ['fontWeight'],
+    letterSpacing: ['dimension'],
+    lineHeight: ['dimension', 'number'],
+  },
+};
+
 /** Resolve aliases, update values, and mutate `token` to add `aliasOf` / `partialAliasOf` */
 function applyAliases(token, { tokens, logger, filename, src, node }) {
+  const $valueNode = (node.type === 'Object' && node.members.find((m) => m.name.value === '$value')?.value) || node;
+  const expectedAliasTypes = COMPOSITE_TYPE_VALUES[token.$type];
+
   // handle simple aliases
   if (isAlias(token.$value)) {
     const aliasOfID = resolveAlias(token.$value, { tokens, logger, filename, node, src });
@@ -441,15 +482,16 @@ function applyAliases(token, { tokens, logger, filename, src, node }) {
     token.aliasOf = aliasOfID;
     token.$value = aliasOf.mode[aliasMode]?.$value || aliasOf.$value;
     if (token.$type && token.$type !== aliasOf.$type) {
-      logger.warn({
-        message: `Token ${token.id} has $type "${token.$type}" but aliased ${aliasOfID} of $type "${aliasOf.$type}"`,
-        node,
+      logger.error({
+        message: `Invalid alias: expected $type: "${token.$type}", received $type: "${aliasOf.$type}".`,
+        node: $valueNode,
+        filename,
+        src,
       });
-      token.$type = aliasOf.$type;
-    } else {
-      token.$type = aliasOf.$type;
     }
+    token.$type = aliasOf.$type;
   }
+
   // handle aliases within array values (e.g. cubicBezier, gradient)
   else if (Array.isArray(token.$value)) {
     // some arrays are primitives, some are objects. handle both
@@ -473,8 +515,20 @@ function applyAliases(token, { tokens, logger, filename, src, node }) {
             }
             const aliasOfID = resolveAlias(token.$value[i][property], { tokens, logger, filename, node, src });
             const { id: aliasID, mode: aliasMode } = parseAlias(token.$value[i][property]);
+            const aliasToken = tokens[aliasOfID];
+
+            if (expectedAliasTypes?.[property] && !expectedAliasTypes[property].includes(aliasToken.$type)) {
+              const elementNode = $valueNode.elements[i].value;
+              logger.error({
+                message: `Invalid alias: expected $type: "${expectedAliasTypes[property].join('" or "')}", received $type: "${aliasToken.$type}".`,
+                node: elementNode.members.find((m) => m.name.value === property).value,
+                filename,
+                src,
+              });
+            }
+
             token.partialAliasOf[i][property] = aliasID; // also keep the shallow alias here, too!
-            token.$value[i][property] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
+            token.$value[i][property] = aliasToken.mode[aliasMode]?.$value || aliasToken.$value;
           }
         }
       }
@@ -494,8 +548,18 @@ function applyAliases(token, { tokens, logger, filename, src, node }) {
         const aliasOfID = resolveAlias(token.$value[property], { tokens, logger, filename, node, src });
         const { id: aliasID, mode: aliasMode } = parseAlias(token.$value[property]);
         token.partialAliasOf[property] = aliasID; // keep the shallow alias!
-        token.$value[property] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
+        const aliasToken = tokens[aliasOfID];
+        if (expectedAliasTypes?.[property] && !expectedAliasTypes[property].includes(aliasToken.$type)) {
+          logger.error({
+            message: `Invalid alias: expected $type: "${expectedAliasTypes[property].join('" or "')}", received $type: "${aliasToken.$type}".`,
+            node: $valueNode.members.find((m) => m.name.value === property).value,
+            filename,
+            src,
+          });
+        }
+        token.$value[property] = aliasToken.mode[aliasMode]?.$value || aliasToken.$value;
       }
+
       // strokeStyle has an array within an object
       else if (Array.isArray(token.$value[property])) {
         for (let i = 0; i < token.$value[property].length; i++) {
@@ -509,6 +573,16 @@ function applyAliases(token, { tokens, logger, filename, src, node }) {
             }
             const { id: aliasID, mode: aliasMode } = parseAlias(token.$value[property][i]);
             token.partialAliasOf[property][i] = aliasID; // keep the shallow alias!
+            const aliasToken = tokens[aliasOfID];
+            if (expectedAliasTypes?.[property] && !expectedAliasTypes[property].includes(aliasToken.$type)) {
+              const arrayNode = $valueNode.members.find((m) => m.name.value === property).value;
+              logger.error({
+                message: `Invalid alias: expected $type: "${expectedAliasTypes[property].join('" or "')}", received $type: "${aliasToken.$type}".`,
+                node: arrayNode.elements[i],
+                filename,
+                src,
+              });
+            }
             token.$value[property][i] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
           }
         }

package/parse/normalize.js

@@ -21,6 +21,8 @@ export const FONT_WEIGHT_MAP = {
   'ultra-black': 950,
 };
 
+const NUMBER_WITH_UNIT_RE = /(-?\d*\.?\d+)(.*)/;
+
 export default function normalizeValue(token) {
   if (isAlias(token.$value)) {
     return token.$value;
@@ -49,15 +51,25 @@ export default function normalizeValue(token) {
     }
     case 'dimension': {
       if (token.$value === 0) {
-        return '0';
+        return { value: 0, unit: 'px' };
+      }
+      // Backwards compat: handle string
+      if (typeof token.$value === 'string') {
+        const match = token.$value.match(NUMBER_WITH_UNIT_RE);
+        return { value: Number.parseFloat(match?.[1] || token.$value), unit: match[2] || 'px' };
       }
-      return typeof token.$value === 'number' ? `${token.$value}px` : token.$value;
+      return token.$value;
     }
     case 'duration': {
       if (token.$value === 0) {
-        return 0;
+        return { value: 0, unit: 'ms' };
       }
-      return typeof token.$value === 'number' ? `${token.$value}ms` : token.$value;
+      // Backwards compat: handle string
+      if (typeof token.$value === 'string') {
+        const match = token.$value.match(NUMBER_WITH_UNIT_RE);
+        return { value: Number.parseFloat(match?.[1] || token.$value), unit: match[2] || 'ms' };
+      }
+      return token.$value;
     }
     case 'fontFamily': {
       return Array.isArray(token.$value) ? token.$value : [token.$value];
@@ -73,7 +85,7 @@ export default function normalizeValue(token) {
       for (let i = 0; i < token.$value.length; i++) {
         const stop = { ...token.$value[i] };
         stop.color = normalizeValue({ $type: 'color', $value: stop.color });
-        if (typeof stop.position !== 'number') {
+        if (stop.position === undefined) {
           stop.position = i / (token.$value.length - 1);
         }
         output.push(stop);

package/parse/validate.d.ts

@@ -12,7 +12,7 @@ export interface ValidateOptions {
   logger: Logger;
 }
 
-export function validateAlias($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+export function validateAliasSyntax($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
 
 export function validateBorder($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
 

package/parse/validate.js

@@ -92,7 +92,7 @@ function validateMembersAs($value, properties, node, { filename, src, logger })
     }
     const value = members[property];
     if (isMaybeAlias(value)) {
-      validateAlias(value, node, { filename, src, logger });
+      validateAliasSyntax(value, node, { filename, src, logger });
     } else {
       validator(value, node, { filename, src, logger });
     }
@@ -100,13 +100,13 @@ function validateMembersAs($value, properties, node, { filename, src, logger })
 }
 
 /**
- * Verify an Alias token is valid
+ * Verify an Alias $value is formatted correctly
  * @param {ValueNode} $value
  * @param {AnyNode} node
  * @param {ValidateOptions} options
  * @return {void}
  */
-export function validateAlias($value, node, { filename, src, logger }) {
+export function validateAliasSyntax($value, node, { filename, src, logger }) {
   if ($value.type !== 'String' || !isAlias($value.value)) {
     logger.error({ message: `Invalid alias: ${print($value)}`, filename, node: $value, src });
   }
@@ -262,13 +262,43 @@ export function validateDimension($value, node, { filename, src, logger }) {
   if ($value.type === 'Number' && $value.value === 0) {
     return; // `0` is a valid number
   }
+  if ($value.type === 'Object') {
+    const { value, unit } = getObjMembers($value);
+    if (!value) {
+      logger.error({ message: 'Missing required property "value".', filename, node: $value, src });
+    }
+    if (!unit) {
+      logger.error({ message: 'Missing required property "unit".', filename, node: $value, src });
+    }
+    if (value.type !== 'Number') {
+      logger.error({ message: `Expected number, received ${value.type}`, filename, node: value, src });
+    }
+    if (!['px', 'em', 'rem'].includes(unit.value)) {
+      logger.error({
+        message: `Expected unit "px", "em", or "rem", received ${print(unit)}`,
+        filename,
+        node: unit,
+        src,
+      });
+    }
+    return;
+  }
+  // Backwards compat: string
   if ($value.type !== 'String') {
     logger.error({ message: `Expected string, received ${$value.type}`, filename, node: $value, src });
-  } else if ($value.value === '') {
+  }
+  const value = $value.value.match(/^-?[0-9.]+/)?.[0];
+  const unit = $value.value.replace(value, '');
+  if ($value.value === '') {
     logger.error({ message: 'Expected dimension, received empty string', filename, node: $value, src });
-  } else if (String(Number.parseFloat($value.value)) === $value.value) {
-    logger.error({ message: 'Missing units', filename, node: $value, src });
-  } else if (!/^-?[0-9]+(\.[0-9]+)?/.test($value.value)) {
+  } else if (!['px', 'em', 'rem'].includes(unit)) {
+    logger.error({
+      message: `Expected unit "px", "em", or "rem", received ${JSON.stringify(unit || $value.value)}`,
+      filename,
+      node: $value,
+      src,
+    });
+  } else if (!Number.isFinite(Number.parseFloat(value))) {
     logger.error({ message: `Expected dimension with units, received ${print($value)}`, filename, node: $value, src });
   }
 }
@@ -284,19 +314,39 @@ export function validateDuration($value, node, { filename, src, logger }) {
   if ($value.type === 'Number' && $value.value === 0) {
     return; // `0` is a valid number
   }
+  if ($value.type === 'Object') {
+    const { value, unit } = getObjMembers($value);
+    if (!value) {
+      logger.error({ message: 'Missing required property "value".', filename, node: $value, src });
+    }
+    if (!unit) {
+      logger.error({ message: 'Missing required property "unit".', filename, node: $value, src });
+    }
+    if (value.type !== 'Number') {
+      logger.error({ message: `Expected number, received ${value.type}`, filename, node: value, src });
+    }
+    if (!['ms', 's'].includes(unit.value)) {
+      logger.error({ message: `Expected unit "ms" or "s", received ${print(unit)}`, filename, node: unit, src });
+    }
+    return;
+  }
+  // Backwards compat: string
   if ($value.type !== 'String') {
     logger.error({ message: `Expected string, received ${$value.type}`, filename, node: $value, src });
-  } else if ($value.value === '') {
+  }
+  const value = $value.value.match(/^-?[0-9.]+/)?.[0];
+  const unit = $value.value.replace(value, '');
+  if ($value.value === '') {
     logger.error({ message: 'Expected duration, received empty string', filename, node: $value, src });
-  } else if (!/m?s$/.test($value.value)) {
-    logger.error({ message: 'Missing unit "ms" or "s"', filename, node: $value, src });
-  } else if (!/^[0-9]+/.test($value.value)) {
+  } else if (!['ms', 's'].includes(unit)) {
     logger.error({
-      message: `Expected duration in \`ms\` or \`s\`, received ${print($value)}`,
+      message: `Expected unit "ms" or "s", received ${JSON.stringify(unit || $value.value)}`,
       filename,
       node: $value,
       src,
     });
+  } else if (!Number.isFinite(Number.parseFloat(value))) {
+    logger.error({ message: `Expected duration with units, received ${print($value)}`, filename, node: $value, src });
   }
 }
 
@@ -476,7 +526,7 @@ export function validateStrokeStyle($value, node, { filename, src, logger }) {
       for (const element of dashArray.elements) {
         if (element.value.type === 'String' && element.value.value !== '') {
           if (isMaybeAlias(element.value)) {
-            validateAlias(element.value, node, { logger, src });
+            validateAliasSyntax(element.value, node, { logger, src });
           } else {
             validateDimension(element.value, node, { logger, src });
           }
@@ -550,7 +600,7 @@ export default function validate(node, { filename, src, logger }) {
   // If top-level value is a valid alias, this is valid (no need for $type)
   // ⚠️ Important: ALL Object and Array nodes below will need to check for aliases within!
   if (isMaybeAlias($value)) {
-    validateAlias($value, node, { logger, src });
+    validateAliasSyntax($value, node, { logger, src });
     return;
   }