@terrazzo/parser 0.0.19 → 0.1.1

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @terrazzo/parser
 
+## 0.1.1
+
+### Patch Changes
+
+- [#339](https://github.com/terrazzoapp/terrazzo/pull/339) [`9197405`](https://github.com/terrazzoapp/terrazzo/commit/9197405209d560f406494b6bd7aa1634608999c6) Thanks [@tomasfrancisco](https://github.com/tomasfrancisco)! - Fix missing letter spacing transformation as a dimension token
+
+- Updated dependencies [[`9197405`](https://github.com/terrazzoapp/terrazzo/commit/9197405209d560f406494b6bd7aa1634608999c6), [`a637f67`](https://github.com/terrazzoapp/terrazzo/commit/a637f67e20009ce5eef1d5bc5b115cfa00b002d4)]:
+  - @terrazzo/token-tools@0.1.1
+
+## 0.1.0
+
+### Minor Changes
+
+- [#319](https://github.com/terrazzoapp/terrazzo/pull/319) [`e7f272d`](https://github.com/terrazzoapp/terrazzo/commit/e7f272defcd889f5a410fdbd30497cf704671b32) Thanks [@drwpow](https://github.com/drwpow)! - ⚠️ Breaking change: dimension and duration tokens normalize to object syntax in plugins (following upcoming changes in DTCG spec; see https://github.com/design-tokens/community-group/pull/244).
+
+### Patch Changes
+
+- Updated dependencies [[`e7f272d`](https://github.com/terrazzoapp/terrazzo/commit/e7f272defcd889f5a410fdbd30497cf704671b32)]:
+  - @terrazzo/token-tools@0.1.0
+
 ## 0.0.19
 
 ### Patch Changes

package/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing to @terrazzo/parser
+
+This document contains developer notes and context for @terrazzo/parser. For general contribution guidelines, see [CONTRIBUTING.md](../../CONTRIBUTING.md) in the root.
+
+## What this package does
+
+- Parses, validates, and normalizes tokens.json
+- Executes plugins and builds output
+- Executes linting
+
+## What this package DOES NOT do
+
+- Write files to disk (that’s a job for Node.js; this can be run in a browser)
+
+## Manual JS and DTS
+
+This package is unique in its manual handling of `.js` and `.d.ts` files (TypeScript definitions). The expected norm is to write `.ts`, and have `.js` and `.d.ts` be generated automatically from source. But in this project, we write `.js` by hand because this specific module has several unique concerns:
+
+1. **It’s dealing with external (possibly invalid) code.** Now, you’d think that `.ts` would be better at handling this, but the reality when dealing with user-submitted code is if you type it too strongly, TS lulls you into a false sense of security and misses necessary assertions. If you type it weakly, then TS requires _too many_ assertions and you end up with boilerplate and noise. Or if you don’t type it at all, then TS doesn’t really provide any value in the first place, and you might as well skip it.
+
+2. **It’s dealing with ASTs.** We’re parsing JSON ASTs using Momoa, and building ad hoc structures on-the-fly. ASTs usually trip TS up because the discrimination isn’t clear (e.g. if `.type == 'foo'`, then `[property]` exists). Most ASTs don’t have perfect types out-of-the-box, and attempting to re-type an AST yourself is overkill. Further, to the previous point, we may be dealing with invalid code, and the AST may not always be in predictable shapes! When faced with these problems, TS at best requires extra boilerplate; at worst guides you into incorrect decisions.
+
+Given both of these, you usually end up with so many `// @ts-ignore`s or `as any`s that TS doesn’t provide benefit.
+
+But this package is unique! Look in any other package in this monorepo, and you’ll see it’s fully written in `.ts` source files. Different concerns require different approaches.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@terrazzo/parser",
-  "version": "0.0.19",
+  "version": "0.1.1",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
   "type": "module",
   "author": {
@@ -30,16 +30,16 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@humanwhocodes/momoa": "^3.2.2",
+    "@humanwhocodes/momoa": "^3.3.3",
     "@types/babel__code-frame": "^7.0.6",
     "@types/culori": "^2.1.1",
     "culori": "^4.0.1",
     "is-what": "^4.1.16",
     "merge-anything": "^5.1.7",
-    "picocolors": "^1.1.0",
+    "picocolors": "^1.1.1",
     "wildcard-match": "^5.1.3",
     "yaml": "^2.6.0",
-    "@terrazzo/token-tools": "^0.0.9"
+    "@terrazzo/token-tools": "^0.1.1"
   },
   "devDependencies": {
     "esbuild": "^0.23.1",

package/parse/index.d.ts

@@ -33,7 +33,7 @@ export interface ParseResult {
 /**
  * Parse and validate Tokens JSON, and lint it
  */
-export default function parse(input: ParseInput[], options?: ParseOptions): Promise<ParseResult>;
+export default function parse(input: ParseInput[], options: ParseOptions): Promise<ParseResult>;
 
 /** Determine if an input is likely a JSON string */
 export function maybeJSONString(input: unknown): boolean;

package/parse/index.js

@@ -240,10 +240,11 @@ async function parseSingle(input, { filename, logger, config, skipLint, continue
           $typeInheritance[path.join('.') || '.'] = node.value.members.find((m) => m.name.value === '$type');
         }
 
+        const id = path.join('.');
+
         if (members.$value) {
           const extensions = members.$extensions ? getObjMembers(members.$extensions) : undefined;
           const sourceNode = structuredClone(node);
-          const id = path.join('.');
 
           // get parent type by taking the closest-scoped $type (length === closer)
           let parent$type;
@@ -309,7 +310,7 @@ async function parseSingle(input, { filename, logger, config, skipLint, continue
           }
 
           tokens[id] = token;
-        } else if (members.value) {
+        } else if (!id.includes('.$value') && members.value) {
           logger.warn({ message: `Group ${id} has "value". Did you mean "$value"?`, filename, node, src });
         }
       }

package/parse/normalize.js

@@ -21,6 +21,8 @@ export const FONT_WEIGHT_MAP = {
   'ultra-black': 950,
 };
 
+const NUMBER_WITH_UNIT_RE = /(-?\d*\.?\d+)(.*)/;
+
 export default function normalizeValue(token) {
   if (isAlias(token.$value)) {
     return token.$value;
@@ -49,15 +51,25 @@ export default function normalizeValue(token) {
     }
     case 'dimension': {
       if (token.$value === 0) {
-        return '0';
+        return { value: 0, unit: 'px' };
+      }
+      // Backwards compat: handle string
+      if (typeof token.$value === 'string') {
+        const match = token.$value.match(NUMBER_WITH_UNIT_RE);
+        return { value: Number.parseFloat(match?.[1] || token.$value), unit: match[2] || 'px' };
       }
-      return typeof token.$value === 'number' ? `${token.$value}px` : token.$value;
+      return token.$value;
     }
     case 'duration': {
       if (token.$value === 0) {
-        return 0;
+        return { value: 0, unit: 'ms' };
       }
-      return typeof token.$value === 'number' ? `${token.$value}ms` : token.$value;
+      // Backwards compat: handle string
+      if (typeof token.$value === 'string') {
+        const match = token.$value.match(NUMBER_WITH_UNIT_RE);
+        return { value: Number.parseFloat(match?.[1] || token.$value), unit: match[2] || 'ms' };
+      }
+      return token.$value;
     }
     case 'fontFamily': {
       return Array.isArray(token.$value) ? token.$value : [token.$value];
@@ -108,10 +120,13 @@ export default function normalizeValue(token) {
     case 'typography': {
       const output = {};
       for (const k in token.$value) {
-        if (k === 'fontSize') {
-          output[k] = normalizeValue({ $type: 'dimension', $value: token.$value[k] });
-        } else {
-          output[k] = token.$value[k];
+        switch (k) {
+          case 'letterSpacing':
+          case 'fontSize':
+            output[k] = normalizeValue({ $type: 'dimension', $value: token.$value[k] });
+            break;
+          default:
+            output[k] = token.$value[k];
         }
       }
       return output;

package/parse/validate.js

@@ -262,13 +262,43 @@ export function validateDimension($value, node, { filename, src, logger }) {
   if ($value.type === 'Number' && $value.value === 0) {
     return; // `0` is a valid number
   }
+  if ($value.type === 'Object') {
+    const { value, unit } = getObjMembers($value);
+    if (!value) {
+      logger.error({ message: 'Missing required property "value".', filename, node: $value, src });
+    }
+    if (!unit) {
+      logger.error({ message: 'Missing required property "unit".', filename, node: $value, src });
+    }
+    if (value.type !== 'Number') {
+      logger.error({ message: `Expected number, received ${value.type}`, filename, node: value, src });
+    }
+    if (!['px', 'em', 'rem'].includes(unit.value)) {
+      logger.error({
+        message: `Expected unit "px", "em", or "rem", received ${print(unit)}`,
+        filename,
+        node: unit,
+        src,
+      });
+    }
+    return;
+  }
+  // Backwards compat: string
   if ($value.type !== 'String') {
     logger.error({ message: `Expected string, received ${$value.type}`, filename, node: $value, src });
-  } else if ($value.value === '') {
+  }
+  const value = $value.value.match(/^-?[0-9.]+/)?.[0];
+  const unit = $value.value.replace(value, '');
+  if ($value.value === '') {
     logger.error({ message: 'Expected dimension, received empty string', filename, node: $value, src });
-  } else if (String(Number.parseFloat($value.value)) === $value.value) {
-    logger.error({ message: 'Missing units', filename, node: $value, src });
-  } else if (!/^-?[0-9]+(\.[0-9]+)?/.test($value.value)) {
+  } else if (!['px', 'em', 'rem'].includes(unit)) {
+    logger.error({
+      message: `Expected unit "px", "em", or "rem", received ${JSON.stringify(unit || $value.value)}`,
+      filename,
+      node: $value,
+      src,
+    });
+  } else if (!Number.isFinite(Number.parseFloat(value))) {
     logger.error({ message: `Expected dimension with units, received ${print($value)}`, filename, node: $value, src });
   }
 }
@@ -284,19 +314,39 @@ export function validateDuration($value, node, { filename, src, logger }) {
   if ($value.type === 'Number' && $value.value === 0) {
     return; // `0` is a valid number
   }
+  if ($value.type === 'Object') {
+    const { value, unit } = getObjMembers($value);
+    if (!value) {
+      logger.error({ message: 'Missing required property "value".', filename, node: $value, src });
+    }
+    if (!unit) {
+      logger.error({ message: 'Missing required property "unit".', filename, node: $value, src });
+    }
+    if (value.type !== 'Number') {
+      logger.error({ message: `Expected number, received ${value.type}`, filename, node: value, src });
+    }
+    if (!['ms', 's'].includes(unit.value)) {
+      logger.error({ message: `Expected unit "ms" or "s", received ${print(unit)}`, filename, node: unit, src });
+    }
+    return;
+  }
+  // Backwards compat: string
   if ($value.type !== 'String') {
     logger.error({ message: `Expected string, received ${$value.type}`, filename, node: $value, src });
-  } else if ($value.value === '') {
+  }
+  const value = $value.value.match(/^-?[0-9.]+/)?.[0];
+  const unit = $value.value.replace(value, '');
+  if ($value.value === '') {
     logger.error({ message: 'Expected duration, received empty string', filename, node: $value, src });
-  } else if (!/m?s$/.test($value.value)) {
-    logger.error({ message: 'Missing unit "ms" or "s"', filename, node: $value, src });
-  } else if (!/^[0-9]+/.test($value.value)) {
+  } else if (!['ms', 's'].includes(unit)) {
     logger.error({
-      message: `Expected duration in \`ms\` or \`s\`, received ${print($value)}`,
+      message: `Expected unit "ms" or "s", received ${JSON.stringify(unit || $value.value)}`,
       filename,
       node: $value,
       src,
     });
+  } else if (!Number.isFinite(Number.parseFloat(value))) {
+    logger.error({ message: `Expected duration with units, received ${print($value)}`, filename, node: $value, src });
   }
 }