@terrazzo/parser 0.7.2 → 0.8.0

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@terrazzo/parser",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
+  "license": "MIT",
   "type": "module",
   "author": {
     "name": "Drew Powers",
@@ -22,33 +23,36 @@
     "lint"
   ],
   "main": "./dist/index.js",
+  "exports": {
+    ".": "./dist/index.js",
+    "./package.json": "./package.json"
+  },
   "homepage": "https://terrazzo.app/docs/cli/api/js",
   "repository": {
     "type": "git",
     "url": "https://github.com/terrazzoapp/terrazzo.git",
     "directory": "./packages/parser/"
   },
-  "license": "MIT",
   "dependencies": {
     "@humanwhocodes/momoa": "^3.3.8",
     "@types/babel__code-frame": "^7.0.6",
     "@types/culori": "^4.0.0",
     "culori": "^4.0.1",
-    "is-what": "^4.1.16",
     "merge-anything": "^5.1.7",
     "picocolors": "^1.1.1",
     "scule": "^1.3.0",
     "wildcard-match": "^5.1.4",
-    "yaml": "^2.8.0",
-    "@terrazzo/token-tools": "^0.7.3"
+    "@terrazzo/token-tools": "^0.8.0"
   },
   "devDependencies": {
     "yaml-to-momoa": "^0.0.3"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.build.json",
+    "build": "rolldown -c && attw --profile esm-only --pack .",
     "dev": "pnpm run build --watch",
-    "lint": "biome check .",
+    "lint": "pnpm --filter @terrazzo/parser run \"/^lint:(js|ts)/\"",
+    "lint:js": "biome check .",
+    "lint:ts": "tsc --noEmit",
     "test": "vitest run"
   }
 }

package/rolldown.config.ts

@@ -0,0 +1,24 @@
+import { defineConfig } from 'rolldown';
+import { dts } from 'rolldown-plugin-dts';
+
+export default defineConfig({
+  input: {
+    index: './src/index.ts',
+  },
+  plugins: [dts()],
+  external: [
+    '@humanwhocodes/mamoa',
+    '@terrazzo/token-tools',
+    'culori',
+    'merge-anything',
+    'picocolors',
+    'wildcard-match',
+    'yaml-to-momoa',
+    'yaml',
+  ],
+  output: {
+    dir: 'dist',
+    format: 'es',
+    sourcemap: true,
+  },
+});

package/src/build/index.ts

@@ -18,7 +18,7 @@ function validateTransformParams({
   params,
   logger,
   pluginName,
-}: { params: TokenTransformed; logger: Logger; pluginName: string }) {
+}: { params: TokenTransformed; logger: Logger; pluginName: string }): void {
   const baseMessage: LogEntry = { group: 'plugin', label: pluginName, message: '' };
 
   // validate value is valid for SINGLE_VALUE or MULTI_VALUE
@@ -48,7 +48,7 @@ function validateTransformParams({
 export default async function build(
   tokens: Record<string, TokenNormalized>,
   { sources, logger = new Logger(), config }: BuildRunnerOptions,
-) {
+): Promise<BuildRunnerResult> {
   const formats: Record<string, TokenTransformed[]> = {};
   const result: BuildRunnerResult = { outputFiles: [] };
 

package/src/index.ts

@@ -15,4 +15,79 @@ export * from './parse/index.js';
 
 export * from './types.js';
 
-export * from '@terrazzo/token-tools/dist/types.js';
+export type {
+  AliasToken,
+  AliasValue,
+  BooleanToken,
+  BooleanTokenNormalized,
+  BooleanValue,
+  BorderToken,
+  BorderTokenNormalized,
+  BorderValue,
+  BorderValueNormalized,
+  ColorSpace,
+  ColorToken,
+  ColorTokenNormalized,
+  ColorValue,
+  ColorValueNormalized,
+  CubicBezierToken,
+  CubicBezierTokenNormalized,
+  CubicBezierValue,
+  CustomTransformOptions,
+  DimensionToken,
+  DimensionTokenNormalized,
+  DimensionValue,
+  DurationToken,
+  DurationTokenNormalized,
+  DurationValue,
+  FontFamilyToken,
+  FontFamilyTokenNormalized,
+  FontFamilyValue,
+  FontFamilyValueNormalized,
+  FontWeightToken,
+  FontWeightTokenNormalized,
+  FontWeightValue,
+  FontWeightValueNormalized,
+  GradientStop,
+  GradientStopNormalized,
+  GradientToken,
+  GradientTokenNormalized,
+  GradientValue,
+  GradientValueNormalized,
+  Group,
+  GroupCore,
+  GroupOrToken,
+  LinkToken,
+  LinkTokenNormalized,
+  LinkValue,
+  ModeMap,
+  NumberToken,
+  NumberTokenNormalized,
+  NumberValue,
+  ShadowToken,
+  ShadowTokenNormalized,
+  ShadowValue,
+  ShadowValueNormalized,
+  StringToken,
+  StringTokenNormalized,
+  StringValue,
+  StrokeStyleToken,
+  StrokeStyleTokenNormalized,
+  StrokeStyleValue,
+  StrokeStyleValueExpanded,
+  Token,
+  TokenCore,
+  TokenMode,
+  TokenNormalized,
+  TokenNormalizedCore,
+  TokenNormalizedSet,
+  TokensSet,
+  TransitionToken,
+  TransitionTokenNormalized,
+  TransitionValue,
+  TransitionValueNormalized,
+  TypographyToken,
+  TypographyTokenNormalized,
+  TypographyValue,
+  TypographyValueNormalized,
+} from '@terrazzo/token-tools';

package/src/parse/index.ts

@@ -1,13 +1,13 @@
-import type { DocumentNode, ObjectNode } from '@humanwhocodes/momoa';
+import { type DocumentNode, type MemberNode, type ObjectNode, evaluate } from '@humanwhocodes/momoa';
 import { type Token, type TokenNormalized, pluralize, splitID } from '@terrazzo/token-tools';
 import type ytm from 'yaml-to-momoa';
 import lintRunner from '../lint/index.js';
 import Logger from '../logger.js';
 import type { ConfigInit, InputSource } from '../types.js';
 import applyAliases from './alias.js';
-import { getObjMembers, toMomoa, traverse } from './json.js';
+import { getObjMembers, parseJSON, replaceObjMembers, toMomoa, traverse } from './json.js';
 import normalize from './normalize.js';
-import validateTokenNode from './validate.js';
+import validateTokenNode, { type Visitors, getInheritedType } from './validate.js';
 
 export * from './alias.js';
 export * from './normalize.js';
@@ -30,6 +30,11 @@ export interface ParseOptions {
   continueOnError?: boolean;
   /** Provide yamlToMomoa module to parse YAML (by default, this isn’t shipped to cut down on package weight) */
   yamlToMomoa?: typeof ytm;
+  /**
+   * Transform API
+   * @see https://terrazzo.app/docs/api/js#transform-api
+   */
+  transform?: Visitors;
   /** (internal cache; do not use) */
   _sources?: Record<string, InputSource>;
 }
@@ -48,6 +53,7 @@ export default async function parse(
     config = {} as ConfigInit,
     continueOnError = false,
     yamlToMomoa,
+    transform,
     _sources = {},
   }: ParseOptions = {} as ParseOptions,
 ): Promise<ParseResult> {
@@ -91,6 +97,7 @@ export default async function parse(
         skipLint,
         continueOnError,
         yamlToMomoa,
+        transform,
       });
       tokensSet = Object.assign(tokensSet, result.tokens);
       if (src.filename) {
@@ -166,6 +173,7 @@ async function parseSingle(
     config,
     skipLint,
     continueOnError = false,
+    transform,
     yamlToMomoa, // optional dependency, declared here so the parser itself doesn’t have to load a heavy dep in-browser
   }: {
     filename: URL;
@@ -173,12 +181,13 @@ async function parseSingle(
     config: ConfigInit;
     skipLint: boolean;
     continueOnError?: boolean;
+    transform: ParseOptions['transform'] | undefined;
     yamlToMomoa?: typeof ytm;
   },
 ): Promise<{ tokens: Record<string, Token>; document: DocumentNode; src?: string }> {
   // 1. Build AST
   const startParsing = performance.now();
-  const { src, document } = toMomoa(input, { filename, logger, continueOnError, yamlToMomoa });
+  let { src, document } = toMomoa(input, { filename, logger, continueOnError, yamlToMomoa });
   logger.debug({
     group: 'parser',
     label: 'json',
@@ -187,27 +196,77 @@ async function parseSingle(
   });
   const tokensSet: Record<string, TokenNormalized> = {};
 
+  // 1a. if there’s a root() transformer, then re-parse
+  if (transform?.root) {
+    const json = typeof input === 'string' ? JSON.parse(input) : input;
+    const result = transform?.root(json, '.', document);
+    if (result) {
+      const reRunResult = toMomoa(result, { filename, logger, continueOnError /* YAML not needed in transform() */ });
+      src = reRunResult.src;
+      document = reRunResult.document;
+    }
+  }
+
   // 2. Walk AST to validate tokens
   let tokenCount = 0;
   const startValidate = performance.now();
-  const $typeInheritance: Record<string, Token['$type']> = {};
+  const $typeInheritance: Record<string, MemberNode> = {};
   traverse(document, {
     enter(node, parent, subpath) {
       // if $type appears at root of tokens.json, collect it
       if (node.type === 'Document' && node.body.type === 'Object' && node.body.members) {
-        const members = getObjMembers(node.body);
-        if (members.$type && members.$type.type === 'String' && !members.$value) {
-          // @ts-ignore
-          $typeInheritance['.'] = node.body.members.find((m) => m.name.value === '$type');
+        const { members: rootMembers } = node.body;
+        const root$type = rootMembers.find((m) => m.name.type === 'String' && m.name.value === '$type');
+        const root$value = rootMembers.find((m) => m.name.type === 'String' && m.name.value === '$value');
+        if (root$type && !root$value) {
+          $typeInheritance['.'] = root$type;
+        }
+      }
+
+      // for transform() visitors, all non-tokens MUST be handled at this point (besides "root", which was handled above)
+      if (
+        node.type === 'Object' && // JSON object
+        subpath.length &&
+        !node.members.some((m) => m.name.type === 'String' && m.name.value === '$value') && // not the token node itself
+        !subpath.includes('$value') && // not a child of a token node, either
+        !subpath.includes('$extensions') // not metadata
+      ) {
+        if (transform?.group) {
+          const newJSON = transform?.group(evaluate(node), subpath.join('.'), node);
+          if (newJSON) {
+            replaceObjMembers(node, parseJSON(newJSON));
+          }
         }
       }
 
       // handle tokens
       if (node.type === 'Member') {
-        const token = validateTokenNode(node, { filename, src, config, logger, parent, subpath, $typeInheritance });
-        if (token) {
-          tokensSet[token.id] = token;
-          tokenCount++;
+        const inheritedTypeNode = getInheritedType(node, { subpath, $typeInheritance });
+        if (node.value.type === 'Object') {
+          const $type =
+            node.value.members.find((m) => m.name.type === 'String' && m.name.value === '$type') || inheritedTypeNode;
+          const $value = node.value.members.find((m) => m.name.type === 'String' && m.name.value === '$value');
+          if ($value && $type?.value.type === 'String' && transform?.[$type.value.value]) {
+            const result = transform[$type.value.value]?.(evaluate(node.value), subpath.join('.'), node);
+            if (result) {
+              node.value = parseJSON(result).body;
+            }
+          }
+
+          const token = validateTokenNode(node, {
+            filename,
+            src,
+            config,
+            logger,
+            parent,
+            subpath,
+            transform,
+            inheritedTypeNode,
+          });
+          if (token) {
+            tokensSet[token.id] = token;
+            tokenCount++;
+          }
         }
       }
     },

package/src/parse/json.ts

@@ -3,15 +3,16 @@ import {
   type DocumentNode,
   type MemberNode,
   type ObjectNode,
+  type ParseOptions,
   type ValueNode,
-  parse as parseJSON,
+  parse as momoaParse,
   print,
 } from '@humanwhocodes/momoa';
 import type yamlToMomoa from 'yaml-to-momoa';
 import type Logger from '../logger.js';
 import type { InputSource } from '../types.js';
 
-export interface Visitor {
+export interface JSONVisitor {
   enter?: (node: AnyNode, parent: AnyNode | undefined, path: string[]) => void;
   exit?: (node: AnyNode, parent: AnyNode | undefined, path: string[]) => void;
 }
@@ -54,11 +55,7 @@ export function getObjMembers(node: ObjectNode): Record<string | number, ValueNo
   return members;
 }
 
-/**
- * Inject members to ObjectNode
- * @param {ObjectNode} node
- * @param {MemberNode[]} members
- */
+/** Inject members to ObjectNode */
 export function injectObjMembers(node: ObjectNode, members: MemberNode[] = []) {
   if (node.type !== 'Object') {
     return;
@@ -66,11 +63,16 @@ export function injectObjMembers(node: ObjectNode, members: MemberNode[] = []) {
   node.members.push(...members);
 }
 
+/** Replace an ObjectNode’s contents outright with another */
+export function replaceObjMembers(a: ObjectNode, b: DocumentNode | ObjectNode) {
+  a.members = (b.type === 'Document' && (b.body as ObjectNode)?.members) || (b as ObjectNode).members;
+}
+
 /**
  * Variation of Momoa’s traverse(), which keeps track of global path.
  * Allows mutation of AST (along with any consequences)
  */
-export function traverse(root: AnyNode, visitor: Visitor) {
+export function traverse(root: AnyNode, visitor: JSONVisitor) {
   /**
    * Recursively visits a node.
    * @param {AnyNode} node The node to visit.
@@ -89,20 +91,15 @@ export function traverse(root: AnyNode, visitor: Visitor) {
     const childNode = CHILD_KEYS[node.type];
     for (const key of childNode ?? []) {
       const value = node[key as keyof typeof node];
-
-      if (value && typeof value === 'object') {
-        if (Array.isArray(value)) {
-          for (let i = 0; i < value.length; i++) {
-            visitNode(
-              // @ts-expect-error this is safe
-              value[i],
-              node,
-              key === 'elements' ? [...nextPath, String(i)] : nextPath,
-            );
-          }
-        } else if (isNode(value)) {
-          visitNode(value as unknown as AnyNode, node, nextPath);
+      if (!value) {
+        continue;
+      }
+      if (Array.isArray(value)) {
+        for (let i = 0; i < value.length; i++) {
+          visitNode(value[i] as unknown as AnyNode, node, key === 'elements' ? [...nextPath, String(i)] : nextPath);
         }
+      } else if (isNode(value)) {
+        visitNode(value as unknown as AnyNode, node, nextPath);
       }
     }
 
@@ -189,17 +186,26 @@ export function toMomoa(
       });
     }
   } else {
-    document = parseJSON(
-      typeof input === 'string' ? input : JSON.stringify(input, undefined, 2), // everything else: assert it’s JSON-serializable
-      {
-        mode: 'jsonc',
-        ranges: true,
-        tokens: true,
-      },
-    );
+    document = parseJSON(input);
   }
   if (!src) {
     src = print(document, { indent: 2 });
   }
   return { src, document };
 }
+
+/** Momoa, just with default options pre-set */
+export function parseJSON(input: string | Record<string, any>, options?: ParseOptions): any {
+  return momoaParse(
+    // note: it seems silly, at first glance, to have JSON.stringify() inside an actual JSON parser. But
+    // this provides a common interface to generate a Momoa AST for JSON created in-memory, which we already
+    // know is 100% valid because it’s already deserialized.
+    typeof input === 'string' ? input : JSON.stringify(input, undefined, 2),
+    {
+      mode: 'jsonc',
+      ranges: true,
+      tokens: true,
+      ...options,
+    },
+  );
+}

package/src/parse/normalize.ts

@@ -34,9 +34,6 @@ export const FONT_WEIGHT_MAP = {
   'ultra-black': 950,
 };
 
-// Note: because we’re handling a lot of input values, the type inference gets lost.
-// This file is expected to have a lot of `@ts-ignore` comments.
-
 const NUMBER_WITH_UNIT_RE = /(-?\d*\.?\d+)(.*)/;
 
 /** Fill in defaults, and return predictable shapes for tokens */
@@ -140,14 +137,10 @@ export default function normalizeValue<T extends Token>(token: T): T['$value'] {
         (layer) =>
           ({
             color: normalizeValue({ $type: 'color', $value: layer.color }),
-            // @ts-ignore
-            offsetX: normalizeValue({ $type: 'dimension', $value: layer.offsetX ?? 0 }),
-            // @ts-ignore
-            offsetY: normalizeValue({ $type: 'dimension', $value: layer.offsetY ?? 0 }),
-            // @ts-ignore
-            blur: normalizeValue({ $type: 'dimension', $value: layer.blur ?? 0 }),
-            // @ts-ignore
-            spread: normalizeValue({ $type: 'dimension', $value: layer.spread ?? 0 }),
+            offsetX: normalizeValue({ $type: 'dimension', $value: layer.offsetX ?? { value: 0, unit: 'px' } }),
+            offsetY: normalizeValue({ $type: 'dimension', $value: layer.offsetY ?? { value: 0, unit: 'px' } }),
+            blur: normalizeValue({ $type: 'dimension', $value: layer.blur ?? { value: 0, unit: 'px' } }),
+            spread: normalizeValue({ $type: 'dimension', $value: layer.spread ?? { value: 0, unit: 'px' } }),
             inset: layer.inset === true,
           }) as ShadowValueNormalized,
       );
@@ -163,11 +156,8 @@ export default function normalizeValue<T extends Token>(token: T): T['$value'] {
         return token.$value;
       }
       return {
-        // @ts-ignore
         duration: normalizeValue({ $type: 'duration', $value: token.$value.duration ?? 0 }),
-        // @ts-ignore
         delay: normalizeValue({ $type: 'duration', $value: token.$value.delay ?? 0 }),
-        // @ts-ignore
         timingFunction: normalizeValue({ $type: 'cubicBezier', $value: token.$value.timingFunction }),
       } as TransitionValue;
     }