@terrazzo/parser 0.10.3 → 2.0.0-alpha.0

package/src/parse/load.ts

@@ -0,0 +1,257 @@
+import * as momoa from '@humanwhocodes/momoa';
+import {
+  type BundleOptions,
+  bundle,
+  getObjMember,
+  type RefMap,
+  replaceNode,
+  traverseAsync,
+} from '@terrazzo/json-schema-tools';
+import type { GroupNormalized, TokenNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
+import { toMomoa } from '../lib/momoa.js';
+import type Logger from '../logger.js';
+import type { InputSource, ParseOptions, TransformVisitors } from '../types.js';
+import { normalize } from './normalize.js';
+import {
+  graphAliases,
+  groupFromNode,
+  refToTokenID,
+  resolveAliases,
+  tokenFromNode,
+  tokenRawValuesFromNode,
+} from './token.js';
+
+/** Ephemeral format that only exists while parsing the document. This is not confirmed to be DTCG yet. */
+export interface IntermediaryToken {
+  id: string;
+  /** Was this token aliasing another? */
+  $ref?: string;
+  $type?: string;
+  $description?: string;
+  $deprecated?: string | boolean;
+  $value: unknown;
+  $extensions?: Record<string, unknown>;
+  group: TokenNormalized['group'];
+  aliasOf?: string;
+  partialAliasOf?: Record<string, any> | any[];
+  mode: Record<
+    string,
+    {
+      $type?: string;
+      $value: unknown;
+      aliasOf?: string;
+      partialAliasOf?: Record<string, any> | any[];
+      source?: { filename?: URL; node: momoa.ObjectNode };
+    }
+  >;
+  source: {
+    filename?: URL;
+    node: momoa.ObjectNode;
+  };
+}
+
+export interface LoadOptions extends Pick<ParseOptions, 'config' | 'continueOnError' | 'yamlToMomoa' | 'transform'> {
+  logger: Logger;
+}
+
+/** Load from multiple entries, while resolving remote files */
+export async function loadSources(
+  inputs: Omit<InputSource, 'document'>[],
+  { config, logger, continueOnError, yamlToMomoa, transform }: LoadOptions,
+): Promise<{ tokens: TokenNormalizedSet; sources: InputSource[] }> {
+  const entry = { group: 'parser' as const, label: 'init' };
+
+  // 1. Bundle root documents together
+  const firstLoad = performance.now();
+  let document = {} as momoa.DocumentNode;
+
+  /** The original user inputs, in original order, with parsed ASTs */
+  const sources = inputs.map((input, i) => ({
+    ...input,
+    document: {} as momoa.DocumentNode,
+    filename: input.filename || new URL(`virtual:${i}`), // for objects created in memory, an index-based ID helps associate tokens with these
+  }));
+  /** The sources array, indexed by filename */
+  let sourceByFilename: Record<string, InputSource> = {};
+  /** Mapping of all final $ref resolutions. This will be used to generate the graph later. */
+  let refMap: RefMap = {};
+
+  try {
+    const result = await bundle(sources, {
+      parse: transform ? transformer(transform) : undefined,
+      yamlToMomoa,
+    });
+    document = result.document;
+    sourceByFilename = result.sources;
+    refMap = result.refMap;
+    for (const [filename, source] of Object.entries(result.sources)) {
+      const i = sources.findIndex((s) => s.filename.href === filename);
+      if (i === -1) {
+        sources.push(source);
+      } else {
+        sources[i]!.src = source.src; // this is a sanitized source that is easier to work with
+        sources[i]!.document = source.document;
+      }
+    }
+  } catch (err) {
+    let src = sources.find((s) => s.filename.href === (err as any).filename)?.src;
+    if (src && typeof src !== 'string') {
+      src = JSON.stringify(src, undefined, 2);
+    }
+    logger.error({
+      ...entry,
+      continueOnError,
+      message: (err as Error).message,
+      node: (err as any).node,
+      src,
+    });
+  }
+
+  logger.debug({ ...entry, message: `JSON loaded`, timing: performance.now() - firstLoad });
+  const artificialSource = { src: momoa.print(document, { indent: 2 }), document };
+
+  // 2. Parse
+  const firstPass = performance.now();
+  const tokens: TokenNormalizedSet = {};
+  // micro-optimization: while we’re iterating over tokens, keeping a “hot”
+  // array in memory saves recreating arrays from object keys over and over again.
+  // it does produce a noticeable speedup > 1,000 tokens.
+  const tokenIDs: string[] = [];
+  const groups: Record<string, GroupNormalized> = {};
+
+  // 2a. Token & group population
+  await traverseAsync(document, {
+    async enter(node, _parent, path) {
+      if (node.type !== 'Object') {
+        return;
+      }
+      groupFromNode(node, { path, groups });
+      const token = tokenFromNode(node, {
+        groups,
+        ignore: config.ignore,
+        path,
+        source: { src: artificialSource, document },
+      });
+      if (token) {
+        tokenIDs.push(token.jsonID);
+        tokens[token.jsonID] = token;
+      }
+    },
+  });
+
+  logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
+  const secondPass = performance.now();
+
+  // 2b. Resolve originalValue and original sources
+  for (const source of Object.values(sourceByFilename)) {
+    await traverseAsync(source.document, {
+      async enter(node, _parent, path) {
+        if (node.type !== 'Object') {
+          return;
+        }
+
+        const tokenRawValues = tokenRawValuesFromNode(node, { filename: source.filename!.href, path });
+        if (tokenRawValues && tokens[tokenRawValues?.jsonID]) {
+          tokens[tokenRawValues.jsonID]!.originalValue = tokenRawValues.originalValue;
+          tokens[tokenRawValues.jsonID]!.source = tokenRawValues.source;
+          for (const mode of Object.keys(tokenRawValues.mode)) {
+            tokens[tokenRawValues.jsonID]!.mode[mode]!.originalValue = tokenRawValues.mode[mode]!.originalValue;
+            tokens[tokenRawValues.jsonID]!.mode[mode]!.source = tokenRawValues.mode[mode]!.source;
+          }
+        }
+      },
+    });
+  }
+
+  // 2c. DTCG alias resolution
+  // Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
+  resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
+  logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
+
+  // 3. Alias graph
+  // We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
+  const aliasStart = performance.now();
+  graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
+  logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
+
+  // 4. normalize
+  // Allow for some minor variance in inputs, and be nice to folks.
+  const normalizeStart = performance.now();
+  for (const id of tokenIDs) {
+    const token = tokens[id]!;
+    normalize(token as any, { logger, src: sourceByFilename[token.source.filename!]?.src });
+  }
+  logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
+
+  // 5. alphabetize & filter
+  // This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
+  const tokensSorted: TokenNormalizedSet = {};
+  tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
+  for (const path of tokenIDs) {
+    // Filter out any tokens in $defs (we needed to reference them earlier, but shouldn’t include them in the final assortment)
+    if (path.includes('/$defs/')) {
+      continue;
+    }
+    const id = refToTokenID(path)!;
+    tokensSorted[id] = tokens[path]!;
+  }
+  // Sort group IDs once, too
+  for (const group of Object.values(groups)) {
+    group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
+  }
+
+  return {
+    tokens: tokensSorted,
+    sources,
+  };
+}
+
+function transformer(transform: TransformVisitors): BundleOptions['parse'] {
+  return async (src, filename) => {
+    let document = toMomoa(src);
+    let lastPath = '#/';
+    let last$type: string | undefined;
+
+    if (transform.root) {
+      const result = transform.root(document, { filename, parent: undefined, path: [] });
+      if (result) {
+        document = result as momoa.DocumentNode;
+      }
+    }
+
+    await traverseAsync(document, {
+      async enter(node, parent, path) {
+        if (node.type !== 'Object' || !path.length) {
+          return;
+        }
+        const ctx = { filename, parent, path };
+        const next$type = getObjMember(node, '$type');
+        if (next$type?.type === 'String') {
+          const jsonPath = `#/${path.join('/')}`;
+          if (jsonPath.startsWith(lastPath)) {
+            last$type = next$type.value;
+          }
+          lastPath = jsonPath;
+        }
+        if (getObjMember(node, '$value')) {
+          let result: any = transform.token?.(structuredClone(node), ctx);
+          if (result) {
+            replaceNode(node, result);
+            result = undefined;
+          }
+          result = transform[last$type as keyof typeof transform]?.(structuredClone(node as any), ctx);
+          if (result) {
+            replaceNode(node, result);
+          }
+        } else if (!path.includes('$value')) {
+          const result = transform.group?.(structuredClone(node), ctx);
+          if (result) {
+            replaceNode(node, result);
+          }
+        }
+      },
+    });
+
+    return document;
+  };
+}

package/src/parse/normalize.ts

@@ -1,199 +1,163 @@
-import {
-  type ColorValueNormalized,
-  type CubicBezierValue,
-  type DimensionValue,
-  type FontFamilyValue,
-  type GradientStopNormalized,
-  type GradientValueNormalized,
-  isAlias,
-  parseColor,
-  type ShadowValueNormalized,
-  type Token,
-  type TransitionValue,
-  type TypographyValueNormalized,
-} from '@terrazzo/token-tools';
+import type * as momoa from '@humanwhocodes/momoa';
+import { getObjMember } from '@terrazzo/json-schema-tools';
+import { FONT_WEIGHTS, isAlias, parseColor } from '@terrazzo/token-tools';
+import type Logger from '../logger.js';
 
-export const FONT_WEIGHT_MAP = {
-  thin: 100,
-  hairline: 100,
-  'extra-light': 200,
-  'ultra-light': 200,
-  light: 300,
-  normal: 400,
-  regular: 400,
-  book: 400,
-  medium: 500,
-  'semi-bold': 600,
-  'demi-bold': 600,
-  bold: 700,
-  'extra-bold': 800,
-  'ultra-bold': 800,
-  black: 900,
-  heavy: 900,
-  'extra-black': 950,
-  'ultra-black': 950,
-};
+interface PreValidatedToken {
+  id: string;
+  $type: string;
+  $value: unknown;
+  mode: Record<string, { $value: unknown; source: { node: any; filename: string | undefined } }>;
+}
 
-const NUMBER_WITH_UNIT_RE = /(-?\d*\.?\d+)(.*)/;
+/**
+ * Normalize token value.
+ * The reason for the “any” typing is this aligns various user-provided inputs to the type
+ */
+export function normalize(token: PreValidatedToken, { logger, src }: { logger: Logger; src: string }) {
+  const entry = { group: 'parser' as const, label: 'init', src };
 
-/** Fill in defaults, and return predictable shapes for tokens */
-export default function normalizeValue<T extends Token>(token: T): T['$value'] {
-  if (typeof token.$value === 'string' && isAlias(token.$value)) {
-    return token.$value;
+  function normalizeFontFamily(value: unknown): string[] {
+    return typeof value === 'string' ? [value] : (value as string[]);
   }
-  switch (token.$type) {
-    case 'boolean': {
-      return !!token.$value;
-    }
-    case 'border': {
-      if (typeof token.$value === 'string') {
-        return token.$value;
-      }
-      return {
-        color: normalizeValue({ $type: 'color', $value: token.$value.color ?? '#000000' }),
-        style: normalizeValue({ $type: 'strokeStyle', $value: token.$value.style ?? 'solid' }),
-        width: normalizeValue({ $type: 'dimension', $value: token.$value.width }),
-      };
-    }
-    case 'color': {
-      if (typeof token.$value === 'string') {
-        return parseColor(token.$value);
-      }
-      const newValue: ColorValueNormalized = {
-        colorSpace: token.$value.colorSpace,
-        components: token.$value.components ?? token.$value.channels,
-        alpha: token.$value.alpha ?? 1,
-      };
-      if ('hex' in token.$value) {
-        newValue.hex = token.$value.hex;
-      }
-      return newValue;
-    }
-    case 'cubicBezier': {
-      if (typeof token.$value === 'string') {
-        return token.$value;
-      }
-      return token.$value.map((value) =>
-        typeof value === 'number' ? normalizeValue({ $type: 'number', $value: value }) : value,
-      ) as CubicBezierValue;
-    }
-    case 'dimension': {
-      if ((token as any).$value === 0) {
-        return { value: 0, unit: 'px' };
+
+  function normalizeFontWeight(value: unknown): number {
+    return (typeof value === 'string' && FONT_WEIGHTS[value as keyof typeof FONT_WEIGHTS]) || (value as number);
+  }
+
+  function normalizeColor(value: unknown, node: momoa.AnyNode | undefined) {
+    if (typeof value === 'string' && !isAlias(value)) {
+      logger.warn({
+        ...entry,
+        node,
+        message: `${token.id}: string colors will be deprecated in a future version. Please update to object notation`,
+      });
+      try {
+        return parseColor(value);
+      } catch {
+        return { colorSpace: 'srgb', components: [0, 0, 0], alpha: 1 };
       }
-      // Backwards compat: handle string
-      if (typeof token.$value === 'string') {
-        const match = token.$value.match(NUMBER_WITH_UNIT_RE);
-        return { value: Number.parseFloat(match?.[1] || token.$value), unit: match?.[2] || 'px' };
+    } else if (value && typeof value === 'object') {
+      if ((value as any).alpha === undefined) {
+        (value as any).alpha = 1;
       }
-      return token.$value;
     }
-    case 'duration': {
-      if ((token as any).$value === 0) {
-        return { value: 0, unit: 'ms' };
-      }
-      // Backwards compat: handle string
-      if (typeof token.$value === 'string') {
-        const match = token.$value.match(NUMBER_WITH_UNIT_RE);
-        return { value: Number.parseFloat(match?.[1] || token.$value), unit: match?.[2] || 'ms' };
+    return value;
+  }
+
+  switch (token.$type) {
+    case 'color': {
+      for (const mode of Object.keys(token.mode)) {
+        token.mode[mode]!.$value = normalizeColor(token.mode[mode]!.$value, token.mode[mode]!.source.node);
       }
-      return token.$value;
+      token.$value = token.mode['.']!.$value;
+      break;
     }
+
     case 'fontFamily': {
-      return Array.isArray(token.$value) ? token.$value : [token.$value];
+      for (const mode of Object.keys(token.mode)) {
+        token.mode[mode]!.$value = normalizeFontFamily(token.mode[mode]!.$value);
+      }
+      token.$value = token.mode['.']!.$value;
+      break;
     }
+
     case 'fontWeight': {
-      if (typeof token.$value === 'string' && FONT_WEIGHT_MAP[token.$value as keyof typeof FONT_WEIGHT_MAP]) {
-        return FONT_WEIGHT_MAP[token.$value as keyof typeof FONT_WEIGHT_MAP];
+      for (const mode of Object.keys(token.mode)) {
+        token.mode[mode]!.$value = normalizeFontWeight(token.mode[mode]!.$value);
       }
-      return Math.min(
-        999,
-        Math.max(1, typeof token.$value === 'string' ? Number.parseInt(token.$value) : token.$value),
-      );
+      token.$value = token.mode['.']!.$value;
+      break;
     }
-    case 'gradient': {
-      if (typeof token.$value === 'string') {
-        return token.$value;
-      }
-      const output: GradientValueNormalized = [];
-      for (let i = 0; i < token.$value.length; i++) {
-        const stop = structuredClone(token.$value[i] as GradientStopNormalized);
-        stop.color = normalizeValue({ $type: 'color', $value: stop.color! });
-        if (stop.position === undefined) {
-          stop.position = i / (token.$value.length - 1);
+
+    case 'border': {
+      for (const mode of Object.keys(token.mode)) {
+        const border = token.mode[mode]!.$value as any;
+        if (!border || typeof border !== 'object') {
+          continue;
+        }
+        if (border.color) {
+          border.color = normalizeColor(
+            border.color,
+            getObjMember(token.mode[mode]!.source.node as momoa.ObjectNode, 'color'),
+          );
         }
-        output.push(stop);
       }
-      return output;
-    }
-    case 'number': {
-      return typeof token.$value === 'number' ? token.$value : Number.parseFloat(token.$value);
+      token.$value = token.mode['.']!.$value;
+      break;
     }
+
     case 'shadow': {
-      if (typeof token.$value === 'string') {
-        return token.$value;
+      for (const mode of Object.keys(token.mode)) {
+        // normalize to array
+        if (!Array.isArray(token.mode[mode]!.$value)) {
+          token.mode[mode]!.$value = [token.mode[mode]!.$value];
+        }
+        const $value = token.mode[mode]!.$value as any[];
+        for (let i = 0; i < $value.length; i++) {
+          const shadow = $value[i]!;
+          if (!shadow || typeof shadow !== 'object') {
+            continue;
+          }
+          const shadowNode = (
+            token.mode[mode]!.source.node.type === 'Array'
+              ? token.mode[mode]!.source.node.elements[i]!.value
+              : token.mode[mode]!.source.node
+          ) as momoa.ObjectNode;
+          if (shadow.color) {
+            shadow.color = normalizeColor(shadow.color, getObjMember(shadowNode, 'color'));
+          }
+          if (!('inset' in shadow)) {
+            shadow.inset = false;
+          }
+        }
       }
-      return (Array.isArray(token.$value) ? token.$value : [token.$value]).map(
-        (layer) =>
-          ({
-            color: normalizeValue({ $type: 'color', $value: layer.color }),
-            offsetX: normalizeValue({ $type: 'dimension', $value: layer.offsetX ?? { value: 0, unit: 'px' } }),
-            offsetY: normalizeValue({ $type: 'dimension', $value: layer.offsetY ?? { value: 0, unit: 'px' } }),
-            blur: normalizeValue({ $type: 'dimension', $value: layer.blur ?? { value: 0, unit: 'px' } }),
-            spread: normalizeValue({ $type: 'dimension', $value: layer.spread ?? { value: 0, unit: 'px' } }),
-            inset: layer.inset === true,
-          }) as ShadowValueNormalized,
-      );
+      token.$value = token.mode['.']!.$value;
+      break;
     }
-    case 'strokeStyle': {
-      return token.$value;
-    }
-    case 'string': {
-      return String(token.$value);
-    }
-    case 'transition': {
-      if (typeof token.$value === 'string') {
-        return token.$value;
+
+    case 'gradient': {
+      for (const mode of Object.keys(token.mode)) {
+        if (!Array.isArray(token.mode[mode]!.$value)) {
+          continue;
+        }
+        const $value = token.mode[mode]!.$value as any[];
+        for (let i = 0; i < $value.length; i++) {
+          const stop = $value[i]!;
+          if (!stop || typeof stop !== 'object') {
+            continue;
+          }
+          const stopNode = (token.mode[mode]!.source.node as momoa.ArrayNode)?.elements?.[i]?.value as momoa.ObjectNode;
+          if (stop.color) {
+            stop.color = normalizeColor(stop.color, getObjMember(stopNode, 'color'));
+          }
+        }
       }
-      return {
-        duration: normalizeValue({ $type: 'duration', $value: token.$value.duration ?? 0 }),
-        delay: normalizeValue({ $type: 'duration', $value: token.$value.delay ?? 0 }),
-        timingFunction: normalizeValue({ $type: 'cubicBezier', $value: token.$value.timingFunction }),
-      } as TransitionValue;
+      token.$value = token.mode['.']!.$value;
+      break;
     }
+
     case 'typography': {
-      if (typeof token.$value === 'string') {
-        return token.$value;
-      }
-      const output: TypographyValueNormalized = {};
-      for (const [k, $value] of Object.entries(token.$value)) {
-        switch (k) {
-          case 'fontFamily': {
-            output[k] = normalizeValue({ $type: 'fontFamily', $value: $value as FontFamilyValue });
-            break;
-          }
-          case 'fontSize':
-          case 'letterSpacing': {
-            output[k] = normalizeValue({ $type: 'dimension', $value: $value as DimensionValue });
-            break;
-          }
-          case 'lineHeight': {
-            output[k] = normalizeValue({
-              $type: typeof token.$value === 'number' ? 'number' : 'dimension',
-              $value: $value as any,
-            });
-            break;
-          }
-          default: {
-            output[k] = $value;
-            break;
+      for (const mode of Object.keys(token.mode)) {
+        const $value = token.mode[mode]!.$value as any;
+        if (typeof $value !== 'object') {
+          return;
+        }
+        for (const [k, v] of Object.entries($value)) {
+          switch (k) {
+            case 'fontFamily': {
+              $value[k] = normalizeFontFamily(v);
+              break;
+            }
+            case 'fontWeight': {
+              $value[k] = normalizeFontWeight(v);
+              break;
+            }
           }
         }
       }
-      return output;
-    }
-    default: {
-      return token.$value;
+      token.$value = token.mode['.']!.$value;
+      break;
     }
   }
 }