@terrazzo/parser 0.0.0

package/parse/index.js

@@ -0,0 +1,372 @@
+import { evaluate, parse as parseJSON } from '@humanwhocodes/momoa';
+import { isAlias, parseAlias, splitID } from '@terrazzo/token-tools';
+import lintRunner from '../lint/index.js';
+import Logger from '../logger.js';
+import normalize from './normalize.js';
+import parseYAML from './yaml.js';
+import validate from './validate.js';
+import { getObjMembers, injectObjMembers, traverse } from './json.js';
+
+export * from './validate.js';
+
+/**
+ * @typedef {import("@humanwhocodes/momoa").DocumentNode} DocumentNode
+ * @typedef {import("../config.js").Plugin} Plugin
+ * @typedef {import("../types.js").TokenNormalized} TokenNormalized
+ * @typedef {object} ParseOptions
+ * @typedef {Logger} ParseOptions.logger
+ * @typedef {boolean=false} ParseOptions.skipLint
+ * @typedef {Plugin[]} ParseOptions.plugins
+ * @typedef {object} ParseResult
+ * @typedef {Record<string, TokenNormalized} ParseResult.tokens
+ * @typedef {DocumentNode} ParseResult.ast
+ */
+
+/**
+ * Parse
+ * @param {string | object} input
+ * @param {ParseOptions} options
+ * @return {Promise<ParseResult>}
+ */
+export default async function parse(input, { logger = new Logger(), skipLint = false, config } = {}) {
+  const { plugins } = config;
+
+  const totalStart = performance.now();
+
+  // 1. Build AST
+  const startParsing = performance.now();
+  logger.debug({ group: 'parser', task: 'parse', message: 'Start tokens parsing' });
+  let ast;
+  if (typeof input === 'string' && !maybeJSONString(input)) {
+    ast = parseYAML(input, { logger }); // if string, but not JSON, attempt YAML
+  } else {
+    ast = parseJSON(typeof input === 'string' ? input : JSON.stringify(input, undefined, 2), {
+      mode: 'jsonc',
+    }); // everything else: assert it’s JSON-serializable
+  }
+  logger.debug({
+    group: 'parser',
+    task: 'parse',
+    message: 'Finish tokens parsing',
+    timing: performance.now() - startParsing,
+  });
+
+  const tokens = {};
+
+  // 2. Walk AST once to validate tokens
+  const startValidation = performance.now();
+  logger.debug({ group: 'parser', task: 'validate', message: 'Start tokens validation' });
+  let last$Type;
+  let last$TypePath = '';
+  traverse(ast, {
+    enter(node, parent, path) {
+      // reset last$Type if not in a direct ancestor tree
+      if (!last$TypePath || !path.join('.').startsWith(last$TypePath)) {
+        last$Type = undefined;
+      }
+
+      if (node.type === 'Member' && node.value.type === 'Object' && node.value.members) {
+        const members = getObjMembers(node.value);
+
+        // keep track of closest-scoped $type
+        // note: this is only reliable in a synchronous, single-pass traversal;
+        // otherwise we’d have to do something more complicated
+        if (members.$type && members.$type.type === 'String' && !members.$value) {
+          last$Type = node.value.members.find((m) => m.name.value === '$type');
+          last$TypePath = path.join('.');
+        }
+
+        if (members.$value) {
+          const extensions = members.$extensions ? getObjMembers(members.$extensions) : undefined;
+          const sourceNode = structuredClone(node);
+          if (last$Type && !members.$type) {
+            sourceNode.value = injectObjMembers(sourceNode.value, [last$Type]);
+          }
+          validate(sourceNode, { ast, logger });
+
+          const id = path.join('.');
+          const group = { id: splitID(id).group, tokens: [] };
+          if (last$Type) {
+            group.$type = last$Type.value.value;
+          }
+          // note: this will also include sibling tokens, so be selective about only accessing group-specific properties
+          const groupMembers = getObjMembers(parent);
+          if (groupMembers.$description) {
+            group.$description = evaluate(groupMembers.$description);
+          }
+          if (groupMembers.$extensions) {
+            group.$extensions = evaluate(groupMembers.$extensions);
+          }
+          const token = {
+            $type: members.$type?.value ?? last$Type?.value.value,
+            $value: evaluate(members.$value),
+            id,
+            mode: {},
+            originalValue: evaluate(node.value),
+            group,
+            sourceNode: sourceNode.value,
+          };
+          if (members.$description?.value) {
+            token.$description = members.$description.value;
+          }
+
+          // handle modes
+          // note that circular refs are avoided here, such as not duplicating `modes`
+          const modeValues = extensions?.mode ? getObjMembers(extensions.mode) : {};
+          for (const mode of ['.', ...Object.keys(modeValues)]) {
+            token.mode[mode] = {
+              id: token.id,
+              $type: token.$type,
+              $value: mode === '.' ? token.$value : evaluate(modeValues[mode]),
+              sourceNode: mode === '.' ? structuredClone(token.sourceNode) : modeValues[mode],
+            };
+            if (token.$description) {
+              token.mode[mode].$description = token.$description;
+            }
+          }
+
+          tokens[id] = token;
+        } else if (members.value) {
+          logger.warn({ message: `Group ${id} has "value". Did you mean "$value"?`, node, ast });
+        }
+      }
+    },
+  });
+  logger.debug({
+    group: 'parser',
+    task: 'validate',
+    message: 'Finish tokens validation',
+    timing: performance.now() - startValidation,
+  });
+
+  // 3. Execute lint runner with loaded plugins
+  if (!skipLint && plugins?.length) {
+    const lintStart = performance.now();
+    logger.debug({
+      group: 'parser',
+      task: 'validate',
+      message: 'Start token linting',
+    });
+    await lintRunner({ ast, config, logger });
+    logger.debug({
+      group: 'parser',
+      task: 'validate',
+      message: 'Finish token linting',
+      timing: performance.now() - lintStart,
+    });
+  }
+
+  // 4. normalize values
+  const normalizeStart = performance.now();
+  logger.debug({
+    group: 'parser',
+    task: 'normalize',
+    message: 'Start token normalization',
+  });
+  for (const id in tokens) {
+    if (!Object.hasOwn(tokens, id)) {
+      continue;
+    }
+    try {
+      tokens[id].$value = normalize(tokens[id]);
+    } catch (err) {
+      logger.error({ message: err.message, ast, node: tokens[id].sourceNode });
+    }
+    for (const mode in tokens[id].mode) {
+      if (mode === '.') {
+        continue;
+      }
+      try {
+        tokens[id].mode[mode].$value = normalize(tokens[id].mode[mode]);
+      } catch (err) {
+        logger.error({ message: err.message, ast, node: tokens[id].mode[mode].sourceNode });
+      }
+    }
+  }
+  logger.debug({
+    group: 'parser',
+    task: 'normalize',
+    message: 'Finish token normalization',
+    timing: performance.now() - normalizeStart,
+  });
+
+  // 5. Resolve aliases and populate groups
+  for (const id in tokens) {
+    if (!Object.hasOwn(tokens, id)) {
+      continue;
+    }
+    const token = tokens[id];
+    applyAliases(token, { tokens, ast, node: token.sourceNode, logger });
+    token.mode['.'].$value = token.$value;
+    if (token.aliasOf) {
+      token.mode['.'].aliasOf = token.aliasOf;
+    }
+    if (token.partialAliasOf) {
+      token.mode['.'].partialAliasOf = token.partialAliasOf;
+    }
+    const { group: parentGroup } = splitID(id);
+    for (const siblingID in tokens) {
+      const { group: siblingGroup } = splitID(siblingID);
+      if (siblingGroup?.startsWith(parentGroup)) {
+        token.group.tokens.push(siblingID);
+      }
+    }
+  }
+
+  // 6. resolve mode aliases
+  const modesStart = performance.now();
+  logger.debug({
+    group: 'parser',
+    task: 'modes',
+    message: 'Start mode resolution',
+  });
+  for (const id in tokens) {
+    if (!Object.hasOwn(tokens, id)) {
+      continue;
+    }
+    for (const mode in tokens[id].mode) {
+      if (mode === '.') {
+        continue; // skip shadow of root value
+      }
+      applyAliases(tokens[id].mode[mode], { tokens, ast, node: tokens[id].mode[mode].sourceNode, logger });
+    }
+  }
+  logger.debug({
+    group: 'parser',
+    task: 'modes',
+    message: 'Finish token modes',
+    timing: performance.now() - modesStart,
+  });
+
+  logger.debug({
+    group: 'parser',
+    task: 'core',
+    message: 'Finish all parser tasks',
+    timing: performance.now() - totalStart,
+  });
+
+  return {
+    tokens,
+    ast,
+  };
+}
+
+/**
+ * Determine if an input is likely a JSON string
+ * @param {string} input
+ * @return {boolean}
+ */
+export function maybeJSONString(input) {
+  return typeof input === 'string' && input.trim().startsWith('{');
+}
+
+/**
+ * Resolve alias
+ * @param {string} alias
+ * @param {Object} options
+ * @param {Record<string, TokenNormalized>} options.tokens
+ * @param {Logger} options.logger
+ * @param {AnyNode | undefined} options.node
+ * @param {DocumentNode | undefined} options.ast
+ * @param {string[]=[]} options.scanned
+ * @param {string}
+ */
+export function resolveAlias(alias, { tokens, logger, ast, node, scanned = [] }) {
+  const { id } = parseAlias(alias);
+  if (!tokens[id]) {
+    logger.error({ message: `Alias "${alias}" not found`, ast, node });
+  }
+  if (scanned.includes(id)) {
+    logger.error({ message: `Circular alias detected from "${alias}"`, ast, node });
+  }
+  const token = tokens[id];
+  if (!isAlias(token.$value)) {
+    return id;
+  }
+  return resolveAlias(alias, { tokens, logger, ast, node, scanned: [...scanned, id] });
+}
+
+/** Resolve aliases, update values, and mutate `token` to add `aliasOf` / `partialAliasOf` */
+function applyAliases(token, { tokens, logger, ast, node }) {
+  // handle simple aliases
+  if (isAlias(token.$value)) {
+    const aliasOfID = resolveAlias(token.$value, { tokens, logger, node, ast });
+    const { mode: aliasMode } = parseAlias(token.$value);
+    const aliasOf = tokens[aliasOfID];
+    token.aliasOf = aliasOfID;
+    token.$value = aliasOf.mode[aliasMode]?.$value || aliasOf.$value;
+    if (token.$type && token.$type !== aliasOf.$type) {
+      logger.warn({
+        message: `Token ${token.id} has $type "${token.$type}" but aliased ${aliasOfID} of $type "${aliasOf.$type}"`,
+        node,
+        ast,
+      });
+      token.$type = aliasOf.$type;
+    }
+  }
+  // handle aliases within array values (e.g. cubicBezier, gradient)
+  else if (Array.isArray(token.$value)) {
+    // some arrays are primitives, some are objects. handle both
+    for (let i = 0; i < token.$value.length; i++) {
+      if (isAlias(token.$value[i])) {
+        if (!token.partialAliasOf) {
+          token.partialAliasOf = [];
+        }
+        const aliasOfID = resolveAlias(token.$value[i], { tokens, logger, node, ast });
+        const { mode: aliasMode } = parseAlias(token.$value[i]);
+        token.partialAliasOf[i] = aliasOfID;
+        token.$value[i] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
+      } else if (typeof token.$value[i] === 'object') {
+        for (const property in token.$value[i]) {
+          if (isAlias(token.$value[i][property])) {
+            if (!token.partialAliasOf) {
+              token.partialAliasOf = [];
+            }
+            if (!token.partialAliasOf[i]) {
+              token.partialAliasOf[i] = {};
+            }
+            const aliasOfID = resolveAlias(token.$value[i][property], { tokens, logger, node, ast });
+            const { mode: aliasMode } = parseAlias(token.$value[i][property]);
+            token.$value[i][property] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
+            token.partialAliasOf[i][property] = aliasOfID;
+          }
+        }
+      }
+    }
+  }
+  // handle aliases within object (composite) values (e.g. border, typography, transition)
+  else if (typeof token.$value === 'object') {
+    for (const property in token.$value) {
+      if (!Object.hasOwn(token.$value, property)) {
+        continue;
+      }
+
+      if (isAlias(token.$value[property])) {
+        if (!token.partialAliasOf) {
+          token.partialAliasOf = {};
+        }
+        const aliasOfID = resolveAlias(token.$value[property], { tokens, logger, node, ast });
+        const { mode: aliasMode } = parseAlias(token.$value[property]);
+        token.partialAliasOf[property] = aliasOfID;
+        token.$value[property] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
+      }
+      // strokeStyle has an array within an object
+      else if (Array.isArray(token.$value[property])) {
+        for (let i = 0; i < token.$value[property].length; i++) {
+          if (isAlias(token.$value[property][i])) {
+            const aliasOfID = resolveAlias(token.$value[property][i], { tokens, logger, node, ast });
+            if (!token.partialAliasOf) {
+              token.partialAliasOf = {};
+            }
+            if (!token.partialAliasOf[property]) {
+              token.partialAliasOf[property] = [];
+            }
+            const { mode: aliasMode } = parseAlias(token.$value[property][i]);
+            token.partialAliasOf[property][i] = aliasOfID;
+            token.$value[property][i] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
+          }
+        }
+      }
+    }
+  }
+}

package/parse/json.d.ts

@@ -0,0 +1,30 @@
+import type { AnyNode, MemberNode, ObjectNode, ValueNode } from '@humanwhocodes/momoa';
+
+export interface Visitor {
+  enter?: (node: AnyNode, parent: AnyNode | undefined, path: string[]) => void;
+  exit?: (node: AnyNode, parent: AnyNode | undefined, path: string[]) => void;
+}
+
+declare const CHILD_KEYS: {
+  Document: readonly ['body'];
+  Object: readonly ['members'];
+  Member: readonly ['name', 'value'];
+  Element: readonly ['value'];
+  Array: readonly ['elements'];
+  String: readonly [];
+  Number: readonly [];
+  Boolean: readonly [];
+  Null: readonly [];
+};
+
+/** Determines if a given value is an AST node. */
+export function isNode(value: unknown): boolean;
+
+/** Get ObjectNode members as object */
+export function getObjMembers(node: ObjectNode): Record<string | number, ValueNode | undefined>;
+
+/** Inject members to ObjectNode and return a clone */
+export function injectObjMembers(node: ObjectNode, members: MemberNode[]): ObjectNode;
+
+/** Variation of Momoa’s traverse(), which keeps track of global path */
+export function traverse(root: AnyNode, visitor: Visitor): void;

package/parse/json.js

@@ -0,0 +1,94 @@
+/**
+ * @typedef {import("@humanwhocodes/momoa").AnyNode} AnyNode
+ * @typedef {import("@humanwhocodes/momoa").ObjectNode} ObjectNode
+ * @typedef {import("@humanwhocodes/momoa").ValueNode} ValueNode
+ */
+
+export const CHILD_KEYS = {
+  Document: ['body'],
+  Object: ['members'],
+  Member: ['name', 'value'],
+  Element: ['value'],
+  Array: ['elements'],
+  String: [],
+  Number: [],
+  Boolean: [],
+  Null: [],
+};
+
+/** Determines if a given value is an AST node. */
+export function isNode(value) {
+  return value && typeof value === 'object' && 'type' in value && typeof value.type === 'string';
+}
+
+/**
+ * Get ObjectNode members as object
+ * @param {ObjectNode} node
+ * @return {Record<string, ValueNode}
+ */
+export function getObjMembers(node) {
+  const members = {};
+  if (node.type !== 'Object') {
+    return members;
+  }
+  for (const m of node.members) {
+    if (m.name.type !== 'String' && m.name.type !== 'Number') {
+      continue;
+    }
+    members[m.name.value] = m.value;
+  }
+  return members;
+}
+
+/**
+ * Inject members to ObjectNode and return a clone
+ * @param {ObjectNode} node
+ * @param {MemberNode[]} members
+ * @return {ObjectNode}
+ */
+export function injectObjMembers(node, members = []) {
+  if (node.type !== 'Object') {
+    return node;
+  }
+  const newNode = structuredClone(node);
+  newNode.members.push(...members);
+  return newNode;
+}
+
+/**
+ * Variation of Momoa’s traverse(), which keeps track of global path
+ */
+export function traverse(root, visitor) {
+  /**
+   * Recursively visits a node.
+   * @param {AnyNode} node The node to visit.
+   * @param {AnyNode} [parent] The parent of the node to visit.
+   * @return {void}
+   */
+  function visitNode(node, parent, path = []) {
+    const nextPath = [...path];
+    if (node.type === 'Member') {
+      nextPath.push(node.name.value);
+    }
+
+    visitor.enter?.(node, parent, nextPath);
+
+    for (const key of CHILD_KEYS[node.type] ?? []) {
+      const value = node[key];
+
+      if (value && typeof value === 'object') {
+        if (Array.isArray(value)) {
+          for (let i = 0; i < value.length; i++) {
+            visitNode(value[i], node, key === 'elements' ? [...nextPath, String(i)] : nextPath);
+          }
+        } else if (isNode(value)) {
+          visitNode(value, node, nextPath);
+        }
+      }
+    }
+
+    visitor.exit?.(node, parent, nextPath);
+  }
+
+  visitNode(root, undefined, []);
+}

package/parse/normalize.d.ts

@@ -0,0 +1,3 @@
+import type { TokenNormalized } from '../types.js';
+
+export default function normalize<T extends TokenNormalized>(token: T): T['$value'];

package/parse/normalize.js

@@ -0,0 +1,114 @@
+import { isAlias, parseColor } from '@terrazzo/token-tools';
+
+export const FONT_WEIGHT_MAP = {
+  thin: 100,
+  hairline: 100,
+  'extra-light': 200,
+  'ultra-light': 200,
+  light: 300,
+  normal: 400,
+  regular: 400,
+  book: 400,
+  medium: 500,
+  'semi-bold': 600,
+  'demi-bold': 600,
+  bold: 700,
+  'extra-bold': 800,
+  'ultra-bold': 800,
+  black: 900,
+  heavy: 900,
+  'extra-black': 950,
+  'ultra-black': 950,
+};
+
+export default function normalizeValue(token) {
+  if (isAlias(token.$value)) {
+    return token.$value;
+  }
+  switch (token.$type) {
+    case 'boolean': {
+      return !!token.$value;
+    }
+    case 'border': {
+      return {
+        color: normalizeValue({ $type: 'color', $value: token.$value.color ?? '#000000' }),
+        style: normalizeValue({ $type: 'strokeStyle', $value: token.$value.style ?? 'solid' }),
+        width: normalizeValue({ $type: 'dimension', $value: token.$value.width }),
+      };
+    }
+    case 'color': {
+      return typeof token.$value === 'string' ? parseColor(token.$value) : token.$value;
+    }
+    case 'cubicBezier': {
+      return token.$value.map((value) =>
+        typeof value === 'number' ? normalizeValue({ $type: 'number', $value: value }) : value,
+      );
+    }
+    case 'dimension': {
+      if (token.$value === 0) {
+        return 0;
+      }
+      return typeof token.$value === 'number' ? `${token.$value}px` : token.$value;
+    }
+    case 'duration': {
+      if (token.$value === 0) {
+        return 0;
+      }
+      return typeof token.$value === 'number' ? `${token.$value}ms` : token.$value;
+    }
+    case 'fontFamily': {
+      return Array.isArray(token.$value) ? token.$value : [token.$value];
+    }
+    case 'fontWeight': {
+      if (typeof token.$value === 'string' && FONT_WEIGHT_MAP[token.$value]) {
+        return FONT_WEIGHT_MAP[token.$value];
+      }
+      return Number.parseInt(token.$value);
+    }
+    case 'gradient': {
+      const output = [];
+      for (let i = 0; i < token.$value.length; i++) {
+        const stop = { ...token.$value[i] };
+        stop.color = normalizeValue({ $type: 'color', $value: stop.color });
+        if (typeof stop.position !== 'number') {
+          stop.position = i / (token.$value.length - 1);
+        }
+        output.push(stop);
+      }
+      return output;
+    }
+    case 'number': {
+      return typeof token.$value === 'number' ? token.$value : Number.parseFloat(token.$value);
+    }
+    case 'shadow': {
+      return Array.isArray(token.$value) ? token.$value : [token.$value];
+    }
+    case 'strokeStyle': {
+      return token.$value;
+    }
+    case 'string': {
+      return String(token.$value);
+    }
+    case 'transition': {
+      return {
+        duration: normalizeValue({ $type: 'duration', $value: token.$value.duration ?? 0 }),
+        delay: normalizeValue({ $type: 'duration', $value: token.$value.delay ?? 0 }),
+        timingFunction: normalizeValue({ $type: 'cubicBezier', $value: token.$value.timingFunction }),
+      };
+    }
+    case 'typography': {
+      const output = {};
+      for (const k in token.$value) {
+        if (k === 'fontSize') {
+          output[k] = normalizeValue({ $type: 'dimension', $value: token.$value[k] });
+        } else {
+          output[k] = token.$value[k];
+        }
+      }
+      return output;
+    }
+    default: {
+      return token.$value;
+    }
+  }
+}

package/parse/validate.d.ts

@@ -0,0 +1,42 @@
+import type { AnyNode, DocumentNode, MemberNode, ValueNode } from '@humanwhocodes/momoa';
+import type Logger from '../logger.js';
+
+declare const FONT_WEIGHT_VALUES: Set<string>;
+
+declare const STROKE_STYLE_VALUES: Set<string>;
+declare const STROKE_STYLE_LINE_CAP_VALUES: Set<string>;
+
+export interface ValidateOptions {
+  ast: DocumentNode;
+  logger: Logger;
+}
+
+export function validateAlias($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateBorder($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateColor($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateCubicBézier($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateDimension($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateDuration($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateFontFamily($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateFontWeight($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateGradient($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateNumber($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateShadowLayer($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateStrokeStyle($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateTransition($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export function validateTypography($value: ValueNode, node: AnyNode, options: ValidateOptions): void;
+
+export default function validate(node: MemberNode, options: ValidateOptions): void;