@terrazzo/parser 2.0.0-alpha.7 → 2.0.0-beta.0

package/src/parse/load.ts

@@ -1,172 +0,0 @@
-import * as momoa from '@humanwhocodes/momoa';
-import {
-  type BundleOptions,
-  bundle,
-  encodeFragment,
-  getObjMember,
-  type InputSource,
-  type InputSourceWithDocument,
-  replaceNode,
-  traverse,
-} from '@terrazzo/json-schema-tools';
-import type { TokenNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
-import { toMomoa } from '../lib/momoa.js';
-import { filterResolverPaths } from '../lib/resolver-utils.js';
-import type Logger from '../logger.js';
-import { isLikelyResolver } from '../resolver/validate.js';
-import type { ParseOptions, TransformVisitors } from '../types.js';
-import { processTokens } from './process.js';
-
-/** Ephemeral format that only exists while parsing the document. This is not confirmed to be DTCG yet. */
-export interface IntermediaryToken {
-  id: string;
-  /** Was this token aliasing another? */
-  $ref?: string;
-  $type?: string;
-  $description?: string;
-  $deprecated?: string | boolean;
-  $value: unknown;
-  $extensions?: Record<string, unknown>;
-  group: TokenNormalized['group'];
-  aliasOf?: string;
-  partialAliasOf?: Record<string, any> | any[];
-  mode: Record<
-    string,
-    {
-      $type?: string;
-      $value: unknown;
-      aliasOf?: string;
-      partialAliasOf?: Record<string, any> | any[];
-      source?: { filename?: URL; node: momoa.ObjectNode };
-    }
-  >;
-  source: {
-    filename?: URL;
-    node: momoa.ObjectNode;
-  };
-}
-
-export interface LoadOptions extends Pick<ParseOptions, 'config' | 'continueOnError' | 'yamlToMomoa' | 'transform'> {
-  req: NonNullable<ParseOptions['req']>;
-  logger: Logger;
-}
-
-export interface LoadSourcesResult {
-  tokens: TokenNormalizedSet;
-  sources: InputSourceWithDocument[];
-}
-
-/** Load from multiple entries, while resolving remote files */
-export async function loadSources(
-  inputs: InputSource[],
-  { config, logger, req, continueOnError, yamlToMomoa, transform }: LoadOptions,
-): Promise<LoadSourcesResult> {
-  const entry = { group: 'parser' as const, label: 'init' };
-
-  // 1. Bundle root documents together
-  const firstLoad = performance.now();
-  let document = {} as momoa.DocumentNode;
-
-  /** The original user inputs, in original order, with parsed ASTs */
-  const sources = inputs.map((input, i) => ({
-    ...input,
-    document: {} as momoa.DocumentNode,
-    filename: input.filename || new URL(`virtual:${i}`), // for objects created in memory, an index-based ID helps associate tokens with these
-  }));
-  /** The sources array, indexed by filename */
-  let sourceByFilename: Record<string, InputSourceWithDocument> = {};
-  try {
-    const result = await bundle(sources, {
-      req,
-      parse: transform ? transformer(transform) : undefined,
-      yamlToMomoa,
-    });
-    document = result.document;
-    sourceByFilename = result.sources;
-    for (const [filename, source] of Object.entries(result.sources)) {
-      const i = sources.findIndex((s) => s.filename.href === filename);
-      if (i === -1) {
-        sources.push(source);
-      } else {
-        sources[i]!.src = source.src; // this is a sanitized source that is easier to work with
-        sources[i]!.document = source.document;
-      }
-    }
-  } catch (err) {
-    let src = sources.find((s) => s.filename.href === (err as any).filename)?.src;
-    if (src && typeof src !== 'string') {
-      src = JSON.stringify(src, undefined, 2);
-    }
-    logger.error({
-      ...entry,
-      continueOnError,
-      message: (err as Error).message,
-      node: (err as any).node,
-      src,
-    });
-  }
-  logger.debug({ ...entry, message: `JSON loaded`, timing: performance.now() - firstLoad });
-
-  const rootSource = {
-    filename: sources[0]!.filename!,
-    document,
-    src: momoa.print(document, { indent: 2 }).replace(/\\\//g, '/'),
-  };
-
-  return {
-    tokens: processTokens(rootSource, { config, logger, sources, sourceByFilename }),
-    sources,
-  };
-}
-
-function transformer(transform: TransformVisitors): BundleOptions['parse'] {
-  return async (src, filename) => {
-    let document = toMomoa(src);
-    let lastPath = '#/';
-    let last$type: string | undefined;
-
-    if (transform.root) {
-      const result = transform.root(document, { filename, parent: undefined, path: [] });
-      if (result) {
-        document = result as momoa.DocumentNode;
-      }
-    }
-
-    const isResolver = isLikelyResolver(document);
-    traverse(document, {
-      enter(node, parent, rawPath) {
-        const path = isResolver ? filterResolverPaths(rawPath) : rawPath;
-        if (node.type !== 'Object' || !path.length) {
-          return;
-        }
-        const ctx = { filename, parent, path };
-        const next$type = getObjMember(node, '$type');
-        if (next$type?.type === 'String') {
-          const jsonPath = encodeFragment(path);
-          if (jsonPath.startsWith(lastPath)) {
-            last$type = next$type.value;
-          }
-          lastPath = jsonPath;
-        }
-        if (getObjMember(node, '$value')) {
-          let result: any = transform.token?.(structuredClone(node), ctx);
-          if (result) {
-            replaceNode(node, result);
-            result = undefined;
-          }
-          result = transform[last$type as keyof typeof transform]?.(structuredClone(node as any), ctx);
-          if (result) {
-            replaceNode(node, result);
-          }
-        } else if (!path.includes('$value')) {
-          const result = transform.group?.(structuredClone(node), ctx);
-          if (result) {
-            replaceNode(node, result);
-          }
-        }
-      },
-    });
-
-    return document;
-  };
-}

package/src/parse/normalize.ts

@@ -1,163 +0,0 @@
-import type * as momoa from '@humanwhocodes/momoa';
-import { getObjMember } from '@terrazzo/json-schema-tools';
-import { FONT_WEIGHTS, isAlias, parseColor } from '@terrazzo/token-tools';
-import type Logger from '../logger.js';
-
-interface PreValidatedToken {
-  id: string;
-  $type: string;
-  $value: unknown;
-  mode: Record<string, { $value: unknown; source: { node: any; filename: string | undefined } }>;
-}
-
-/**
- * Normalize token value.
- * The reason for the “any” typing is this aligns various user-provided inputs to the type
- */
-export function normalize(token: PreValidatedToken, { logger, src }: { logger: Logger; src: string }) {
-  const entry = { group: 'parser' as const, label: 'init', src };
-
-  function normalizeFontFamily(value: unknown): string[] {
-    return typeof value === 'string' ? [value] : (value as string[]);
-  }
-
-  function normalizeFontWeight(value: unknown): number {
-    return (typeof value === 'string' && FONT_WEIGHTS[value as keyof typeof FONT_WEIGHTS]) || (value as number);
-  }
-
-  function normalizeColor(value: unknown, node: momoa.AnyNode | undefined) {
-    if (typeof value === 'string' && !isAlias(value)) {
-      logger.warn({
-        ...entry,
-        node,
-        message: `${token.id}: string colors will be deprecated in a future version. Please update to object notation`,
-      });
-      try {
-        return parseColor(value);
-      } catch {
-        return { colorSpace: 'srgb', components: [0, 0, 0], alpha: 1 };
-      }
-    } else if (value && typeof value === 'object') {
-      if ((value as any).alpha === undefined) {
-        (value as any).alpha = 1;
-      }
-    }
-    return value;
-  }
-
-  switch (token.$type) {
-    case 'color': {
-      for (const mode of Object.keys(token.mode)) {
-        token.mode[mode]!.$value = normalizeColor(token.mode[mode]!.$value, token.mode[mode]!.source.node);
-      }
-      token.$value = token.mode['.']!.$value;
-      break;
-    }
-
-    case 'fontFamily': {
-      for (const mode of Object.keys(token.mode)) {
-        token.mode[mode]!.$value = normalizeFontFamily(token.mode[mode]!.$value);
-      }
-      token.$value = token.mode['.']!.$value;
-      break;
-    }
-
-    case 'fontWeight': {
-      for (const mode of Object.keys(token.mode)) {
-        token.mode[mode]!.$value = normalizeFontWeight(token.mode[mode]!.$value);
-      }
-      token.$value = token.mode['.']!.$value;
-      break;
-    }
-
-    case 'border': {
-      for (const mode of Object.keys(token.mode)) {
-        const border = token.mode[mode]!.$value as any;
-        if (!border || typeof border !== 'object') {
-          continue;
-        }
-        if (border.color) {
-          border.color = normalizeColor(
-            border.color,
-            getObjMember(token.mode[mode]!.source.node as momoa.ObjectNode, 'color'),
-          );
-        }
-      }
-      token.$value = token.mode['.']!.$value;
-      break;
-    }
-
-    case 'shadow': {
-      for (const mode of Object.keys(token.mode)) {
-        // normalize to array
-        if (!Array.isArray(token.mode[mode]!.$value)) {
-          token.mode[mode]!.$value = [token.mode[mode]!.$value];
-        }
-        const $value = token.mode[mode]!.$value as any[];
-        for (let i = 0; i < $value.length; i++) {
-          const shadow = $value[i]!;
-          if (!shadow || typeof shadow !== 'object') {
-            continue;
-          }
-          const shadowNode = (
-            token.mode[mode]!.source.node.type === 'Array'
-              ? token.mode[mode]!.source.node.elements[i]!.value
-              : token.mode[mode]!.source.node
-          ) as momoa.ObjectNode;
-          if (shadow.color) {
-            shadow.color = normalizeColor(shadow.color, getObjMember(shadowNode, 'color'));
-          }
-          if (!('inset' in shadow)) {
-            shadow.inset = false;
-          }
-        }
-      }
-      token.$value = token.mode['.']!.$value;
-      break;
-    }
-
-    case 'gradient': {
-      for (const mode of Object.keys(token.mode)) {
-        if (!Array.isArray(token.mode[mode]!.$value)) {
-          continue;
-        }
-        const $value = token.mode[mode]!.$value as any[];
-        for (let i = 0; i < $value.length; i++) {
-          const stop = $value[i]!;
-          if (!stop || typeof stop !== 'object') {
-            continue;
-          }
-          const stopNode = (token.mode[mode]!.source.node as momoa.ArrayNode)?.elements?.[i]?.value as momoa.ObjectNode;
-          if (stop.color) {
-            stop.color = normalizeColor(stop.color, getObjMember(stopNode, 'color'));
-          }
-        }
-      }
-      token.$value = token.mode['.']!.$value;
-      break;
-    }
-
-    case 'typography': {
-      for (const mode of Object.keys(token.mode)) {
-        const $value = token.mode[mode]!.$value as any;
-        if (typeof $value !== 'object') {
-          return;
-        }
-        for (const [k, v] of Object.entries($value)) {
-          switch (k) {
-            case 'fontFamily': {
-              $value[k] = normalizeFontFamily(v);
-              break;
-            }
-            case 'fontWeight': {
-              $value[k] = normalizeFontWeight(v);
-              break;
-            }
-          }
-        }
-      }
-      token.$value = token.mode['.']!.$value;
-      break;
-    }
-  }
-}

package/src/parse/process.ts

@@ -1,251 +0,0 @@
-import * as momoa from '@humanwhocodes/momoa';
-import {
-  encodeFragment,
-  findNode,
-  getObjMember,
-  type InputSourceWithDocument,
-  mergeObjects,
-  parseRef,
-  replaceNode,
-  traverse,
-} from '@terrazzo/json-schema-tools';
-import { type GroupNormalized, isAlias, type TokenNormalizedSet } from '@terrazzo/token-tools';
-import { filterResolverPaths } from '../lib/resolver-utils.js';
-import type Logger from '../logger.js';
-import { isLikelyResolver } from '../resolver/validate.js';
-import type { ConfigInit, RefMap } from '../types.js';
-import { normalize } from './normalize.js';
-import {
-  aliasToGroupRef,
-  graphAliases,
-  groupFromNode,
-  refToTokenID,
-  resolveAliases,
-  tokenFromNode,
-  tokenRawValuesFromNode,
-} from './token.js';
-
-export interface ProcessTokensOptions {
-  config: ConfigInit;
-  logger: Logger;
-  sourceByFilename: Record<string, InputSourceWithDocument>;
-  sources: InputSourceWithDocument[];
-}
-
-export function processTokens(
-  rootSource: InputSourceWithDocument,
-  { config, logger, sourceByFilename }: ProcessTokensOptions,
-): TokenNormalizedSet {
-  const entry = { group: 'parser' as const, label: 'init' };
-
-  // 1. Inline $refs to discover any additional tokens
-  const refMap: RefMap = {};
-  function resolveRef(node: momoa.StringNode, chain: string[]): momoa.AnyNode {
-    const { subpath } = parseRef(node.value);
-    if (!subpath) {
-      logger.error({ ...entry, message: 'Can’t resolve $ref', node, src: rootSource.src });
-      // exit
-    }
-    const next = findNode(rootSource.document, subpath);
-    if (next?.type === 'Object') {
-      const next$ref = getObjMember(next, '$ref');
-      if (next$ref && next$ref.type === 'String') {
-        if (chain.includes(next$ref.value)) {
-          logger.error({
-            ...entry,
-            message: `Circular $ref detected: ${JSON.stringify(next$ref.value)}`,
-            node: next$ref,
-            src: rootSource.src,
-          });
-        }
-        chain.push(next$ref.value);
-        return resolveRef(next$ref, chain);
-      }
-    }
-    return next;
-  }
-  const inlineStart = performance.now();
-  traverse(rootSource.document, {
-    enter(node, _parent, rawPath) {
-      if (rawPath.includes('$extensions') || node.type !== 'Object') {
-        return;
-      }
-      const $ref = node.type === 'Object' ? (getObjMember(node, '$ref') as momoa.StringNode) : undefined;
-      if (!$ref) {
-        return;
-      }
-      if ($ref.type !== 'String') {
-        logger.error({ ...entry, message: 'Invalid $ref. Expected string.', node: $ref, src: rootSource.src });
-      }
-      const jsonID = encodeFragment(rawPath);
-      refMap[jsonID] = { filename: rootSource.filename.href, refChain: [$ref.value] };
-      const resolved = resolveRef($ref, refMap[jsonID]!.refChain);
-      if (resolved.type === 'Object') {
-        node.members.splice(
-          node.members.findIndex((m) => m.name.type === 'String' && m.name.value === '$ref'),
-          1,
-        );
-        replaceNode(node, mergeObjects(resolved, node));
-      } else {
-        replaceNode(node, resolved);
-      }
-    },
-  });
-  logger.debug({ ...entry, message: 'Inline aliases', timing: performance.now() - inlineStart });
-
-  // 2. Resolve $extends to discover any more additional tokens
-  function flatten$extends(node: momoa.ObjectNode, chain: string[]) {
-    const memberKeys = node.members.map((m) => m.name.type === 'String' && m.name.value).filter(Boolean) as string[];
-
-    let extended: momoa.ObjectNode | undefined;
-
-    if (memberKeys.includes('$extends')) {
-      const $extends = getObjMember(node, '$extends') as momoa.StringNode;
-      if ($extends.type !== 'String') {
-        logger.error({ ...entry, message: '$extends must be a string', node: $extends, src: rootSource.src });
-      }
-      if (memberKeys.includes('$value')) {
-        logger.error({ ...entry, message: '$extends can’t exist within a token', node: $extends, src: rootSource.src });
-      }
-      const next = isAlias($extends.value) ? aliasToGroupRef($extends.value) : undefined;
-      if (!next) {
-        logger.error({ ...entry, message: '$extends must be a valid alias', node: $extends, src: rootSource.src });
-      }
-
-      if (
-        chain.includes(next!.$ref) ||
-        // Check that $extends is not importing from higher up (could go in either direction, which is why we check both ways)
-        chain.some((value) => value.startsWith(next!.$ref) || next!.$ref.startsWith(value))
-      ) {
-        logger.error({ ...entry, message: 'Circular $extends detected', node: $extends, src: rootSource.src });
-      }
-
-      chain.push(next!.$ref);
-      extended = findNode(rootSource.document, parseRef(next!.$ref).subpath ?? []);
-      if (!extended) {
-        logger.error({ ...entry, message: 'Could not resolve $extends', node: $extends, src: rootSource.src });
-      }
-      if (extended!.type !== 'Object') {
-        logger.error({ ...entry, message: '$extends must resolve to a group of tokens', node });
-      }
-
-      // To ensure this is resolvable, try and flatten this node first (will catch circular refs)
-      flatten$extends(extended!, chain);
-
-      replaceNode(node, mergeObjects(extended!, node));
-    }
-
-    // Deeply-traverse for any interior $extends (even if it wasn’t at the top level)
-    for (const member of node.members) {
-      if (
-        member.value.type === 'Object' &&
-        member.name.type === 'String' &&
-        !['$value', '$extensions'].includes(member.name.value)
-      ) {
-        traverse(member.value, {
-          enter(subnode, _parent) {
-            if (subnode.type === 'Object') {
-              flatten$extends(subnode, chain);
-            }
-          },
-        });
-      }
-    }
-  }
-
-  const extendsStart = performance.now();
-  const extendsChain: string[] = [];
-  flatten$extends(rootSource.document.body as momoa.ObjectNode, extendsChain);
-  logger.debug({ ...entry, message: 'Resolving $extends', timing: performance.now() - extendsStart });
-
-  // 3. Parse discovered tokens
-  const firstPass = performance.now();
-  const tokens: TokenNormalizedSet = {};
-  // micro-optimization: while we’re iterating over tokens, keeping a “hot”
-  // array in memory saves recreating arrays from object keys over and over again.
-  // it does produce a noticeable speedup > 1,000 tokens.
-  const tokenIDs: string[] = [];
-  const groups: Record<string, GroupNormalized> = {};
-
-  // 3a. Token & group population
-  const isResolver = isLikelyResolver(rootSource.document);
-  traverse(rootSource.document, {
-    enter(node, _parent, rawPath) {
-      if (node.type !== 'Object') {
-        return;
-      }
-      groupFromNode(node, { path: isResolver ? filterResolverPaths(rawPath) : rawPath, groups });
-      const token = tokenFromNode(node, {
-        groups,
-        ignore: config.ignore,
-        path: isResolver ? filterResolverPaths(rawPath) : rawPath,
-        source: rootSource,
-      });
-      if (token) {
-        tokenIDs.push(token.jsonID);
-        tokens[token.jsonID] = token;
-      }
-    },
-  });
-
-  logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
-  const secondPass = performance.now();
-
-  // 3b. Resolve originalValue and original sources
-  for (const source of Object.values(sourceByFilename)) {
-    traverse(source.document, {
-      enter(node, _parent, path) {
-        if (node.type !== 'Object') {
-          return;
-        }
-
-        const tokenRawValues = tokenRawValuesFromNode(node, { filename: source.filename!.href, path });
-        if (tokenRawValues && tokens[tokenRawValues?.jsonID]) {
-          tokens[tokenRawValues.jsonID]!.originalValue = tokenRawValues.originalValue;
-          tokens[tokenRawValues.jsonID]!.source = tokenRawValues.source;
-          for (const mode of Object.keys(tokenRawValues.mode)) {
-            tokens[tokenRawValues.jsonID]!.mode[mode]!.originalValue = tokenRawValues.mode[mode]!.originalValue;
-            tokens[tokenRawValues.jsonID]!.mode[mode]!.source = tokenRawValues.mode[mode]!.source;
-          }
-        }
-      },
-    });
-  }
-
-  // 3c. DTCG alias resolution
-  // Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
-  resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
-  logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
-
-  // 4. Alias graph
-  // We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
-  const aliasStart = performance.now();
-  graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
-  logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
-
-  // 5. normalize
-  // Allow for some minor variance in inputs, and be nice to folks.
-  const normalizeStart = performance.now();
-  for (const id of tokenIDs) {
-    const token = tokens[id]!;
-    normalize(token as any, { logger, src: sourceByFilename[token.source.filename!]?.src });
-  }
-  logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
-
-  // 6. alphabetize & filter
-  // This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
-  const sortStart = performance.now();
-  const tokensSorted: TokenNormalizedSet = {};
-  tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
-  for (const path of tokenIDs) {
-    const id = refToTokenID(path)!;
-    tokensSorted[id] = tokens[path]!;
-  }
-  // Sort group IDs once, too
-  for (const group of Object.values(groups)) {
-    group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
-  }
-  logger.debug({ ...entry, message: 'Sorted tokens', timing: performance.now() - sortStart });
-
-  return tokensSorted;
-}