@terrazzo/parser 2.0.0-alpha.2 → 2.0.0-alpha.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@terrazzo/parser",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
   "license": "MIT",
   "type": "module",
@@ -27,26 +27,27 @@
     ".": "./dist/index.js",
     "./package.json": "./package.json"
   },
-  "homepage": "https://terrazzo.app/docs/cli/api/js",
+  "homepage": "https://terrazzo.app/docs/reference/js-api",
   "repository": {
     "type": "git",
     "url": "https://github.com/terrazzoapp/terrazzo.git",
     "directory": "./packages/parser/"
   },
   "dependencies": {
-    "@humanwhocodes/momoa": "^3.3.9",
+    "@humanwhocodes/momoa": "^3.3.10",
     "@types/babel__code-frame": "^7.0.6",
     "@types/culori": "^4.0.1",
     "culori": "^4.0.2",
+    "fast-deep-equal": "^3.1.3",
     "merge-anything": "^5.1.7",
     "picocolors": "^1.1.1",
     "scule": "^1.3.0",
     "wildcard-match": "^5.1.4",
-    "@terrazzo/json-schema-tools": "^0.0.1",
-    "@terrazzo/token-tools": "^2.0.0-alpha.2"
+    "@terrazzo/json-schema-tools": "^0.1.0-alpha.0",
+    "@terrazzo/token-tools": "^2.0.0-alpha.4"
   },
   "devDependencies": {
-    "yaml-to-momoa": "0.0.6"
+    "yaml-to-momoa": "0.0.8"
   },
   "scripts": {
     "build": "rolldown -c && attw --profile esm-only --pack .",

package/src/build/index.ts

@@ -1,11 +1,11 @@
-import type * as momoa from '@humanwhocodes/momoa';
+import type { InputSourceWithDocument } from '@terrazzo/json-schema-tools';
 import type { TokenNormalized } from '@terrazzo/token-tools';
 import wcmatch from 'wildcard-match';
 import Logger, { type LogEntry } from '../logger.js';
 import type { BuildRunnerResult, ConfigInit, TokenTransformed, TransformParams } from '../types.js';
 
 export interface BuildRunnerOptions {
-  sources: { filename?: URL; src: string; document: momoa.DocumentNode }[];
+  sources: InputSourceWithDocument[];
   config: ConfigInit;
   logger?: Logger;
 }

package/src/config.ts

@@ -206,7 +206,6 @@ function normalizeLint({ config, logger }: { config: ConfigInit; logger: Logger
           // Note: sometimes plugins will be loaded multiple times, in which case it’s expected
           // they’re register rules again for lint(). Only throw an error if plugin A and plugin B’s
           // rules conflict.
-
           if (allRules.get(rule) && allRules.get(rule) !== plugin.name) {
             logger.error({
               group: 'config',
@@ -229,7 +228,7 @@ function normalizeLint({ config, logger }: { config: ConfigInit; logger: Logger
 
         const value = config.lint.rules[id];
         let severity: LintRuleSeverity = 'off';
-        let options: any;
+        let options: any = {};
         if (typeof value === 'number' || typeof value === 'string') {
           severity = value;
         } else if (Array.isArray(value)) {

package/src/index.ts

@@ -90,4 +90,6 @@ export { default as Logger } from './logger.js';
 export * from './parse/index.js';
 export { default as parse } from './parse/index.js';
 
+export * from './resolver/index.js';
+
 export * from './types.js';

package/src/lib/resolver-utils.ts

@@ -0,0 +1,35 @@
+/**
+ * If tokens are found inside a resolver, strip out the resolver paths (don’t
+ * include "sets"/"modifiers" in the token ID etc.)
+ */
+export function filterResolverPaths(path: string[]): string[] {
+  switch (path[0]) {
+    case 'sets': {
+      return path.slice(4);
+    }
+    case 'modifiers': {
+      return path.slice(5);
+    }
+    case 'resolutionOrder': {
+      switch (path[2]) {
+        case 'sources': {
+          return path.slice(4);
+        }
+        case 'contexts': {
+          return path.slice(5);
+        }
+      }
+      break;
+    }
+  }
+  return path;
+}
+
+/**
+ * Make a deterministic string from an object
+ */
+export function makeInputKey(input: Record<string, string | undefined>): string {
+  return JSON.stringify(
+    Object.fromEntries(Object.entries(input).sort((a, b) => a[0].localeCompare(b[0], 'en-us', { numeric: true }))),
+  );
+}

package/src/lint/index.ts

@@ -1,7 +1,8 @@
+import type { InputSourceWithDocument } from '@terrazzo/json-schema-tools';
 import { pluralize, type TokenNormalizedSet } from '@terrazzo/token-tools';
 import { merge } from 'merge-anything';
 import type { LogEntry, default as Logger } from '../logger.js';
-import type { ConfigInit, InputSource } from '../types.js';
+import type { ConfigInit } from '../types.js';
 
 export { RECOMMENDED_CONFIG } from './plugin-core/index.js';
 
@@ -9,7 +10,7 @@ export interface LintRunnerOptions {
   tokens: TokenNormalizedSet;
   filename?: URL;
   config: ConfigInit;
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
   logger: Logger;
 }
 
@@ -21,7 +22,7 @@ export default async function lintRunner({
   logger,
 }: LintRunnerOptions): Promise<void> {
   const { plugins = [], lint } = config;
-  const sourceByFilename: Record<string, InputSource> = {};
+  const sourceByFilename: Record<string, InputSourceWithDocument> = {};
   for (const source of sources) {
     sourceByFilename[source.filename!.href] = source;
   }

package/src/lint/plugin-core/index.ts

@@ -8,9 +8,9 @@ export * from './rules/consistent-naming.js';
 export * from './rules/descriptions.js';
 export * from './rules/duplicate-values.js';
 export * from './rules/max-gamut.js';
-export * from './rules/no-type-on-alias.js';
 export * from './rules/required-children.js';
 export * from './rules/required-modes.js';
+export * from './rules/required-type.js';
 export * from './rules/required-typography-properties.js';
 
 import a11yMinContrast, { A11Y_MIN_CONTRAST } from './rules/a11y-min-contrast.js';
@@ -20,9 +20,9 @@ import consistentNaming, { CONSISTENT_NAMING } from './rules/consistent-naming.j
 import descriptions, { DESCRIPTIONS } from './rules/descriptions.js';
 import duplicateValues, { DUPLICATE_VALUES } from './rules/duplicate-values.js';
 import maxGamut, { MAX_GAMUT } from './rules/max-gamut.js';
-import noTypeOnAlias, { NO_TYPE_ON_ALIAS } from './rules/no-type-on-alias.js';
 import requiredChildren, { REQUIRED_CHILDREN } from './rules/required-children.js';
 import requiredModes, { REQUIRED_MODES } from './rules/required-modes.js';
+import requiredType, { REQUIRED_TYPE } from './rules/required-type.js';
 import requiredTypographyProperties, {
   REQUIRED_TYPOGRAPHY_PROPERTIES,
 } from './rules/required-typography-properties.js';
@@ -65,9 +65,9 @@ const ALL_RULES = {
   [DESCRIPTIONS]: descriptions,
   [DUPLICATE_VALUES]: duplicateValues,
   [MAX_GAMUT]: maxGamut,
-  [NO_TYPE_ON_ALIAS]: noTypeOnAlias,
   [REQUIRED_CHILDREN]: requiredChildren,
   [REQUIRED_MODES]: requiredModes,
+  [REQUIRED_TYPE]: requiredType,
   [REQUIRED_TYPOGRAPHY_PROPERTIES]: requiredTypographyProperties,
   [A11Y_MIN_CONTRAST]: a11yMinContrast,
   [A11Y_MIN_FONT_SIZE]: a11yMinFontSize,
@@ -100,5 +100,4 @@ export const RECOMMENDED_CONFIG: Record<string, LintRuleLonghand> = {
   [VALID_GRADIENT]: ['error', {}],
   [VALID_TYPOGRAPHY]: ['error', {}],
   [CONSISTENT_NAMING]: ['warn', { format: 'kebab-case' }],
-  [NO_TYPE_ON_ALIAS]: ['warn', {}],
 };

package/src/lint/plugin-core/lib/docs.ts

@@ -1,3 +1,3 @@
 export function docsLink(ruleName: string): string {
-  return `https://terrazzo.app/docs/cli/lint#${ruleName.replaceAll('/', '')}`;
+  return `https://terrazzo.app/docs/linting#${ruleName.replaceAll('/', '')}`;
 }

package/src/lint/plugin-core/rules/{no-type-on-alias.ts → required-type.ts}

@@ -1,25 +1,24 @@
-import { isAlias } from '@terrazzo/token-tools';
 import type { LintRule } from '../../../types.js';
 import { docsLink } from '../lib/docs.js';
 
-export const NO_TYPE_ON_ALIAS = 'core/no-type-on-alias';
+export const REQUIRED_TYPE = 'core/required-type';
 
 export const ERROR = 'ERROR';
 
 const rule: LintRule<typeof ERROR> = {
   meta: {
     messages: {
-      [ERROR]: 'Remove $type from aliased value.',
+      [ERROR]: 'Token missing $type.',
     },
     docs: {
-      description: 'If a $value is aliased it already has a $type defined.',
-      url: docsLink(NO_TYPE_ON_ALIAS),
+      description: 'Requiring every token to have $type, even aliases, simplifies computation.',
+      url: docsLink(REQUIRED_TYPE),
     },
   },
   defaultOptions: {},
   create({ tokens, report }) {
     for (const t of Object.values(tokens)) {
-      if (isAlias(t.originalValue!.$value as any) && t.originalValue?.$type) {
+      if (!t.originalValue?.$type) {
         report({ messageId: ERROR, node: t.source.node, filename: t.source.filename });
       }
     }

package/src/parse/index.ts

@@ -1,19 +1,25 @@
+import type fsType from 'node:fs/promises';
+import type { InputSource, InputSourceWithDocument } from '@terrazzo/json-schema-tools';
 import { pluralize, type TokenNormalizedSet } from '@terrazzo/token-tools';
 import lintRunner from '../lint/index.js';
 import Logger from '../logger.js';
-import type { ConfigInit, InputSource, ParseOptions } from '../types.js';
+import { createSyntheticResolver } from '../resolver/create-synthetic-resolver.js';
+import { loadResolver } from '../resolver/load.js';
+import type { ConfigInit, ParseOptions, Resolver } from '../types.js';
 import { loadSources } from './load.js';
 
 export interface ParseResult {
   tokens: TokenNormalizedSet;
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
+  resolver: Resolver;
 }
 
 /** Parse */
 export default async function parse(
-  _input: Omit<InputSource, 'document'> | Omit<InputSource, 'document'>[],
+  _input: InputSource | InputSource[],
   {
     logger = new Logger(),
+    req = defaultReq,
     skipLint = false,
     config = {} as ConfigInit,
     continueOnError = false,
@@ -22,10 +28,33 @@ export default async function parse(
   }: ParseOptions = {} as ParseOptions,
 ): Promise<ParseResult> {
   const inputs = Array.isArray(_input) ? _input : [_input];
+  let tokens: TokenNormalizedSet = {};
+  let resolver: Resolver | undefined;
+  let sources: InputSourceWithDocument[] = [];
 
   const totalStart = performance.now();
+
+  // 1. Load tokens
   const initStart = performance.now();
-  const { tokens, sources } = await loadSources(inputs, { logger, config, continueOnError, yamlToMomoa, transform });
+  const resolverResult = await loadResolver(inputs, { config, logger, req, yamlToMomoa });
+  // 1a. Resolver
+  if (resolverResult.resolver) {
+    tokens = resolverResult.tokens;
+    sources = resolverResult.sources;
+    resolver = resolverResult.resolver;
+  } else {
+    // 1b. No resolver
+    const tokenResult = await loadSources(inputs, {
+      req,
+      logger,
+      config,
+      continueOnError,
+      yamlToMomoa,
+      transform,
+    });
+    tokens = tokenResult.tokens;
+    sources = tokenResult.sources;
+  }
   logger.debug({
     message: 'Loaded tokens',
     group: 'parser',
@@ -64,5 +93,23 @@ export default async function parse(
   return {
     tokens,
     sources,
+    resolver: resolver || (await createSyntheticResolver(tokens, { config, logger, req, sources })),
   };
 }
+
+let fs: typeof fsType | undefined;
+
+/** Fallback req */
+async function defaultReq(src: URL, _origin: URL) {
+  if (src.protocol === 'file:') {
+    if (!fs) {
+      fs = await import('node:fs/promises');
+    }
+    return await fs.readFile(src, 'utf8');
+  }
+  const res = await fetch(src);
+  if (!res.ok) {
+    throw new Error(`${src} responded with ${res.status}\n${await res.text()}`);
+  }
+  return await res.text();
+}

package/src/parse/load.ts

@@ -3,23 +3,19 @@ import {
   type BundleOptions,
   bundle,
   getObjMember,
+  type InputSource,
+  type InputSourceWithDocument,
   type RefMap,
   replaceNode,
-  traverseAsync,
+  traverse,
 } from '@terrazzo/json-schema-tools';
-import type { GroupNormalized, TokenNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
+import type { TokenNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
 import { toMomoa } from '../lib/momoa.js';
+import { filterResolverPaths } from '../lib/resolver-utils.js';
 import type Logger from '../logger.js';
-import type { InputSource, ParseOptions, TransformVisitors } from '../types.js';
-import { normalize } from './normalize.js';
-import {
-  graphAliases,
-  groupFromNode,
-  refToTokenID,
-  resolveAliases,
-  tokenFromNode,
-  tokenRawValuesFromNode,
-} from './token.js';
+import { isLikelyResolver } from '../resolver/validate.js';
+import type { ParseOptions, TransformVisitors } from '../types.js';
+import { processTokens } from './process.js';
 
 /** Ephemeral format that only exists while parsing the document. This is not confirmed to be DTCG yet. */
 export interface IntermediaryToken {
@@ -51,14 +47,20 @@ export interface IntermediaryToken {
 }
 
 export interface LoadOptions extends Pick<ParseOptions, 'config' | 'continueOnError' | 'yamlToMomoa' | 'transform'> {
+  req: NonNullable<ParseOptions['req']>;
   logger: Logger;
 }
 
+export interface LoadSourcesResult {
+  tokens: TokenNormalizedSet;
+  sources: InputSourceWithDocument[];
+}
+
 /** Load from multiple entries, while resolving remote files */
 export async function loadSources(
-  inputs: Omit<InputSource, 'document'>[],
-  { config, logger, continueOnError, yamlToMomoa, transform }: LoadOptions,
-): Promise<{ tokens: TokenNormalizedSet; sources: InputSource[] }> {
+  inputs: InputSource[],
+  { config, logger, req, continueOnError, yamlToMomoa, transform }: LoadOptions,
+): Promise<LoadSourcesResult> {
   const entry = { group: 'parser' as const, label: 'init' };
 
   // 1. Bundle root documents together
@@ -72,12 +74,13 @@ export async function loadSources(
     filename: input.filename || new URL(`virtual:${i}`), // for objects created in memory, an index-based ID helps associate tokens with these
   }));
   /** The sources array, indexed by filename */
-  let sourceByFilename: Record<string, InputSource> = {};
+  let sourceByFilename: Record<string, InputSourceWithDocument> = {};
   /** Mapping of all final $ref resolutions. This will be used to generate the graph later. */
   let refMap: RefMap = {};
 
   try {
     const result = await bundle(sources, {
+      req,
       parse: transform ? transformer(transform) : undefined,
       yamlToMomoa,
     });
@@ -106,102 +109,11 @@ export async function loadSources(
       src,
     });
   }
-
   logger.debug({ ...entry, message: `JSON loaded`, timing: performance.now() - firstLoad });
-  const artificialSource = { src: momoa.print(document, { indent: 2 }), document };
-
-  // 2. Parse
-  const firstPass = performance.now();
-  const tokens: TokenNormalizedSet = {};
-  // micro-optimization: while we’re iterating over tokens, keeping a “hot”
-  // array in memory saves recreating arrays from object keys over and over again.
-  // it does produce a noticeable speedup > 1,000 tokens.
-  const tokenIDs: string[] = [];
-  const groups: Record<string, GroupNormalized> = {};
-
-  // 2a. Token & group population
-  await traverseAsync(document, {
-    async enter(node, _parent, path) {
-      if (node.type !== 'Object') {
-        return;
-      }
-      groupFromNode(node, { path, groups });
-      const token = tokenFromNode(node, {
-        groups,
-        ignore: config.ignore,
-        path,
-        source: { src: artificialSource, document },
-      });
-      if (token) {
-        tokenIDs.push(token.jsonID);
-        tokens[token.jsonID] = token;
-      }
-    },
-  });
-
-  logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
-  const secondPass = performance.now();
-
-  // 2b. Resolve originalValue and original sources
-  for (const source of Object.values(sourceByFilename)) {
-    await traverseAsync(source.document, {
-      async enter(node, _parent, path) {
-        if (node.type !== 'Object') {
-          return;
-        }
-
-        const tokenRawValues = tokenRawValuesFromNode(node, { filename: source.filename!.href, path });
-        if (tokenRawValues && tokens[tokenRawValues?.jsonID]) {
-          tokens[tokenRawValues.jsonID]!.originalValue = tokenRawValues.originalValue;
-          tokens[tokenRawValues.jsonID]!.source = tokenRawValues.source;
-          for (const mode of Object.keys(tokenRawValues.mode)) {
-            tokens[tokenRawValues.jsonID]!.mode[mode]!.originalValue = tokenRawValues.mode[mode]!.originalValue;
-            tokens[tokenRawValues.jsonID]!.mode[mode]!.source = tokenRawValues.mode[mode]!.source;
-          }
-        }
-      },
-    });
-  }
-
-  // 2c. DTCG alias resolution
-  // Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
-  resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
-  logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
-
-  // 3. Alias graph
-  // We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
-  const aliasStart = performance.now();
-  graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
-  logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
-
-  // 4. normalize
-  // Allow for some minor variance in inputs, and be nice to folks.
-  const normalizeStart = performance.now();
-  for (const id of tokenIDs) {
-    const token = tokens[id]!;
-    normalize(token as any, { logger, src: sourceByFilename[token.source.filename!]?.src });
-  }
-  logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
-
-  // 5. alphabetize & filter
-  // This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
-  const tokensSorted: TokenNormalizedSet = {};
-  tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
-  for (const path of tokenIDs) {
-    // Filter out any tokens in $defs (we needed to reference them earlier, but shouldn’t include them in the final assortment)
-    if (path.includes('/$defs/')) {
-      continue;
-    }
-    const id = refToTokenID(path)!;
-    tokensSorted[id] = tokens[path]!;
-  }
-  // Sort group IDs once, too
-  for (const group of Object.values(groups)) {
-    group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
-  }
 
+  const rootSource = { filename: sources[0]!.filename!, document, src: momoa.print(document, { indent: 2 }) };
   return {
-    tokens: tokensSorted,
+    tokens: processTokens(rootSource, { config, logger, refMap, sources, sourceByFilename }),
     sources,
   };
 }
@@ -219,8 +131,10 @@ function transformer(transform: TransformVisitors): BundleOptions['parse'] {
       }
     }
 
-    await traverseAsync(document, {
-      async enter(node, parent, path) {
+    const isResolver = isLikelyResolver(document);
+    traverse(document, {
+      enter(node, parent, rawPath) {
+        const path = isResolver ? filterResolverPaths(rawPath) : rawPath;
         if (node.type !== 'Object' || !path.length) {
           return;
         }

package/src/parse/process.ts

@@ -0,0 +1,124 @@
+import { type InputSourceWithDocument, type RefMap, traverse } from '@terrazzo/json-schema-tools';
+import type { GroupNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
+import { filterResolverPaths } from '../lib/resolver-utils.js';
+import type Logger from '../logger.js';
+import { isLikelyResolver } from '../resolver/validate.js';
+import type { ConfigInit } from '../types.js';
+import { normalize } from './normalize.js';
+import {
+  graphAliases,
+  groupFromNode,
+  refToTokenID,
+  resolveAliases,
+  tokenFromNode,
+  tokenRawValuesFromNode,
+} from './token.js';
+
+export interface ProcessTokensOptions {
+  config: ConfigInit;
+  logger: Logger;
+  sourceByFilename: Record<string, InputSourceWithDocument>;
+  refMap: RefMap;
+  sources: InputSourceWithDocument[];
+}
+
+export function processTokens(
+  rootSource: InputSourceWithDocument,
+  { config, logger, sourceByFilename, refMap }: ProcessTokensOptions,
+): TokenNormalizedSet {
+  const entry = { group: 'parser' as const, label: 'init' };
+
+  // 2. Parse
+  const firstPass = performance.now();
+  const tokens: TokenNormalizedSet = {};
+  // micro-optimization: while we’re iterating over tokens, keeping a “hot”
+  // array in memory saves recreating arrays from object keys over and over again.
+  // it does produce a noticeable speedup > 1,000 tokens.
+  const tokenIDs: string[] = [];
+  const groups: Record<string, GroupNormalized> = {};
+
+  // 2a. Token & group population
+  const isResolver = isLikelyResolver(rootSource.document);
+  traverse(rootSource.document, {
+    enter(node, _parent, rawPath) {
+      if (node.type !== 'Object') {
+        return;
+      }
+      const path = isResolver ? filterResolverPaths(rawPath) : rawPath;
+      groupFromNode(node, { path, groups });
+      const token = tokenFromNode(node, {
+        groups,
+        ignore: config.ignore,
+        path,
+        source: rootSource,
+      });
+      if (token) {
+        tokenIDs.push(token.jsonID);
+        tokens[token.jsonID] = token;
+      }
+    },
+  });
+
+  logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
+  const secondPass = performance.now();
+
+  // 2b. Resolve originalValue and original sources
+  for (const source of Object.values(sourceByFilename)) {
+    traverse(source.document, {
+      enter(node, _parent, path) {
+        if (node.type !== 'Object') {
+          return;
+        }
+
+        const tokenRawValues = tokenRawValuesFromNode(node, { filename: source.filename!.href, path });
+        if (tokenRawValues && tokens[tokenRawValues?.jsonID]) {
+          tokens[tokenRawValues.jsonID]!.originalValue = tokenRawValues.originalValue;
+          tokens[tokenRawValues.jsonID]!.source = tokenRawValues.source;
+          for (const mode of Object.keys(tokenRawValues.mode)) {
+            tokens[tokenRawValues.jsonID]!.mode[mode]!.originalValue = tokenRawValues.mode[mode]!.originalValue;
+            tokens[tokenRawValues.jsonID]!.mode[mode]!.source = tokenRawValues.mode[mode]!.source;
+          }
+        }
+      },
+    });
+  }
+
+  // 2c. DTCG alias resolution
+  // Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
+  resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
+  logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
+
+  // 3. Alias graph
+  // We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
+  const aliasStart = performance.now();
+  graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
+  logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
+
+  // 4. normalize
+  // Allow for some minor variance in inputs, and be nice to folks.
+  const normalizeStart = performance.now();
+  for (const id of tokenIDs) {
+    const token = tokens[id]!;
+    normalize(token as any, { logger, src: sourceByFilename[token.source.filename!]?.src });
+  }
+  logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
+
+  // 5. alphabetize & filter
+  // This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
+  const tokensSorted: TokenNormalizedSet = {};
+  tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
+  for (const path of tokenIDs) {
+    // Filter out any tokens in $defs (we needed to reference them earlier, but shouldn’t include them in the final assortment)
+    if (path.includes('/$defs/')) {
+      continue;
+    }
+    const id = refToTokenID(path)!;
+    tokensSorted[id] = tokens[path]!;
+  }
+  // Sort group IDs once, too
+  for (const group of Object.values(groups)) {
+    group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
+  }
+
+  return tokensSorted;
+}