@terrazzo/parser 2.0.0-alpha.3 → 2.0.0-alpha.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@terrazzo/parser",
-  "version": "2.0.0-alpha.3",
+  "version": "2.0.0-alpha.4",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
   "license": "MIT",
   "type": "module",
@@ -43,11 +43,11 @@
     "picocolors": "^1.1.1",
     "scule": "^1.3.0",
     "wildcard-match": "^5.1.4",
-    "@terrazzo/json-schema-tools": "^0.1.0",
-    "@terrazzo/token-tools": "^2.0.0-alpha.3"
+    "@terrazzo/json-schema-tools": "^0.1.0-alpha.0",
+    "@terrazzo/token-tools": "^2.0.0-alpha.4"
   },
   "devDependencies": {
-    "yaml-to-momoa": "0.0.6"
+    "yaml-to-momoa": "0.0.8"
   },
   "scripts": {
     "build": "rolldown -c && attw --profile esm-only --pack .",

package/src/build/index.ts

@@ -1,11 +1,11 @@
-import type * as momoa from '@humanwhocodes/momoa';
+import type { InputSourceWithDocument } from '@terrazzo/json-schema-tools';
 import type { TokenNormalized } from '@terrazzo/token-tools';
 import wcmatch from 'wildcard-match';
 import Logger, { type LogEntry } from '../logger.js';
 import type { BuildRunnerResult, ConfigInit, TokenTransformed, TransformParams } from '../types.js';
 
 export interface BuildRunnerOptions {
-  sources: { filename?: URL; src: string; document: momoa.DocumentNode }[];
+  sources: InputSourceWithDocument[];
   config: ConfigInit;
   logger?: Logger;
 }

package/src/lib/resolver-utils.ts

@@ -0,0 +1,35 @@
+/**
+ * If tokens are found inside a resolver, strip out the resolver paths (don’t
+ * include "sets"/"modifiers" in the token ID etc.)
+ */
+export function filterResolverPaths(path: string[]): string[] {
+  switch (path[0]) {
+    case 'sets': {
+      return path.slice(4);
+    }
+    case 'modifiers': {
+      return path.slice(5);
+    }
+    case 'resolutionOrder': {
+      switch (path[2]) {
+        case 'sources': {
+          return path.slice(4);
+        }
+        case 'contexts': {
+          return path.slice(5);
+        }
+      }
+      break;
+    }
+  }
+  return path;
+}
+
+/**
+ * Make a deterministic string from an object
+ */
+export function makeInputKey(input: Record<string, string | undefined>): string {
+  return JSON.stringify(
+    Object.fromEntries(Object.entries(input).sort((a, b) => a[0].localeCompare(b[0], 'en-us', { numeric: true }))),
+  );
+}

package/src/lint/index.ts

@@ -1,7 +1,8 @@
+import type { InputSourceWithDocument } from '@terrazzo/json-schema-tools';
 import { pluralize, type TokenNormalizedSet } from '@terrazzo/token-tools';
 import { merge } from 'merge-anything';
 import type { LogEntry, default as Logger } from '../logger.js';
-import type { ConfigInit, InputSource } from '../types.js';
+import type { ConfigInit } from '../types.js';
 
 export { RECOMMENDED_CONFIG } from './plugin-core/index.js';
 
@@ -9,7 +10,7 @@ export interface LintRunnerOptions {
   tokens: TokenNormalizedSet;
   filename?: URL;
   config: ConfigInit;
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
   logger: Logger;
 }
 
@@ -21,7 +22,7 @@ export default async function lintRunner({
   logger,
 }: LintRunnerOptions): Promise<void> {
   const { plugins = [], lint } = config;
-  const sourceByFilename: Record<string, InputSource> = {};
+  const sourceByFilename: Record<string, InputSourceWithDocument> = {};
   for (const source of sources) {
     sourceByFilename[source.filename!.href] = source;
   }

package/src/parse/index.ts

@@ -1,20 +1,22 @@
 import type fsType from 'node:fs/promises';
+import type { InputSource, InputSourceWithDocument } from '@terrazzo/json-schema-tools';
 import { pluralize, type TokenNormalizedSet } from '@terrazzo/token-tools';
 import lintRunner from '../lint/index.js';
 import Logger from '../logger.js';
+import { createSyntheticResolver } from '../resolver/create-synthetic-resolver.js';
 import { loadResolver } from '../resolver/load.js';
-import type { ConfigInit, InputSource, ParseOptions, Resolver } from '../types.js';
+import type { ConfigInit, ParseOptions, Resolver } from '../types.js';
 import { loadSources } from './load.js';
 
 export interface ParseResult {
   tokens: TokenNormalizedSet;
-  sources: InputSource[];
-  resolver?: Resolver | undefined;
+  sources: InputSourceWithDocument[];
+  resolver: Resolver;
 }
 
 /** Parse */
 export default async function parse(
-  _input: Omit<InputSource, 'document'> | Omit<InputSource, 'document'>[],
+  _input: InputSource | InputSource[],
   {
     logger = new Logger(),
     req = defaultReq,
@@ -26,22 +28,33 @@ export default async function parse(
   }: ParseOptions = {} as ParseOptions,
 ): Promise<ParseResult> {
   const inputs = Array.isArray(_input) ? _input : [_input];
+  let tokens: TokenNormalizedSet = {};
+  let resolver: Resolver | undefined;
+  let sources: InputSourceWithDocument[] = [];
 
   const totalStart = performance.now();
 
-  // 1. Resolver
-  const resolver = await loadResolver(inputs, { logger, req, yamlToMomoa });
-
-  // 2. No resolver (tokens)
+  // 1. Load tokens
   const initStart = performance.now();
-  const { tokens, sources } = await loadSources(inputs, {
-    req,
-    logger,
-    config,
-    continueOnError,
-    yamlToMomoa,
-    transform,
-  });
+  const resolverResult = await loadResolver(inputs, { config, logger, req, yamlToMomoa });
+  // 1a. Resolver
+  if (resolverResult.resolver) {
+    tokens = resolverResult.tokens;
+    sources = resolverResult.sources;
+    resolver = resolverResult.resolver;
+  } else {
+    // 1b. No resolver
+    const tokenResult = await loadSources(inputs, {
+      req,
+      logger,
+      config,
+      continueOnError,
+      yamlToMomoa,
+      transform,
+    });
+    tokens = tokenResult.tokens;
+    sources = tokenResult.sources;
+  }
   logger.debug({
     message: 'Loaded tokens',
     group: 'parser',
@@ -80,7 +93,7 @@ export default async function parse(
   return {
     tokens,
     sources,
-    resolver,
+    resolver: resolver || (await createSyntheticResolver(tokens, { config, logger, req, sources })),
   };
 }
 

package/src/parse/load.ts

@@ -3,23 +3,19 @@ import {
   type BundleOptions,
   bundle,
   getObjMember,
+  type InputSource,
+  type InputSourceWithDocument,
   type RefMap,
   replaceNode,
-  traverseAsync,
+  traverse,
 } from '@terrazzo/json-schema-tools';
-import type { GroupNormalized, TokenNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
+import type { TokenNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
 import { toMomoa } from '../lib/momoa.js';
+import { filterResolverPaths } from '../lib/resolver-utils.js';
 import type Logger from '../logger.js';
-import type { InputSource, ParseOptions, TransformVisitors } from '../types.js';
-import { normalize } from './normalize.js';
-import {
-  graphAliases,
-  groupFromNode,
-  refToTokenID,
-  resolveAliases,
-  tokenFromNode,
-  tokenRawValuesFromNode,
-} from './token.js';
+import { isLikelyResolver } from '../resolver/validate.js';
+import type { ParseOptions, TransformVisitors } from '../types.js';
+import { processTokens } from './process.js';
 
 /** Ephemeral format that only exists while parsing the document. This is not confirmed to be DTCG yet. */
 export interface IntermediaryToken {
@@ -57,12 +53,12 @@ export interface LoadOptions extends Pick<ParseOptions, 'config' | 'continueOnEr
 
 export interface LoadSourcesResult {
   tokens: TokenNormalizedSet;
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
 }
 
 /** Load from multiple entries, while resolving remote files */
 export async function loadSources(
-  inputs: Omit<InputSource, 'document'>[],
+  inputs: InputSource[],
   { config, logger, req, continueOnError, yamlToMomoa, transform }: LoadOptions,
 ): Promise<LoadSourcesResult> {
   const entry = { group: 'parser' as const, label: 'init' };
@@ -78,7 +74,7 @@ export async function loadSources(
     filename: input.filename || new URL(`virtual:${i}`), // for objects created in memory, an index-based ID helps associate tokens with these
   }));
   /** The sources array, indexed by filename */
-  let sourceByFilename: Record<string, InputSource> = {};
+  let sourceByFilename: Record<string, InputSourceWithDocument> = {};
   /** Mapping of all final $ref resolutions. This will be used to generate the graph later. */
   let refMap: RefMap = {};
 
@@ -113,101 +109,13 @@ export async function loadSources(
       src,
     });
   }
-
   logger.debug({ ...entry, message: `JSON loaded`, timing: performance.now() - firstLoad });
-  const artificialSource = { src: momoa.print(document, { indent: 2 }), document };
-
-  // 2. Parse
-  const firstPass = performance.now();
-  const tokens: TokenNormalizedSet = {};
-  // micro-optimization: while we’re iterating over tokens, keeping a “hot”
-  // array in memory saves recreating arrays from object keys over and over again.
-  // it does produce a noticeable speedup > 1,000 tokens.
-  const tokenIDs: string[] = [];
-  const groups: Record<string, GroupNormalized> = {};
-
-  // 2a. Token & group population
-  await traverseAsync(document, {
-    async enter(node, _parent, path) {
-      if (node.type !== 'Object') {
-        return;
-      }
-      groupFromNode(node, { path, groups });
-      const token = tokenFromNode(node, {
-        groups,
-        ignore: config.ignore,
-        path,
-        source: { src: artificialSource, document },
-      });
-      if (token) {
-        tokenIDs.push(token.jsonID);
-        tokens[token.jsonID] = token;
-      }
-    },
-  });
-
-  logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
-  const secondPass = performance.now();
 
-  // 2b. Resolve originalValue and original sources
-  for (const source of Object.values(sourceByFilename)) {
-    await traverseAsync(source.document, {
-      async enter(node, _parent, path) {
-        if (node.type !== 'Object') {
-          return;
-        }
-
-        const tokenRawValues = tokenRawValuesFromNode(node, { filename: source.filename!.href, path });
-        if (tokenRawValues && tokens[tokenRawValues?.jsonID]) {
-          tokens[tokenRawValues.jsonID]!.originalValue = tokenRawValues.originalValue;
-          tokens[tokenRawValues.jsonID]!.source = tokenRawValues.source;
-          for (const mode of Object.keys(tokenRawValues.mode)) {
-            tokens[tokenRawValues.jsonID]!.mode[mode]!.originalValue = tokenRawValues.mode[mode]!.originalValue;
-            tokens[tokenRawValues.jsonID]!.mode[mode]!.source = tokenRawValues.mode[mode]!.source;
-          }
-        }
-      },
-    });
-  }
-
-  // 2c. DTCG alias resolution
-  // Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
-  resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
-  logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
-
-  // 3. Alias graph
-  // We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
-  const aliasStart = performance.now();
-  graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
-  logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
-
-  // 4. normalize
-  // Allow for some minor variance in inputs, and be nice to folks.
-  const normalizeStart = performance.now();
-  for (const id of tokenIDs) {
-    const token = tokens[id]!;
-    normalize(token as any, { logger, src: sourceByFilename[token.source.filename!]?.src });
-  }
-  logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
-
-  // 5. alphabetize & filter
-  // This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
-  const tokensSorted: TokenNormalizedSet = {};
-  tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
-  for (const path of tokenIDs) {
-    // Filter out any tokens in $defs (we needed to reference them earlier, but shouldn’t include them in the final assortment)
-    if (path.includes('/$defs/')) {
-      continue;
-    }
-    const id = refToTokenID(path)!;
-    tokensSorted[id] = tokens[path]!;
-  }
-  // Sort group IDs once, too
-  for (const group of Object.values(groups)) {
-    group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
-  }
-
-  return { tokens: tokensSorted, sources };
+  const rootSource = { filename: sources[0]!.filename!, document, src: momoa.print(document, { indent: 2 }) };
+  return {
+    tokens: processTokens(rootSource, { config, logger, refMap, sources, sourceByFilename }),
+    sources,
+  };
 }
 
 function transformer(transform: TransformVisitors): BundleOptions['parse'] {
@@ -223,8 +131,10 @@ function transformer(transform: TransformVisitors): BundleOptions['parse'] {
       }
     }
 
-    await traverseAsync(document, {
-      async enter(node, parent, path) {
+    const isResolver = isLikelyResolver(document);
+    traverse(document, {
+      enter(node, parent, rawPath) {
+        const path = isResolver ? filterResolverPaths(rawPath) : rawPath;
         if (node.type !== 'Object' || !path.length) {
           return;
         }

package/src/parse/process.ts

@@ -0,0 +1,124 @@
+import { type InputSourceWithDocument, type RefMap, traverse } from '@terrazzo/json-schema-tools';
+import type { GroupNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
+import { filterResolverPaths } from '../lib/resolver-utils.js';
+import type Logger from '../logger.js';
+import { isLikelyResolver } from '../resolver/validate.js';
+import type { ConfigInit } from '../types.js';
+import { normalize } from './normalize.js';
+import {
+  graphAliases,
+  groupFromNode,
+  refToTokenID,
+  resolveAliases,
+  tokenFromNode,
+  tokenRawValuesFromNode,
+} from './token.js';
+
+export interface ProcessTokensOptions {
+  config: ConfigInit;
+  logger: Logger;
+  sourceByFilename: Record<string, InputSourceWithDocument>;
+  refMap: RefMap;
+  sources: InputSourceWithDocument[];
+}
+
+export function processTokens(
+  rootSource: InputSourceWithDocument,
+  { config, logger, sourceByFilename, refMap }: ProcessTokensOptions,
+): TokenNormalizedSet {
+  const entry = { group: 'parser' as const, label: 'init' };
+
+  // 2. Parse
+  const firstPass = performance.now();
+  const tokens: TokenNormalizedSet = {};
+  // micro-optimization: while we’re iterating over tokens, keeping a “hot”
+  // array in memory saves recreating arrays from object keys over and over again.
+  // it does produce a noticeable speedup > 1,000 tokens.
+  const tokenIDs: string[] = [];
+  const groups: Record<string, GroupNormalized> = {};
+
+  // 2a. Token & group population
+  const isResolver = isLikelyResolver(rootSource.document);
+  traverse(rootSource.document, {
+    enter(node, _parent, rawPath) {
+      if (node.type !== 'Object') {
+        return;
+      }
+      const path = isResolver ? filterResolverPaths(rawPath) : rawPath;
+      groupFromNode(node, { path, groups });
+      const token = tokenFromNode(node, {
+        groups,
+        ignore: config.ignore,
+        path,
+        source: rootSource,
+      });
+      if (token) {
+        tokenIDs.push(token.jsonID);
+        tokens[token.jsonID] = token;
+      }
+    },
+  });
+
+  logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
+  const secondPass = performance.now();
+
+  // 2b. Resolve originalValue and original sources
+  for (const source of Object.values(sourceByFilename)) {
+    traverse(source.document, {
+      enter(node, _parent, path) {
+        if (node.type !== 'Object') {
+          return;
+        }
+
+        const tokenRawValues = tokenRawValuesFromNode(node, { filename: source.filename!.href, path });
+        if (tokenRawValues && tokens[tokenRawValues?.jsonID]) {
+          tokens[tokenRawValues.jsonID]!.originalValue = tokenRawValues.originalValue;
+          tokens[tokenRawValues.jsonID]!.source = tokenRawValues.source;
+          for (const mode of Object.keys(tokenRawValues.mode)) {
+            tokens[tokenRawValues.jsonID]!.mode[mode]!.originalValue = tokenRawValues.mode[mode]!.originalValue;
+            tokens[tokenRawValues.jsonID]!.mode[mode]!.source = tokenRawValues.mode[mode]!.source;
+          }
+        }
+      },
+    });
+  }
+
+  // 2c. DTCG alias resolution
+  // Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
+  resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
+  logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
+
+  // 3. Alias graph
+  // We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
+  const aliasStart = performance.now();
+  graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
+  logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
+
+  // 4. normalize
+  // Allow for some minor variance in inputs, and be nice to folks.
+  const normalizeStart = performance.now();
+  for (const id of tokenIDs) {
+    const token = tokens[id]!;
+    normalize(token as any, { logger, src: sourceByFilename[token.source.filename!]?.src });
+  }
+  logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
+
+  // 5. alphabetize & filter
+  // This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
+  const tokensSorted: TokenNormalizedSet = {};
+  tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
+  for (const path of tokenIDs) {
+    // Filter out any tokens in $defs (we needed to reference them earlier, but shouldn’t include them in the final assortment)
+    if (path.includes('/$defs/')) {
+      continue;
+    }
+    const id = refToTokenID(path)!;
+    tokensSorted[id] = tokens[path]!;
+  }
+  // Sort group IDs once, too
+  for (const group of Object.values(groups)) {
+    group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
+  }
+
+  return tokensSorted;
+}

package/src/parse/token.ts

@@ -1,5 +1,5 @@
 import * as momoa from '@humanwhocodes/momoa';
-import { getObjMember, parseRef, type RefMap } from '@terrazzo/json-schema-tools';
+import { getObjMember, type InputSourceWithDocument, parseRef, type RefMap } from '@terrazzo/json-schema-tools';
 import {
   type GroupNormalized,
   isAlias,
@@ -9,7 +9,7 @@ import {
 } from '@terrazzo/token-tools';
 import wcmatch from 'wildcard-match';
 import type { default as Logger } from '../logger.js';
-import type { Config, InputSource, ReferenceObject } from '../types.js';
+import type { Config, ReferenceObject } from '../types.js';
 
 /** Convert valid DTCG alias to $ref */
 export function aliasToRef(alias: string, mode?: string): ReferenceObject | undefined {
@@ -26,7 +26,7 @@ export function aliasToRef(alias: string, mode?: string): ReferenceObject | unde
 export interface TokenFromNodeOptions {
   groups: Record<string, GroupNormalized>;
   path: string[];
-  source: InputSource;
+  source: InputSourceWithDocument;
   ignore: Config['ignore'];
 }
 
@@ -51,7 +51,7 @@ export function tokenFromNode(
     group.tokens.push(id);
   }
 
-  const nodeSource = { filename: source.filename?.href, node };
+  const nodeSource = { filename: source.filename.href, node };
   const token: TokenNormalized = {
     id,
     $type: originalToken.$type || group.$type,
@@ -211,7 +211,7 @@ export function groupFromNode(
 
 export interface GraphAliasesOptions {
   tokens: TokenNormalizedSet;
-  sources: Record<string, InputSource>;
+  sources: Record<string, InputSourceWithDocument>;
   logger: Logger;
 }
 
@@ -425,7 +425,7 @@ const EXPECTED_NESTED_ALIAS: Record<string, Record<string, string[]>> = {
  */
 export function resolveAliases(
   tokens: TokenNormalizedSet,
-  { logger, refMap, sources }: { logger: Logger; refMap: RefMap; sources: Record<string, InputSource> },
+  { logger, refMap, sources }: { logger: Logger; refMap: RefMap; sources: Record<string, InputSourceWithDocument> },
 ): void {
   for (const token of Object.values(tokens)) {
     const aliasEntry = {

package/src/resolver/create-synthetic-resolver.ts

@@ -0,0 +1,86 @@
+import * as momoa from '@humanwhocodes/momoa';
+import type { InputSourceWithDocument } from '@terrazzo/json-schema-tools';
+import type Logger from '../logger.js';
+import type { ConfigInit, Group, Resolver, TokenNormalized, TokenNormalizedSet } from '../types.js';
+import { createResolver } from './load.js';
+import { normalizeResolver } from './normalize.js';
+
+export interface CreateSyntheticResolverOptions {
+  config: ConfigInit;
+  logger: Logger;
+  req: (url: URL, origin: URL) => Promise<string>;
+  sources: InputSourceWithDocument[];
+}
+
+/**
+ * Interop layer upgrading legacy Terrazzo modes to resolvers
+ */
+export async function createSyntheticResolver(
+  tokens: TokenNormalizedSet,
+  { config, logger, req, sources }: CreateSyntheticResolverOptions,
+): Promise<Resolver> {
+  const contexts: Record<string, any[]> = {};
+  for (const token of Object.values(tokens)) {
+    for (const [mode, value] of Object.entries(token.mode)) {
+      if (mode === '.') {
+        continue;
+      }
+      if (!(mode in contexts)) {
+        contexts[mode] = [{}];
+      }
+      addToken(contexts[mode]![0], { ...token, $value: value.$value }, { logger });
+    }
+  }
+
+  const src = JSON.stringify(
+    {
+      name: 'Terrazzo',
+      version: '2025.10',
+      resolutionOrder: [{ $ref: '#/sets/allTokens' }, { $ref: '#/modifiers/tzMode' }],
+      sets: {
+        allTokens: { sources: [simpleFlatten(tokens, { logger })] },
+      },
+      modifiers: {
+        tzMode: {
+          description: 'Automatically built from $extensions.mode',
+          contexts,
+        },
+      },
+    },
+    undefined,
+    2,
+  );
+  const normalized = await normalizeResolver(momoa.parse(src), {
+    filename: new URL('file:///virtual:resolver.json'),
+    logger,
+    req,
+    src,
+  });
+  return createResolver(normalized, { config, logger, sources });
+}
+
+/** Add a normalized token back into an arbitrary, hierarchial structure */
+function addToken(structure: any, token: TokenNormalized, { logger }: { logger: Logger }): void {
+  let node = structure;
+  const parts = token.id.split('.');
+  const localID = parts.pop()!;
+  for (const part of parts) {
+    if (!(part in node)) {
+      node[part] = {};
+    }
+    node = node[part];
+  }
+  if (localID in node) {
+    logger.error({ group: 'parser', label: 'resolver', message: `${localID} already exists!` });
+  }
+  node[localID] = { $type: token.$type, $value: token.$value };
+}
+
+/** Downconvert normalized tokens back into a simplified, hierarchial shape. This is extremely lossy, and only done to build a resolver. */
+function simpleFlatten(tokens: TokenNormalizedSet, { logger }: { logger: Logger }): Group {
+  const group: Group = {};
+  for (const token of Object.values(tokens)) {
+    addToken(group, token, { logger });
+  }
+  return group;
+}