@terrazzo/parser 2.0.0-alpha.5 → 2.0.0-alpha.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@terrazzo/parser",
-  "version": "2.0.0-alpha.5",
+  "version": "2.0.0-alpha.7",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
   "license": "MIT",
   "type": "module",
@@ -43,8 +43,8 @@
     "picocolors": "^1.1.1",
     "scule": "^1.3.0",
     "wildcard-match": "^5.1.4",
-    "@terrazzo/json-schema-tools": "^0.1.0-alpha.0",
-    "@terrazzo/token-tools": "^2.0.0-alpha.5"
+    "@terrazzo/json-schema-tools": "^0.2.0",
+    "@terrazzo/token-tools": "^2.0.0-alpha.7"
   },
   "peerDependencies": {
     "yaml-to-momoa": "0.0.8"

package/src/lint/plugin-core/rules/valid-border.ts

@@ -12,7 +12,7 @@ const ERROR_INVALID_PROP = 'ERROR_INVALID_PROP';
 const rule: LintRule<typeof ERROR | typeof ERROR_INVALID_PROP, {}> = {
   meta: {
     messages: {
-      [ERROR]: `Border token missing required properties: ${new Intl.ListFormat(undefined, { type: 'conjunction' }).format(BORDER_REQUIRED_PROPERTIES)}.`,
+      [ERROR]: `Border token missing required properties: ${new Intl.ListFormat('en-us', { type: 'conjunction' }).format(BORDER_REQUIRED_PROPERTIES)}.`,
       [ERROR_INVALID_PROP]: 'Unknown property: {{ key }}.',
     },
     docs: {

package/src/lint/plugin-core/rules/valid-color.ts

@@ -54,7 +54,7 @@ const rule: LintRule<
   meta: {
     messages: {
       [ERROR_ALPHA]: `Alpha {{ alpha }} not in range 0 – 1.`,
-      [ERROR_INVALID_COLOR_SPACE]: `Invalid color space: {{ colorSpace }}. Expected ${new Intl.ListFormat(undefined, { type: 'disjunction' }).format(Object.keys(COLORSPACE))}`,
+      [ERROR_INVALID_COLOR_SPACE]: `Invalid color space: {{ colorSpace }}. Expected ${new Intl.ListFormat('en-us', { type: 'disjunction' }).format(Object.keys(COLORSPACE))}`,
       [ERROR_INVALID_COLOR]: `Could not parse color {{ color }}.`,
       [ERROR_INVALID_COMPONENT_LENGTH]: 'Expected {{ expected }} components, received {{ got }}.',
       [ERROR_INVALID_HEX8]: `Hex value can’t be semi-transparent.`,

package/src/lint/plugin-core/rules/valid-font-weight.ts

@@ -21,7 +21,7 @@ export interface RuleFontWeightOptions {
 const rule: LintRule<typeof ERROR | typeof ERROR_STYLE, RuleFontWeightOptions> = {
   meta: {
     messages: {
-      [ERROR]: `Must either be a valid number (0 - 999) or a valid font weight: ${new Intl.ListFormat(undefined, { type: 'disjunction' }).format(Object.keys(FONT_WEIGHTS))}.`,
+      [ERROR]: `Must either be a valid number (0 - 999) or a valid font weight: ${new Intl.ListFormat('en-us', { type: 'disjunction' }).format(Object.keys(FONT_WEIGHTS))}.`,
       [ERROR_STYLE]: 'Expected style {{ style }}, received {{ value }}.',
     },
     docs: {

package/src/lint/plugin-core/rules/valid-shadow.ts

@@ -12,7 +12,7 @@ const ERROR_INVALID_PROP = 'ERROR_INVALID_PROP';
 const rule: LintRule<typeof ERROR | typeof ERROR_INVALID_PROP> = {
   meta: {
     messages: {
-      [ERROR]: `Missing required properties: ${new Intl.ListFormat(undefined, { type: 'conjunction' }).format(SHADOW_REQUIRED_PROPERTIES)}.`,
+      [ERROR]: `Missing required properties: ${new Intl.ListFormat('en-us', { type: 'conjunction' }).format(SHADOW_REQUIRED_PROPERTIES)}.`,
       [ERROR_INVALID_PROP]: 'Unknown property {{ key }}.',
     },
     docs: {

package/src/lint/plugin-core/rules/valid-stroke-style.ts

@@ -20,9 +20,9 @@ const ERROR_INVALID_PROP = 'ERROR_INVALID_PROP';
 const rule: LintRule<typeof ERROR_STR | typeof ERROR_OBJ | typeof ERROR_LINE_CAP | typeof ERROR_INVALID_PROP> = {
   meta: {
     messages: {
-      [ERROR_STR]: `Value most be one of ${new Intl.ListFormat(undefined, { type: 'disjunction' }).format(STROKE_STYLE_STRING_VALUES)}.`,
-      [ERROR_OBJ]: `Missing required properties: ${new Intl.ListFormat(undefined, { type: 'conjunction' }).format(TRANSITION_REQUIRED_PROPERTIES)}.`,
-      [ERROR_LINE_CAP]: `lineCap must be one of ${new Intl.ListFormat(undefined, { type: 'disjunction' }).format(STROKE_STYLE_LINE_CAP_VALUES)}.`,
+      [ERROR_STR]: `Value most be one of ${new Intl.ListFormat('en-us', { type: 'disjunction' }).format(STROKE_STYLE_STRING_VALUES)}.`,
+      [ERROR_OBJ]: `Missing required properties: ${new Intl.ListFormat('en-us', { type: 'conjunction' }).format(TRANSITION_REQUIRED_PROPERTIES)}.`,
+      [ERROR_LINE_CAP]: `lineCap must be one of ${new Intl.ListFormat('en-us', { type: 'disjunction' }).format(STROKE_STYLE_LINE_CAP_VALUES)}.`,
       [ERROR_INVALID_PROP]: 'Unknown property: {{ key }}.',
     },
     docs: {

package/src/lint/plugin-core/rules/valid-transition.ts

@@ -12,7 +12,7 @@ const ERROR_INVALID_PROP = 'ERROR_INVALID_PROP';
 const rule: LintRule<typeof ERROR | typeof ERROR_INVALID_PROP> = {
   meta: {
     messages: {
-      [ERROR]: `Missing required properties: ${new Intl.ListFormat(undefined, { type: 'conjunction' }).format(TRANSITION_REQUIRED_PROPERTIES)}.`,
+      [ERROR]: `Missing required properties: ${new Intl.ListFormat('en-us', { type: 'conjunction' }).format(TRANSITION_REQUIRED_PROPERTIES)}.`,
       [ERROR_INVALID_PROP]: 'Unknown property: {{ key }}.',
     },
     docs: {

package/src/parse/load.ts

@@ -2,10 +2,10 @@ import * as momoa from '@humanwhocodes/momoa';
 import {
   type BundleOptions,
   bundle,
+  encodeFragment,
   getObjMember,
   type InputSource,
   type InputSourceWithDocument,
-  type RefMap,
   replaceNode,
   traverse,
 } from '@terrazzo/json-schema-tools';
@@ -75,9 +75,6 @@ export async function loadSources(
   }));
   /** The sources array, indexed by filename */
   let sourceByFilename: Record<string, InputSourceWithDocument> = {};
-  /** Mapping of all final $ref resolutions. This will be used to generate the graph later. */
-  let refMap: RefMap = {};
-
   try {
     const result = await bundle(sources, {
       req,
@@ -86,7 +83,6 @@ export async function loadSources(
     });
     document = result.document;
     sourceByFilename = result.sources;
-    refMap = result.refMap;
     for (const [filename, source] of Object.entries(result.sources)) {
       const i = sources.findIndex((s) => s.filename.href === filename);
       if (i === -1) {
@@ -111,9 +107,14 @@ export async function loadSources(
   }
   logger.debug({ ...entry, message: `JSON loaded`, timing: performance.now() - firstLoad });
 
-  const rootSource = { filename: sources[0]!.filename!, document, src: momoa.print(document, { indent: 2 }) };
+  const rootSource = {
+    filename: sources[0]!.filename!,
+    document,
+    src: momoa.print(document, { indent: 2 }).replace(/\\\//g, '/'),
+  };
+
   return {
-    tokens: processTokens(rootSource, { config, logger, refMap, sources, sourceByFilename }),
+    tokens: processTokens(rootSource, { config, logger, sources, sourceByFilename }),
     sources,
   };
 }
@@ -141,7 +142,7 @@ function transformer(transform: TransformVisitors): BundleOptions['parse'] {
         const ctx = { filename, parent, path };
         const next$type = getObjMember(node, '$type');
         if (next$type?.type === 'String') {
-          const jsonPath = `#/${path.join('/')}`;
+          const jsonPath = encodeFragment(path);
           if (jsonPath.startsWith(lastPath)) {
             last$type = next$type.value;
           }

package/src/parse/process.ts

@@ -1,11 +1,22 @@
-import { type InputSourceWithDocument, type RefMap, traverse } from '@terrazzo/json-schema-tools';
-import type { GroupNormalized, TokenNormalizedSet } from '@terrazzo/token-tools';
+import * as momoa from '@humanwhocodes/momoa';
+import {
+  encodeFragment,
+  findNode,
+  getObjMember,
+  type InputSourceWithDocument,
+  mergeObjects,
+  parseRef,
+  replaceNode,
+  traverse,
+} from '@terrazzo/json-schema-tools';
+import { type GroupNormalized, isAlias, type TokenNormalizedSet } from '@terrazzo/token-tools';
 import { filterResolverPaths } from '../lib/resolver-utils.js';
 import type Logger from '../logger.js';
 import { isLikelyResolver } from '../resolver/validate.js';
-import type { ConfigInit } from '../types.js';
+import type { ConfigInit, RefMap } from '../types.js';
 import { normalize } from './normalize.js';
 import {
+  aliasToGroupRef,
   graphAliases,
   groupFromNode,
   refToTokenID,
@@ -18,17 +29,136 @@ export interface ProcessTokensOptions {
   config: ConfigInit;
   logger: Logger;
   sourceByFilename: Record<string, InputSourceWithDocument>;
-  refMap: RefMap;
   sources: InputSourceWithDocument[];
 }
 
 export function processTokens(
   rootSource: InputSourceWithDocument,
-  { config, logger, sourceByFilename, refMap }: ProcessTokensOptions,
+  { config, logger, sourceByFilename }: ProcessTokensOptions,
 ): TokenNormalizedSet {
   const entry = { group: 'parser' as const, label: 'init' };
 
-  // 2. Parse
+  // 1. Inline $refs to discover any additional tokens
+  const refMap: RefMap = {};
+  function resolveRef(node: momoa.StringNode, chain: string[]): momoa.AnyNode {
+    const { subpath } = parseRef(node.value);
+    if (!subpath) {
+      logger.error({ ...entry, message: 'Can’t resolve $ref', node, src: rootSource.src });
+      // exit
+    }
+    const next = findNode(rootSource.document, subpath);
+    if (next?.type === 'Object') {
+      const next$ref = getObjMember(next, '$ref');
+      if (next$ref && next$ref.type === 'String') {
+        if (chain.includes(next$ref.value)) {
+          logger.error({
+            ...entry,
+            message: `Circular $ref detected: ${JSON.stringify(next$ref.value)}`,
+            node: next$ref,
+            src: rootSource.src,
+          });
+        }
+        chain.push(next$ref.value);
+        return resolveRef(next$ref, chain);
+      }
+    }
+    return next;
+  }
+  const inlineStart = performance.now();
+  traverse(rootSource.document, {
+    enter(node, _parent, rawPath) {
+      if (rawPath.includes('$extensions') || node.type !== 'Object') {
+        return;
+      }
+      const $ref = node.type === 'Object' ? (getObjMember(node, '$ref') as momoa.StringNode) : undefined;
+      if (!$ref) {
+        return;
+      }
+      if ($ref.type !== 'String') {
+        logger.error({ ...entry, message: 'Invalid $ref. Expected string.', node: $ref, src: rootSource.src });
+      }
+      const jsonID = encodeFragment(rawPath);
+      refMap[jsonID] = { filename: rootSource.filename.href, refChain: [$ref.value] };
+      const resolved = resolveRef($ref, refMap[jsonID]!.refChain);
+      if (resolved.type === 'Object') {
+        node.members.splice(
+          node.members.findIndex((m) => m.name.type === 'String' && m.name.value === '$ref'),
+          1,
+        );
+        replaceNode(node, mergeObjects(resolved, node));
+      } else {
+        replaceNode(node, resolved);
+      }
+    },
+  });
+  logger.debug({ ...entry, message: 'Inline aliases', timing: performance.now() - inlineStart });
+
+  // 2. Resolve $extends to discover any more additional tokens
+  function flatten$extends(node: momoa.ObjectNode, chain: string[]) {
+    const memberKeys = node.members.map((m) => m.name.type === 'String' && m.name.value).filter(Boolean) as string[];
+
+    let extended: momoa.ObjectNode | undefined;
+
+    if (memberKeys.includes('$extends')) {
+      const $extends = getObjMember(node, '$extends') as momoa.StringNode;
+      if ($extends.type !== 'String') {
+        logger.error({ ...entry, message: '$extends must be a string', node: $extends, src: rootSource.src });
+      }
+      if (memberKeys.includes('$value')) {
+        logger.error({ ...entry, message: '$extends can’t exist within a token', node: $extends, src: rootSource.src });
+      }
+      const next = isAlias($extends.value) ? aliasToGroupRef($extends.value) : undefined;
+      if (!next) {
+        logger.error({ ...entry, message: '$extends must be a valid alias', node: $extends, src: rootSource.src });
+      }
+
+      if (
+        chain.includes(next!.$ref) ||
+        // Check that $extends is not importing from higher up (could go in either direction, which is why we check both ways)
+        chain.some((value) => value.startsWith(next!.$ref) || next!.$ref.startsWith(value))
+      ) {
+        logger.error({ ...entry, message: 'Circular $extends detected', node: $extends, src: rootSource.src });
+      }
+
+      chain.push(next!.$ref);
+      extended = findNode(rootSource.document, parseRef(next!.$ref).subpath ?? []);
+      if (!extended) {
+        logger.error({ ...entry, message: 'Could not resolve $extends', node: $extends, src: rootSource.src });
+      }
+      if (extended!.type !== 'Object') {
+        logger.error({ ...entry, message: '$extends must resolve to a group of tokens', node });
+      }
+
+      // To ensure this is resolvable, try and flatten this node first (will catch circular refs)
+      flatten$extends(extended!, chain);
+
+      replaceNode(node, mergeObjects(extended!, node));
+    }
+
+    // Deeply-traverse for any interior $extends (even if it wasn’t at the top level)
+    for (const member of node.members) {
+      if (
+        member.value.type === 'Object' &&
+        member.name.type === 'String' &&
+        !['$value', '$extensions'].includes(member.name.value)
+      ) {
+        traverse(member.value, {
+          enter(subnode, _parent) {
+            if (subnode.type === 'Object') {
+              flatten$extends(subnode, chain);
+            }
+          },
+        });
+      }
+    }
+  }
+
+  const extendsStart = performance.now();
+  const extendsChain: string[] = [];
+  flatten$extends(rootSource.document.body as momoa.ObjectNode, extendsChain);
+  logger.debug({ ...entry, message: 'Resolving $extends', timing: performance.now() - extendsStart });
+
+  // 3. Parse discovered tokens
   const firstPass = performance.now();
   const tokens: TokenNormalizedSet = {};
   // micro-optimization: while we’re iterating over tokens, keeping a “hot”
@@ -37,7 +167,7 @@ export function processTokens(
   const tokenIDs: string[] = [];
   const groups: Record<string, GroupNormalized> = {};
 
-  // 2a. Token & group population
+  // 3a. Token & group population
   const isResolver = isLikelyResolver(rootSource.document);
   traverse(rootSource.document, {
     enter(node, _parent, rawPath) {
@@ -61,7 +191,7 @@ export function processTokens(
   logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
   const secondPass = performance.now();
 
-  // 2b. Resolve originalValue and original sources
+  // 3b. Resolve originalValue and original sources
   for (const source of Object.values(sourceByFilename)) {
     traverse(source.document, {
       enter(node, _parent, path) {
@@ -82,18 +212,18 @@ export function processTokens(
     });
   }
 
-  // 2c. DTCG alias resolution
+  // 3c. DTCG alias resolution
   // Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
   resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
   logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
 
-  // 3. Alias graph
+  // 4. Alias graph
   // We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
   const aliasStart = performance.now();
   graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
   logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
 
-  // 4. normalize
+  // 5. normalize
   // Allow for some minor variance in inputs, and be nice to folks.
   const normalizeStart = performance.now();
   for (const id of tokenIDs) {
@@ -102,15 +232,12 @@ export function processTokens(
   }
   logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
 
-  // 5. alphabetize & filter
+  // 6. alphabetize & filter
   // This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
+  const sortStart = performance.now();
   const tokensSorted: TokenNormalizedSet = {};
   tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
   for (const path of tokenIDs) {
-    // Filter out any tokens in $defs (we needed to reference them earlier, but shouldn’t include them in the final assortment)
-    if (path.includes('/$defs/')) {
-      continue;
-    }
     const id = refToTokenID(path)!;
     tokensSorted[id] = tokens[path]!;
   }
@@ -118,6 +245,7 @@ export function processTokens(
   for (const group of Object.values(groups)) {
     group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
   }
+  logger.debug({ ...entry, message: 'Sorted tokens', timing: performance.now() - sortStart });
 
   return tokensSorted;
 }

package/src/parse/token.ts

@@ -1,5 +1,5 @@
 import * as momoa from '@humanwhocodes/momoa';
-import { getObjMember, type InputSourceWithDocument, parseRef, type RefMap } from '@terrazzo/json-schema-tools';
+import { encodeFragment, getObjMember, type InputSourceWithDocument, parseRef } from '@terrazzo/json-schema-tools';
 import {
   type GroupNormalized,
   isAlias,
@@ -9,10 +9,20 @@ import {
 } from '@terrazzo/token-tools';
 import wcmatch from 'wildcard-match';
 import type { default as Logger } from '../logger.js';
-import type { Config, ReferenceObject } from '../types.js';
+import type { Config, ReferenceObject, RefMap } from '../types.js';
 
 /** Convert valid DTCG alias to $ref */
-export function aliasToRef(alias: string, mode?: string): ReferenceObject | undefined {
+export function aliasToGroupRef(alias: string): ReferenceObject | undefined {
+  const id = parseAlias(alias);
+  // if this is invalid, stop
+  if (id === alias) {
+    return;
+  }
+  return { $ref: `#/${id.replace(/~/g, '~0').replace(/\//g, '~1').replace(/\./g, '/')}` };
+}
+
+/** Convert valid DTCG alias to $ref */
+export function aliasToTokenRef(alias: string, mode?: string): ReferenceObject | undefined {
   const id = parseAlias(alias);
   // if this is invalid, stop
   if (id === alias) {
@@ -40,12 +50,12 @@ export function tokenFromNode(
     return undefined;
   }
 
-  const jsonID = `#/${path.join('/')}`;
+  const jsonID = encodeFragment(path);
   const id = path.join('.');
 
   const originalToken = momoa.evaluate(node) as any;
 
-  const groupID = `#/${path.slice(0, -1).join('/')}`;
+  const groupID = encodeFragment(path.slice(0, -1));
   const group = groups[groupID]!;
   if (group?.tokens && !group.tokens.includes(id)) {
     group.tokens.push(id);
@@ -132,7 +142,7 @@ export function tokenRawValuesFromNode(
     return undefined;
   }
 
-  const jsonID = `#/${path.join('/')}`;
+  const jsonID = encodeFragment(path);
   const rawValues: TokenRawValues = {
     jsonID,
     originalValue: momoa.evaluate(node),
@@ -173,7 +183,7 @@ export function groupFromNode(
   { path, groups }: { path: string[]; groups: Record<string, GroupNormalized> },
 ): GroupNormalized {
   const id = path.join('.');
-  const jsonID = `#/${path.join('/')}`;
+  const jsonID = encodeFragment(path);
 
   // group
   if (!groups[jsonID]) {
@@ -354,7 +364,7 @@ export function aliasToMomoa(
     end: { line: -1, column: -1, offset: 0 },
   },
 ): momoa.ObjectNode | undefined {
-  const $ref = aliasToRef(alias);
+  const $ref = aliasToTokenRef(alias);
   if (!$ref) {
     return;
   }
@@ -383,6 +393,10 @@ export function refToTokenID($ref: ReferenceObject | string): string | undefined
     return;
   }
   const { subpath } = parseRef(path);
+  // if this ID comes from #/$defs/…, strip the first 2 segments to get the global ID
+  if (subpath?.[0] === '$defs') {
+    subpath.splice(0, 2);
+  }
   return (subpath?.length && subpath.join('.').replace(/\.(\$value|\$extensions).*$/, '')) || undefined;
 }
 
@@ -422,7 +436,7 @@ const EXPECTED_NESTED_ALIAS: Record<string, Record<string, string[]>> = {
 };
 
 /**
- * Resolve DTCG aliases
+ * Resolve DTCG aliases, $extends, and $ref
  */
 export function resolveAliases(
   tokens: TokenNormalizedSet,
@@ -438,7 +452,7 @@ export function resolveAliases(
 
     for (const mode of Object.keys(token.mode)) {
       function resolveInner(alias: string, refChain: string[]): string {
-        const nextRef = aliasToRef(alias, mode)?.$ref;
+        const nextRef = aliasToTokenRef(alias, mode)?.$ref;
         if (!nextRef) {
           logger.error({ ...aliasEntry, message: `Internal error resolving ${JSON.stringify(refChain)}` });
           throw new Error('Internal error');

package/src/resolver/load.ts

@@ -161,7 +161,6 @@ export function createResolver(
         config,
         logger,
         sourceByFilename: { [resolverSource._source.filename!.href]: rootSource },
-        refMap: {},
         sources,
       });
       resolverCache[inputKey] = tokens;