@terrazzo/parser 2.0.0-alpha.2 → 2.0.0-alpha.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@terrazzo/parser",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.3",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
   "license": "MIT",
   "type": "module",
@@ -27,23 +27,24 @@
     ".": "./dist/index.js",
     "./package.json": "./package.json"
   },
-  "homepage": "https://terrazzo.app/docs/cli/api/js",
+  "homepage": "https://terrazzo.app/docs/reference/js-api",
   "repository": {
     "type": "git",
     "url": "https://github.com/terrazzoapp/terrazzo.git",
     "directory": "./packages/parser/"
   },
   "dependencies": {
-    "@humanwhocodes/momoa": "^3.3.9",
+    "@humanwhocodes/momoa": "^3.3.10",
     "@types/babel__code-frame": "^7.0.6",
     "@types/culori": "^4.0.1",
     "culori": "^4.0.2",
+    "fast-deep-equal": "^3.1.3",
     "merge-anything": "^5.1.7",
     "picocolors": "^1.1.1",
     "scule": "^1.3.0",
     "wildcard-match": "^5.1.4",
-    "@terrazzo/json-schema-tools": "^0.0.1",
-    "@terrazzo/token-tools": "^2.0.0-alpha.2"
+    "@terrazzo/json-schema-tools": "^0.1.0",
+    "@terrazzo/token-tools": "^2.0.0-alpha.3"
   },
   "devDependencies": {
     "yaml-to-momoa": "0.0.6"

package/src/config.ts

@@ -206,7 +206,6 @@ function normalizeLint({ config, logger }: { config: ConfigInit; logger: Logger
           // Note: sometimes plugins will be loaded multiple times, in which case it’s expected
           // they’re register rules again for lint(). Only throw an error if plugin A and plugin B’s
           // rules conflict.
-
           if (allRules.get(rule) && allRules.get(rule) !== plugin.name) {
             logger.error({
               group: 'config',
@@ -229,7 +228,7 @@ function normalizeLint({ config, logger }: { config: ConfigInit; logger: Logger
 
         const value = config.lint.rules[id];
         let severity: LintRuleSeverity = 'off';
-        let options: any;
+        let options: any = {};
         if (typeof value === 'number' || typeof value === 'string') {
           severity = value;
         } else if (Array.isArray(value)) {

package/src/index.ts

@@ -90,4 +90,6 @@ export { default as Logger } from './logger.js';
 export * from './parse/index.js';
 export { default as parse } from './parse/index.js';
 
+export * from './resolver/index.js';
+
 export * from './types.js';

package/src/lint/plugin-core/index.ts

@@ -8,9 +8,9 @@ export * from './rules/consistent-naming.js';
 export * from './rules/descriptions.js';
 export * from './rules/duplicate-values.js';
 export * from './rules/max-gamut.js';
-export * from './rules/no-type-on-alias.js';
 export * from './rules/required-children.js';
 export * from './rules/required-modes.js';
+export * from './rules/required-type.js';
 export * from './rules/required-typography-properties.js';
 
 import a11yMinContrast, { A11Y_MIN_CONTRAST } from './rules/a11y-min-contrast.js';
@@ -20,9 +20,9 @@ import consistentNaming, { CONSISTENT_NAMING } from './rules/consistent-naming.j
 import descriptions, { DESCRIPTIONS } from './rules/descriptions.js';
 import duplicateValues, { DUPLICATE_VALUES } from './rules/duplicate-values.js';
 import maxGamut, { MAX_GAMUT } from './rules/max-gamut.js';
-import noTypeOnAlias, { NO_TYPE_ON_ALIAS } from './rules/no-type-on-alias.js';
 import requiredChildren, { REQUIRED_CHILDREN } from './rules/required-children.js';
 import requiredModes, { REQUIRED_MODES } from './rules/required-modes.js';
+import requiredType, { REQUIRED_TYPE } from './rules/required-type.js';
 import requiredTypographyProperties, {
   REQUIRED_TYPOGRAPHY_PROPERTIES,
 } from './rules/required-typography-properties.js';
@@ -65,9 +65,9 @@ const ALL_RULES = {
   [DESCRIPTIONS]: descriptions,
   [DUPLICATE_VALUES]: duplicateValues,
   [MAX_GAMUT]: maxGamut,
-  [NO_TYPE_ON_ALIAS]: noTypeOnAlias,
   [REQUIRED_CHILDREN]: requiredChildren,
   [REQUIRED_MODES]: requiredModes,
+  [REQUIRED_TYPE]: requiredType,
   [REQUIRED_TYPOGRAPHY_PROPERTIES]: requiredTypographyProperties,
   [A11Y_MIN_CONTRAST]: a11yMinContrast,
   [A11Y_MIN_FONT_SIZE]: a11yMinFontSize,
@@ -100,5 +100,4 @@ export const RECOMMENDED_CONFIG: Record<string, LintRuleLonghand> = {
   [VALID_GRADIENT]: ['error', {}],
   [VALID_TYPOGRAPHY]: ['error', {}],
   [CONSISTENT_NAMING]: ['warn', { format: 'kebab-case' }],
-  [NO_TYPE_ON_ALIAS]: ['warn', {}],
 };

package/src/lint/plugin-core/lib/docs.ts

@@ -1,3 +1,3 @@
 export function docsLink(ruleName: string): string {
-  return `https://terrazzo.app/docs/cli/lint#${ruleName.replaceAll('/', '')}`;
+  return `https://terrazzo.app/docs/linting#${ruleName.replaceAll('/', '')}`;
 }

package/src/lint/plugin-core/rules/{no-type-on-alias.ts → required-type.ts}

@@ -1,25 +1,24 @@
-import { isAlias } from '@terrazzo/token-tools';
 import type { LintRule } from '../../../types.js';
 import { docsLink } from '../lib/docs.js';
 
-export const NO_TYPE_ON_ALIAS = 'core/no-type-on-alias';
+export const REQUIRED_TYPE = 'core/required-type';
 
 export const ERROR = 'ERROR';
 
 const rule: LintRule<typeof ERROR> = {
   meta: {
     messages: {
-      [ERROR]: 'Remove $type from aliased value.',
+      [ERROR]: 'Token missing $type.',
     },
     docs: {
-      description: 'If a $value is aliased it already has a $type defined.',
-      url: docsLink(NO_TYPE_ON_ALIAS),
+      description: 'Requiring every token to have $type, even aliases, simplifies computation.',
+      url: docsLink(REQUIRED_TYPE),
     },
   },
   defaultOptions: {},
   create({ tokens, report }) {
     for (const t of Object.values(tokens)) {
-      if (isAlias(t.originalValue!.$value as any) && t.originalValue?.$type) {
+      if (!t.originalValue?.$type) {
         report({ messageId: ERROR, node: t.source.node, filename: t.source.filename });
       }
     }

package/src/parse/index.ts

@@ -1,12 +1,15 @@
+import type fsType from 'node:fs/promises';
 import { pluralize, type TokenNormalizedSet } from '@terrazzo/token-tools';
 import lintRunner from '../lint/index.js';
 import Logger from '../logger.js';
-import type { ConfigInit, InputSource, ParseOptions } from '../types.js';
+import { loadResolver } from '../resolver/load.js';
+import type { ConfigInit, InputSource, ParseOptions, Resolver } from '../types.js';
 import { loadSources } from './load.js';
 
 export interface ParseResult {
   tokens: TokenNormalizedSet;
   sources: InputSource[];
+  resolver?: Resolver | undefined;
 }
 
 /** Parse */
@@ -14,6 +17,7 @@ export default async function parse(
   _input: Omit<InputSource, 'document'> | Omit<InputSource, 'document'>[],
   {
     logger = new Logger(),
+    req = defaultReq,
     skipLint = false,
     config = {} as ConfigInit,
     continueOnError = false,
@@ -24,8 +28,20 @@ export default async function parse(
   const inputs = Array.isArray(_input) ? _input : [_input];
 
   const totalStart = performance.now();
+
+  // 1. Resolver
+  const resolver = await loadResolver(inputs, { logger, req, yamlToMomoa });
+
+  // 2. No resolver (tokens)
   const initStart = performance.now();
-  const { tokens, sources } = await loadSources(inputs, { logger, config, continueOnError, yamlToMomoa, transform });
+  const { tokens, sources } = await loadSources(inputs, {
+    req,
+    logger,
+    config,
+    continueOnError,
+    yamlToMomoa,
+    transform,
+  });
   logger.debug({
     message: 'Loaded tokens',
     group: 'parser',
@@ -64,5 +80,23 @@ export default async function parse(
   return {
     tokens,
     sources,
+    resolver,
   };
 }
+
+let fs: typeof fsType | undefined;
+
+/** Fallback req */
+async function defaultReq(src: URL, _origin: URL) {
+  if (src.protocol === 'file:') {
+    if (!fs) {
+      fs = await import('node:fs/promises');
+    }
+    return await fs.readFile(src, 'utf8');
+  }
+  const res = await fetch(src);
+  if (!res.ok) {
+    throw new Error(`${src} responded with ${res.status}\n${await res.text()}`);
+  }
+  return await res.text();
+}

package/src/parse/load.ts

@@ -51,14 +51,20 @@ export interface IntermediaryToken {
 }
 
 export interface LoadOptions extends Pick<ParseOptions, 'config' | 'continueOnError' | 'yamlToMomoa' | 'transform'> {
+  req: NonNullable<ParseOptions['req']>;
   logger: Logger;
 }
 
+export interface LoadSourcesResult {
+  tokens: TokenNormalizedSet;
+  sources: InputSource[];
+}
+
 /** Load from multiple entries, while resolving remote files */
 export async function loadSources(
   inputs: Omit<InputSource, 'document'>[],
-  { config, logger, continueOnError, yamlToMomoa, transform }: LoadOptions,
-): Promise<{ tokens: TokenNormalizedSet; sources: InputSource[] }> {
+  { config, logger, req, continueOnError, yamlToMomoa, transform }: LoadOptions,
+): Promise<LoadSourcesResult> {
   const entry = { group: 'parser' as const, label: 'init' };
 
   // 1. Bundle root documents together
@@ -78,6 +84,7 @@ export async function loadSources(
 
   try {
     const result = await bundle(sources, {
+      req,
       parse: transform ? transformer(transform) : undefined,
       yamlToMomoa,
     });
@@ -200,10 +207,7 @@ export async function loadSources(
     group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
   }
 
-  return {
-    tokens: tokensSorted,
-    sources,
-  };
+  return { tokens: tokensSorted, sources };
 }
 
 function transformer(transform: TransformVisitors): BundleOptions['parse'] {

package/src/parse/token.ts

@@ -437,7 +437,11 @@ export function resolveAliases(
 
     for (const mode of Object.keys(token.mode)) {
       function resolveInner(alias: string, refChain: string[]): string {
-        const nextRef = aliasToRef(alias, mode)?.$ref!;
+        const nextRef = aliasToRef(alias, mode)?.$ref;
+        if (!nextRef) {
+          logger.error({ ...aliasEntry, message: `Internal error resolving ${JSON.stringify(refChain)}` });
+          throw new Error('Internal error');
+        }
         if (refChain.includes(nextRef)) {
           logger.error({ ...aliasEntry, message: 'Circular alias detected.' });
         }
@@ -464,6 +468,7 @@ export function resolveAliases(
                 continue;
               }
               value[i] = traverseAndResolve(value[i], {
+                // biome-ignore lint/suspicious/noNonNullAssertedOptionalChain: we checked for this earlier
                 node: (node as momoa.ArrayNode).elements?.[i]?.value!,
                 // special case: cubicBezier
                 expectedTypes: expectedTypes?.includes('cubicBezier') ? ['number'] : expectedTypes,

package/src/resolver/index.ts

@@ -0,0 +1,7 @@
+import { loadResolver } from './load.js';
+
+export * from './load.js';
+export * from './normalize.js';
+export * from './validate.js';
+
+export default loadResolver;

package/src/resolver/load.ts

@@ -0,0 +1,161 @@
+import type { DocumentNode } from '@humanwhocodes/momoa';
+import { maybeRawJSON } from '@terrazzo/json-schema-tools';
+import type { TokenNormalizedSet } from '@terrazzo/token-tools';
+import eq from 'fast-deep-equal';
+import { merge } from 'merge-anything';
+import type yamlToMomoa from 'yaml-to-momoa';
+import { toMomoa } from '../lib/momoa.js';
+import type Logger from '../logger.js';
+import type { InputSource, Resolver, ResolverSourceNormalized } from '../types.js';
+import { normalizeResolver } from './normalize.js';
+import { isLikelyResolver, validateResolver } from './validate.js';
+
+export interface LoadResolverOptions {
+  logger: Logger;
+  req: (url: URL, origin: URL) => Promise<string>;
+  yamlToMomoa?: typeof yamlToMomoa;
+}
+
+/** Quick-parse input sources and find a resolver */
+export async function loadResolver(
+  inputs: Omit<InputSource, 'document'>[],
+  { logger, req, yamlToMomoa }: LoadResolverOptions,
+): Promise<Resolver | undefined> {
+  let resolverDoc: DocumentNode | undefined;
+  const entry = {
+    group: 'parser',
+    label: 'init',
+  } as const;
+
+  for (const input of inputs) {
+    let document: DocumentNode | undefined;
+    if (typeof input.src === 'string') {
+      if (maybeRawJSON(input.src)) {
+        document = toMomoa(input.src);
+      } else if (yamlToMomoa) {
+        document = yamlToMomoa(input.src);
+      } else {
+        logger.error({
+          ...entry,
+          message: `Install yaml-to-momoa package to parse YAML, and pass in as option, e.g.:
+
+  import { bundle } from '@terrazzo/json-schema-tools';
+  import yamlToMomoa from 'yaml-to-momoa';
+
+  bundle(yamlString, { yamlToMomoa });`,
+        });
+      }
+    } else if (input.src && typeof input.src === 'object') {
+      document = toMomoa(JSON.stringify(input.src, undefined, 2));
+    } else {
+      logger.error({ ...entry, message: `Could not parse ${input.filename}. Is this valid JSON or YAML?` });
+    }
+    if (!document || !isLikelyResolver(document)) {
+      continue;
+    }
+    if (inputs.length > 1) {
+      logger.error({ ...entry, message: `Resolver must be the only input, found ${inputs.length} sources.` });
+    }
+    resolverDoc = document;
+    break;
+  }
+
+  if (resolverDoc) {
+    validateResolver(resolverDoc, { logger, src: inputs[0]!.src });
+    const normalized = await normalizeResolver(resolverDoc, {
+      filename: inputs[0]!.filename!,
+      logger,
+      req,
+      src: inputs[0]!.src,
+      yamlToMomoa,
+    });
+    return createResolver(normalized, { logger });
+  }
+}
+
+export interface CreateResolverOptions {
+  logger: Logger;
+}
+
+/** Create an interface to resolve permutations */
+export function createResolver(resolverSource: ResolverSourceNormalized, { logger }: CreateResolverOptions): Resolver {
+  const inputDefaults: Record<string, string> = {};
+  const modifierPermutations: [string, string[]][] = []; // figure out modifiers
+  for (const [name, m] of Object.entries(resolverSource.modifiers ?? {})) {
+    if (typeof m.default === 'string') {
+      inputDefaults[name] = m.default;
+    }
+    modifierPermutations.push([name, Object.keys(m.contexts)]);
+  }
+  for (const m of resolverSource.resolutionOrder) {
+    if (!('type' in m) || m.type !== 'modifier') {
+      continue;
+    }
+    if (typeof m.default === 'string') {
+      inputDefaults[m.name] = m.default;
+    }
+    modifierPermutations.push([m.name, Object.keys(m.contexts)]);
+  }
+
+  const permutations = calculatePermutations(modifierPermutations);
+
+  return {
+    apply(inputRaw): TokenNormalizedSet {
+      let tokens: TokenNormalizedSet = {};
+      const input = { ...inputDefaults, ...inputRaw };
+      for (const item of resolverSource.resolutionOrder) {
+        switch (item.type) {
+          case 'set': {
+            for (const s of item.sources) {
+              tokens = merge(tokens, s) as TokenNormalizedSet;
+            }
+            break;
+          }
+          case 'modifier': {
+            const context = input[item.name]!;
+            const sources = item.contexts[context];
+            if (!sources) {
+              logger.error({
+                group: 'parser',
+                label: 'resolver',
+                message: `Modifier ${item.name} has no context ${JSON.stringify(context)}.`,
+              });
+            }
+            for (const s of sources ?? []) {
+              tokens = merge(tokens, s) as TokenNormalizedSet;
+            }
+            break;
+          }
+        }
+      }
+      return tokens;
+    },
+    source: resolverSource,
+    permutations,
+    isValidInput(inputRaw: Record<string, string>) {
+      if (!inputRaw || typeof inputRaw !== 'object') {
+        logger.error({ group: 'parser', label: 'resolver', message: `Invalid input: ${JSON.stringify(inputRaw)}.` });
+      }
+      const input = { ...inputDefaults, ...inputRaw };
+      return permutations.findIndex((p) => eq(input, p)) !== -1;
+    },
+  };
+}
+
+/** Calculate all permutations */
+export function calculatePermutations(options: [string, string[]][]) {
+  const permutationCount = [1];
+  for (const [_name, contexts] of options) {
+    permutationCount.push(contexts.length * (permutationCount.at(-1) || 1));
+  }
+  const permutations: Record<string, string>[] = [];
+  for (let i = 0; i < permutationCount.at(-1)!; i++) {
+    const input: Record<string, string> = {};
+    for (let j = 0; j < options.length; j++) {
+      const [name, contexts] = options[j]!;
+      input[name] = contexts[Math.floor(i / permutationCount[j]!) % contexts.length]!;
+    }
+    permutations.push(input);
+  }
+  return permutations.length ? permutations : [{}];
+}

package/src/resolver/normalize.ts

@@ -0,0 +1,99 @@
+import { type ArrayNode, type DocumentNode, evaluate, type ObjectNode, type StringNode } from '@humanwhocodes/momoa';
+import { bundle, getObjMember, getObjMembers, parseRef } from '@terrazzo/json-schema-tools';
+import type yamlToMomoa from 'yaml-to-momoa';
+import type Logger from '../logger.js';
+import type {
+  ResolverModifierInline,
+  ResolverModifierNormalized,
+  ResolverSetInline,
+  ResolverSetNormalized,
+  ResolverSourceNormalized,
+} from '../types.js';
+import { validateModifier, validateSet } from './validate.js';
+
+export interface NormalizeResolverOptions {
+  logger: Logger;
+  yamlToMomoa?: typeof yamlToMomoa;
+  filename: URL;
+  req: (url: URL, origin: URL) => Promise<string>;
+  src?: any;
+}
+
+/** Normalize resolver (assuming it’s been validated) */
+export async function normalizeResolver(
+  node: DocumentNode,
+  { filename, req, src, logger, yamlToMomoa }: NormalizeResolverOptions,
+): Promise<ResolverSourceNormalized> {
+  const resolverSource = evaluate(node) as unknown as ResolverSourceNormalized;
+  const resolutionOrder = getObjMember(node.body as ObjectNode, 'resolutionOrder') as ArrayNode;
+
+  return {
+    name: resolverSource.name,
+    version: resolverSource.version,
+    description: resolverSource.description,
+    sets: resolverSource.sets,
+    modifiers: resolverSource.modifiers,
+    resolutionOrder: await Promise.all(
+      resolutionOrder.elements.map(async (element, i) => {
+        const layer = element.value as ObjectNode;
+        const members = getObjMembers(layer);
+
+        // If this is an inline set or modifier it’s already been validated; we only need
+        // to resolve & validate $refs here which haven’t yet been parsed
+        let item = layer as unknown as ResolverSetInline | ResolverModifierInline;
+
+        // 1. $ref
+        if (members.$ref) {
+          const entry = { group: 'parser', label: 'init', node: members.$ref, src } as const;
+          const { url, subpath } = parseRef((members.$ref as unknown as StringNode).value);
+          if (url === '.') {
+            // 1a. local $ref: pull from local document
+            if (!subpath?.[0]) {
+              logger.error({ ...entry, message: '$ref can’t refer to the root document.' });
+            } else if (subpath[0] !== 'sets' && subpath[0] !== 'modifiers') {
+              // Note: technically we could allow $defs, but that’s just unnecessary shenanigans here.
+              logger.error({
+                ...entry,
+                message: 'Local $ref in resolutionOrder must point to either #/sets/[set] or #/modifiers/[modifiers].',
+              });
+            } else {
+              const resolvedItem = resolverSource[subpath[0] as 'sets' | 'modifiers']?.[subpath[1]!];
+              if (!resolvedItem) {
+                logger.error({ ...entry, message: 'Invalid $ref' });
+              } else {
+                item = {
+                  type: subpath[0] === 'sets' ? 'set' : 'modifier',
+                  ...(resolvedItem as any), // Note: as long as this exists, this has already been validated to be correct
+                };
+              }
+            }
+          } else {
+            // 1b. remote $ref: load and validate
+            const result = await bundle(
+              [{ filename: new URL(url, filename), src: resolverSource.resolutionOrder[i]! }],
+              {
+                req,
+                yamlToMomoa,
+              },
+            );
+            if (result.document.body.type === 'Object') {
+              const type = getObjMember(result.document.body, 'type');
+              if (type?.type === 'String' && type.value === 'set') {
+                validateSet(result.document.body as ObjectNode, true, src);
+                item = evaluate(result.document.body) as unknown as ResolverSetInline;
+              } else if (type?.type === 'String' && type.value === 'modifier') {
+                validateModifier(result.document.body as ObjectNode, true, src);
+                item = evaluate(result.document.body) as unknown as ResolverModifierInline;
+              }
+            }
+            logger.error({ ...entry, message: '$ref did not resolve to a valid Set or Modifier.' });
+          }
+        }
+
+        // 2. resolve inline sources & contexts
+        const finalResult = await bundle([{ filename, src: item }], { req, yamlToMomoa });
+        return evaluate(finalResult.document.body) as unknown as ResolverSetNormalized | ResolverModifierNormalized;
+      }),
+    ),
+  };
+}