@terrazzo/parser 2.0.0-alpha.2 → 2.0.0-alpha.4

package/src/parse/token.ts

@@ -1,5 +1,5 @@
 import * as momoa from '@humanwhocodes/momoa';
-import { getObjMember, parseRef, type RefMap } from '@terrazzo/json-schema-tools';
+import { getObjMember, type InputSourceWithDocument, parseRef, type RefMap } from '@terrazzo/json-schema-tools';
 import {
   type GroupNormalized,
   isAlias,
@@ -9,7 +9,7 @@ import {
 } from '@terrazzo/token-tools';
 import wcmatch from 'wildcard-match';
 import type { default as Logger } from '../logger.js';
-import type { Config, InputSource, ReferenceObject } from '../types.js';
+import type { Config, ReferenceObject } from '../types.js';
 
 /** Convert valid DTCG alias to $ref */
 export function aliasToRef(alias: string, mode?: string): ReferenceObject | undefined {
@@ -26,7 +26,7 @@ export function aliasToRef(alias: string, mode?: string): ReferenceObject | unde
 export interface TokenFromNodeOptions {
   groups: Record<string, GroupNormalized>;
   path: string[];
-  source: InputSource;
+  source: InputSourceWithDocument;
   ignore: Config['ignore'];
 }
 
@@ -51,7 +51,7 @@ export function tokenFromNode(
     group.tokens.push(id);
   }
 
-  const nodeSource = { filename: source.filename?.href, node };
+  const nodeSource = { filename: source.filename.href, node };
   const token: TokenNormalized = {
     id,
     $type: originalToken.$type || group.$type,
@@ -211,7 +211,7 @@ export function groupFromNode(
 
 export interface GraphAliasesOptions {
   tokens: TokenNormalizedSet;
-  sources: Record<string, InputSource>;
+  sources: Record<string, InputSourceWithDocument>;
   logger: Logger;
 }
 
@@ -425,7 +425,7 @@ const EXPECTED_NESTED_ALIAS: Record<string, Record<string, string[]>> = {
  */
 export function resolveAliases(
   tokens: TokenNormalizedSet,
-  { logger, refMap, sources }: { logger: Logger; refMap: RefMap; sources: Record<string, InputSource> },
+  { logger, refMap, sources }: { logger: Logger; refMap: RefMap; sources: Record<string, InputSourceWithDocument> },
 ): void {
   for (const token of Object.values(tokens)) {
     const aliasEntry = {
@@ -437,7 +437,11 @@ export function resolveAliases(
 
     for (const mode of Object.keys(token.mode)) {
       function resolveInner(alias: string, refChain: string[]): string {
-        const nextRef = aliasToRef(alias, mode)?.$ref!;
+        const nextRef = aliasToRef(alias, mode)?.$ref;
+        if (!nextRef) {
+          logger.error({ ...aliasEntry, message: `Internal error resolving ${JSON.stringify(refChain)}` });
+          throw new Error('Internal error');
+        }
         if (refChain.includes(nextRef)) {
           logger.error({ ...aliasEntry, message: 'Circular alias detected.' });
         }
@@ -464,6 +468,7 @@ export function resolveAliases(
                 continue;
               }
               value[i] = traverseAndResolve(value[i], {
+                // biome-ignore lint/suspicious/noNonNullAssertedOptionalChain: we checked for this earlier
                 node: (node as momoa.ArrayNode).elements?.[i]?.value!,
                 // special case: cubicBezier
                 expectedTypes: expectedTypes?.includes('cubicBezier') ? ['number'] : expectedTypes,

package/src/resolver/create-synthetic-resolver.ts

@@ -0,0 +1,86 @@
+import * as momoa from '@humanwhocodes/momoa';
+import type { InputSourceWithDocument } from '@terrazzo/json-schema-tools';
+import type Logger from '../logger.js';
+import type { ConfigInit, Group, Resolver, TokenNormalized, TokenNormalizedSet } from '../types.js';
+import { createResolver } from './load.js';
+import { normalizeResolver } from './normalize.js';
+
+export interface CreateSyntheticResolverOptions {
+  config: ConfigInit;
+  logger: Logger;
+  req: (url: URL, origin: URL) => Promise<string>;
+  sources: InputSourceWithDocument[];
+}
+
+/**
+ * Interop layer upgrading legacy Terrazzo modes to resolvers
+ */
+export async function createSyntheticResolver(
+  tokens: TokenNormalizedSet,
+  { config, logger, req, sources }: CreateSyntheticResolverOptions,
+): Promise<Resolver> {
+  const contexts: Record<string, any[]> = {};
+  for (const token of Object.values(tokens)) {
+    for (const [mode, value] of Object.entries(token.mode)) {
+      if (mode === '.') {
+        continue;
+      }
+      if (!(mode in contexts)) {
+        contexts[mode] = [{}];
+      }
+      addToken(contexts[mode]![0], { ...token, $value: value.$value }, { logger });
+    }
+  }
+
+  const src = JSON.stringify(
+    {
+      name: 'Terrazzo',
+      version: '2025.10',
+      resolutionOrder: [{ $ref: '#/sets/allTokens' }, { $ref: '#/modifiers/tzMode' }],
+      sets: {
+        allTokens: { sources: [simpleFlatten(tokens, { logger })] },
+      },
+      modifiers: {
+        tzMode: {
+          description: 'Automatically built from $extensions.mode',
+          contexts,
+        },
+      },
+    },
+    undefined,
+    2,
+  );
+  const normalized = await normalizeResolver(momoa.parse(src), {
+    filename: new URL('file:///virtual:resolver.json'),
+    logger,
+    req,
+    src,
+  });
+  return createResolver(normalized, { config, logger, sources });
+}
+
+/** Add a normalized token back into an arbitrary, hierarchial structure */
+function addToken(structure: any, token: TokenNormalized, { logger }: { logger: Logger }): void {
+  let node = structure;
+  const parts = token.id.split('.');
+  const localID = parts.pop()!;
+  for (const part of parts) {
+    if (!(part in node)) {
+      node[part] = {};
+    }
+    node = node[part];
+  }
+  if (localID in node) {
+    logger.error({ group: 'parser', label: 'resolver', message: `${localID} already exists!` });
+  }
+  node[localID] = { $type: token.$type, $value: token.$value };
+}
+
+/** Downconvert normalized tokens back into a simplified, hierarchial shape. This is extremely lossy, and only done to build a resolver. */
+function simpleFlatten(tokens: TokenNormalizedSet, { logger }: { logger: Logger }): Group {
+  const group: Group = {};
+  for (const token of Object.values(tokens)) {
+    addToken(group, token, { logger });
+  }
+  return group;
+}

package/src/resolver/index.ts

@@ -0,0 +1,7 @@
+import { loadResolver } from './load.js';
+
+export * from './load.js';
+export * from './normalize.js';
+export * from './validate.js';
+
+export default loadResolver;

package/src/resolver/load.ts

@@ -0,0 +1,216 @@
+import * as momoa from '@humanwhocodes/momoa';
+import { type InputSource, type InputSourceWithDocument, maybeRawJSON } from '@terrazzo/json-schema-tools';
+import type { TokenNormalizedSet } from '@terrazzo/token-tools';
+import { merge } from 'merge-anything';
+import type yamlToMomoa from 'yaml-to-momoa';
+import { toMomoa } from '../lib/momoa.js';
+import { makeInputKey } from '../lib/resolver-utils.js';
+import type Logger from '../logger.js';
+import { processTokens } from '../parse/process.js';
+import type { ConfigInit, Resolver, ResolverSourceNormalized } from '../types.js';
+import { normalizeResolver } from './normalize.js';
+import { isLikelyResolver, validateResolver } from './validate.js';
+
+export interface LoadResolverOptions {
+  config: ConfigInit;
+  logger: Logger;
+  req: (url: URL, origin: URL) => Promise<string>;
+  yamlToMomoa?: typeof yamlToMomoa;
+}
+
+/** Quick-parse input sources and find a resolver */
+export async function loadResolver(
+  inputs: InputSource[],
+  { config, logger, req, yamlToMomoa }: LoadResolverOptions,
+): Promise<{ resolver: Resolver | undefined; tokens: TokenNormalizedSet; sources: InputSourceWithDocument[] }> {
+  let resolverDoc: momoa.DocumentNode | undefined;
+  let tokens: TokenNormalizedSet = {};
+  const entry = {
+    group: 'parser',
+    label: 'init',
+  } as const;
+
+  for (const input of inputs) {
+    let document: momoa.DocumentNode | undefined;
+    if (typeof input.src === 'string') {
+      if (maybeRawJSON(input.src)) {
+        document = toMomoa(input.src);
+      } else if (yamlToMomoa) {
+        document = yamlToMomoa(input.src);
+      } else {
+        logger.error({
+          ...entry,
+          message: `Install yaml-to-momoa package to parse YAML, and pass in as option, e.g.:
+
+  import { bundle } from '@terrazzo/json-schema-tools';
+  import yamlToMomoa from 'yaml-to-momoa';
+
+  bundle(yamlString, { yamlToMomoa });`,
+        });
+      }
+    } else if (input.src && typeof input.src === 'object') {
+      document = toMomoa(JSON.stringify(input.src, undefined, 2));
+    } else {
+      logger.error({ ...entry, message: `Could not parse ${input.filename}. Is this valid JSON or YAML?` });
+    }
+    if (!document || !isLikelyResolver(document)) {
+      continue;
+    }
+    if (inputs.length > 1) {
+      logger.error({ ...entry, message: `Resolver must be the only input, found ${inputs.length} sources.` });
+    }
+    resolverDoc = document;
+    break;
+  }
+
+  let resolver: Resolver | undefined;
+  if (resolverDoc) {
+    validateResolver(resolverDoc, { logger, src: inputs[0]!.src });
+    const normalized = await normalizeResolver(resolverDoc, {
+      filename: inputs[0]!.filename!,
+      logger,
+      req,
+      src: inputs[0]!.src,
+      yamlToMomoa,
+    });
+    resolver = createResolver(normalized, { config, logger, sources: [{ ...inputs[0]!, document: resolverDoc }] });
+
+    // If a resolver is present, load a single permutation to get a base token set.
+    const firstInput: Record<string, string> = {};
+    for (const m of resolver.source.resolutionOrder) {
+      if (m.type !== 'modifier') {
+        continue;
+      }
+      firstInput[m.name] = typeof m.default === 'string' ? m.default : Object.keys(m.contexts)[0]!;
+    }
+    tokens = resolver.apply(firstInput);
+  }
+
+  return {
+    resolver,
+    tokens,
+    sources: [{ ...inputs[0]!, document: resolverDoc! }],
+  };
+}
+
+export interface CreateResolverOptions {
+  config: ConfigInit;
+  logger: Logger;
+  sources: InputSourceWithDocument[];
+}
+
+/** Create an interface to resolve permutations */
+export function createResolver(
+  resolverSource: ResolverSourceNormalized,
+  { config, logger, sources }: CreateResolverOptions,
+): Resolver {
+  const inputDefaults: Record<string, string> = {};
+  const validContexts: Record<string, string[]> = {};
+  const allPermutations: Record<string, string>[] = [];
+
+  const resolverCache: Record<string, any> = {};
+
+  for (const m of resolverSource.resolutionOrder) {
+    if (m.type === 'modifier') {
+      if (typeof m.default === 'string') {
+        inputDefaults[m.name] = m.default!;
+      }
+      validContexts[m.name] = Object.keys(m.contexts);
+    }
+  }
+
+  return {
+    apply(inputRaw): TokenNormalizedSet {
+      let tokensRaw: TokenNormalizedSet = {};
+      const input = { ...inputDefaults, ...inputRaw };
+      const inputKey = makeInputKey(input);
+
+      if (resolverCache[inputKey]) {
+        return resolverCache[inputKey];
+      }
+
+      for (const item of resolverSource.resolutionOrder) {
+        switch (item.type) {
+          case 'set': {
+            for (const s of item.sources) {
+              tokensRaw = merge(tokensRaw, s) as TokenNormalizedSet;
+            }
+            break;
+          }
+          case 'modifier': {
+            const context = input[item.name]!;
+            const sources = item.contexts[context];
+            if (!sources) {
+              logger.error({
+                group: 'parser',
+                label: 'resolver',
+                message: `Modifier ${item.name} has no context ${JSON.stringify(context)}.`,
+              });
+            }
+            for (const s of sources ?? []) {
+              tokensRaw = merge(tokensRaw, s) as TokenNormalizedSet;
+            }
+            break;
+          }
+        }
+      }
+
+      const src = JSON.stringify(tokensRaw, undefined, 2);
+      const rootSource = { filename: resolverSource._source.filename!, document: toMomoa(src), src };
+      const tokens = processTokens(rootSource, {
+        config,
+        logger,
+        sourceByFilename: {},
+        refMap: {},
+        sources,
+      });
+      resolverCache[inputKey] = tokens;
+      return tokens;
+    },
+    source: resolverSource,
+    listPermutations() {
+      // only do work on first call, then cache subsequent work. this could be thousands of possible values!
+      if (!allPermutations.length) {
+        allPermutations.push(...calculatePermutations(Object.entries(validContexts)));
+      }
+      return allPermutations;
+    },
+    isValidInput(input: Record<string, string>) {
+      if (!input || typeof input !== 'object') {
+        logger.error({ group: 'parser', label: 'resolver', message: `Invalid input: ${JSON.stringify(input)}.` });
+      }
+      if (!Object.keys(input).every((k) => k in validContexts)) {
+        return false; // 1. invalid if unknown modifier name
+      }
+      for (const [name, contexts] of Object.entries(validContexts)) {
+        // Note: empty strings are valid! Don’t check for truthiness.
+        if (name in input) {
+          if (!contexts.includes(input[name]!)) {
+            return false; // 2. invalid if unknown context
+          }
+        } else if (!(name in inputDefaults)) {
+          return false; // 3. invalid if omitted, and no default
+        }
+      }
+      return true;
+    },
+  };
+}
+
+/** Calculate all permutations */
+export function calculatePermutations(options: [string, string[]][]) {
+  const permutationCount = [1];
+  for (const [_name, contexts] of options) {
+    permutationCount.push(contexts.length * (permutationCount.at(-1) || 1));
+  }
+  const permutations: Record<string, string>[] = [];
+  for (let i = 0; i < permutationCount.at(-1)!; i++) {
+    const input: Record<string, string> = {};
+    for (let j = 0; j < options.length; j++) {
+      const [name, contexts] = options[j]!;
+      input[name] = contexts[Math.floor(i / permutationCount[j]!) % contexts.length]!;
+    }
+    permutations.push(input);
+  }
+  return permutations.length ? permutations : [{}];
+}

package/src/resolver/normalize.ts

@@ -0,0 +1,106 @@
+import * as momoa from '@humanwhocodes/momoa';
+import { bundle, getObjMember, getObjMembers, parseRef } from '@terrazzo/json-schema-tools';
+import type yamlToMomoa from 'yaml-to-momoa';
+import type Logger from '../logger.js';
+import type {
+  ResolverModifierInline,
+  ResolverModifierNormalized,
+  ResolverSetInline,
+  ResolverSetNormalized,
+  ResolverSourceNormalized,
+} from '../types.js';
+import { validateModifier, validateSet } from './validate.js';
+
+export interface NormalizeResolverOptions {
+  logger: Logger;
+  yamlToMomoa?: typeof yamlToMomoa;
+  filename: URL;
+  req: (url: URL, origin: URL) => Promise<string>;
+  src?: any;
+}
+
+/** Normalize resolver (assuming it’s been validated) */
+export async function normalizeResolver(
+  node: momoa.DocumentNode,
+  { filename, req, src, logger, yamlToMomoa }: NormalizeResolverOptions,
+): Promise<ResolverSourceNormalized> {
+  const resolverSource = momoa.evaluate(node) as unknown as ResolverSourceNormalized;
+  const resolutionOrder = getObjMember(node.body as momoa.ObjectNode, 'resolutionOrder') as momoa.ArrayNode;
+
+  return {
+    name: resolverSource.name,
+    version: resolverSource.version,
+    description: resolverSource.description,
+    sets: resolverSource.sets,
+    modifiers: resolverSource.modifiers,
+    resolutionOrder: await Promise.all(
+      resolutionOrder.elements.map(async (element, i) => {
+        const layer = element.value as momoa.ObjectNode;
+        const members = getObjMembers(layer);
+
+        // If this is an inline set or modifier it’s already been validated; we only need
+        // to resolve & validate $refs here which haven’t yet been parsed
+        let item = layer as unknown as ResolverSetInline | ResolverModifierInline;
+
+        // 1. $ref
+        if (members.$ref) {
+          const entry = { group: 'parser', label: 'init', node: members.$ref, src } as const;
+          const { url, subpath } = parseRef((members.$ref as unknown as momoa.StringNode).value);
+          if (url === '.') {
+            // 1a. local $ref: pull from local document
+            if (!subpath?.[0]) {
+              logger.error({ ...entry, message: '$ref can’t refer to the root document.' });
+            } else if (subpath[0] !== 'sets' && subpath[0] !== 'modifiers') {
+              // Note: technically we could allow $defs, but that’s just unnecessary shenanigans here.
+              logger.error({
+                ...entry,
+                message: 'Local $ref in resolutionOrder must point to either #/sets/[set] or #/modifiers/[modifiers].',
+              });
+            } else {
+              const resolvedItem = resolverSource[subpath[0] as 'sets' | 'modifiers']?.[subpath[1]!];
+              if (!resolvedItem) {
+                logger.error({ ...entry, message: 'Invalid $ref' });
+              } else {
+                item = {
+                  type: subpath[0] === 'sets' ? 'set' : 'modifier',
+                  name: subpath[1],
+                  ...(resolvedItem as any), // Note: as long as this exists, this has already been validated to be correct
+                };
+              }
+            }
+          } else {
+            // 1b. remote $ref: load and validate
+            const result = await bundle(
+              [{ filename: new URL(url, filename), src: resolverSource.resolutionOrder[i]! }],
+              {
+                req,
+                yamlToMomoa,
+              },
+            );
+            if (result.document.body.type === 'Object') {
+              const type = getObjMember(result.document.body, 'type');
+              if (type?.type === 'String' && type.value === 'set') {
+                validateSet(result.document.body as momoa.ObjectNode, true, src);
+                item = momoa.evaluate(result.document.body) as unknown as ResolverSetInline;
+              } else if (type?.type === 'String' && type.value === 'modifier') {
+                validateModifier(result.document.body as momoa.ObjectNode, true, src);
+                item = momoa.evaluate(result.document.body) as unknown as ResolverModifierInline;
+              }
+            }
+            logger.error({ ...entry, message: '$ref did not resolve to a valid Set or Modifier.' });
+          }
+        }
+
+        // 2. resolve inline sources & contexts
+        const finalResult = await bundle([{ filename, src: item }], { req, yamlToMomoa });
+        return momoa.evaluate(finalResult.document.body) as unknown as
+          | ResolverSetNormalized
+          | ResolverModifierNormalized;
+      }),
+    ),
+    _source: {
+      filename,
+      node,
+    },
+  };
+}