@terrazzo/parser 2.0.0-alpha.3 → 2.0.0-alpha.4

package/src/resolver/load.ts

@@ -1,16 +1,18 @@
-import type { DocumentNode } from '@humanwhocodes/momoa';
-import { maybeRawJSON } from '@terrazzo/json-schema-tools';
+import * as momoa from '@humanwhocodes/momoa';
+import { type InputSource, type InputSourceWithDocument, maybeRawJSON } from '@terrazzo/json-schema-tools';
 import type { TokenNormalizedSet } from '@terrazzo/token-tools';
-import eq from 'fast-deep-equal';
 import { merge } from 'merge-anything';
 import type yamlToMomoa from 'yaml-to-momoa';
 import { toMomoa } from '../lib/momoa.js';
+import { makeInputKey } from '../lib/resolver-utils.js';
 import type Logger from '../logger.js';
-import type { InputSource, Resolver, ResolverSourceNormalized } from '../types.js';
+import { processTokens } from '../parse/process.js';
+import type { ConfigInit, Resolver, ResolverSourceNormalized } from '../types.js';
 import { normalizeResolver } from './normalize.js';
 import { isLikelyResolver, validateResolver } from './validate.js';
 
 export interface LoadResolverOptions {
+  config: ConfigInit;
   logger: Logger;
   req: (url: URL, origin: URL) => Promise<string>;
   yamlToMomoa?: typeof yamlToMomoa;
@@ -18,17 +20,18 @@ export interface LoadResolverOptions {
 
 /** Quick-parse input sources and find a resolver */
 export async function loadResolver(
-  inputs: Omit<InputSource, 'document'>[],
-  { logger, req, yamlToMomoa }: LoadResolverOptions,
-): Promise<Resolver | undefined> {
-  let resolverDoc: DocumentNode | undefined;
+  inputs: InputSource[],
+  { config, logger, req, yamlToMomoa }: LoadResolverOptions,
+): Promise<{ resolver: Resolver | undefined; tokens: TokenNormalizedSet; sources: InputSourceWithDocument[] }> {
+  let resolverDoc: momoa.DocumentNode | undefined;
+  let tokens: TokenNormalizedSet = {};
   const entry = {
     group: 'parser',
     label: 'init',
   } as const;
 
   for (const input of inputs) {
-    let document: DocumentNode | undefined;
+    let document: momoa.DocumentNode | undefined;
     if (typeof input.src === 'string') {
       if (maybeRawJSON(input.src)) {
         document = toMomoa(input.src);
@@ -60,6 +63,7 @@ export async function loadResolver(
     break;
   }
 
+  let resolver: Resolver | undefined;
   if (resolverDoc) {
     validateResolver(resolverDoc, { logger, src: inputs[0]!.src });
     const normalized = await normalizeResolver(resolverDoc, {
@@ -69,45 +73,67 @@ export async function loadResolver(
       src: inputs[0]!.src,
       yamlToMomoa,
     });
-    return createResolver(normalized, { logger });
+    resolver = createResolver(normalized, { config, logger, sources: [{ ...inputs[0]!, document: resolverDoc }] });
+
+    // If a resolver is present, load a single permutation to get a base token set.
+    const firstInput: Record<string, string> = {};
+    for (const m of resolver.source.resolutionOrder) {
+      if (m.type !== 'modifier') {
+        continue;
+      }
+      firstInput[m.name] = typeof m.default === 'string' ? m.default : Object.keys(m.contexts)[0]!;
+    }
+    tokens = resolver.apply(firstInput);
   }
+
+  return {
+    resolver,
+    tokens,
+    sources: [{ ...inputs[0]!, document: resolverDoc! }],
+  };
 }
 
 export interface CreateResolverOptions {
+  config: ConfigInit;
   logger: Logger;
+  sources: InputSourceWithDocument[];
 }
 
 /** Create an interface to resolve permutations */
-export function createResolver(resolverSource: ResolverSourceNormalized, { logger }: CreateResolverOptions): Resolver {
+export function createResolver(
+  resolverSource: ResolverSourceNormalized,
+  { config, logger, sources }: CreateResolverOptions,
+): Resolver {
   const inputDefaults: Record<string, string> = {};
-  const modifierPermutations: [string, string[]][] = []; // figure out modifiers
-  for (const [name, m] of Object.entries(resolverSource.modifiers ?? {})) {
-    if (typeof m.default === 'string') {
-      inputDefaults[name] = m.default;
-    }
-    modifierPermutations.push([name, Object.keys(m.contexts)]);
-  }
+  const validContexts: Record<string, string[]> = {};
+  const allPermutations: Record<string, string>[] = [];
+
+  const resolverCache: Record<string, any> = {};
+
   for (const m of resolverSource.resolutionOrder) {
-    if (!('type' in m) || m.type !== 'modifier') {
-      continue;
-    }
-    if (typeof m.default === 'string') {
-      inputDefaults[m.name] = m.default;
+    if (m.type === 'modifier') {
+      if (typeof m.default === 'string') {
+        inputDefaults[m.name] = m.default!;
+      }
+      validContexts[m.name] = Object.keys(m.contexts);
     }
-    modifierPermutations.push([m.name, Object.keys(m.contexts)]);
   }
 
-  const permutations = calculatePermutations(modifierPermutations);
-
   return {
     apply(inputRaw): TokenNormalizedSet {
-      let tokens: TokenNormalizedSet = {};
+      let tokensRaw: TokenNormalizedSet = {};
       const input = { ...inputDefaults, ...inputRaw };
+      const inputKey = makeInputKey(input);
+
+      if (resolverCache[inputKey]) {
+        return resolverCache[inputKey];
+      }
+
       for (const item of resolverSource.resolutionOrder) {
         switch (item.type) {
           case 'set': {
             for (const s of item.sources) {
-              tokens = merge(tokens, s) as TokenNormalizedSet;
+              tokensRaw = merge(tokensRaw, s) as TokenNormalizedSet;
             }
             break;
           }
@@ -122,22 +148,51 @@ export function createResolver(resolverSource: ResolverSourceNormalized, { logge
               });
             }
             for (const s of sources ?? []) {
-              tokens = merge(tokens, s) as TokenNormalizedSet;
+              tokensRaw = merge(tokensRaw, s) as TokenNormalizedSet;
             }
             break;
           }
         }
       }
+
+      const src = JSON.stringify(tokensRaw, undefined, 2);
+      const rootSource = { filename: resolverSource._source.filename!, document: toMomoa(src), src };
+      const tokens = processTokens(rootSource, {
+        config,
+        logger,
+        sourceByFilename: {},
+        refMap: {},
+        sources,
+      });
+      resolverCache[inputKey] = tokens;
       return tokens;
     },
     source: resolverSource,
-    permutations,
-    isValidInput(inputRaw: Record<string, string>) {
-      if (!inputRaw || typeof inputRaw !== 'object') {
-        logger.error({ group: 'parser', label: 'resolver', message: `Invalid input: ${JSON.stringify(inputRaw)}.` });
+    listPermutations() {
+      // only do work on first call, then cache subsequent work. this could be thousands of possible values!
+      if (!allPermutations.length) {
+        allPermutations.push(...calculatePermutations(Object.entries(validContexts)));
       }
-      const input = { ...inputDefaults, ...inputRaw };
-      return permutations.findIndex((p) => eq(input, p)) !== -1;
+      return allPermutations;
+    },
+    isValidInput(input: Record<string, string>) {
+      if (!input || typeof input !== 'object') {
+        logger.error({ group: 'parser', label: 'resolver', message: `Invalid input: ${JSON.stringify(input)}.` });
+      }
+      if (!Object.keys(input).every((k) => k in validContexts)) {
+        return false; // 1. invalid if unknown modifier name
+      }
+      for (const [name, contexts] of Object.entries(validContexts)) {
+        // Note: empty strings are valid! Don’t check for truthiness.
+        if (name in input) {
+          if (!contexts.includes(input[name]!)) {
+            return false; // 2. invalid if unknown context
+          }
+        } else if (!(name in inputDefaults)) {
+          return false; // 3. invalid if omitted, and no default
+        }
+      }
+      return true;
     },
   };
 }

package/src/resolver/normalize.ts

@@ -1,4 +1,4 @@
-import { type ArrayNode, type DocumentNode, evaluate, type ObjectNode, type StringNode } from '@humanwhocodes/momoa';
+import * as momoa from '@humanwhocodes/momoa';
 import { bundle, getObjMember, getObjMembers, parseRef } from '@terrazzo/json-schema-tools';
 import type yamlToMomoa from 'yaml-to-momoa';
 import type Logger from '../logger.js';
@@ -21,11 +21,11 @@ export interface NormalizeResolverOptions {
 
 /** Normalize resolver (assuming it’s been validated) */
 export async function normalizeResolver(
-  node: DocumentNode,
+  node: momoa.DocumentNode,
   { filename, req, src, logger, yamlToMomoa }: NormalizeResolverOptions,
 ): Promise<ResolverSourceNormalized> {
-  const resolverSource = evaluate(node) as unknown as ResolverSourceNormalized;
-  const resolutionOrder = getObjMember(node.body as ObjectNode, 'resolutionOrder') as ArrayNode;
+  const resolverSource = momoa.evaluate(node) as unknown as ResolverSourceNormalized;
+  const resolutionOrder = getObjMember(node.body as momoa.ObjectNode, 'resolutionOrder') as momoa.ArrayNode;
 
   return {
     name: resolverSource.name,
@@ -35,7 +35,7 @@ export async function normalizeResolver(
     modifiers: resolverSource.modifiers,
     resolutionOrder: await Promise.all(
       resolutionOrder.elements.map(async (element, i) => {
-        const layer = element.value as ObjectNode;
+        const layer = element.value as momoa.ObjectNode;
         const members = getObjMembers(layer);
 
         // If this is an inline set or modifier it’s already been validated; we only need
@@ -45,7 +45,7 @@ export async function normalizeResolver(
         // 1. $ref
         if (members.$ref) {
           const entry = { group: 'parser', label: 'init', node: members.$ref, src } as const;
-          const { url, subpath } = parseRef((members.$ref as unknown as StringNode).value);
+          const { url, subpath } = parseRef((members.$ref as unknown as momoa.StringNode).value);
           if (url === '.') {
             // 1a. local $ref: pull from local document
             if (!subpath?.[0]) {
@@ -63,6 +63,7 @@ export async function normalizeResolver(
               } else {
                 item = {
                   type: subpath[0] === 'sets' ? 'set' : 'modifier',
+                  name: subpath[1],
                   ...(resolvedItem as any), // Note: as long as this exists, this has already been validated to be correct
                 };
               }
@@ -79,11 +80,11 @@ export async function normalizeResolver(
             if (result.document.body.type === 'Object') {
               const type = getObjMember(result.document.body, 'type');
               if (type?.type === 'String' && type.value === 'set') {
-                validateSet(result.document.body as ObjectNode, true, src);
-                item = evaluate(result.document.body) as unknown as ResolverSetInline;
+                validateSet(result.document.body as momoa.ObjectNode, true, src);
+                item = momoa.evaluate(result.document.body) as unknown as ResolverSetInline;
               } else if (type?.type === 'String' && type.value === 'modifier') {
-                validateModifier(result.document.body as ObjectNode, true, src);
-                item = evaluate(result.document.body) as unknown as ResolverModifierInline;
+                validateModifier(result.document.body as momoa.ObjectNode, true, src);
+                item = momoa.evaluate(result.document.body) as unknown as ResolverModifierInline;
               }
             }
             logger.error({ ...entry, message: '$ref did not resolve to a valid Set or Modifier.' });
@@ -92,8 +93,14 @@ export async function normalizeResolver(
 
         // 2. resolve inline sources & contexts
         const finalResult = await bundle([{ filename, src: item }], { req, yamlToMomoa });
-        return evaluate(finalResult.document.body) as unknown as ResolverSetNormalized | ResolverModifierNormalized;
+        return momoa.evaluate(finalResult.document.body) as unknown as
+          | ResolverSetNormalized
+          | ResolverModifierNormalized;
       }),
     ),
+    _source: {
+      filename,
+      node,
+    },
   };
 }

package/src/resolver/validate.ts

@@ -1,4 +1,4 @@
-import type { DocumentNode, ObjectNode } from '@humanwhocodes/momoa';
+import type * as momoa from '@humanwhocodes/momoa';
 import { getObjMember, getObjMembers } from '@terrazzo/json-schema-tools';
 import type { LogEntry, default as Logger } from '../logger.js';
 
@@ -10,7 +10,7 @@ import type { LogEntry, default as Logger } from '../logger.js';
  * guesswork here, but we try and find a reasonable edge case where we sniff out
  * invalid DTCG syntax that a resolver doc would have.
  */
-export function isLikelyResolver(doc: DocumentNode): boolean {
+export function isLikelyResolver(doc: momoa.DocumentNode): boolean {
   if (doc.body.type !== 'Object') {
     return false;
   }
@@ -79,7 +79,7 @@ const MESSAGE_EXPECTED = {
  * Validate a resolver document.
  * There’s a ton of boilerplate here, only to surface detailed code frames. Is there a better abstraction?
  */
-export function validateResolver(node: DocumentNode, { logger, src }: ValidateResolverOptions) {
+export function validateResolver(node: momoa.DocumentNode, { logger, src }: ValidateResolverOptions) {
   const entry = { group: 'parser', label: 'resolver', src } as const;
   if (node.body.type !== 'Object') {
     logger.error({ ...entry, message: MESSAGE_EXPECTED.OBJECT, node });
@@ -89,7 +89,7 @@ export function validateResolver(node: DocumentNode, { logger, src }: ValidateRe
   let hasVersion = false;
   let hasResolutionOrder = false;
 
-  for (const member of (node.body as ObjectNode).members) {
+  for (const member of (node.body as momoa.ObjectNode).members) {
     if (member.name.type !== 'String') {
       continue; // IDK, don’t ask
     }
@@ -202,7 +202,7 @@ export function validateResolver(node: DocumentNode, { logger, src }: ValidateRe
   }
 }
 
-export function validateSet(node: ObjectNode, isInline = false, { src }: ValidateResolverOptions): LogEntry[] {
+export function validateSet(node: momoa.ObjectNode, isInline = false, { src }: ValidateResolverOptions): LogEntry[] {
   const entry = { group: 'parser', label: 'resolver', src } as const;
   const errors: LogEntry[] = [];
   let hasName = !isInline;
@@ -277,7 +277,11 @@ export function validateSet(node: ObjectNode, isInline = false, { src }: Validat
   return errors;
 }
 
-export function validateModifier(node: ObjectNode, isInline = false, { src }: ValidateResolverOptions): LogEntry[] {
+export function validateModifier(
+  node: momoa.ObjectNode,
+  isInline = false,
+  { src }: ValidateResolverOptions,
+): LogEntry[] {
   const errors: LogEntry[] = [];
   const entry = { group: 'parser', label: 'resolver', src } as const;
   let hasName = !isInline;

package/src/types.ts

@@ -1,4 +1,5 @@
 import type * as momoa from '@humanwhocodes/momoa';
+import type { InputSourceWithDocument } from '@terrazzo/json-schema-tools';
 import type {
   Group,
   TokenNormalized,
@@ -30,7 +31,7 @@ export interface BuildHookOptions {
   /** Query transformed values */
   getTransforms(params: TransformParams): TokenTransformed[];
   /** Momoa documents */
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
   outputFile: (
     /** Filename to output (relative to outDir) */
     filename: string,
@@ -51,7 +52,7 @@ export interface BuildEndHookOptions {
   /** Query transformed values */
   getTransforms(params: TransformParams): TokenTransformed[];
   /** Momoa documents */
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
   /** Final files to be written */
   outputFiles: OutputFileExpanded[];
 }
@@ -147,12 +148,6 @@ export interface ConfigOptions {
   cwd: URL;
 }
 
-export interface InputSource {
-  filename?: URL;
-  src: any;
-  document: momoa.DocumentNode;
-}
-
 export interface LintNotice {
   /** Lint message shown to the user */
   message: string;
@@ -226,7 +221,7 @@ export interface LintRuleContext<MessageIds extends string, LintRuleOptions exte
    * All source files present in this run. To find the original source, match a
    * token’s `source.loc` filename to one of the source’s `filename`s.
    */
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
   /** Source file location. */
   filename?: URL;
   /** ID:Token map of all tokens. */
@@ -309,7 +304,7 @@ export interface ParseOptions {
    */
   transform?: TransformVisitors;
   /** (internal cache; do not use) */
-  _sources?: Record<string, InputSource>;
+  _sources?: Record<string, InputSourceWithDocument>;
 }
 
 export interface Plugin {
@@ -342,7 +337,7 @@ export interface Resolver<
   /** Supply values to modifiers to produce a final tokens set */
   apply: (input: Partial<Input>) => TokenNormalizedSet;
   /** List all possible valid input combinations. Ignores default values, as they would duplicate some other permutations. */
-  permutations: Input[];
+  listPermutations: () => Input[];
   /** The original resolver document, simplified */
   source: ResolverSourceNormalized;
   /** Helper function for permutations—see if a particular input is valid. Automatically applies default values. */
@@ -378,6 +373,10 @@ export interface ResolverSourceNormalized {
    * pass over the resolutionOrder array is needed.
    */
   resolutionOrder: (ResolverSetNormalized | ResolverModifierNormalized)[];
+  _source: {
+    filename?: URL;
+    node: momoa.DocumentNode;
+  };
 }
 
 export interface ResolverModifier<Context extends string = string> {
@@ -454,5 +453,5 @@ export interface TransformHookOptions {
     },
   ): void;
   /** Momoa documents */
-  sources: InputSource[];
+  sources: InputSourceWithDocument[];
 }