@terrazzo/parser 0.3.2 → 0.3.4

package/src/parse/index.ts

@@ -1,20 +1,19 @@
-import { type DocumentNode, type ObjectNode, evaluate, parse as parseJSON, print } from '@humanwhocodes/momoa';
-import { type Token, type TokenNormalized, isTokenMatch, pluralize, splitID } from '@terrazzo/token-tools';
+import type { DocumentNode, ObjectNode } from '@humanwhocodes/momoa';
+import { type Token, type TokenNormalized, pluralize, splitID } from '@terrazzo/token-tools';
 import type ytm from 'yaml-to-momoa';
 import lintRunner from '../lint/index.js';
 import Logger from '../logger.js';
 import type { ConfigInit, InputSource } from '../types.js';
 import { applyAliases } from './alias.js';
-import { getObjMembers, injectObjMembers, maybeJSONString, traverse } from './json.js';
+import { getObjMembers, toMomoa, traverse } from './json.js';
 import normalize from './normalize.js';
-import validate from './validate.js';
+import validateTokenNode from './validate.js';
 
 export * from './alias.js';
-export { default as normalize } from './normalize.js';
 export * from './normalize.js';
 export * from './json.js';
-export { default as validate } from './validate.js';
 export * from './validate.js';
+export { normalize, validateTokenNode };
 
 export interface ParseOptions {
   logger?: Logger;
@@ -31,6 +30,8 @@ export interface ParseOptions {
   continueOnError?: boolean;
   /** Provide yamlToMomoa module to parse YAML (by default, this isn’t shipped to cut down on package weight) */
   yamlToMomoa?: typeof ytm;
+  /** (internal cache; do not use) */
+  _sources?: Record<string, InputSource>;
 }
 
 export interface ParseResult {
@@ -47,53 +48,59 @@ export default async function parse(
     config = {} as ConfigInit,
     continueOnError = false,
     yamlToMomoa,
+    _sources = {},
   }: ParseOptions = {} as ParseOptions,
 ): Promise<ParseResult> {
   let tokens: Record<string, TokenNormalized> = {};
-  // note: only keeps track of sources with locations on disk; in-memory sources are discarded
-  // (it’s only for reporting line numbers, which doesn’t mean as much for dynamic sources)
-  const sources: Record<string, InputSource> = {};
 
   if (!Array.isArray(input)) {
     logger.error({ group: 'parser', label: 'init', message: 'Input must be an array of input objects.' });
   }
-  for (let i = 0; i < input.length; i++) {
-    if (!input[i] || typeof input[i] !== 'object') {
-      logger.error({ group: 'parser', label: 'init', message: `Input (${i}) must be an object.` });
-    }
-    if (!input[i]!.src || (typeof input[i]!.src !== 'string' && typeof input[i]!.src !== 'object')) {
-      logger.error({
-        group: 'parser',
-        label: 'init',
-        message: `Input (${i}) missing "src" with a JSON/YAML string, or JSON object.`,
-      });
-    }
-    if (input[i]!.filename && !(input[i]!.filename instanceof URL)) {
-      logger.error({
-        group: 'parser',
-        label: 'init',
-        message: `Input (${i}) "filename" must be a URL (remote or file URL).`,
-      });
-    }
+  await Promise.all(
+    input.map(async (src, i) => {
+      if (!src || typeof src !== 'object') {
+        logger.error({ group: 'parser', label: 'init', message: `Input (${i}) must be an object.` });
+      }
+      if (!src.src || (typeof src.src !== 'string' && typeof src.src !== 'object')) {
+        logger.error({
+          message: `Input (${i}) missing "src" with a JSON/YAML string, or JSON object.`,
+          group: 'parser',
+          label: 'init',
+        });
+      }
+      if (src.filename) {
+        if (!(src.filename instanceof URL)) {
+          logger.error({
+            message: `Input (${i}) "filename" must be a URL (remote or file URL).`,
+            group: 'parser',
+            label: 'init',
+          });
+        }
 
-    const result = await parseSingle(input[i]!.src, {
-      filename: input[i]!.filename!,
-      logger,
-      config,
-      skipLint,
-      continueOnError,
-      yamlToMomoa,
-    });
+        // if already parsed/scanned, skip
+        if (_sources[src.filename.href]) {
+          return;
+        }
+      }
 
-    tokens = Object.assign(tokens, result.tokens);
-    if (input[i]!.filename) {
-      sources[input[i]!.filename!.protocol === 'file:' ? input[i]!.filename!.href : input[i]!.filename!.href] = {
-        filename: input[i]!.filename,
-        src: result.src,
-        document: result.document,
-      };
-    }
-  }
+      const result = await parseSingle(src.src, {
+        filename: src.filename!,
+        logger,
+        config,
+        skipLint,
+        continueOnError,
+        yamlToMomoa,
+      });
+      tokens = Object.assign(tokens, result.tokens);
+      if (src.filename) {
+        _sources[src.filename.href] = {
+          filename: src.filename,
+          src: result.src,
+          document: result.document,
+        };
+      }
+    }),
+  );
 
   const totalStart = performance.now();
 
@@ -104,8 +111,8 @@ export default async function parse(
     }
     applyAliases(token, {
       tokens,
-      filename: sources[token.source.loc!]?.filename!,
-      src: sources[token.source.loc!]?.src as string,
+      filename: _sources[token.source.loc!]?.filename!,
+      src: _sources[token.source.loc!]?.src as string,
       node: token.source.node,
       logger,
     });
@@ -130,9 +137,9 @@ export default async function parse(
   // 6. resolve mode aliases
   const modesStart = performance.now();
   logger.debug({
+    message: 'Start mode resolution',
     group: 'parser',
     label: 'modes',
-    message: 'Start mode resolution',
   });
   for (const [id, token] of Object.entries(tokens)) {
     if (!Object.hasOwn(tokens, id)) {
@@ -146,7 +153,7 @@ export default async function parse(
         tokens,
         node: modeValue.source.node,
         logger,
-        src: sources[token.source.loc!]?.src as string,
+        src: _sources[token.source.loc!]?.src as string,
       });
     }
   }
@@ -158,9 +165,9 @@ export default async function parse(
   });
 
   logger.debug({
+    message: 'Finish all parser tasks',
     group: 'parser',
     label: 'core',
-    message: 'Finish all parser tasks',
     timing: performance.now() - totalStart,
   });
 
@@ -175,7 +182,7 @@ export default async function parse(
 
   return {
     tokens,
-    sources: Object.values(sources),
+    sources: Object.values(_sources),
   };
 }
 
@@ -197,185 +204,26 @@ async function parseSingle(
     continueOnError?: boolean;
     yamlToMomoa?: typeof ytm;
   },
-) {
+): Promise<{ tokens: Record<string, Token>; document: DocumentNode; src?: string }> {
   // 1. Build AST
-  let src: any;
-  if (typeof input === 'string') {
-    src = input;
-  }
   const startParsing = performance.now();
-  logger.debug({ group: 'parser', label: 'parse', message: 'Start tokens parsing' });
-  let document = {} as DocumentNode;
-  if (typeof input === 'string' && !maybeJSONString(input)) {
-    if (yamlToMomoa) {
-      try {
-        document = yamlToMomoa(input); // if string, but not JSON, attempt YAML
-      } catch (err) {
-        logger.error({ message: String(err), filename, src: input, continueOnError });
-      }
-    } else {
-      logger.error({
-        group: 'parser',
-        message: `Install \`yaml-to-momoa\` package to parse YAML, and pass in as option, e.g.:
-
-    import { parse } from '@terrazzo/parser';
-    import yamlToMomoa from 'yaml-to-momoa';
-
-    parse(yamlString, { yamlToMomoa });`,
-        continueOnError: false, // fail here; no point in continuing
-      });
-    }
-  } else {
-    document = parseJSON(
-      typeof input === 'string' ? input : JSON.stringify(input, undefined, 2), // everything else: assert it’s JSON-serializable
-      {
-        mode: 'jsonc',
-        ranges: true,
-        tokens: true,
-      },
-    );
-  }
-  if (!src) {
-    src = print(document, { indent: 2 });
-  }
+  logger.debug({ group: 'parser', label: 'parse', message: 'Start JSON parsing' });
+  const { src, document } = toMomoa(input, { filename, logger, continueOnError, yamlToMomoa });
   logger.debug({
     group: 'parser',
     label: 'parse',
-    message: 'Finish tokens parsing',
+    message: 'Finish JSON parsing',
     timing: performance.now() - startParsing,
   });
-
   const tokens: Record<string, TokenNormalized> = {};
 
-  // 2. Walk AST once to validate tokens
-  const startValidation = performance.now();
-  logger.debug({ group: 'parser', label: 'validate', message: 'Start tokens validation' });
+  // 2. Walk AST to validate tokens
+  const startValidate = performance.now();
   const $typeInheritance: Record<string, Token['$type']> = {};
+  logger.debug({ message: 'Start token validation', group: 'parser', label: 'validate' });
   traverse(document, {
-    enter(node, parent, path) {
-      if (node.type === 'Member' && node.value.type === 'Object' && node.value.members && !path.includes('$value')) {
-        const members = getObjMembers(node.value);
-
-        // keep track of $types
-        if (members.$type && members.$type.type === 'String' && !members.$value) {
-          // @ts-ignore
-          $typeInheritance[path.join('.') || '.'] = node.value.members.find((m) => m.name.value === '$type');
-        }
-
-        const id = path.join('.');
-
-        if (
-          members.$value &&
-          !path.includes('$extensions') // don’t validate anything in $extensions
-        ) {
-          const extensions = members.$extensions ? getObjMembers(members.$extensions as ObjectNode) : undefined;
-          const sourceNode = structuredClone(node);
-
-          // get parent type by taking the closest-scoped $type (length === closer)
-          let parent$type: Token['$type'] | undefined;
-          let longestPath = '';
-          for (const [k, v] of Object.entries($typeInheritance)) {
-            if (k === '.' || id.startsWith(k)) {
-              if (k.length > longestPath.length) {
-                parent$type = v;
-                longestPath = k;
-              }
-            }
-          }
-          if (parent$type && !members.$type) {
-            sourceNode.value = injectObjMembers(
-              // @ts-ignore
-              sourceNode.value,
-              [parent$type],
-            );
-          }
-
-          validate(sourceNode, { filename, src, logger });
-
-          // All tokens must be valid, so we want to validate it up till this
-          // point. However, if we are ignoring this token (or respecting
-          // $deprecated, we can omit it from the output.
-          const $deprecated = members.$deprecated && (evaluate(members.$deprecated) as string | boolean | undefined);
-          if (
-            (config.ignore.deprecated && $deprecated) ||
-            (config.ignore.tokens && isTokenMatch(id, config.ignore.tokens))
-          ) {
-            return;
-          }
-
-          const group: TokenNormalized['group'] = { id: splitID(id).group!, tokens: [] };
-          if (parent$type) {
-            group.$type =
-              // @ts-ignore
-              parent$type.value.value;
-          }
-          // note: this will also include sibling tokens, so be selective about only accessing group-specific properties
-          const groupMembers = getObjMembers(
-            // @ts-ignore
-            parent,
-          );
-          if (groupMembers.$description) {
-            group.$description = evaluate(groupMembers.$description) as string;
-          }
-          if (groupMembers.$extensions) {
-            group.$extensions = evaluate(groupMembers.$extensions) as Record<string, unknown>;
-          }
-          const token: TokenNormalized = {
-            // @ts-ignore
-            $type: members.$type?.value ?? parent$type?.value.value,
-            // @ts-ignore
-            $value: evaluate(members.$value),
-            id,
-            // @ts-ignore
-            mode: {},
-            // @ts-ignore
-            originalValue: evaluate(node.value),
-            group,
-            source: {
-              loc: filename ? filename.href : undefined,
-              // @ts-ignore
-              node: sourceNode.value,
-            },
-          };
-          // @ts-ignore
-          if (members.$description?.value) {
-            // @ts-ignore
-            token.$description = members.$description.value;
-          }
-
-          // handle modes
-          // note that circular refs are avoided here, such as not duplicating `modes`
-          const modeValues = extensions?.mode
-            ? getObjMembers(
-                // @ts-ignore
-                extensions.mode,
-              )
-            : {};
-          for (const mode of ['.', ...Object.keys(modeValues)]) {
-            token.mode[mode] = {
-              id: token.id,
-              // @ts-ignore
-              $type: token.$type,
-              // @ts-ignore
-              $value: mode === '.' ? token.$value : evaluate(modeValues[mode]),
-              source: {
-                loc: filename ? filename.href : undefined,
-                // @ts-ignore
-                node: mode === '.' ? structuredClone(token.source.node) : modeValues[mode],
-              },
-            };
-            if (token.$description) {
-              token.mode[mode]!.$description = token.$description;
-            }
-          }
-
-          tokens[id] = token;
-        } else if (!path.includes('.$value') && members.value) {
-          logger.warn({ message: `Group ${id} has "value". Did you mean "$value"?`, filename, node, src });
-        }
-      }
-
-      // edge case: if $type appears at root of tokens.json, collect it
+    enter(node, parent, subpath) {
+      // if $type appears at root of tokens.json, collect it
       if (node.type === 'Document' && node.body.type === 'Object' && node.body.members) {
         const members = getObjMembers(node.body);
         if (members.$type && members.$type.type === 'String' && !members.$value) {
@@ -383,21 +231,24 @@ async function parseSingle(
           $typeInheritance['.'] = node.body.members.find((m) => m.name.value === '$type');
         }
       }
+
+      // handle tokens
+      if (node.type === 'Member') {
+        const token = validateTokenNode(node, { filename, src, config, logger, parent, subpath, $typeInheritance });
+        if (token) {
+          tokens[token.id] = token;
+        }
+      }
     },
   });
-  logger.debug({
-    group: 'parser',
-    label: 'validate',
-    message: 'Finish tokens validation',
-    timing: performance.now() - startValidation,
-  });
+  logger.debug({ message: 'Finish token validation', group: 'parser', label: 'validate', timing: startValidate });
 
   // 3. normalize values
   const normalizeStart = performance.now();
   logger.debug({
+    message: 'Start token normalization',
     group: 'parser',
     label: 'normalize',
-    message: 'Start token normalization',
   });
   for (const [id, token] of Object.entries(tokens)) {
     try {
@@ -422,13 +273,7 @@ async function parseSingle(
         if (members.$value) {
           node = members.$value as ObjectNode;
         }
-        logger.error({
-          message: (err as Error).message,
-          filename,
-          src,
-          node: modeValue.source.node,
-          continueOnError,
-        });
+        logger.error({ message: (err as Error).message, filename, src, node: modeValue.source.node, continueOnError });
       }
     }
   }
@@ -436,24 +281,20 @@ async function parseSingle(
   // 4. Execute lint runner with loaded plugins
   if (!skipLint && config?.plugins?.length) {
     const lintStart = performance.now();
-    logger.debug({
-      group: 'parser',
-      label: 'validate',
-      message: 'Start token linting',
-    });
+    logger.debug({ message: 'Start token linting', group: 'parser', label: 'validate' });
     await lintRunner({ tokens, src, config, logger });
     logger.debug({
+      message: 'Finish token linting',
       group: 'parser',
       label: 'validate',
-      message: 'Finish token linting',
       timing: performance.now() - lintStart,
     });
   }
 
   logger.debug({
+    message: 'Finish token normalization',
     group: 'parser',
     label: 'normalize',
-    message: 'Finish token normalization',
     timing: performance.now() - normalizeStart,
   });
 

package/src/parse/json.ts

@@ -1,4 +1,15 @@
-import type { AnyNode, MemberNode, ObjectNode, ValueNode } from '@humanwhocodes/momoa';
+import {
+  type AnyNode,
+  type DocumentNode,
+  type MemberNode,
+  type ObjectNode,
+  type ValueNode,
+  parse as parseJSON,
+  print,
+} from '@humanwhocodes/momoa';
+import type yamlToMomoa from 'yaml-to-momoa';
+import type Logger from '../logger.js';
+import type { InputSource } from '../types.js';
 
 export interface Visitor {
   enter?: (node: AnyNode, parent: AnyNode | undefined, path: string[]) => void;
@@ -25,17 +36,20 @@ export function isNode(value: unknown): boolean {
   return !!value && typeof value === 'object' && 'type' in value && typeof value.type === 'string';
 }
 
+export type ValueNodeWithIndex = ValueNode & { index: number };
+
 /** Get ObjectNode members as object */
-export function getObjMembers(node: ObjectNode): Record<string | number, ValueNode> {
-  const members: Record<string | number, ValueNode> = {};
+export function getObjMembers(node: ObjectNode): Record<string | number, ValueNodeWithIndex> {
+  const members: Record<string | number, ValueNodeWithIndex> = {};
   if (node.type !== 'Object') {
     return members;
   }
-  for (const m of node.members) {
+  for (let i = 0; i < node.members.length; i++) {
+    const m = node.members[i]!;
     if (m.name.type !== 'String') {
       continue;
     }
-    members[m.name.value] = m.value;
+    members[m.name.value] = { ...m.value, index: i };
   }
   return members;
 }
@@ -56,7 +70,8 @@ export function injectObjMembers(node: ObjectNode, members: MemberNode[] = []):
 }
 
 /**
- * Variation of Momoa’s traverse(), which keeps track of global path
+ * Variation of Momoa’s traverse(), which keeps track of global path.
+ * Allows mutation of AST (along with any consequences)
  */
 export function traverse(root: AnyNode, visitor: Visitor) {
   /**
@@ -101,6 +116,92 @@ export function traverse(root: AnyNode, visitor: Visitor) {
 }
 
 /** Determine if an input is likely a JSON string */
-export function maybeJSONString(input: string): boolean {
+export function maybeRawJSON(input: string): boolean {
   return typeof input === 'string' && input.trim().startsWith('{');
 }
+
+/** Find Momoa node by traversing paths */
+export function findNode<T = AnyNode>(node: AnyNode, path: string[]): T | undefined {
+  if (!path.length) {
+    return;
+  }
+
+  let nextNode: AnyNode | undefined;
+
+  switch (node.type) {
+    // for Document nodes, dive into body for “free” (not part of the path)
+    case 'Document': {
+      return findNode(node.body, path);
+    }
+    case 'Object': {
+      const [member, ...rest] = path;
+      nextNode = node.members.find((m) => m.name.type === 'String' && m.name.value === member)?.value;
+      if (nextNode && rest.length) {
+        return findNode(nextNode, path.slice(1));
+      }
+      break;
+    }
+    case 'Array': {
+      const [_index, ...rest] = path;
+      const index = Number.parseInt(_index!, 10);
+      nextNode = node.elements[index]?.value;
+      if (nextNode && rest.length) {
+        return findNode(nextNode, path.slice(1));
+      }
+      break;
+    }
+  }
+
+  return nextNode as T;
+}
+
+export interface ToMomoaOptions {
+  filename?: URL;
+  continueOnError?: boolean;
+  logger: Logger;
+  yamlToMomoa?: typeof yamlToMomoa;
+}
+
+export function toMomoa(
+  input: string | Record<string, any>,
+  { continueOnError, filename, logger, yamlToMomoa }: ToMomoaOptions,
+): InputSource {
+  let src = '';
+  if (typeof input === 'string') {
+    src = input;
+  }
+  let document = {} as DocumentNode;
+  if (typeof input === 'string' && !maybeRawJSON(input)) {
+    if (yamlToMomoa) {
+      try {
+        document = yamlToMomoa(input); // if string, but not JSON, attempt YAML
+      } catch (err) {
+        logger.error({ message: String(err), filename, src: input, continueOnError });
+      }
+    } else {
+      logger.error({
+        group: 'parser',
+        message: `Install \`yaml-to-momoa\` package to parse YAML, and pass in as option, e.g.:
+
+    import { parse } from '@terrazzo/parser';
+    import yamlToMomoa from 'yaml-to-momoa';
+
+    parse(yamlString, { yamlToMomoa });`,
+        continueOnError: false, // fail here; no point in continuing
+      });
+    }
+  } else {
+    document = parseJSON(
+      typeof input === 'string' ? input : JSON.stringify(input, undefined, 2), // everything else: assert it’s JSON-serializable
+      {
+        mode: 'jsonc',
+        ranges: true,
+        tokens: true,
+      },
+    );
+  }
+  if (!src) {
+    src = print(document, { indent: 2 });
+  }
+  return { src, document };
+}