@terrazzo/parser 0.0.11 → 0.0.13

package/parse/index.js

@@ -3,56 +3,214 @@ import { isAlias, parseAlias, pluralize, splitID } from '@terrazzo/token-tools';
 import lintRunner from '../lint/index.js';
 import Logger from '../logger.js';
 import normalize from './normalize.js';
-import parseYAML from './yaml.js';
 import validate from './validate.js';
 import { getObjMembers, injectObjMembers, traverse } from './json.js';
 
 export * from './validate.js';
 
+/** @typedef {import("@humanwhocodes/momoa").DocumentNode} DocumentNode */
+/** @typedef {import("../config.js").Plugin} Plugin */
+/** @typedef {import("../types.js").TokenNormalized} TokenNormalized */
 /**
- * @typedef {import("@humanwhocodes/momoa").DocumentNode} DocumentNode
- * @typedef {import("../config.js").Plugin} Plugin
- * @typedef {import("../types.js").TokenNormalized} TokenNormalized
- * @typedef {object} ParseOptions
- * @typedef {Logger} ParseOptions.logger
- * @typedef {boolean=false} ParseOptions.skipLint
- * @typedef {Plugin[]} ParseOptions.plugins
  * @typedef {object} ParseResult
- * @typedef {Record<string, TokenNormalized} ParseResult.tokens
- * @typedef {DocumentNode} ParseResult.ast
+ * @property {Record<string, TokenNormalized} tokens
+ * @property {Object[]} src
+ */
+/**
+ * @typedef {object} ParseInput
+ * @property {string | object} src
+ * @property {URL} [filename]
+ */
+/**
+ * @typedef {object} ParseOptions
+ * @property {Logger} logger
+ * @property {import("../config.js").Config} config
+ * @property {import("yamlToMomoa")} yamlToMomoa
+ * @property {boolean} [skipLint=false]
+ * @property {boolean} [continueOnError=false]
  */
-
 /**
  * Parse
- * @param {string | object} input
- * @param {ParseOptions} options
+ * @param {ParseInput[]} input
+ * @param {ParseOptions} [options]
  * @return {Promise<ParseResult>}
  */
 export default async function parse(
   input,
-  { logger = new Logger(), skipLint = false, config, continueOnError = false } = {},
+  { logger = new Logger(), skipLint = false, config = {}, continueOnError = false, yamlToMomoa } = {},
 ) {
-  const { plugins } = config;
+  let tokens = {};
+  // note: only keeps track of sources with locations on disk; in-memory sources are discarded
+  // (it’s only for reporting line numbers, which doesn’t mean as much for dynamic sources)
+  const sources = {};
+
+  if (!Array.isArray(input)) {
+    logger.error({ group: 'parser', task: 'init', message: 'Input must be an array of input objects.' });
+  }
+  for (let i = 0; i < input.length; i++) {
+    if (!input[i] || typeof input[i] !== 'object') {
+      logger.error({ group: 'parser', task: 'init', message: `Input (${i}) must be an object.` });
+    }
+    if (!input[i].src || (typeof input[i].src !== 'string' && typeof input[i].src !== 'object')) {
+      logger.error({
+        group: 'parser',
+        task: 'init',
+        message: `Input (${i}) missing "src" with a JSON/YAML string, or JSON object.`,
+      });
+    }
+    if (input[i].filename && !(input[i].filename instanceof URL)) {
+      logger.error({
+        group: 'parser',
+        task: 'init',
+        message: `Input (${i}) "filename" must be a URL (remote or file URL).`,
+      });
+    }
+
+    const result = await parseSingle(input[i].src, {
+      filename: input[i].filename,
+      logger,
+      config,
+      skipLint,
+      continueOnError,
+      yamlToMomoa,
+    });
+
+    tokens = Object.assign(tokens, result.tokens);
+    if (input[i].filename) {
+      sources[input[i].filename.protocol === 'file:' ? input[i].filename.href : input[i].filename.href] = {
+        filename: input[i].filename,
+        src: result.src,
+        document: result.document,
+      };
+    }
+  }
 
   const totalStart = performance.now();
 
+  // 5. Resolve aliases and populate groups
+  for (const id in tokens) {
+    if (!Object.hasOwn(tokens, id)) {
+      continue;
+    }
+    const token = tokens[id];
+    applyAliases(token, {
+      tokens,
+      filename: sources[token.source.loc]?.filename,
+      src: sources[token.source.loc]?.src,
+      node: token.source.node,
+      logger,
+    });
+    token.mode['.'].$value = token.$value;
+    if (token.aliasOf) {
+      token.mode['.'].aliasOf = token.aliasOf;
+    }
+    if (token.partialAliasOf) {
+      token.mode['.'].partialAliasOf = token.partialAliasOf;
+    }
+    const { group: parentGroup } = splitID(id);
+    for (const siblingID in tokens) {
+      const { group: siblingGroup } = splitID(siblingID);
+      if (siblingGroup?.startsWith(parentGroup)) {
+        token.group.tokens.push(siblingID);
+      }
+    }
+  }
+
+  // 6. resolve mode aliases
+  const modesStart = performance.now();
+  logger.debug({
+    group: 'parser',
+    task: 'modes',
+    message: 'Start mode resolution',
+  });
+  for (const id in tokens) {
+    if (!Object.hasOwn(tokens, id)) {
+      continue;
+    }
+    for (const mode in tokens[id].mode) {
+      if (mode === '.') {
+        continue; // skip shadow of root value
+      }
+      applyAliases(tokens[id].mode[mode], { tokens, node: tokens[id].mode[mode].source.node, logger });
+    }
+  }
+  logger.debug({
+    group: 'parser',
+    task: 'modes',
+    message: 'Finish token modes',
+    timing: performance.now() - modesStart,
+  });
+
+  logger.debug({
+    group: 'parser',
+    task: 'core',
+    message: 'Finish all parser tasks',
+    timing: performance.now() - totalStart,
+  });
+
+  if (continueOnError) {
+    const { errorCount } = logger.stats();
+    if (errorCount > 0) {
+      logger.error({
+        message: `Parser encountered ${errorCount} ${pluralize(errorCount, 'error', 'errors')}. Exiting.`,
+      });
+    }
+  }
+
+  return {
+    tokens,
+    sources,
+  };
+}
+
+/**
+ * Parse a single input
+ * @param {string | object} input
+ * @param {object} options
+ * @param {URL} [options.filename]
+ * @param {Logger} [options.logger]
+ * @param {import("../config.js").Config} [options.config]
+ * @param {boolean} [options.skipLint]
+ */
+async function parseSingle(input, { filename, logger, config, skipLint, continueOnError = false, yamlToMomoa }) {
   // 1. Build AST
-  let source;
+  let src;
   if (typeof input === 'string') {
-    source = input;
+    src = input;
   }
   const startParsing = performance.now();
   logger.debug({ group: 'parser', task: 'parse', message: 'Start tokens parsing' });
-  let ast;
+  let document;
   if (typeof input === 'string' && !maybeJSONString(input)) {
-    ast = parseYAML(input, { logger }); // if string, but not JSON, attempt YAML
+    if (yamlToMomoa) {
+      try {
+        document = yamlToMomoa(input); // if string, but not JSON, attempt YAML
+      } catch (err) {
+        logger.error({ message: String(err), filename, src: input, continueOnError });
+      }
+    } else {
+      logger.error({
+        group: 'parser',
+        task: 'parse',
+        message: `Install \`yaml-to-momoa\` package to parse YAML, and pass in as option, e.g.:
+
+    import { parse } from '@terrazzo/parser';
+    import yamlToMomoa from 'yaml-to-momoa';
+
+    parse(yamlString, { yamlToMomoa });`,
+        continueOnError: false, // fail here; no point in continuing
+      });
+    }
   } else {
-    ast = parseJSON(typeof input === 'string' ? input : JSON.stringify(input, undefined, 2), {
-      mode: 'jsonc',
-    }); // everything else: assert it’s JSON-serializable
+    document = parseJSON(
+      typeof input === 'string' ? input : JSON.stringify(input, undefined, 2), // everything else: assert it’s JSON-serializable
+      {
+        mode: 'jsonc',
+      },
+    );
   }
-  if (!source) {
-    source = print(ast, { indent: 2 });
+  if (!src) {
+    src = print(document, { indent: 2 });
   }
   logger.debug({
     group: 'parser',
@@ -67,7 +225,7 @@ export default async function parse(
   const startValidation = performance.now();
   logger.debug({ group: 'parser', task: 'validate', message: 'Start tokens validation' });
   const $typeInheritance = {};
-  traverse(ast, {
+  traverse(document, {
     enter(node, parent, path) {
       if (node.type === 'Member' && node.value.type === 'Object' && node.value.members) {
         const members = getObjMembers(node.value);
@@ -96,7 +254,8 @@ export default async function parse(
           if (parent$type && !members.$type) {
             sourceNode.value = injectObjMembers(sourceNode.value, [parent$type]);
           }
-          validate(sourceNode, { source, logger });
+
+          validate(sourceNode, { filename, src, logger });
 
           const group = { id: splitID(id).group, tokens: [] };
           if (parent$type) {
@@ -117,7 +276,10 @@ export default async function parse(
             mode: {},
             originalValue: evaluate(node.value),
             group,
-            sourceNode: sourceNode.value,
+            source: {
+              loc: filename ? filename.href : undefined,
+              node: sourceNode.value,
+            },
           };
           if (members.$description?.value) {
             token.$description = members.$description.value;
@@ -131,7 +293,10 @@ export default async function parse(
               id: token.id,
               $type: token.$type,
               $value: mode === '.' ? token.$value : evaluate(modeValues[mode]),
-              sourceNode: mode === '.' ? structuredClone(token.sourceNode) : modeValues[mode],
+              source: {
+                loc: filename ? filename.href : undefined,
+                node: mode === '.' ? structuredClone(token.source.node) : modeValues[mode],
+              },
             };
             if (token.$description) {
               token.mode[mode].$description = token.$description;
@@ -140,7 +305,7 @@ export default async function parse(
 
           tokens[id] = token;
         } else if (members.value) {
-          logger.warn({ message: `Group ${id} has "value". Did you mean "$value"?`, node, source });
+          logger.warn({ message: `Group ${id} has "value". Did you mean "$value"?`, filename, node, src });
         }
       }
 
@@ -161,14 +326,14 @@ export default async function parse(
   });
 
   // 3. Execute lint runner with loaded plugins
-  if (!skipLint && plugins?.length) {
+  if (!skipLint && config?.plugins?.length) {
     const lintStart = performance.now();
     logger.debug({
       group: 'parser',
       task: 'validate',
       message: 'Start token linting',
     });
-    await lintRunner({ ast, source, config, logger });
+    await lintRunner({ document, filename, src, config, logger });
     logger.debug({
       group: 'parser',
       task: 'validate',
@@ -191,12 +356,12 @@ export default async function parse(
     try {
       tokens[id].$value = normalize(tokens[id]);
     } catch (err) {
-      let node = tokens[id].sourceNode;
+      let { node } = tokens[id].source;
       const members = getObjMembers(node);
       if (members.$value) {
         node = members.$value;
       }
-      logger.error({ message: err.message, source, node, continueOnError });
+      logger.error({ message: err.message, filename, src, node, continueOnError });
     }
     for (const mode in tokens[id].mode) {
       if (mode === '.') {
@@ -205,12 +370,18 @@ export default async function parse(
       try {
         tokens[id].mode[mode].$value = normalize(tokens[id].mode[mode]);
       } catch (err) {
-        let node = tokens[id].sourceNode;
+        let { node } = tokens[id].source;
         const members = getObjMembers(node);
         if (members.$value) {
           node = members.$value;
         }
-        logger.error({ message: err.message, source, node: tokens[id].mode[mode].sourceNode, continueOnError });
+        logger.error({
+          message: err.message,
+          filename,
+          src,
+          node: tokens[id].mode[mode].source.node,
+          continueOnError,
+        });
       }
     }
   }
@@ -221,74 +392,7 @@ export default async function parse(
     timing: performance.now() - normalizeStart,
   });
 
-  // 5. Resolve aliases and populate groups
-  for (const id in tokens) {
-    if (!Object.hasOwn(tokens, id)) {
-      continue;
-    }
-    const token = tokens[id];
-    applyAliases(token, { tokens, source, node: token.sourceNode, logger });
-    token.mode['.'].$value = token.$value;
-    if (token.aliasOf) {
-      token.mode['.'].aliasOf = token.aliasOf;
-    }
-    if (token.partialAliasOf) {
-      token.mode['.'].partialAliasOf = token.partialAliasOf;
-    }
-    const { group: parentGroup } = splitID(id);
-    for (const siblingID in tokens) {
-      const { group: siblingGroup } = splitID(siblingID);
-      if (siblingGroup?.startsWith(parentGroup)) {
-        token.group.tokens.push(siblingID);
-      }
-    }
-  }
-
-  // 6. resolve mode aliases
-  const modesStart = performance.now();
-  logger.debug({
-    group: 'parser',
-    task: 'modes',
-    message: 'Start mode resolution',
-  });
-  for (const id in tokens) {
-    if (!Object.hasOwn(tokens, id)) {
-      continue;
-    }
-    for (const mode in tokens[id].mode) {
-      if (mode === '.') {
-        continue; // skip shadow of root value
-      }
-      applyAliases(tokens[id].mode[mode], { tokens, source, node: tokens[id].mode[mode].sourceNode, logger });
-    }
-  }
-  logger.debug({
-    group: 'parser',
-    task: 'modes',
-    message: 'Finish token modes',
-    timing: performance.now() - modesStart,
-  });
-
-  logger.debug({
-    group: 'parser',
-    task: 'core',
-    message: 'Finish all parser tasks',
-    timing: performance.now() - totalStart,
-  });
-
-  if (continueOnError) {
-    const { errorCount } = logger.stats();
-    if (errorCount > 0) {
-      logger.error({
-        message: `Parser encountered ${errorCount} ${pluralize(errorCount, 'error', 'errors')}. Exiting.`,
-      });
-    }
-  }
-
-  return {
-    tokens,
-    ast,
-  };
+  return { tokens, document, src };
 }
 
 /**
@@ -306,31 +410,32 @@ export function maybeJSONString(input) {
  * @param {Object} options
  * @param {Record<string, TokenNormalized>} options.tokens
  * @param {Logger} options.logger
+ * @param {string} [options.filename]
  * @param {AnyNode} [options.node]
  * @param {string} [options.string]
- * @param {string[]=[]} options.scanned
+ * @param {string} [options.scanned=[]]
  * @param {string}
  */
-export function resolveAlias(alias, { tokens, logger, source, node, scanned = [] }) {
+export function resolveAlias(alias, { tokens, logger, filename, src, node, scanned = [] }) {
   const { id } = parseAlias(alias);
   if (!tokens[id]) {
-    logger.error({ message: `Alias "${alias}" not found`, source, node });
+    logger.error({ message: `Alias "${alias}" not found`, filename, src, node });
   }
   if (scanned.includes(id)) {
-    logger.error({ message: `Circular alias detected from "${alias}"`, source, node });
+    logger.error({ message: `Circular alias detected from "${alias}"`, filename, src, node });
   }
   const token = tokens[id];
   if (!isAlias(token.$value)) {
     return id;
   }
-  return resolveAlias(token.$value, { tokens, logger, node, source, scanned: [...scanned, id] });
+  return resolveAlias(token.$value, { tokens, logger, filename, node, src, scanned: [...scanned, id] });
 }
 
 /** Resolve aliases, update values, and mutate `token` to add `aliasOf` / `partialAliasOf` */
-function applyAliases(token, { tokens, logger, source, node }) {
+function applyAliases(token, { tokens, logger, filename, src, node }) {
   // handle simple aliases
   if (isAlias(token.$value)) {
-    const aliasOfID = resolveAlias(token.$value, { tokens, logger, node, source });
+    const aliasOfID = resolveAlias(token.$value, { tokens, logger, filename, node, src });
     const { mode: aliasMode } = parseAlias(token.$value);
     const aliasOf = tokens[aliasOfID];
     token.aliasOf = aliasOfID;
@@ -353,7 +458,7 @@ function applyAliases(token, { tokens, logger, source, node }) {
         if (!token.partialAliasOf) {
           token.partialAliasOf = [];
         }
-        const aliasOfID = resolveAlias(token.$value[i], { tokens, logger, node, source });
+        const aliasOfID = resolveAlias(token.$value[i], { tokens, logger, filename, node, src });
         const { mode: aliasMode } = parseAlias(token.$value[i]);
         token.partialAliasOf[i] = aliasOfID;
         token.$value[i] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
@@ -366,7 +471,7 @@ function applyAliases(token, { tokens, logger, source, node }) {
             if (!token.partialAliasOf[i]) {
               token.partialAliasOf[i] = {};
             }
-            const aliasOfID = resolveAlias(token.$value[i][property], { tokens, logger, node, source });
+            const aliasOfID = resolveAlias(token.$value[i][property], { tokens, logger, filename, node, src });
             const { mode: aliasMode } = parseAlias(token.$value[i][property]);
             token.$value[i][property] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
             token.partialAliasOf[i][property] = aliasOfID;
@@ -386,7 +491,7 @@ function applyAliases(token, { tokens, logger, source, node }) {
         if (!token.partialAliasOf) {
           token.partialAliasOf = {};
         }
-        const aliasOfID = resolveAlias(token.$value[property], { tokens, logger, node, source });
+        const aliasOfID = resolveAlias(token.$value[property], { tokens, logger, filename, node, src });
         const { mode: aliasMode } = parseAlias(token.$value[property]);
         token.partialAliasOf[property] = aliasOfID;
         token.$value[property] = tokens[aliasOfID].mode[aliasMode]?.$value || tokens[aliasOfID].$value;
@@ -395,7 +500,7 @@ function applyAliases(token, { tokens, logger, source, node }) {
       else if (Array.isArray(token.$value[property])) {
         for (let i = 0; i < token.$value[property].length; i++) {
           if (isAlias(token.$value[property][i])) {
-            const aliasOfID = resolveAlias(token.$value[property][i], { tokens, logger, node, source });
+            const aliasOfID = resolveAlias(token.$value[property][i], { tokens, logger, filename, node, src });
             if (!token.partialAliasOf) {
               token.partialAliasOf = {};
             }

package/parse/validate.d.ts

@@ -7,7 +7,8 @@ declare const STROKE_STYLE_VALUES: Set<string>;
 declare const STROKE_STYLE_LINE_CAP_VALUES: Set<string>;
 
 export interface ValidateOptions {
-  source: string;
+  filename?: URL;
+  src: string;
   logger: Logger;
 }