@asyncapi/generator 1.15.9 → 1.17.0

package/docs/template.md

@@ -16,7 +16,7 @@ Examples outputs:
 
 A template is an independent Node.js project unrelated to the `generator` repository. AsyncAPI templates are managed, released, and published separately. You can also create templates and manage templates on your own.
 
-The generator uses the official [Arborist](https://www.npmjs.com/package/@npmcli/arborist) NPM library. (This means templates do not have to be published to package managers to use them.) Arborist helps the generator fetch the template's source code and use it for the generation process. 
+The generator uses the official [Arborist](https://www.npmjs.com/package/@npmcli/arborist) NPM library. (This means templates do not have to be published to package managers to use them.) Arborist helps the generator fetch the template's source code and use it for the generation process. By default, this library pulls data from the default NPM registry, which is https://registry.npmjs.org. You can also configure the generator to fetch templates that are private or hosted in different NPM registry
 
 You can store template projects on a local drive or as a `git` repository during the development process. 
 

package/docs/using-private-template.md

@@ -0,0 +1,29 @@
+---
+title: "Using private templates"
+weight: 180
+---
+Generator allows fetching the template from private repositories like Verdaccio, Nexus, npm, etc.
+
+
+## Private registry options:
+
+* **registry.url**: The URL of the registry where the private template is located. Defaults to `registry.npmjs.org`.
+* **registry.auth**: An optional parameter to pass the npm registry username and password encoded with base64, formatted as `username:password`. For example, if the username and password are `admin` and `nimda`, you need to encode them with the base64 value like `admin:nimda` which results in `YWRtaW46bmltZGE=`.
+**registry.token**: An optional parameter to pass to the npm registry authentication token. To get the token, you can first authenticate with the registry using `npm login` and then grab the generated token from the `.npmrc` file.
+
+## Pulling private template using library:
+
+```javascript
+const generator = new Generator('@asyncapi/html-template', 'output',
+      { 
+        debug: true,
+        registry: {
+          url: 'http://verdaccio:4873',  
+          auth: 'YWRtaW46bmltZGE=' 
+            // base64 encoded username and password 
+            // represented as admin:nimda
+          
+        }
+      });
+```
+Assuming you host `@asyncapi/html-template` in a private package registry like Verdaccio. To pull this template, you need to provide `registry.url` option that points to the registry URL and `registry.auth` as a base64 encoded value that represents the username and password. Instead of username and password, you can also pass `registry.token`.

package/lib/filtersRegistry.js

@@ -12,7 +12,7 @@ const { isAsyncFunction } = require('./utils');
  */
 module.exports.registerFilters = async (nunjucks, templateConfig, templateDir, filtersDir) => {
   await registerLocalFilters(nunjucks, templateDir, filtersDir);
-  await registerConfigFilters(nunjucks, templateDir, templateConfig);
+  registerConfigFilters(nunjucks, templateDir, templateConfig);
 };
 
 /**
@@ -71,16 +71,31 @@ async function registerConfigFilters(nunjucks, templateDir, templateConfig) {
 
   const promises = confFilters.map(async filtersModule => {
     let mod;
+    let filterName = filtersModule;
     try {
       //first we try to grab module with filters by the module name
       //this is when generation is used on production using remote templates
-      mod = require(filtersModule);
+      mod = require(filterName);
     } catch (error) {
       //in case template is local but was not installed in node_modules of the generator then we need to explicitly provide modules location
-      mod = require(path.resolve(templateDir, DEFAULT_MODULES_DIR, filtersModule));
+      try {
+        filterName = path.resolve(templateDir, DEFAULT_MODULES_DIR, filtersModule);
+        mod = require(filterName);
+      } catch (e) {
+        //sometimes it may happen that template is located in node_modules with other templates and its filter package is on the same level as template, as it is shared with other templates
+        try {
+          filterName = path.resolve(templateDir, '../..', filtersModule);
+          mod = require(filterName);
+        } catch (error) {
+          //in rare cases, especially in isolated tests, it may happen that installation 
+          //ends but is not yet fully completed, so initial require of the same path do not work
+          //but in next attempt it works
+          //we need to keep this workaround until we find a solution
+          mod = require(filterName);
+        }
+      }
     }
-
-    addFilters(nunjucks, mod);
+    return addFilters(nunjucks, mod);
   });
 
   await Promise.all(promises);

package/lib/generator.js

@@ -21,7 +21,6 @@ const {
   copyFile,
   exists,
   fetchSpec,
-  getInvalidOptions,
   isReactTemplate,
   isJsFile,
   registerSourceMap,
@@ -44,8 +43,7 @@ const DEFAULT_TEMPLATES_DIR = path.resolve(ROOT_DIR, 'node_modules');
 
 const TRANSPILED_TEMPLATE_LOCATION = '__transpiled';
 const TEMPLATE_CONTENT_DIRNAME = 'template';
-const GENERATOR_OPTIONS = ['debug', 'disabledHooks', 'entrypoint', 'forceWrite', 'install', 'noOverwriteGlobs', 'output', 'templateParams', 'mapBaseUrlToFolder'];
-
+const GENERATOR_OPTIONS = ['debug', 'disabledHooks', 'entrypoint', 'forceWrite', 'install', 'noOverwriteGlobs', 'output', 'templateParams', 'mapBaseUrlToFolder', 'url', 'auth', 'token', 'registry'];
 const logMessage = require('./logMessages');
 
 const shouldIgnoreFile = filePath =>
@@ -86,14 +84,21 @@ class Generator {
    * @param {Boolean} [options.install=false] Install the template and its dependencies, even when the template has already been installed.
    * @param {Boolean} [options.debug=false] Enable more specific errors in the console. At the moment it only shows specific errors about filters. Keep in mind that as a result errors about template are less descriptive.
    * @param {Object<String, String>} [options.mapBaseUrlToFolder] Optional parameter to map schema references from a base url to a local base folder e.g. url=https://schema.example.com/crm/  folder=./test/docs/ .
+   * @param {Object} [options.registry]           Optional parameter with private registry configuration
+   * @param {String} [options.registry.url]       Parameter to pass npm registry url
+   * @param {String} [options.registry.auth]      Optional parameter to pass npm registry username and password encoded with base64, formatted like username:password value should be encoded
+   * @param {String} [options.registry.token]     Optional parameter to pass npm registry auth token that you can grab from .npmrc file
    */
-  constructor(templateName, targetDir, { templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, install = false, debug = false, mapBaseUrlToFolder = {} } = {}) {
-    const invalidOptions = getInvalidOptions(GENERATOR_OPTIONS, arguments[arguments.length - 1] || []);
-    if (invalidOptions.length) throw new Error(`These options are not supported by the generator: ${invalidOptions.join(', ')}`);
+
+  constructor(templateName, targetDir, { templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, install = false, debug = false, mapBaseUrlToFolder = {}, registry = {}} = {}) {
+    const options = arguments[arguments.length - 1];
+    this.verifyoptions(options);
     if (!templateName) throw new Error('No template name has been specified.');
     if (!entrypoint && !targetDir) throw new Error('No target directory has been specified.');
     if (!['fs', 'string'].includes(output)) throw new Error(`Invalid output type ${output}. Valid values are 'fs' and 'string'.`);
 
+    /** @type {Object} Npm registry information. */
+    this.registry = registry;
     /** @type {String} Name of the template to generate. */
     this.templateName = templateName;
     /** @type {String} Path to the directory where the files will be generated. */
@@ -136,6 +141,23 @@ class Generator {
     });
   }
 
+  /**
+    * Check if the Registry Options are valid or not.
+    *
+    * @private
+    * @param {Object} invalidRegOptions Invalid Registry Options.
+    *
+    */
+
+  verifyoptions(Options) {
+    if (typeof Options !== 'object') return [];
+    const invalidOptions = Object.keys(Options).filter(param => !GENERATOR_OPTIONS.includes(param));
+
+    if (invalidOptions.length > 0) {
+      throw new Error(`These options are not supported by the generator: ${invalidOptions.join(', ')}`);
+    }
+  }
+
   /**
    * Generates files from a given template and an AsyncAPIDocument object.
    *
@@ -143,7 +165,7 @@ class Generator {
    * @example
    * await generator.generate(myAsyncAPIdocument);
    * console.log('Done!');
-   * 
+   *
    * @example
    * generator
    *   .generate(myAsyncAPIdocument)
@@ -201,9 +223,9 @@ class Generator {
    * @example
    * const generator = new Generator();
    * await generator.setupOutput();
-   * 
+   *
    * @async
-   * 
+   *
    * @throws {Error} If 'output' is set to 'string' without providing 'entrypoint'.
    */
   async setupOutput() {
@@ -259,7 +281,6 @@ class Generator {
    */
   async installAndSetupTemplate() {
     const { name: templatePkgName, path: templatePkgPath } = await this.installTemplate(this.install);
-
     this.templateDir = templatePkgPath;
     this.templateName = templatePkgName;
     this.templateContentDir = path.resolve(this.templateDir, TEMPLATE_CONTENT_DIRNAME);
@@ -287,11 +308,9 @@ class Generator {
     await this.parseInput(this.asyncapi, parseOptions);
     validateTemplateConfig(this.templateConfig, this.templateParams, this.asyncapi);
     await this.configureTemplate();
-
     if (!isReactTemplate(this.templateConfig)) {
       await registerFilters(this.nunjucks, this.templateConfig, this.templateDir, FILTERS_DIRNAME);
     }
-
     await registerHooks(this.hooks, this.templateConfig, this.templateDir, HOOKS_DIRNAME);
     await this.launchHook('generate:before');
   }
@@ -313,16 +332,14 @@ class Generator {
   async handleEntrypoint() {
     if (this.entrypoint) {
       const entrypointPath = path.resolve(this.templateContentDir, this.entrypoint);
-
       if (!(await exists(entrypointPath))) {
         throw new Error(`Template entrypoint "${entrypointPath}" couldn't be found.`);
       }
-
       if (this.output === 'fs') {
         await this.generateFile(this.asyncapi, path.basename(entrypointPath), path.dirname(entrypointPath));
         await this.launchHook('generate:after');
       } else if (this.output === 'string') {
-        return this.renderFile(this.asyncapi, entrypointPath);
+        return await this.renderFile(this.asyncapi, entrypointPath);
       }
     } else {
       await this.generateDirectoryStructure(this.asyncapi);
@@ -355,6 +372,13 @@ class Generator {
       if (!document) {
         const err = new Error('Input is not a correct AsyncAPI document so it cannot be processed.');
         err.diagnostics = diagnostics;
+        for (const diag of diagnostics) {
+          console.error(
+            `Diagnostic err: ${diag['message']} in path ${JSON.stringify(diag['path'])} starting `+
+            `L${diag['range']['start']['line'] + 1} C${diag['range']['start']['character']}, ending `+
+            `L${diag['range']['end']['line'] + 1} C${diag['range']['end']['character']}`
+          );
+        }
         throw err;
       } else {
         this.asyncapi = document;
@@ -417,7 +441,7 @@ class Generator {
     if (!isParsableCompatible) {
       throw new Error('Parameter "asyncapiString" must be a non-empty string.');
     }
-    return this.generate(asyncapiString, parseOptions);
+    return await this.generate(asyncapiString, parseOptions);
   }
 
   /**
@@ -444,7 +468,7 @@ class Generator {
    */
   async generateFromURL(asyncapiURL) {
     const doc = await fetchSpec(asyncapiURL);
-    return this.generate(doc, { path: asyncapiURL });
+    return await this.generate(doc, { path: asyncapiURL });
   }
 
   /**
@@ -471,7 +495,7 @@ class Generator {
    */
   async generateFromFile(asyncapiFile) {
     const doc = await readFile(asyncapiFile, { encoding: 'utf8' });
-    return this.generate(doc, { path: asyncapiFile });
+    return await this.generate(doc, { path: asyncapiFile });
   }
 
   /**
@@ -495,69 +519,99 @@ class Generator {
     return await readFile(path.resolve(templatesDir, templateName, filePath), 'utf8');
   }
 
+  /**
+   * @private
+   * @param {Object} arbOptions ArbOptions to intialise the Registry details.
+   */
+  initialiseArbOptions(arbOptions) {
+    let registryUrl = 'registry.npmjs.org';
+    let authorizationName = 'anonymous';
+    const providedRegistry = this.registry.url;
+
+    if (providedRegistry) {
+      arbOptions.registry = providedRegistry;
+      registryUrl = providedRegistry;
+    }
+    
+    const domainName = registryUrl.replace(/^https?:\/\//, '');
+    //doing basic if/else so basically only one auth type is used and token as more secure is primary
+    if (this.registry.token) {
+      authorizationName = `//${domainName}:_authToken`;
+      arbOptions[authorizationName] = this.registry.token;
+    } else if (this.registry.auth) {
+      authorizationName = `//${domainName}:_auth`;
+      arbOptions[authorizationName] = this.registry.auth;
+    } 
+
+    //not sharing in logs neither token nor auth for security reasons
+    log.debug(`Using npm registry ${registryUrl} and authorization type ${authorizationName} to handle template installation.`);
+  }
   /**
    * Downloads and installs a template and its dependencies
    *
    * @param {Boolean} [force=false] Whether to force installation (and skip cache) or not.
    */
-  installTemplate(force = false) {
-    return new Promise(async (resolve, reject) => {
-      if (!force) {
-        let pkgPath;
-        let installedPkg;
-        let packageVersion;
-
-        try {
-          installedPkg = getTemplateDetails(this.templateName, PACKAGE_JSON_FILENAME);
-          pkgPath = installedPkg && installedPkg.pkgPath;
-          packageVersion = installedPkg && installedPkg.version;
-          log.debug(logMessage.templateSource(pkgPath));
-          if (packageVersion) log.debug(logMessage.templateVersion(packageVersion));
-
-          return resolve({
-            name: installedPkg.name,
-            path: pkgPath
-          });
-        } catch (e) {
-          log.debug(logMessage.packageNotAvailable(pkgPath), e);
-          // We did our best. Proceed with installation...
-        }
-      }
-
-      const debugMessage = force ? logMessage.TEMPLATE_INSTALL_FLAG_MSG : logMessage.TEMPLATE_INSTALL_DISK_MSG;
-      log.debug(logMessage.installationDebugMessage(debugMessage));
-
-      if (isFileSystemPath(this.templateName)) log.debug(logMessage.NPM_INSTALL_TRIGGER);
-
-      const arb = new Arborist({
-        path: ROOT_DIR
-      });
-
+  async installTemplate(force = false) {
+    if (!force) {
+      let pkgPath;
+      let installedPkg;
+      let packageVersion;
+  
       try {
-        const installResult = await arb.reify({
-          add: [this.templateName],
-          saveType: 'prod',
-          save: false
-        });
-
-        const addResult = arb[Symbol.for('resolvedAdd')];
-        if (!addResult) return reject('Unable to resolve the name of the added package. It was most probably not added to node_modules successfully');
-
-        const packageName = addResult[0].name;
-        const packageVersion = installResult.children.get(packageName).version;
-        const packagePath = installResult.children.get(packageName).path;
-
-        if (!isFileSystemPath(this.templateName)) log.debug(logMessage.templateSuccessfullyInstalled(packageName, packagePath));
+        installedPkg = getTemplateDetails(this.templateName, PACKAGE_JSON_FILENAME);
+        pkgPath = installedPkg && installedPkg.pkgPath;
+        packageVersion = installedPkg && installedPkg.version;
+        log.debug(logMessage.templateSource(pkgPath));
         if (packageVersion) log.debug(logMessage.templateVersion(packageVersion));
-
-        return resolve({
-          name: packageName,
-          path: packagePath,
-        });
-      } catch (err) {
-        reject(err);
+  
+        return {
+          name: installedPkg.name,
+          path: pkgPath
+        };
+      } catch (e) {
+        log.debug(logMessage.packageNotAvailable(installedPkg), e);
+        // We did our best. Proceed with installation...
       }
-    });
+    }
+  
+    const debugMessage = force ? logMessage.TEMPLATE_INSTALL_FLAG_MSG : logMessage.TEMPLATE_INSTALL_DISK_MSG;
+    log.debug(logMessage.installationDebugMessage(debugMessage));
+  
+    if (isFileSystemPath(this.templateName)) log.debug(logMessage.NPM_INSTALL_TRIGGER);
+  
+    const arbOptions = {
+      path: ROOT_DIR,
+    };
+    if (this.registry) {
+      this.initialiseArbOptions(arbOptions);
+    }
+  
+    const arb = new Arborist(arbOptions);
+    
+    try {
+      const installResult = await arb.reify({
+        add: [this.templateName],
+        saveType: 'prod',
+        save: false
+      });
+  
+      const addResult = arb[Symbol.for('resolvedAdd')];
+      if (!addResult) throw new Error('Unable to resolve the name of the added package. It was most probably not added to node_modules successfully');
+  
+      const packageName = addResult[0].name;
+      const packageVersion = installResult.children.get(packageName).version;
+      const packagePath = installResult.children.get(packageName).path;
+  
+      if (!isFileSystemPath(this.templateName)) log.debug(logMessage.templateSuccessfullyInstalled(packageName, packagePath));
+      if (packageVersion) log.debug(logMessage.templateVersion(packageVersion));
+  
+      return {
+        name: packageName,
+        path: packagePath,
+      };
+    } catch (err) {
+      throw new Error('Installation failed', err);
+    }
   }
 
   /**
@@ -722,7 +776,7 @@ class Generator {
    * @param  {String} baseDir Base directory of the given file name.
    * @returns {Promise}
    */
-  generateSeparateFiles(asyncapiDocument, array, template, fileName, baseDir) {
+  async generateSeparateFiles(asyncapiDocument, array, template, fileName, baseDir) {
     const promises = [];
 
     Object.keys(array).forEach((name) => {
@@ -761,7 +815,7 @@ class Generator {
     const newFileName = fileName.replace(`\$\$${template}\$\$`, filename);
     const targetFile = path.resolve(this.targetDir, relativeBaseDir, newFileName);
     const relativeTargetFile = path.relative(this.targetDir, targetFile);
-    const shouldOverwriteFile = this.shouldOverwriteFile(relativeTargetFile);
+    const shouldOverwriteFile = await this.shouldOverwriteFile(relativeTargetFile);
     if (!shouldOverwriteFile) return;
     //Ensure the same object are parsed to the renderFile method as before.
     const temp = {};
@@ -808,7 +862,7 @@ class Generator {
 
     if (shouldIgnoreFile(relativeSourceFile)) return;
 
-    const shouldOverwriteFile = this.shouldOverwriteFile(relativeTargetFile);
+    const shouldOverwriteFile = await this.shouldOverwriteFile(relativeTargetFile);
     if (!shouldOverwriteFile) return;
 
     if (this.templateConfig.conditionalFiles && this.templateConfig.conditionalFiles[relativeSourceFile]) {
@@ -889,7 +943,7 @@ class Generator {
    *
    * @private
    * @param  {string} filePath Path to the file to check against a list of glob patterns.
-   * @return {boolean}
+   * @return {Promise<boolean>}
    */
   async shouldOverwriteFile(filePath) {
     if (!Array.isArray(this.noOverwriteGlobs)) return true;

package/lib/logMessages.js

@@ -18,8 +18,12 @@ function templateNotFound(templateName) {
   return `${templateName} not found in local dependencies but found it installed as a global package.`;
 } 
 
-function packageNotAvailable(pkgPath) {
-  return `Unable to resolve template location at ${pkgPath}. Package is not available locally.`;
+function packageNotAvailable(packageDetails) {
+  if (packageDetails && packageDetails.pkgPath) {
+    return `Unable to resolve template location at ${packageDetails.pkgPath}. Package is not available locally.`;
+  } 
+
+  return `Template is not available locally and expected location is undefined. Known details are: ${JSON.stringify(packageDetails, null, 2)}`;
 }
 
 function installationDebugMessage(debugMessage) {

package/lib/utils.js

@@ -124,17 +124,6 @@ utils.getGeneratorVersion = () => {
   return packageJson.version;
 };
 
-/**
- * Filters out the Generator invalid options given
- *
- * @param {Array}
- * @returns {Array}
- */
-utils.getInvalidOptions = (generatorOptions, options) => {
-  if (typeof options !== 'object') return [];
-  return Object.keys(options).filter(param => !generatorOptions.includes(param));
-};
-
 /**
  * Determine whether the given function is asynchronous.
  * @private

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@asyncapi/generator",
-  "version": "1.15.9",
+  "version": "1.17.0",
   "description": "The AsyncAPI generator. It can generate documentation, code, anything!",
   "main": "./lib/generator.js",
   "bin": {
@@ -50,7 +50,7 @@
   "dependencies": {
     "@asyncapi/generator-react-sdk": "^1.0.6",
     "@asyncapi/parser": "^3.0.2",
-    "@npmcli/arborist": "^2.2.4",
+    "@npmcli/arborist": "5.6.3",
     "@smoya/multi-parser": "^5.0.0",
     "ajv": "^8.12.0",
     "chokidar": "^3.4.0",