@instana/shared-metrics 1.134.0 → 1.137.1

package/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+## [1.137.1](https://github.com/instana/nodejs/compare/v1.137.0...v1.137.1) (2021-11-23)
+
+
+### Bug Fixes
+
+* **dependency:** pinned semver to 7.3.3 ([d32f23e](https://github.com/instana/nodejs/commit/d32f23ea6807989d57ec6165c407b64e04d8d7c1)), closes [/github.com/npm/node-semver/blob/v7.3.3/package.json#L39](https://github.com//github.com/npm/node-semver/blob/v7.3.3/package.json/issues/L39) [/github.com/npm/node-semver/blob/v7.3.5/package.json#L39](https://github.com//github.com/npm/node-semver/blob/v7.3.5/package.json/issues/L39)
+* **dependency:** updated tar to 6.x in shared-metrics ([#415](https://github.com/instana/nodejs/issues/415)) ([5288ba5](https://github.com/instana/nodejs/commit/5288ba5241acd23d54f11c76edb3cffc0ffe6a66))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@instana/shared-metrics",
-  "version": "1.134.0",
+  "version": "1.137.1",
   "description": "Internal metrics plug-in package for Node.js monitoring with Instana",
   "author": {
     "name": "Bastian Krol",
@@ -26,7 +26,7 @@
   "scripts": {
     "audit": "bin/prepare-audit.sh && npm audit --production; AUDIT_RESULT=$?; git checkout package-lock.json; exit $AUDIT_RESULT",
     "test": "npm run test:mocha",
-    "test:mocha": "mocha --sort --reporter mocha-multi --reporter-options spec=-,xunit=../../test-results/shared-metris/results.xml $(find test -iname '*test.js' -not -path '*node_modules*')",
+    "test:mocha": "echo \"******* Files to be tested:\n $CI_SHARED_METRICS_TEST_FILES\" && mocha --sort --reporter mocha-multi --reporter-options spec=-,xunit=../../test-results/shared-metrics/results.xml ${CI_SHARED_METRICS_TEST_FILES:=$(find test -iname '*test.js' -not -path '*node_modules*')} test/globalHooks_test.js",
     "test:debug": "WITH_STDOUT=true npm run test:mocha",
     "lint": "eslint src test",
     "verify": "npm run lint && npm test",
@@ -52,6 +52,14 @@
     {
       "name": "Richard Gebhardt",
       "email": "gebhardt@us.ibm.com"
+    },
+    {
+      "name": "Willian Carvalho",
+      "email": "willian.carvalho@instana.com"
+    },
+    {
+      "name": "Katharina Irrgang",
+      "email": "katharina.irrgang@instana.com"
     }
   ],
   "bugs": {
@@ -59,13 +67,15 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@instana/core": "1.134.0",
+    "@instana/core": "1.137.1",
     "detect-libc": "^1.0.3",
     "event-loop-lag": "^1.4.0",
     "recursive-copy": "^2.0.13",
-    "tar": "^5.0.11"
+    "semver": "7.3.3",
+    "tar": "^6.1.11"
   },
   "devDependencies": {
+    "@types/tar": "^4.0.5",
     "eslint": "^7.30.0",
     "eslint-config-airbnb-base": "^14.2.1",
     "eslint-plugin-import": "^2.23.4",
@@ -75,8 +85,7 @@
   },
   "optionalDependencies": {
     "event-loop-stats": "1.3.0",
-    "gcstats.js": "1.0.0",
-    "node-gyp": "^7.1.2"
+    "gcstats.js": "1.0.0"
   },
-  "gitHead": "222ed99d7ce0fa90529e071a9395d726eafa9fb6"
+  "gitHead": "3245c3c62510c75760763669a4834c789e75fa91"
 }

package/src/dependencies.js

@@ -7,7 +7,6 @@
 
 const path = require('path');
 const fs = require('fs');
-
 const { applicationUnderMonitoring } = require('@instana/core').util;
 
 let logger = require('@instana/core').logger.getLogger('metrics');
@@ -32,6 +31,7 @@ exports.payloadPrefix = 'dependencies';
 const preliminaryPayload = {};
 
 /** @type {Object.<string, *>} */
+// @ts-ignore: Cannot redeclare exported variable 'currentPayload'
 exports.currentPayload = {};
 
 exports.MAX_ATTEMPTS = 20;
@@ -43,14 +43,14 @@ exports.activate = function activate() {
   attempts++;
 
   const started = Date.now();
-  applicationUnderMonitoring.getMainPackageJsonPathStartingAtMainModule((err, packageJsonPath) => {
+  applicationUnderMonitoring.getMainPackageJsonPathStartingAtMainModule((err, mainPackageJsonPath) => {
     if (err) {
       return logger.warn('Failed to determine main package.json. Reason: %s %s ', err.message, err.stack);
-    } else if (!packageJsonPath && attempts < exports.MAX_ATTEMPTS) {
+    } else if (!mainPackageJsonPath && attempts < exports.MAX_ATTEMPTS) {
       logger.debug('Main package.json could not be found. Will try again later.');
       setTimeout(exports.activate, DELAY).unref();
       return;
-    } else if (!packageJsonPath) {
+    } else if (!mainPackageJsonPath) {
       logger.info(
         `Main package.json could not be found after ${attempts} retries. Looking for node_modules folder now.`
       );
@@ -70,11 +70,11 @@ exports.activate = function activate() {
 
     let dependencyDir;
     if (applicationUnderMonitoring.isAppInstalledIntoNodeModules()) {
-      dependencyDir = path.join(path.dirname(packageJsonPath), '..', '..', 'node_modules');
+      dependencyDir = path.join(path.dirname(mainPackageJsonPath), '..', '..', 'node_modules');
     } else {
-      dependencyDir = path.join(path.dirname(packageJsonPath), 'node_modules');
+      dependencyDir = path.join(path.dirname(mainPackageJsonPath), 'node_modules');
     }
-    addAllDependencies(dependencyDir, started, packageJsonPath);
+    addAllDependencies(dependencyDir, started, mainPackageJsonPath);
   });
 };
 
@@ -89,6 +89,7 @@ exports.activate = function activate() {
 function addAllDependencies(dependencyDir, started, packageJsonPath) {
   addDependenciesFromDir(dependencyDir, () => {
     if (Object.keys(preliminaryPayload).length <= exports.MAX_DEPENDENCIES) {
+      // @ts-ignore: Cannot redeclare exported variable 'currentPayload'
       exports.currentPayload = preliminaryPayload;
       logger.debug(`Collection of dependencies took ${Date.now() - started} ms.`);
       return;
@@ -238,6 +239,7 @@ function limitAndSet(distances = {}) {
   for (let i = 0; i < keys.length - exports.MAX_DEPENDENCIES; i++) {
     delete preliminaryPayload[keys[i]];
   }
+  // @ts-ignore: Cannot redeclare exported variable 'currentPayload'
   exports.currentPayload = preliminaryPayload;
 }
 

package/src/gc.js

@@ -37,7 +37,8 @@ exports.currentPayload = {
   majorGcs: 0,
   incrementalMarkings: 0,
   weakCallbackProcessing: 0,
-  gcPause: 0
+  gcPause: 0,
+  statsSupported: false
 };
 
 exports.activate = function activate() {

package/src/libuv.js

@@ -47,5 +47,7 @@ function sense() {
     stats.statsSupported = true;
     return stats;
   }
-  return {};
+  return {
+    statsSupported: false
+  };
 }

package/src/util/nativeModuleRetry.js

@@ -11,9 +11,8 @@ const EventEmitter = require('events');
 const copy = require('recursive-copy');
 const fs = require('fs');
 const os = require('os');
+const semver = require('semver');
 const path = require('path');
-const tar = require('tar');
-const { fork } = require('child_process');
 const detectLibc = require('detect-libc');
 
 /**
@@ -26,20 +25,9 @@ const detectLibc = require('detect-libc');
  * @property {string} [loadFrom]
  */
 
-/** @type {Array.<string>} */
-const retryMechanisms = [];
-if (
-  !process.env.INSTANA_COPY_PRECOMPILED_NATIVE_ADDONS ||
-  process.env.INSTANA_COPY_PRECOMPILED_NATIVE_ADDONS.toLowerCase() !== 'false'
-) {
-  retryMechanisms.push('copy-precompiled');
-}
-if (
-  process.env.INSTANA_REBUILD_NATIVE_ADDONS_ON_DEMAND &&
-  process.env.INSTANA_REBUILD_NATIVE_ADDONS_ON_DEMAND.toLowerCase() === 'true'
-) {
-  retryMechanisms.push('rebuild');
-}
+const copyPrecompiledDisabled =
+  process.env.INSTANA_COPY_PRECOMPILED_NATIVE_ADDONS &&
+  process.env.INSTANA_COPY_PRECOMPILED_NATIVE_ADDONS.toLowerCase() === 'false';
 
 const platform = os.platform();
 const arch = process.arch;
@@ -60,18 +48,16 @@ function loadNativeAddOn(opts) {
   // Give clients a chance to register event listeners on the emitter that we return by attempting to load the module
   // asynchronously on the next tick.
   opts.loadFrom = opts.nativeModuleName;
-  process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, 0));
+  process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter));
   return loaderEmitter;
 }
 
 /**
  * @param {InstanaSharedMetricsOptions} opts
  * @param {EventEmitter} loaderEmitter
- * @param {number} retryIndex
- * @param {boolean} skipAttempt
- * @returns
+ * @returns {boolean}
  */
-function loadNativeAddOnInternal(opts, loaderEmitter, retryIndex, skipAttempt) {
+function loadNativeAddOnInternal(opts, loaderEmitter) {
   try {
     const { isMainThread } = require('worker_threads');
     if (!isMainThread) {
@@ -83,21 +69,32 @@ function loadNativeAddOnInternal(opts, loaderEmitter, retryIndex, skipAttempt) {
     // worker threads are not available, so we know that this is the main thread
   }
 
-  if (skipAttempt) {
-    // The logic of the previous retry mechanism figured out that it cannot complete successfully, so there is no reason
-    // to try to require the module again. Skip directly to the next retry.
-    logger.debug(`Skipping attempt ${retryIndex + 1} to load native add-on ${opts.nativeModuleName}.`);
-    prepareNextRetry(opts, loaderEmitter, retryIndex);
-  } else {
-    logger.debug(`Attempt ${retryIndex + 1} to load native add-on ${opts.nativeModuleName} from ${opts.loadFrom}.`);
-    try {
-      // Try to actually require the native add-on module.
-      const nativeModule = require(opts.loadFrom);
-      loaderEmitter.emit('loaded', nativeModule);
-      logger.debug(`Attempt ${retryIndex + 1} to load native add-on ${opts.nativeModuleName} has been successful.`);
-    } catch (e) {
-      logger.debug(`Attempt ${retryIndex + 1} to load native add-on ${opts.nativeModuleName} has failed.`, e);
-      prepareNextRetry(opts, loaderEmitter, retryIndex);
+  let nativeModuleHasBeenRequiredSuccessfully = attemptRequire(opts, loaderEmitter, 'directly');
+  if (!nativeModuleHasBeenRequiredSuccessfully) {
+    if (!copyPrecompiledDisabled) {
+      copyPrecompiled(opts, loaderEmitter, success => {
+        if (success) {
+          // The initial attempt to require the native add-on directly has failed but copying the precompiled add-on
+          // binaries has been successful. Try to require the precompiled add-on now.
+          nativeModuleHasBeenRequiredSuccessfully = attemptRequire(
+            opts,
+            loaderEmitter,
+            'after copying precompiled binaries'
+          );
+          if (!nativeModuleHasBeenRequiredSuccessfully) {
+            // Requiring the precompiled add-on has failed after successfully copying them.
+            giveUp(opts, loaderEmitter);
+          }
+        } else {
+          // The initial attempt to require the native add-on directly has failed and copying the precompiled add-on
+          // binaries has also failed.
+          giveUp(opts, loaderEmitter);
+        }
+      });
+    } else {
+      // The initial attempt to require the native add-on directly has failed and copying the precompiled add-on
+      // binaries has been explicitly disabled via INSTANA_COPY_PRECOMPILED_NATIVE_ADDONS=false.
+      giveUp(opts, loaderEmitter);
     }
   }
 }
@@ -105,49 +102,50 @@ function loadNativeAddOnInternal(opts, loaderEmitter, retryIndex, skipAttempt) {
 /**
  * @param {InstanaSharedMetricsOptions} opts
  * @param {EventEmitter} loaderEmitter
- * @param {number} retryIndex
+ * @param {string} mechanism
  */
-function prepareNextRetry(opts, loaderEmitter, retryIndex) {
-  // The first pre-condition for all retry mechanisms is that we can find the path to the native add-on that can not be
-  // required.
-  if (!opts.nativeModulePath || !opts.nativeModuleParentPath) {
-    findNativeModulePath(opts);
-    if (!opts.nativeModulePath || !opts.nativeModuleParentPath) {
-      logger.warn(opts.message + ' (No retry attempted.)');
-      loaderEmitter.emit('failed');
-      return;
-    }
+function attemptRequire(opts, loaderEmitter, mechanism) {
+  try {
+    // Try to actually require the native add-on module.
+    const nativeModule = require(opts.loadFrom);
+    loaderEmitter.emit('loaded', nativeModule);
+    logger.debug(`Attempt to load native add-on ${opts.nativeModuleName} ${mechanism} has been successful.`);
+    return true;
+  } catch (e) {
+    logger.debug(`Attempt to load native add-on ${opts.nativeModuleName} ${mechanism} has failed.`, e);
+    return false;
   }
+}
 
-  const nextRetryMechanism = retryMechanisms[retryIndex];
-  if (!nextRetryMechanism) {
-    // We have exhausted all possible mechanisms to cope with the failure to load the native add-on.
-    logger.warn(opts.message);
-    loaderEmitter.emit('failed');
-  } else if (nextRetryMechanism === 'copy-precompiled') {
-    copyPrecompiled(opts, loaderEmitter, retryIndex);
-  } else if (nextRetryMechanism === 'rebuild') {
-    rebuildOnDemand(opts, loaderEmitter, retryIndex);
-  } else {
-    logger.error(
-      `Unknown retry mechanism for loading the native module ${opts.nativeModuleName}: ${nextRetryMechanism}.`
-    );
-    process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
-  }
+/**
+ * @param {InstanaSharedMetricsOptions} opts
+ * @param {EventEmitter} loaderEmitter
+ */
+function giveUp(opts, loaderEmitter) {
+  logger.warn(opts.message);
+  loaderEmitter.emit('failed');
 }
 
 /**
  * @param {InstanaSharedMetricsOptions} opts
  * @param {EventEmitter} loaderEmitter
- * @param {number} retryIndex
+ * @param {(success: boolean) => void} callback
  */
-function copyPrecompiled(opts, loaderEmitter, retryIndex) {
+function copyPrecompiled(opts, loaderEmitter, callback) {
   logger.debug(`Trying to copy precompiled version of ${opts.nativeModuleName} for Node.js ${process.version}.`);
 
+  if (!opts.nativeModulePath || !opts.nativeModuleParentPath) {
+    if (!findNativeModulePath(opts)) {
+      logger.warn(`Unable to find or construct a path for native add-on ${opts.nativeModuleName}.`);
+      process.nextTick(callback.bind(false));
+      return;
+    }
+  }
+
   const abi = process.versions.modules;
   if (!abi) {
     logger.warn(`Could not determine ABI version for Node.js version ${process.version}.`);
-    process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
+    process.nextTick(callback.bind(false));
     return;
   }
 
@@ -163,19 +161,30 @@ function copyPrecompiled(opts, loaderEmitter, retryIndex) {
       logger.info(
         `A precompiled version for ${opts.nativeModuleName} is not available ${label} (at ${precompiledTarGzPath}).`
       );
-      process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
+      callback(false);
       return;
-      // Note: We could combine copying and recompiling for cases where we do not have a precompiled version but
-      // node-gyp is available. That is, copy the precompiled package (from an arbitrary architecture/ABI version), then
-      // rebuild that.
     } else if (statsErr) {
       logger.warn(`Looking for a precompiled version for ${opts.nativeModuleName} ${label} failed.`, statsErr);
-      process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
+      callback(false);
       return;
     }
 
     logger.info(`Found a precompiled version for ${opts.nativeModuleName} ${label}, unpacking.`);
 
+    /**
+     * tar@6 has dropped support for Node < 10
+     * It might work to require tar@6 or to execute commands with tar@6 and Node < 10,
+     * but we don't want to risk that a customers application fails - especially if tar@6 adds
+     * breaking changes. We decided to disallow this feature.
+     */
+    if (semver.lt(process.version, '10.0.0')) {
+      logger.info(`Skipped copying precompiled version for ${opts.nativeModuleName} ${label} with Node < 10.`);
+      callback(false);
+      return;
+    }
+
+    const tar = require('tar');
+
     tar
       .x({
         cwd: os.tmpdir(),
@@ -197,7 +206,7 @@ function copyPrecompiled(opts, loaderEmitter, retryIndex) {
           cpErr => {
             if (cpErr) {
               logger.warn(`Copying the precompiled build for ${opts.nativeModuleName} ${label} failed.`, cpErr);
-              process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
+              callback(false);
               return;
             }
 
@@ -205,86 +214,25 @@ function copyPrecompiled(opts, loaderEmitter, retryIndex) {
             // dependency should work.
             //
             // However, we must not use any of the paths from which Node.js has tried to load the module before (that
-            // is, node_modules/${opts.nativeModuleName}). Node.js has module loading infrastructure
+            // is, node_modules/${opts.nativeModuleName}). Node.js' module loading infrastructure
             // (lib/internal/modules/cjs/loader.js and lib/internal/modules/package_json_reader.js) have built-in
             // caching on multiple levels (for example, package.json locations and package.json contents). If Node.js
-            // has tried unsuccessfully to load a module or read a package.json from a particular path, it will remember
-            // and not try to load anything from that path again (a `false` will be put into the cache for that cache
-            // key). Instead, we force a new path, by adding precompiled to the module path and use the absolute path to
-            // the module to load it.
+            // has tried unsuccessfully to load a module or read a package.json from a particular path,
+            // it will remember and not try to load anything from that path again (a `false` will be
+            // put into the cache for that cache key). Instead, we force a new path, by adding precompiled
+            // to the module path and use the absolute path to the module to load it.
             opts.loadFrom = targetDir;
-            process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex));
+            callback(true);
           }
         );
       })
       .catch(tarErr => {
         logger.warn(`Unpacking the precompiled build for ${opts.nativeModuleName} ${label} failed.`, tarErr);
-        process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
+        callback(false);
       });
   });
 }
 
-/**
- * This is mainly for the scenario where @instana/collector and its dependencies has been provided by  an external
- * mechanism (like copying a bundled version contained in the Instana agent package) without running an npm install on
- * the target system. Native addons are present, but might have been build for a different operating system/Node ABI,
- * libc familiy, ...). Thus, they need to be rebuilt.
- *
- * @param {InstanaSharedMetricsOptions} opts
- * @param {EventEmitter} loaderEmitter
- * @param {number} retryIndex
- */
-function rebuildOnDemand(opts, loaderEmitter, retryIndex) {
-  /** @type {string} */
-  let nodeGypExecutable;
-  try {
-    const nodeGypPath = require.resolve('node-gyp');
-    if (!nodeGypPath) {
-      logger.warn(
-        // eslint-disable-next-line max-len
-        `Could not find node-gyp (require.resolve didn't return anything) to rebuild ${opts.nativeModuleName} on demand.`
-      );
-      process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
-      return;
-    }
-    nodeGypExecutable = path.join(nodeGypPath, '..', '..', 'bin', 'node-gyp.js');
-  } catch (e) {
-    logger.warn(`Could not load node-gyp to rebuild ${opts.nativeModuleName} on demand.`, e);
-    process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
-    return;
-  }
-
-  logger.info(`Rebuilding ${opts.nativeModulePath} via ${nodeGypExecutable}.`);
-  const nodeGyp = fork(nodeGypExecutable, ['rebuild'], {
-    cwd: opts.nativeModulePath
-  });
-  nodeGyp.on('error', err => {
-    logger.warn(
-      // eslint-disable-next-line max-len
-      `Attempt to rebuild ${opts.nativeModulePath} via ${nodeGypExecutable} has failed with an error.`,
-      err
-    );
-  });
-  nodeGyp.on('close', code => {
-    if (code === 0) {
-      logger.info(
-        // eslint-disable-next-line max-len
-        `Attempt to rebuild ${opts.nativeModulePath} via ${nodeGypExecutable} has finished, will try to load the module again.`
-      );
-      // In contrast to copyPrecompiled we do not need to force a different module path here, since in this scenario the
-      // package contents (including the package.json) was present in the node_modules folder, it was only the binary
-      // that didn't match the platform/ABI version.
-      opts.loadFrom = opts.nativeModuleName;
-      process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex));
-    } else {
-      logger.warn(
-        `Attempt to rebuild ${opts.nativeModulePath} via ${nodeGypExecutable} has failed with exit code ${code}.`
-      );
-      process.nextTick(loadNativeAddOnInternal.bind(null, opts, loaderEmitter, ++retryIndex, true));
-    }
-  });
-}
-
 /**
  * @param {InstanaSharedMetricsOptions} opts
  */
@@ -297,24 +245,24 @@ function findNativeModulePath(opts) {
         `Could not find location for ${opts.nativeModuleName} (require.resolve didn't return anything). ` +
           'Will create a path for it.'
       );
-      createNativeModulePath(opts);
-      return;
+      return createNativeModulePath(opts);
     }
     // We found a path to the module in node_modules, that means the directory exist (and we will reuse it) but the
     // module installation is incomplete and it could not be loaded earlier (otherwise we wouldn't have gotten here).
     const idx = nativeModulePath.lastIndexOf('node_modules');
     if (idx < 0) {
       logger.warn(`Could not find node_modules substring in ${nativeModulePath}.`);
-      return;
+      return false;
     }
     opts.nativeModulePath = nativeModulePath.substring(
       0,
       idx + 'node_modules'.length + opts.nativeModuleName.length + 2
     );
     opts.nativeModuleParentPath = path.join(opts.nativeModulePath, '..');
+    return true;
   } catch (e) {
     logger.debug(`Could not find location for ${opts.nativeModuleName}. Will create a path for it.`, e);
-    createNativeModulePath(opts);
+    return createNativeModulePath(opts);
   }
 }
 
@@ -332,7 +280,7 @@ function createNativeModulePath(opts) {
       logger.warn(
         `Could not find node_modules substring in ${selfPath}. Will give up loading ${opts.nativeModuleName}.`
       );
-      return;
+      return false;
     }
 
     // cut off everything after module path
@@ -343,6 +291,7 @@ function createNativeModulePath(opts) {
   // we need to go up two directory levels.
   opts.nativeModuleParentPath = loadNativeAddOn.selfNodeModulesPath;
   opts.nativeModulePath = path.join(loadNativeAddOn.selfNodeModulesPath, opts.nativeModuleName);
+  return true;
 }
 
 loadNativeAddOn.setLogger = setLogger;