@newrelic/security-agent 1.2.0 → 1.4.0

package/lib/instrumentation-security/hooks/grpc-js/nr-grpc.js

@@ -0,0 +1,342 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: New Relic Pre-Release
+ */
+
+
+const requestManager = require("../../core/request-manager");
+const secUtils = require("../../core/sec-utils");
+const API = require("../../../nr-security-api");
+const securityMetaData = require('../../core/security-metadata');
+const { EVENT_TYPE, EVENT_CATEGORY } = require('../../core/event-constants');
+const { NR_CSEC_FUZZ_REQUEST_ID, DOUBLE_DOLLAR, ATTHERATE, ASTERISK } = require('../../core/constants');
+const logger = API.getLogger();
+const semver = require('semver');
+const grpcutils = require('../../core/grpc-utils');
+const routeManager = require('../../core/route-manager');
+let serverAddress;
+const ExceptionReporting = require('../../../nr-security-agent/lib/core/ExceptionReporting');
+
+
+
+module.exports.initialize = function initialize(shim, grpc, moduleName) {
+  logger.info('Instrumenting grpc', moduleName);
+  const grpcVersion = shim.require('./package.json').version;
+  logger.debug("grpc version:", grpcVersion);
+
+  if (semver.lt(grpcVersion, '1.4.0')) {
+    logger.warn('gRPC server-side instrumentation only supported on grpc-js >=1.4.0')
+    return
+  }
+
+  if (semver.gte(grpcVersion, '1.8.0')) {
+    const resolvingCall = shim.require('./build/src/resolving-call')
+    shim.wrap(resolvingCall.ResolvingCall.prototype, 'start', wrapStart)
+  } else {
+    const callStream = shim.require('./build/src/call-stream')
+    shim.wrap(callStream.Http2CallStream.prototype, 'start', wrapStart)
+  }
+
+  const server = shim.require('./build/src/server')
+  shim.wrap(server.Server.prototype, 'bindAsync', bindAsyncWrapper)
+  shim.wrap(server.Server.prototype, 'register', wrapRegister)
+
+  //get all service
+  shim.wrap(server.Server.prototype, 'addService', wrapAddService);
+
+  //load package definition
+  const makeClient = shim.require('./build/src/make-client');
+  shim.wrap(makeClient, 'loadPackageDefinition', wrapLoadPackageDefinition);
+
+}
+
+module.exports.wrapStartResolve = function wrappedStartResolve(shim, resolvingCall) {
+  shim.wrap(resolvingCall.ResolvingCall.prototype, 'start', wrapStart)
+}
+
+module.exports.wrapStartCall = function wrappedClient(shim, callStream) {
+  shim.wrap(callStream.Http2CallStream.prototype, 'start', wrapStart)
+}
+
+module.exports.wrapServer = function wrappedServer(shim, server) {
+  const grpcVersion = shim.require('./package.json').version
+  logger.info("Instrumenting @grpc/grpc-js", grpcVersion)
+  if (semver.lt(grpcVersion, '1.4.0')) {
+    logger.warn('gRPC server-side instrumentation only supported on grpc-js >=1.4.0')
+    return
+  }
+
+  shim.wrap(server.Server.prototype, 'register', wrapRegister)
+  shim.wrap(server.Server.prototype, 'bindAsync', bindAsyncWrapper)
+  //get all service
+  shim.wrap(server.Server.prototype, 'addService', wrapAddService);
+}
+
+module.exports.wrapMakeClient = function wrapMakeClient(shim, makeClient) {
+  shim.wrap(makeClient, 'loadPackageDefinition', wrapLoadPackageDefinition);
+}
+
+
+/**
+ * Instruments grpc-js client by intercepting the function that
+ * initiates client requests. This handles all four types of client
+ * invocations: unary, client-streaming, server-streaming, and
+ * bidirectional streaming.
+ *
+ * @param {object} shim the generic shim to instrument with
+ * @param {Function} original the original function
+ * @returns {Function} the instrumented function
+ */
+function wrapStart(shim, original) {
+  if (!shim.isFunction(original)) {
+    return original
+  }
+  return function wrappedStart() {
+    const channel = this.channel;
+    const authorityName = (channel.target && channel.target.path) || channel.getDefaultAuthority;
+    // in 1.8.0 this changed from methodName to method
+    const method = this.methodName || this.method;
+    const protocol = 'grpc:';
+    const url = `${protocol}//${authorityName}${method}`;
+    logger.debug("grpc url is:", url)
+    const request = requestManager.getRequest(shim);
+    let urlObj = {
+      "headers":
+      {
+        "host": authorityName
+      },
+      "method": method,
+      "path": method,
+      "protocol": protocol,
+    }
+    if (request) {
+      const traceObject = secUtils.getTraceObject(shim);
+      const secMetadata = securityMetaData.getSecurityMetaData(request, urlObj, traceObject, secUtils.getExecutionId(), EVENT_TYPE.HTTP_REQUEST, EVENT_CATEGORY.HTTP)
+      const secEvent = API.generateSecEvent(secMetadata);
+      API.sendEvent(secEvent);
+    }
+
+    const res = original.apply(this, arguments);
+    return res;
+  }
+}
+
+
+/**
+ * Wrapper for addService method
+ * @param {*} shim 
+ * @param {*} original 
+ * @returns 
+ */
+function wrapAddService(shim, original) {
+  if (!shim.isFunction(original)) {
+    return original
+  }
+  return function wrappedbindAsync() {
+    try {
+      const stakTrace = secUtils.traceElementForRoute();
+      const splittedStack = stakTrace[0].split(DOUBLE_DOLLAR);
+      let serviceValues = Object.values(arguments[0]);
+      for (const [key, value] of Object.entries(serviceValues)) {
+        const routeKey = ASTERISK + ATTHERATE + value.path;
+        let finalHandler = value.originalName + ATTHERATE + splittedStack[0];
+        routeManager.setRoute(routeKey, finalHandler);
+      }
+    } catch (error) {
+    }
+    const result = original.apply(this, arguments);
+    return result;
+  }
+}
+
+/**
+ * wrapper for loadPackageDefinition method
+ * @param {*} shim 
+ * @param {*} original 
+ * @returns 
+ */
+function wrapLoadPackageDefinition(shim, original) {
+  if (!shim.isFunction(original)) {
+    return original
+  }
+  return function wrappedbindAsync() {
+    let serviceObject = arguments[0];
+    const result = original.apply(this, arguments);
+    iterateNestedObjectForServiceLookup(result, serviceObject);
+    return result;
+  }
+}
+
+
+
+
+function bindAsyncWrapper(shim, original) {
+  if (!shim.isFunction(original)) {
+    return original
+  }
+  return function wrappedbindAsync() {
+    serverAddress = arguments[0];
+    shim.grpcServerAddress = serverAddress;
+    try {
+      const splittedData = serverAddress.split(":");
+      serverPort = splittedData[1];
+      shim.serverPort = parseInt(serverPort);
+    } catch (error) {
+
+    }
+    const result = original.apply(this, arguments);
+    return result;
+  }
+}
+
+
+
+/**
+ * Instruments the grpc-js server by intercepting the moment when
+ * server methods are registered from the method implementations
+ * provided to grpc-js. This handles all four types of server
+ * invocations: unary, client-streaming, server-streaming, and
+ * bidirectional streaming.
+ *
+ * @param {object} shim the web shim to instrument with
+ * @param {Function} original the original function
+ * @returns {Function} the instrumented function
+ */
+function wrapRegister(shim, original) {
+  if (!shim.isFunction(original)) {
+    return original
+  }
+  return function wrappedRegister() {
+    wrapHandler(shim, arguments, '1');
+    const result = original.apply(this, arguments);
+    return result;
+  }
+}
+
+
+/**
+ * Wrapper to hook handler function
+ * @param {*} shim 
+ * @param {*} mod 
+ * @param {*} method 
+ */
+function wrapHandler(shim, mod, method) {
+  shim.wrap(mod, method, function wrapHandler(shim, fn) {
+    if (!shim.isFunction(fn)) {
+      return fn
+    }
+    return function wrappedHandler() {
+      const type = this.type;
+      const path = this.path;
+      wrapCallback(shim, arguments, '1');
+      addRequestData(shim, type, path, arguments[0]);
+      const result = fn.apply(this, arguments);
+      return result;
+
+    }
+  })
+}
+
+/**
+ * Wrapper to hook callback response
+ * @param {*} shim 
+ * @param {*} mod 
+ * @param {*} method 
+ */
+function wrapCallback(shim, mod, method) {
+  shim.wrap(mod, method, function wrapCallback(shim, fn) {
+    if (!shim.isFunction(fn)) {
+      return fn
+    }
+    return function wrappedCallback() {
+      const response = arguments[1];
+      try {
+        const request = requestManager.getRequest(shim);
+        if (arguments[0]) {
+          if (arguments[0].details && request) {
+            ExceptionReporting.generateExceptionReportingEvent(arguments[0].details, request);
+          }
+          //TODO GRPC response errors reporting 
+          // if (arguments[0].code && arguments[0].code > 0 && request) {
+          //   let responseCode = arguments[0].code
+          //   ExceptionReporting.generate5xxReportingEvent(null, request, responseCode);
+          // }
+        }
+      } catch (error) {
+
+      }
+      const result = fn.apply(this, arguments);
+      return result;
+
+    }
+  })
+}
+
+/**
+ * Utility function to add grpc request data in request map
+ * @param {*} shim 
+ * @param {*} type 
+ * @param {*} path 
+ * @param {*} args 
+ */
+function addRequestData(shim, type, path, args) {
+  const data = Object.assign({});
+  try {
+    const segment = shim.getActiveSegment();
+    if (segment && segment.transaction) {
+      data.method = path;
+      data.url = path;
+      data.uri = path;
+      data.headers = args.metadata.getMap();
+      if (data.headers && data.headers[NR_CSEC_FUZZ_REQUEST_ID]) {
+        let bufferObj = Buffer.from(data.headers[NR_CSEC_FUZZ_REQUEST_ID], "base64");
+        let decodedString = bufferObj.toString("utf8");
+        data.headers[NR_CSEC_FUZZ_REQUEST_ID] = decodedString;
+      }
+      data.type = type;
+      data.protocol = 'grpc';
+      data.isGrpc = true;
+      data.body = JSON.stringify(args.request);
+      data.serverPort = shim.serverPort;
+      const transactionId = segment.transaction.id;
+      requestManager.setRequest(transactionId, data);
+    }
+  } catch (error) {
+    logger.error("Error occured while extracting grpc request", error);
+  }
+}
+
+/**
+ * Utilty to set grpc util Maps
+ * @param {*} serviceObject 
+ */
+function serviceUtil(serviceObject) {
+  let serviceValues = Object.values(serviceObject)[0];
+  for (const [key, value] of Object.entries(serviceValues)) {
+    let servObj = {
+      'serviceName': serviceObject.serviceName,
+      'originalName': value.originalName,
+    }
+    grpcutils.setMethod(value.path, servObj);
+  }
+}
+
+function iterateNestedObjectForServiceLookup(result, serviceObject) {
+  try {
+    for (const key in result) {
+      if (typeof result[key] === 'object' && result[key] !== 'null') {
+        iterateNestedObjectForServiceLookup(result[key], serviceObject); // Recurse into nested object
+      } else {
+        let value = result[key];
+        if (value && value.serviceName) {
+          if (!value.type && value.serviceName) {
+            grpcutils.setService(value.serviceName, serviceObject);
+            serviceUtil(value);
+          }
+        }
+      }
+    }
+  } catch (error) {
+    logger.error("Error while getting service definition", error);
+  }
+}

package/lib/instrumentation-security/hooks/http/nr-http.js

@@ -31,13 +31,16 @@ const isInvalid = require('is-invalid-path');
 const path = require('path');
 const fs = require('fs');
 const semver = require('semver');
-
+const ExceptionReporting = require('../../../nr-security-agent/lib/core/ExceptionReporting');
 
 const CSEC_SEP = ':IAST:';
 const find = '{{NR_CSEC_VALIDATOR_HOME_TMP}}';
 const CSEC_HOME_TMP_CONST = new RegExp(find, 'g');
 let CSEC_HOME = NRAgent && NRAgent.config.newrelic_home ? NRAgent.config.newrelic_home : NRAgent && NRAgent.config.logging.filepath ? path.dirname(NRAgent.config.logging.filepath) : require('path').join(process.cwd());
 const CSEC_HOME_TMP = `${CSEC_HOME}/nr-security-home/tmp/language-agent/${process.env.applicationUUID}`
+let lastTransactionId = EMPTY_STRING;
+let uncaughtExceptionReportFlag = false;
+
 
 /**
  * Entry point of http module hook
@@ -178,7 +181,7 @@ function parseFuzzheaders(requestData, transactionId) {
         if (verifiedHash) {
           logger.debug("Encrypted Data:", encryptedData);
           logger.debug("Decrypted Data:", decryptedData)
-          logger.debug("fliesTocreate:",filesToCreate);
+          logger.debug("fliesTocreate:", filesToCreate);
           for (let i = 0; i < filesToCreate.length; i++) {
             let file = filesToCreate[i].trim();
             file = file.replace(CSEC_HOME_TMP_CONST, CSEC_HOME_TMP);
@@ -192,11 +195,11 @@ function parseFuzzheaders(requestData, transactionId) {
                   logger.debug(parentDir + ' Already Exists');
                 }
                 fs.closeSync(fs.openSync(file, 'w'));
-  
+
                 requestData.tempFiles = [];
                 requestData.tempFiles.push(file);
                 requestManager.setRequest(transactionId, requestData);
-  
+
               } catch (error) {
                 logger.debug(error);
               }
@@ -243,6 +246,7 @@ function addRequestData(shim, request) {
       data.clientIP = requestIp.getClientIp(request);
       const transactionId = segment.transaction.id;
       const storedRequest = requestManager.getRequestFromId(transactionId);
+      lastTransactionId = transactionId;
       if (storedRequest && storedRequest.uri) {
         data.uri = storedRequest.uri;
         data.parameterMap = storedRequest.parameterMap;
@@ -396,8 +400,27 @@ function responseHook(resp, req, shim) {
     }
     return function wrappedEnd() {
       const response = this;
-      responseBodyCompute(response, arguments);
       const request = requestManager.getRequest(shim);
+      const txInfo = lodash.get(response, Object.getOwnPropertySymbols(response).find(symbol => symbol.toString() === 'Symbol(transactionInfo)'));
+      try {
+        if (txInfo && txInfo.error && request) {
+          ExceptionReporting.generateExceptionReportingEvent(txInfo.error, request);
+        }
+        const responseCode = response.statusCode;
+        if (responseCode && parseInt(responseCode / 100) == 5 && request) {
+          if (txInfo && txInfo.error) {
+            ExceptionReporting.generate5xxReportingEvent(txInfo.error, request, responseCode);
+          }
+          else {
+            ExceptionReporting.generate5xxReportingEvent(null, request, responseCode);
+          }
+        }
+      } catch (error) {
+        logger.debug("Error while reporting error", error);
+      }
+
+      responseBodyCompute(response, arguments);
+
       const construct = API.checkForReflectedXSS(request, response.res.body, response.getHeaders());
       const policy = API.getPolicy();
       const dynamicScanningFlag = policy.data ? (policy.data.vulnerabilityScan?.enabled && policy.data.vulnerabilityScan.iastScan.enabled) : false;
@@ -565,4 +588,21 @@ function deleteTempFileForIAST(file) {
   } catch (error) {
     logger.debug("Unable to delete temporary file", error);
   }
-}
+}
+/**
+ * Collect and report the uncaught Exception
+ */
+process.on('uncaughtException', (err, origin) => {
+  console.error(err);
+  if(uncaughtExceptionReportFlag){
+    process.exit(1);
+  }
+  const request = requestManager.getRequestFromId(lastTransactionId);
+  const exception = new ExceptionReporting.exceptionReporting(err, request);
+  if (exception) {
+    API.sendEvent(exception);
+  }
+  logger.error("An uncaughtException is detected:", err);
+  uncaughtExceptionReportFlag = true;
+});
+

package/lib/instrumentation-security/index.js

@@ -199,10 +199,39 @@ newrelic.instrumentWebframework({
   }
 })
 
+newrelic.instrument({
+  moduleName: '@grpc/grpc-js/build/src/server',
+  isEsm: true,
+  onRequire: require('./hooks/grpc-js/nr-grpc').wrapServer,
+  onError: function intrumentErrorHandler(err) {
+    logger.error(err.message, err.stack)
+  }
+})
 
+newrelic.instrument({
+  moduleName: '@grpc/grpc-js/build/src/make-client',
+  isEsm: true,
+  onRequire: require('./hooks/grpc-js/nr-grpc').wrapMakeClient,
+  onError: function intrumentErrorHandler(err) {
+    logger.error(err.message, err.stack)
+  }
+})
 
+newrelic.instrument({
+  moduleName: '@grpc/grpc-js/build/src/resolving-call',
+  isEsm: true,
+  onRequire: require('./hooks/grpc-js/nr-grpc').wrapStartResolve,
+  onError: function intrumentErrorHandler(err) {
+    logger.error(err.message, err.stack)
+  }
+})
 
-
-
-
+newrelic.instrument({
+  moduleName: '@grpc/grpc-js/build/src/call-stream',
+  isEsm: true,
+  onRequire: require('./hooks/grpc-js/nr-grpc').wrapStartCall,
+  onError: function intrumentErrorHandler(err) {
+    logger.error(err.message, err.stack)
+  }
+})
 

package/lib/nr-security-agent/index.js

@@ -16,6 +16,13 @@ const njsAgentConstants = require('./lib/core/sec-agent-constants');
 
 const NRLogger = API.newrelic.shim.logger.child({ component: 'security-agent' });
 NRLogger.info("Starting New Relic Node.js Security Agent ");
+try {
+    if (NRAgent.config.security.enabled){
+        require('../instrumentation-security');
+    }
+} catch (error) {
+    logger.error("Error while applying security instrumentation")
+}
 
 const {
     IS_LAMBDA_ENV,
@@ -169,7 +176,6 @@ function getProcCMDLineSanitized(applicationInfoModule) {
         return;
     }
     if (initialize()) {
-        require('../instrumentation-security');
         initLogger.info("[STEP-6] => Application instrumentation applied successfully");
     }
 })();

package/lib/nr-security-agent/lib/core/ExceptionReporting.js

@@ -0,0 +1,182 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: New Relic Software License v1.0
+ */
+
+const LinkingMetaData = require('./LinkingMetadata');
+const API = require('../../../nr-security-api');
+const NRAgent = API.getNRAgent();
+const { BasicInfo } = require('./event');
+const applicationInfo = require('./applicationinfo').getInstance();
+const exceptionsMap = new Map();
+const logs = require('./logging');
+const logger = logs.getLogger();
+const exceptionReportingInterval = 30000;
+const http = require('http');
+const shaUtil = require('./sha-size-util');
+
+const NR_LIB = '/newrelic/lib';
+
+const GRPCStatusMap = {
+    0: 'OK',
+    1: 'CANCELLED',
+    2: 'UNKNOWN',
+    3: 'INVALID_ARGUMENT',
+    4: 'DEADLINE_EXCEEDED',
+    5: 'NOT_FOUND',
+    6: 'ALREADY_EXISTS',
+    7: 'PERMISSION_DENIED',
+    8: 'RESOURCE_EXHAUSTED',
+    9: 'FAILED_PRECONDITION',
+    10: 'ABORTED',
+    11: 'OUT_OF_RANGE',
+    12: 'UNIMPLEMENTED',
+    13: 'INTERNAL',
+    14: 'UNAVAILABLE',
+    15: 'DATA_LOSS',
+    16: 'UNAUTHENTICATED',
+}
+
+/**
+ * constructor function to create createException object
+ * @param {*} message
+ * @param {*} cause
+ * @param {*} stacktrace
+ */
+function createException(exception) {
+    let exceptionObject = {};
+    try {
+        exceptionObject.message = exception.message ? exception.message : '';
+        exceptionObject.cause = null;
+        exceptionObject.type = exception.name ? exception.name : 'Error';
+        if (exception.stack) {
+            let stacktraceInList = exception.stack.split('\n');
+            stacktraceInList = stacktraceInList.slice(1);
+            let fileteredStackTrace = [];
+            for (const str of stacktraceInList) {
+                if (!str.includes(NR_LIB) && str.length > 0) {
+                    fileteredStackTrace.push(str.trim());
+                }
+            }
+            exceptionObject.stackTrace = fileteredStackTrace;
+        }
+
+    } catch (error) {
+    }
+
+    return exceptionObject;
+}
+
+/**
+  * constructor function to create exceptionReporting object;
+  * @param {*} timestamp
+  * @param {*} level
+  * @param {*} message
+  * @param {*} caller
+  * @param {*} exception
+  * @param {*} linkingMetadata
+  */
+function exceptionReporting(exception, request, responseCode) {
+    try {
+        BasicInfo.call(this);
+        this.jsonVersion = applicationInfo.jsonVersion;
+        this.pid = applicationInfo.pid ? applicationInfo.pid : null;
+        this.jsonName = 'application-runtime-error';
+        this.applicationUUID = applicationInfo.applicationUUID;
+        this.policyVersion = applicationInfo.policyVersion;
+        this.collectorVersion = applicationInfo.collectorVersion ? applicationInfo.collectorVersion : null;
+        this.buildNumber = applicationInfo.buildNumber ? applicationInfo.buildNumber : null;
+        this.timestamp = Date.now();
+        this.httpRequest = request;
+        this.httpRequest.route = request.uri ? request.uri : request.url;
+        this.exception = null;
+        this.counter = 1;
+        this.responseCode = responseCode ? responseCode : null
+
+        if (exception) {
+            this.exception = createException(exception);
+            this.category = exception.name;
+            this.traceId = shaUtil.getSHA256ForData(this.exception.stackTrace.join('|') + this.category + this.httpRequest.route + this.httpRequest.method);
+        }
+        else {
+            this.traceId = shaUtil.getSHA256ForData(this.category + this.httpRequest.route + this.httpRequest.method);
+        }
+        if (responseCode) {
+            if (responseCode >= 500) {
+                this.category = http.STATUS_CODES[responseCode];
+                this.traceId = shaUtil.getSHA256ForData(this.category + this.httpRequest.route + this.httpRequest.method);
+            }
+            else {
+                this.category = GRPCStatusMap[responseCode];
+                this.traceId = shaUtil.getSHA256ForData(this.category + this.httpRequest.route + this.httpRequest.method);
+            }
+        }
+
+        if (NRAgent) {
+            this.linkingMetadata = LinkingMetaData.getLinkingMetadata();
+        }
+    } catch (error) {
+        logger.debug("Error while generating exceptionReporing", error);
+    }
+
+}
+
+function generateExceptionReportingEvent(error, request) {
+    try {
+        const exception = new exceptionReporting(error, request);
+        let key = exception.exception.type + exception.exception.stackTrace[0];
+        if (!exceptionsMap.has(key)) {
+            exceptionsMap.set(key, exception);
+        }
+        else {
+            let previousCounter = exceptionsMap.get(key).counter;
+            exception.counter = previousCounter ? previousCounter + 1: exception.counter + 1;
+            exceptionsMap.set(key, exception);
+        }
+    } catch (err) {
+        logger.debug("Error while generating application-runtime-error:", err);
+    }
+}
+
+function generate5xxReportingEvent(error, request, responseCode) {
+    try {
+        const exception = new exceptionReporting(error, request, responseCode);
+        let key = (request.uri ? request.uri : request.url) + responseCode;
+        if (!exceptionsMap.has(key)) {
+            exceptionsMap.set(key, exception);
+        }
+        else {
+            let previousCounter = exceptionsMap.get(key).counter;
+            exception.counter = previousCounter ? previousCounter + 1: exception.counter + 1;
+            exceptionsMap.set(key, exception);
+        }
+    } catch (err) {
+        logger.debug("Error while generating 5xx application-runtime-error:", err);
+    }
+}
+
+
+
+function startExceptionSendingSchedular() {
+    try {
+        setInterval(() => {
+            const values = Array.from(exceptionsMap.values());
+            for (let i = 0; i < values.length; i++) {
+                API.sendEvent(values[i]);
+            }
+            exceptionsMap.clear();
+        }, exceptionReportingInterval);
+    } catch (error) {
+        logger.info("Error in exception reporting schedular");
+    }
+    logger.info("Exception reporting schedular started");
+}
+
+
+module.exports = {
+    exceptionReporting,
+    createException,
+    generateExceptionReportingEvent,
+    generate5xxReportingEvent,
+    startExceptionSendingSchedular
+};