@netlify/build 29.15.2 → 29.15.4

package/lib/log/messages/core_steps.js

@@ -80,7 +80,7 @@ export const logSecretsScanSuccessMessage = function (logs, msg) {
     log(logs, msg, { color: THEME.highlightWords });
 };
 export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) {
-    logErrorSubHeader(logs, `Secrets scanning found ${scanResults.matches.length} instance(s) of secrets in build output or repo code.\n`);
+    logErrorSubHeader(logs, `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${scanResults.matches.length} instance(s) of secrets in build output or repo code.\n`);
     Object.keys(groupedResults).forEach((key) => {
         logError(logs, `Secret env var "${key}"'s value detected:`);
         groupedResults[key]

package/lib/plugins_core/secrets_scanning/index.js

@@ -1,17 +1,26 @@
+import { trace } from '@opentelemetry/api';
 import { addErrorInfo } from '../../error/info.js';
+import { log } from '../../log/logger.js';
 import { logSecretsScanFailBuildMessage, logSecretsScanSkipMessage, logSecretsScanSuccessMessage, } from '../../log/messages/core_steps.js';
 import { getFilePathsToScan, getSecretKeysToScanFor, groupScanResultsByKey, isSecretsScanningEnabled, scanFilesForKeyValues, } from './utils.js';
+const tracer = trace.getTracer('secrets-scanning');
 const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecretKeys, systemLog }) {
     const stepResults = {};
     const passedSecretKeys = (explicitSecretKeys || '').split(',');
     const envVars = netlifyConfig.build.environment;
-    systemLog({ envVars, passedSecretKeys });
+    systemLog({ passedSecretKeys, buildDir });
     if (!isSecretsScanningEnabled(envVars)) {
         logSecretsScanSkipMessage(logs, 'Secrets scanning disabled via SECRETS_SCAN_ENABLED flag set to false.');
         return stepResults;
     }
+    // transparently log if there are scanning values being omitted
+    if (envVars['SECRETS_SCAN_OMIT_KEYS'] !== undefined) {
+        log(logs, `SECRETS_SCAN_OMIT_KEYS override option set to: ${envVars['SECRETS_SCAN_OMIT_KEYS']}\n`);
+    }
+    if (envVars['SECRETS_SCAN_OMIT_PATHS'] !== undefined) {
+        log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`);
+    }
     const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys);
-    systemLog({ keysToSearchFor });
     if (keysToSearchFor.length === 0) {
         logSecretsScanSkipMessage(logs, 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.');
         return stepResults;
@@ -20,19 +29,30 @@ const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecret
     // The scanning will look at builddir so that it can review both repo pulled files
     // and post build files
     const filePaths = await getFilePathsToScan({ env: envVars, base: buildDir });
-    systemLog({ buildDir, filePaths });
     if (filePaths.length === 0) {
         logSecretsScanSkipMessage(logs, 'Secrets scanning skipped because there are no files or all files were omitted with SECRETS_SCAN_OMIT_PATHS env var setting.');
         return stepResults;
     }
-    const scanResults = await scanFilesForKeyValues({
-        env: envVars,
-        keys: keysToSearchFor,
-        base: buildDir,
-        filePaths,
+    let scanResults;
+    await tracer.startActiveSpan('scanning-files', { attributes: { keysToSearchFor, totalFiles: filePaths.length } }, async (span) => {
+        scanResults = await scanFilesForKeyValues({
+            env: envVars,
+            keys: keysToSearchFor,
+            base: buildDir,
+            filePaths,
+        });
+        const attributesForLogsAndSpan = {
+            secretsScanFoundSecrets: scanResults.matches.length > 0,
+            secretsScanMatchesCount: scanResults.matches.length,
+            secretsFilesCount: scanResults.scannedFilesCount,
+            keysToSearchFor,
+        };
+        systemLog(attributesForLogsAndSpan);
+        span.setAttributes(attributesForLogsAndSpan);
+        span.end();
     });
-    if (scanResults.matches.length === 0) {
-        logSecretsScanSuccessMessage(logs, 'Secrets scanning complete. No secrets detected in build output or repo code.');
+    if (!scanResults || scanResults.matches.length === 0) {
+        logSecretsScanSuccessMessage(logs, `Secrets scanning complete. ${scanResults?.scannedFilesCount} file(s) scanned. No secrets detected in build output or repo code!`);
         return stepResults;
     }
     // at this point we have found matching secrets

package/lib/plugins_core/secrets_scanning/utils.js

@@ -1,4 +1,4 @@
-import { createReadStream } from 'node:fs';
+import { createReadStream, promises as fs, existsSync } from 'node:fs';
 import path from 'node:path';
 import { createInterface } from 'node:readline';
 import { fdir } from 'fdir';
@@ -63,13 +63,35 @@ export function getSecretKeysToScanFor(env, secretKeys) {
  * @returns string[] of relative paths from base of files that should be searched
  */
 export async function getFilePathsToScan({ env, base }) {
-    let files = await new fdir().withRelativePaths().crawl(base).withPromise();
+    const omitPathsAlways = ['.git'];
+    // node modules is dense and is only useful to scan if the repo itself commits these
+    // files. As a simple check to understand if the repo would commit these files, we expect
+    // that they would not ignore them from their git settings. So if gitignore includes
+    // node_modules anywhere we will omit looking in those folders - this will allow repos
+    // that do commit node_modules to still scan them.
+    let ignoreNodeModules = false;
+    const gitignorePath = path.resolve(base, '.gitignore');
+    const gitignoreContents = existsSync(gitignorePath) ? await fs.readFile(gitignorePath, 'utf-8') : '';
+    if (gitignoreContents?.includes('node_modules')) {
+        ignoreNodeModules = true;
+    }
+    let files = await new fdir()
+        .withRelativePaths()
+        .filter((path) => {
+        if (ignoreNodeModules && path.includes('node_modules')) {
+            return false;
+        }
+        return true;
+    })
+        .crawl(base)
+        .withPromise();
     // normalize the path separators to all use the forward slash
     // this is needed for windows machines and snapshot tests consistency.
     files = files.map((f) => f.split(path.sep).join('/'));
     let omitPaths = [];
     if (typeof env.SECRETS_SCAN_OMIT_PATHS === 'string') {
         omitPaths = env.SECRETS_SCAN_OMIT_PATHS.split(',')
+            .concat(omitPathsAlways)
             .map((s) => s.trim())
             .filter(Boolean);
     }
@@ -92,6 +114,7 @@ const omitPathMatches = (relativePath, omitPaths) => omitPaths.some((oPath) => r
 export async function scanFilesForKeyValues({ env, keys, filePaths, base }) {
     const scanResults = {
         matches: [],
+        scannedFilesCount: 0,
     };
     const keyValues = keys.reduce((kvs, key) => {
         let val = env[key];
@@ -107,9 +130,17 @@ export async function scanFilesForKeyValues({ env, keys, filePaths, base }) {
         }
         return kvs;
     }, {});
-    const settledPromises = await Promise.allSettled(filePaths.map((file) => {
-        return searchStream(base, file, keyValues);
-    }));
+    scanResults.scannedFilesCount = filePaths.length;
+    let settledPromises = [];
+    // process the scanning in batches to not run into memory issues by
+    // processing all files at the same time.
+    while (filePaths.length > 0) {
+        const chunkSize = 200;
+        const batch = filePaths.splice(0, chunkSize);
+        settledPromises = settledPromises.concat(await Promise.allSettled(batch.map((file) => {
+            return searchStream(base, file, keyValues);
+        })));
+    }
     settledPromises.forEach((result) => {
         if (result.status === 'fulfilled' && result.value?.length > 0) {
             scanResults.matches = scanResults.matches.concat(result.value);
@@ -118,10 +149,10 @@ export async function scanFilesForKeyValues({ env, keys, filePaths, base }) {
     return scanResults;
 }
 const searchStream = (basePath, file, keyValues) => {
-    return new Promise((resolve) => {
+    return new Promise((resolve, reject) => {
         const filePath = path.resolve(basePath, file);
         const inStream = createReadStream(filePath);
-        const rl = createInterface(inStream);
+        const rl = createInterface({ input: inStream, terminal: false });
         const matches = [];
         const keyVals = [].concat(...Object.values(keyValues));
         function getKeyForValue(val) {
@@ -138,7 +169,6 @@ const searchStream = (basePath, file, keyValues) => {
         keyVals.forEach((valVariant) => {
             maxMultiLineCount = Math.max(maxMultiLineCount, valVariant.split('\n').length);
         });
-        // search if not multi line or current accumulated lines length is less than num of lines
         const lines = [];
         let lineNumber = 0;
         rl.on('line', function (line) {
@@ -156,7 +186,7 @@ const searchStream = (basePath, file, keyValues) => {
                 }
                 keyVals.forEach((valVariant) => {
                     // matching of single/whole values
-                    if (line.search(new RegExp(valVariant)) >= 0) {
+                    if (line.includes(valVariant)) {
                         matches.push({
                             file,
                             lineNumber,
@@ -210,6 +240,15 @@ const searchStream = (basePath, file, keyValues) => {
                 });
             }
         });
+        rl.on('error', function (error) {
+            if (error?.code === 'EISDIR') {
+                // file path is a directory - do nothing
+                resolve(matches);
+            }
+            else {
+                reject(error);
+            }
+        });
         rl.on('close', function () {
             resolve(matches);
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@netlify/build",
-  "version": "29.15.2",
+  "version": "29.15.4",
   "description": "Netlify build module",
   "type": "module",
   "exports": "./lib/core/main.js",
@@ -68,11 +68,11 @@
     "@netlify/config": "^20.5.2",
     "@netlify/edge-bundler": "8.16.2",
     "@netlify/framework-info": "^9.8.10",
-    "@netlify/functions-utils": "^5.2.15",
+    "@netlify/functions-utils": "^5.2.17",
     "@netlify/git-utils": "^5.1.1",
     "@netlify/plugins-list": "^6.68.0",
     "@netlify/run-utils": "^5.1.1",
-    "@netlify/zip-it-and-ship-it": "9.12.0",
+    "@netlify/zip-it-and-ship-it": "9.12.2",
     "@opentelemetry/api": "^1.4.1",
     "@sindresorhus/slugify": "^2.0.0",
     "ansi-escapes": "^6.0.0",
@@ -148,5 +148,5 @@
       "module": "commonjs"
     }
   },
-  "gitHead": "2bb08a1fb4ad1bca50cee1131b70cbd94a9f08a3"
+  "gitHead": "c274972d83e29fcde2496b765c141521ffcdd4c1"
 }