@netlify/build 33.1.5 → 33.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/log/messages/core_steps.js +15 -7
- package/lib/plugins_core/blobs_upload/index.js +0 -7
- package/lib/plugins_core/dev_blobs_upload/index.js +0 -7
- package/lib/plugins_core/secrets_scanning/index.js +20 -12
- package/lib/plugins_core/secrets_scanning/secret_prefixes.d.ts +4 -0
- package/lib/plugins_core/secrets_scanning/secret_prefixes.js +4 -1
- package/lib/plugins_core/secrets_scanning/utils.d.ts +37 -9
- package/lib/plugins_core/secrets_scanning/utils.js +87 -46
- package/package.json +3 -4
|
@@ -95,9 +95,11 @@ export const logSecretsScanSuccessMessage = function (logs, msg) {
|
|
|
95
95
|
};
|
|
96
96
|
export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) {
|
|
97
97
|
const { secretMatches, enhancedSecretMatches } = groupedResults;
|
|
98
|
-
|
|
98
|
+
const secretMatchesKeys = Object.keys(secretMatches);
|
|
99
|
+
const enhancedSecretMatchesKeys = Object.keys(enhancedSecretMatches);
|
|
100
|
+
logErrorSubHeader(logs, `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${secretMatchesKeys.length} instance(s) of secrets${enhancedSecretMatchesKeys.length > 0 ? ` and ${enhancedSecretMatchesKeys.length} instance(s) of likely secrets` : ''} in build output or repo code.\n`);
|
|
99
101
|
// Explicit secret matches
|
|
100
|
-
|
|
102
|
+
secretMatchesKeys.forEach((key) => {
|
|
101
103
|
logError(logs, `Secret env var "${key}"'s value detected:`);
|
|
102
104
|
secretMatches[key]
|
|
103
105
|
.sort((a, b) => {
|
|
@@ -107,9 +109,13 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro
|
|
|
107
109
|
logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true });
|
|
108
110
|
});
|
|
109
111
|
});
|
|
112
|
+
if (secretMatchesKeys.length) {
|
|
113
|
+
logError(logs, `\nTo prevent exposing secrets, the build will fail until these secret values are not found in build output or repo files.`);
|
|
114
|
+
logError(logs, `\nIf these are expected, use SECRETS_SCAN_OMIT_PATHS, SECRETS_SCAN_OMIT_KEYS, or SECRETS_SCAN_ENABLED to prevent detecting.`);
|
|
115
|
+
}
|
|
110
116
|
// Likely secret matches from enhanced scan
|
|
111
|
-
|
|
112
|
-
logError(logs,
|
|
117
|
+
enhancedSecretMatchesKeys.forEach((key, index) => {
|
|
118
|
+
logError(logs, `${index === 0 && secretMatchesKeys.length ? '\n' : ''}"${key}***" detected as a likely secret:`);
|
|
113
119
|
enhancedSecretMatches[key]
|
|
114
120
|
.sort((a, b) => {
|
|
115
121
|
return a.file > b.file ? 0 : 1;
|
|
@@ -118,7 +124,9 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro
|
|
|
118
124
|
logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true });
|
|
119
125
|
});
|
|
120
126
|
});
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
127
|
+
if (enhancedSecretMatchesKeys.length) {
|
|
128
|
+
logError(logs, `\nTo prevent exposing secrets, the build will fail until these likely secret values are not found in build output or repo files.`);
|
|
129
|
+
logError(logs, `\nIf these are expected, use ENHANCED_SECRETS_SCAN_OMIT_VALUES, or ENHANCED_SECRETS_SCAN_ENABLED to prevent detecting.`);
|
|
130
|
+
}
|
|
131
|
+
logError(logs, `\nFor more information on secrets scanning, see the Netlify Docs: https://ntl.fyi/configure-secrets-scanning`);
|
|
124
132
|
};
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import { version as nodeVersion } from 'node:process';
|
|
2
1
|
import { getDeployStore } from '@netlify/blobs';
|
|
3
2
|
import pMap from 'p-map';
|
|
4
|
-
import semver from 'semver';
|
|
5
3
|
import { DEFAULT_API_HOST } from '../../core/normalize_flags.js';
|
|
6
4
|
import { logError } from '../../log/logger.js';
|
|
7
5
|
import { getFileWithMetadata, getKeysToUpload, scanForBlobs } from '../../utils/blobs.js';
|
|
@@ -19,11 +17,6 @@ const coreStep = async function ({ logs, deployId, buildDir, packagePath, consta
|
|
|
19
17
|
token: NETLIFY_API_TOKEN,
|
|
20
18
|
apiURL: `https://${apiHost}`,
|
|
21
19
|
};
|
|
22
|
-
// If we don't have native `fetch` in the global scope, add a polyfill.
|
|
23
|
-
if (semver.lt(nodeVersion, '18.0.0')) {
|
|
24
|
-
const nodeFetch = (await import('node-fetch')).default;
|
|
25
|
-
storeOpts.fetch = nodeFetch;
|
|
26
|
-
}
|
|
27
20
|
const blobs = await scanForBlobs(buildDir, packagePath);
|
|
28
21
|
// We checked earlier, but let's be extra safe
|
|
29
22
|
if (blobs === null) {
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import { version as nodeVersion } from 'node:process';
|
|
2
1
|
import { getDeployStore } from '@netlify/blobs';
|
|
3
2
|
import pMap from 'p-map';
|
|
4
|
-
import semver from 'semver';
|
|
5
3
|
import { log, logError } from '../../log/logger.js';
|
|
6
4
|
import { getFileWithMetadata, getKeysToUpload, scanForBlobs } from '../../utils/blobs.js';
|
|
7
5
|
import { getBlobs } from '../../utils/frameworks_api.js';
|
|
@@ -18,11 +16,6 @@ const coreStep = async function ({ debug, logs, deployId, buildDir, quiet, packa
|
|
|
18
16
|
token: NETLIFY_API_TOKEN,
|
|
19
17
|
apiURL: `https://${apiHost}`,
|
|
20
18
|
};
|
|
21
|
-
// If we don't have native `fetch` in the global scope, add a polyfill.
|
|
22
|
-
if (semver.lt(nodeVersion, '18.0.0')) {
|
|
23
|
-
const nodeFetch = (await import('node-fetch')).default;
|
|
24
|
-
storeOpts.fetch = nodeFetch;
|
|
25
|
-
}
|
|
26
19
|
const blobs = await scanForBlobs(buildDir, packagePath);
|
|
27
20
|
// We checked earlier, but let's be extra safe
|
|
28
21
|
if (blobs === null) {
|
|
@@ -3,7 +3,7 @@ import { addErrorInfo } from '../../error/info.js';
|
|
|
3
3
|
import { log } from '../../log/logger.js';
|
|
4
4
|
import { logSecretsScanFailBuildMessage, logSecretsScanSkipMessage, logSecretsScanSuccessMessage, } from '../../log/messages/core_steps.js';
|
|
5
5
|
import { reportValidations } from '../../status/validations.js';
|
|
6
|
-
import { getFilePathsToScan,
|
|
6
|
+
import { getFilePathsToScan, getOmitValuesFromEnhancedScanForEnhancedScanFromEnv, getSecretKeysToScanFor, groupScanResultsByKeyAndScanType, isEnhancedSecretsScanningEnabled, isSecretsScanningEnabled, scanFilesForKeyValues, } from './utils.js';
|
|
7
7
|
const tracer = trace.getTracer('secrets-scanning');
|
|
8
8
|
const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecretKeys, enhancedSecretScan, systemLog, deployId, api, }) {
|
|
9
9
|
const stepResults = {};
|
|
@@ -21,14 +21,18 @@ const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecret
|
|
|
21
21
|
if (envVars['SECRETS_SCAN_OMIT_PATHS'] !== undefined) {
|
|
22
22
|
log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`);
|
|
23
23
|
}
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
24
|
+
const enhancedScanningEnabledInEnv = isEnhancedSecretsScanningEnabled(envVars);
|
|
25
|
+
if (enhancedSecretScan && !enhancedScanningEnabledInEnv) {
|
|
26
|
+
logSecretsScanSkipMessage(logs, 'Enhanced secrets detection disabled via ENHANCED_SECRETS_SCAN_ENABLED flag set to false.');
|
|
27
|
+
}
|
|
28
|
+
if (enhancedSecretScan &&
|
|
29
|
+
enhancedScanningEnabledInEnv &&
|
|
30
|
+
envVars['ENHANCED_SECRETS_SCAN_OMIT_VALUES'] !== undefined) {
|
|
31
|
+
log(logs, `ENHANCED_SECRETS_SCAN_OMIT_VALUES override option set to: ${envVars['ENHANCED_SECRETS_SCAN_OMIT_VALUES']}\n`);
|
|
32
|
+
}
|
|
33
|
+
const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys);
|
|
34
|
+
if (keysToSearchFor.length === 0 && !enhancedSecretScan) {
|
|
35
|
+
logSecretsScanSkipMessage(logs, 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.');
|
|
32
36
|
return stepResults;
|
|
33
37
|
}
|
|
34
38
|
// buildDir is the repository root or the base folder
|
|
@@ -48,9 +52,11 @@ const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecret
|
|
|
48
52
|
keys: keysToSearchFor,
|
|
49
53
|
base: buildDir,
|
|
50
54
|
filePaths,
|
|
55
|
+
enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv,
|
|
56
|
+
omitValuesFromEnhancedScan: getOmitValuesFromEnhancedScanForEnhancedScanFromEnv(envVars),
|
|
51
57
|
});
|
|
52
|
-
secretMatches = scanResults.matches.filter((match) =>
|
|
53
|
-
enhancedSecretMatches = scanResults.matches.filter((match) =>
|
|
58
|
+
secretMatches = scanResults.matches.filter((match) => !match.enhancedMatch);
|
|
59
|
+
enhancedSecretMatches = scanResults.matches.filter((match) => match.enhancedMatch);
|
|
54
60
|
const attributesForLogsAndSpan = {
|
|
55
61
|
secretsScanFoundSecrets: secretMatches.length > 0,
|
|
56
62
|
enhancedSecretsScanFoundSecrets: enhancedSecretMatches.length > 0,
|
|
@@ -58,6 +64,8 @@ const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecret
|
|
|
58
64
|
enhancedSecretsScanMatchesCount: enhancedSecretMatches.length,
|
|
59
65
|
secretsFilesCount: scanResults.scannedFilesCount,
|
|
60
66
|
keysToSearchFor,
|
|
67
|
+
enhancedPrefixMatches: enhancedSecretMatches.length ? enhancedSecretMatches.map((match) => match.key) : [],
|
|
68
|
+
enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv,
|
|
61
69
|
};
|
|
62
70
|
systemLog?.(attributesForLogsAndSpan);
|
|
63
71
|
span.setAttributes(attributesForLogsAndSpan);
|
|
@@ -80,7 +88,7 @@ const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecret
|
|
|
80
88
|
logSecretsScanFailBuildMessage({
|
|
81
89
|
logs,
|
|
82
90
|
scanResults,
|
|
83
|
-
groupedResults: groupScanResultsByKeyAndScanType(scanResults
|
|
91
|
+
groupedResults: groupScanResultsByKeyAndScanType(scanResults),
|
|
84
92
|
});
|
|
85
93
|
const error = new Error(`Secrets scanning found secrets in build.`);
|
|
86
94
|
addErrorInfo(error, { type: 'secretScanningFoundSecrets' });
|
|
@@ -3,3 +3,7 @@
|
|
|
3
3
|
* Note: string comparison is case-insensitive so we use all lowercase here.
|
|
4
4
|
*/
|
|
5
5
|
export declare const LIKELY_SECRET_PREFIXES: string[];
|
|
6
|
+
/**
|
|
7
|
+
* Known values that we do not want to trigger secret detection failures (e.g. common to framework build output)
|
|
8
|
+
*/
|
|
9
|
+
export declare const SAFE_LISTED_VALUES: string[];
|
|
@@ -10,7 +10,6 @@ const GITHUB_PREFIXES = ['ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_', 'github_pat_'];
|
|
|
10
10
|
const SHOPIFY_PREFIXES = ['shpss_', 'shpat_', 'shpca_', 'shppa_'];
|
|
11
11
|
const SQUARE_PREFIXES = ['sq0atp-'];
|
|
12
12
|
const OTHER_COMMON_PREFIXES = [
|
|
13
|
-
'pk_',
|
|
14
13
|
'sk_',
|
|
15
14
|
'pat_',
|
|
16
15
|
'sk-',
|
|
@@ -33,3 +32,7 @@ export const LIKELY_SECRET_PREFIXES = [
|
|
|
33
32
|
...SQUARE_PREFIXES,
|
|
34
33
|
...OTHER_COMMON_PREFIXES,
|
|
35
34
|
];
|
|
35
|
+
/**
|
|
36
|
+
* Known values that we do not want to trigger secret detection failures (e.g. common to framework build output)
|
|
37
|
+
*/
|
|
38
|
+
export const SAFE_LISTED_VALUES = ['SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED']; // Common to code using React PropTypes
|
|
@@ -7,11 +7,14 @@ interface ScanArgs {
|
|
|
7
7
|
keys: string[];
|
|
8
8
|
base: string;
|
|
9
9
|
filePaths: string[];
|
|
10
|
+
enhancedScanning?: boolean;
|
|
11
|
+
omitValuesFromEnhancedScan?: unknown[];
|
|
10
12
|
}
|
|
11
13
|
interface MatchResult {
|
|
12
14
|
lineNumber: number;
|
|
13
15
|
key: string;
|
|
14
16
|
file: string;
|
|
17
|
+
enhancedMatch: boolean;
|
|
15
18
|
}
|
|
16
19
|
export type SecretScanResult = {
|
|
17
20
|
scannedFilesCount: number;
|
|
@@ -24,6 +27,14 @@ export type SecretScanResult = {
|
|
|
24
27
|
* @returns
|
|
25
28
|
*/
|
|
26
29
|
export declare function isSecretsScanningEnabled(env: Record<string, unknown>): boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Determine if the user disabled enhanced scanning via env var
|
|
32
|
+
* @param env current envars
|
|
33
|
+
* @returns
|
|
34
|
+
*/
|
|
35
|
+
export declare function isEnhancedSecretsScanningEnabled(env: Record<string, unknown>): boolean;
|
|
36
|
+
export declare function getStringArrayFromEnvValue(env: Record<string, unknown>, envVarName: string): string[];
|
|
37
|
+
export declare function getOmitValuesFromEnhancedScanForEnhancedScanFromEnv(env: Record<string, unknown>): unknown[];
|
|
27
38
|
/**
|
|
28
39
|
* given the explicit secret keys and env vars, return the list of secret keys which have non-empty or non-trivial values. This
|
|
29
40
|
* will also filter out keys passed in the SECRETS_SCAN_OMIT_KEYS env var.
|
|
@@ -38,15 +49,32 @@ export declare function isSecretsScanningEnabled(env: Record<string, unknown>):
|
|
|
38
49
|
*/
|
|
39
50
|
export declare function getSecretKeysToScanFor(env: Record<string, unknown>, secretKeys: string[]): string[];
|
|
40
51
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
52
|
+
* Checks a line of text for likely secrets based on known prefixes and patterns.
|
|
53
|
+
* The function works by:
|
|
54
|
+
* 1. Splitting the line into tokens using quotes, whitespace, equals signs, colons, and commas as delimiters
|
|
55
|
+
* 2. For each token, checking if it matches our secret pattern:
|
|
56
|
+
* - Must start (^) with one of our known prefixes (e.g. aws_, github_pat_, etc)
|
|
57
|
+
* - Must be followed by at least MIN_CHARS_AFTER_PREFIX non-whitespace characters
|
|
58
|
+
* - Must extend to the end ($) of the token
|
|
44
59
|
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
60
|
+
* For example, given the line: secretKey='aws_123456789012345678'
|
|
61
|
+
* 1. It's split into tokens: ['secretKey', 'aws_123456789012345678']
|
|
62
|
+
* 2. Each token is checked against the regex pattern:
|
|
63
|
+
* - 'secretKey' doesn't match (doesn't start with a known prefix)
|
|
64
|
+
* - 'aws_123456789012345678' matches (starts with 'aws_' and has sufficient length)
|
|
65
|
+
*
|
|
66
|
+
* @param line The line of text to check
|
|
67
|
+
* @param file The file path where this line was found
|
|
68
|
+
* @param lineNumber The line number in the file
|
|
69
|
+
* @param omitValuesFromEnhancedScan Optional array of values to exclude from matching
|
|
70
|
+
* @returns Array of matches found in the line
|
|
48
71
|
*/
|
|
49
|
-
export declare function
|
|
72
|
+
export declare function findLikelySecrets({ line, file, lineNumber, omitValuesFromEnhancedScan, }: {
|
|
73
|
+
line: string;
|
|
74
|
+
file: string;
|
|
75
|
+
lineNumber: number;
|
|
76
|
+
omitValuesFromEnhancedScan?: unknown[];
|
|
77
|
+
}): MatchResult[];
|
|
50
78
|
/**
|
|
51
79
|
* Given the env and base directory, find all file paths to scan. It will look at the
|
|
52
80
|
* env vars to decide if it should omit certain paths.
|
|
@@ -67,7 +95,7 @@ export declare function getFilePathsToScan({ env, base }: {
|
|
|
67
95
|
* @param scanArgs {ScanArgs} scan options
|
|
68
96
|
* @returns promise with all of the scan results, if any
|
|
69
97
|
*/
|
|
70
|
-
export declare function scanFilesForKeyValues({ env, keys, filePaths, base }: ScanArgs): Promise<ScanResults>;
|
|
98
|
+
export declare function scanFilesForKeyValues({ env, keys, filePaths, base, enhancedScanning, omitValuesFromEnhancedScan, }: ScanArgs): Promise<ScanResults>;
|
|
71
99
|
/**
|
|
72
100
|
* ScanResults are all of the finds for all keys and their disparate locations. Scanning is
|
|
73
101
|
* async in streams so order can change a lot. Some matches are the result of an env var explictly being marked as secret,
|
|
@@ -79,7 +107,7 @@ export declare function scanFilesForKeyValues({ env, keys, filePaths, base }: Sc
|
|
|
79
107
|
* @param scanResults
|
|
80
108
|
* @returns
|
|
81
109
|
*/
|
|
82
|
-
export declare function groupScanResultsByKeyAndScanType(scanResults: ScanResults
|
|
110
|
+
export declare function groupScanResultsByKeyAndScanType(scanResults: ScanResults): {
|
|
83
111
|
secretMatches: {
|
|
84
112
|
[key: string]: MatchResult[];
|
|
85
113
|
};
|
|
@@ -3,7 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
import { createInterface } from 'node:readline';
|
|
4
4
|
import { fdir } from 'fdir';
|
|
5
5
|
import { minimatch } from 'minimatch';
|
|
6
|
-
import { LIKELY_SECRET_PREFIXES } from './secret_prefixes.js';
|
|
6
|
+
import { LIKELY_SECRET_PREFIXES, SAFE_LISTED_VALUES } from './secret_prefixes.js';
|
|
7
7
|
/**
|
|
8
8
|
* Determine if the user disabled scanning via env var
|
|
9
9
|
* @param env current envars
|
|
@@ -15,13 +15,32 @@ export function isSecretsScanningEnabled(env) {
|
|
|
15
15
|
}
|
|
16
16
|
return true;
|
|
17
17
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
/**
|
|
19
|
+
* Determine if the user disabled enhanced scanning via env var
|
|
20
|
+
* @param env current envars
|
|
21
|
+
* @returns
|
|
22
|
+
*/
|
|
23
|
+
export function isEnhancedSecretsScanningEnabled(env) {
|
|
24
|
+
if (env.ENHANCED_SECRETS_SCAN_ENABLED === false || env.ENHANCED_SECRETS_SCAN_ENABLED === 'false') {
|
|
25
|
+
return false;
|
|
21
26
|
}
|
|
22
|
-
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
export function getStringArrayFromEnvValue(env, envVarName) {
|
|
30
|
+
if (typeof env[envVarName] !== 'string') {
|
|
31
|
+
return [];
|
|
32
|
+
}
|
|
33
|
+
const omitKeys = env[envVarName]
|
|
34
|
+
.split(',')
|
|
23
35
|
.map((s) => s.trim())
|
|
24
36
|
.filter(Boolean);
|
|
37
|
+
return omitKeys;
|
|
38
|
+
}
|
|
39
|
+
export function getOmitValuesFromEnhancedScanForEnhancedScanFromEnv(env) {
|
|
40
|
+
return getStringArrayFromEnvValue(env, 'ENHANCED_SECRETS_SCAN_OMIT_VALUES');
|
|
41
|
+
}
|
|
42
|
+
function filterOmittedKeys(env, envKeys = []) {
|
|
43
|
+
const omitKeys = getStringArrayFromEnvValue(env, 'SECRETS_SCAN_OMIT_KEYS');
|
|
25
44
|
return envKeys.filter((key) => !omitKeys.includes(key));
|
|
26
45
|
}
|
|
27
46
|
/**
|
|
@@ -65,46 +84,64 @@ export function getSecretKeysToScanFor(env, secretKeys) {
|
|
|
65
84
|
const filteredSecretKeys = filterOmittedKeys(env, secretKeys);
|
|
66
85
|
return filteredSecretKeys.filter((key) => !isValueTrivial(env[key]));
|
|
67
86
|
}
|
|
87
|
+
// Most prefixes are 4-5 chars, so requiring 12 chars after ensures a reasonable secret length
|
|
88
|
+
const MIN_CHARS_AFTER_PREFIX = 12;
|
|
89
|
+
// Escape special regex characters (like $, *, +, etc) in prefixes so they're treated as literal characters
|
|
90
|
+
const prefixMatchingRegex = LIKELY_SECRET_PREFIXES.map((p) => p.replace(/[$*+?.()|[\]{}]/g, '\\$&')).join('|');
|
|
91
|
+
// Build regex pattern for matching secrets with various delimiters and quotes:
|
|
92
|
+
// (?:["'`]|^|[=:,]) - match either quotes, start of line, or delimiters (=:,) at the start
|
|
93
|
+
// Named capturing groups:
|
|
94
|
+
// - <token>: captures the entire secret value including its prefix
|
|
95
|
+
// - <prefix>: captures just the prefix part (e.g. 'aws_', 'github_pat_')
|
|
96
|
+
// (?:${prefixMatchingRegex}) - non-capturing group containing our escaped prefixes (e.g. aws_|github_pat_|etc)
|
|
97
|
+
// [^ "'`=:,]{${MIN_CHARS_AFTER_PREFIX}} - match exactly MIN_CHARS_AFTER_PREFIX chars after the prefix
|
|
98
|
+
// [^ "'`=:,]*? - lazily match any additional chars that aren't quotes/delimiters
|
|
99
|
+
// (?:["'`]|[ =:,]|$) - end with either quotes, delimiters, whitespace, or end of line
|
|
100
|
+
// gi - global and case insensitive flags
|
|
101
|
+
// Note: Using the global flag (g) means this regex object maintains state between executions.
|
|
102
|
+
// We would need to reset lastIndex to 0 if we wanted to reuse it on the same string multiple times.
|
|
103
|
+
const likelySecretRegex = new RegExp(`(?:["'\`]|^|[=:,]) *(?<token>(?<prefix>${prefixMatchingRegex})[^ "'\`=:,]{${MIN_CHARS_AFTER_PREFIX}}[^ "'\`=:,]*?)(?:["'\`]|[ =:,]|$)`, 'gi');
|
|
68
104
|
/**
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
105
|
+
* Checks a line of text for likely secrets based on known prefixes and patterns.
|
|
106
|
+
* The function works by:
|
|
107
|
+
* 1. Splitting the line into tokens using quotes, whitespace, equals signs, colons, and commas as delimiters
|
|
108
|
+
* 2. For each token, checking if it matches our secret pattern:
|
|
109
|
+
* - Must start (^) with one of our known prefixes (e.g. aws_, github_pat_, etc)
|
|
110
|
+
* - Must be followed by at least MIN_CHARS_AFTER_PREFIX non-whitespace characters
|
|
111
|
+
* - Must extend to the end ($) of the token
|
|
72
112
|
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
const nonSecretKeys = Object.keys(env).filter((key) => !secretKeys.includes(key));
|
|
79
|
-
const filteredNonSecretKeys = filterOmittedKeys(env, nonSecretKeys);
|
|
80
|
-
const nonSecretKeysToScanFor = filteredNonSecretKeys.filter((key) => {
|
|
81
|
-
const val = env[key];
|
|
82
|
-
if (isValueTrivial(val)) {
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
return isLikelySecretValue(val);
|
|
86
|
-
});
|
|
87
|
-
return nonSecretKeysToScanFor;
|
|
88
|
-
}
|
|
89
|
-
const LIKELY_SECRET_MIN_LENGTH = 16;
|
|
90
|
-
/**
|
|
91
|
-
* When the enhanced secret scan is run, we check any env vars _not_ marked as secret if they are highly likely to be secret values.
|
|
92
|
-
* For now, this means the value is a string of at least 16 chars and starts with one of the known prefixes.
|
|
113
|
+
* For example, given the line: secretKey='aws_123456789012345678'
|
|
114
|
+
* 1. It's split into tokens: ['secretKey', 'aws_123456789012345678']
|
|
115
|
+
* 2. Each token is checked against the regex pattern:
|
|
116
|
+
* - 'secretKey' doesn't match (doesn't start with a known prefix)
|
|
117
|
+
* - 'aws_123456789012345678' matches (starts with 'aws_' and has sufficient length)
|
|
93
118
|
*
|
|
94
|
-
* @param
|
|
95
|
-
* @
|
|
119
|
+
* @param line The line of text to check
|
|
120
|
+
* @param file The file path where this line was found
|
|
121
|
+
* @param lineNumber The line number in the file
|
|
122
|
+
* @param omitValuesFromEnhancedScan Optional array of values to exclude from matching
|
|
123
|
+
* @returns Array of matches found in the line
|
|
96
124
|
*/
|
|
97
|
-
function
|
|
98
|
-
if (
|
|
99
|
-
return
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
125
|
+
export function findLikelySecrets({ line, file, lineNumber, omitValuesFromEnhancedScan = [], }) {
|
|
126
|
+
if (!line)
|
|
127
|
+
return [];
|
|
128
|
+
const matches = [];
|
|
129
|
+
let match;
|
|
130
|
+
const allOmittedValues = [...omitValuesFromEnhancedScan, ...SAFE_LISTED_VALUES];
|
|
131
|
+
while ((match = likelySecretRegex.exec(line)) !== null) {
|
|
132
|
+
const token = match.groups?.token;
|
|
133
|
+
const prefix = match.groups?.prefix;
|
|
134
|
+
if (!token || !prefix || allOmittedValues.includes(token)) {
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
matches.push({
|
|
138
|
+
file,
|
|
139
|
+
lineNumber,
|
|
140
|
+
key: prefix,
|
|
141
|
+
enhancedMatch: true,
|
|
142
|
+
});
|
|
106
143
|
}
|
|
107
|
-
return
|
|
144
|
+
return matches;
|
|
108
145
|
}
|
|
109
146
|
/**
|
|
110
147
|
* Given the env and base directory, find all file paths to scan. It will look at the
|
|
@@ -167,7 +204,7 @@ const omitPathMatches = (relativePath, omitPaths) => {
|
|
|
167
204
|
* @param scanArgs {ScanArgs} scan options
|
|
168
205
|
* @returns promise with all of the scan results, if any
|
|
169
206
|
*/
|
|
170
|
-
export async function scanFilesForKeyValues({ env, keys, filePaths, base }) {
|
|
207
|
+
export async function scanFilesForKeyValues({ env, keys, filePaths, base, enhancedScanning, omitValuesFromEnhancedScan = [], }) {
|
|
171
208
|
const scanResults = {
|
|
172
209
|
matches: [],
|
|
173
210
|
scannedFilesCount: 0,
|
|
@@ -194,7 +231,7 @@ export async function scanFilesForKeyValues({ env, keys, filePaths, base }) {
|
|
|
194
231
|
const chunkSize = 200;
|
|
195
232
|
const batch = filePaths.splice(0, chunkSize);
|
|
196
233
|
settledPromises = settledPromises.concat(await Promise.allSettled(batch.map((file) => {
|
|
197
|
-
return searchStream(base, file, keyValues);
|
|
234
|
+
return searchStream({ basePath: base, file, keyValues, enhancedScanning, omitValuesFromEnhancedScan });
|
|
198
235
|
})));
|
|
199
236
|
}
|
|
200
237
|
settledPromises.forEach((result) => {
|
|
@@ -204,7 +241,7 @@ export async function scanFilesForKeyValues({ env, keys, filePaths, base }) {
|
|
|
204
241
|
});
|
|
205
242
|
return scanResults;
|
|
206
243
|
}
|
|
207
|
-
const searchStream = (basePath, file, keyValues) => {
|
|
244
|
+
const searchStream = ({ basePath, file, keyValues, enhancedScanning, omitValuesFromEnhancedScan = [], }) => {
|
|
208
245
|
return new Promise((resolve, reject) => {
|
|
209
246
|
const filePath = path.resolve(basePath, file);
|
|
210
247
|
const inStream = createReadStream(filePath);
|
|
@@ -232,6 +269,9 @@ const searchStream = (basePath, file, keyValues) => {
|
|
|
232
269
|
// and match what an IDE would show for a line number.
|
|
233
270
|
lineNumber++;
|
|
234
271
|
if (typeof line === 'string') {
|
|
272
|
+
if (enhancedScanning) {
|
|
273
|
+
matches.push(...findLikelySecrets({ line, file, lineNumber, omitValuesFromEnhancedScan }));
|
|
274
|
+
}
|
|
235
275
|
if (maxMultiLineCount > 1) {
|
|
236
276
|
lines.push(line);
|
|
237
277
|
}
|
|
@@ -247,6 +287,7 @@ const searchStream = (basePath, file, keyValues) => {
|
|
|
247
287
|
file,
|
|
248
288
|
lineNumber,
|
|
249
289
|
key: getKeyForValue(valVariant),
|
|
290
|
+
enhancedMatch: false,
|
|
250
291
|
});
|
|
251
292
|
return;
|
|
252
293
|
}
|
|
@@ -289,6 +330,7 @@ const searchStream = (basePath, file, keyValues) => {
|
|
|
289
330
|
file,
|
|
290
331
|
lineNumber: lineNumber - lines.length + 1,
|
|
291
332
|
key: getKeyForValue(valVariant),
|
|
333
|
+
enhancedMatch: false,
|
|
292
334
|
});
|
|
293
335
|
return;
|
|
294
336
|
}
|
|
@@ -321,12 +363,11 @@ const searchStream = (basePath, file, keyValues) => {
|
|
|
321
363
|
* @param scanResults
|
|
322
364
|
* @returns
|
|
323
365
|
*/
|
|
324
|
-
export function groupScanResultsByKeyAndScanType(scanResults
|
|
366
|
+
export function groupScanResultsByKeyAndScanType(scanResults) {
|
|
325
367
|
const secretMatchesByKeys = {};
|
|
326
368
|
const enhancedSecretMatchesByKeys = {};
|
|
327
369
|
scanResults.matches.forEach((matchResult) => {
|
|
328
|
-
|
|
329
|
-
if (isEnhancedCheck) {
|
|
370
|
+
if (matchResult.enhancedMatch) {
|
|
330
371
|
if (!enhancedSecretMatchesByKeys[matchResult.key]) {
|
|
331
372
|
enhancedSecretMatchesByKeys[matchResult.key] = [];
|
|
332
373
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@netlify/build",
|
|
3
|
-
"version": "33.1
|
|
3
|
+
"version": "33.2.1",
|
|
4
4
|
"description": "Netlify build module",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": "./lib/index.js",
|
|
@@ -69,7 +69,7 @@
|
|
|
69
69
|
"@bugsnag/js": "^8.0.0",
|
|
70
70
|
"@netlify/blobs": "^8.2.0",
|
|
71
71
|
"@netlify/cache-utils": "^6.0.2",
|
|
72
|
-
"@netlify/config": "^23.0.
|
|
72
|
+
"@netlify/config": "^23.0.7",
|
|
73
73
|
"@netlify/edge-bundler": "14.0.4",
|
|
74
74
|
"@netlify/framework-info": "^10.0.4",
|
|
75
75
|
"@netlify/functions-utils": "^6.0.6",
|
|
@@ -97,7 +97,6 @@
|
|
|
97
97
|
"map-obj": "^5.0.0",
|
|
98
98
|
"memoize-one": "^6.0.0",
|
|
99
99
|
"minimatch": "^9.0.4",
|
|
100
|
-
"node-fetch": "^3.3.2",
|
|
101
100
|
"os-name": "^6.0.0",
|
|
102
101
|
"p-event": "^6.0.0",
|
|
103
102
|
"p-every": "^2.0.0",
|
|
@@ -159,5 +158,5 @@
|
|
|
159
158
|
"engines": {
|
|
160
159
|
"node": ">=18.14.0"
|
|
161
160
|
},
|
|
162
|
-
"gitHead": "
|
|
161
|
+
"gitHead": "f85f7e9a41f2c1698a320402d09072ee04f4dc6d"
|
|
163
162
|
}
|