@netlify/build 29.14.1 → 29.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/core/build.js +2 -2
- package/lib/core/flags.js +10 -0
- package/lib/core/main.js +3 -3
- package/lib/core/normalize_flags.js +9 -1
- package/lib/error/type.js +5 -0
- package/lib/log/messages/core_steps.js +23 -1
- package/lib/plugins_core/secrets_scanning/index.js +61 -0
- package/lib/plugins_core/secrets_scanning/utils.js +258 -0
- package/lib/steps/get.js +2 -1
- package/lib/steps/run_step.js +3 -2
- package/lib/tracing/main.js +11 -11
- package/package.json +6 -7
package/lib/core/build.js
CHANGED
|
@@ -30,8 +30,8 @@ export const startBuild = function (flags) {
|
|
|
30
30
|
}
|
|
31
31
|
const { bugsnagKey, tracingOpts, debug, systemLogFile, ...flagsA } = normalizeFlags(flags, logs);
|
|
32
32
|
const errorMonitor = startErrorMonitor({ flags: { tracingOpts, debug, systemLogFile, ...flagsA }, logs, bugsnagKey });
|
|
33
|
-
startTracing(tracingOpts, getSystemLogger(logs, debug, systemLogFile));
|
|
34
|
-
return { ...flagsA, debug, systemLogFile, errorMonitor, logs, timers };
|
|
33
|
+
const rootTracingContext = startTracing(tracingOpts, getSystemLogger(logs, debug, systemLogFile));
|
|
34
|
+
return { ...flagsA, rootTracingContext, debug, systemLogFile, errorMonitor, logs, timers };
|
|
35
35
|
};
|
|
36
36
|
const tExecBuild = async function ({ config, defaultConfig, cachedConfig, cachedConfigPath, outputConfigPath, cwd, repositoryRoot, apiHost, token, siteId, context, branch, baseRelDir, env: envOpt, debug, systemLogFile, verbose, nodePath, functionsDistDir, edgeFunctionsDistDir, cacheDir, dry, mode, offline, deployId, buildId, testOpts, errorMonitor, errorParams, logs, timers, buildbotServerSocket, sendStatus, saveConfig, featureFlags, timeline, devCommand, quiet, framework, explicitSecretKeys, }) {
|
|
37
37
|
const configOpts = getConfigOpts({
|
package/lib/core/flags.js
CHANGED
|
@@ -198,6 +198,16 @@ Default: false`,
|
|
|
198
198
|
describe: 'Enable distributed tracing for build',
|
|
199
199
|
hidden: true,
|
|
200
200
|
},
|
|
201
|
+
'tracing.apiKey': {
|
|
202
|
+
string: true,
|
|
203
|
+
describe: 'API Key for the tracing backend provider',
|
|
204
|
+
hidden: true,
|
|
205
|
+
},
|
|
206
|
+
'tracing.httpProtocol': {
|
|
207
|
+
string: true,
|
|
208
|
+
describe: 'Traces backend protocol. HTTP or HTTPS.',
|
|
209
|
+
hidden: true,
|
|
210
|
+
},
|
|
201
211
|
'tracing.host': {
|
|
202
212
|
string: true,
|
|
203
213
|
describe: 'Traces backend host',
|
package/lib/core/main.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { trace } from '@opentelemetry/api';
|
|
1
|
+
import { trace, context } from '@opentelemetry/api';
|
|
2
2
|
import { handleBuildError } from '../error/handle.js';
|
|
3
3
|
import { reportError } from '../error/report.js';
|
|
4
4
|
import { getSystemLogger } from '../log/logger.js';
|
|
@@ -19,7 +19,7 @@ const tracer = trace.getTracer('core');
|
|
|
19
19
|
* @param flags - build configuration CLI flags
|
|
20
20
|
*/
|
|
21
21
|
export default async function buildSite(flags = {}) {
|
|
22
|
-
const { errorMonitor, framework, mode, logs, debug, systemLogFile, testOpts, statsdOpts, dry, telemetry, buildId, deployId, ...flagsA } = startBuild(flags);
|
|
22
|
+
const { errorMonitor, framework, mode, logs, debug, systemLogFile, testOpts, statsdOpts, dry, telemetry, buildId, deployId, rootTracingContext, ...flagsA } = startBuild(flags);
|
|
23
23
|
const errorParams = { errorMonitor, mode, logs, debug, testOpts };
|
|
24
24
|
const systemLog = getSystemLogger(logs, debug, systemLogFile);
|
|
25
25
|
const attributes = {
|
|
@@ -28,7 +28,7 @@ export default async function buildSite(flags = {}) {
|
|
|
28
28
|
'deploy.context': flagsA.context,
|
|
29
29
|
'site.id': flagsA.siteId,
|
|
30
30
|
};
|
|
31
|
-
const rootCtx = setMultiSpanAttributes(attributes);
|
|
31
|
+
const rootCtx = context.with(rootTracingContext, () => setMultiSpanAttributes(attributes));
|
|
32
32
|
return await tracer.startActiveSpan('exec-build', {}, rootCtx, async (span) => {
|
|
33
33
|
try {
|
|
34
34
|
const { pluginsOptions, netlifyConfig: netlifyConfigA, siteInfo, userNodeVersion, stepsCount, timers, durationNs, configMutations, metrics, } = await execBuild({
|
|
@@ -8,6 +8,7 @@ const DEFAULT_FUNCTIONS_DIST = '.netlify/functions/';
|
|
|
8
8
|
const DEFAULT_CACHE_DIR = '.netlify/cache/';
|
|
9
9
|
const DEFAULT_STATSD_PORT = 8125;
|
|
10
10
|
const DEFAULT_OTEL_TRACING_PORT = 4317;
|
|
11
|
+
const DEFAULT_OTEL_ENDPOINT_PROTOCOL = 'http';
|
|
11
12
|
/** Normalize CLI flags */
|
|
12
13
|
export const normalizeFlags = function (flags, logs) {
|
|
13
14
|
const rawFlags = removeFalsy(flags);
|
|
@@ -53,7 +54,14 @@ const getDefaultFlags = function ({ env: envOpt = {} }, combinedEnv) {
|
|
|
53
54
|
testOpts: {},
|
|
54
55
|
featureFlags: DEFAULT_FEATURE_FLAGS,
|
|
55
56
|
statsd: { port: DEFAULT_STATSD_PORT },
|
|
56
|
-
tracing
|
|
57
|
+
// tracing.apiKey defaults to '-' else we'll get warning logs if not using
|
|
58
|
+
// honeycomb directly - https://github.com/honeycombio/honeycomb-opentelemetry-node/issues/201
|
|
59
|
+
tracing: {
|
|
60
|
+
enabled: false,
|
|
61
|
+
apiKey: '-',
|
|
62
|
+
httpProtocol: DEFAULT_OTEL_ENDPOINT_PROTOCOL,
|
|
63
|
+
port: DEFAULT_OTEL_TRACING_PORT,
|
|
64
|
+
},
|
|
57
65
|
timeline: 'build',
|
|
58
66
|
quiet: false,
|
|
59
67
|
};
|
package/lib/error/type.js
CHANGED
|
@@ -92,6 +92,11 @@ const TYPES = {
|
|
|
92
92
|
locationType: 'functionsBundling',
|
|
93
93
|
severity: 'info',
|
|
94
94
|
},
|
|
95
|
+
secretScanningFoundSecrets: {
|
|
96
|
+
title: 'Secrets scanning detected secrets in files during build.',
|
|
97
|
+
stackType: 'none',
|
|
98
|
+
severity: 'error',
|
|
99
|
+
},
|
|
95
100
|
// Plugin called `utils.build.failBuild()`
|
|
96
101
|
failBuild: {
|
|
97
102
|
title: ({ location: { packageName } }) => `Plugin "${packageName}" failed`,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
|
-
import { log, logArray, logErrorSubHeader, logWarningSubHeader } from '../logger.js';
|
|
2
|
+
import { log, logArray, logError, logErrorSubHeader, logWarningSubHeader } from '../logger.js';
|
|
3
3
|
import { THEME } from '../theme.js';
|
|
4
4
|
const logBundleResultFunctions = ({ functions, headerMessage, logs, error }) => {
|
|
5
5
|
const functionNames = functions.map(({ path: functionPath }) => path.basename(functionPath));
|
|
@@ -73,3 +73,25 @@ const logModulesWithDynamicImports = ({ logs, modulesWithDynamicImports }) => {
|
|
|
73
73
|
Visit https://ntl.fyi/dynamic-imports for more information.
|
|
74
74
|
`);
|
|
75
75
|
};
|
|
76
|
+
export const logSecretsScanSkipMessage = function (logs, msg) {
|
|
77
|
+
log(logs, msg, { color: THEME.warningHighlightWords });
|
|
78
|
+
};
|
|
79
|
+
export const logSecretsScanSuccessMessage = function (logs, msg) {
|
|
80
|
+
log(logs, msg, { color: THEME.highlightWords });
|
|
81
|
+
};
|
|
82
|
+
export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) {
|
|
83
|
+
logErrorSubHeader(logs, `Secrets scanning found ${scanResults.matches.length} instance(s) of secrets in build output or repo code.\n`);
|
|
84
|
+
Object.keys(groupedResults).forEach((key) => {
|
|
85
|
+
logError(logs, `Secret env var "${key}"'s value detected:`);
|
|
86
|
+
groupedResults[key]
|
|
87
|
+
.sort((a, b) => {
|
|
88
|
+
return a.file > b.file ? 0 : 1;
|
|
89
|
+
})
|
|
90
|
+
.forEach(({ lineNumber, file }) => {
|
|
91
|
+
logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true });
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
logError(logs, `\nTo prevent exposing secrets, the build will fail until these secret values are not found in build output or repo files.`);
|
|
95
|
+
logError(logs, `If these are expected, use SECRETS_SCAN_OMIT_PATHS, SECRETS_SCAN_OMIT_KEYS, or SECRETS_SCAN_ENABLED to prevent detecting.`);
|
|
96
|
+
logError(logs, `See the Netlify Docs for more information on secrets scanning.`);
|
|
97
|
+
};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { addErrorInfo } from '../../error/info.js';
|
|
2
|
+
import { logSecretsScanFailBuildMessage, logSecretsScanSkipMessage, logSecretsScanSuccessMessage, } from '../../log/messages/core_steps.js';
|
|
3
|
+
import { getFilePathsToScan, getSecretKeysToScanFor, groupScanResultsByKey, isSecretsScanningEnabled, scanFilesForKeyValues, } from './utils.js';
|
|
4
|
+
const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecretKeys, systemLog }) {
|
|
5
|
+
const stepResults = {};
|
|
6
|
+
const passedSecretKeys = (explicitSecretKeys || '').split(',');
|
|
7
|
+
const envVars = netlifyConfig.build.environment;
|
|
8
|
+
systemLog({ envVars, passedSecretKeys });
|
|
9
|
+
if (!isSecretsScanningEnabled(envVars)) {
|
|
10
|
+
logSecretsScanSkipMessage(logs, 'Secrets scanning disabled via SECRETS_SCAN_ENABLED flag set to false.');
|
|
11
|
+
return stepResults;
|
|
12
|
+
}
|
|
13
|
+
const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys);
|
|
14
|
+
systemLog({ keysToSearchFor });
|
|
15
|
+
if (keysToSearchFor.length === 0) {
|
|
16
|
+
logSecretsScanSkipMessage(logs, 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.');
|
|
17
|
+
return stepResults;
|
|
18
|
+
}
|
|
19
|
+
// buildDir is the repository root or the base folder
|
|
20
|
+
// The scanning will look at builddir so that it can review both repo pulled files
|
|
21
|
+
// and post build files
|
|
22
|
+
const filePaths = await getFilePathsToScan({ env: envVars, base: buildDir });
|
|
23
|
+
systemLog({ buildDir, filePaths });
|
|
24
|
+
if (filePaths.length === 0) {
|
|
25
|
+
logSecretsScanSkipMessage(logs, 'Secrets scanning skipped because there are no files or all files were omitted with SECRETS_SCAN_OMIT_PATHS env var setting.');
|
|
26
|
+
return stepResults;
|
|
27
|
+
}
|
|
28
|
+
const scanResults = await scanFilesForKeyValues({
|
|
29
|
+
env: envVars,
|
|
30
|
+
keys: keysToSearchFor,
|
|
31
|
+
base: buildDir,
|
|
32
|
+
filePaths,
|
|
33
|
+
});
|
|
34
|
+
if (scanResults.matches.length === 0) {
|
|
35
|
+
logSecretsScanSuccessMessage(logs, 'Secrets scanning complete. No secrets detected in build output or repo code.');
|
|
36
|
+
return stepResults;
|
|
37
|
+
}
|
|
38
|
+
// at this point we have found matching secrets
|
|
39
|
+
// Output the results and fail the build
|
|
40
|
+
logSecretsScanFailBuildMessage({ logs, scanResults, groupedResults: groupScanResultsByKey(scanResults) });
|
|
41
|
+
const error = new Error(`Secrets scanning found secrets in build.`);
|
|
42
|
+
addErrorInfo(error, { type: 'secretScanningFoundSecrets' });
|
|
43
|
+
throw error;
|
|
44
|
+
};
|
|
45
|
+
// We run this core step if the build was run with explicit secret keys. This
|
|
46
|
+
// is passed from BB to build so only accounts that are allowed to have explicit
|
|
47
|
+
// secrets and actually have them will have them.
|
|
48
|
+
const hasExplicitSecretsKeys = function ({ explicitSecretKeys }) {
|
|
49
|
+
if (typeof explicitSecretKeys !== 'string') {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
return explicitSecretKeys.length > 0;
|
|
53
|
+
};
|
|
54
|
+
export const scanForSecrets = {
|
|
55
|
+
event: 'onPostBuild',
|
|
56
|
+
coreStep,
|
|
57
|
+
coreStepId: 'secrets_scanning',
|
|
58
|
+
coreStepName: 'Secrets scanning',
|
|
59
|
+
coreStepDescription: () => 'Scanning for secrets in code and build output.',
|
|
60
|
+
condition: hasExplicitSecretsKeys,
|
|
61
|
+
};
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import { createReadStream } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { createInterface } from 'node:readline';
|
|
4
|
+
import { fdir } from 'fdir';
|
|
5
|
+
/**
|
|
6
|
+
* Determine if the user disabled scanning via env var
|
|
7
|
+
* @param env current envars
|
|
8
|
+
* @returns
|
|
9
|
+
*/
|
|
10
|
+
export function isSecretsScanningEnabled(env) {
|
|
11
|
+
if (env.SECRETS_SCAN_ENABLED === false || env.SECRETS_SCAN_ENABLED === 'false') {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* given the explicit secret keys and env vars, return the list of secret keys which have non-empty or non-trivial values. This
|
|
18
|
+
* will also filter out keys passed in the SECRETS_SCAN_OMIT_KEYS env var.
|
|
19
|
+
*
|
|
20
|
+
* non-trivial values are values that are:
|
|
21
|
+
* - >4 characters/digits
|
|
22
|
+
* - not booleans
|
|
23
|
+
*
|
|
24
|
+
* @param env env vars list
|
|
25
|
+
* @param secretKeys
|
|
26
|
+
* @returns string[]
|
|
27
|
+
*/
|
|
28
|
+
export function getSecretKeysToScanFor(env, secretKeys) {
|
|
29
|
+
let omitKeys = [];
|
|
30
|
+
if (typeof env.SECRETS_SCAN_OMIT_KEYS === 'string') {
|
|
31
|
+
omitKeys = env.SECRETS_SCAN_OMIT_KEYS.split(',')
|
|
32
|
+
.map((s) => s.trim())
|
|
33
|
+
.filter(Boolean);
|
|
34
|
+
}
|
|
35
|
+
return secretKeys.filter((key) => {
|
|
36
|
+
if (omitKeys.includes(key)) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
const val = env[key];
|
|
40
|
+
if (typeof val === 'string') {
|
|
41
|
+
// string forms of booleans
|
|
42
|
+
if (val === 'true' || val === 'false') {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
// non-trivial/non-empty values only
|
|
46
|
+
return val.trim().length > 4;
|
|
47
|
+
}
|
|
48
|
+
else if (typeof val === 'boolean') {
|
|
49
|
+
// booleans are trivial values
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
else if (typeof val === 'number' || typeof val === 'object') {
|
|
53
|
+
return JSON.stringify(val).length > 4;
|
|
54
|
+
}
|
|
55
|
+
return !!val;
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Given the env and base directory, find all file paths to scan. It will look at the
|
|
60
|
+
* env vars to decide if it should omit certain paths.
|
|
61
|
+
*
|
|
62
|
+
* @param options
|
|
63
|
+
* @returns string[] of relative paths from base of files that should be searched
|
|
64
|
+
*/
|
|
65
|
+
export async function getFilePathsToScan({ env, base }) {
|
|
66
|
+
let files = await new fdir().withRelativePaths().crawl(base).withPromise();
|
|
67
|
+
// normalize the path separators to all use the forward slash
|
|
68
|
+
// this is needed for windows machines and snapshot tests consistency.
|
|
69
|
+
files = files.map((f) => f.split(path.sep).join('/'));
|
|
70
|
+
let omitPaths = [];
|
|
71
|
+
if (typeof env.SECRETS_SCAN_OMIT_PATHS === 'string') {
|
|
72
|
+
omitPaths = env.SECRETS_SCAN_OMIT_PATHS.split(',')
|
|
73
|
+
.map((s) => s.trim())
|
|
74
|
+
.filter(Boolean);
|
|
75
|
+
}
|
|
76
|
+
if (omitPaths.length > 0) {
|
|
77
|
+
files = files.filter((relativePath) => !omitPathMatches(relativePath, omitPaths));
|
|
78
|
+
}
|
|
79
|
+
return files;
|
|
80
|
+
}
|
|
81
|
+
// omit paths are relative path substrings.
|
|
82
|
+
const omitPathMatches = (relativePath, omitPaths) => omitPaths.some((oPath) => relativePath.startsWith(oPath));
|
|
83
|
+
/**
|
|
84
|
+
* Given the env vars, the current keys, paths, etc. Look across the provided files to find the values
|
|
85
|
+
* of the secrets based on the keys provided. It will process files separately in different read streams.
|
|
86
|
+
* The values that it looks for will be a unique set of plaintext, base64 encoded, and uri encoded permutations
|
|
87
|
+
* of each value - to catch common permutations that occur post build.
|
|
88
|
+
*
|
|
89
|
+
* @param scanArgs {ScanArgs} scan options
|
|
90
|
+
* @returns promise with all of the scan results, if any
|
|
91
|
+
*/
|
|
92
|
+
export async function scanFilesForKeyValues({ env, keys, filePaths, base }) {
|
|
93
|
+
const scanResults = {
|
|
94
|
+
matches: [],
|
|
95
|
+
};
|
|
96
|
+
const keyValues = keys.reduce((kvs, key) => {
|
|
97
|
+
let val = env[key];
|
|
98
|
+
if (typeof val === 'number' || typeof val === 'object') {
|
|
99
|
+
val = JSON.stringify(val);
|
|
100
|
+
}
|
|
101
|
+
if (typeof val === 'string') {
|
|
102
|
+
// to detect the secrets effectively
|
|
103
|
+
// normalize the value so that we remove leading and
|
|
104
|
+
// ending whitespace and newline characters
|
|
105
|
+
const normalizedVal = val.replace(/^\s*/, '').replace(/\s*$/, '');
|
|
106
|
+
kvs[key] = Array.from(new Set([normalizedVal, Buffer.from(normalizedVal).toString('base64'), encodeURIComponent(normalizedVal)]));
|
|
107
|
+
}
|
|
108
|
+
return kvs;
|
|
109
|
+
}, {});
|
|
110
|
+
const settledPromises = await Promise.allSettled(filePaths.map((file) => {
|
|
111
|
+
return searchStream(base, file, keyValues);
|
|
112
|
+
}));
|
|
113
|
+
settledPromises.forEach((result) => {
|
|
114
|
+
if (result.status === 'fulfilled' && result.value?.length > 0) {
|
|
115
|
+
scanResults.matches = scanResults.matches.concat(result.value);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
return scanResults;
|
|
119
|
+
}
|
|
120
|
+
const searchStream = (basePath, file, keyValues) => {
|
|
121
|
+
return new Promise((resolve) => {
|
|
122
|
+
const filePath = path.resolve(basePath, file);
|
|
123
|
+
const inStream = createReadStream(filePath);
|
|
124
|
+
const rl = createInterface(inStream);
|
|
125
|
+
const matches = [];
|
|
126
|
+
const keyVals = [].concat(...Object.values(keyValues));
|
|
127
|
+
function getKeyForValue(val) {
|
|
128
|
+
let key = '';
|
|
129
|
+
for (const [secretKeyName, valuePermutations] of Object.entries(keyValues)) {
|
|
130
|
+
if (valuePermutations.includes(val)) {
|
|
131
|
+
key = secretKeyName;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return key;
|
|
135
|
+
}
|
|
136
|
+
// how many lines is the largest multiline string
|
|
137
|
+
let maxMultiLineCount = 1;
|
|
138
|
+
keyVals.forEach((valVariant) => {
|
|
139
|
+
maxMultiLineCount = Math.max(maxMultiLineCount, valVariant.split('\n').length);
|
|
140
|
+
});
|
|
141
|
+
// search if not multi line or current accumulated lines length is less than num of lines
|
|
142
|
+
const lines = [];
|
|
143
|
+
let lineNumber = 0;
|
|
144
|
+
rl.on('line', function (line) {
|
|
145
|
+
// iterating here so the first line will always appear as line 1 to be human friendly
|
|
146
|
+
// and match what an IDE would show for a line number.
|
|
147
|
+
lineNumber++;
|
|
148
|
+
if (typeof line === 'string') {
|
|
149
|
+
if (maxMultiLineCount > 1) {
|
|
150
|
+
lines.push(line);
|
|
151
|
+
}
|
|
152
|
+
// only track the max number of lines needed to match our largest
|
|
153
|
+
// multiline value. If we get above that remove the first value from the list
|
|
154
|
+
if (lines.length > maxMultiLineCount) {
|
|
155
|
+
lines.shift();
|
|
156
|
+
}
|
|
157
|
+
keyVals.forEach((valVariant) => {
|
|
158
|
+
// matching of single/whole values
|
|
159
|
+
if (line.search(new RegExp(valVariant)) >= 0) {
|
|
160
|
+
matches.push({
|
|
161
|
+
file,
|
|
162
|
+
lineNumber,
|
|
163
|
+
key: getKeyForValue(valVariant),
|
|
164
|
+
});
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
// matching of multiline values
|
|
168
|
+
if (isMultiLineVal(valVariant)) {
|
|
169
|
+
// drop empty values at beginning and end
|
|
170
|
+
const multiStringLines = valVariant.split('\n');
|
|
171
|
+
// drop early if we don't have enough lines for all values
|
|
172
|
+
if (lines.length < multiStringLines.length) {
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
let stillMatches = true;
|
|
176
|
+
let fullMatch = false;
|
|
177
|
+
multiStringLines.forEach((valLine, valIndex) => {
|
|
178
|
+
if (valIndex === 0) {
|
|
179
|
+
// first lines have to end with the line value
|
|
180
|
+
if (!lines[valIndex].endsWith(valLine)) {
|
|
181
|
+
stillMatches = false;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
else if (valIndex !== multiStringLines.length - 1) {
|
|
185
|
+
// middle lines have to have full line match
|
|
186
|
+
// middle lines
|
|
187
|
+
if (lines[valIndex] !== valLine) {
|
|
188
|
+
stillMatches = false;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
// last lines have start with the value
|
|
193
|
+
if (!lines[valIndex].startsWith(valLine)) {
|
|
194
|
+
stillMatches = false;
|
|
195
|
+
}
|
|
196
|
+
if (stillMatches === true) {
|
|
197
|
+
fullMatch = true;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
if (fullMatch) {
|
|
202
|
+
matches.push({
|
|
203
|
+
file,
|
|
204
|
+
lineNumber: lineNumber - lines.length + 1,
|
|
205
|
+
key: getKeyForValue(valVariant),
|
|
206
|
+
});
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
rl.on('close', function () {
|
|
214
|
+
resolve(matches);
|
|
215
|
+
});
|
|
216
|
+
});
|
|
217
|
+
};
|
|
218
|
+
/**
|
|
219
|
+
* ScanResults are all of the finds for all keys and their disparate locations. Scanning is
|
|
220
|
+
* async in streams so order can change a lot. This function groups the results into an object
|
|
221
|
+
* where the keys are the env var keys and the values are all match results for that key
|
|
222
|
+
*
|
|
223
|
+
* @param scanResults
|
|
224
|
+
* @returns
|
|
225
|
+
*/
|
|
226
|
+
export function groupScanResultsByKey(scanResults) {
|
|
227
|
+
const matchesByKeys = {};
|
|
228
|
+
scanResults.matches.forEach((matchResult) => {
|
|
229
|
+
if (!matchesByKeys[matchResult.key]) {
|
|
230
|
+
matchesByKeys[matchResult.key] = [];
|
|
231
|
+
}
|
|
232
|
+
matchesByKeys[matchResult.key].push(matchResult);
|
|
233
|
+
});
|
|
234
|
+
// sort results to get a consistent output and logically ordered match results
|
|
235
|
+
Object.keys(matchesByKeys).forEach((key) => {
|
|
236
|
+
matchesByKeys[key].sort((a, b) => {
|
|
237
|
+
// sort by file name first
|
|
238
|
+
if (a.file > b.file) {
|
|
239
|
+
return 1;
|
|
240
|
+
}
|
|
241
|
+
// sort by line number second
|
|
242
|
+
if (a.file === b.file) {
|
|
243
|
+
if (a.lineNumber > b.lineNumber) {
|
|
244
|
+
return 1;
|
|
245
|
+
}
|
|
246
|
+
if (a.lineNumber === b.lineNumber) {
|
|
247
|
+
return 0;
|
|
248
|
+
}
|
|
249
|
+
return -1;
|
|
250
|
+
}
|
|
251
|
+
return -1;
|
|
252
|
+
});
|
|
253
|
+
});
|
|
254
|
+
return matchesByKeys;
|
|
255
|
+
}
|
|
256
|
+
function isMultiLineVal(v) {
|
|
257
|
+
return typeof v === 'string' && v.includes('\n');
|
|
258
|
+
}
|
package/lib/steps/get.js
CHANGED
|
@@ -4,6 +4,7 @@ import { deploySite } from '../plugins_core/deploy/index.js';
|
|
|
4
4
|
import { bundleEdgeFunctions } from '../plugins_core/edge_functions/index.js';
|
|
5
5
|
import { bundleFunctions } from '../plugins_core/functions/index.js';
|
|
6
6
|
import { saveArtifacts } from '../plugins_core/save_artifacts/index.js';
|
|
7
|
+
import { scanForSecrets } from '../plugins_core/secrets_scanning/index.js';
|
|
7
8
|
// Get all build steps
|
|
8
9
|
export const getSteps = function (steps) {
|
|
9
10
|
const stepsA = addCoreSteps(steps);
|
|
@@ -28,7 +29,7 @@ export const getDevSteps = function (command, steps) {
|
|
|
28
29
|
return { steps: sortedSteps, events };
|
|
29
30
|
};
|
|
30
31
|
const addCoreSteps = function (steps) {
|
|
31
|
-
return [buildCommandCore, ...steps, bundleFunctions, bundleEdgeFunctions, deploySite, saveArtifacts];
|
|
32
|
+
return [buildCommandCore, ...steps, bundleFunctions, bundleEdgeFunctions, scanForSecrets, deploySite, saveArtifacts];
|
|
32
33
|
};
|
|
33
34
|
// Sort plugin steps by event order.
|
|
34
35
|
const sortSteps = function (steps, events) {
|
package/lib/steps/run_step.js
CHANGED
|
@@ -14,7 +14,6 @@ export const runStep = async function ({ event, childProcess, packageName, coreS
|
|
|
14
14
|
// Add relevant attributes to the upcoming span context
|
|
15
15
|
const attributes = {
|
|
16
16
|
'build.execution.step.name': coreStepName,
|
|
17
|
-
'build.execution.step.description': coreStepDescription,
|
|
18
17
|
'build.execution.step.package_name': packageName,
|
|
19
18
|
'build.execution.step.id': coreStepId,
|
|
20
19
|
'build.execution.step.loaded_from': loadedFrom,
|
|
@@ -22,7 +21,9 @@ export const runStep = async function ({ event, childProcess, packageName, coreS
|
|
|
22
21
|
'build.execution.step.event': event,
|
|
23
22
|
};
|
|
24
23
|
const spanCtx = setMultiSpanAttributes(attributes);
|
|
25
|
-
|
|
24
|
+
// If there's no `coreStepId` then this is a plugin execution
|
|
25
|
+
const spanName = `run-step-${coreStepId || 'plugin'}`;
|
|
26
|
+
return tracer.startActiveSpan(spanName, {}, spanCtx, async (span) => {
|
|
26
27
|
const constantsA = await addMutableConstants({ constants, buildDir, netlifyConfig });
|
|
27
28
|
const shouldRun = await shouldRunStep({
|
|
28
29
|
event,
|
package/lib/tracing/main.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
|
+
import { HoneycombSDK } from '@honeycombio/opentelemetry-node';
|
|
1
2
|
import { context, trace, propagation, SpanStatusCode, diag, DiagLogLevel } from '@opentelemetry/api';
|
|
2
|
-
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
|
|
3
|
-
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
|
|
4
|
-
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
5
3
|
import { ROOT_PACKAGE_JSON } from '../utils/json.js';
|
|
6
4
|
let sdk;
|
|
7
5
|
/** Given a simple logging function return a `DiagLogger`. Used to setup our system logger as the diag logger.*/
|
|
8
6
|
const getOtelLogger = function (logger) {
|
|
9
|
-
const otelLogger = (...args) =>
|
|
7
|
+
const otelLogger = (...args) => {
|
|
8
|
+
// Debug log msgs can be an array of 1 or 2 elements with the second element being an array fo multiple elements
|
|
9
|
+
const msgs = args.flat(1);
|
|
10
|
+
logger('[otel-traces]', ...msgs);
|
|
11
|
+
};
|
|
10
12
|
return {
|
|
11
13
|
debug: otelLogger,
|
|
12
14
|
info: otelLogger,
|
|
@@ -21,13 +23,11 @@ export const startTracing = function (options, logger) {
|
|
|
21
23
|
return;
|
|
22
24
|
if (sdk)
|
|
23
25
|
return;
|
|
24
|
-
|
|
25
|
-
url: `http://${options.host}:${options.port}`,
|
|
26
|
-
});
|
|
27
|
-
sdk = new NodeSDK({
|
|
26
|
+
sdk = new HoneycombSDK({
|
|
28
27
|
serviceName: ROOT_PACKAGE_JSON.name,
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
protocol: 'grpc',
|
|
29
|
+
apiKey: options.apiKey,
|
|
30
|
+
endpoint: `${options.httpProtocol}://${options.host}:${options.port}`,
|
|
31
31
|
});
|
|
32
32
|
// Set the diagnostics logger to our system logger. We also need to suppress the override msg
|
|
33
33
|
// in case there's a default console logger already registered (it would log a msg to it)
|
|
@@ -35,7 +35,7 @@ export const startTracing = function (options, logger) {
|
|
|
35
35
|
sdk.start();
|
|
36
36
|
// Sets the current trace ID and span ID based on the options received
|
|
37
37
|
// this is used as a way to propagate trace context from Buildbot
|
|
38
|
-
trace.setSpanContext(context.active(), {
|
|
38
|
+
return trace.setSpanContext(context.active(), {
|
|
39
39
|
traceId: options.traceId,
|
|
40
40
|
spanId: options.parentSpanId,
|
|
41
41
|
traceFlags: options.traceFlags,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@netlify/build",
|
|
3
|
-
"version": "29.
|
|
3
|
+
"version": "29.15.1",
|
|
4
4
|
"description": "Netlify build module",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": "./lib/core/main.js",
|
|
@@ -64,24 +64,23 @@
|
|
|
64
64
|
"license": "MIT",
|
|
65
65
|
"dependencies": {
|
|
66
66
|
"@bugsnag/js": "^7.0.0",
|
|
67
|
+
"@honeycombio/opentelemetry-node": "^0.4.0",
|
|
67
68
|
"@netlify/cache-utils": "^5.1.5",
|
|
68
69
|
"@netlify/config": "^20.5.1",
|
|
69
70
|
"@netlify/edge-bundler": "8.16.2",
|
|
70
71
|
"@netlify/framework-info": "^9.8.10",
|
|
71
|
-
"@netlify/functions-utils": "^5.2.
|
|
72
|
+
"@netlify/functions-utils": "^5.2.15",
|
|
72
73
|
"@netlify/git-utils": "^5.1.1",
|
|
73
74
|
"@netlify/plugins-list": "^6.68.0",
|
|
74
75
|
"@netlify/run-utils": "^5.1.1",
|
|
75
|
-
"@netlify/zip-it-and-ship-it": "9.
|
|
76
|
+
"@netlify/zip-it-and-ship-it": "9.12.0",
|
|
76
77
|
"@opentelemetry/api": "^1.4.1",
|
|
77
|
-
"@opentelemetry/exporter-trace-otlp-grpc": "^0.40.0",
|
|
78
|
-
"@opentelemetry/instrumentation-http": "^0.40.0",
|
|
79
|
-
"@opentelemetry/sdk-node": "^0.40.0",
|
|
80
78
|
"@sindresorhus/slugify": "^2.0.0",
|
|
81
79
|
"ansi-escapes": "^6.0.0",
|
|
82
80
|
"chalk": "^5.0.0",
|
|
83
81
|
"clean-stack": "^4.0.0",
|
|
84
82
|
"execa": "^6.0.0",
|
|
83
|
+
"fdir": "^6.0.1",
|
|
85
84
|
"figures": "^5.0.0",
|
|
86
85
|
"filter-obj": "^5.0.0",
|
|
87
86
|
"got": "^12.0.0",
|
|
@@ -150,5 +149,5 @@
|
|
|
150
149
|
"module": "commonjs"
|
|
151
150
|
}
|
|
152
151
|
},
|
|
153
|
-
"gitHead": "
|
|
152
|
+
"gitHead": "8710be74e31adad1729680e7202f77fb72e0d2a4"
|
|
154
153
|
}
|