@netlify/build 33.5.1 → 34.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/core/build.d.ts +1 -0
- package/lib/core/build.js +8 -4
- package/lib/core/dev.d.ts +2 -0
- package/lib/core/dev.js +10 -2
- package/lib/core/main.js +10 -2
- package/lib/log/messages/core_steps.js +12 -8
- package/lib/plugins_core/edge_functions/index.js +1 -1
- package/lib/plugins_core/functions/index.d.ts +1 -1
- package/lib/plugins_core/functions/index.js +19 -22
- package/lib/plugins_core/functions/zisi.js +0 -1
- package/lib/plugins_core/secrets_scanning/index.js +0 -2
- package/lib/plugins_core/secrets_scanning/utils.d.ts +1 -2
- package/lib/plugins_core/secrets_scanning/utils.js +8 -126
- package/lib/steps/return_values.d.ts +18 -0
- package/lib/steps/return_values.js +13 -0
- package/lib/steps/run_steps.js +1 -1
- package/package.json +7 -7
- package/lib/types/step.d.ts +0 -9
- package/lib/types/step.js +0 -1
package/lib/core/build.d.ts
CHANGED
package/lib/core/build.js
CHANGED
|
@@ -85,7 +85,7 @@ const tExecBuild = async function ({ config, defaultConfig, cachedConfig, cached
|
|
|
85
85
|
const pluginsOptions = addCorePlugins({ netlifyConfig, constants });
|
|
86
86
|
// `errorParams` is purposely stateful
|
|
87
87
|
Object.assign(errorParams, { netlifyConfig, pluginsOptions, siteInfo, childEnv, userNodeVersion });
|
|
88
|
-
const { pluginsOptions: pluginsOptionsA, netlifyConfig: netlifyConfigA, stepsCount, timers: timersB, configMutations, metrics, } = await runAndReportBuild({
|
|
88
|
+
const { pluginsOptions: pluginsOptionsA, netlifyConfig: netlifyConfigA, stepsCount, timers: timersB, configMutations, metrics, returnValues, } = await runAndReportBuild({
|
|
89
89
|
pluginsOptions,
|
|
90
90
|
netlifyConfig,
|
|
91
91
|
defaultConfig,
|
|
@@ -141,13 +141,14 @@ const tExecBuild = async function ({ config, defaultConfig, cachedConfig, cached
|
|
|
141
141
|
timers: timersB,
|
|
142
142
|
configMutations,
|
|
143
143
|
metrics,
|
|
144
|
+
returnValues,
|
|
144
145
|
};
|
|
145
146
|
};
|
|
146
147
|
export const execBuild = measureDuration(tExecBuild, 'total', { parentTag: 'build_site' });
|
|
147
148
|
// Runs a build then report any plugin statuses
|
|
148
149
|
export const runAndReportBuild = async function ({ pluginsOptions, netlifyConfig, defaultConfig, configOpts, siteInfo, configPath, outputConfigPath, headersPath, redirectsPath, packagePath, buildDir, repositoryRoot, nodePath, packageJson, userNodeVersion, childEnv, context, branch, buildbotServerSocket, constants, dry, mode, api, token, errorMonitor, deployId, errorParams, logs, debug, systemLog, systemLogFile, verbose, timers, sendStatus, saveConfig, testOpts, featureFlags, timeline, devCommand, quiet, integrations, explicitSecretKeys, enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) {
|
|
149
150
|
try {
|
|
150
|
-
const { stepsCount, netlifyConfig: netlifyConfigA, statuses, pluginsOptions: pluginsOptionsA, failedPlugins, timers: timersA, configMutations, metrics, } = await initAndRunBuild({
|
|
151
|
+
const { stepsCount, netlifyConfig: netlifyConfigA, statuses, pluginsOptions: pluginsOptionsA, failedPlugins, timers: timersA, configMutations, metrics, returnValues, } = await initAndRunBuild({
|
|
151
152
|
pluginsOptions,
|
|
152
153
|
netlifyConfig,
|
|
153
154
|
defaultConfig,
|
|
@@ -231,6 +232,7 @@ export const runAndReportBuild = async function ({ pluginsOptions, netlifyConfig
|
|
|
231
232
|
timers: timersA,
|
|
232
233
|
configMutations,
|
|
233
234
|
metrics,
|
|
235
|
+
returnValues,
|
|
234
236
|
};
|
|
235
237
|
}
|
|
236
238
|
catch (error) {
|
|
@@ -303,7 +305,7 @@ const initAndRunBuild = async function ({ pluginsOptions, netlifyConfig, default
|
|
|
303
305
|
systemLogFile,
|
|
304
306
|
});
|
|
305
307
|
try {
|
|
306
|
-
const { stepsCount, netlifyConfig: netlifyConfigA, statuses, failedPlugins, timers: timersC, configMutations, metrics, } = await runBuild({
|
|
308
|
+
const { stepsCount, netlifyConfig: netlifyConfigA, statuses, failedPlugins, timers: timersC, configMutations, metrics, returnValues, } = await runBuild({
|
|
307
309
|
childProcesses,
|
|
308
310
|
pluginsOptions: pluginsOptionsA,
|
|
309
311
|
netlifyConfig,
|
|
@@ -359,6 +361,7 @@ const initAndRunBuild = async function ({ pluginsOptions, netlifyConfig, default
|
|
|
359
361
|
timers: timersC,
|
|
360
362
|
configMutations,
|
|
361
363
|
metrics,
|
|
364
|
+
returnValues,
|
|
362
365
|
};
|
|
363
366
|
}
|
|
364
367
|
finally {
|
|
@@ -396,7 +399,7 @@ const runBuild = async function ({ childProcesses, pluginsOptions, netlifyConfig
|
|
|
396
399
|
await doDryRun({ buildDir, steps, netlifyConfig, constants, buildbotServerSocket, logs, featureFlags });
|
|
397
400
|
return { netlifyConfig };
|
|
398
401
|
}
|
|
399
|
-
const { stepsCount, netlifyConfig: netlifyConfigA, statuses, failedPlugins, timers: timersB, configMutations, metrics, } = await runSteps({
|
|
402
|
+
const { stepsCount, netlifyConfig: netlifyConfigA, statuses, failedPlugins, timers: timersB, configMutations, metrics, returnValues, } = await runSteps({
|
|
400
403
|
steps,
|
|
401
404
|
buildbotServerSocket,
|
|
402
405
|
events,
|
|
@@ -442,5 +445,6 @@ const runBuild = async function ({ childProcesses, pluginsOptions, netlifyConfig
|
|
|
442
445
|
timers: timersB,
|
|
443
446
|
configMutations,
|
|
444
447
|
metrics,
|
|
448
|
+
returnValues,
|
|
445
449
|
};
|
|
446
450
|
};
|
package/lib/core/dev.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export function startDev(devCommand: any, flags?: {}): Promise<{
|
|
|
4
4
|
netlifyConfig: any;
|
|
5
5
|
logs: import("../log/logger.js").BufferedLogs | undefined;
|
|
6
6
|
configMutations: any;
|
|
7
|
+
generatedFunctions: import("../steps/return_values.js").GeneratedFunction[];
|
|
7
8
|
error?: undefined;
|
|
8
9
|
} | {
|
|
9
10
|
success: boolean;
|
|
@@ -15,4 +16,5 @@ export function startDev(devCommand: any, flags?: {}): Promise<{
|
|
|
15
16
|
};
|
|
16
17
|
netlifyConfig?: undefined;
|
|
17
18
|
configMutations?: undefined;
|
|
19
|
+
generatedFunctions?: undefined;
|
|
18
20
|
}>;
|
package/lib/core/dev.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { handleBuildError } from '../error/handle.js';
|
|
2
|
+
import { getGeneratedFunctions } from '../steps/return_values.js';
|
|
2
3
|
import { execBuild, startBuild } from './build.js';
|
|
3
4
|
import { getSeverity } from './severity.js';
|
|
4
5
|
export const startDev = async (devCommand, flags = {}) => {
|
|
5
6
|
const { errorMonitor, mode, logs, debug, testOpts, ...normalizedFlags } = startBuild(flags);
|
|
6
7
|
const errorParams = { errorMonitor, mode, logs, debug, testOpts };
|
|
7
8
|
try {
|
|
8
|
-
const { netlifyConfig: netlifyConfigA, configMutations } = await execBuild({
|
|
9
|
+
const { netlifyConfig: netlifyConfigA, configMutations, returnValues, } = await execBuild({
|
|
9
10
|
...normalizedFlags,
|
|
10
11
|
errorMonitor,
|
|
11
12
|
errorParams,
|
|
@@ -17,7 +18,14 @@ export const startDev = async (devCommand, flags = {}) => {
|
|
|
17
18
|
devCommand,
|
|
18
19
|
});
|
|
19
20
|
const { success, severityCode } = getSeverity('success');
|
|
20
|
-
return {
|
|
21
|
+
return {
|
|
22
|
+
success,
|
|
23
|
+
severityCode,
|
|
24
|
+
netlifyConfig: netlifyConfigA,
|
|
25
|
+
logs,
|
|
26
|
+
configMutations,
|
|
27
|
+
generatedFunctions: getGeneratedFunctions(returnValues),
|
|
28
|
+
};
|
|
21
29
|
}
|
|
22
30
|
catch (error) {
|
|
23
31
|
const { severity, message, stack } = await handleBuildError(error, errorParams);
|
package/lib/core/main.js
CHANGED
|
@@ -4,6 +4,7 @@ import { handleBuildError } from '../error/handle.js';
|
|
|
4
4
|
import { reportError } from '../error/report.js';
|
|
5
5
|
import { getSystemLogger } from '../log/logger.js';
|
|
6
6
|
import { logTimer, logBuildSuccess } from '../log/messages/core.js';
|
|
7
|
+
import { getGeneratedFunctions } from '../steps/return_values.js';
|
|
7
8
|
import { trackBuildComplete } from '../telemetry/main.js';
|
|
8
9
|
import { reportTimers } from '../time/report.js';
|
|
9
10
|
import { execBuild, startBuild } from './build.js';
|
|
@@ -32,7 +33,7 @@ export async function buildSite(flags = {}) {
|
|
|
32
33
|
const rootCtx = context.with(getGlobalContext(), () => setMultiSpanAttributes(attributes));
|
|
33
34
|
return await tracer.startActiveSpan('exec-build', {}, rootCtx, async (span) => {
|
|
34
35
|
try {
|
|
35
|
-
const { pluginsOptions, netlifyConfig: netlifyConfigA, siteInfo, userNodeVersion, stepsCount, timers, durationNs, configMutations, metrics, } = await execBuild({
|
|
36
|
+
const { pluginsOptions, netlifyConfig: netlifyConfigA, siteInfo, userNodeVersion, stepsCount, timers, durationNs, configMutations, metrics, returnValues, } = await execBuild({
|
|
36
37
|
...flagsA,
|
|
37
38
|
buildId,
|
|
38
39
|
systemLogFile,
|
|
@@ -77,7 +78,14 @@ export async function buildSite(flags = {}) {
|
|
|
77
78
|
testOpts,
|
|
78
79
|
errorParams,
|
|
79
80
|
});
|
|
80
|
-
return {
|
|
81
|
+
return {
|
|
82
|
+
success,
|
|
83
|
+
severityCode,
|
|
84
|
+
netlifyConfig: netlifyConfigA,
|
|
85
|
+
logs,
|
|
86
|
+
configMutations,
|
|
87
|
+
generatedFunctions: getGeneratedFunctions(returnValues),
|
|
88
|
+
};
|
|
81
89
|
}
|
|
82
90
|
catch (error) {
|
|
83
91
|
const { severity } = await handleBuildError(error, errorParams);
|
|
@@ -50,15 +50,19 @@ export const logFunctionsNonExistingDir = function (logs, relativeFunctionsSrc)
|
|
|
50
50
|
// Print the list of Netlify Functions about to be bundled
|
|
51
51
|
export const logFunctionsToBundle = function ({ logs, userFunctions, userFunctionsSrc, userFunctionsSrcExists, internalFunctions, internalFunctionsSrc, frameworkFunctions, generatedFunctions, type = 'Functions', }) {
|
|
52
52
|
let needsSpace = false;
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
if (needsSpace)
|
|
57
|
-
log(logs, '');
|
|
58
|
-
log(logs, `Packaging ${type} generated by ${THEME.highlightWords(displayName)} ${generatorType}:`);
|
|
59
|
-
logArray(logs, functionNames, { indent: false });
|
|
60
|
-
needsSpace = true;
|
|
53
|
+
for (const id in generatedFunctions) {
|
|
54
|
+
if (generatedFunctions[id].length === 0) {
|
|
55
|
+
continue;
|
|
61
56
|
}
|
|
57
|
+
// Getting the generator block from the first function, since it will be
|
|
58
|
+
// the same for all of them.
|
|
59
|
+
const { generator } = generatedFunctions[id][0];
|
|
60
|
+
const functionNames = generatedFunctions[id].map((func) => path.basename(func.path));
|
|
61
|
+
if (needsSpace)
|
|
62
|
+
log(logs, '');
|
|
63
|
+
log(logs, `Packaging ${type} generated by ${THEME.highlightWords(generator.displayName)} ${generator.type}:`);
|
|
64
|
+
logArray(logs, functionNames, { indent: false });
|
|
65
|
+
needsSpace = true;
|
|
62
66
|
}
|
|
63
67
|
if (internalFunctions.length !== 0) {
|
|
64
68
|
if (needsSpace)
|
|
@@ -123,7 +123,7 @@ const logFunctions = async ({ frameworksAPISrcPath, internalSrcDirectory, intern
|
|
|
123
123
|
internalFunctionsSrc: internalSrcDirectory,
|
|
124
124
|
frameworkFunctions: frameworkFunctions.map(({ name }) => name),
|
|
125
125
|
type: 'Edge Functions',
|
|
126
|
-
generatedFunctions:
|
|
126
|
+
generatedFunctions: {},
|
|
127
127
|
});
|
|
128
128
|
};
|
|
129
129
|
export const bundleEdgeFunctions = {
|
|
@@ -49,5 +49,5 @@ export declare const bundleFunctions: {
|
|
|
49
49
|
}) => Promise<boolean>;
|
|
50
50
|
};
|
|
51
51
|
export declare const zipItAndShipIt: {
|
|
52
|
-
zipFunctions(
|
|
52
|
+
zipFunctions(input: import("packages/zip-it-and-ship-it/dist/zip.js").ZipFunctionsPaths, destFolder: string, args_2?: import("@netlify/zip-it-and-ship-it").ZipFunctionsOptions | undefined): Promise<FunctionResult[]>;
|
|
53
53
|
};
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolve } from 'path';
|
|
2
2
|
import { RUNTIME, zipFunctions } from '@netlify/zip-it-and-ship-it';
|
|
3
3
|
import { pathExists } from 'path-exists';
|
|
4
4
|
import { addErrorInfo } from '../../error/info.js';
|
|
5
5
|
import { log } from '../../log/logger.js';
|
|
6
|
+
import { getGeneratedFunctions } from '../../steps/return_values.js';
|
|
6
7
|
import { logBundleResults, logFunctionsNonExistingDir, logFunctionsToBundle } from '../../log/messages/core_steps.js';
|
|
7
8
|
import { FRAMEWORKS_API_FUNCTIONS_ENDPOINT } from '../../utils/frameworks_api.js';
|
|
8
9
|
import { getZipError } from './error.js';
|
|
@@ -63,8 +64,15 @@ const zipFunctionsAndLogResults = async ({ branch, buildDir, childEnv, featureFl
|
|
|
63
64
|
try {
|
|
64
65
|
// Printing an empty line before bundling output.
|
|
65
66
|
log(logs, '');
|
|
66
|
-
const
|
|
67
|
-
|
|
67
|
+
const results = await zipItAndShipIt.zipFunctions({
|
|
68
|
+
generated: {
|
|
69
|
+
directories: [internalFunctionsSrc, frameworkFunctionsSrc].filter(Boolean),
|
|
70
|
+
functions: generatedFunctions,
|
|
71
|
+
},
|
|
72
|
+
user: {
|
|
73
|
+
directories: [functionsSrc].filter(Boolean),
|
|
74
|
+
},
|
|
75
|
+
}, functionsDist, zisiParameters);
|
|
68
76
|
validateCustomRoutes(results);
|
|
69
77
|
const bundlers = Array.from(getBundlers(results));
|
|
70
78
|
logBundleResults({ logs, results });
|
|
@@ -98,7 +106,7 @@ const coreStep = async function ({ childEnv, constants: { INTERNAL_FUNCTIONS_SRC
|
|
|
98
106
|
log(logs, '');
|
|
99
107
|
}
|
|
100
108
|
}
|
|
101
|
-
const generatedFunctions =
|
|
109
|
+
const generatedFunctions = getGeneratedFunctions(returnValues);
|
|
102
110
|
logFunctionsToBundle({
|
|
103
111
|
logs,
|
|
104
112
|
userFunctions,
|
|
@@ -107,7 +115,7 @@ const coreStep = async function ({ childEnv, constants: { INTERNAL_FUNCTIONS_SRC
|
|
|
107
115
|
internalFunctions,
|
|
108
116
|
internalFunctionsSrc: relativeInternalFunctionsSrc,
|
|
109
117
|
frameworkFunctions,
|
|
110
|
-
generatedFunctions: getGeneratedFunctionsByGenerator(
|
|
118
|
+
generatedFunctions: getGeneratedFunctionsByGenerator(generatedFunctions),
|
|
111
119
|
});
|
|
112
120
|
if (userFunctions.length === 0 &&
|
|
113
121
|
internalFunctions.length === 0 &&
|
|
@@ -130,7 +138,7 @@ const coreStep = async function ({ childEnv, constants: { INTERNAL_FUNCTIONS_SRC
|
|
|
130
138
|
repositoryRoot,
|
|
131
139
|
userNodeVersion,
|
|
132
140
|
systemLog,
|
|
133
|
-
generatedFunctions,
|
|
141
|
+
generatedFunctions: generatedFunctions.map((func) => func.path),
|
|
134
142
|
});
|
|
135
143
|
const metrics = getMetrics(internalFunctions, userFunctions);
|
|
136
144
|
return {
|
|
@@ -166,25 +174,14 @@ const hasFunctionsDirectories = async function ({ buildDir, constants: { INTERNA
|
|
|
166
174
|
}
|
|
167
175
|
return false;
|
|
168
176
|
};
|
|
169
|
-
// Takes a list of return values and produces an array with the paths of all
|
|
170
|
-
// generated functions.
|
|
171
|
-
const getGeneratedFunctionPaths = (returnValues) => {
|
|
172
|
-
return Object.values(returnValues).flatMap((returnValue) => returnValue.generatedFunctions?.map((func) => func.path) || []);
|
|
173
|
-
};
|
|
174
177
|
// Takes a list of return values and produces an object with the names of the
|
|
175
178
|
// generated functions for each generator. This is used for printing logs only.
|
|
176
|
-
const getGeneratedFunctionsByGenerator = (
|
|
179
|
+
const getGeneratedFunctionsByGenerator = (generatedFunctions) => {
|
|
177
180
|
const result = {};
|
|
178
|
-
for (const
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
}
|
|
183
|
-
result[id] = {
|
|
184
|
-
displayName,
|
|
185
|
-
generatorType,
|
|
186
|
-
functionNames: generatedFunctions.map((func) => basename(func.path)),
|
|
187
|
-
};
|
|
181
|
+
for (const func of generatedFunctions) {
|
|
182
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
183
|
+
result[func.generator.name] = result[func.generator.name] || [];
|
|
184
|
+
result[func.generator.name].push(func);
|
|
188
185
|
}
|
|
189
186
|
return result;
|
|
190
187
|
};
|
|
@@ -12,7 +12,6 @@ const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecret
|
|
|
12
12
|
// When the flag is disabled, we may still run the scan if a secrets scan would otherwise take place anyway
|
|
13
13
|
// In this case, we hide any output to the user and simply gather the information in our logs
|
|
14
14
|
const enhancedScanShouldRunInActiveMode = featureFlags?.enhanced_secret_scan_impacts_builds ?? false;
|
|
15
|
-
const useMinimalChunks = featureFlags?.secret_scanning_minimal_chunks;
|
|
16
15
|
systemLog?.({ passedSecretKeys, buildDir });
|
|
17
16
|
if (!isSecretsScanningEnabled(envVars)) {
|
|
18
17
|
logSecretsScanSkipMessage(logs, 'Secrets scanning disabled via SECRETS_SCAN_ENABLED flag set to false.');
|
|
@@ -63,7 +62,6 @@ const coreStep = async function ({ buildDir, logs, netlifyConfig, explicitSecret
|
|
|
63
62
|
filePaths,
|
|
64
63
|
enhancedScanning: enhancedScanShouldRun,
|
|
65
64
|
omitValuesFromEnhancedScan: getOmitValuesFromEnhancedScanForEnhancedScanFromEnv(envVars),
|
|
66
|
-
useMinimalChunks,
|
|
67
65
|
});
|
|
68
66
|
secretMatches = scanResults.matches.filter((match) => !match.enhancedMatch);
|
|
69
67
|
enhancedSecretMatches = scanResults.matches.filter((match) => match.enhancedMatch);
|
|
@@ -9,7 +9,6 @@ interface ScanArgs {
|
|
|
9
9
|
filePaths: string[];
|
|
10
10
|
enhancedScanning?: boolean;
|
|
11
11
|
omitValuesFromEnhancedScan?: unknown[];
|
|
12
|
-
useMinimalChunks: boolean;
|
|
13
12
|
}
|
|
14
13
|
interface MatchResult {
|
|
15
14
|
lineNumber: number;
|
|
@@ -98,7 +97,7 @@ export declare function getFilePathsToScan({ env, base }: {
|
|
|
98
97
|
* @param scanArgs {ScanArgs} scan options
|
|
99
98
|
* @returns promise with all of the scan results, if any
|
|
100
99
|
*/
|
|
101
|
-
export declare function scanFilesForKeyValues({ env, keys, filePaths, base, enhancedScanning, omitValuesFromEnhancedScan,
|
|
100
|
+
export declare function scanFilesForKeyValues({ env, keys, filePaths, base, enhancedScanning, omitValuesFromEnhancedScan, }: ScanArgs): Promise<ScanResults>;
|
|
102
101
|
/**
|
|
103
102
|
* ScanResults are all of the finds for all keys and their disparate locations. Scanning is
|
|
104
103
|
* async in streams so order can change a lot. Some matches are the result of an env var explictly being marked as secret,
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { createReadStream, promises as fs, existsSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { createInterface } from 'node:readline';
|
|
4
3
|
import { fdir } from 'fdir';
|
|
5
4
|
import { minimatch } from 'minimatch';
|
|
6
5
|
import { LIKELY_SECRET_PREFIXES, SAFE_LISTED_VALUES } from './secret_prefixes.js';
|
|
@@ -221,7 +220,7 @@ const omitPathMatches = (relativePath, omitPaths) => {
|
|
|
221
220
|
* @param scanArgs {ScanArgs} scan options
|
|
222
221
|
* @returns promise with all of the scan results, if any
|
|
223
222
|
*/
|
|
224
|
-
export async function scanFilesForKeyValues({ env, keys, filePaths, base, enhancedScanning, omitValuesFromEnhancedScan = [],
|
|
223
|
+
export async function scanFilesForKeyValues({ env, keys, filePaths, base, enhancedScanning, omitValuesFromEnhancedScan = [], }) {
|
|
225
224
|
const scanResults = {
|
|
226
225
|
matches: [],
|
|
227
226
|
scannedFilesCount: 0,
|
|
@@ -242,14 +241,19 @@ export async function scanFilesForKeyValues({ env, keys, filePaths, base, enhanc
|
|
|
242
241
|
}, {});
|
|
243
242
|
scanResults.scannedFilesCount = filePaths.length;
|
|
244
243
|
let settledPromises = [];
|
|
245
|
-
const searchStream = useMinimalChunks ? searchStreamMinimalChunks : searchStreamReadline;
|
|
246
244
|
// process the scanning in batches to not run into memory issues by
|
|
247
245
|
// processing all files at the same time.
|
|
248
246
|
while (filePaths.length > 0) {
|
|
249
247
|
const chunkSize = 200;
|
|
250
248
|
const batch = filePaths.splice(0, chunkSize);
|
|
251
249
|
settledPromises = settledPromises.concat(await Promise.allSettled(batch.map((file) => {
|
|
252
|
-
return
|
|
250
|
+
return searchStreamMinimalChunks({
|
|
251
|
+
basePath: base,
|
|
252
|
+
file,
|
|
253
|
+
keyValues,
|
|
254
|
+
enhancedScanning,
|
|
255
|
+
omitValuesFromEnhancedScan,
|
|
256
|
+
});
|
|
253
257
|
})));
|
|
254
258
|
}
|
|
255
259
|
settledPromises.forEach((result) => {
|
|
@@ -259,125 +263,6 @@ export async function scanFilesForKeyValues({ env, keys, filePaths, base, enhanc
|
|
|
259
263
|
});
|
|
260
264
|
return scanResults;
|
|
261
265
|
}
|
|
262
|
-
/**
|
|
263
|
-
* Search stream implementation using node:readline
|
|
264
|
-
*/
|
|
265
|
-
const searchStreamReadline = ({ basePath, file, keyValues, enhancedScanning, omitValuesFromEnhancedScan = [], }) => {
|
|
266
|
-
return new Promise((resolve, reject) => {
|
|
267
|
-
const filePath = path.resolve(basePath, file);
|
|
268
|
-
const inStream = createReadStream(filePath);
|
|
269
|
-
const rl = createInterface({ input: inStream, terminal: false });
|
|
270
|
-
const matches = [];
|
|
271
|
-
const keyVals = [].concat(...Object.values(keyValues));
|
|
272
|
-
function getKeyForValue(val) {
|
|
273
|
-
let key = '';
|
|
274
|
-
for (const [secretKeyName, valuePermutations] of Object.entries(keyValues)) {
|
|
275
|
-
if (valuePermutations.includes(val)) {
|
|
276
|
-
key = secretKeyName;
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
return key;
|
|
280
|
-
}
|
|
281
|
-
// how many lines is the largest multiline string
|
|
282
|
-
let maxMultiLineCount = 1;
|
|
283
|
-
keyVals.forEach((valVariant) => {
|
|
284
|
-
maxMultiLineCount = Math.max(maxMultiLineCount, valVariant.split('\n').length);
|
|
285
|
-
});
|
|
286
|
-
const lines = [];
|
|
287
|
-
let lineNumber = 0;
|
|
288
|
-
rl.on('line', function (line) {
|
|
289
|
-
// iterating here so the first line will always appear as line 1 to be human friendly
|
|
290
|
-
// and match what an IDE would show for a line number.
|
|
291
|
-
lineNumber++;
|
|
292
|
-
if (typeof line === 'string') {
|
|
293
|
-
if (enhancedScanning) {
|
|
294
|
-
matches.push(...findLikelySecrets({ text: line, omitValuesFromEnhancedScan }).map(({ prefix }) => ({
|
|
295
|
-
key: prefix,
|
|
296
|
-
file,
|
|
297
|
-
lineNumber,
|
|
298
|
-
enhancedMatch: true,
|
|
299
|
-
})));
|
|
300
|
-
}
|
|
301
|
-
if (maxMultiLineCount > 1) {
|
|
302
|
-
lines.push(line);
|
|
303
|
-
}
|
|
304
|
-
// only track the max number of lines needed to match our largest
|
|
305
|
-
// multiline value. If we get above that remove the first value from the list
|
|
306
|
-
if (lines.length > maxMultiLineCount) {
|
|
307
|
-
lines.shift();
|
|
308
|
-
}
|
|
309
|
-
keyVals.forEach((valVariant) => {
|
|
310
|
-
// matching of single/whole values
|
|
311
|
-
if (line.includes(valVariant)) {
|
|
312
|
-
matches.push({
|
|
313
|
-
file,
|
|
314
|
-
lineNumber,
|
|
315
|
-
key: getKeyForValue(valVariant),
|
|
316
|
-
enhancedMatch: false,
|
|
317
|
-
});
|
|
318
|
-
return;
|
|
319
|
-
}
|
|
320
|
-
// matching of multiline values
|
|
321
|
-
if (isMultiLineVal(valVariant)) {
|
|
322
|
-
// drop empty values at beginning and end
|
|
323
|
-
const multiStringLines = valVariant.split('\n');
|
|
324
|
-
// drop early if we don't have enough lines for all values
|
|
325
|
-
if (lines.length < multiStringLines.length) {
|
|
326
|
-
return;
|
|
327
|
-
}
|
|
328
|
-
let stillMatches = true;
|
|
329
|
-
let fullMatch = false;
|
|
330
|
-
multiStringLines.forEach((valLine, valIndex) => {
|
|
331
|
-
if (valIndex === 0) {
|
|
332
|
-
// first lines have to end with the line value
|
|
333
|
-
if (!lines[valIndex].endsWith(valLine)) {
|
|
334
|
-
stillMatches = false;
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
else if (valIndex !== multiStringLines.length - 1) {
|
|
338
|
-
// middle lines have to have full line match
|
|
339
|
-
// middle lines
|
|
340
|
-
if (lines[valIndex] !== valLine) {
|
|
341
|
-
stillMatches = false;
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
else {
|
|
345
|
-
// last lines have start with the value
|
|
346
|
-
if (!lines[valIndex].startsWith(valLine)) {
|
|
347
|
-
stillMatches = false;
|
|
348
|
-
}
|
|
349
|
-
if (stillMatches === true) {
|
|
350
|
-
fullMatch = true;
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
});
|
|
354
|
-
if (fullMatch) {
|
|
355
|
-
matches.push({
|
|
356
|
-
file,
|
|
357
|
-
lineNumber: lineNumber - lines.length + 1,
|
|
358
|
-
key: getKeyForValue(valVariant),
|
|
359
|
-
enhancedMatch: false,
|
|
360
|
-
});
|
|
361
|
-
return;
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
});
|
|
365
|
-
}
|
|
366
|
-
});
|
|
367
|
-
rl.on('error', function (error) {
|
|
368
|
-
if (error?.code === 'EISDIR') {
|
|
369
|
-
// file path is a directory - do nothing
|
|
370
|
-
resolve(matches);
|
|
371
|
-
}
|
|
372
|
-
else {
|
|
373
|
-
reject(error);
|
|
374
|
-
}
|
|
375
|
-
});
|
|
376
|
-
rl.on('close', function () {
|
|
377
|
-
resolve(matches);
|
|
378
|
-
});
|
|
379
|
-
});
|
|
380
|
-
};
|
|
381
266
|
/**
|
|
382
267
|
* Search stream implementation using just read stream that allows to buffer less content
|
|
383
268
|
*/
|
|
@@ -610,6 +495,3 @@ export function groupScanResultsByKeyAndScanType(scanResults) {
|
|
|
610
495
|
sortMatches(enhancedSecretMatchesByKeys);
|
|
611
496
|
return { secretMatches: secretMatchesByKeys, enhancedSecretMatches: enhancedSecretMatchesByKeys };
|
|
612
497
|
}
|
|
613
|
-
function isMultiLineVal(v) {
|
|
614
|
-
return typeof v === 'string' && v.includes('\n');
|
|
615
|
-
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
type GeneratorType = 'build plugin' | 'extension';
|
|
2
|
+
export interface GeneratedFunction {
|
|
3
|
+
generator: {
|
|
4
|
+
displayName: string;
|
|
5
|
+
name: string;
|
|
6
|
+
type: GeneratorType;
|
|
7
|
+
};
|
|
8
|
+
path: string;
|
|
9
|
+
}
|
|
10
|
+
export interface ReturnValue {
|
|
11
|
+
displayName?: string;
|
|
12
|
+
generatedFunctions?: {
|
|
13
|
+
path: string;
|
|
14
|
+
}[];
|
|
15
|
+
generatorType: GeneratorType;
|
|
16
|
+
}
|
|
17
|
+
export declare const getGeneratedFunctions: (returnValues?: Record<string, ReturnValue>) => GeneratedFunction[];
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export const getGeneratedFunctions = (returnValues) => {
|
|
2
|
+
return Object.entries(returnValues ?? {}).flatMap(([name, returnValue]) => {
|
|
3
|
+
const generator = {
|
|
4
|
+
displayName: returnValue.displayName ?? name,
|
|
5
|
+
name,
|
|
6
|
+
type: returnValue.generatorType,
|
|
7
|
+
};
|
|
8
|
+
return (returnValue.generatedFunctions?.map((func) => ({
|
|
9
|
+
generator,
|
|
10
|
+
path: func.path,
|
|
11
|
+
})) ?? []);
|
|
12
|
+
});
|
|
13
|
+
};
|
package/lib/steps/run_steps.js
CHANGED
|
@@ -67,7 +67,7 @@ export const runSteps = async function ({ defaultConfig, steps, buildbotServerSo
|
|
|
67
67
|
edgeFunctionsBootstrapURL,
|
|
68
68
|
});
|
|
69
69
|
const statusesA = addStatus({ newStatus, statuses, event, packageName, pluginPackageJson });
|
|
70
|
-
/** @type import('../
|
|
70
|
+
/** @type import('../steps/return_values.js').ReturnValue */
|
|
71
71
|
const augmentedReturnValue = returnValue
|
|
72
72
|
? {
|
|
73
73
|
...returnValue,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@netlify/build",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "34.0.1",
|
|
4
4
|
"description": "Netlify build module",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": "./lib/index.js",
|
|
@@ -67,16 +67,16 @@
|
|
|
67
67
|
"license": "MIT",
|
|
68
68
|
"dependencies": {
|
|
69
69
|
"@bugsnag/js": "^8.0.0",
|
|
70
|
-
"@netlify/blobs": "^10.0.
|
|
70
|
+
"@netlify/blobs": "^10.0.4",
|
|
71
71
|
"@netlify/cache-utils": "^6.0.3",
|
|
72
|
-
"@netlify/config": "^23.0
|
|
73
|
-
"@netlify/edge-bundler": "14.0.
|
|
74
|
-
"@netlify/functions-utils": "^6.0.
|
|
72
|
+
"@netlify/config": "^23.1.0",
|
|
73
|
+
"@netlify/edge-bundler": "14.0.7",
|
|
74
|
+
"@netlify/functions-utils": "^6.0.15",
|
|
75
75
|
"@netlify/git-utils": "^6.0.2",
|
|
76
76
|
"@netlify/opentelemetry-utils": "^2.0.1",
|
|
77
77
|
"@netlify/plugins-list": "^6.80.0",
|
|
78
78
|
"@netlify/run-utils": "^6.0.2",
|
|
79
|
-
"@netlify/zip-it-and-ship-it": "
|
|
79
|
+
"@netlify/zip-it-and-ship-it": "13.0.1",
|
|
80
80
|
"@sindresorhus/slugify": "^2.0.0",
|
|
81
81
|
"ansi-escapes": "^7.0.0",
|
|
82
82
|
"chalk": "^5.0.0",
|
|
@@ -156,5 +156,5 @@
|
|
|
156
156
|
"engines": {
|
|
157
157
|
"node": ">=18.14.0"
|
|
158
158
|
},
|
|
159
|
-
"gitHead": "
|
|
159
|
+
"gitHead": "e471abe7b07fb8a34110d06bd8857c89f1d84142"
|
|
160
160
|
}
|
package/lib/types/step.d.ts
DELETED
package/lib/types/step.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|