artifact-engine 1.283.1 → 1.283.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/Engine/artifactEngine.js
CHANGED
|
@@ -127,11 +127,34 @@ class ArtifactEngine {
|
|
|
127
127
|
}
|
|
128
128
|
return value;
|
|
129
129
|
});
|
|
130
|
+
const isPathValidationEnabled = tl.getPipelineFeature("EnableArtifactEnginePathValidation") && !tl.getPipelineFeature("EnableLegacyArtifactPathValidation");
|
|
131
|
+
if (!isPathValidationEnabled) {
|
|
132
|
+
if (items.length > 0) {
|
|
133
|
+
this.artifactItemStore.addItems(items);
|
|
134
|
+
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
destProvider.putArtifactItem(item, null)
|
|
138
|
+
.then((item) => {
|
|
139
|
+
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
140
|
+
resolve();
|
|
141
|
+
}, (err) => {
|
|
142
|
+
logger_1.Logger.infoExternalOutput("Error creating folder " + item.path + ": " + err);
|
|
143
|
+
retryIfRequired(err);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
logger_1.Logger.logInfo("Enqueued " + items.length + " for processing.");
|
|
147
|
+
resolve();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
130
150
|
if (items.length > 0) {
|
|
131
151
|
this.artifactItemStore.addItems(items);
|
|
132
152
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
153
|
+
logger_1.Logger.logInfo("Enqueued " + items.length + " for processing.");
|
|
154
|
+
resolve();
|
|
133
155
|
}
|
|
134
156
|
else {
|
|
157
|
+
logger_1.Logger.logInfo("Enqueued " + items.length + " for processing.");
|
|
135
158
|
destProvider.putArtifactItem(item, null)
|
|
136
159
|
.then((item) => {
|
|
137
160
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
@@ -141,8 +164,6 @@ class ArtifactEngine {
|
|
|
141
164
|
retryIfRequired(err);
|
|
142
165
|
});
|
|
143
166
|
}
|
|
144
|
-
logger_1.Logger.logInfo("Enqueued " + items.length + " for processing.");
|
|
145
|
-
resolve();
|
|
146
167
|
}, (err) => {
|
|
147
168
|
logger_1.Logger.infoExternalOutput("Error getting " + item.path + ":" + err);
|
|
148
169
|
retryIfRequired(err);
|
package/Engine/worker.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Worker = void 0;
|
|
4
4
|
const logger_1 = require("./logger");
|
|
5
|
+
var tl = require('azure-pipelines-task-lib/task');
|
|
5
6
|
class Worker {
|
|
6
7
|
constructor(id, execute, getNextItem, canExit, hasDownloadFailed) {
|
|
7
8
|
this.id = id;
|
|
@@ -20,6 +21,9 @@ class Worker {
|
|
|
20
21
|
try {
|
|
21
22
|
if (this.hasDownloadFailed()) {
|
|
22
23
|
logger_1.Logger.logInfo(`Aborting respawning worker, as download failed for some file(s).`);
|
|
24
|
+
if (tl.getPipelineFeature("EnableArtifactEnginePathValidation") && !tl.getPipelineFeature("EnableLegacyArtifactPathValidation")) {
|
|
25
|
+
reject(new Error('Aborting worker: download failed for one or more item(s).'));
|
|
26
|
+
}
|
|
23
27
|
return;
|
|
24
28
|
}
|
|
25
29
|
let item = this.getNextItem();
|
|
@@ -11,7 +11,10 @@ export declare class FilesystemProvider implements models.IArtifactProvider {
|
|
|
11
11
|
private putArtifactItemLegacy;
|
|
12
12
|
private putArtifactItemSecure;
|
|
13
13
|
private getWriteOpenFlags;
|
|
14
|
+
private getTempWritePath;
|
|
14
15
|
private createSecurityRejection;
|
|
16
|
+
private createValidatedDirectory;
|
|
17
|
+
private validateExistingIntermediateSegment;
|
|
15
18
|
private resolveSafeOutputPath;
|
|
16
19
|
isCaseInsensitiveFilesystem(): boolean;
|
|
17
20
|
dispose(): void;
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.FilesystemProvider = void 0;
|
|
4
|
+
const os = require("os");
|
|
4
5
|
const path = require("path");
|
|
5
6
|
const fs = require("fs");
|
|
6
7
|
const models = require("../Models");
|
|
7
8
|
const logger_1 = require("../Engine/logger");
|
|
9
|
+
const ci = require("../Engine/cilogger");
|
|
8
10
|
const filesystemCaseSensitivity_1 = require("./filesystemCaseSensitivity");
|
|
9
11
|
var tl = require('azure-pipelines-task-lib');
|
|
10
12
|
class FilesystemProvider {
|
|
@@ -107,8 +109,6 @@ class FilesystemProvider {
|
|
|
107
109
|
}
|
|
108
110
|
});
|
|
109
111
|
}
|
|
110
|
-
// New hardened implementation. Runs only when path validation is on and the legacy
|
|
111
|
-
// rollback flag is off (the new default).
|
|
112
112
|
putArtifactItemSecure(item, stream) {
|
|
113
113
|
return new Promise((resolve, reject) => {
|
|
114
114
|
let outputItemPath;
|
|
@@ -116,16 +116,24 @@ class FilesystemProvider {
|
|
|
116
116
|
outputItemPath = this.resolveSafeOutputPath(item.path);
|
|
117
117
|
}
|
|
118
118
|
catch (err) {
|
|
119
|
+
if (stream && typeof stream.destroy === 'function') {
|
|
120
|
+
stream.destroy();
|
|
121
|
+
}
|
|
119
122
|
reject(err);
|
|
120
123
|
return;
|
|
121
124
|
}
|
|
122
125
|
if (item.itemType === models.ItemType.File) {
|
|
123
126
|
const folder = path.dirname(outputItemPath);
|
|
127
|
+
const tempItemPath = this.getTempWritePath(outputItemPath);
|
|
128
|
+
let settled = false;
|
|
129
|
+
const cleanupTempFile = () => {
|
|
130
|
+
fs.unlink(tempItemPath, () => { });
|
|
131
|
+
};
|
|
124
132
|
try {
|
|
125
133
|
tl.mkdirP(folder);
|
|
126
134
|
logger_1.Logger.messageExternalOutput(tl.loc("DownloadingTo", item.path, outputItemPath));
|
|
127
135
|
const writeOptions = { flags: this.getWriteOpenFlags() };
|
|
128
|
-
const outputStream = fs.createWriteStream(
|
|
136
|
+
const outputStream = fs.createWriteStream(tempItemPath, writeOptions);
|
|
129
137
|
stream.pipe(outputStream);
|
|
130
138
|
stream.on("end", () => {
|
|
131
139
|
logger_1.Logger.messageExternalOutput(tl.loc("DownloadedTo", item.path, outputItemPath));
|
|
@@ -135,23 +143,47 @@ class FilesystemProvider {
|
|
|
135
143
|
item.metadata[models.Constants.DestinationUrlKey] = outputItemPath;
|
|
136
144
|
});
|
|
137
145
|
stream.on("error", (error) => {
|
|
146
|
+
if (settled) {
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
settled = true;
|
|
150
|
+
cleanupTempFile();
|
|
138
151
|
reject(error);
|
|
139
152
|
});
|
|
140
153
|
outputStream.on("error", (error) => {
|
|
154
|
+
if (settled) {
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
settled = true;
|
|
158
|
+
cleanupTempFile();
|
|
141
159
|
reject(error);
|
|
142
160
|
});
|
|
143
161
|
outputStream.on("finish", () => {
|
|
162
|
+
if (settled) {
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
try {
|
|
166
|
+
fs.renameSync(tempItemPath, outputItemPath);
|
|
167
|
+
}
|
|
168
|
+
catch (renameErr) {
|
|
169
|
+
settled = true;
|
|
170
|
+
cleanupTempFile();
|
|
171
|
+
reject(renameErr);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
settled = true;
|
|
144
175
|
this.artifactItemStore.updateFileSize(item, outputStream.bytesWritten);
|
|
145
176
|
resolve(item);
|
|
146
177
|
});
|
|
147
178
|
}
|
|
148
179
|
catch (err) {
|
|
180
|
+
cleanupTempFile();
|
|
149
181
|
reject(err);
|
|
150
182
|
}
|
|
151
183
|
}
|
|
152
184
|
else {
|
|
153
185
|
try {
|
|
154
|
-
|
|
186
|
+
this.createValidatedDirectory(outputItemPath, item.path);
|
|
155
187
|
resolve(item);
|
|
156
188
|
}
|
|
157
189
|
catch (err) {
|
|
@@ -167,11 +199,85 @@ class FilesystemProvider {
|
|
|
167
199
|
}
|
|
168
200
|
return flags;
|
|
169
201
|
}
|
|
170
|
-
|
|
202
|
+
getTempWritePath(outputItemPath) {
|
|
203
|
+
const dir = path.dirname(outputItemPath);
|
|
204
|
+
const base = path.basename(outputItemPath);
|
|
205
|
+
const unique = `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
206
|
+
return path.join(dir, `.${base}.tmp-${unique}`);
|
|
207
|
+
}
|
|
208
|
+
createSecurityRejection(messageKey, itemPath, cause) {
|
|
171
209
|
const err = new Error(tl.loc(messageKey, itemPath));
|
|
172
210
|
err.nonRetryable = true;
|
|
211
|
+
if (cause !== undefined && cause !== null) {
|
|
212
|
+
err.cause = cause;
|
|
213
|
+
}
|
|
214
|
+
try {
|
|
215
|
+
ci.publishEvent('pathValidationRejection', {
|
|
216
|
+
reason: messageKey,
|
|
217
|
+
causeCode: cause && cause.code ? cause.code : undefined,
|
|
218
|
+
pathValidationFlag: !!tl.getPipelineFeature("EnableArtifactEnginePathValidation"),
|
|
219
|
+
legacyPathValidationFlag: !!tl.getPipelineFeature("EnableLegacyArtifactPathValidation"),
|
|
220
|
+
os: os.platform(),
|
|
221
|
+
provider: 'filesystem'
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
catch (telemetryErr) {
|
|
225
|
+
// Telemetry must never block or fail the rejection it is reporting on.
|
|
226
|
+
}
|
|
173
227
|
return err;
|
|
174
228
|
}
|
|
229
|
+
// Creates a directory-type artifact's final path segment with the same symlink-safe
|
|
230
|
+
// guarantee as the intermediate segments in resolveSafeOutputPath, instead of an
|
|
231
|
+
// unchecked mkdirP call that would reopen the TOCTOU window the latter closes.
|
|
232
|
+
createValidatedDirectory(dirPath, itemPath) {
|
|
233
|
+
try {
|
|
234
|
+
fs.mkdirSync(dirPath);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
catch (err) {
|
|
238
|
+
if (!err || err.code !== 'EEXIST') {
|
|
239
|
+
throw err;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
let raceStat;
|
|
243
|
+
try {
|
|
244
|
+
raceStat = fs.lstatSync(dirPath);
|
|
245
|
+
}
|
|
246
|
+
catch (raceErr) {
|
|
247
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath, raceErr);
|
|
248
|
+
}
|
|
249
|
+
const resolvedRoot = path.resolve(this._rootLocation);
|
|
250
|
+
let realRoot;
|
|
251
|
+
try {
|
|
252
|
+
realRoot = fs.realpathSync(resolvedRoot);
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
realRoot = resolvedRoot;
|
|
256
|
+
}
|
|
257
|
+
this.validateExistingIntermediateSegment(dirPath, raceStat, realRoot, itemPath);
|
|
258
|
+
}
|
|
259
|
+
validateExistingIntermediateSegment(current, stat, realRoot, itemPath) {
|
|
260
|
+
if (stat.isSymbolicLink()) {
|
|
261
|
+
let real;
|
|
262
|
+
try {
|
|
263
|
+
real = fs.realpathSync(current);
|
|
264
|
+
}
|
|
265
|
+
catch (err) {
|
|
266
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath, err);
|
|
267
|
+
}
|
|
268
|
+
const relToRoot = path.relative(realRoot, real);
|
|
269
|
+
if (relToRoot === '..'
|
|
270
|
+
|| relToRoot.startsWith('..' + path.sep)
|
|
271
|
+
|| path.isAbsolute(relToRoot)) {
|
|
272
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath);
|
|
273
|
+
}
|
|
274
|
+
return real;
|
|
275
|
+
}
|
|
276
|
+
if (!stat.isDirectory()) {
|
|
277
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath);
|
|
278
|
+
}
|
|
279
|
+
return current;
|
|
280
|
+
}
|
|
175
281
|
resolveSafeOutputPath(itemPath) {
|
|
176
282
|
const resolvedRoot = path.resolve(this._rootLocation);
|
|
177
283
|
const resolvedTarget = path.resolve(resolvedRoot, itemPath);
|
|
@@ -181,15 +287,14 @@ class FilesystemProvider {
|
|
|
181
287
|
|| path.isAbsolute(relative)) {
|
|
182
288
|
throw this.createSecurityRejection("InvalidArtifactPath", itemPath);
|
|
183
289
|
}
|
|
184
|
-
|
|
290
|
+
if (relative !== '') {
|
|
291
|
+
tl.mkdirP(resolvedRoot);
|
|
292
|
+
}
|
|
185
293
|
let realRoot;
|
|
186
294
|
try {
|
|
187
295
|
realRoot = fs.realpathSync(resolvedRoot);
|
|
188
296
|
}
|
|
189
297
|
catch (err) {
|
|
190
|
-
// Root does not exist yet (e.g. mkdirP was a no-op, or creation is deferred until
|
|
191
|
-
// the first write). Nothing can be a pre-existing symlink at a path that doesn't
|
|
192
|
-
// exist, so fall back to the lexically resolved root.
|
|
193
298
|
realRoot = resolvedRoot;
|
|
194
299
|
}
|
|
195
300
|
const segments = relative.split(path.sep);
|
|
@@ -201,24 +306,29 @@ class FilesystemProvider {
|
|
|
201
306
|
stat = fs.lstatSync(current);
|
|
202
307
|
}
|
|
203
308
|
catch (err) {
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
if (stat.isSymbolicLink()) {
|
|
207
|
-
let real;
|
|
208
|
-
try {
|
|
209
|
-
real = fs.realpathSync(current);
|
|
309
|
+
if (err && err.code !== 'ENOENT') {
|
|
310
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath, err);
|
|
210
311
|
}
|
|
211
|
-
|
|
212
|
-
|
|
312
|
+
try {
|
|
313
|
+
fs.mkdirSync(current);
|
|
314
|
+
continue;
|
|
213
315
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
316
|
+
catch (mkdirErr) {
|
|
317
|
+
if (!mkdirErr || mkdirErr.code !== 'EEXIST') {
|
|
318
|
+
throw mkdirErr;
|
|
319
|
+
}
|
|
320
|
+
let raceStat;
|
|
321
|
+
try {
|
|
322
|
+
raceStat = fs.lstatSync(current);
|
|
323
|
+
}
|
|
324
|
+
catch (raceErr) {
|
|
325
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath, raceErr);
|
|
326
|
+
}
|
|
327
|
+
current = this.validateExistingIntermediateSegment(current, raceStat, realRoot, itemPath);
|
|
328
|
+
continue;
|
|
219
329
|
}
|
|
220
|
-
current = real;
|
|
221
330
|
}
|
|
331
|
+
current = this.validateExistingIntermediateSegment(current, stat, realRoot, itemPath);
|
|
222
332
|
}
|
|
223
333
|
const finalPath = path.join(current, segments[segments.length - 1]);
|
|
224
334
|
try {
|
|
@@ -229,14 +339,14 @@ class FilesystemProvider {
|
|
|
229
339
|
if (!finalStat.isFile() && !finalStat.isDirectory()) {
|
|
230
340
|
throw this.createSecurityRejection("ArtifactPathIsSpecialFile", itemPath);
|
|
231
341
|
}
|
|
232
|
-
if (finalStat.isFile() && finalStat.nlink > 1) {
|
|
233
|
-
throw this.createSecurityRejection("ArtifactPathIsHardLinked", itemPath);
|
|
234
|
-
}
|
|
235
342
|
}
|
|
236
343
|
catch (err) {
|
|
237
344
|
if (err && err.nonRetryable) {
|
|
238
345
|
throw err;
|
|
239
346
|
}
|
|
347
|
+
if (err && err.code !== 'ENOENT') {
|
|
348
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath, err);
|
|
349
|
+
}
|
|
240
350
|
}
|
|
241
351
|
return finalPath;
|
|
242
352
|
}
|