artifact-engine 2.281.2 → 2.283.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Engine/artifactEngine.js +17 -10
- package/Engine/artifactEngineOptions.d.ts +2 -0
- package/Engine/artifactEngineOptions.js +1 -0
- package/Engine/logger.d.ts +8 -0
- package/Engine/logger.js +32 -3
- package/Providers/filesystemProvider.d.ts +5 -0
- package/Providers/filesystemProvider.js +144 -3
- package/Providers/webProvider.js +3 -3
- package/README.md +50 -0
- package/Strings/resources.resjson/de-DE/resources.resjson +4 -1
- package/Strings/resources.resjson/en-US/resources.resjson +3 -0
- package/Strings/resources.resjson/es-ES/resources.resjson +4 -1
- package/Strings/resources.resjson/fr-FR/resources.resjson +4 -1
- package/Strings/resources.resjson/it-IT/resources.resjson +4 -1
- package/Strings/resources.resjson/ja-JP/resources.resjson +4 -1
- package/Strings/resources.resjson/ko-KR/resources.resjson +4 -1
- package/Strings/resources.resjson/ru-RU/resources.resjson +4 -1
- package/Strings/resources.resjson/zh-CN/resources.resjson +4 -1
- package/Strings/resources.resjson/zh-TW/resources.resjson +4 -1
- package/lib.json +3 -0
- package/package.json +2 -2
package/Engine/artifactEngine.js
CHANGED
|
@@ -16,6 +16,7 @@ class ArtifactEngine {
|
|
|
16
16
|
}
|
|
17
17
|
processItems(sourceProvider, destProvider, artifactEngineOptions) {
|
|
18
18
|
var artifactDownloadTicketsPromise = new Promise((resolve, reject) => {
|
|
19
|
+
var _a;
|
|
19
20
|
const workers = [];
|
|
20
21
|
artifactEngineOptions = artifactEngineOptions || new artifactEngineOptions_1.ArtifactEngineOptions();
|
|
21
22
|
this.createPatternList(artifactEngineOptions);
|
|
@@ -23,6 +24,7 @@ class ArtifactEngine {
|
|
|
23
24
|
this.isCaseInsensitiveArtifactMatchingEnabled(destProvider);
|
|
24
25
|
this.artifactItemStore.flush();
|
|
25
26
|
logger_1.Logger.verbose = artifactEngineOptions.verbose;
|
|
27
|
+
logger_1.Logger.vsoCommandMode = (_a = artifactEngineOptions.vsoCommandMode) !== null && _a !== void 0 ? _a : 'blockOnly';
|
|
26
28
|
this.logger = new logger_1.Logger(this.artifactItemStore);
|
|
27
29
|
this.logger.logProgress();
|
|
28
30
|
sourceProvider.artifactItemStore = this.artifactItemStore;
|
|
@@ -60,13 +62,18 @@ class ArtifactEngine {
|
|
|
60
62
|
}
|
|
61
63
|
processArtifactItemImplementation(sourceProvider, item, destProvider, artifactEngineOptions, resolve, reject, retryCount) {
|
|
62
64
|
var retryIfRequired = (err) => {
|
|
65
|
+
if (err && err.nonRetryable) {
|
|
66
|
+
this.artifactItemStore.updateState(item, models.TicketState.Failed);
|
|
67
|
+
reject(err);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
63
70
|
if (retryCount === artifactEngineOptions.retryLimit - 1) {
|
|
64
71
|
this.artifactItemStore.updateState(item, models.TicketState.Failed);
|
|
65
72
|
reject(err);
|
|
66
73
|
}
|
|
67
74
|
else {
|
|
68
75
|
this.artifactItemStore.increaseRetryCount(item);
|
|
69
|
-
logger_1.Logger.
|
|
76
|
+
logger_1.Logger.messageExternalOutput(tl.loc("RetryingDownload", item.path, (retryCount + 1)));
|
|
70
77
|
// retry interval sequence 5s 15s 45s 135s
|
|
71
78
|
// 2 min is the time limit for freeing of ports
|
|
72
79
|
setTimeout(() => this
|
|
@@ -90,24 +97,24 @@ class ArtifactEngine {
|
|
|
90
97
|
flipNegate: false
|
|
91
98
|
};
|
|
92
99
|
if (tl.match([pathToMatch], this.patternList, null, matchOptions).length > 0) {
|
|
93
|
-
logger_1.Logger.
|
|
100
|
+
logger_1.Logger.infoExternalOutput("Processing " + item.path);
|
|
94
101
|
sourceProvider.getArtifactItem(item).then((contentStream) => {
|
|
95
|
-
logger_1.Logger.
|
|
102
|
+
logger_1.Logger.infoExternalOutput("Got download stream for item: " + item.path);
|
|
96
103
|
destProvider.putArtifactItem(item, contentStream)
|
|
97
104
|
.then((item) => {
|
|
98
105
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
99
106
|
resolve();
|
|
100
107
|
}, (err) => {
|
|
101
|
-
logger_1.Logger.
|
|
108
|
+
logger_1.Logger.infoExternalOutput("Error placing file " + item.path + ": " + err);
|
|
102
109
|
retryIfRequired(err);
|
|
103
110
|
});
|
|
104
111
|
}, (err) => {
|
|
105
|
-
logger_1.Logger.
|
|
112
|
+
logger_1.Logger.infoExternalOutput("Error getting file " + item.path + ": " + err);
|
|
106
113
|
retryIfRequired(err);
|
|
107
114
|
});
|
|
108
115
|
}
|
|
109
116
|
else {
|
|
110
|
-
logger_1.Logger.
|
|
117
|
+
logger_1.Logger.messageExternalOutput(tl.loc("SkippingItem", pathToMatch));
|
|
111
118
|
this.artifactItemStore.updateState(item, models.TicketState.Skipped);
|
|
112
119
|
resolve();
|
|
113
120
|
}
|
|
@@ -130,14 +137,14 @@ class ArtifactEngine {
|
|
|
130
137
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
131
138
|
resolve();
|
|
132
139
|
}, (err) => {
|
|
133
|
-
logger_1.Logger.
|
|
140
|
+
logger_1.Logger.infoExternalOutput("Error creating folder " + item.path + ": " + err);
|
|
134
141
|
retryIfRequired(err);
|
|
135
142
|
});
|
|
136
143
|
}
|
|
137
144
|
logger_1.Logger.logInfo("Enqueued " + items.length + " for processing.");
|
|
138
145
|
resolve();
|
|
139
146
|
}, (err) => {
|
|
140
|
-
logger_1.Logger.
|
|
147
|
+
logger_1.Logger.infoExternalOutput("Error getting " + item.path + ":" + err);
|
|
141
148
|
retryIfRequired(err);
|
|
142
149
|
});
|
|
143
150
|
}
|
|
@@ -175,11 +182,11 @@ exports.ArtifactEngine = ArtifactEngine;
|
|
|
175
182
|
tl.setResourcePath(path.join(path.dirname(__dirname), 'lib.json'));
|
|
176
183
|
process.on('unhandledRejection', (err) => {
|
|
177
184
|
ci.publishEvent('reliability', { issueType: 'unhandledRejection', errorMessage: JSON.stringify(err, Object.getOwnPropertyNames(err)) });
|
|
178
|
-
logger_1.Logger.
|
|
185
|
+
logger_1.Logger.errorExternalOutput(tl.loc("UnhandledRejection", err));
|
|
179
186
|
throw err;
|
|
180
187
|
});
|
|
181
188
|
process.on('uncaughtException', (err) => {
|
|
182
189
|
ci.publishEvent('reliability', { issueType: 'uncaughtException', errorMessage: JSON.stringify(err, Object.getOwnPropertyNames(err)) });
|
|
183
|
-
logger_1.Logger.
|
|
190
|
+
logger_1.Logger.errorExternalOutput(tl.loc("UnhandledException", err));
|
|
184
191
|
throw err;
|
|
185
192
|
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { VsoCommandMode } from 'azure-pipelines-task-lib/externaloutput';
|
|
1
2
|
export declare class ArtifactEngineOptions {
|
|
2
3
|
retryLimit: number;
|
|
3
4
|
retryIntervalInSeconds: number;
|
|
@@ -5,4 +6,5 @@ export declare class ArtifactEngineOptions {
|
|
|
5
6
|
parallelProcessingLimit: number;
|
|
6
7
|
itemPattern: string;
|
|
7
8
|
verbose: boolean;
|
|
9
|
+
vsoCommandMode?: VsoCommandMode;
|
|
8
10
|
}
|
package/Engine/logger.d.ts
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
|
+
import type { VsoCommandMode } from 'azure-pipelines-task-lib/externaloutput';
|
|
1
2
|
import { ArtifactItemStore } from '../Store/artifactItemStore';
|
|
2
3
|
export declare class Logger {
|
|
3
4
|
constructor(store: ArtifactItemStore);
|
|
4
5
|
static logInfo(message: string): void;
|
|
5
6
|
static logError(message: string): void;
|
|
6
7
|
static logMessage(message: string): void;
|
|
8
|
+
/** Logs external text while preserving debug output semantics. */
|
|
9
|
+
static infoExternalOutput(message: string): void;
|
|
10
|
+
/** Logs external text while preserving error output semantics. */
|
|
11
|
+
static errorExternalOutput(message: string): void;
|
|
12
|
+
/** Logs external text using the configured mode and native line ending. */
|
|
13
|
+
static messageExternalOutput(message: string): void;
|
|
7
14
|
logProgress(): void;
|
|
8
15
|
logSummary(): void;
|
|
9
16
|
private padText;
|
|
10
17
|
static verbose: boolean;
|
|
18
|
+
static vsoCommandMode: VsoCommandMode;
|
|
11
19
|
private store;
|
|
12
20
|
private startTime;
|
|
13
21
|
}
|
package/Engine/logger.js
CHANGED
|
@@ -10,11 +10,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.Logger = void 0;
|
|
13
|
+
const os = require("os");
|
|
14
|
+
const tl = require("azure-pipelines-task-lib/task");
|
|
13
15
|
const Models_1 = require("../Models");
|
|
14
16
|
const ticketState_1 = require("../Models/ticketState");
|
|
15
|
-
var tl = require('azure-pipelines-task-lib');
|
|
16
|
-
var os = require('os');
|
|
17
17
|
const ci = require("./cilogger");
|
|
18
|
+
const externalOutputFeature = 'EnableArtifactEngineExternalOutput';
|
|
18
19
|
class Logger {
|
|
19
20
|
constructor(store) {
|
|
20
21
|
this.store = store;
|
|
@@ -29,6 +30,33 @@ class Logger {
|
|
|
29
30
|
static logMessage(message) {
|
|
30
31
|
tl.writeExternalOutput(message + os.EOL, { source: 'remote' });
|
|
31
32
|
}
|
|
33
|
+
/** Logs external text while preserving debug output semantics. */
|
|
34
|
+
static infoExternalOutput(message) {
|
|
35
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
36
|
+
tl.debugExternalOutput(message, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
Logger.logInfo(message);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Logs external text while preserving error output semantics. */
|
|
43
|
+
static errorExternalOutput(message) {
|
|
44
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
45
|
+
tl.errorExternalOutput(message, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
Logger.logError(message);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Logs external text using the configured mode and native line ending. */
|
|
52
|
+
static messageExternalOutput(message) {
|
|
53
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
54
|
+
tl.writeExternalOutput(message + os.EOL, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
Logger.logMessage(message);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
32
60
|
logProgress() {
|
|
33
61
|
if (Logger.verbose) {
|
|
34
62
|
var progressLogger = () => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -97,7 +125,7 @@ class Logger {
|
|
|
97
125
|
tl.debug(this.padText("", maxPathLength + 25 + 25 + 10 + 10 + 15, '-'));
|
|
98
126
|
fileTickets.forEach(ticket => {
|
|
99
127
|
var duration = (ticket.finishTime.valueOf() - ticket.startTime.valueOf()) / 1000 + " secs";
|
|
100
|
-
|
|
128
|
+
Logger.infoExternalOutput("| " + this.padText(ticket.artifactItem.path, maxPathLength) + " | " + this.padText(ticket.startTime.toISOString(), 25) + " | " + this.padText(ticket.finishTime.toISOString(), 25) + " | " + this.padText(duration, 10) + " | " + this.padText(ticket.state.toString().toUpperCase(), 10) + "|");
|
|
101
129
|
tl.debug(this.padText("", maxPathLength + 25 + 25 + 10 + 10 + 15, '-'));
|
|
102
130
|
});
|
|
103
131
|
}
|
|
@@ -110,3 +138,4 @@ class Logger {
|
|
|
110
138
|
}
|
|
111
139
|
}
|
|
112
140
|
exports.Logger = Logger;
|
|
141
|
+
Logger.vsoCommandMode = 'blockOnly';
|
|
@@ -8,6 +8,11 @@ export declare class FilesystemProvider implements models.IArtifactProvider {
|
|
|
8
8
|
getArtifactItems(artifactItem: models.ArtifactItem): Promise<models.ArtifactItem[]>;
|
|
9
9
|
getArtifactItem(artifactItem: models.ArtifactItem): Promise<NodeJS.ReadableStream>;
|
|
10
10
|
putArtifactItem(item: models.ArtifactItem, stream: NodeJS.ReadableStream): Promise<models.ArtifactItem>;
|
|
11
|
+
private putArtifactItemLegacy;
|
|
12
|
+
private putArtifactItemSecure;
|
|
13
|
+
private getWriteOpenFlags;
|
|
14
|
+
private createSecurityRejection;
|
|
15
|
+
private resolveSafeOutputPath;
|
|
11
16
|
isCaseInsensitiveFilesystem(): boolean;
|
|
12
17
|
dispose(): void;
|
|
13
18
|
private getItems;
|
|
@@ -45,6 +45,14 @@ class FilesystemProvider {
|
|
|
45
45
|
return promise;
|
|
46
46
|
}
|
|
47
47
|
putArtifactItem(item, stream) {
|
|
48
|
+
if (!tl.getPipelineFeature("EnableArtifactEnginePathValidation") || tl.getPipelineFeature("EnableLegacyArtifactPathValidation")) {
|
|
49
|
+
return this.putArtifactItemLegacy(item, stream);
|
|
50
|
+
}
|
|
51
|
+
return this.putArtifactItemSecure(item, stream);
|
|
52
|
+
}
|
|
53
|
+
// Unmodified pre-fix implementation. Runs verbatim whenever path validation is off, or
|
|
54
|
+
// EnableLegacyArtifactPathValidation is On, so rollback restores the exact legacy behavior.
|
|
55
|
+
putArtifactItemLegacy(item, stream) {
|
|
48
56
|
return new Promise((resolve, reject) => {
|
|
49
57
|
let outputItemPath;
|
|
50
58
|
if (!tl.getPipelineFeature("EnableArtifactEnginePathValidation")) {
|
|
@@ -68,11 +76,61 @@ class FilesystemProvider {
|
|
|
68
76
|
try {
|
|
69
77
|
// create parent folder if it has not already been created
|
|
70
78
|
tl.mkdirP(folder);
|
|
71
|
-
logger_1.Logger.
|
|
79
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadingTo", item.path, outputItemPath));
|
|
72
80
|
const outputStream = fs.createWriteStream(outputItemPath);
|
|
73
81
|
stream.pipe(outputStream);
|
|
74
82
|
stream.on("end", () => {
|
|
75
|
-
logger_1.Logger.
|
|
83
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadedTo", item.path, outputItemPath));
|
|
84
|
+
if (!item.metadata) {
|
|
85
|
+
item.metadata = {};
|
|
86
|
+
}
|
|
87
|
+
item.metadata[models.Constants.DestinationUrlKey] = outputItemPath;
|
|
88
|
+
});
|
|
89
|
+
stream.on("error", (error) => {
|
|
90
|
+
reject(error);
|
|
91
|
+
});
|
|
92
|
+
outputStream.on("finish", () => {
|
|
93
|
+
this.artifactItemStore.updateFileSize(item, outputStream.bytesWritten);
|
|
94
|
+
resolve(item);
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
reject(err);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
try {
|
|
103
|
+
tl.mkdirP(outputItemPath);
|
|
104
|
+
resolve(item);
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
reject(err);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
// New hardened implementation. Runs only when path validation is on and the legacy
|
|
113
|
+
// rollback flag is off (the new default).
|
|
114
|
+
putArtifactItemSecure(item, stream) {
|
|
115
|
+
return new Promise((resolve, reject) => {
|
|
116
|
+
let outputItemPath;
|
|
117
|
+
try {
|
|
118
|
+
outputItemPath = this.resolveSafeOutputPath(item.path);
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
reject(err);
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
if (item.itemType === models.ItemType.File) {
|
|
125
|
+
const folder = path.dirname(outputItemPath);
|
|
126
|
+
try {
|
|
127
|
+
tl.mkdirP(folder);
|
|
128
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadingTo", item.path, outputItemPath));
|
|
129
|
+
const writeOptions = { flags: this.getWriteOpenFlags() };
|
|
130
|
+
const outputStream = fs.createWriteStream(outputItemPath, writeOptions);
|
|
131
|
+
stream.pipe(outputStream);
|
|
132
|
+
stream.on("end", () => {
|
|
133
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadedTo", item.path, outputItemPath));
|
|
76
134
|
if (!item.metadata) {
|
|
77
135
|
item.metadata = {};
|
|
78
136
|
}
|
|
@@ -81,6 +139,9 @@ class FilesystemProvider {
|
|
|
81
139
|
stream.on("error", (error) => {
|
|
82
140
|
reject(error);
|
|
83
141
|
});
|
|
142
|
+
outputStream.on("error", (error) => {
|
|
143
|
+
reject(error);
|
|
144
|
+
});
|
|
84
145
|
outputStream.on("finish", () => {
|
|
85
146
|
this.artifactItemStore.updateFileSize(item, outputStream.bytesWritten);
|
|
86
147
|
resolve(item);
|
|
@@ -101,6 +162,86 @@ class FilesystemProvider {
|
|
|
101
162
|
}
|
|
102
163
|
});
|
|
103
164
|
}
|
|
165
|
+
getWriteOpenFlags() {
|
|
166
|
+
let flags = fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_TRUNC;
|
|
167
|
+
if (typeof fs.constants.O_NOFOLLOW === 'number') {
|
|
168
|
+
flags |= fs.constants.O_NOFOLLOW;
|
|
169
|
+
}
|
|
170
|
+
return flags;
|
|
171
|
+
}
|
|
172
|
+
createSecurityRejection(messageKey, itemPath) {
|
|
173
|
+
const err = new Error(tl.loc(messageKey, itemPath));
|
|
174
|
+
err.nonRetryable = true;
|
|
175
|
+
return err;
|
|
176
|
+
}
|
|
177
|
+
resolveSafeOutputPath(itemPath) {
|
|
178
|
+
const resolvedRoot = path.resolve(this._rootLocation);
|
|
179
|
+
const resolvedTarget = path.resolve(resolvedRoot, itemPath);
|
|
180
|
+
const relative = path.relative(resolvedRoot, resolvedTarget);
|
|
181
|
+
if (relative === '..'
|
|
182
|
+
|| relative.startsWith('..' + path.sep)
|
|
183
|
+
|| path.isAbsolute(relative)) {
|
|
184
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath);
|
|
185
|
+
}
|
|
186
|
+
tl.mkdirP(resolvedRoot);
|
|
187
|
+
let realRoot;
|
|
188
|
+
try {
|
|
189
|
+
realRoot = fs.realpathSync(resolvedRoot);
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
// Root does not exist yet (e.g. mkdirP was a no-op, or creation is deferred until
|
|
193
|
+
// the first write). Nothing can be a pre-existing symlink at a path that doesn't
|
|
194
|
+
// exist, so fall back to the lexically resolved root.
|
|
195
|
+
realRoot = resolvedRoot;
|
|
196
|
+
}
|
|
197
|
+
const segments = relative.split(path.sep);
|
|
198
|
+
let current = realRoot;
|
|
199
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
200
|
+
current = path.join(current, segments[i]);
|
|
201
|
+
let stat;
|
|
202
|
+
try {
|
|
203
|
+
stat = fs.lstatSync(current);
|
|
204
|
+
}
|
|
205
|
+
catch (err) {
|
|
206
|
+
break;
|
|
207
|
+
}
|
|
208
|
+
if (stat.isSymbolicLink()) {
|
|
209
|
+
let real;
|
|
210
|
+
try {
|
|
211
|
+
real = fs.realpathSync(current);
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath);
|
|
215
|
+
}
|
|
216
|
+
const relToRoot = path.relative(realRoot, real);
|
|
217
|
+
if (relToRoot === '..'
|
|
218
|
+
|| relToRoot.startsWith('..' + path.sep)
|
|
219
|
+
|| path.isAbsolute(relToRoot)) {
|
|
220
|
+
throw this.createSecurityRejection("InvalidArtifactPath", itemPath);
|
|
221
|
+
}
|
|
222
|
+
current = real;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
const finalPath = path.join(current, segments[segments.length - 1]);
|
|
226
|
+
try {
|
|
227
|
+
const finalStat = fs.lstatSync(finalPath);
|
|
228
|
+
if (finalStat.isSymbolicLink()) {
|
|
229
|
+
throw this.createSecurityRejection("ArtifactPathIsSymlink", itemPath);
|
|
230
|
+
}
|
|
231
|
+
if (!finalStat.isFile() && !finalStat.isDirectory()) {
|
|
232
|
+
throw this.createSecurityRejection("ArtifactPathIsSpecialFile", itemPath);
|
|
233
|
+
}
|
|
234
|
+
if (finalStat.isFile() && finalStat.nlink > 1) {
|
|
235
|
+
throw this.createSecurityRejection("ArtifactPathIsHardLinked", itemPath);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
catch (err) {
|
|
239
|
+
if (err && err.nonRetryable) {
|
|
240
|
+
throw err;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return finalPath;
|
|
244
|
+
}
|
|
104
245
|
isCaseInsensitiveFilesystem() {
|
|
105
246
|
if (!this._hasDeterminedFilesystemCaseSensitivity) {
|
|
106
247
|
try {
|
|
@@ -120,7 +261,7 @@ class FilesystemProvider {
|
|
|
120
261
|
var items = [];
|
|
121
262
|
fs.readdir(itemsPath, (error, files) => {
|
|
122
263
|
if (!!error) {
|
|
123
|
-
logger_1.Logger.
|
|
264
|
+
logger_1.Logger.messageExternalOutput(tl.loc("UnableToReadDirectory", itemsPath, error));
|
|
124
265
|
reject(error);
|
|
125
266
|
return;
|
|
126
267
|
}
|
package/Providers/webProvider.js
CHANGED
|
@@ -56,7 +56,7 @@ class WebProvider {
|
|
|
56
56
|
res.message.on('error', (error) => {
|
|
57
57
|
if (zipStream) {
|
|
58
58
|
zipStream.destroy(error);
|
|
59
|
-
logger_1.Logger.
|
|
59
|
+
logger_1.Logger.messageExternalOutput(error.toString());
|
|
60
60
|
}
|
|
61
61
|
reject(error);
|
|
62
62
|
});
|
|
@@ -91,7 +91,7 @@ class WebProvider {
|
|
|
91
91
|
res.readBody().then((body) => {
|
|
92
92
|
fs.readFile(this.getTemplateFilePath(), 'utf8', (err, templateFileContent) => {
|
|
93
93
|
if (err) {
|
|
94
|
-
logger_1.Logger.
|
|
94
|
+
logger_1.Logger.messageExternalOutput(err ? JSON.stringify(err) : "");
|
|
95
95
|
reject(err);
|
|
96
96
|
return;
|
|
97
97
|
}
|
|
@@ -108,7 +108,7 @@ class WebProvider {
|
|
|
108
108
|
resolve(items);
|
|
109
109
|
}
|
|
110
110
|
catch (error) {
|
|
111
|
-
logger_1.Logger.
|
|
111
|
+
logger_1.Logger.messageExternalOutput(tl.loc("FailedToParseResponse", body, error));
|
|
112
112
|
reject(error);
|
|
113
113
|
}
|
|
114
114
|
});
|
package/README.md
CHANGED
|
@@ -14,6 +14,56 @@ To use Artifact engine in your tasks or app have a look at [E2E.ts](E2ETests/jen
|
|
|
14
14
|
## Usage
|
|
15
15
|

|
|
16
16
|
|
|
17
|
+
### External-output logging
|
|
18
|
+
|
|
19
|
+
Use `Logger.infoExternalOutput`, `Logger.errorExternalOutput`, and
|
|
20
|
+
`Logger.messageExternalOutput` for external or mixed-origin messages such as
|
|
21
|
+
artifact paths, metadata, remote responses, and error text. A trusted format
|
|
22
|
+
string does not make interpolated external data trusted.
|
|
23
|
+
|
|
24
|
+
The pipeline feature `EnableArtifactEngineExternalOutput` enables configurable
|
|
25
|
+
handling through task-lib's matching debug, error, and ordinary output helpers.
|
|
26
|
+
The task receives the feature as
|
|
27
|
+
`DISTRIBUTEDTASK_TASKS_ENABLEARTIFACTENGINEEXTERNALOUTPUT=true`.
|
|
28
|
+
|
|
29
|
+
```ts
|
|
30
|
+
const options = new ArtifactEngineOptions();
|
|
31
|
+
options.vsoCommandMode = 'blockAndTelemetry';
|
|
32
|
+
await engine.processItems(sourceProvider, destinationProvider, options);
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
| Mode | External-output behavior when the feature is enabled |
|
|
36
|
+
| --- | --- |
|
|
37
|
+
| `blockOnly` (V2 default) | Block commands without command telemetry. |
|
|
38
|
+
| `blockAndTelemetry` | Block commands and report command telemetry. |
|
|
39
|
+
| `telemetryOnly` | Report commands without blocking them. |
|
|
40
|
+
| `disabled` | Neither block commands nor report command telemetry. |
|
|
41
|
+
|
|
42
|
+
V2 defaults to `blockOnly`, preserving its existing blocking behavior for
|
|
43
|
+
ordinary messages. V1's compatibility default is `disabled`; do not assume
|
|
44
|
+
identical defaults across major versions. Explicitly choosing `disabled` or
|
|
45
|
+
`telemetryOnly` with the feature enabled opts out of blocking, including for
|
|
46
|
+
ordinary messages that V2 previously blocked.
|
|
47
|
+
|
|
48
|
+
With the feature absent or off, the new methods delegate to their corresponding
|
|
49
|
+
legacy methods. `Logger.logMessage` keeps its existing task-lib filtering and
|
|
50
|
+
native `os.EOL` newline, independently of the new mode. `Logger.logInfo` and
|
|
51
|
+
`Logger.logError` retain their existing debug/error behavior. Disabling the
|
|
52
|
+
feature does not disable V2's pre-existing ordinary-output protection.
|
|
53
|
+
|
|
54
|
+
Keep the legacy APIs for trusted internal messages. Only external/mixed-origin
|
|
55
|
+
call sites, including verbose artifact-path rows, use the new boundary. Internal
|
|
56
|
+
counts, progress, headings, worker status, and intentional telemetry remain
|
|
57
|
+
unchanged. Filtering changes only the logged copy, not artifact paths or data.
|
|
58
|
+
|
|
59
|
+
Logger's mode is process-wide static state, like `verbose`. Every `processItems`
|
|
60
|
+
call resets it, including when options or the property are omitted. Concurrent
|
|
61
|
+
engines with differing modes are not isolated.
|
|
62
|
+
|
|
63
|
+
Consuming-task updates, server-side feature registration/propagation, package
|
|
64
|
+
publication, and rollout are separate follow-ups. Keep existing task-level
|
|
65
|
+
filtering in place.
|
|
66
|
+
|
|
17
67
|
## Architecture
|
|
18
68
|

|
|
19
69
|
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "Fehler beim Analysieren des Antworttexts: %s. Erhaltener Fehler: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "Download von \"%s\" nach \"%s\" wird durchgeführt.",
|
|
5
5
|
"loc.messages.DownloadedTo": "Download von \"%s\" nach \"%s\" wurde durchgeführt.",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "Der Artifactpfad \"%s\" liegt außerhalb des Downloadverzeichnisses.",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "Der Artifactpfad \"%s\" ist bereits ein symbolischer Link und wurde abgelehnt.",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "Der Artifactpfad \"%s\" ist bereits eine Datei mit einem Hardlink und wurde abgelehnt.",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "Der Artifactpfad \"%s\" ist bereits eine Datei, die kein regulärer Dateityp ist (z. B. ein Gerät, eine Pipe oder ein Socket), und wurde abgelehnt.",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "Das Verzeichnis \"%s\" kann nicht gelesen werden. Fehler: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "Download von \"%s\" wird erneut versucht, Wiederholungsanzahl: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "Die Verarbeitung des Elements \"%s\" wird übersprungen.",
|
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
"loc.messages.DownloadingTo": "Downloading %s to %s",
|
|
5
5
|
"loc.messages.DownloadedTo": "Downloaded %s to %s",
|
|
6
6
|
"loc.messages.InvalidArtifactPath": "Artifact path %s resolves outside the download directory.",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "Artifact path %s already exists as a symbolic link and was rejected.",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "Artifact path %s already exists as a hard-linked file and was rejected.",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "Artifact path %s already exists as a non-regular file (e.g. a device, pipe, or socket) and was rejected.",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "Unable to read directory %s. Error: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "Retrying download of %s, retry count: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "Skipped processing item %s",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "No se pudo analizar el cuerpo de la respuesta: %s. Error: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "Descargando %s en %s",
|
|
5
5
|
"loc.messages.DownloadedTo": "%s descargado en %s",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "La ruta de acceso del artefacto %s se resuelve fuera del directorio de descarga.",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "La ruta de acceso del artefacto %s ya existe como vínculo simbólico y se rechazó.",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "La ruta de acceso del artefacto %s ya existe como archivo con vínculo físico y se rechazó.",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "La ruta de acceso del artefacto %s ya existe como archivo no habitual (por ejemplo, un dispositivo, una canalización o un socket) y se rechazó.",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "No se puede leer el directorio %s. Error: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "Reintentando la descarga de %s, número de reintentos: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "Procesamiento de %s omitido",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "Échec d’analyse du corps de la réponse : %s, erreur : %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "Téléchargement de %s à %s",
|
|
5
5
|
"loc.messages.DownloadedTo": "Téléchargé de %s à %s",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "Le chemin d’accès de l’artefact %s se résout en dehors du répertoire de téléchargement.",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "Le chemin d’accès de l’artefact %s existe déjà en tant que lien symbolique et a été rejeté.",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "Le chemin d’accès de l’artefact %s existe déjà en tant que fichier à liens physiques et a été rejeté.",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "Le chemin d’accès de l’artefact %s existe déjà en tant que fichier non standard (par exemple, un périphérique, un tube ou un socket) et a été rejeté.",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "Impossible de lire le répertoire %s. Erreur : %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "Nouvelle tentative de téléchargement de %s, nombre de tentatives : %s",
|
|
9
12
|
"loc.messages.SkippingItem": "Traitement de l’élément ignoré %s",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "Non è stato possibile analizzare il corpo della risposta: %s. Errore restituito: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "Download di %s in %s",
|
|
5
5
|
"loc.messages.DownloadedTo": "%s scaricato in %s",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "Il percorso dell'artifact %s viene risolto all'esterno della directory di download.",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "Il percorso dell'artifact %s esiste già come collegamento simbolico ed è stato rifiutato.",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "Il percorso dell'artifact %s esiste già come file con collegamento fisico ed è stato rifiutato.",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "Il percorso dell'artifact %s esiste già come file non normale (ad esempio un dispositivo, una pipe o un socket) ed è stato rifiutato.",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "Non è possibile leggere la directory %s. Errore: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "Nuovo tentativo di download di %s. Conteggio dei tentativi: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "Elaborazione dell'elemento %s ignorata",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "応答本文を解析できませんでした: %s、エラー発生: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "%s を %s へダウンロード中",
|
|
5
5
|
"loc.messages.DownloadedTo": "%s を %s へダウンロード済み",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "アーティファクト パス %s はダウンロード ディレクトリの外部に解決されます。",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "アーティファクト パス %s は既にシンボリック リンクとして存在するため、拒否されました。",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "アーティファクト パス %s は既にハード リンクされたファイルとして存在するため、拒否されました。",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "アーティファクト パス %s は既に特殊ファイル (デバイス、パイプ、ソケットなど) として存在するため、拒否されました。",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "ディレクトリ %s を読み取ることができません。エラー: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "%s のダウンロードを再試行しています。再試行回数: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "項目 %s の処理をスキップしました",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "응답 본문 %s을(를) 구문 분석하지 못했습니다. 오류: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "%s을(를) %s에 다운로드하는 중",
|
|
5
5
|
"loc.messages.DownloadedTo": "%s을(를) %s에 다운로드함",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "아티팩트 경로 %s이(가) 다운로드 디렉터리 외부로 확인됩니다.",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "아티팩트 경로 %s이(가) 이미 심볼릭 링크로 존재하여 거부되었습니다.",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "아티팩트 경로 %s이(가) 이미 하드 링크된 파일로 존재하여 거부되었습니다.",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "아티팩트 경로 %s이(가) 이미 일반 파일이 아닌 파일(예: 장치, 파이프 또는 소켓)로 존재하여 거부되었습니다.",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "디렉터리 %s을(를) 읽을 수 없습니다. 오류: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "%s 다운로드를 다시 시도하는 중, 다시 시도 횟수: %s회",
|
|
9
12
|
"loc.messages.SkippingItem": "항목 %s의 처리를 건너뜀",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "Не удалось проанализировать текст ответа: %s. Ошибка: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "Идет скачивание %s в %s",
|
|
5
5
|
"loc.messages.DownloadedTo": "Скачано %s в %s",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "Путь к артефакту %s указывает за пределы каталога скачивания.",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "Путь к артефакту %s уже существует как символическая ссылка и был отклонен.",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "Путь к артефакту %s уже существует как файл с жесткой ссылкой и был отклонен.",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "Путь к артефакту %s уже существует как нестандартный файл (например, устройство, канал или сокет) и был отклонен.",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "Не удается считать каталог %s. Ошибка: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "Повторная попытка скачать %s, всего попыток: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "Пропущена обработка элемента %s",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "未能分析响应正文: %s,显示错误: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "正在将 %s 下载至 %s",
|
|
5
5
|
"loc.messages.DownloadedTo": "已将 %s 下载至 %s",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "项目路径 %s 解析到下载目录之外。",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "项目路径 %s 已作为符号链接存在,已被拒绝。",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "项目路径 %s 已作为硬链接文件存在,已被拒绝。",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "项目路径 %s 已作为非常规文件(例如设备、管道或套接字)存在,已被拒绝。",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "未能读取目录 %s。错误: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "重新尝试下载 %s,重试计数: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "已跳过处理项目 %s",
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
"loc.messages.FailedToParseResponse": "無法剖析回應主體: %s,收到錯誤: %s",
|
|
4
4
|
"loc.messages.DownloadingTo": "正在將 %s 下載到 %s",
|
|
5
5
|
"loc.messages.DownloadedTo": "已將 %s 下載到 %s",
|
|
6
|
-
"loc.messages.InvalidArtifactPath": "
|
|
6
|
+
"loc.messages.InvalidArtifactPath": "成品路徑 %s 解析到下載目錄外部。",
|
|
7
|
+
"loc.messages.ArtifactPathIsSymlink": "成品路徑 %s 已作為符號連結存在,已遭拒絕。",
|
|
8
|
+
"loc.messages.ArtifactPathIsHardLinked": "成品路徑 %s 已作為硬連結檔案存在,已遭拒絕。",
|
|
9
|
+
"loc.messages.ArtifactPathIsSpecialFile": "成品路徑 %s 已作為非一般檔案 (例如裝置、管道或通訊端) 存在,已遭拒絕。",
|
|
7
10
|
"loc.messages.UnableToReadDirectory": "無法讀取目錄 %s。錯誤: %s",
|
|
8
11
|
"loc.messages.RetryingDownload": "正在重試下載 %s,重試次數: %s",
|
|
9
12
|
"loc.messages.SkippingItem": "已跳過處理項目 %s",
|
package/lib.json
CHANGED
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
"DownloadingTo": "Downloading %s to %s",
|
|
6
6
|
"DownloadedTo": "Downloaded %s to %s",
|
|
7
7
|
"InvalidArtifactPath": "Artifact path %s resolves outside the download directory.",
|
|
8
|
+
"ArtifactPathIsSymlink": "Artifact path %s already exists as a symbolic link and was rejected.",
|
|
9
|
+
"ArtifactPathIsHardLinked": "Artifact path %s already exists as a hard-linked file and was rejected.",
|
|
10
|
+
"ArtifactPathIsSpecialFile": "Artifact path %s already exists as a non-regular file (e.g. a device, pipe, or socket) and was rejected.",
|
|
8
11
|
"UnableToReadDirectory": "Unable to read directory %s. Error: %s",
|
|
9
12
|
"RetryingDownload": "Retrying download of %s, retry count: %s",
|
|
10
13
|
"SkippingItem": "Skipped processing item %s",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "artifact-engine",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.283.1",
|
|
4
4
|
"description": "Artifact Engine to download artifacts from jenkins, teamcity, vsts",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"email": "omeshp@microsoft.com"
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"azure-pipelines-task-lib": "^5.281.
|
|
19
|
+
"azure-pipelines-task-lib": "^5.281.2",
|
|
20
20
|
"handlebars": "4.7.10",
|
|
21
21
|
"typed-rest-client": "^3.0.0"
|
|
22
22
|
},
|