artifact-engine 1.280.0 → 1.282.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Engine/artifactEngine.js +12 -10
- package/Engine/artifactEngineOptions.d.ts +2 -0
- package/Engine/artifactEngineOptions.js +1 -0
- package/Engine/logger.d.ts +8 -0
- package/Engine/logger.js +31 -2
- package/Providers/filesystemProvider.js +3 -3
- package/Providers/webProvider.js +3 -3
- package/README.md +50 -0
- package/package.json +4 -4
package/Engine/artifactEngine.js
CHANGED
|
@@ -16,6 +16,7 @@ class ArtifactEngine {
|
|
|
16
16
|
}
|
|
17
17
|
processItems(sourceProvider, destProvider, artifactEngineOptions) {
|
|
18
18
|
var artifactDownloadTicketsPromise = new Promise((resolve, reject) => {
|
|
19
|
+
var _a;
|
|
19
20
|
const workers = [];
|
|
20
21
|
artifactEngineOptions = artifactEngineOptions || new artifactEngineOptions_1.ArtifactEngineOptions();
|
|
21
22
|
this.createPatternList(artifactEngineOptions);
|
|
@@ -23,6 +24,7 @@ class ArtifactEngine {
|
|
|
23
24
|
this.isCaseInsensitiveArtifactMatchingEnabled(destProvider);
|
|
24
25
|
this.artifactItemStore.flush();
|
|
25
26
|
logger_1.Logger.verbose = artifactEngineOptions.verbose;
|
|
27
|
+
logger_1.Logger.vsoCommandMode = (_a = artifactEngineOptions.vsoCommandMode) !== null && _a !== void 0 ? _a : 'disabled';
|
|
26
28
|
this.logger = new logger_1.Logger(this.artifactItemStore);
|
|
27
29
|
this.logger.logProgress();
|
|
28
30
|
sourceProvider.artifactItemStore = this.artifactItemStore;
|
|
@@ -66,7 +68,7 @@ class ArtifactEngine {
|
|
|
66
68
|
}
|
|
67
69
|
else {
|
|
68
70
|
this.artifactItemStore.increaseRetryCount(item);
|
|
69
|
-
logger_1.Logger.
|
|
71
|
+
logger_1.Logger.messageExternalOutput(tl.loc("RetryingDownload", item.path, (retryCount + 1)));
|
|
70
72
|
// retry interval sequence 5s 15s 45s 135s
|
|
71
73
|
// 2 min is the time limit for freeing of ports
|
|
72
74
|
setTimeout(() => this
|
|
@@ -90,24 +92,24 @@ class ArtifactEngine {
|
|
|
90
92
|
flipNegate: false
|
|
91
93
|
};
|
|
92
94
|
if (tl.match([pathToMatch], this.patternList, null, matchOptions).length > 0) {
|
|
93
|
-
logger_1.Logger.
|
|
95
|
+
logger_1.Logger.infoExternalOutput("Processing " + item.path);
|
|
94
96
|
sourceProvider.getArtifactItem(item).then((contentStream) => {
|
|
95
|
-
logger_1.Logger.
|
|
97
|
+
logger_1.Logger.infoExternalOutput("Got download stream for item: " + item.path);
|
|
96
98
|
destProvider.putArtifactItem(item, contentStream)
|
|
97
99
|
.then((item) => {
|
|
98
100
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
99
101
|
resolve();
|
|
100
102
|
}, (err) => {
|
|
101
|
-
logger_1.Logger.
|
|
103
|
+
logger_1.Logger.infoExternalOutput("Error placing file " + item.path + ": " + err);
|
|
102
104
|
retryIfRequired(err);
|
|
103
105
|
});
|
|
104
106
|
}, (err) => {
|
|
105
|
-
logger_1.Logger.
|
|
107
|
+
logger_1.Logger.infoExternalOutput("Error getting file " + item.path + ": " + err);
|
|
106
108
|
retryIfRequired(err);
|
|
107
109
|
});
|
|
108
110
|
}
|
|
109
111
|
else {
|
|
110
|
-
logger_1.Logger.
|
|
112
|
+
logger_1.Logger.messageExternalOutput(tl.loc("SkippingItem", pathToMatch));
|
|
111
113
|
this.artifactItemStore.updateState(item, models.TicketState.Skipped);
|
|
112
114
|
resolve();
|
|
113
115
|
}
|
|
@@ -130,14 +132,14 @@ class ArtifactEngine {
|
|
|
130
132
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
131
133
|
resolve();
|
|
132
134
|
}, (err) => {
|
|
133
|
-
logger_1.Logger.
|
|
135
|
+
logger_1.Logger.infoExternalOutput("Error creating folder " + item.path + ": " + err);
|
|
134
136
|
retryIfRequired(err);
|
|
135
137
|
});
|
|
136
138
|
}
|
|
137
139
|
logger_1.Logger.logInfo("Enqueued " + items.length + " for processing.");
|
|
138
140
|
resolve();
|
|
139
141
|
}, (err) => {
|
|
140
|
-
logger_1.Logger.
|
|
142
|
+
logger_1.Logger.infoExternalOutput("Error getting " + item.path + ":" + err);
|
|
141
143
|
retryIfRequired(err);
|
|
142
144
|
});
|
|
143
145
|
}
|
|
@@ -178,11 +180,11 @@ exports.ArtifactEngine = ArtifactEngine;
|
|
|
178
180
|
tl.setResourcePath(path.join(path.dirname(__dirname), 'lib.json'));
|
|
179
181
|
process.on('unhandledRejection', (err) => {
|
|
180
182
|
ci.publishEvent('reliability', { issueType: 'unhandledRejection', errorMessage: JSON.stringify(err, Object.getOwnPropertyNames(err)) });
|
|
181
|
-
logger_1.Logger.
|
|
183
|
+
logger_1.Logger.errorExternalOutput(tl.loc("UnhandledRejection", err));
|
|
182
184
|
throw err;
|
|
183
185
|
});
|
|
184
186
|
process.on('uncaughtException', (err) => {
|
|
185
187
|
ci.publishEvent('reliability', { issueType: 'uncaughtException', errorMessage: JSON.stringify(err, Object.getOwnPropertyNames(err)) });
|
|
186
|
-
logger_1.Logger.
|
|
188
|
+
logger_1.Logger.errorExternalOutput(tl.loc("UnhandledException", err));
|
|
187
189
|
throw err;
|
|
188
190
|
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { VsoCommandMode } from 'azure-pipelines-task-lib/externaloutput';
|
|
1
2
|
export declare class ArtifactEngineOptions {
|
|
2
3
|
retryLimit: number;
|
|
3
4
|
retryIntervalInSeconds: number;
|
|
@@ -5,4 +6,5 @@ export declare class ArtifactEngineOptions {
|
|
|
5
6
|
parallelProcessingLimit: number;
|
|
6
7
|
itemPattern: string;
|
|
7
8
|
verbose: boolean;
|
|
9
|
+
vsoCommandMode?: VsoCommandMode;
|
|
8
10
|
}
|
package/Engine/logger.d.ts
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
|
+
import type { VsoCommandMode } from 'azure-pipelines-task-lib/externaloutput';
|
|
1
2
|
import { ArtifactItemStore } from '../Store/artifactItemStore';
|
|
2
3
|
export declare class Logger {
|
|
3
4
|
constructor(store: ArtifactItemStore);
|
|
4
5
|
static logInfo(message: string): void;
|
|
5
6
|
static logError(message: string): void;
|
|
6
7
|
static logMessage(message: string): void;
|
|
8
|
+
/** Logs external text while preserving debug output semantics. */
|
|
9
|
+
static infoExternalOutput(message: string): void;
|
|
10
|
+
/** Logs external text while preserving error output semantics. */
|
|
11
|
+
static errorExternalOutput(message: string): void;
|
|
12
|
+
/** Logs external text rather than trusted internal messages or agent commands. */
|
|
13
|
+
static messageExternalOutput(message: string): void;
|
|
7
14
|
logProgress(): void;
|
|
8
15
|
logSummary(): void;
|
|
9
16
|
private padText;
|
|
10
17
|
static verbose: boolean;
|
|
18
|
+
static vsoCommandMode: VsoCommandMode;
|
|
11
19
|
private store;
|
|
12
20
|
private startTime;
|
|
13
21
|
}
|
package/Engine/logger.js
CHANGED
|
@@ -10,10 +10,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.Logger = void 0;
|
|
13
|
+
const tl = require("azure-pipelines-task-lib/task");
|
|
13
14
|
const Models_1 = require("../Models");
|
|
14
15
|
const ticketState_1 = require("../Models/ticketState");
|
|
15
|
-
var tl = require('azure-pipelines-task-lib');
|
|
16
16
|
const ci = require("./cilogger");
|
|
17
|
+
const externalOutputFeature = 'EnableArtifactEngineExternalOutput';
|
|
17
18
|
class Logger {
|
|
18
19
|
constructor(store) {
|
|
19
20
|
this.store = store;
|
|
@@ -28,6 +29,33 @@ class Logger {
|
|
|
28
29
|
static logMessage(message) {
|
|
29
30
|
console.log(message);
|
|
30
31
|
}
|
|
32
|
+
/** Logs external text while preserving debug output semantics. */
|
|
33
|
+
static infoExternalOutput(message) {
|
|
34
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
35
|
+
tl.debugExternalOutput(message, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
Logger.logInfo(message);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** Logs external text while preserving error output semantics. */
|
|
42
|
+
static errorExternalOutput(message) {
|
|
43
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
44
|
+
tl.errorExternalOutput(message, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
Logger.logError(message);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** Logs external text rather than trusted internal messages or agent commands. */
|
|
51
|
+
static messageExternalOutput(message) {
|
|
52
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
53
|
+
tl.writeExternalOutput(message + '\n', { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
Logger.logMessage(message);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
31
59
|
logProgress() {
|
|
32
60
|
if (Logger.verbose) {
|
|
33
61
|
var progressLogger = () => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -96,7 +124,7 @@ class Logger {
|
|
|
96
124
|
tl.debug(this.padText("", maxPathLength + 25 + 25 + 10 + 10 + 15, '-'));
|
|
97
125
|
fileTickets.forEach(ticket => {
|
|
98
126
|
var duration = (ticket.finishTime.valueOf() - ticket.startTime.valueOf()) / 1000 + " secs";
|
|
99
|
-
|
|
127
|
+
Logger.infoExternalOutput("| " + this.padText(ticket.artifactItem.path, maxPathLength) + " | " + this.padText(ticket.startTime.toISOString(), 25) + " | " + this.padText(ticket.finishTime.toISOString(), 25) + " | " + this.padText(duration, 10) + " | " + this.padText(ticket.state.toString().toUpperCase(), 10) + "|");
|
|
100
128
|
tl.debug(this.padText("", maxPathLength + 25 + 25 + 10 + 10 + 15, '-'));
|
|
101
129
|
});
|
|
102
130
|
}
|
|
@@ -109,3 +137,4 @@ class Logger {
|
|
|
109
137
|
}
|
|
110
138
|
}
|
|
111
139
|
exports.Logger = Logger;
|
|
140
|
+
Logger.vsoCommandMode = 'disabled';
|
|
@@ -68,11 +68,11 @@ class FilesystemProvider {
|
|
|
68
68
|
try {
|
|
69
69
|
// create parent folder if it has not already been created
|
|
70
70
|
tl.mkdirP(folder);
|
|
71
|
-
logger_1.Logger.
|
|
71
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadingTo", item.path, outputItemPath));
|
|
72
72
|
const outputStream = fs.createWriteStream(outputItemPath);
|
|
73
73
|
stream.pipe(outputStream);
|
|
74
74
|
stream.on("end", () => {
|
|
75
|
-
logger_1.Logger.
|
|
75
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadedTo", item.path, outputItemPath));
|
|
76
76
|
if (!item.metadata) {
|
|
77
77
|
item.metadata = {};
|
|
78
78
|
}
|
|
@@ -120,7 +120,7 @@ class FilesystemProvider {
|
|
|
120
120
|
var items = [];
|
|
121
121
|
fs.readdir(itemsPath, (error, files) => {
|
|
122
122
|
if (!!error) {
|
|
123
|
-
logger_1.Logger.
|
|
123
|
+
logger_1.Logger.messageExternalOutput(tl.loc("UnableToReadDirectory", itemsPath, error));
|
|
124
124
|
reject(error);
|
|
125
125
|
return;
|
|
126
126
|
}
|
package/Providers/webProvider.js
CHANGED
|
@@ -56,7 +56,7 @@ class WebProvider {
|
|
|
56
56
|
res.message.on('error', (error) => {
|
|
57
57
|
if (zipStream) {
|
|
58
58
|
zipStream.destroy(error);
|
|
59
|
-
logger_1.Logger.
|
|
59
|
+
logger_1.Logger.messageExternalOutput(error.toString());
|
|
60
60
|
}
|
|
61
61
|
reject(error);
|
|
62
62
|
});
|
|
@@ -91,7 +91,7 @@ class WebProvider {
|
|
|
91
91
|
res.readBody().then((body) => {
|
|
92
92
|
fs.readFile(this.getTemplateFilePath(), 'utf8', (err, templateFileContent) => {
|
|
93
93
|
if (err) {
|
|
94
|
-
logger_1.Logger.
|
|
94
|
+
logger_1.Logger.messageExternalOutput(err ? JSON.stringify(err) : "");
|
|
95
95
|
reject(err);
|
|
96
96
|
return;
|
|
97
97
|
}
|
|
@@ -108,7 +108,7 @@ class WebProvider {
|
|
|
108
108
|
resolve(items);
|
|
109
109
|
}
|
|
110
110
|
catch (error) {
|
|
111
|
-
logger_1.Logger.
|
|
111
|
+
logger_1.Logger.messageExternalOutput(tl.loc("FailedToParseResponse", body, error));
|
|
112
112
|
reject(error);
|
|
113
113
|
}
|
|
114
114
|
});
|
package/README.md
CHANGED
|
@@ -11,6 +11,56 @@ To use Artifact engine in your tasks or app have a look at [E2E.ts](E2ETests/jen
|
|
|
11
11
|
## Usage
|
|
12
12
|

|
|
13
13
|
|
|
14
|
+
### External-output logging
|
|
15
|
+
|
|
16
|
+
`ArtifactEngineOptions.vsoCommandMode` controls handling of agent logging-command
|
|
17
|
+
markers in external log messages. The pipeline feature
|
|
18
|
+
`EnableArtifactEngineExternalOutput` must also be enabled (the task receives
|
|
19
|
+
`DISTRIBUTEDTASK_TASKS_ENABLEARTIFACTENGINEEXTERNALOUTPUT=true`).
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
const options = new ArtifactEngineOptions();
|
|
23
|
+
options.vsoCommandMode = 'blockAndTelemetry';
|
|
24
|
+
await engine.processItems(sourceProvider, destinationProvider, options);
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
| Mode | External-output behavior when the feature is enabled |
|
|
28
|
+
| --- | --- |
|
|
29
|
+
| `disabled` (default) | Neither block commands nor report command telemetry. |
|
|
30
|
+
| `telemetryOnly` | Report commands without blocking them. |
|
|
31
|
+
| `blockOnly` | Block commands without command telemetry. |
|
|
32
|
+
| `blockAndTelemetry` | Block commands and report command telemetry. |
|
|
33
|
+
|
|
34
|
+
The feature is off when absent. With the feature off, external messages use the
|
|
35
|
+
original logging behavior regardless of the mode. Enabling the feature alone is
|
|
36
|
+
not a mitigation: the caller must explicitly select a blocking mode.
|
|
37
|
+
`telemetryOnly` is observation, not protection.
|
|
38
|
+
|
|
39
|
+
For library contributors, use `Logger.infoExternalOutput`,
|
|
40
|
+
`Logger.errorExternalOutput`, and `Logger.messageExternalOutput` for external
|
|
41
|
+
or mixed-origin messages, including artifact paths, metadata,
|
|
42
|
+
response bodies, and error text. These methods use task-lib's external-output
|
|
43
|
+
helpers, preserving debug, error, and ordinary output respectively:
|
|
44
|
+
|
|
45
|
+
```ts
|
|
46
|
+
Logger.messageExternalOutput(tl.loc('DownloadingTo', item.path, outputItemPath));
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Keep `Logger.logInfo`, `Logger.logError`, and `Logger.logMessage` for trusted
|
|
50
|
+
internal messages. A trusted format string does not make interpolated external
|
|
51
|
+
values trusted. Only the logged text is filtered; artifact data and paths are
|
|
52
|
+
not modified. The verbose summary's artifact-path row uses the external API;
|
|
53
|
+
internal counts, progress, headings, and intentional telemetry commands do not.
|
|
54
|
+
|
|
55
|
+
The logging mode is process-wide static Logger state, like `verbose`.
|
|
56
|
+
Each `processItems` invocation resets it, including when options are omitted.
|
|
57
|
+
Concurrent engine runs with different modes are not isolated.
|
|
58
|
+
|
|
59
|
+
Consuming tasks must update their bundled library and explicitly pass a mode.
|
|
60
|
+
Server-side feature registration/propagation and package/task rollout are
|
|
61
|
+
separate from this library change. Existing task-level output filtering should
|
|
62
|
+
not be removed as part of adopting this API.
|
|
63
|
+
|
|
14
64
|
## Architecture
|
|
15
65
|

|
|
16
66
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "artifact-engine",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.282.0",
|
|
4
4
|
"description": "Artifact Engine to download artifacts from jenkins, teamcity, vsts",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
"email": "omeshp@microsoft.com"
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"azure-pipelines-task-lib": "^5.
|
|
20
|
-
"handlebars": "4.7.
|
|
19
|
+
"azure-pipelines-task-lib": "^5.281.2",
|
|
20
|
+
"handlebars": "4.7.10",
|
|
21
21
|
"minimatch": "3.1.5",
|
|
22
22
|
"tunnel": "0.0.4"
|
|
23
23
|
},
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"nock": "^14.0.6",
|
|
38
38
|
"nyc": "^15.0.0",
|
|
39
39
|
"sinon": "4.0.1",
|
|
40
|
-
"typescript": "4.
|
|
40
|
+
"typescript": "4.9.5",
|
|
41
41
|
"xml2js": "^0.6.2"
|
|
42
42
|
},
|
|
43
43
|
"files": [
|