artifact-engine 2.281.2 → 2.282.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Engine/artifactEngine.js +12 -10
- package/Engine/artifactEngineOptions.d.ts +2 -0
- package/Engine/artifactEngineOptions.js +1 -0
- package/Engine/logger.d.ts +8 -0
- package/Engine/logger.js +32 -3
- package/Providers/filesystemProvider.js +3 -3
- package/Providers/webProvider.js +3 -3
- package/README.md +50 -0
- package/package.json +2 -2
package/Engine/artifactEngine.js
CHANGED
|
@@ -16,6 +16,7 @@ class ArtifactEngine {
|
|
|
16
16
|
}
|
|
17
17
|
processItems(sourceProvider, destProvider, artifactEngineOptions) {
|
|
18
18
|
var artifactDownloadTicketsPromise = new Promise((resolve, reject) => {
|
|
19
|
+
var _a;
|
|
19
20
|
const workers = [];
|
|
20
21
|
artifactEngineOptions = artifactEngineOptions || new artifactEngineOptions_1.ArtifactEngineOptions();
|
|
21
22
|
this.createPatternList(artifactEngineOptions);
|
|
@@ -23,6 +24,7 @@ class ArtifactEngine {
|
|
|
23
24
|
this.isCaseInsensitiveArtifactMatchingEnabled(destProvider);
|
|
24
25
|
this.artifactItemStore.flush();
|
|
25
26
|
logger_1.Logger.verbose = artifactEngineOptions.verbose;
|
|
27
|
+
logger_1.Logger.vsoCommandMode = (_a = artifactEngineOptions.vsoCommandMode) !== null && _a !== void 0 ? _a : 'blockOnly';
|
|
26
28
|
this.logger = new logger_1.Logger(this.artifactItemStore);
|
|
27
29
|
this.logger.logProgress();
|
|
28
30
|
sourceProvider.artifactItemStore = this.artifactItemStore;
|
|
@@ -66,7 +68,7 @@ class ArtifactEngine {
|
|
|
66
68
|
}
|
|
67
69
|
else {
|
|
68
70
|
this.artifactItemStore.increaseRetryCount(item);
|
|
69
|
-
logger_1.Logger.
|
|
71
|
+
logger_1.Logger.messageExternalOutput(tl.loc("RetryingDownload", item.path, (retryCount + 1)));
|
|
70
72
|
// retry interval sequence 5s 15s 45s 135s
|
|
71
73
|
// 2 min is the time limit for freeing of ports
|
|
72
74
|
setTimeout(() => this
|
|
@@ -90,24 +92,24 @@ class ArtifactEngine {
|
|
|
90
92
|
flipNegate: false
|
|
91
93
|
};
|
|
92
94
|
if (tl.match([pathToMatch], this.patternList, null, matchOptions).length > 0) {
|
|
93
|
-
logger_1.Logger.
|
|
95
|
+
logger_1.Logger.infoExternalOutput("Processing " + item.path);
|
|
94
96
|
sourceProvider.getArtifactItem(item).then((contentStream) => {
|
|
95
|
-
logger_1.Logger.
|
|
97
|
+
logger_1.Logger.infoExternalOutput("Got download stream for item: " + item.path);
|
|
96
98
|
destProvider.putArtifactItem(item, contentStream)
|
|
97
99
|
.then((item) => {
|
|
98
100
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
99
101
|
resolve();
|
|
100
102
|
}, (err) => {
|
|
101
|
-
logger_1.Logger.
|
|
103
|
+
logger_1.Logger.infoExternalOutput("Error placing file " + item.path + ": " + err);
|
|
102
104
|
retryIfRequired(err);
|
|
103
105
|
});
|
|
104
106
|
}, (err) => {
|
|
105
|
-
logger_1.Logger.
|
|
107
|
+
logger_1.Logger.infoExternalOutput("Error getting file " + item.path + ": " + err);
|
|
106
108
|
retryIfRequired(err);
|
|
107
109
|
});
|
|
108
110
|
}
|
|
109
111
|
else {
|
|
110
|
-
logger_1.Logger.
|
|
112
|
+
logger_1.Logger.messageExternalOutput(tl.loc("SkippingItem", pathToMatch));
|
|
111
113
|
this.artifactItemStore.updateState(item, models.TicketState.Skipped);
|
|
112
114
|
resolve();
|
|
113
115
|
}
|
|
@@ -130,14 +132,14 @@ class ArtifactEngine {
|
|
|
130
132
|
this.artifactItemStore.updateState(item, models.TicketState.Processed);
|
|
131
133
|
resolve();
|
|
132
134
|
}, (err) => {
|
|
133
|
-
logger_1.Logger.
|
|
135
|
+
logger_1.Logger.infoExternalOutput("Error creating folder " + item.path + ": " + err);
|
|
134
136
|
retryIfRequired(err);
|
|
135
137
|
});
|
|
136
138
|
}
|
|
137
139
|
logger_1.Logger.logInfo("Enqueued " + items.length + " for processing.");
|
|
138
140
|
resolve();
|
|
139
141
|
}, (err) => {
|
|
140
|
-
logger_1.Logger.
|
|
142
|
+
logger_1.Logger.infoExternalOutput("Error getting " + item.path + ":" + err);
|
|
141
143
|
retryIfRequired(err);
|
|
142
144
|
});
|
|
143
145
|
}
|
|
@@ -175,11 +177,11 @@ exports.ArtifactEngine = ArtifactEngine;
|
|
|
175
177
|
tl.setResourcePath(path.join(path.dirname(__dirname), 'lib.json'));
|
|
176
178
|
process.on('unhandledRejection', (err) => {
|
|
177
179
|
ci.publishEvent('reliability', { issueType: 'unhandledRejection', errorMessage: JSON.stringify(err, Object.getOwnPropertyNames(err)) });
|
|
178
|
-
logger_1.Logger.
|
|
180
|
+
logger_1.Logger.errorExternalOutput(tl.loc("UnhandledRejection", err));
|
|
179
181
|
throw err;
|
|
180
182
|
});
|
|
181
183
|
process.on('uncaughtException', (err) => {
|
|
182
184
|
ci.publishEvent('reliability', { issueType: 'uncaughtException', errorMessage: JSON.stringify(err, Object.getOwnPropertyNames(err)) });
|
|
183
|
-
logger_1.Logger.
|
|
185
|
+
logger_1.Logger.errorExternalOutput(tl.loc("UnhandledException", err));
|
|
184
186
|
throw err;
|
|
185
187
|
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { VsoCommandMode } from 'azure-pipelines-task-lib/externaloutput';
|
|
1
2
|
export declare class ArtifactEngineOptions {
|
|
2
3
|
retryLimit: number;
|
|
3
4
|
retryIntervalInSeconds: number;
|
|
@@ -5,4 +6,5 @@ export declare class ArtifactEngineOptions {
|
|
|
5
6
|
parallelProcessingLimit: number;
|
|
6
7
|
itemPattern: string;
|
|
7
8
|
verbose: boolean;
|
|
9
|
+
vsoCommandMode?: VsoCommandMode;
|
|
8
10
|
}
|
package/Engine/logger.d.ts
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
|
+
import type { VsoCommandMode } from 'azure-pipelines-task-lib/externaloutput';
|
|
1
2
|
import { ArtifactItemStore } from '../Store/artifactItemStore';
|
|
2
3
|
export declare class Logger {
|
|
3
4
|
constructor(store: ArtifactItemStore);
|
|
4
5
|
static logInfo(message: string): void;
|
|
5
6
|
static logError(message: string): void;
|
|
6
7
|
static logMessage(message: string): void;
|
|
8
|
+
/** Logs external text while preserving debug output semantics. */
|
|
9
|
+
static infoExternalOutput(message: string): void;
|
|
10
|
+
/** Logs external text while preserving error output semantics. */
|
|
11
|
+
static errorExternalOutput(message: string): void;
|
|
12
|
+
/** Logs external text using the configured mode and native line ending. */
|
|
13
|
+
static messageExternalOutput(message: string): void;
|
|
7
14
|
logProgress(): void;
|
|
8
15
|
logSummary(): void;
|
|
9
16
|
private padText;
|
|
10
17
|
static verbose: boolean;
|
|
18
|
+
static vsoCommandMode: VsoCommandMode;
|
|
11
19
|
private store;
|
|
12
20
|
private startTime;
|
|
13
21
|
}
|
package/Engine/logger.js
CHANGED
|
@@ -10,11 +10,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.Logger = void 0;
|
|
13
|
+
const os = require("os");
|
|
14
|
+
const tl = require("azure-pipelines-task-lib/task");
|
|
13
15
|
const Models_1 = require("../Models");
|
|
14
16
|
const ticketState_1 = require("../Models/ticketState");
|
|
15
|
-
var tl = require('azure-pipelines-task-lib');
|
|
16
|
-
var os = require('os');
|
|
17
17
|
const ci = require("./cilogger");
|
|
18
|
+
const externalOutputFeature = 'EnableArtifactEngineExternalOutput';
|
|
18
19
|
class Logger {
|
|
19
20
|
constructor(store) {
|
|
20
21
|
this.store = store;
|
|
@@ -29,6 +30,33 @@ class Logger {
|
|
|
29
30
|
static logMessage(message) {
|
|
30
31
|
tl.writeExternalOutput(message + os.EOL, { source: 'remote' });
|
|
31
32
|
}
|
|
33
|
+
/** Logs external text while preserving debug output semantics. */
|
|
34
|
+
static infoExternalOutput(message) {
|
|
35
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
36
|
+
tl.debugExternalOutput(message, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
Logger.logInfo(message);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Logs external text while preserving error output semantics. */
|
|
43
|
+
static errorExternalOutput(message) {
|
|
44
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
45
|
+
tl.errorExternalOutput(message, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
Logger.logError(message);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Logs external text using the configured mode and native line ending. */
|
|
52
|
+
static messageExternalOutput(message) {
|
|
53
|
+
if (tl.getPipelineFeature(externalOutputFeature)) {
|
|
54
|
+
tl.writeExternalOutput(message + os.EOL, { source: 'remote', vsoCommandMode: Logger.vsoCommandMode });
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
Logger.logMessage(message);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
32
60
|
logProgress() {
|
|
33
61
|
if (Logger.verbose) {
|
|
34
62
|
var progressLogger = () => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -97,7 +125,7 @@ class Logger {
|
|
|
97
125
|
tl.debug(this.padText("", maxPathLength + 25 + 25 + 10 + 10 + 15, '-'));
|
|
98
126
|
fileTickets.forEach(ticket => {
|
|
99
127
|
var duration = (ticket.finishTime.valueOf() - ticket.startTime.valueOf()) / 1000 + " secs";
|
|
100
|
-
|
|
128
|
+
Logger.infoExternalOutput("| " + this.padText(ticket.artifactItem.path, maxPathLength) + " | " + this.padText(ticket.startTime.toISOString(), 25) + " | " + this.padText(ticket.finishTime.toISOString(), 25) + " | " + this.padText(duration, 10) + " | " + this.padText(ticket.state.toString().toUpperCase(), 10) + "|");
|
|
101
129
|
tl.debug(this.padText("", maxPathLength + 25 + 25 + 10 + 10 + 15, '-'));
|
|
102
130
|
});
|
|
103
131
|
}
|
|
@@ -110,3 +138,4 @@ class Logger {
|
|
|
110
138
|
}
|
|
111
139
|
}
|
|
112
140
|
exports.Logger = Logger;
|
|
141
|
+
Logger.vsoCommandMode = 'blockOnly';
|
|
@@ -68,11 +68,11 @@ class FilesystemProvider {
|
|
|
68
68
|
try {
|
|
69
69
|
// create parent folder if it has not already been created
|
|
70
70
|
tl.mkdirP(folder);
|
|
71
|
-
logger_1.Logger.
|
|
71
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadingTo", item.path, outputItemPath));
|
|
72
72
|
const outputStream = fs.createWriteStream(outputItemPath);
|
|
73
73
|
stream.pipe(outputStream);
|
|
74
74
|
stream.on("end", () => {
|
|
75
|
-
logger_1.Logger.
|
|
75
|
+
logger_1.Logger.messageExternalOutput(tl.loc("DownloadedTo", item.path, outputItemPath));
|
|
76
76
|
if (!item.metadata) {
|
|
77
77
|
item.metadata = {};
|
|
78
78
|
}
|
|
@@ -120,7 +120,7 @@ class FilesystemProvider {
|
|
|
120
120
|
var items = [];
|
|
121
121
|
fs.readdir(itemsPath, (error, files) => {
|
|
122
122
|
if (!!error) {
|
|
123
|
-
logger_1.Logger.
|
|
123
|
+
logger_1.Logger.messageExternalOutput(tl.loc("UnableToReadDirectory", itemsPath, error));
|
|
124
124
|
reject(error);
|
|
125
125
|
return;
|
|
126
126
|
}
|
package/Providers/webProvider.js
CHANGED
|
@@ -56,7 +56,7 @@ class WebProvider {
|
|
|
56
56
|
res.message.on('error', (error) => {
|
|
57
57
|
if (zipStream) {
|
|
58
58
|
zipStream.destroy(error);
|
|
59
|
-
logger_1.Logger.
|
|
59
|
+
logger_1.Logger.messageExternalOutput(error.toString());
|
|
60
60
|
}
|
|
61
61
|
reject(error);
|
|
62
62
|
});
|
|
@@ -91,7 +91,7 @@ class WebProvider {
|
|
|
91
91
|
res.readBody().then((body) => {
|
|
92
92
|
fs.readFile(this.getTemplateFilePath(), 'utf8', (err, templateFileContent) => {
|
|
93
93
|
if (err) {
|
|
94
|
-
logger_1.Logger.
|
|
94
|
+
logger_1.Logger.messageExternalOutput(err ? JSON.stringify(err) : "");
|
|
95
95
|
reject(err);
|
|
96
96
|
return;
|
|
97
97
|
}
|
|
@@ -108,7 +108,7 @@ class WebProvider {
|
|
|
108
108
|
resolve(items);
|
|
109
109
|
}
|
|
110
110
|
catch (error) {
|
|
111
|
-
logger_1.Logger.
|
|
111
|
+
logger_1.Logger.messageExternalOutput(tl.loc("FailedToParseResponse", body, error));
|
|
112
112
|
reject(error);
|
|
113
113
|
}
|
|
114
114
|
});
|
package/README.md
CHANGED
|
@@ -14,6 +14,56 @@ To use Artifact engine in your tasks or app have a look at [E2E.ts](E2ETests/jen
|
|
|
14
14
|
## Usage
|
|
15
15
|

|
|
16
16
|
|
|
17
|
+
### External-output logging
|
|
18
|
+
|
|
19
|
+
Use `Logger.infoExternalOutput`, `Logger.errorExternalOutput`, and
|
|
20
|
+
`Logger.messageExternalOutput` for external or mixed-origin messages such as
|
|
21
|
+
artifact paths, metadata, remote responses, and error text. A trusted format
|
|
22
|
+
string does not make interpolated external data trusted.
|
|
23
|
+
|
|
24
|
+
The pipeline feature `EnableArtifactEngineExternalOutput` enables configurable
|
|
25
|
+
handling through task-lib's matching debug, error, and ordinary output helpers.
|
|
26
|
+
The task receives the feature as
|
|
27
|
+
`DISTRIBUTEDTASK_TASKS_ENABLEARTIFACTENGINEEXTERNALOUTPUT=true`.
|
|
28
|
+
|
|
29
|
+
```ts
|
|
30
|
+
const options = new ArtifactEngineOptions();
|
|
31
|
+
options.vsoCommandMode = 'blockAndTelemetry';
|
|
32
|
+
await engine.processItems(sourceProvider, destinationProvider, options);
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
| Mode | External-output behavior when the feature is enabled |
|
|
36
|
+
| --- | --- |
|
|
37
|
+
| `blockOnly` (V2 default) | Block commands without command telemetry. |
|
|
38
|
+
| `blockAndTelemetry` | Block commands and report command telemetry. |
|
|
39
|
+
| `telemetryOnly` | Report commands without blocking them. |
|
|
40
|
+
| `disabled` | Neither block commands nor report command telemetry. |
|
|
41
|
+
|
|
42
|
+
V2 defaults to `blockOnly`, preserving its existing blocking behavior for
|
|
43
|
+
ordinary messages. V1's compatibility default is `disabled`; do not assume
|
|
44
|
+
identical defaults across major versions. Explicitly choosing `disabled` or
|
|
45
|
+
`telemetryOnly` with the feature enabled opts out of blocking, including for
|
|
46
|
+
ordinary messages that V2 previously blocked.
|
|
47
|
+
|
|
48
|
+
With the feature absent or off, the new methods delegate to their corresponding
|
|
49
|
+
legacy methods. `Logger.logMessage` keeps its existing task-lib filtering and
|
|
50
|
+
native `os.EOL` newline, independently of the new mode. `Logger.logInfo` and
|
|
51
|
+
`Logger.logError` retain their existing debug/error behavior. Disabling the
|
|
52
|
+
feature does not disable V2's pre-existing ordinary-output protection.
|
|
53
|
+
|
|
54
|
+
Keep the legacy APIs for trusted internal messages. Only external/mixed-origin
|
|
55
|
+
call sites, including verbose artifact-path rows, use the new boundary. Internal
|
|
56
|
+
counts, progress, headings, worker status, and intentional telemetry remain
|
|
57
|
+
unchanged. Filtering changes only the logged copy, not artifact paths or data.
|
|
58
|
+
|
|
59
|
+
Logger's mode is process-wide static state, like `verbose`. Every `processItems`
|
|
60
|
+
call resets it, including when options or the property are omitted. Concurrent
|
|
61
|
+
engines with differing modes are not isolated.
|
|
62
|
+
|
|
63
|
+
Consuming-task updates, server-side feature registration/propagation, package
|
|
64
|
+
publication, and rollout are separate follow-ups. Keep existing task-level
|
|
65
|
+
filtering in place.
|
|
66
|
+
|
|
17
67
|
## Architecture
|
|
18
68
|

|
|
19
69
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "artifact-engine",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.282.0",
|
|
4
4
|
"description": "Artifact Engine to download artifacts from jenkins, teamcity, vsts",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"email": "omeshp@microsoft.com"
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"azure-pipelines-task-lib": "^5.281.
|
|
19
|
+
"azure-pipelines-task-lib": "^5.281.2",
|
|
20
20
|
"handlebars": "4.7.10",
|
|
21
21
|
"typed-rest-client": "^3.0.0"
|
|
22
22
|
},
|