@taskcluster/client 100.3.0 → 100.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/apis.js +4 -5
- package/src/client.js +334 -326
- package/src/download.js +30 -13
- package/src/hashstream.js +4 -4
- package/src/index.js +2 -2
- package/src/parsetime.js +18 -15
- package/src/retry.js +3 -3
- package/src/upload.js +4 -4
- package/src/utils.js +11 -13
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@taskcluster/client",
|
|
3
|
-
"version": "100.
|
|
3
|
+
"version": "100.5.0",
|
|
4
4
|
"author": "Jonas Finnemann Jensen <jopsen@gmail.com>",
|
|
5
5
|
"description": "Client for interfacing taskcluster components",
|
|
6
6
|
"license": "MPL-2.0",
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
"nock": "^14.0.13"
|
|
24
24
|
},
|
|
25
25
|
"engines": {
|
|
26
|
-
"node": "24.
|
|
26
|
+
"node": "24.17.0"
|
|
27
27
|
},
|
|
28
28
|
"type": "module",
|
|
29
29
|
"files": [
|
package/src/apis.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/* eslint-disable */
|
|
2
1
|
export default {
|
|
3
2
|
"Auth": {
|
|
4
3
|
"reference": {
|
|
@@ -576,7 +575,7 @@ export default {
|
|
|
576
575
|
"wstClient"
|
|
577
576
|
],
|
|
578
577
|
"category": "Websocktunnel Credentials",
|
|
579
|
-
"description": "Get a temporary token suitable for use connecting to a\n[websocktunnel](https://github.com/taskcluster/taskcluster/tree/main/tools/websocktunnel) server.\n\nThe resulting token will only be accepted by servers with a matching audience\nvalue. Reaching such a server is the
|
|
578
|
+
"description": "Get a temporary token suitable for use connecting to a\n[websocktunnel](https://github.com/taskcluster/taskcluster/tree/main/tools/websocktunnel) server.\n\nThe resulting token will only be accepted by servers with a matching audience\nvalue. Reaching such a server is the caller's responsibility. In general,\na server URL or set of URLs should be provided to the caller as configuration\nalong with the audience value.\n\nThe token is valid for a limited time (on the scale of hours). Callers should\nrefresh it before expiration.",
|
|
580
579
|
"method": "get",
|
|
581
580
|
"name": "websocktunnelToken",
|
|
582
581
|
"output": "v1/websocktunnel-token-response.json#",
|
|
@@ -1352,7 +1351,7 @@ export default {
|
|
|
1352
1351
|
"hookId"
|
|
1353
1352
|
],
|
|
1354
1353
|
"category": "Hooks",
|
|
1355
|
-
"description": "This endpoint will trigger the creation of a task from a hook definition.\n\nThe HTTP payload must match the
|
|
1354
|
+
"description": "This endpoint will trigger the creation of a task from a hook definition.\n\nThe HTTP payload must match the hook's `triggerSchema`. If it does, it is\nprovided as the `payload` property of the JSON-e context used to render the\ntask template.\n\nOptionally, a `taskId` can be provided in the payload which the hook task\nwill use. It must be unique and follow the slugid format.",
|
|
1356
1355
|
"input": "v1/trigger-hook.json#",
|
|
1357
1356
|
"method": "post",
|
|
1358
1357
|
"name": "triggerHook",
|
|
@@ -1408,7 +1407,7 @@ export default {
|
|
|
1408
1407
|
"token"
|
|
1409
1408
|
],
|
|
1410
1409
|
"category": "Hooks",
|
|
1411
|
-
"description": "This endpoint triggers a defined hook with a valid token.\n\nThe HTTP payload must match the
|
|
1410
|
+
"description": "This endpoint triggers a defined hook with a valid token.\n\nThe HTTP payload must match the hook's `triggerSchema`. If it does, it is\nprovided as the `payload` property of the JSON-e context used to render the\ntask template.\n\nOptionally, a `taskId` can be provided in the payload which the hook task\nwill use. It must be unique and follow the slugid format.",
|
|
1412
1411
|
"input": "v1/trigger-hook.json#",
|
|
1413
1412
|
"method": "post",
|
|
1414
1413
|
"name": "triggerHookWithToken",
|
|
@@ -4088,7 +4087,7 @@ export default {
|
|
|
4088
4087
|
"args": [
|
|
4089
4088
|
],
|
|
4090
4089
|
"category": "Secrets Service",
|
|
4091
|
-
"description": "List the names of all secrets.\n\nBy default this end-point will try to return up to 1000 secret names in one\nrequest. But it **may return less**, even if more
|
|
4090
|
+
"description": "List the names of all secrets.\n\nBy default this end-point will try to return up to 1000 secret names in one\nrequest. But it **may return less**, even if more secrets are available.\nIt may also return a `continuationToken` even though there are no more\nresults. However, you can only be sure to have seen all results if you\nkeep calling `list` with the last `continuationToken` until you\nget a result without a `continuationToken`.\n\nIf you are not interested in listing all the members at once, you may\nuse the query-string option `limit` to return fewer.",
|
|
4092
4091
|
"method": "get",
|
|
4093
4092
|
"name": "list",
|
|
4094
4093
|
"output": "v1/secret-list.json#",
|
package/src/client.js
CHANGED
|
@@ -7,19 +7,19 @@ import got, { TimeoutError } from 'got';
|
|
|
7
7
|
import debugFactory from 'debug';
|
|
8
8
|
const debug = debugFactory('@taskcluster/client');
|
|
9
9
|
import _ from 'lodash';
|
|
10
|
-
import assert from 'assert';
|
|
10
|
+
import assert from 'node:assert';
|
|
11
11
|
import hawk from 'hawk';
|
|
12
|
-
import url from 'url';
|
|
13
|
-
import crypto from 'crypto';
|
|
12
|
+
import url from 'node:url';
|
|
13
|
+
import crypto from 'node:crypto';
|
|
14
14
|
import slugid from 'slugid';
|
|
15
|
-
import http from 'http';
|
|
16
|
-
import https from 'https';
|
|
17
|
-
import querystring from 'querystring';
|
|
15
|
+
import http from 'node:http';
|
|
16
|
+
import https from 'node:https';
|
|
17
|
+
import querystring from 'node:querystring';
|
|
18
18
|
import tcUrl from 'taskcluster-lib-urls';
|
|
19
19
|
import retry from './retry.js';
|
|
20
20
|
|
|
21
21
|
/** Default options for our http/https global agents */
|
|
22
|
-
|
|
22
|
+
const AGENT_OPTIONS = {
|
|
23
23
|
maxSockets: 50,
|
|
24
24
|
maxFreeSockets: 0,
|
|
25
25
|
keepAlive: false,
|
|
@@ -30,7 +30,7 @@ let AGENT_OPTIONS = {
|
|
|
30
30
|
* defaulting to the global node agents primarily so we can tweak this across
|
|
31
31
|
* all our components if needed...
|
|
32
32
|
*/
|
|
33
|
-
|
|
33
|
+
const DEFAULT_AGENTS = {
|
|
34
34
|
http: new http.Agent(AGENT_OPTIONS),
|
|
35
35
|
https: new https.Agent(AGENT_OPTIONS),
|
|
36
36
|
};
|
|
@@ -93,12 +93,12 @@ let _defaultOptions = {
|
|
|
93
93
|
};
|
|
94
94
|
|
|
95
95
|
/** Make a request for a Client instance */
|
|
96
|
-
export const makeRequest = async
|
|
96
|
+
export const makeRequest = async (client, method, url, payload, query) => {
|
|
97
97
|
// Add query to url if present
|
|
98
98
|
if (query) {
|
|
99
99
|
query = querystring.stringify(query);
|
|
100
100
|
if (query.length > 0) {
|
|
101
|
-
url +=
|
|
101
|
+
url += `?${query}`;
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
|
|
@@ -115,13 +115,15 @@ export const makeRequest = async function(client, method, url, payload, query) {
|
|
|
115
115
|
limit: 0,
|
|
116
116
|
},
|
|
117
117
|
hooks: {
|
|
118
|
-
afterResponse: [
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
res.body
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
118
|
+
afterResponse: [
|
|
119
|
+
res => {
|
|
120
|
+
// parse the body, if one was given (Got's `responseType: json` fails to check content-type)
|
|
121
|
+
if (res.body && (res.headers['content-type'] || '').startsWith('application/json')) {
|
|
122
|
+
res.body = JSON.parse(res.body);
|
|
123
|
+
}
|
|
124
|
+
return res;
|
|
125
|
+
},
|
|
126
|
+
],
|
|
125
127
|
},
|
|
126
128
|
};
|
|
127
129
|
|
|
@@ -130,11 +132,9 @@ export const makeRequest = async function(client, method, url, payload, query) {
|
|
|
130
132
|
}
|
|
131
133
|
|
|
132
134
|
// Authenticate, if credentials are provided
|
|
133
|
-
if (client._options.credentials &&
|
|
134
|
-
client._options.credentials.clientId &&
|
|
135
|
-
client._options.credentials.accessToken) {
|
|
135
|
+
if (client._options.credentials?.clientId && client._options.credentials.accessToken) {
|
|
136
136
|
// Create hawk authentication header
|
|
137
|
-
|
|
137
|
+
const header = hawk.client.header(url, method.toUpperCase(), {
|
|
138
138
|
credentials: {
|
|
139
139
|
id: client._options.credentials.clientId,
|
|
140
140
|
key: client._options.credentials.accessToken,
|
|
@@ -142,7 +142,7 @@ export const makeRequest = async function(client, method, url, payload, query) {
|
|
|
142
142
|
},
|
|
143
143
|
ext: client._extData,
|
|
144
144
|
});
|
|
145
|
-
options.headers
|
|
145
|
+
options.headers.Authorization = header.header;
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
// Send payload if defined
|
|
@@ -190,17 +190,17 @@ export const makeRequest = async function(client, method, url, payload, query) {
|
|
|
190
190
|
*
|
|
191
191
|
* `rootUrl` and `baseUrl` are mutually exclusive.
|
|
192
192
|
*/
|
|
193
|
-
export const createClient =
|
|
193
|
+
export const createClient = (reference, name) => {
|
|
194
194
|
if (!name || typeof name !== 'string') {
|
|
195
195
|
name = 'Unknown';
|
|
196
196
|
}
|
|
197
197
|
|
|
198
198
|
// Client class constructor
|
|
199
|
-
|
|
200
|
-
if (options
|
|
199
|
+
const Client = function (options) {
|
|
200
|
+
if (options?.baseUrl) {
|
|
201
201
|
throw new Error('baseUrl has been deprecated!');
|
|
202
202
|
}
|
|
203
|
-
if (options
|
|
203
|
+
if (options?.exchangePrefix) {
|
|
204
204
|
throw new Error('exchangePrefix has been deprecated!');
|
|
205
205
|
}
|
|
206
206
|
let serviceName = reference.serviceName;
|
|
@@ -216,11 +216,16 @@ export const createClient = function(reference, name) {
|
|
|
216
216
|
serviceName = reference.exchangePrefix.split('/')[1].replace('taskcluster-', '');
|
|
217
217
|
}
|
|
218
218
|
}
|
|
219
|
-
this._options = _.defaults(
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
219
|
+
this._options = _.defaults(
|
|
220
|
+
{},
|
|
221
|
+
options || {},
|
|
222
|
+
{
|
|
223
|
+
exchangePrefix: reference.exchangePrefix,
|
|
224
|
+
serviceName,
|
|
225
|
+
serviceVersion: 'v1',
|
|
226
|
+
},
|
|
227
|
+
_defaultOptions
|
|
228
|
+
);
|
|
224
229
|
assert(this._options.rootUrl, 'Must provide a rootUrl'); // We always assert this even with service discovery
|
|
225
230
|
this._options.rootUrl = this._options.rootUrl.replace(/\/$/, '');
|
|
226
231
|
this._options._trueRootUrl = this._options.rootUrl.replace(/\/$/, ''); // Useful for buildUrl/buildSignedUrl in certain cases
|
|
@@ -238,8 +243,7 @@ export const createClient = function(reference, name) {
|
|
|
238
243
|
throw new Error('monitoring client calls is no longer supported');
|
|
239
244
|
}
|
|
240
245
|
|
|
241
|
-
if (this._options.randomizationFactor < 0 ||
|
|
242
|
-
this._options.randomizationFactor >= 1) {
|
|
246
|
+
if (this._options.randomizationFactor < 0 || this._options.randomizationFactor >= 1) {
|
|
243
247
|
throw new Error('options.randomizationFactor must be between 0 and 1!');
|
|
244
248
|
}
|
|
245
249
|
|
|
@@ -262,10 +266,8 @@ export const createClient = function(reference, name) {
|
|
|
262
266
|
|
|
263
267
|
// Build ext for hawk requests
|
|
264
268
|
this._extData = undefined;
|
|
265
|
-
if (this._options.credentials &&
|
|
266
|
-
|
|
267
|
-
this._options.credentials.accessToken) {
|
|
268
|
-
let ext = {};
|
|
269
|
+
if (this._options.credentials?.clientId && this._options.credentials.accessToken) {
|
|
270
|
+
const ext = {};
|
|
269
271
|
|
|
270
272
|
// If there is a certificate we have temporary credentials, and we
|
|
271
273
|
// must provide the certificate
|
|
@@ -276,8 +278,7 @@ export const createClient = function(reference, name) {
|
|
|
276
278
|
try {
|
|
277
279
|
ext.certificate = JSON.parse(ext.certificate);
|
|
278
280
|
} catch (err) {
|
|
279
|
-
debug('Failed to parse credentials.certificate, err: %s, JSON: %j',
|
|
280
|
-
err, err);
|
|
281
|
+
debug('Failed to parse credentials.certificate, err: %s, JSON: %j', err, err);
|
|
281
282
|
throw new Error('JSON.parse(): Failed for configured certificate');
|
|
282
283
|
}
|
|
283
284
|
}
|
|
@@ -285,7 +286,7 @@ export const createClient = function(reference, name) {
|
|
|
285
286
|
|
|
286
287
|
// If set of authorized scopes is provided, we'll restrict the request
|
|
287
288
|
// to only use these scopes
|
|
288
|
-
if (this._options.authorizedScopes
|
|
289
|
+
if (Array.isArray(this._options.authorizedScopes)) {
|
|
289
290
|
ext.authorizedScopes = this._options.authorizedScopes;
|
|
290
291
|
}
|
|
291
292
|
|
|
@@ -299,248 +300,253 @@ export const createClient = function(reference, name) {
|
|
|
299
300
|
if (this._options.fake) {
|
|
300
301
|
debug('Creating @taskcluster/client object in "fake" mode');
|
|
301
302
|
this.fakeCalls = {};
|
|
302
|
-
reference.entries
|
|
303
|
+
reference.entries
|
|
304
|
+
.filter(e => e.type === 'function')
|
|
305
|
+
.forEach(e => {
|
|
306
|
+
this.fakeCalls[e.name] = [];
|
|
307
|
+
});
|
|
303
308
|
// Throw an error if creating fakes in production
|
|
304
309
|
if (process.env.NODE_ENV === 'production') {
|
|
305
|
-
new Error('@taskcluster/client object created in "fake" mode, when NODE_ENV == "production"');
|
|
310
|
+
throw new Error('@taskcluster/client object created in "fake" mode, when NODE_ENV == "production"');
|
|
306
311
|
}
|
|
307
312
|
}
|
|
308
313
|
};
|
|
309
314
|
|
|
310
|
-
Client.prototype.use = function(optionsUpdates) {
|
|
311
|
-
|
|
315
|
+
Client.prototype.use = function (optionsUpdates) {
|
|
316
|
+
const options = _.defaults({}, optionsUpdates, { rootUrl: this._options._trueRootUrl }, this._options);
|
|
312
317
|
return new Client(options);
|
|
313
318
|
};
|
|
314
319
|
|
|
315
|
-
Client.prototype.taskclusterPerRequestInstance = function({
|
|
320
|
+
Client.prototype.taskclusterPerRequestInstance = function ({ traceId }) {
|
|
316
321
|
return this.use({ traceId });
|
|
317
322
|
};
|
|
318
323
|
|
|
319
324
|
// For each function entry create a method on the Client class
|
|
320
|
-
reference.entries
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
if (entry.input) {
|
|
326
|
-
nb_args += 1;
|
|
327
|
-
}
|
|
328
|
-
// Get the query-string options taken
|
|
329
|
-
let optKeys = entry.query || [];
|
|
330
|
-
|
|
331
|
-
// Create method on prototype
|
|
332
|
-
Client.prototype[entry.name] = function() {
|
|
333
|
-
// Convert arguments to actual array
|
|
334
|
-
let args = Array.prototype.slice.call(arguments);
|
|
335
|
-
// Validate number of arguments
|
|
336
|
-
let N = args.length;
|
|
337
|
-
if (N !== nb_args && (optKeys.length === 0 || N !== nb_args + 1)) {
|
|
338
|
-
throw new Error('Function ' + entry.name + ' takes ' + nb_args +
|
|
339
|
-
' arguments, but was given ' + N +
|
|
340
|
-
' arguments');
|
|
341
|
-
}
|
|
342
|
-
// Substitute parameters into route
|
|
343
|
-
let endpoint = entry.route.replace(/<([^<>]+)>/g, function(text, arg) {
|
|
344
|
-
let index = entry.args.indexOf(arg);
|
|
345
|
-
if (index !== -1) {
|
|
346
|
-
let param = args[index];
|
|
347
|
-
if (typeof param !== 'string' && typeof param !== 'number') {
|
|
348
|
-
throw new Error('URL parameter ' + arg + ' must be a string, but ' +
|
|
349
|
-
'we received a: ' + typeof param);
|
|
350
|
-
}
|
|
351
|
-
return encodeURIComponent(param);
|
|
352
|
-
}
|
|
353
|
-
return text; // Preserve original
|
|
354
|
-
});
|
|
355
|
-
// Create url for the request
|
|
356
|
-
let url = tcUrl.api(this._options.rootUrl, this._options.serviceName, this._options.serviceVersion, endpoint);
|
|
357
|
-
// Add payload if one is given
|
|
358
|
-
let payload = undefined;
|
|
325
|
+
reference.entries
|
|
326
|
+
.filter(entry => entry.type === 'function')
|
|
327
|
+
.forEach(entry => {
|
|
328
|
+
// Get number of arguments
|
|
329
|
+
let nb_args = entry.args.length;
|
|
359
330
|
if (entry.input) {
|
|
360
|
-
|
|
331
|
+
nb_args += 1;
|
|
361
332
|
}
|
|
362
|
-
//
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
333
|
+
// Get the query-string options taken
|
|
334
|
+
const optKeys = entry.query || [];
|
|
335
|
+
|
|
336
|
+
// Create method on prototype
|
|
337
|
+
Client.prototype[entry.name] = function (...args) {
|
|
338
|
+
// Validate number of arguments
|
|
339
|
+
const N = args.length;
|
|
340
|
+
if (N !== nb_args && (optKeys.length === 0 || N !== nb_args + 1)) {
|
|
341
|
+
throw new Error(`Function ${entry.name} takes ${nb_args} arguments, but was given ${N} arguments`);
|
|
342
|
+
}
|
|
343
|
+
// Substitute parameters into route
|
|
344
|
+
const endpoint = entry.route.replace(/<([^<>]+)>/g, (text, arg) => {
|
|
345
|
+
const index = entry.args.indexOf(arg);
|
|
346
|
+
if (index !== -1) {
|
|
347
|
+
const param = args[index];
|
|
348
|
+
if (typeof param !== 'string' && typeof param !== 'number') {
|
|
349
|
+
throw new Error(`URL parameter ${arg} must be a string, but we received a: ${typeof param}`);
|
|
350
|
+
}
|
|
351
|
+
return encodeURIComponent(param);
|
|
369
352
|
}
|
|
353
|
+
return text; // Preserve original
|
|
370
354
|
});
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
let record = {};
|
|
378
|
-
if (payload !== undefined) {
|
|
379
|
-
record.payload = _.cloneDeep(payload);
|
|
380
|
-
}
|
|
381
|
-
if (query !== null) {
|
|
382
|
-
record.query = _.cloneDeep(query);
|
|
355
|
+
// Create url for the request
|
|
356
|
+
const url = tcUrl.api(this._options.rootUrl, this._options.serviceName, this._options.serviceVersion, endpoint);
|
|
357
|
+
// Add payload if one is given
|
|
358
|
+
let payload;
|
|
359
|
+
if (entry.input) {
|
|
360
|
+
payload = args[nb_args - 1];
|
|
383
361
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
362
|
+
// Find query string options (if present)
|
|
363
|
+
const query = args[nb_args] || null;
|
|
364
|
+
if (query) {
|
|
365
|
+
_.keys(query).forEach(key => {
|
|
366
|
+
if (!_.includes(optKeys, key)) {
|
|
367
|
+
throw new Error(`Function ${entry.name} takes options: ${optKeys.join(', ')} but was given ${key}`);
|
|
368
|
+
}
|
|
369
|
+
});
|
|
391
370
|
}
|
|
392
|
-
return this._options.fake[entry.name].apply(this, args);
|
|
393
|
-
}
|
|
394
371
|
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
payload,
|
|
403
|
-
query,
|
|
404
|
-
).then(function(res) {
|
|
405
|
-
// If request was successful, accept the result
|
|
406
|
-
debug('Success calling: %s, (%s retries)', entry.name, attempt);
|
|
407
|
-
if (!_.includes(res.headers['content-type'], 'application/json') || !res.body) {
|
|
408
|
-
debug('Empty response from server: call: %s, method: %s', entry.name, entry.method);
|
|
409
|
-
return undefined;
|
|
372
|
+
// call out to the fake version, if set
|
|
373
|
+
if (this._options.fake) {
|
|
374
|
+
debug('Faking call to %s(%s)', entry.name, args.map(a => JSON.stringify(a, null, 2)).join(', '));
|
|
375
|
+
// Add a call record to fakeCalls[<method>]
|
|
376
|
+
const record = {};
|
|
377
|
+
if (payload !== undefined) {
|
|
378
|
+
record.payload = _.cloneDeep(payload);
|
|
410
379
|
}
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
if (res.statusCode >= 300 && res.statusCode < 400) {
|
|
424
|
-
message = 'Unexpected Redirect';
|
|
425
|
-
}
|
|
426
|
-
err = new Error(res.body.message || message);
|
|
427
|
-
err.body = res.body;
|
|
428
|
-
err.code = res.body.code || 'UnknownError';
|
|
429
|
-
err.statusCode = res.statusCode;
|
|
430
|
-
|
|
431
|
-
// Decide if we should retry or just throw
|
|
432
|
-
if (res.statusCode >= 500 && // Check if it's a 5xx error
|
|
433
|
-
res.statusCode < 600) {
|
|
434
|
-
debug('Error calling: %s now retrying, info: %j',
|
|
435
|
-
entry.name, res.body);
|
|
436
|
-
return retriableError(err);
|
|
437
|
-
} else {
|
|
438
|
-
debug('Error calling: %s NOT retrying!, info: %j',
|
|
439
|
-
entry.name, res.body);
|
|
440
|
-
throw err;
|
|
441
|
-
}
|
|
380
|
+
if (query !== null) {
|
|
381
|
+
record.query = _.cloneDeep(query);
|
|
382
|
+
}
|
|
383
|
+
entry.args.forEach((k, i) => {
|
|
384
|
+
record[k] = _.cloneDeep(args[i]);
|
|
385
|
+
});
|
|
386
|
+
this.fakeCalls[entry.name].push(record);
|
|
387
|
+
// Call fake[<method>]
|
|
388
|
+
if (!this._options.fake[entry.name]) {
|
|
389
|
+
return Promise.reject(
|
|
390
|
+
new Error(`Faked ${this._options.serviceName} object does not have an implementation of ${entry.name}`)
|
|
391
|
+
);
|
|
442
392
|
}
|
|
393
|
+
return this._options.fake[entry.name].apply(this, args);
|
|
394
|
+
}
|
|
443
395
|
|
|
444
|
-
|
|
445
|
-
debug('
|
|
446
|
-
|
|
447
|
-
return
|
|
396
|
+
return retry(this._options, (retriableError, attempt) => {
|
|
397
|
+
debug('Calling: %s, retry: %s', entry.name, attempt);
|
|
398
|
+
// Make request and handle response or error
|
|
399
|
+
return makeRequest(this, entry.method, url, payload, query).then(
|
|
400
|
+
res => {
|
|
401
|
+
// If request was successful, accept the result
|
|
402
|
+
debug('Success calling: %s, (%s retries)', entry.name, attempt);
|
|
403
|
+
if (!_.includes(res.headers['content-type'], 'application/json') || !res.body) {
|
|
404
|
+
debug('Empty response from server: call: %s, method: %s', entry.name, entry.method);
|
|
405
|
+
return undefined;
|
|
406
|
+
}
|
|
407
|
+
return res.body;
|
|
408
|
+
},
|
|
409
|
+
err => {
|
|
410
|
+
// If we got a response we read the error code from the response
|
|
411
|
+
const res = err.response;
|
|
412
|
+
if (res) {
|
|
413
|
+
let message = 'Unknown Server Error';
|
|
414
|
+
if (res.statusCode === 401) {
|
|
415
|
+
message = 'Authentication Error';
|
|
416
|
+
}
|
|
417
|
+
if (res.statusCode === 500) {
|
|
418
|
+
message = 'Internal Server Error';
|
|
419
|
+
}
|
|
420
|
+
if (res.statusCode >= 300 && res.statusCode < 400) {
|
|
421
|
+
message = 'Unexpected Redirect';
|
|
422
|
+
}
|
|
423
|
+
err = new Error(res.body.message || message);
|
|
424
|
+
err.body = res.body;
|
|
425
|
+
err.code = res.body.code || 'UnknownError';
|
|
426
|
+
err.statusCode = res.statusCode;
|
|
427
|
+
|
|
428
|
+
// Decide if we should retry or just throw
|
|
429
|
+
if (
|
|
430
|
+
res.statusCode >= 500 && // Check if it's a 5xx error
|
|
431
|
+
res.statusCode < 600
|
|
432
|
+
) {
|
|
433
|
+
debug('Error calling: %s now retrying, info: %j', entry.name, res.body);
|
|
434
|
+
return retriableError(err);
|
|
435
|
+
} else {
|
|
436
|
+
debug('Error calling: %s NOT retrying!, info: %j', entry.name, res.body);
|
|
437
|
+
throw err;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
// All errors without a response are treated as retriable
|
|
442
|
+
debug('Request error calling %s (retrying), err: %s, JSON: %s', entry.name, err, err);
|
|
443
|
+
return retriableError(err);
|
|
444
|
+
}
|
|
445
|
+
);
|
|
448
446
|
});
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
});
|
|
447
|
+
};
|
|
448
|
+
// Add reference for buildUrl and signUrl
|
|
449
|
+
Client.prototype[entry.name].entryReference = entry;
|
|
450
|
+
});
|
|
454
451
|
|
|
455
452
|
// For each topic-exchange entry
|
|
456
|
-
reference.entries
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
routingKeyPattern
|
|
465
|
-
routingKeyPattern = {};
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// Check that the routing key pattern is an object
|
|
469
|
-
assert(routingKeyPattern instanceof Object,
|
|
470
|
-
'routingKeyPattern must be an object');
|
|
471
|
-
|
|
472
|
-
// Construct routingkey pattern as string from reference
|
|
473
|
-
routingKeyPattern = entry.routingKey.map(function(key) {
|
|
474
|
-
// Get value for key
|
|
475
|
-
let value = routingKeyPattern[key.name];
|
|
476
|
-
// Routing key constant entries cannot be modified
|
|
477
|
-
if (key.constant) {
|
|
478
|
-
value = key.constant;
|
|
479
|
-
}
|
|
480
|
-
// If number convert to string
|
|
481
|
-
if (typeof value === 'number') {
|
|
482
|
-
return '' + value;
|
|
453
|
+
reference.entries
|
|
454
|
+
.filter(entry => entry.type === 'topic-exchange')
|
|
455
|
+
.forEach(entry => {
|
|
456
|
+
// Create function for routing-key pattern construction
|
|
457
|
+
Client.prototype[entry.name] = function (routingKeyPattern) {
|
|
458
|
+
if (typeof routingKeyPattern !== 'string') {
|
|
459
|
+
// Allow for empty routing key patterns
|
|
460
|
+
if (routingKeyPattern === undefined || routingKeyPattern === null) {
|
|
461
|
+
routingKeyPattern = {};
|
|
483
462
|
}
|
|
484
|
-
// Validate string and return
|
|
485
|
-
if (typeof value === 'string') {
|
|
486
|
-
// Check for multiple words
|
|
487
|
-
assert(key.multipleWords || value.indexOf('.') === -1,
|
|
488
|
-
'routingKey pattern \'' + value + '\' for ' + key.name +
|
|
489
|
-
' cannot contain dots as it does not hold multiple words');
|
|
490
|
-
return value;
|
|
491
|
-
}
|
|
492
|
-
// Check that we haven't got an invalid value
|
|
493
|
-
assert(value === null || value === undefined,
|
|
494
|
-
'Value: \'' + value + '\' is not supported as routingKey ' +
|
|
495
|
-
'pattern for ' + key.name);
|
|
496
|
-
// Return default pattern for entry not being matched
|
|
497
|
-
return key.multipleWords ? '#' : '*';
|
|
498
|
-
}).join('.');
|
|
499
|
-
}
|
|
500
463
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
464
|
+
// Check that the routing key pattern is an object
|
|
465
|
+
assert(routingKeyPattern instanceof Object, 'routingKeyPattern must be an object');
|
|
466
|
+
|
|
467
|
+
// Construct routingkey pattern as string from reference
|
|
468
|
+
routingKeyPattern = entry.routingKey
|
|
469
|
+
.map(key => {
|
|
470
|
+
// Get value for key
|
|
471
|
+
let value = routingKeyPattern[key.name];
|
|
472
|
+
// Routing key constant entries cannot be modified
|
|
473
|
+
if (key.constant) {
|
|
474
|
+
value = key.constant;
|
|
475
|
+
}
|
|
476
|
+
// If number convert to string
|
|
477
|
+
if (typeof value === 'number') {
|
|
478
|
+
return `${value}`;
|
|
479
|
+
}
|
|
480
|
+
// Validate string and return
|
|
481
|
+
if (typeof value === 'string') {
|
|
482
|
+
// Check for multiple words
|
|
483
|
+
assert(
|
|
484
|
+
key.multipleWords || value.indexOf('.') === -1,
|
|
485
|
+
"routingKey pattern '" +
|
|
486
|
+
value +
|
|
487
|
+
"' for " +
|
|
488
|
+
key.name +
|
|
489
|
+
' cannot contain dots as it does not hold multiple words'
|
|
490
|
+
);
|
|
491
|
+
return value;
|
|
492
|
+
}
|
|
493
|
+
// Check that we haven't got an invalid value
|
|
494
|
+
assert(
|
|
495
|
+
value === null || value === undefined,
|
|
496
|
+
`Value: '${value}' is not supported as routingKey pattern for ${key.name}`
|
|
497
|
+
);
|
|
498
|
+
// Return default pattern for entry not being matched
|
|
499
|
+
return key.multipleWords ? '#' : '*';
|
|
500
|
+
})
|
|
501
|
+
.join('.');
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// Return values necessary to bind with EventHandler
|
|
505
|
+
return {
|
|
506
|
+
exchange: this._options.exchangePrefix + entry.exchange,
|
|
507
|
+
routingKeyPattern: routingKeyPattern,
|
|
508
|
+
routingKeyReference: _.cloneDeep(entry.routingKey),
|
|
509
|
+
};
|
|
506
510
|
};
|
|
507
|
-
};
|
|
508
|
-
});
|
|
511
|
+
});
|
|
509
512
|
|
|
510
|
-
Client.prototype._buildUrl = function(rootUrl, args) {
|
|
513
|
+
Client.prototype._buildUrl = function (rootUrl, args) {
|
|
511
514
|
if (args.length === 0) {
|
|
512
|
-
throw new Error('buildUrl(method, arg1, arg2, ...) takes a least one ' +
|
|
513
|
-
'argument!');
|
|
515
|
+
throw new Error('buildUrl(method, arg1, arg2, ...) takes a least one ' + 'argument!');
|
|
514
516
|
}
|
|
515
517
|
// Find the method
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
if (
|
|
519
|
-
throw new Error('method in buildUrl(method, arg1, arg2, ...) must be ' +
|
|
520
|
-
'an API method from the same object!');
|
|
518
|
+
const method = args.shift();
|
|
519
|
+
const entry = method.entryReference;
|
|
520
|
+
if (entry?.type !== 'function') {
|
|
521
|
+
throw new Error('method in buildUrl(method, arg1, arg2, ...) must be ' + 'an API method from the same object!');
|
|
521
522
|
}
|
|
522
523
|
|
|
523
524
|
// Get the query-string options taken
|
|
524
|
-
|
|
525
|
-
|
|
525
|
+
const optKeys = entry.query || [];
|
|
526
|
+
const supportsOpts = optKeys.length !== 0;
|
|
526
527
|
|
|
527
|
-
debug(
|
|
528
|
+
debug(`build url for: ${entry.name}`);
|
|
528
529
|
// Validate number of arguments
|
|
529
|
-
|
|
530
|
+
const N = entry.args.length;
|
|
530
531
|
if (args.length !== N && (!supportsOpts || args.length !== N + 1)) {
|
|
531
|
-
throw new Error(
|
|
532
|
-
|
|
533
|
-
|
|
532
|
+
throw new Error(
|
|
533
|
+
'Function ' +
|
|
534
|
+
entry.name +
|
|
535
|
+
'buildUrl() takes ' +
|
|
536
|
+
(N + 1) +
|
|
537
|
+
' arguments, but was given ' +
|
|
538
|
+
(args.length + 1) +
|
|
539
|
+
' arguments'
|
|
540
|
+
);
|
|
534
541
|
}
|
|
535
542
|
|
|
536
543
|
// Substitute parameters into route
|
|
537
|
-
|
|
538
|
-
|
|
544
|
+
const endpoint = entry.route.replace(/<([^<>]+)>/g, (text, arg) => {
|
|
545
|
+
const index = entry.args.indexOf(arg);
|
|
539
546
|
if (index !== -1) {
|
|
540
|
-
|
|
547
|
+
const param = args[index];
|
|
541
548
|
if (typeof param !== 'string' && typeof param !== 'number') {
|
|
542
|
-
throw new Error(
|
|
543
|
-
'we received a: ' + typeof param);
|
|
549
|
+
throw new Error(`URL parameter ${arg} must be a string, but we received a: ${typeof param}`);
|
|
544
550
|
}
|
|
545
551
|
return encodeURIComponent(param);
|
|
546
552
|
}
|
|
@@ -550,16 +556,15 @@ export const createClient = function(reference, name) {
|
|
|
550
556
|
// Find query string options (if present)
|
|
551
557
|
let query = args[N] || '';
|
|
552
558
|
if (query) {
|
|
553
|
-
_.keys(query).forEach(
|
|
559
|
+
_.keys(query).forEach(key => {
|
|
554
560
|
if (!_.includes(optKeys, key)) {
|
|
555
|
-
throw new Error(
|
|
556
|
-
optKeys.join(', ') + ' but was given ' + key);
|
|
561
|
+
throw new Error(`Function ${entry.name} takes options: ${optKeys.join(', ')} but was given ${key}`);
|
|
557
562
|
}
|
|
558
563
|
});
|
|
559
564
|
|
|
560
565
|
query = querystring.stringify(query);
|
|
561
566
|
if (query.length > 0) {
|
|
562
|
-
query =
|
|
567
|
+
query = `?${query}`;
|
|
563
568
|
}
|
|
564
569
|
}
|
|
565
570
|
|
|
@@ -569,22 +574,21 @@ export const createClient = function(reference, name) {
|
|
|
569
574
|
// Utility functions to build the request URL for given method and
|
|
570
575
|
// input parameters. The first builds with whatever rootUrl currently
|
|
571
576
|
// is while the latter builds with trueRootUrl for sending to users
|
|
572
|
-
Client.prototype.buildUrl = function() {
|
|
573
|
-
return this._buildUrl(this._options.rootUrl,
|
|
577
|
+
Client.prototype.buildUrl = function (...args) {
|
|
578
|
+
return this._buildUrl(this._options.rootUrl, args);
|
|
574
579
|
};
|
|
575
|
-
Client.prototype.externalBuildUrl = function() {
|
|
576
|
-
return this._buildUrl(this._options._trueRootUrl,
|
|
580
|
+
Client.prototype.externalBuildUrl = function (...args) {
|
|
581
|
+
return this._buildUrl(this._options._trueRootUrl, args);
|
|
577
582
|
};
|
|
578
583
|
|
|
579
|
-
Client.prototype._buildSignedUrl = function(builder, args) {
|
|
584
|
+
Client.prototype._buildSignedUrl = function (builder, args) {
|
|
580
585
|
if (args.length === 0) {
|
|
581
|
-
throw new Error('buildSignedUrl(method, arg1, arg2, ..., [options]) ' +
|
|
582
|
-
'takes a least one argument!');
|
|
586
|
+
throw new Error('buildSignedUrl(method, arg1, arg2, ..., [options]) ' + 'takes a least one argument!');
|
|
583
587
|
}
|
|
584
588
|
|
|
585
589
|
// Find method and reference entry
|
|
586
|
-
|
|
587
|
-
|
|
590
|
+
const method = args[0];
|
|
591
|
+
const entry = method.entryReference;
|
|
588
592
|
if (entry.method !== 'get') {
|
|
589
593
|
throw new Error('buildSignedUrl only works for GET requests');
|
|
590
594
|
}
|
|
@@ -593,7 +597,7 @@ export const createClient = function(reference, name) {
|
|
|
593
597
|
let expiration = 15 * 60;
|
|
594
598
|
|
|
595
599
|
// Check if method supports query-string options
|
|
596
|
-
|
|
600
|
+
const supportsOpts = (entry.query || []).length !== 0;
|
|
597
601
|
|
|
598
602
|
// if longer than method + args, then we have options too
|
|
599
603
|
let N = entry.args.length + 1;
|
|
@@ -602,7 +606,7 @@ export const createClient = function(reference, name) {
|
|
|
602
606
|
}
|
|
603
607
|
if (args.length > N) {
|
|
604
608
|
// Get request options
|
|
605
|
-
|
|
609
|
+
const options = args.pop();
|
|
606
610
|
|
|
607
611
|
// Get expiration from options
|
|
608
612
|
expiration = options.expiration || expiration;
|
|
@@ -614,7 +618,7 @@ export const createClient = function(reference, name) {
|
|
|
614
618
|
}
|
|
615
619
|
|
|
616
620
|
// Build URL
|
|
617
|
-
|
|
621
|
+
const requestUrl = builder.apply(this, args);
|
|
618
622
|
|
|
619
623
|
// Check that we have credentials
|
|
620
624
|
if (!this._options.credentials.clientId) {
|
|
@@ -625,7 +629,7 @@ export const createClient = function(reference, name) {
|
|
|
625
629
|
}
|
|
626
630
|
|
|
627
631
|
// Create bewit
|
|
628
|
-
|
|
632
|
+
const bewit = hawk.client.getBewit(requestUrl, {
|
|
629
633
|
credentials: {
|
|
630
634
|
id: this._options.credentials.clientId,
|
|
631
635
|
key: this._options.credentials.accessToken,
|
|
@@ -636,11 +640,11 @@ export const createClient = function(reference, name) {
|
|
|
636
640
|
});
|
|
637
641
|
|
|
638
642
|
// Add bewit to requestUrl
|
|
639
|
-
|
|
643
|
+
const urlParts = url.parse(requestUrl);
|
|
640
644
|
if (urlParts.search) {
|
|
641
|
-
urlParts.search +=
|
|
645
|
+
urlParts.search += `&bewit=${bewit}`;
|
|
642
646
|
} else {
|
|
643
|
-
urlParts.search =
|
|
647
|
+
urlParts.search = `?bewit=${bewit}`;
|
|
644
648
|
}
|
|
645
649
|
|
|
646
650
|
// Return formatted URL
|
|
@@ -649,11 +653,11 @@ export const createClient = function(reference, name) {
|
|
|
649
653
|
|
|
650
654
|
// Utility function to construct a bewit URL for GET requests. Same convention
|
|
651
655
|
// as unsigned buildUrl applies here too
|
|
652
|
-
Client.prototype.buildSignedUrl = function() {
|
|
653
|
-
return this._buildSignedUrl(this.buildUrl,
|
|
656
|
+
Client.prototype.buildSignedUrl = function (...args) {
|
|
657
|
+
return this._buildSignedUrl(this.buildUrl, args);
|
|
654
658
|
};
|
|
655
|
-
Client.prototype.externalBuildSignedUrl = function() {
|
|
656
|
-
return this._buildSignedUrl(this.externalBuildUrl,
|
|
659
|
+
Client.prototype.externalBuildSignedUrl = function (...args) {
|
|
660
|
+
return this._buildSignedUrl(this.externalBuildUrl, args);
|
|
657
661
|
};
|
|
658
662
|
|
|
659
663
|
// Return client class
|
|
@@ -664,23 +668,23 @@ export const createClient = function(reference, name) {
|
|
|
664
668
|
import apis from './apis.js';
|
|
665
669
|
|
|
666
670
|
export const clients = {
|
|
667
|
-
Auth: createClient(apis.Auth.reference,
|
|
668
|
-
AuthEvents: createClient(apis.AuthEvents.reference,
|
|
669
|
-
Github: createClient(apis.Github.reference,
|
|
670
|
-
GithubEvents: createClient(apis.GithubEvents.reference,
|
|
671
|
-
Hooks: createClient(apis.Hooks.reference,
|
|
672
|
-
HooksEvents: createClient(apis.HooksEvents.reference,
|
|
673
|
-
Index: createClient(apis.Index.reference,
|
|
674
|
-
Notify: createClient(apis.Notify.reference,
|
|
675
|
-
NotifyEvents: createClient(apis.NotifyEvents.reference,
|
|
676
|
-
Object: createClient(apis.Object.reference,
|
|
677
|
-
PurgeCache: createClient(apis.PurgeCache.reference,
|
|
678
|
-
Queue: createClient(apis.Queue.reference,
|
|
679
|
-
QueueEvents: createClient(apis.QueueEvents.reference,
|
|
680
|
-
Secrets: createClient(apis.Secrets.reference,
|
|
681
|
-
WebServer: createClient(apis.WebServer.reference,
|
|
682
|
-
WorkerManager: createClient(apis.WorkerManager.reference,
|
|
683
|
-
WorkerManagerEvents: createClient(apis.WorkerManagerEvents.reference,
|
|
671
|
+
Auth: createClient(apis.Auth.reference, 'Auth'),
|
|
672
|
+
AuthEvents: createClient(apis.AuthEvents.reference, 'AuthEvents'),
|
|
673
|
+
Github: createClient(apis.Github.reference, 'Github'),
|
|
674
|
+
GithubEvents: createClient(apis.GithubEvents.reference, 'GithubEvents'),
|
|
675
|
+
Hooks: createClient(apis.Hooks.reference, 'Hooks'),
|
|
676
|
+
HooksEvents: createClient(apis.HooksEvents.reference, 'HooksEvents'),
|
|
677
|
+
Index: createClient(apis.Index.reference, 'Index'),
|
|
678
|
+
Notify: createClient(apis.Notify.reference, 'Notify'),
|
|
679
|
+
NotifyEvents: createClient(apis.NotifyEvents.reference, 'NotifyEvents'),
|
|
680
|
+
Object: createClient(apis.Object.reference, 'Object'),
|
|
681
|
+
PurgeCache: createClient(apis.PurgeCache.reference, 'PurgeCache'),
|
|
682
|
+
Queue: createClient(apis.Queue.reference, 'Queue'),
|
|
683
|
+
QueueEvents: createClient(apis.QueueEvents.reference, 'QueueEvents'),
|
|
684
|
+
Secrets: createClient(apis.Secrets.reference, 'Secrets'),
|
|
685
|
+
WebServer: createClient(apis.WebServer.reference, 'WebServer'),
|
|
686
|
+
WorkerManager: createClient(apis.WorkerManager.reference, 'WorkerManager'),
|
|
687
|
+
WorkerManagerEvents: createClient(apis.WorkerManagerEvents.reference, 'WorkerManagerEvents'),
|
|
684
688
|
};
|
|
685
689
|
|
|
686
690
|
/**
|
|
@@ -688,13 +692,13 @@ export const clients = {
|
|
|
688
692
|
*
|
|
689
693
|
* Example: `Client.config({credentials: {...}});`
|
|
690
694
|
*/
|
|
691
|
-
export const config =
|
|
695
|
+
export const config = options => {
|
|
692
696
|
_defaultOptions = _.defaults({}, options, _defaultOptions);
|
|
693
697
|
};
|
|
694
698
|
|
|
695
|
-
export const fromEnvVars =
|
|
696
|
-
|
|
697
|
-
for (
|
|
699
|
+
export const fromEnvVars = () => {
|
|
700
|
+
const results = {};
|
|
701
|
+
for (const { env, path } of [
|
|
698
702
|
{ env: 'TASKCLUSTER_ROOT_URL', path: 'rootUrl' },
|
|
699
703
|
{ env: 'TASKCLUSTER_CLIENT_ID', path: 'credentials.clientId' },
|
|
700
704
|
{ env: 'TASKCLUSTER_ACCESS_TOKEN', path: 'credentials.accessToken' },
|
|
@@ -728,48 +732,52 @@ export const fromEnvVars = function() {
|
|
|
728
732
|
*
|
|
729
733
|
* Returns an object on the form: {clientId, accessToken, certificate}
|
|
730
734
|
*/
|
|
731
|
-
export const createTemporaryCredentials =
|
|
735
|
+
export const createTemporaryCredentials = options => {
|
|
732
736
|
assert(options, 'options are required');
|
|
733
737
|
|
|
734
|
-
|
|
738
|
+
const now = new Date();
|
|
735
739
|
|
|
736
740
|
// Set default options
|
|
737
|
-
options = _.defaults(
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
741
|
+
options = _.defaults(
|
|
742
|
+
{},
|
|
743
|
+
options,
|
|
744
|
+
{
|
|
745
|
+
// Clock drift is handled in auth service (PR #117)
|
|
746
|
+
// so no clock skew required.
|
|
747
|
+
start: now,
|
|
748
|
+
scopes: [],
|
|
749
|
+
},
|
|
750
|
+
_defaultOptions
|
|
751
|
+
);
|
|
743
752
|
|
|
744
753
|
// Validate options
|
|
745
754
|
assert(options.credentials, 'options.credentials is required');
|
|
746
|
-
assert(options.credentials.clientId,
|
|
747
|
-
|
|
748
|
-
assert(
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
'credentials; ensure that options.credentials.certificate is null');
|
|
755
|
+
assert(options.credentials.clientId, 'options.credentials.clientId is required');
|
|
756
|
+
assert(options.credentials.accessToken, 'options.credentials.accessToken is required');
|
|
757
|
+
assert(
|
|
758
|
+
options.credentials.certificate === undefined || options.credentials.certificate === null,
|
|
759
|
+
'temporary credentials cannot be used to make new temporary ' +
|
|
760
|
+
'credentials; ensure that options.credentials.certificate is null'
|
|
761
|
+
);
|
|
754
762
|
assert(options.start instanceof Date, 'options.start must be a Date');
|
|
755
763
|
assert(options.expiry instanceof Date, 'options.expiry must be a Date');
|
|
756
|
-
assert(options.scopes
|
|
757
|
-
options.scopes.forEach(
|
|
758
|
-
assert(typeof scope === 'string',
|
|
759
|
-
'options.scopes must be an array of strings');
|
|
764
|
+
assert(Array.isArray(options.scopes), 'options.scopes must be an array');
|
|
765
|
+
options.scopes.forEach(scope => {
|
|
766
|
+
assert(typeof scope === 'string', 'options.scopes must be an array of strings');
|
|
760
767
|
});
|
|
761
|
-
assert(
|
|
762
|
-
|
|
768
|
+
assert(
|
|
769
|
+
options.expiry.getTime() - options.start.getTime() <= 31 * 24 * 60 * 60 * 1000,
|
|
770
|
+
'Credentials cannot span more than 31 days'
|
|
771
|
+
);
|
|
763
772
|
|
|
764
|
-
|
|
773
|
+
const isNamed = !!options.clientId;
|
|
765
774
|
|
|
766
775
|
if (isNamed) {
|
|
767
|
-
assert(options.clientId !== options.credentials.clientId,
|
|
768
|
-
'Credential issuer must be different from the name');
|
|
776
|
+
assert(options.clientId !== options.credentials.clientId, 'Credential issuer must be different from the name');
|
|
769
777
|
}
|
|
770
778
|
|
|
771
779
|
// Construct certificate
|
|
772
|
-
|
|
780
|
+
const cert = {
|
|
773
781
|
version: 1,
|
|
774
782
|
scopes: _.cloneDeep(options.scopes),
|
|
775
783
|
start: options.start.getTime(),
|
|
@@ -782,21 +790,21 @@ export const createTemporaryCredentials = function(options) {
|
|
|
782
790
|
}
|
|
783
791
|
|
|
784
792
|
// Construct signature
|
|
785
|
-
|
|
786
|
-
sig.update(
|
|
793
|
+
const sig = crypto.createHmac('sha256', options.credentials.accessToken);
|
|
794
|
+
sig.update(`version:${cert.version}\n`);
|
|
787
795
|
if (isNamed) {
|
|
788
|
-
sig.update(
|
|
789
|
-
sig.update(
|
|
796
|
+
sig.update(`clientId:${options.clientId}\n`);
|
|
797
|
+
sig.update(`issuer:${options.credentials.clientId}\n`);
|
|
790
798
|
}
|
|
791
|
-
sig.update(
|
|
792
|
-
sig.update(
|
|
793
|
-
sig.update(
|
|
799
|
+
sig.update(`seed:${cert.seed}\n`);
|
|
800
|
+
sig.update(`start:${cert.start}\n`);
|
|
801
|
+
sig.update(`expiry:${cert.expiry}\n`);
|
|
794
802
|
sig.update('scopes:\n');
|
|
795
803
|
sig.update(cert.scopes.join('\n'));
|
|
796
804
|
cert.signature = sig.digest('base64');
|
|
797
805
|
|
|
798
806
|
// Construct temporary key
|
|
799
|
-
|
|
807
|
+
const accessToken = crypto
|
|
800
808
|
.createHmac('sha256', options.credentials.accessToken)
|
|
801
809
|
.update(cert.seed)
|
|
802
810
|
.digest('base64')
|
|
@@ -831,8 +839,8 @@ export const createTemporaryCredentials = function(options) {
|
|
|
831
839
|
* scopes: [...], // associated scopes (if available)
|
|
832
840
|
* }
|
|
833
841
|
*/
|
|
834
|
-
export const credentialInformation =
|
|
835
|
-
|
|
842
|
+
export const credentialInformation = (rootUrl, credentials) => {
|
|
843
|
+
const result = {};
|
|
836
844
|
let issuer = credentials.clientId;
|
|
837
845
|
|
|
838
846
|
result.clientId = issuer;
|
|
@@ -862,9 +870,9 @@ export const credentialInformation = function(rootUrl, credentials) {
|
|
|
862
870
|
result.type = 'permanent';
|
|
863
871
|
}
|
|
864
872
|
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
873
|
+
const anonClient = new clients.Auth({ rootUrl });
|
|
874
|
+
const clientLookup = anonClient.client(issuer).then(client => {
|
|
875
|
+
const expires = new Date(client.expires);
|
|
868
876
|
if (!result.expiry || result.expiry > expires) {
|
|
869
877
|
result.expiry = expires;
|
|
870
878
|
}
|
|
@@ -873,14 +881,14 @@ export const credentialInformation = function(rootUrl, credentials) {
|
|
|
873
881
|
}
|
|
874
882
|
});
|
|
875
883
|
|
|
876
|
-
|
|
877
|
-
|
|
884
|
+
const credClient = new clients.Auth({ rootUrl, credentials });
|
|
885
|
+
const scopeLookup = credClient.currentScopes().then(response => {
|
|
878
886
|
result.scopes = response.scopes;
|
|
879
887
|
});
|
|
880
888
|
|
|
881
|
-
return Promise.all([clientLookup, scopeLookup]).then(
|
|
889
|
+
return Promise.all([clientLookup, scopeLookup]).then(() => {
|
|
882
890
|
// re-calculate "active" based on updated start/expiration
|
|
883
|
-
|
|
891
|
+
const now = new Date();
|
|
884
892
|
if (result.start && result.start > now) {
|
|
885
893
|
result.active = false;
|
|
886
894
|
} else if (result.expiry && now > result.expiry) {
|
package/src/download.js
CHANGED
|
@@ -13,7 +13,7 @@ const makeRetryCfg = ({ retries, delayFactor, randomizationFactor, maxDelay }) =
|
|
|
13
13
|
});
|
|
14
14
|
|
|
15
15
|
const s3 = async ({ url, streamFactory, retryCfg }) => {
|
|
16
|
-
return await retry(retryCfg, async
|
|
16
|
+
return await retry(retryCfg, async retriableError => {
|
|
17
17
|
let contentType = 'application/binary';
|
|
18
18
|
try {
|
|
19
19
|
const src = got.stream(url, { retry: { limit: 0 } });
|
|
@@ -38,7 +38,7 @@ const getUrl = async ({ object, name, resp, streamFactory, retryCfg }) => {
|
|
|
38
38
|
let hashStream;
|
|
39
39
|
let contentType = 'application/binary';
|
|
40
40
|
|
|
41
|
-
await retry(retryCfg, async
|
|
41
|
+
await retry(retryCfg, async retriableError => {
|
|
42
42
|
// renew the download URL if necessary (note that we assume the object-sevice
|
|
43
43
|
// credentials are good for long enough)
|
|
44
44
|
if (responseUsed && new Date(resp.expires) < new Date()) {
|
|
@@ -78,7 +78,7 @@ const getUrl = async ({ object, name, resp, streamFactory, retryCfg }) => {
|
|
|
78
78
|
// "acceptable" hash algorithm. Throws an exception on verification failure.
|
|
79
79
|
const verifyHashes = (observedHashes, expectedHashes) => {
|
|
80
80
|
let someValidAcceptableHash = false;
|
|
81
|
-
for (
|
|
81
|
+
for (const algo of Object.keys(expectedHashes)) {
|
|
82
82
|
const computed = observedHashes[algo];
|
|
83
83
|
if (!computed) {
|
|
84
84
|
// ignore unknown hash algorithms
|
|
@@ -94,12 +94,19 @@ const verifyHashes = (observedHashes, expectedHashes) => {
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
if (!someValidAcceptableHash) {
|
|
97
|
-
throw new Error(
|
|
97
|
+
throw new Error('No acceptable hash algorithm found');
|
|
98
98
|
}
|
|
99
99
|
};
|
|
100
100
|
|
|
101
|
-
export const download = async ({
|
|
102
|
-
|
|
101
|
+
export const download = async ({
|
|
102
|
+
name,
|
|
103
|
+
object,
|
|
104
|
+
streamFactory,
|
|
105
|
+
retries,
|
|
106
|
+
delayFactor,
|
|
107
|
+
randomizationFactor,
|
|
108
|
+
maxDelay,
|
|
109
|
+
}) => {
|
|
103
110
|
const retryCfg = makeRetryCfg({ retries, delayFactor, randomizationFactor, maxDelay });
|
|
104
111
|
|
|
105
112
|
const acceptDownloadMethods = {
|
|
@@ -111,24 +118,34 @@ export const download = async ({ name, object, streamFactory,
|
|
|
111
118
|
if (resp.method === 'getUrl') {
|
|
112
119
|
return await getUrl({ object, name, resp, streamFactory, retryCfg });
|
|
113
120
|
} else {
|
|
114
|
-
throw new Error(
|
|
121
|
+
throw new Error('Could not negotiate a download method');
|
|
115
122
|
}
|
|
116
123
|
};
|
|
117
124
|
|
|
118
125
|
export const downloadArtifact = async ({
|
|
119
|
-
taskId,
|
|
126
|
+
taskId,
|
|
127
|
+
runId,
|
|
128
|
+
name,
|
|
129
|
+
queue,
|
|
130
|
+
streamFactory,
|
|
131
|
+
retries,
|
|
132
|
+
delayFactor,
|
|
133
|
+
randomizationFactor,
|
|
134
|
+
maxDelay,
|
|
120
135
|
}) => {
|
|
121
136
|
const retryCfg = makeRetryCfg({ retries, delayFactor, randomizationFactor, maxDelay });
|
|
122
137
|
|
|
123
|
-
|
|
138
|
+
const artifact = await (runId === undefined
|
|
139
|
+
? queue.latestArtifact(taskId, name)
|
|
140
|
+
: queue.artifact(taskId, runId, name));
|
|
124
141
|
|
|
125
142
|
switch (artifact.storageType) {
|
|
126
|
-
case
|
|
127
|
-
case
|
|
143
|
+
case 'reference':
|
|
144
|
+
case 's3': {
|
|
128
145
|
return await s3({ url: artifact.url, streamFactory, retryCfg });
|
|
129
146
|
}
|
|
130
147
|
|
|
131
|
-
case
|
|
148
|
+
case 'object': {
|
|
132
149
|
const object = new clients.Object({
|
|
133
150
|
rootUrl: queue._options._trueRootUrl,
|
|
134
151
|
credentials: artifact.credentials,
|
|
@@ -136,7 +153,7 @@ export const downloadArtifact = async ({
|
|
|
136
153
|
return await download({ name: artifact.name, object, streamFactory, ...retryCfg });
|
|
137
154
|
}
|
|
138
155
|
|
|
139
|
-
case
|
|
156
|
+
case 'error': {
|
|
140
157
|
const err = new Error(artifact.message);
|
|
141
158
|
err.reason = artifact.reason;
|
|
142
159
|
throw err;
|
package/src/hashstream.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { Transform } from 'stream';
|
|
2
|
-
import { createHash } from 'crypto';
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
3
|
|
|
4
4
|
// The subset of hashes supported by HashStream which are "accepted" as per the
|
|
5
5
|
// object service's schemas.
|
|
6
|
-
export const ACCEPTABLE_HASHES = new Set([
|
|
6
|
+
export const ACCEPTABLE_HASHES = new Set(['sha256', 'sha512']);
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* A stream that hashes the bytes passing through it
|
|
@@ -16,7 +16,7 @@ export class HashStream extends Transform {
|
|
|
16
16
|
this.bytes = 0;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
_transform(chunk,
|
|
19
|
+
_transform(chunk, _enc, cb) {
|
|
20
20
|
this.sha256.update(chunk);
|
|
21
21
|
this.sha512.update(chunk);
|
|
22
22
|
this.bytes += chunk.length;
|
package/src/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
2
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
4
4
|
|
|
5
5
|
export * from './client.js';
|
|
6
6
|
export * from './utils.js';
|
package/src/parsetime.js
CHANGED
|
@@ -1,26 +1,29 @@
|
|
|
1
1
|
// Regular expression matching:
|
|
2
2
|
// A years B months C days D hours E minutes F seconds
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
const timeExp = new RegExp(
|
|
4
|
+
[
|
|
5
|
+
'^(\\s*(-|\\+))?',
|
|
6
|
+
'(\\s*(\\d+)\\s*y((ears?)|r)?)?',
|
|
7
|
+
'(\\s*(\\d+)\\s*mo(nths?)?)?',
|
|
8
|
+
'(\\s*(\\d+)\\s*w((eeks?)|k)?)?',
|
|
9
|
+
'(\\s*(\\d+)\\s*d(ays?)?)?',
|
|
10
|
+
'(\\s*(\\d+)\\s*h((ours?)|r)?)?',
|
|
11
|
+
'(\\s*(\\d+)\\s*m(in(utes?)?)?)?',
|
|
12
|
+
'(\\s*(\\d+)\\s*s(ec(onds?)?)?)?',
|
|
13
|
+
'\\s*$',
|
|
14
|
+
].join(''),
|
|
15
|
+
'i'
|
|
16
|
+
);
|
|
14
17
|
|
|
15
18
|
/** Parse time string */
|
|
16
|
-
|
|
19
|
+
const parseTime = str => {
|
|
17
20
|
// Parse the string
|
|
18
|
-
|
|
21
|
+
const match = timeExp.exec(str || '');
|
|
19
22
|
if (!match) {
|
|
20
|
-
throw new Error(
|
|
23
|
+
throw new Error(`String: '${str}' isn't a time expression`);
|
|
21
24
|
}
|
|
22
25
|
// Negate if needed
|
|
23
|
-
|
|
26
|
+
const neg = match[2] === '-' ? -1 : 1;
|
|
24
27
|
// Return parsed values
|
|
25
28
|
return {
|
|
26
29
|
years: parseInt(match[4] || 0, 10) * neg,
|
package/src/retry.js
CHANGED
|
@@ -15,7 +15,7 @@ export default async ({ retries, delayFactor, randomizationFactor, maxDelay }, f
|
|
|
15
15
|
|
|
16
16
|
let retriableError = null;
|
|
17
17
|
|
|
18
|
-
const rv = await func(err => retriableError = err, attempt);
|
|
18
|
+
const rv = await func(err => (retriableError = err), attempt);
|
|
19
19
|
if (!retriableError) {
|
|
20
20
|
// success!
|
|
21
21
|
return rv;
|
|
@@ -27,9 +27,9 @@ export default async ({ retries, delayFactor, randomizationFactor, maxDelay }, f
|
|
|
27
27
|
|
|
28
28
|
// Sleep for 2 * delayFactor on the first attempt, and 2x as long
|
|
29
29
|
// each time thereafter
|
|
30
|
-
let delay =
|
|
30
|
+
let delay = 2 ** attempt * delayFactor;
|
|
31
31
|
// Apply randomization factor
|
|
32
|
-
|
|
32
|
+
const rf = randomizationFactor;
|
|
33
33
|
delay *= Math.random() * 2 * rf + 1 - rf;
|
|
34
34
|
// Always limit with a maximum delay
|
|
35
35
|
delay = Math.min(delay, maxDelay);
|
package/src/upload.js
CHANGED
|
@@ -5,9 +5,9 @@ import { HashStream } from './hashstream.js';
|
|
|
5
5
|
|
|
6
6
|
const DATA_INLINE_MAX_SIZE = 8192;
|
|
7
7
|
|
|
8
|
-
const putUrl = async ({ streamFactory,
|
|
8
|
+
const putUrl = async ({ streamFactory, uploadMethod, retryCfg }) => {
|
|
9
9
|
const { url, headers } = uploadMethod.putUrl;
|
|
10
|
-
await retry(retryCfg, async
|
|
10
|
+
await retry(retryCfg, async retriableError => {
|
|
11
11
|
try {
|
|
12
12
|
await got.put(url, {
|
|
13
13
|
headers,
|
|
@@ -88,9 +88,9 @@ export const upload = async ({
|
|
|
88
88
|
if (res.uploadMethod.dataInline) {
|
|
89
89
|
// nothing to do
|
|
90
90
|
} else if (res.uploadMethod.putUrl) {
|
|
91
|
-
await putUrl({ streamFactory: hashStreamFactory,
|
|
91
|
+
await putUrl({ streamFactory: hashStreamFactory, uploadMethod: res.uploadMethod, retryCfg });
|
|
92
92
|
} else {
|
|
93
|
-
throw new Error(
|
|
93
|
+
throw new Error('Could not negotiate an upload method');
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
const hashes = hashStream.hashes(contentLength);
|
package/src/utils.js
CHANGED
|
@@ -10,7 +10,7 @@ import sluglib from 'slugid';
|
|
|
10
10
|
* short hand `1d2h3min`, it's fairly tolerant of different spelling forms and
|
|
11
11
|
* whitespace. But only really meant to be used with constants.
|
|
12
12
|
*/
|
|
13
|
-
export const fromNow =
|
|
13
|
+
export const fromNow = (offset, reference) => {
|
|
14
14
|
if (reference === undefined) {
|
|
15
15
|
reference = new Date();
|
|
16
16
|
}
|
|
@@ -19,15 +19,15 @@ export const fromNow = function(offset, reference) {
|
|
|
19
19
|
offset.days += 30 * offset.months;
|
|
20
20
|
offset.days += 365 * offset.years;
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
reference.getTime()
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
22
|
+
const retval = new Date(
|
|
23
|
+
reference.getTime() +
|
|
24
|
+
// + offset.years * 365 * 24 * 60 * 60 * 1000
|
|
25
|
+
// + offset.month * 30 * 24 * 60 * 60 * 1000
|
|
26
|
+
offset.weeks * 7 * 24 * 60 * 60 * 1000 +
|
|
27
|
+
offset.days * 24 * 60 * 60 * 1000 +
|
|
28
|
+
offset.hours * 60 * 60 * 1000 +
|
|
29
|
+
offset.minutes * 60 * 1000 +
|
|
30
|
+
offset.seconds * 1000
|
|
31
31
|
);
|
|
32
32
|
return retval;
|
|
33
33
|
};
|
|
@@ -45,9 +45,7 @@ export const fromNow = function(offset, reference) {
|
|
|
45
45
|
* short hand `1d2h3min`, it's fairly tolerant of different spelling forms and
|
|
46
46
|
* whitespace. But only really meant to be used with constants.
|
|
47
47
|
*/
|
|
48
|
-
export const fromNowJSON =
|
|
49
|
-
return fromNow(offset, reference).toJSON();
|
|
50
|
-
};
|
|
48
|
+
export const fromNowJSON = (offset, reference) => fromNow(offset, reference).toJSON();
|
|
51
49
|
|
|
52
50
|
// Export function to generate _nice_ slugids
|
|
53
51
|
export const slugid = () => sluglib.nice();
|