@taskcluster/client 88.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +572 -0
- package/package.json +36 -0
- package/src/apis.js +5089 -0
- package/src/client.js +891 -0
- package/src/download.js +150 -0
- package/src/hashstream.js +37 -0
- package/src/index.js +21 -0
- package/src/parsetime.js +37 -0
- package/src/retry.js +39 -0
- package/src/upload.js +101 -0
- package/src/utils.js +53 -0
package/src/client.js
ADDED
|
@@ -0,0 +1,891 @@
|
|
|
1
|
+
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
4
|
+
|
|
5
|
+
import got, { TimeoutError } from 'got';
|
|
6
|
+
|
|
7
|
+
import debugFactory from 'debug';
|
|
8
|
+
const debug = debugFactory('@taskcluster/client');
|
|
9
|
+
import _ from 'lodash';
|
|
10
|
+
import assert from 'assert';
|
|
11
|
+
import hawk from 'hawk';
|
|
12
|
+
import url from 'url';
|
|
13
|
+
import crypto from 'crypto';
|
|
14
|
+
import slugid from 'slugid';
|
|
15
|
+
import http from 'http';
|
|
16
|
+
import https from 'https';
|
|
17
|
+
import querystring from 'querystring';
|
|
18
|
+
import tcUrl from 'taskcluster-lib-urls';
|
|
19
|
+
import retry from './retry.js';
|
|
20
|
+
|
|
21
|
+
/** Default options for our http/https global agents */
|
|
22
|
+
let AGENT_OPTIONS = {
|
|
23
|
+
maxSockets: 50,
|
|
24
|
+
maxFreeSockets: 0,
|
|
25
|
+
keepAlive: false,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Generally shared agents is optimal we are creating our own rather then
|
|
30
|
+
* defaulting to the global node agents primarily so we can tweak this across
|
|
31
|
+
* all our components if needed...
|
|
32
|
+
*/
|
|
33
|
+
let DEFAULT_AGENTS = {
|
|
34
|
+
http: new http.Agent(AGENT_OPTIONS),
|
|
35
|
+
https: new https.Agent(AGENT_OPTIONS),
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
// Exports agents, consumers can provide their own default agents and tests
|
|
39
|
+
// can call taskcluster.agents.http.destroy() when running locally, otherwise
|
|
40
|
+
// tests won't terminate (if they are configured with keepAlive)
|
|
41
|
+
export const agents = DEFAULT_AGENTS;
|
|
42
|
+
|
|
43
|
+
// By default, all requests to a service go the the rootUrl and
|
|
44
|
+
// are load-balanced to services from there. However, if running
|
|
45
|
+
// inside a kubernetes cluster, you can opt to
|
|
46
|
+
// use kubernetes DNS to access other Taskcluster services
|
|
47
|
+
// These are something like `my-svc.my-namespace.svc.cluster-domain.example`
|
|
48
|
+
// but k8s also sets up search domains where the first one is
|
|
49
|
+
// `my-namespace.svc.cluster-domain.example` so you can just make a
|
|
50
|
+
// request to `my-svc` and it will route correctly
|
|
51
|
+
const SERVICE_DISCOVERY_SCHEMES = ['default', 'k8s-dns'];
|
|
52
|
+
let DEFAULT_SERVICE_DISCOVERY_SCHEME = 'default';
|
|
53
|
+
|
|
54
|
+
export const setServiceDiscoveryScheme = scheme => {
|
|
55
|
+
DEFAULT_SERVICE_DISCOVERY_SCHEME = scheme;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// Default options stored globally for convenience
|
|
59
|
+
let _defaultOptions = {
|
|
60
|
+
credentials: {
|
|
61
|
+
clientId: undefined,
|
|
62
|
+
accessToken: undefined,
|
|
63
|
+
certificate: undefined,
|
|
64
|
+
},
|
|
65
|
+
// Request time out (defaults to 30 seconds)
|
|
66
|
+
timeout: 30 * 1000,
|
|
67
|
+
// Max number of request retries
|
|
68
|
+
retries: 5,
|
|
69
|
+
// Multiplier for computation of retry delay: 2 ^ retry * delayFactor,
|
|
70
|
+
// 100 ms is solid for servers, and 500ms - 1s is suitable for background
|
|
71
|
+
// processes
|
|
72
|
+
delayFactor: 100,
|
|
73
|
+
// Randomization factor added as.
|
|
74
|
+
// delay = delay * random([1 - randomizationFactor; 1 + randomizationFactor])
|
|
75
|
+
randomizationFactor: 0.25,
|
|
76
|
+
// Maximum retry delay (defaults to 30 seconds)
|
|
77
|
+
maxDelay: 30 * 1000,
|
|
78
|
+
|
|
79
|
+
// The prefix of any api calls. e.g. https://taskcluster.net/api/
|
|
80
|
+
rootUrl: undefined,
|
|
81
|
+
|
|
82
|
+
// See above for what this means
|
|
83
|
+
serviceDiscoveryScheme: undefined,
|
|
84
|
+
|
|
85
|
+
// Fake methods, if given this will produce a fake client object.
|
|
86
|
+
// Methods called won't make expected HTTP requests, but instead:
|
|
87
|
+
// 1. Add arguments to `Client.fakeCalls.<method>.push({...params, payload, query})`
|
|
88
|
+
// 2. Invoke and return `fake.<method>(...args)`
|
|
89
|
+
//
|
|
90
|
+
// This allows `Client.fakeCalls.<method>` to be used for assertions, and
|
|
91
|
+
// `fake.<method>` can be used inject fake implementations.
|
|
92
|
+
fake: null,
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
/** Make a request for a Client instance */
|
|
96
|
+
export const makeRequest = async function(client, method, url, payload, query) {
|
|
97
|
+
// Add query to url if present
|
|
98
|
+
if (query) {
|
|
99
|
+
query = querystring.stringify(query);
|
|
100
|
+
if (query.length > 0) {
|
|
101
|
+
url += '?' + query;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const options = {
|
|
106
|
+
method: method.toUpperCase(),
|
|
107
|
+
agent: client._httpAgent,
|
|
108
|
+
followRedirect: false,
|
|
109
|
+
timeout: {
|
|
110
|
+
request: client._timeout,
|
|
111
|
+
},
|
|
112
|
+
headers: {},
|
|
113
|
+
responseType: 'text',
|
|
114
|
+
retry: {
|
|
115
|
+
limit: 0,
|
|
116
|
+
},
|
|
117
|
+
hooks: {
|
|
118
|
+
afterResponse: [res => {
|
|
119
|
+
// parse the body, if one was given (Got's `responseType: json` fails to check content-type)
|
|
120
|
+
if (res.rawBody.length > 0 && (res.headers['content-type'] || '').startsWith('application/json')) {
|
|
121
|
+
res.body = JSON.parse(res.rawBody);
|
|
122
|
+
}
|
|
123
|
+
return res;
|
|
124
|
+
}],
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
if (client._options.traceId) {
|
|
129
|
+
options.headers['x-taskcluster-trace-id'] = client._options.traceId;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Authenticate, if credentials are provided
|
|
133
|
+
if (client._options.credentials &&
|
|
134
|
+
client._options.credentials.clientId &&
|
|
135
|
+
client._options.credentials.accessToken) {
|
|
136
|
+
// Create hawk authentication header
|
|
137
|
+
let header = hawk.client.header(url, method.toUpperCase(), {
|
|
138
|
+
credentials: {
|
|
139
|
+
id: client._options.credentials.clientId,
|
|
140
|
+
key: client._options.credentials.accessToken,
|
|
141
|
+
algorithm: 'sha256',
|
|
142
|
+
},
|
|
143
|
+
ext: client._extData,
|
|
144
|
+
});
|
|
145
|
+
options.headers['Authorization'] = header.header;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Send payload if defined
|
|
149
|
+
if (payload !== undefined) {
|
|
150
|
+
options.json = payload;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
let res;
|
|
154
|
+
try {
|
|
155
|
+
res = await got(url, options);
|
|
156
|
+
} catch (err) {
|
|
157
|
+
// translate errors as users expect them, for compatibility
|
|
158
|
+
if (err instanceof TimeoutError) {
|
|
159
|
+
err.code = 'ECONNABORTED';
|
|
160
|
+
}
|
|
161
|
+
throw err;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
return res;
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Create a client class from a JSON reference, and an optional `name`, which is
|
|
169
|
+
* mostly intended for debugging, error messages and stats.
|
|
170
|
+
*
|
|
171
|
+
* Returns a Client class which can be initialized with following options:
|
|
172
|
+
* options:
|
|
173
|
+
* {
|
|
174
|
+
* // Taskcluster credentials, if not provided fallback to defaults from
|
|
175
|
+
* // environment variables, if defaults are not explicitly set with
|
|
176
|
+
* // taskcluster.config({...}).
|
|
177
|
+
* // To create a client without authentication (and not using defaults)
|
|
178
|
+
* // use `credentials: {}`
|
|
179
|
+
* credentials: {
|
|
180
|
+
* clientId: '...', // ClientId
|
|
181
|
+
* accessToken: '...', // AccessToken for clientId
|
|
182
|
+
* certificate: {...} // Certificate, if temporary credentials
|
|
183
|
+
* },
|
|
184
|
+
* // Limit the set of scopes requests with this client may make.
|
|
185
|
+
* // Note, that your clientId must have a superset of the these scopes.
|
|
186
|
+
* authorizedScopes: ['scope1', 'scope2', ...]
|
|
187
|
+
* retries: 5, // Maximum number of retries
|
|
188
|
+
* rootUrl: 'https://taskcluster.net/api/' // prefix for all api calls
|
|
189
|
+
* }
|
|
190
|
+
*
|
|
191
|
+
* `rootUrl` and `baseUrl` are mutually exclusive.
|
|
192
|
+
*/
|
|
193
|
+
export const createClient = function(reference, name) {
|
|
194
|
+
if (!name || typeof name !== 'string') {
|
|
195
|
+
name = 'Unknown';
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Client class constructor
|
|
199
|
+
let Client = function(options) {
|
|
200
|
+
if (options && options.baseUrl) {
|
|
201
|
+
throw new Error('baseUrl has been deprecated!');
|
|
202
|
+
}
|
|
203
|
+
if (options && options.exchangePrefix) {
|
|
204
|
+
throw new Error('exchangePrefix has been deprecated!');
|
|
205
|
+
}
|
|
206
|
+
let serviceName = reference.serviceName;
|
|
207
|
+
|
|
208
|
+
// allow for older schemas; this should be deleted once it is no longer used.
|
|
209
|
+
if (!serviceName) {
|
|
210
|
+
if (reference.name) {
|
|
211
|
+
// it was called this for a while; https://bugzilla.mozilla.org/show_bug.cgi?id=1463207
|
|
212
|
+
serviceName = reference.name;
|
|
213
|
+
} else if (reference.baseUrl) {
|
|
214
|
+
serviceName = reference.baseUrl.split('//')[1].split('.')[0];
|
|
215
|
+
} else if (reference.exchangePrefix) {
|
|
216
|
+
serviceName = reference.exchangePrefix.split('/')[1].replace('taskcluster-', '');
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
this._options = _.defaults({}, options || {}, {
|
|
220
|
+
exchangePrefix: reference.exchangePrefix,
|
|
221
|
+
serviceName,
|
|
222
|
+
serviceVersion: 'v1',
|
|
223
|
+
}, _defaultOptions);
|
|
224
|
+
assert(this._options.rootUrl, 'Must provide a rootUrl'); // We always assert this even with service discovery
|
|
225
|
+
this._options.rootUrl = this._options.rootUrl.replace(/\/$/, '');
|
|
226
|
+
this._options._trueRootUrl = this._options.rootUrl.replace(/\/$/, ''); // Useful for buildUrl/buildSignedUrl in certain cases
|
|
227
|
+
|
|
228
|
+
this._options.serviceDiscoveryScheme = this._options.serviceDiscoveryScheme || DEFAULT_SERVICE_DISCOVERY_SCHEME;
|
|
229
|
+
if (!SERVICE_DISCOVERY_SCHEMES.includes(this._options.serviceDiscoveryScheme)) {
|
|
230
|
+
throw new Error(`Invalid Taskcluster client service discovery scheme: ${this._options.serviceDiscoveryScheme}`);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
if (this._options.serviceDiscoveryScheme === 'k8s-dns') {
|
|
234
|
+
this._options.rootUrl = `http://taskcluster-${serviceName}`; // Notice this is http, not https
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (this._options.stats || this._options.monitor) {
|
|
238
|
+
throw new Error('monitoring client calls is no longer supported');
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
if (this._options.randomizationFactor < 0 ||
|
|
242
|
+
this._options.randomizationFactor >= 1) {
|
|
243
|
+
throw new Error('options.randomizationFactor must be between 0 and 1!');
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (this._options.agent) {
|
|
247
|
+
// We have explicit options for new agent create one...
|
|
248
|
+
this._httpAgent = {
|
|
249
|
+
https: new https.Agent(this._options.agent),
|
|
250
|
+
http: new http.Agent(this._options.agent),
|
|
251
|
+
};
|
|
252
|
+
} else {
|
|
253
|
+
// Use default global agent(s)...
|
|
254
|
+
this._httpAgent = {
|
|
255
|
+
https: DEFAULT_AGENTS.https,
|
|
256
|
+
http: DEFAULT_AGENTS.http,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// Timeout for each _individual_ http request.
|
|
261
|
+
this._timeout = this._options.timeout;
|
|
262
|
+
|
|
263
|
+
// Build ext for hawk requests
|
|
264
|
+
this._extData = undefined;
|
|
265
|
+
if (this._options.credentials &&
|
|
266
|
+
this._options.credentials.clientId &&
|
|
267
|
+
this._options.credentials.accessToken) {
|
|
268
|
+
let ext = {};
|
|
269
|
+
|
|
270
|
+
// If there is a certificate we have temporary credentials, and we
|
|
271
|
+
// must provide the certificate
|
|
272
|
+
if (this._options.credentials.certificate) {
|
|
273
|
+
ext.certificate = this._options.credentials.certificate;
|
|
274
|
+
// Parse as JSON if it's a string
|
|
275
|
+
if (typeof ext.certificate === 'string') {
|
|
276
|
+
try {
|
|
277
|
+
ext.certificate = JSON.parse(ext.certificate);
|
|
278
|
+
} catch (err) {
|
|
279
|
+
debug('Failed to parse credentials.certificate, err: %s, JSON: %j',
|
|
280
|
+
err, err);
|
|
281
|
+
throw new Error('JSON.parse(): Failed for configured certificate');
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// If set of authorized scopes is provided, we'll restrict the request
|
|
287
|
+
// to only use these scopes
|
|
288
|
+
if (this._options.authorizedScopes instanceof Array) {
|
|
289
|
+
ext.authorizedScopes = this._options.authorizedScopes;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// ext has any keys we better base64 encode it, and set ext on extra
|
|
293
|
+
if (_.keys(ext).length > 0) {
|
|
294
|
+
this._extData = Buffer.from(JSON.stringify(ext)).toString('base64');
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// If fake, we create an array this.fakeCalls[method] = [] for each method
|
|
299
|
+
if (this._options.fake) {
|
|
300
|
+
debug('Creating @taskcluster/client object in "fake" mode');
|
|
301
|
+
this.fakeCalls = {};
|
|
302
|
+
reference.entries.filter(e => e.type === 'function').forEach(e => this.fakeCalls[e.name] = []);
|
|
303
|
+
// Throw an error if creating fakes in production
|
|
304
|
+
if (process.env.NODE_ENV === 'production') {
|
|
305
|
+
new Error('@taskcluster/client object created in "fake" mode, when NODE_ENV == "production"');
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
Client.prototype.use = function(optionsUpdates) {
|
|
311
|
+
let options = _.defaults({}, optionsUpdates, { rootUrl: this._options._trueRootUrl }, this._options);
|
|
312
|
+
return new Client(options);
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
Client.prototype.taskclusterPerRequestInstance = function({ requestId, traceId }) {
|
|
316
|
+
return this.use({ traceId });
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
// For each function entry create a method on the Client class
|
|
320
|
+
reference.entries.filter(function(entry) {
|
|
321
|
+
return entry.type === 'function';
|
|
322
|
+
}).forEach(function(entry) {
|
|
323
|
+
// Get number of arguments
|
|
324
|
+
let nb_args = entry.args.length;
|
|
325
|
+
if (entry.input) {
|
|
326
|
+
nb_args += 1;
|
|
327
|
+
}
|
|
328
|
+
// Get the query-string options taken
|
|
329
|
+
let optKeys = entry.query || [];
|
|
330
|
+
|
|
331
|
+
// Create method on prototype
|
|
332
|
+
Client.prototype[entry.name] = function() {
|
|
333
|
+
// Convert arguments to actual array
|
|
334
|
+
let args = Array.prototype.slice.call(arguments);
|
|
335
|
+
// Validate number of arguments
|
|
336
|
+
let N = args.length;
|
|
337
|
+
if (N !== nb_args && (optKeys.length === 0 || N !== nb_args + 1)) {
|
|
338
|
+
throw new Error('Function ' + entry.name + ' takes ' + nb_args +
|
|
339
|
+
' arguments, but was given ' + N +
|
|
340
|
+
' arguments');
|
|
341
|
+
}
|
|
342
|
+
// Substitute parameters into route
|
|
343
|
+
let endpoint = entry.route.replace(/<([^<>]+)>/g, function(text, arg) {
|
|
344
|
+
let index = entry.args.indexOf(arg);
|
|
345
|
+
if (index !== -1) {
|
|
346
|
+
let param = args[index];
|
|
347
|
+
if (typeof param !== 'string' && typeof param !== 'number') {
|
|
348
|
+
throw new Error('URL parameter ' + arg + ' must be a string, but ' +
|
|
349
|
+
'we received a: ' + typeof param);
|
|
350
|
+
}
|
|
351
|
+
return encodeURIComponent(param);
|
|
352
|
+
}
|
|
353
|
+
return text; // Preserve original
|
|
354
|
+
});
|
|
355
|
+
// Create url for the request
|
|
356
|
+
let url = tcUrl.api(this._options.rootUrl, this._options.serviceName, this._options.serviceVersion, endpoint);
|
|
357
|
+
// Add payload if one is given
|
|
358
|
+
let payload = undefined;
|
|
359
|
+
if (entry.input) {
|
|
360
|
+
payload = args[nb_args - 1];
|
|
361
|
+
}
|
|
362
|
+
// Find query string options (if present)
|
|
363
|
+
let query = args[nb_args] || null;
|
|
364
|
+
if (query) {
|
|
365
|
+
_.keys(query).forEach(function(key) {
|
|
366
|
+
if (!_.includes(optKeys, key)) {
|
|
367
|
+
throw new Error('Function ' + entry.name + ' takes options: ' +
|
|
368
|
+
optKeys.join(', ') + ' but was given ' + key);
|
|
369
|
+
}
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// call out to the fake version, if set
|
|
374
|
+
if (this._options.fake) {
|
|
375
|
+
debug('Faking call to %s(%s)', entry.name, args.map(a => JSON.stringify(a, null, 2)).join(', '));
|
|
376
|
+
// Add a call record to fakeCalls[<method>]
|
|
377
|
+
let record = {};
|
|
378
|
+
if (payload !== undefined) {
|
|
379
|
+
record.payload = _.cloneDeep(payload);
|
|
380
|
+
}
|
|
381
|
+
if (query !== null) {
|
|
382
|
+
record.query = _.cloneDeep(query);
|
|
383
|
+
}
|
|
384
|
+
entry.args.forEach((k, i) => record[k] = _.cloneDeep(args[i]));
|
|
385
|
+
this.fakeCalls[entry.name].push(record);
|
|
386
|
+
// Call fake[<method>]
|
|
387
|
+
if (!this._options.fake[entry.name]) {
|
|
388
|
+
return Promise.reject(new Error(
|
|
389
|
+
`Faked ${this._options.serviceName} object does not have an implementation of ${entry.name}`,
|
|
390
|
+
));
|
|
391
|
+
}
|
|
392
|
+
return this._options.fake[entry.name].apply(this, args);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
return retry(this._options, (retriableError, attempt) => {
|
|
396
|
+
debug('Calling: %s, retry: %s', entry.name, attempt);
|
|
397
|
+
// Make request and handle response or error
|
|
398
|
+
return makeRequest(
|
|
399
|
+
this,
|
|
400
|
+
entry.method,
|
|
401
|
+
url,
|
|
402
|
+
payload,
|
|
403
|
+
query,
|
|
404
|
+
).then(function(res) {
|
|
405
|
+
// If request was successful, accept the result
|
|
406
|
+
debug('Success calling: %s, (%s retries)', entry.name, attempt);
|
|
407
|
+
if (!_.includes(res.headers['content-type'], 'application/json') || !res.body) {
|
|
408
|
+
debug('Empty response from server: call: %s, method: %s', entry.name, entry.method);
|
|
409
|
+
return undefined;
|
|
410
|
+
}
|
|
411
|
+
return res.body;
|
|
412
|
+
}, function(err) {
|
|
413
|
+
// If we got a response we read the error code from the response
|
|
414
|
+
let res = err.response;
|
|
415
|
+
if (res) {
|
|
416
|
+
let message = 'Unknown Server Error';
|
|
417
|
+
if (res.statusCode === 401) {
|
|
418
|
+
message = 'Authentication Error';
|
|
419
|
+
}
|
|
420
|
+
if (res.statusCode === 500) {
|
|
421
|
+
message = 'Internal Server Error';
|
|
422
|
+
}
|
|
423
|
+
if (res.statusCode >= 300 && res.statusCode < 400) {
|
|
424
|
+
message = 'Unexpected Redirect';
|
|
425
|
+
}
|
|
426
|
+
err = new Error(res.body.message || message);
|
|
427
|
+
err.body = res.body;
|
|
428
|
+
err.code = res.body.code || 'UnknownError';
|
|
429
|
+
err.statusCode = res.statusCode;
|
|
430
|
+
|
|
431
|
+
// Decide if we should retry or just throw
|
|
432
|
+
if (res.statusCode >= 500 && // Check if it's a 5xx error
|
|
433
|
+
res.statusCode < 600) {
|
|
434
|
+
debug('Error calling: %s now retrying, info: %j',
|
|
435
|
+
entry.name, res.body);
|
|
436
|
+
return retriableError(err);
|
|
437
|
+
} else {
|
|
438
|
+
debug('Error calling: %s NOT retrying!, info: %j',
|
|
439
|
+
entry.name, res.body);
|
|
440
|
+
throw err;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// All errors without a response are treated as retriable
|
|
445
|
+
debug('Request error calling %s (retrying), err: %s, JSON: %s',
|
|
446
|
+
entry.name, err, err);
|
|
447
|
+
return retriableError(err);
|
|
448
|
+
});
|
|
449
|
+
});
|
|
450
|
+
};
|
|
451
|
+
// Add reference for buildUrl and signUrl
|
|
452
|
+
Client.prototype[entry.name].entryReference = entry;
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
// For each topic-exchange entry
|
|
456
|
+
reference.entries.filter(function(entry) {
|
|
457
|
+
return entry.type === 'topic-exchange';
|
|
458
|
+
}).forEach(function(entry) {
|
|
459
|
+
// Create function for routing-key pattern construction
|
|
460
|
+
Client.prototype[entry.name] = function(routingKeyPattern) {
|
|
461
|
+
if (typeof routingKeyPattern !== 'string') {
|
|
462
|
+
// Allow for empty routing key patterns
|
|
463
|
+
if (routingKeyPattern === undefined ||
|
|
464
|
+
routingKeyPattern === null) {
|
|
465
|
+
routingKeyPattern = {};
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// Check that the routing key pattern is an object
|
|
469
|
+
assert(routingKeyPattern instanceof Object,
|
|
470
|
+
'routingKeyPattern must be an object');
|
|
471
|
+
|
|
472
|
+
// Construct routingkey pattern as string from reference
|
|
473
|
+
routingKeyPattern = entry.routingKey.map(function(key) {
|
|
474
|
+
// Get value for key
|
|
475
|
+
let value = routingKeyPattern[key.name];
|
|
476
|
+
// Routing key constant entries cannot be modified
|
|
477
|
+
if (key.constant) {
|
|
478
|
+
value = key.constant;
|
|
479
|
+
}
|
|
480
|
+
// If number convert to string
|
|
481
|
+
if (typeof value === 'number') {
|
|
482
|
+
return '' + value;
|
|
483
|
+
}
|
|
484
|
+
// Validate string and return
|
|
485
|
+
if (typeof value === 'string') {
|
|
486
|
+
// Check for multiple words
|
|
487
|
+
assert(key.multipleWords || value.indexOf('.') === -1,
|
|
488
|
+
'routingKey pattern \'' + value + '\' for ' + key.name +
|
|
489
|
+
' cannot contain dots as it does not hold multiple words');
|
|
490
|
+
return value;
|
|
491
|
+
}
|
|
492
|
+
// Check that we haven't got an invalid value
|
|
493
|
+
assert(value === null || value === undefined,
|
|
494
|
+
'Value: \'' + value + '\' is not supported as routingKey ' +
|
|
495
|
+
'pattern for ' + key.name);
|
|
496
|
+
// Return default pattern for entry not being matched
|
|
497
|
+
return key.multipleWords ? '#' : '*';
|
|
498
|
+
}).join('.');
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// Return values necessary to bind with EventHandler
|
|
502
|
+
return {
|
|
503
|
+
exchange: this._options.exchangePrefix + entry.exchange,
|
|
504
|
+
routingKeyPattern: routingKeyPattern,
|
|
505
|
+
routingKeyReference: _.cloneDeep(entry.routingKey),
|
|
506
|
+
};
|
|
507
|
+
};
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
Client.prototype._buildUrl = function(rootUrl, args) {
|
|
511
|
+
if (args.length === 0) {
|
|
512
|
+
throw new Error('buildUrl(method, arg1, arg2, ...) takes a least one ' +
|
|
513
|
+
'argument!');
|
|
514
|
+
}
|
|
515
|
+
// Find the method
|
|
516
|
+
let method = args.shift();
|
|
517
|
+
let entry = method.entryReference;
|
|
518
|
+
if (!entry || entry.type !== 'function') {
|
|
519
|
+
throw new Error('method in buildUrl(method, arg1, arg2, ...) must be ' +
|
|
520
|
+
'an API method from the same object!');
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// Get the query-string options taken
|
|
524
|
+
let optKeys = entry.query || [];
|
|
525
|
+
let supportsOpts = optKeys.length !== 0;
|
|
526
|
+
|
|
527
|
+
debug('build url for: ' + entry.name);
|
|
528
|
+
// Validate number of arguments
|
|
529
|
+
let N = entry.args.length;
|
|
530
|
+
if (args.length !== N && (!supportsOpts || args.length !== N + 1)) {
|
|
531
|
+
throw new Error('Function ' + entry.name + 'buildUrl() takes ' +
|
|
532
|
+
(N + 1) + ' arguments, but was given ' +
|
|
533
|
+
(args.length + 1) + ' arguments');
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// Substitute parameters into route
|
|
537
|
+
let endpoint = entry.route.replace(/<([^<>]+)>/g, function(text, arg) {
|
|
538
|
+
let index = entry.args.indexOf(arg);
|
|
539
|
+
if (index !== -1) {
|
|
540
|
+
let param = args[index];
|
|
541
|
+
if (typeof param !== 'string' && typeof param !== 'number') {
|
|
542
|
+
throw new Error('URL parameter ' + arg + ' must be a string, but ' +
|
|
543
|
+
'we received a: ' + typeof param);
|
|
544
|
+
}
|
|
545
|
+
return encodeURIComponent(param);
|
|
546
|
+
}
|
|
547
|
+
return text; // Preserve original
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
// Find query string options (if present)
|
|
551
|
+
let query = args[N] || '';
|
|
552
|
+
if (query) {
|
|
553
|
+
_.keys(query).forEach(function(key) {
|
|
554
|
+
if (!_.includes(optKeys, key)) {
|
|
555
|
+
throw new Error('Function ' + entry.name + ' takes options: ' +
|
|
556
|
+
optKeys.join(', ') + ' but was given ' + key);
|
|
557
|
+
}
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
query = querystring.stringify(query);
|
|
561
|
+
if (query.length > 0) {
|
|
562
|
+
query = '?' + query;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
return tcUrl.api(rootUrl, this._options.serviceName, this._options.serviceVersion, endpoint) + query;
|
|
567
|
+
};
|
|
568
|
+
|
|
569
|
+
// Utility functions to build the request URL for given method and
|
|
570
|
+
// input parameters. The first builds with whatever rootUrl currently
|
|
571
|
+
// is while the latter builds with trueRootUrl for sending to users
|
|
572
|
+
Client.prototype.buildUrl = function() {
|
|
573
|
+
return this._buildUrl(this._options.rootUrl, Array.prototype.slice.call(arguments));
|
|
574
|
+
};
|
|
575
|
+
Client.prototype.externalBuildUrl = function() {
|
|
576
|
+
return this._buildUrl(this._options._trueRootUrl, Array.prototype.slice.call(arguments));
|
|
577
|
+
};
|
|
578
|
+
|
|
579
|
+
Client.prototype._buildSignedUrl = function(builder, args) {
|
|
580
|
+
if (args.length === 0) {
|
|
581
|
+
throw new Error('buildSignedUrl(method, arg1, arg2, ..., [options]) ' +
|
|
582
|
+
'takes a least one argument!');
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// Find method and reference entry
|
|
586
|
+
let method = args[0];
|
|
587
|
+
let entry = method.entryReference;
|
|
588
|
+
if (entry.method !== 'get') {
|
|
589
|
+
throw new Error('buildSignedUrl only works for GET requests');
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
// Default to 15 minutes before expiration
|
|
593
|
+
let expiration = 15 * 60;
|
|
594
|
+
|
|
595
|
+
// Check if method supports query-string options
|
|
596
|
+
let supportsOpts = (entry.query || []).length !== 0;
|
|
597
|
+
|
|
598
|
+
// if longer than method + args, then we have options too
|
|
599
|
+
let N = entry.args.length + 1;
|
|
600
|
+
if (supportsOpts) {
|
|
601
|
+
N += 1;
|
|
602
|
+
}
|
|
603
|
+
if (args.length > N) {
|
|
604
|
+
// Get request options
|
|
605
|
+
let options = args.pop();
|
|
606
|
+
|
|
607
|
+
// Get expiration from options
|
|
608
|
+
expiration = options.expiration || expiration;
|
|
609
|
+
|
|
610
|
+
// Complain if expiration isn't a number
|
|
611
|
+
if (typeof expiration !== 'number') {
|
|
612
|
+
throw new Error('options.expiration must be a number');
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// Build URL
|
|
617
|
+
let requestUrl = builder.apply(this, args);
|
|
618
|
+
|
|
619
|
+
// Check that we have credentials
|
|
620
|
+
if (!this._options.credentials.clientId) {
|
|
621
|
+
throw new Error('credentials must be given');
|
|
622
|
+
}
|
|
623
|
+
if (!this._options.credentials.accessToken) {
|
|
624
|
+
throw new Error('accessToken must be given');
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
// Create bewit
|
|
628
|
+
let bewit = hawk.client.getBewit(requestUrl, {
|
|
629
|
+
credentials: {
|
|
630
|
+
id: this._options.credentials.clientId,
|
|
631
|
+
key: this._options.credentials.accessToken,
|
|
632
|
+
algorithm: 'sha256',
|
|
633
|
+
},
|
|
634
|
+
ttlSec: expiration,
|
|
635
|
+
ext: this._extData,
|
|
636
|
+
});
|
|
637
|
+
|
|
638
|
+
// Add bewit to requestUrl
|
|
639
|
+
let urlParts = url.parse(requestUrl);
|
|
640
|
+
if (urlParts.search) {
|
|
641
|
+
urlParts.search += '&bewit=' + bewit;
|
|
642
|
+
} else {
|
|
643
|
+
urlParts.search = '?bewit=' + bewit;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Return formatted URL
|
|
647
|
+
return url.format(urlParts);
|
|
648
|
+
};
|
|
649
|
+
|
|
650
|
+
// Utility function to construct a bewit URL for GET requests. Same convention
|
|
651
|
+
// as unsigned buildUrl applies here too
|
|
652
|
+
Client.prototype.buildSignedUrl = function() {
|
|
653
|
+
return this._buildSignedUrl(this.buildUrl, Array.prototype.slice.call(arguments));
|
|
654
|
+
};
|
|
655
|
+
Client.prototype.externalBuildSignedUrl = function() {
|
|
656
|
+
return this._buildSignedUrl(this.externalBuildUrl, Array.prototype.slice.call(arguments));
|
|
657
|
+
};
|
|
658
|
+
|
|
659
|
+
// Return client class
|
|
660
|
+
return Client;
|
|
661
|
+
};
|
|
662
|
+
|
|
663
|
+
// Load data from apis.js
|
|
664
|
+
import apis from './apis.js';
|
|
665
|
+
|
|
666
|
+
export const clients = {
|
|
667
|
+
Auth: createClient(apis.Auth.reference, "Auth"),
|
|
668
|
+
AuthEvents: createClient(apis.AuthEvents.reference, "AuthEvents"),
|
|
669
|
+
Github: createClient(apis.Github.reference, "Github"),
|
|
670
|
+
GithubEvents: createClient(apis.GithubEvents.reference, "GithubEvents"),
|
|
671
|
+
Hooks: createClient(apis.Hooks.reference, "Hooks"),
|
|
672
|
+
HooksEvents: createClient(apis.HooksEvents.reference, "HooksEvents"),
|
|
673
|
+
Index: createClient(apis.Index.reference, "Index"),
|
|
674
|
+
Notify: createClient(apis.Notify.reference, "Notify"),
|
|
675
|
+
NotifyEvents: createClient(apis.NotifyEvents.reference, "NotifyEvents"),
|
|
676
|
+
Object: createClient(apis.Object.reference, "Object"),
|
|
677
|
+
PurgeCache: createClient(apis.PurgeCache.reference, "PurgeCache"),
|
|
678
|
+
Queue: createClient(apis.Queue.reference, "Queue"),
|
|
679
|
+
QueueEvents: createClient(apis.QueueEvents.reference, "QueueEvents"),
|
|
680
|
+
Secrets: createClient(apis.Secrets.reference, "Secrets"),
|
|
681
|
+
WorkerManager: createClient(apis.WorkerManager.reference, "WorkerManager"),
|
|
682
|
+
WorkerManagerEvents: createClient(apis.WorkerManagerEvents.reference, "WorkerManagerEvents"),
|
|
683
|
+
};
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Update default configuration
|
|
687
|
+
*
|
|
688
|
+
* Example: `Client.config({credentials: {...}});`
|
|
689
|
+
*/
|
|
690
|
+
export const config = function(options) {
|
|
691
|
+
_defaultOptions = _.defaults({}, options, _defaultOptions);
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
export const fromEnvVars = function() {
|
|
695
|
+
let results = {};
|
|
696
|
+
for (let { env, path } of [
|
|
697
|
+
{ env: 'TASKCLUSTER_ROOT_URL', path: 'rootUrl' },
|
|
698
|
+
{ env: 'TASKCLUSTER_CLIENT_ID', path: 'credentials.clientId' },
|
|
699
|
+
{ env: 'TASKCLUSTER_ACCESS_TOKEN', path: 'credentials.accessToken' },
|
|
700
|
+
{ env: 'TASKCLUSTER_CERTIFICATE', path: 'credentials.certificate' },
|
|
701
|
+
]) {
|
|
702
|
+
if (process.env[env]) {
|
|
703
|
+
_.set(results, path, process.env[env]);
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
return results;
|
|
707
|
+
};
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* Construct a set of temporary credentials.
|
|
711
|
+
*
|
|
712
|
+
* options:
|
|
713
|
+
* {
|
|
714
|
+
* start: new Date(), // Start time of credentials (defaults to now)
|
|
715
|
+
* expiry: new Date(), // Credentials expiration time
|
|
716
|
+
* scopes: ['scope'...], // Scopes granted (defaults to empty-set)
|
|
717
|
+
* clientId: '...', // *optional* name to create named temporary credential
|
|
718
|
+
* credentials: { // (defaults to use global config, if available)
|
|
719
|
+
* clientId: '...', // ClientId
|
|
720
|
+
* accessToken: '...', // AccessToken for clientId
|
|
721
|
+
* },
|
|
722
|
+
* }
|
|
723
|
+
*
|
|
724
|
+
* Note that a named temporary credential is only valid if the issuing credentials
|
|
725
|
+
* have the scope 'auth:create-client:<name>'. This function does not check for
|
|
726
|
+
* this scope, but it will be checked when the credentials are used.
|
|
727
|
+
*
|
|
728
|
+
* Returns an object on the form: {clientId, accessToken, certificate}
|
|
729
|
+
*/
|
|
730
|
+
export const createTemporaryCredentials = function(options) {
|
|
731
|
+
assert(options, 'options are required');
|
|
732
|
+
|
|
733
|
+
let now = new Date();
|
|
734
|
+
|
|
735
|
+
// Set default options
|
|
736
|
+
options = _.defaults({}, options, {
|
|
737
|
+
// Clock drift is handled in auth service (PR #117)
|
|
738
|
+
// so no clock skew required.
|
|
739
|
+
start: now,
|
|
740
|
+
scopes: [],
|
|
741
|
+
}, _defaultOptions);
|
|
742
|
+
|
|
743
|
+
// Validate options
|
|
744
|
+
assert(options.credentials, 'options.credentials is required');
|
|
745
|
+
assert(options.credentials.clientId,
|
|
746
|
+
'options.credentials.clientId is required');
|
|
747
|
+
assert(options.credentials.accessToken,
|
|
748
|
+
'options.credentials.accessToken is required');
|
|
749
|
+
assert(options.credentials.certificate === undefined ||
|
|
750
|
+
options.credentials.certificate === null,
|
|
751
|
+
'temporary credentials cannot be used to make new temporary ' +
|
|
752
|
+
'credentials; ensure that options.credentials.certificate is null');
|
|
753
|
+
assert(options.start instanceof Date, 'options.start must be a Date');
|
|
754
|
+
assert(options.expiry instanceof Date, 'options.expiry must be a Date');
|
|
755
|
+
assert(options.scopes instanceof Array, 'options.scopes must be an array');
|
|
756
|
+
options.scopes.forEach(function(scope) {
|
|
757
|
+
assert(typeof scope === 'string',
|
|
758
|
+
'options.scopes must be an array of strings');
|
|
759
|
+
});
|
|
760
|
+
assert(options.expiry.getTime() - options.start.getTime() <=
|
|
761
|
+
31 * 24 * 60 * 60 * 1000, 'Credentials cannot span more than 31 days');
|
|
762
|
+
|
|
763
|
+
let isNamed = !!options.clientId;
|
|
764
|
+
|
|
765
|
+
if (isNamed) {
|
|
766
|
+
assert(options.clientId !== options.credentials.clientId,
|
|
767
|
+
'Credential issuer must be different from the name');
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// Construct certificate
|
|
771
|
+
let cert = {
|
|
772
|
+
version: 1,
|
|
773
|
+
scopes: _.cloneDeep(options.scopes),
|
|
774
|
+
start: options.start.getTime(),
|
|
775
|
+
expiry: options.expiry.getTime(),
|
|
776
|
+
seed: slugid.v4() + slugid.v4(),
|
|
777
|
+
signature: null, // generated later
|
|
778
|
+
};
|
|
779
|
+
if (isNamed) {
|
|
780
|
+
cert.issuer = options.credentials.clientId;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
// Construct signature
|
|
784
|
+
let sig = crypto.createHmac('sha256', options.credentials.accessToken);
|
|
785
|
+
sig.update('version:' + cert.version + '\n');
|
|
786
|
+
if (isNamed) {
|
|
787
|
+
sig.update('clientId:' + options.clientId + '\n');
|
|
788
|
+
sig.update('issuer:' + options.credentials.clientId + '\n');
|
|
789
|
+
}
|
|
790
|
+
sig.update('seed:' + cert.seed + '\n');
|
|
791
|
+
sig.update('start:' + cert.start + '\n');
|
|
792
|
+
sig.update('expiry:' + cert.expiry + '\n');
|
|
793
|
+
sig.update('scopes:\n');
|
|
794
|
+
sig.update(cert.scopes.join('\n'));
|
|
795
|
+
cert.signature = sig.digest('base64');
|
|
796
|
+
|
|
797
|
+
// Construct temporary key
|
|
798
|
+
let accessToken = crypto
|
|
799
|
+
.createHmac('sha256', options.credentials.accessToken)
|
|
800
|
+
.update(cert.seed)
|
|
801
|
+
.digest('base64')
|
|
802
|
+
.replace(/\+/g, '-') // Replace + with - (see RFC 4648, sec. 5)
|
|
803
|
+
.replace(/\//g, '_') // Replace / with _ (see RFC 4648, sec. 5)
|
|
804
|
+
.replace(/=/g, ''); // Drop '==' padding
|
|
805
|
+
|
|
806
|
+
// Return the generated temporary credentials
|
|
807
|
+
return {
|
|
808
|
+
clientId: isNamed ? options.clientId : options.credentials.clientId,
|
|
809
|
+
accessToken: accessToken,
|
|
810
|
+
certificate: JSON.stringify(cert),
|
|
811
|
+
};
|
|
812
|
+
};
|
|
813
|
+
|
|
814
|
+
/**
|
|
815
|
+
* Get information about a set of credentials.
|
|
816
|
+
*
|
|
817
|
+
* credentials: {
|
|
818
|
+
* clientId,
|
|
819
|
+
* accessToken,
|
|
820
|
+
* certificate, // optional
|
|
821
|
+
* }
|
|
822
|
+
*
|
|
823
|
+
* result: Promise for
|
|
824
|
+
* {
|
|
825
|
+
* clientId: .., // name of the credential
|
|
826
|
+
* type: .., // type of credential, e.g., "temporary"
|
|
827
|
+
* active: .., // active (valid, not disabled, etc.)
|
|
828
|
+
* start: .., // validity start time (if applicable)
|
|
829
|
+
* expiry: .., // validity end time (if applicable)
|
|
830
|
+
* scopes: [...], // associated scopes (if available)
|
|
831
|
+
* }
|
|
832
|
+
*/
|
|
833
|
+
export const credentialInformation = function(rootUrl, credentials) {
|
|
834
|
+
let result = {};
|
|
835
|
+
let issuer = credentials.clientId;
|
|
836
|
+
|
|
837
|
+
result.clientId = issuer;
|
|
838
|
+
result.active = true;
|
|
839
|
+
|
|
840
|
+
// distinguish permacreds from temporary creds
|
|
841
|
+
if (credentials.certificate) {
|
|
842
|
+
result.type = 'temporary';
|
|
843
|
+
let cert;
|
|
844
|
+
if (typeof credentials.certificate === 'string') {
|
|
845
|
+
try {
|
|
846
|
+
cert = JSON.parse(credentials.certificate);
|
|
847
|
+
} catch (err) {
|
|
848
|
+
return Promise.reject(err);
|
|
849
|
+
}
|
|
850
|
+
} else {
|
|
851
|
+
cert = credentials.certificate;
|
|
852
|
+
}
|
|
853
|
+
result.scopes = cert.scopes;
|
|
854
|
+
result.start = new Date(cert.start);
|
|
855
|
+
result.expiry = new Date(cert.expiry);
|
|
856
|
+
|
|
857
|
+
if (cert.issuer) {
|
|
858
|
+
issuer = cert.issuer;
|
|
859
|
+
}
|
|
860
|
+
} else {
|
|
861
|
+
result.type = 'permanent';
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
let anonClient = new clients.Auth({ rootUrl });
|
|
865
|
+
let clientLookup = anonClient.client(issuer).then(function(client) {
|
|
866
|
+
let expires = new Date(client.expires);
|
|
867
|
+
if (!result.expiry || result.expiry > expires) {
|
|
868
|
+
result.expiry = expires;
|
|
869
|
+
}
|
|
870
|
+
if (client.disabled) {
|
|
871
|
+
result.active = false;
|
|
872
|
+
}
|
|
873
|
+
});
|
|
874
|
+
|
|
875
|
+
let credClient = new clients.Auth({ rootUrl, credentials });
|
|
876
|
+
let scopeLookup = credClient.currentScopes().then(function(response) {
|
|
877
|
+
result.scopes = response.scopes;
|
|
878
|
+
});
|
|
879
|
+
|
|
880
|
+
return Promise.all([clientLookup, scopeLookup]).then(function() {
|
|
881
|
+
// re-calculate "active" based on updated start/expiration
|
|
882
|
+
let now = new Date();
|
|
883
|
+
if (result.start && result.start > now) {
|
|
884
|
+
result.active = false;
|
|
885
|
+
} else if (result.expiry && now > result.expiry) {
|
|
886
|
+
result.active = false;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
return result;
|
|
890
|
+
});
|
|
891
|
+
};
|