@taskcluster/client 88.0.1

package/src/download.js

@@ -0,0 +1,150 @@
+import got, { HTTPError } from 'got';
+import { pipeline } from 'node:stream/promises';
+import retry from './retry.js';
+import { HashStream, ACCEPTABLE_HASHES } from './hashstream.js';
+import { clients } from './client.js';
+
+// apply default retry config
+const makeRetryCfg = ({ retries, delayFactor, randomizationFactor, maxDelay }) => ({
+  retries: retries === undefined ? 5 : retries,
+  delayFactor: delayFactor === undefined ? 100 : delayFactor,
+  randomizationFactor: randomizationFactor === undefined ? randomizationFactor : 0.25,
+  maxDelay: maxDelay === undefined ? 30 * 1000 : maxDelay,
+});
+
+const s3 = async ({ url, streamFactory, retryCfg }) => {
+  return await retry(retryCfg, async (retriableError, attempt) => {
+    let contentType = 'application/binary';
+    try {
+      const src = got.stream(url, { retry: { limit: 0 } });
+      src.on('response', res => {
+        contentType = res.headers['content-type'] || contentType;
+      });
+      const dest = await streamFactory();
+      await pipeline(src, dest);
+      return contentType;
+    } catch (err) {
+      // treat non-500 HTTP responses as fatal errors, and retry everything else
+      if (err instanceof HTTPError && err.response.statusCode < 500) {
+        throw err;
+      }
+      return retriableError(err);
+    }
+  });
+};
+
+const getUrl = async ({ object, name, resp, streamFactory, retryCfg }) => {
+  let responseUsed = false;
+  let hashStream;
+  let contentType = 'application/binary';
+
+  await retry(retryCfg, async (retriableError, attempt) => {
+    // renew the download URL if necessary (note that we assume the object-sevice
+    // credentials are good for long enough)
+    if (responseUsed && new Date(resp.expires) < new Date()) {
+      resp = await object.startDownload(name, { acceptDownloadMethods: { getUrl: true } });
+      responseUsed = false;
+    }
+
+    try {
+      responseUsed = true;
+      const src = got.stream(resp.url, { retry: { limit: 0 } });
+      src.on('response', res => {
+        contentType = res.headers['content-type'] || contentType;
+      });
+      const dest = await streamFactory();
+      hashStream = new HashStream();
+      await pipeline(src, hashStream, dest);
+
+      return;
+    } catch (err) {
+      // treat non-500 HTTP responses as fatal errors, and retry everything else
+      if (err instanceof HTTPError && err.response.statusCode < 500) {
+        throw err;
+      }
+      return retriableError(err);
+    }
+  });
+
+  // now that the download is complete, check the hashes.  Note that a hash
+  // verification failure does not result in a retry.
+  const observedHashes = hashStream.hashes();
+  verifyHashes(observedHashes, resp.hashes);
+
+  return contentType;
+};
+
+// verify that all known hashes match, and that at least one of them is an
+// "acceptable" hash algorithm.  Throws an exception on verification failure.
+const verifyHashes = (observedHashes, expectedHashes) => {
+  let someValidAcceptableHash = false;
+  for (let algo of Object.keys(expectedHashes)) {
+    const computed = observedHashes[algo];
+    if (!computed) {
+      // ignore unknown hash algorithms
+      continue;
+    }
+    if (computed !== expectedHashes[algo]) {
+      throw new Error(`Computed ${algo} hash does not match that from object service`);
+    }
+
+    if (ACCEPTABLE_HASHES.has(algo)) {
+      someValidAcceptableHash = true;
+    }
+  }
+
+  if (!someValidAcceptableHash) {
+    throw new Error("No acceptable hash algorithm found");
+  }
+};
+
+export const download = async ({ name, object, streamFactory,
+  retries, delayFactor, randomizationFactor, maxDelay }) => {
+  const retryCfg = makeRetryCfg({ retries, delayFactor, randomizationFactor, maxDelay });
+
+  const acceptDownloadMethods = {
+    getUrl: true,
+  };
+
+  const resp = await object.startDownload(name, { acceptDownloadMethods });
+
+  if (resp.method === 'getUrl') {
+    return await getUrl({ object, name, resp, streamFactory, retryCfg });
+  } else {
+    throw new Error("Could not negotiate a download method");
+  }
+};
+
+export const downloadArtifact = async ({
+  taskId, runId, name, queue, streamFactory, retries, delayFactor, randomizationFactor, maxDelay,
+}) => {
+  const retryCfg = makeRetryCfg({ retries, delayFactor, randomizationFactor, maxDelay });
+
+  let artifact = await (runId === undefined ? queue.latestArtifact(taskId, name) : queue.artifact(taskId, runId, name));
+
+  switch (artifact.storageType) {
+    case "reference":
+    case "s3": {
+      return await s3({ url: artifact.url, streamFactory, retryCfg });
+    }
+
+    case "object": {
+      const object = new clients.Object({
+        rootUrl: queue._options._trueRootUrl,
+        credentials: artifact.credentials,
+      });
+      return await download({ name: artifact.name, object, streamFactory, ...retryCfg });
+    }
+
+    case "error": {
+      const err = new Error(artifact.message);
+      err.reason = artifact.reason;
+      throw err;
+    }
+
+    default:
+      throw new Error(`Unsupported artifact storageType '${artifact.storageType}'`);
+  }
+};
+
+export default { download, downloadArtifact };

package/src/hashstream.js

@@ -0,0 +1,37 @@
+import { Transform } from 'stream';
+import { createHash } from 'crypto';
+
+// The subset of hashes supported by HashStream which are "accepted" as per the
+// object service's schemas.
+export const ACCEPTABLE_HASHES = new Set(["sha256", "sha512"]);
+
+/**
+ * A stream that hashes the bytes passing through it
+ */
+export class HashStream extends Transform {
+  constructor() {
+    super();
+    this.sha256 = createHash('sha256');
+    this.sha512 = createHash('sha512');
+    this.bytes = 0;
+  }
+
+  _transform(chunk, enc, cb) {
+    this.sha256.update(chunk);
+    this.sha512.update(chunk);
+    this.bytes += chunk.length;
+    cb(null, chunk);
+  }
+
+  // Return the calculated hashes in a format suitable for finishUpload,
+  // checking that the content length matches the bytes hashed, if given.
+  hashes(contentLength) {
+    if (contentLength !== undefined && contentLength !== this.bytes) {
+      throw new Error(`Hashed ${this.bytes} bytes but content length is ${contentLength}`);
+    }
+    return {
+      sha256: this.sha256.digest('hex'),
+      sha512: this.sha512.digest('hex'),
+    };
+  }
+}

package/src/index.js

@@ -0,0 +1,21 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export * from './client.js';
+export * from './utils.js';
+export * from './upload.js';
+export * from './download.js';
+
+import * as client from './client.js';
+import * as utils from './utils.js';
+import * as upload from './upload.js';
+import * as download from './download.js';
+
+export default {
+  ...client,
+  ...client.clients,
+  ...utils,
+  ...upload,
+  ...download,
+};

package/src/parsetime.js

@@ -0,0 +1,37 @@
+// Regular expression matching:
+// A years B months C days D hours E minutes F seconds
+let timeExp = new RegExp([
+  '^(\\s*(-|\\+))?',
+  '(\\s*(\\d+)\\s*y((ears?)|r)?)?',
+  '(\\s*(\\d+)\\s*mo(nths?)?)?',
+  '(\\s*(\\d+)\\s*w((eeks?)|k)?)?',
+  '(\\s*(\\d+)\\s*d(ays?)?)?',
+  '(\\s*(\\d+)\\s*h((ours?)|r)?)?',
+  '(\\s*(\\d+)\\s*m(in(utes?)?)?)?',
+  '(\\s*(\\d+)\\s*s(ec(onds?)?)?)?',
+  '\\s*$',
+].join(''), 'i');
+
+/** Parse time string */
+let parseTime = function(str) {
+  // Parse the string
+  let match = timeExp.exec(str || '');
+  if (!match) {
+    throw new Error('String: \'' + str + '\' isn\'t a time expression');
+  }
+  // Negate if needed
+  let neg = match[2] === '-' ? - 1 : 1;
+  // Return parsed values
+  return {
+    years: parseInt(match[4] || 0, 10) * neg,
+    months: parseInt(match[8] || 0, 10) * neg,
+    weeks: parseInt(match[11] || 0, 10) * neg,
+    days: parseInt(match[15] || 0, 10) * neg,
+    hours: parseInt(match[18] || 0, 10) * neg,
+    minutes: parseInt(match[22] || 0, 10) * neg,
+    seconds: parseInt(match[25] || 0, 10) * neg,
+  };
+};
+
+// Export parseTime
+export default parseTime;

package/src/retry.js

@@ -0,0 +1,39 @@
+/**
+ * Call the given function based on the given retry configuration.  Each call to the
+ * function is given a `retriableError` callback.  For retriable failures, call this
+ * callback with an Error object (in case retries are exhausted) and return.  For
+ * fatal errors, simply throw the error as usual.  The second argument is the attempt number.
+ *
+ * Note that as per the existing API, `retries` is the number of retries after the first
+ * try; that is, a `retries` value of 3 means that `func` will be called 4 times.
+ */
+export default async ({ retries, delayFactor, randomizationFactor, maxDelay }, func) => {
+  let attempt = 0;
+
+  while (true) {
+    attempt += 1;
+
+    let retriableError = null;
+
+    const rv = await func(err => retriableError = err, attempt);
+    if (!retriableError) {
+      // success!
+      return rv;
+    }
+
+    if (attempt > retries) {
+      throw retriableError;
+    }
+
+    // Sleep for 2 * delayFactor on the first attempt, and 2x as long
+    // each time thereafter
+    let delay = Math.pow(2, attempt) * delayFactor;
+    // Apply randomization factor
+    let rf = randomizationFactor;
+    delay *= Math.random() * 2 * rf + 1 - rf;
+    // Always limit with a maximum delay
+    delay = Math.min(delay, maxDelay);
+    // Sleep before looping again
+    await new Promise(accept => setTimeout(accept, delay));
+  }
+};

package/src/upload.js

@@ -0,0 +1,101 @@
+import got, { HTTPError } from 'got';
+import { slugid } from './utils.js';
+import retry from './retry.js';
+import { HashStream } from './hashstream.js';
+
+const DATA_INLINE_MAX_SIZE = 8192;
+
+const putUrl = async ({ streamFactory, contentLength, uploadMethod, retryCfg }) => {
+  const { url, headers } = uploadMethod.putUrl;
+  await retry(retryCfg, async (retriableError, attempt) => {
+    try {
+      await got.put(url, {
+        headers,
+        retry: { limit: 0 }, // use our own retry logic
+        body: await streamFactory(),
+      });
+    } catch (err) {
+      // treat non-500 HTTP responses as fatal errors, and retry everything else
+      if (err instanceof HTTPError && err.response.statusCode < 500) {
+        throw err;
+      }
+      return retriableError(err);
+    }
+  });
+};
+
+const readFullStream = stream => {
+  const chunks = [];
+  return new Promise((accept, reject) => {
+    stream.on('data', chunk => chunks.push(chunk));
+    stream.on('error', err => reject(err));
+    stream.on('end', () => accept(Buffer.concat(chunks)));
+  });
+};
+
+export const upload = async ({
+  projectId,
+  name,
+  contentType,
+  contentLength,
+  expires,
+  object,
+  streamFactory,
+  retries,
+  delayFactor,
+  randomizationFactor,
+  maxDelay,
+  uploadId,
+}) => {
+  if (!uploadId) {
+    uploadId = slugid();
+  }
+
+  // set up to hash streams as we read them
+  let hashStream;
+  const hashStreamFactory = async () => {
+    hashStream = new HashStream();
+    return (await streamFactory()).pipe(hashStream);
+  };
+
+  // apply default retry config
+  const retryCfg = {
+    retries: retries === undefined ? 5 : retries,
+    delayFactor: delayFactor === undefined ? 100 : delayFactor,
+    randomizationFactor: randomizationFactor === undefined ? 0.25 : randomizationFactor,
+    maxDelay: maxDelay === undefined ? 30 * 1000 : maxDelay,
+  };
+
+  const proposedUploadMethods = {};
+
+  if (contentLength < DATA_INLINE_MAX_SIZE) {
+    // get the (small) data as a buffer to include in the request
+    const data = await readFullStream(await hashStreamFactory());
+
+    proposedUploadMethods.dataInline = {
+      contentType,
+      objectData: data.toString('base64'),
+    };
+  }
+
+  proposedUploadMethods.putUrl = {
+    contentType,
+    contentLength,
+  };
+
+  const res = await object.createUpload(name, { expires, projectId, uploadId, proposedUploadMethods });
+
+  if (res.uploadMethod.dataInline) {
+    // nothing to do
+  } else if (res.uploadMethod.putUrl) {
+    await putUrl({ streamFactory: hashStreamFactory, contentLength, uploadMethod: res.uploadMethod, retryCfg });
+  } else {
+    throw new Error("Could not negotiate an upload method");
+  }
+
+  const hashes = hashStream.hashes(contentLength);
+
+  await object.finishUpload(name, { projectId, uploadId, hashes });
+};
+
+export default upload;

package/src/utils.js

@@ -0,0 +1,53 @@
+import parseTime from './parsetime.js';
+import sluglib from 'slugid';
+
+/**
+ * Create a Date object offset = '1d 2h 3min' into the future
+ *
+ * Offset format: The argument `offset` (if given) is a string on the form
+ *   `1 day 2 hours 3 minutes`
+ * where specification of day, hours and minutes is optional. You can also the
+ * short hand `1d2h3min`, it's fairly tolerant of different spelling forms and
+ * whitespace. But only really meant to be used with constants.
+ */
+export const fromNow = function(offset, reference) {
+  if (reference === undefined) {
+    reference = new Date();
+  }
+  offset = parseTime(offset || '');
+
+  offset.days += 30 * offset.months;
+  offset.days += 365 * offset.years;
+
+  let retval = new Date(
+    reference.getTime()
+    //    + offset.years * 365 * 24 * 60 * 60 * 1000
+    //   + offset.month  * 30 * 24 * 60 * 60 * 1000
+    + offset.weeks * 7 * 24 * 60 * 60 * 1000
+    + offset.days * 24 * 60 * 60 * 1000
+    + offset.hours * 60 * 60 * 1000
+    + offset.minutes * 60 * 1000
+    + offset.seconds * 1000,
+  );
+  return retval;
+};
+
+/**
+ * Create an ISO 8601 time stamp offset = '1d 2h 3min' into the future
+ *
+ * This returns a time stamp in the format expected by taskcluster.
+ * Compatible with Date.toJSON() from Javascript. These time stamps are string
+ * that with UTC as timezone.
+ *
+ * Offset format: The argument `offset` (if given) is a string on the form
+ *   `1 day 2 hours 3 minutes`
+ * where specification of day, hours and minutes is optional. You can also the
+ * short hand `1d2h3min`, it's fairly tolerant of different spelling forms and
+ * whitespace. But only really meant to be used with constants.
+ */
+export const fromNowJSON = function(offset, reference) {
+  return fromNow(offset, reference).toJSON();
+};
+
+// Export function to generate _nice_ slugids
+export const slugid = () => sluglib.nice();