@flakiness/sdk 0.95.0 → 0.96.0

package/lib/cli/cmd-upload.js

@@ -1,256 +1,37 @@
 #!/usr/bin/env node
 
 // src/cli/cmd-upload.ts
-import { FlakinessReport } from "@flakiness/report";
-import fs2 from "fs/promises";
-import path from "path";
+import fs3 from "fs/promises";
+import path2 from "path";
 
 // src/reportUploader.ts
-import fs from "fs";
-import { URL as URL2 } from "url";
-import { brotliCompressSync as brotliCompressSync2 } from "zlib";
+import { compressTextAsync, compressTextSync } from "@flakiness/shared/node/compression.js";
+import assert from "assert";
+import fs2 from "fs";
+import { URL } from "url";
 
-// ../server/lib/common/typedHttp.js
-var TypedHTTP;
-((TypedHTTP2) => {
-  TypedHTTP2.StatusCodes = {
-    Informational: {
-      CONTINUE: 100,
-      SWITCHING_PROTOCOLS: 101,
-      PROCESSING: 102,
-      EARLY_HINTS: 103
-    },
-    Success: {
-      OK: 200,
-      CREATED: 201,
-      ACCEPTED: 202,
-      NON_AUTHORITATIVE_INFORMATION: 203,
-      NO_CONTENT: 204,
-      RESET_CONTENT: 205,
-      PARTIAL_CONTENT: 206,
-      MULTI_STATUS: 207
-    },
-    Redirection: {
-      MULTIPLE_CHOICES: 300,
-      MOVED_PERMANENTLY: 301,
-      MOVED_TEMPORARILY: 302,
-      SEE_OTHER: 303,
-      NOT_MODIFIED: 304,
-      USE_PROXY: 305,
-      TEMPORARY_REDIRECT: 307,
-      PERMANENT_REDIRECT: 308
-    },
-    ClientErrors: {
-      BAD_REQUEST: 400,
-      UNAUTHORIZED: 401,
-      PAYMENT_REQUIRED: 402,
-      FORBIDDEN: 403,
-      NOT_FOUND: 404,
-      METHOD_NOT_ALLOWED: 405,
-      NOT_ACCEPTABLE: 406,
-      PROXY_AUTHENTICATION_REQUIRED: 407,
-      REQUEST_TIMEOUT: 408,
-      CONFLICT: 409,
-      GONE: 410,
-      LENGTH_REQUIRED: 411,
-      PRECONDITION_FAILED: 412,
-      REQUEST_TOO_LONG: 413,
-      REQUEST_URI_TOO_LONG: 414,
-      UNSUPPORTED_MEDIA_TYPE: 415,
-      REQUESTED_RANGE_NOT_SATISFIABLE: 416,
-      EXPECTATION_FAILED: 417,
-      IM_A_TEAPOT: 418,
-      INSUFFICIENT_SPACE_ON_RESOURCE: 419,
-      METHOD_FAILURE: 420,
-      MISDIRECTED_REQUEST: 421,
-      UNPROCESSABLE_ENTITY: 422,
-      LOCKED: 423,
-      FAILED_DEPENDENCY: 424,
-      UPGRADE_REQUIRED: 426,
-      PRECONDITION_REQUIRED: 428,
-      TOO_MANY_REQUESTS: 429,
-      REQUEST_HEADER_FIELDS_TOO_LARGE: 431,
-      UNAVAILABLE_FOR_LEGAL_REASONS: 451
-    },
-    ServerErrors: {
-      INTERNAL_SERVER_ERROR: 500,
-      NOT_IMPLEMENTED: 501,
-      BAD_GATEWAY: 502,
-      SERVICE_UNAVAILABLE: 503,
-      GATEWAY_TIMEOUT: 504,
-      HTTP_VERSION_NOT_SUPPORTED: 505,
-      INSUFFICIENT_STORAGE: 507,
-      NETWORK_AUTHENTICATION_REQUIRED: 511
-    }
-  };
-  const AllErrorCodes = {
-    ...TypedHTTP2.StatusCodes.ClientErrors,
-    ...TypedHTTP2.StatusCodes.ServerErrors
-  };
-  class HttpError extends Error {
-    constructor(status, message) {
-      super(message);
-      this.status = status;
-    }
-    static withCode(code, message) {
-      const statusCode = AllErrorCodes[code];
-      const defaultMessage = code.split("_").map((word) => word.charAt(0) + word.slice(1).toLowerCase()).join(" ");
-      return new HttpError(statusCode, message ?? defaultMessage);
-    }
-  }
-  TypedHTTP2.HttpError = HttpError;
-  function isInformationalResponse(response) {
-    return response.status >= 100 && response.status < 200;
-  }
-  TypedHTTP2.isInformationalResponse = isInformationalResponse;
-  function isSuccessResponse(response) {
-    return response.status >= 200 && response.status < 300;
-  }
-  TypedHTTP2.isSuccessResponse = isSuccessResponse;
-  function isRedirectResponse(response) {
-    return response.status >= 300 && response.status < 400;
-  }
-  TypedHTTP2.isRedirectResponse = isRedirectResponse;
-  function isErrorResponse(response) {
-    return response.status >= 400 && response.status < 600;
-  }
-  TypedHTTP2.isErrorResponse = isErrorResponse;
-  function info(status) {
-    return { status };
-  }
-  TypedHTTP2.info = info;
-  function ok(data, status) {
-    return {
-      status: status ?? TypedHTTP2.StatusCodes.Success.OK,
-      data
-    };
-  }
-  TypedHTTP2.ok = ok;
-  function redirect(url, status = 302) {
-    return { status, url };
-  }
-  TypedHTTP2.redirect = redirect;
-  function error(message, status = TypedHTTP2.StatusCodes.ServerErrors.INTERNAL_SERVER_ERROR) {
-    return { status, message };
-  }
-  TypedHTTP2.error = error;
-  class Router {
-    constructor(_resolveContext) {
-      this._resolveContext = _resolveContext;
-    }
-    static create() {
-      return new Router(async (e) => e.ctx);
-    }
-    rawMethod(method, route) {
-      return {
-        [method]: {
-          method,
-          input: route.input,
-          etag: route.etag,
-          resolveContext: this._resolveContext,
-          handler: route.handler
-        }
-      };
-    }
-    get(route) {
-      return this.rawMethod("GET", {
-        ...route,
-        handler: (...args) => Promise.resolve(route.handler(...args)).then((result) => TypedHTTP2.ok(result))
-      });
-    }
-    post(route) {
-      return this.rawMethod("POST", {
-        ...route,
-        handler: (...args) => Promise.resolve(route.handler(...args)).then((result) => TypedHTTP2.ok(result))
-      });
-    }
-    use(resolveContext) {
-      return new Router(async (options) => {
-        const m = await this._resolveContext(options);
-        return resolveContext({ ...options, ctx: m });
-      });
-    }
-  }
-  TypedHTTP2.Router = Router;
-  function createClient(base, fetchCallback) {
-    function buildUrl(path2, input, options) {
-      const method = path2.at(-1);
-      const url = new URL(path2.slice(0, path2.length - 1).join("/"), base);
-      const signal = options?.signal;
-      let body = void 0;
-      if (method === "GET" && input)
-        url.searchParams.set("input", JSON.stringify(input));
-      else if (method !== "GET" && input)
-        body = JSON.stringify(input);
-      return {
-        url,
-        method,
-        headers: body ? { "Content-Type": "application/json" } : void 0,
-        body,
-        signal
-      };
-    }
-    function createProxy(path2 = []) {
-      return new Proxy(() => {
-      }, {
-        get(target, prop) {
-          if (typeof prop === "symbol")
-            return void 0;
-          if (prop === "prepare")
-            return (input, options) => buildUrl(path2, input, options);
-          const newPath = [...path2, prop];
-          return createProxy(newPath);
-        },
-        apply(target, thisArg, args) {
-          const options = buildUrl(path2, args[0], args[1]);
-          return fetchCallback(options.url, {
-            method: options.method,
-            body: options.body,
-            headers: options.headers,
-            signal: options.signal
-          }).then(async (response) => {
-            if (response.status >= 200 && response.status < 300) {
-              if (response.headers.get("content-type")?.includes("application/json")) {
-                const text = await response.text();
-                return text.length ? JSON.parse(text) : void 0;
-              }
-              return await response.blob();
-            }
-            if (response.status >= 400 && response.status < 600) {
-              const text = await response.text();
-              if (text)
-                throw new Error(`HTTP request failed with status ${response.status}: ${text}`);
-              else
-                throw new Error(`HTTP request failed with status ${response.status}`);
-            }
-          });
-        }
-      });
-    }
-    return createProxy();
-  }
-  TypedHTTP2.createClient = createClient;
-})(TypedHTTP || (TypedHTTP = {}));
+// src/serverapi.ts
+import { TypedHTTP } from "@flakiness/shared/common/typedHttp.js";
 
 // src/utils.ts
+import { FlakinessReport } from "@flakiness/report";
+import fs from "fs";
 import http from "http";
 import https from "https";
-import util from "util";
-import zlib from "zlib";
-var gzipAsync = util.promisify(zlib.gzip);
-var gunzipAsync = util.promisify(zlib.gunzip);
-var gunzipSync = zlib.gunzipSync;
-var brotliCompressAsync = util.promisify(zlib.brotliCompress);
-var brotliCompressSync = zlib.brotliCompressSync;
+import path, { posix as posixPath, win32 as win32Path } from "path";
+var FLAKINESS_DBG = !!process.env.FLAKINESS_DBG;
+function errorText(error) {
+  return FLAKINESS_DBG ? error.stack : error.message;
+}
 async function retryWithBackoff(job, backoff = []) {
   for (const timeout of backoff) {
     try {
       return await job();
     } catch (e) {
       if (e instanceof AggregateError)
-        console.error(`[flakiness.io err]`, e.errors[0].message);
+        console.error(`[flakiness.io err]`, errorText(e.errors[0]));
       else if (e instanceof Error)
-        console.error(`[flakiness.io err]`, e.message);
+        console.error(`[flakiness.io err]`, errorText(e));
       else
         console.error(`[flakiness.io err]`, e);
       await new Promise((x) => setTimeout(x, timeout));
@@ -268,6 +49,7 @@ var httpUtils;
       reject = b;
     });
     const protocol = url.startsWith("https") ? https : http;
+    headers = Object.fromEntries(Object.entries(headers).filter(([key, value]) => value !== void 0));
     const request = protocol.request(url, { method, headers }, (res) => {
       const chunks = [];
       res.on("data", (chunk) => chunks.push(chunk));
@@ -320,6 +102,40 @@ var httpUtils;
   httpUtils2.postJSON = postJSON;
 })(httpUtils || (httpUtils = {}));
 var ansiRegex = new RegExp("[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))", "g");
+async function resolveAttachmentPaths(report, attachmentsDir) {
+  const attachmentFiles = await listFilesRecursively(attachmentsDir);
+  const filenameToPath = new Map(attachmentFiles.map((file) => [path.basename(file), file]));
+  const attachmentIdToPath = /* @__PURE__ */ new Map();
+  const missingAttachments = /* @__PURE__ */ new Set();
+  FlakinessReport.visitTests(report, (test) => {
+    for (const attempt of test.attempts) {
+      for (const attachment of attempt.attachments ?? []) {
+        const attachmentPath = filenameToPath.get(attachment.id);
+        if (!attachmentPath) {
+          missingAttachments.add(attachment.id);
+        } else {
+          attachmentIdToPath.set(attachment.id, {
+            contentType: attachment.contentType,
+            id: attachment.id,
+            path: attachmentPath
+          });
+        }
+      }
+    }
+  });
+  return { attachmentIdToPath, missingAttachments: Array.from(missingAttachments) };
+}
+async function listFilesRecursively(dir, result = []) {
+  const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory())
+      await listFilesRecursively(fullPath, result);
+    else
+      result.push(fullPath);
+  }
+  return result;
+}
 var IS_WIN32_PATH = new RegExp("^[a-zA-Z]:\\\\", "i");
 var IS_ALMOST_POSIX_PATH = new RegExp("^[a-zA-Z]:/", "i");
 
@@ -383,11 +199,10 @@ var ReportUpload = class {
     this._options = options;
     this._report = report;
     this._attachments = attachments;
-    this._api = createServerAPI(this._options.flakinessEndpoint, { retries: HTTP_BACKOFF });
+    this._api = createServerAPI(this._options.flakinessEndpoint, { retries: HTTP_BACKOFF, auth: this._options.flakinessAccessToken });
   }
   async upload(options) {
     const response = await this._api.run.startUpload.POST({
-      flakinessAccessToken: this._options.flakinessAccessToken,
       attachmentIds: this._attachments.map((attachment) => attachment.id)
     }).then((result) => ({ result, error: void 0 })).catch((e) => ({ result: void 0, error: e }));
     if (response?.error || !response.result)
@@ -398,17 +213,17 @@ var ReportUpload = class {
         const uploadURL = response.result.attachment_upload_urls[attachment.id];
         if (!uploadURL)
           throw new Error("Internal error: missing upload URL for attachment!");
-        return this._uploadAttachment(attachment, uploadURL);
+        return this._uploadAttachment(attachment, uploadURL, options?.syncCompression ?? false);
       })
     ]);
     const response2 = await this._api.run.completeUpload.POST({
       upload_token: response.result.upload_token
     }).then((result) => ({ result, error: void 0 })).catch((e) => ({ error: e, result: void 0 }));
-    const url = response2?.result?.report_url ? new URL2(response2?.result.report_url, this._options.flakinessEndpoint).toString() : void 0;
+    const url = response2?.result?.report_url ? new URL(response2?.result.report_url, this._options.flakinessEndpoint).toString() : void 0;
     return { success: true, reportUrl: url };
   }
   async _uploadReport(data, uploadUrl, syncCompression) {
-    const compressed = syncCompression ? brotliCompressSync2(data) : await brotliCompressAsync(data);
+    const compressed = syncCompression ? compressTextSync(data) : await compressTextAsync(data);
     const headers = {
       "Content-Type": "application/json",
       "Content-Length": Buffer.byteLength(compressed) + "",
@@ -425,11 +240,34 @@ var ReportUpload = class {
       await responseDataPromise;
     }, HTTP_BACKOFF);
   }
-  async _uploadAttachment(attachment, uploadUrl) {
-    const bytesLength = attachment.path ? (await fs.promises.stat(attachment.path)).size : attachment.body ? Buffer.byteLength(attachment.body) : 0;
+  async _uploadAttachment(attachment, uploadUrl, syncCompression) {
+    const mimeType = attachment.contentType.toLocaleLowerCase().trim();
+    const compressable = mimeType.startsWith("text/") || mimeType.endsWith("+json") || mimeType.endsWith("+text") || mimeType.endsWith("+xml");
+    if (!compressable && attachment.path) {
+      const attachmentPath = attachment.path;
+      await retryWithBackoff(async () => {
+        const { request, responseDataPromise } = httpUtils.createRequest({
+          url: uploadUrl,
+          headers: {
+            "Content-Type": attachment.contentType,
+            "Content-Length": (await fs2.promises.stat(attachmentPath)).size + ""
+          },
+          method: "put"
+        });
+        fs2.createReadStream(attachmentPath).pipe(request);
+        await responseDataPromise;
+      }, HTTP_BACKOFF);
+      return;
+    }
+    let buffer = attachment.body ? attachment.body : attachment.path ? await fs2.promises.readFile(attachment.path) : void 0;
+    assert(buffer);
+    const encoding = compressable ? "br" : void 0;
+    if (compressable)
+      buffer = syncCompression ? compressTextSync(buffer) : await compressTextAsync(buffer);
     const headers = {
       "Content-Type": attachment.contentType,
-      "Content-Length": bytesLength + ""
+      "Content-Length": Buffer.byteLength(buffer) + "",
+      "Content-Encoding": encoding
     };
     await retryWithBackoff(async () => {
       const { request, responseDataPromise } = httpUtils.createRequest({
@@ -437,13 +275,8 @@ var ReportUpload = class {
         headers,
         method: "put"
       });
-      if (attachment.path) {
-        fs.createReadStream(attachment.path).pipe(request);
-      } else {
-        if (attachment.body)
-          request.write(attachment.body);
-        request.end();
-      }
+      request.write(buffer);
+      request.end();
       await responseDataPromise;
     }, HTTP_BACKOFF);
   }
@@ -451,34 +284,15 @@ var ReportUpload = class {
 
 // src/cli/cmd-upload.ts
 async function cmdUpload(relativePath, options) {
-  const fullPath = path.resolve(relativePath);
-  if (!await fs2.access(fullPath, fs2.constants.F_OK).then(() => true).catch(() => false)) {
+  const fullPath = path2.resolve(relativePath);
+  if (!await fs3.access(fullPath, fs3.constants.F_OK).then(() => true).catch(() => false)) {
     console.error(`Error: path ${fullPath} is not accessible`);
     process.exit(1);
   }
-  const attachmentsDir = options.attachmentsDir ?? path.dirname(fullPath);
-  const attachmentFiles = await listFilesRecursively(attachmentsDir);
-  const attachmentIdToPath = new Map(attachmentFiles.map((file) => [path.basename(file), file]));
-  const text = await fs2.readFile(fullPath, "utf-8");
+  const text = await fs3.readFile(fullPath, "utf-8");
   const report = JSON.parse(text);
-  const attachments = [];
-  const missingAttachments = [];
-  FlakinessReport.visitTests(report, (test) => {
-    for (const attempt of test.attempts) {
-      for (const attachment of attempt.attachments ?? []) {
-        const file = attachmentIdToPath.get(attachment.id);
-        if (!file) {
-          missingAttachments.push(attachment);
-          continue;
-        }
-        attachments.push({
-          contentType: attachment.contentType,
-          id: attachment.id,
-          path: file
-        });
-      }
-    }
-  });
+  const attachmentsDir = options.attachmentsDir ?? path2.dirname(fullPath);
+  const { attachmentIdToPath, missingAttachments } = await resolveAttachmentPaths(report, attachmentsDir);
   if (missingAttachments.length && !options.ignoreMissingAttachments) {
     console.log(`Missing ${missingAttachments.length} attachments - exiting. Use --ignore-missing-attachments to force upload.`);
     process.exit(1);
@@ -487,7 +301,7 @@ async function cmdUpload(relativePath, options) {
     flakinessAccessToken: options.accessToken,
     flakinessEndpoint: options.endpoint
   });
-  const upload = uploader.createUpload(report, attachments);
+  const upload = uploader.createUpload(report, Array.from(attachmentIdToPath.values()));
   const uploadResult = await upload.upload();
   if (!uploadResult.success) {
     console.log(`[flakiness.io] X Failed to upload to ${options.endpoint}: ${uploadResult.message}`);
@@ -495,17 +309,6 @@ async function cmdUpload(relativePath, options) {
     console.log(`[flakiness.io] \u2713 Report uploaded ${uploadResult.reportUrl ?? uploadResult.message ?? ""}`);
   }
 }
-async function listFilesRecursively(dir, result = []) {
-  const entries = await fs2.readdir(dir, { withFileTypes: true });
-  for (const entry of entries) {
-    const fullPath = path.join(dir, entry.name);
-    if (entry.isDirectory())
-      await listFilesRecursively(fullPath, result);
-    else
-      result.push(fullPath);
-  }
-  return result;
-}
 export {
   cmdUpload
 };