@intuned/runtime-dev 1.0.4-stuck.1 → 1.0.6-cli-auth.0.0.1

package/WebTemplate/jobs.ts

@@ -1,356 +0,0 @@
-import { runApi, RunBody } from "./controllers/runApi";
-import {
-  ATTEMPT_NUMBER_HEADER,
-  JOB_ID_HEADER,
-  JOB_RUN_ID_HEADER,
-  RUN_ID_HEADER,
-  SHOULD_SHUTDOWN_HEADER,
-} from "./headers";
-import {
-  getErrorResponse,
-  isHeadless,
-  isJobRunMachine,
-  ProxyConfig,
-  proxyToUrl,
-} from "./utils";
-import {
-  callBackendFunctionWithToken,
-  backendFunctionsTokenManager,
-} from "@intuned/runtime/dist/common/jwtTokenManager";
-import retry from "async-retry";
-import { runWithContext } from "@intuned/runtime";
-import { setTimeout } from "timers/promises";
-import { deleteTrace, uploadTrace } from "./controllers/traces";
-
-type JobPayload = {
-  headers: Record<string, string>;
-  originalPayload: any;
-  payload: RunBody;
-  startTime: number;
-  functionName: string;
-  receiptHandle: string;
-  traceSignedUrl?: string;
-};
-
-if (isJobRunMachine()) {
-  console.log("Running in job v3 mode");
-  void jobsV3();
-}
-
-async function runApiInContext(
-  jobPayload: JobPayload
-): Promise<Awaited<ReturnType<typeof runApi>>> {
-  const runId = jobPayload.headers[RUN_ID_HEADER] as string;
-  const jobId = process.env.JOB_ID as string;
-  const jobRunId = process.env.JOB_RUN_ID as string;
-  const proxy = jobPayload.payload.proxy
-    ? (proxyToUrl(jobPayload.payload.proxy as ProxyConfig) as string)
-    : undefined;
-  const contextData = {
-    runId: runId ?? "",
-    jobId,
-    jobRunId,
-    queueId: undefined,
-    proxy,
-    timeoutInfo: {
-      extendTimeoutCallback: async () => {
-        try {
-          await extendTimeout(jobPayload);
-        } catch (error: any) {
-          console.error("Error extending timeout", error);
-        }
-      },
-    },
-    ...(jobPayload.payload.executionContext ?? {}),
-  };
-
-  const backendFunctionsToken = jobPayload.payload.functionsToken;
-  if (backendFunctionsToken) {
-    backendFunctionsTokenManager.token = backendFunctionsToken;
-  }
-
-  try {
-    return await runWithContext(contextData, runApi, {
-      functionName: jobPayload.functionName,
-      ...jobPayload.payload,
-      runId: jobPayload.headers[RUN_ID_HEADER],
-      attemptNumber: jobPayload.headers[ATTEMPT_NUMBER_HEADER],
-      shouldSaveTrace: true,
-      headless: isHeadless(),
-    });
-  } catch (error: any) {
-    return getErrorResponse(error);
-  }
-}
-
-async function jobsV3() {
-  await reportReady();
-  const initialDelay = 1000;
-  let delay = initialDelay;
-  while (true as boolean) {
-    await setTimeout(delay);
-    try {
-      const { payload, shouldShutdown } = await requestPayload();
-      if (!payload) {
-        if (shouldShutdown) {
-          console.log("Received shutdown signal from job");
-          process.exit(0);
-        }
-        delay = Math.min(delay * 2, 60000);
-        continue;
-      }
-      delay = initialDelay;
-      console.log("Running payload", payload.functionName);
-      const { body, status } = await runApiInContext(payload);
-
-      await reportResultsAndUploadTraces(payload, status, body);
-
-      if (shouldShutdown) {
-        process.exit(0);
-      }
-    } catch (error: any) {
-      console.error("Error in payload", error);
-    }
-  }
-}
-
-async function reportReady() {
-  console.log("Ready");
-}
-
-async function requestPayload(): Promise<{
-  payload: JobPayload | null;
-  shouldShutdown: boolean;
-}> {
-  console.log("Requesting payload");
-  const result = await callBackendFunctionWithToken("jobs/requestPayload", {
-    headers: {
-      "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
-      [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
-      [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
-    },
-  });
-  const shouldShutdown = result.headers.get(SHOULD_SHUTDOWN_HEADER) === "true";
-  if (result.status === 200) {
-    const payload = (await result.json()) as JobPayload;
-    console.log("Received payload", payload);
-    return {
-      payload: {
-        ...payload,
-        headers: {
-          ...payload.headers,
-          [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
-          [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
-        },
-      },
-      shouldShutdown,
-    };
-  }
-  if (result.status === 204) {
-    console.log("No payload available");
-    return {
-      payload: null,
-      shouldShutdown,
-    };
-  }
-  console.log("Failed to obtain payload", result.status, await result.text());
-  return {
-    payload: null,
-    shouldShutdown,
-  };
-}
-
-async function reportResultsAndUploadTraces(
-  payload: JobPayload,
-  statusCode: number,
-  body: any
-) {
-  await reportJobRunResult({
-    runId: payload.headers[RUN_ID_HEADER] as string,
-    result: {
-      statusCode,
-      body,
-      receiptHandle: payload.receiptHandle,
-    },
-    originalPayload: payload.originalPayload,
-    startTime: payload.startTime,
-    functionsToken: backendFunctionsTokenManager.token,
-  });
-  await uploadJobRunTrace(payload);
-}
-
-async function reportJobRunResult({
-  runId,
-  result,
-  originalPayload,
-  startTime,
-  functionsToken,
-}: {
-  runId: string;
-  result: {
-    statusCode: number;
-    body: any;
-    receiptHandle: string;
-  };
-  originalPayload: any;
-  startTime: number;
-  functionsToken?: string;
-}) {
-  console.log("Reporting payload result");
-  const reqResult = await retry(
-    async (bail) => {
-      const response = await callBackendFunctionWithToken("jobs/reportResult", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
-          [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
-          [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
-        },
-        body: JSON.stringify({
-          runId,
-          result,
-          originalPayload,
-          startTime,
-        }),
-      });
-
-      if (!response.ok) {
-        if ([401, 403, 404, 413].includes(response.status)) {
-          bail(
-            new Error(
-              `Reporting result failed (non-retryable), status ${response.status}`
-            )
-          );
-        }
-        throw new Error(`Reporting result failed, status ${response.status}`);
-      }
-
-      return response;
-    },
-    {
-      retries: 5,
-      factor: 2,
-      maxTimeout: 1000 * 60, // 1 minute
-      minTimeout: 1000 * 5, // 5 seconds
-    }
-  );
-  console.log("Reported payload result", reqResult.ok, await reqResult.text());
-}
-
-async function uploadJobRunTrace(
-  payload: NonNullable<
-    NonNullable<Awaited<ReturnType<typeof requestPayload>>>["payload"]
-  >
-) {
-  if (!payload.traceSignedUrl) {
-    return;
-  }
-  const runId = payload.headers[RUN_ID_HEADER] as string;
-  const attemptNumber = payload.headers[ATTEMPT_NUMBER_HEADER] as
-    | string
-    | undefined;
-  console.log("Uploading trace", runId, attemptNumber);
-  const uploadTraceResult = await uploadTrace({
-    runId,
-    attemptNumber,
-    signedUrl: payload.traceSignedUrl,
-  });
-  if (uploadTraceResult.isErr()) {
-    console.error("Error uploading trace", uploadTraceResult.error, {
-      runId,
-      attemptNumber,
-    });
-  } else {
-    console.log(
-      "Trace uploaded successfully, reporting trace",
-      runId,
-      attemptNumber
-    );
-    await retry(
-      async (bail) => {
-        const response = await callBackendFunctionWithToken(
-          "jobs/reportTrace",
-          {
-            method: "POST",
-            headers: {
-              "Content-Type": "application/json",
-              "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
-              [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
-              [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
-            },
-            body: JSON.stringify({
-              runId,
-              attemptNumber,
-              traceSignedUrl: payload.traceSignedUrl,
-            }),
-          }
-        );
-
-        if (!response.ok) {
-          if ([401, 403, 404].includes(response.status)) {
-            bail(
-              new Error(
-                `Reporting trace failed (non-retryable), status ${response.status}`
-              )
-            );
-          }
-          throw new Error(`Reporting trace failed, status ${response.status}`);
-        }
-
-        console.log("Trace reported successfully", runId, attemptNumber);
-        return response;
-      },
-      {
-        retries: 2,
-        minTimeout: 1000 * 5,
-      }
-    ).catch(() => undefined);
-  }
-  console.log("Deleting trace", runId, attemptNumber);
-  const deleteTraceResult = await deleteTrace({
-    runId,
-    attemptNumber,
-  });
-  if (deleteTraceResult.isErr()) {
-    console.error("Error deleting trace", deleteTraceResult.error, {
-      runId,
-      attemptNumber,
-    });
-  }
-}
-
-async function extendTimeout(payload: JobPayload) {
-  console.log("Requesting timeout extension for", payload.functionName);
-  const extendTimeoutRes = await callBackendFunctionWithToken(
-    "jobs/extendTimeout",
-    {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
-        [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
-        [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
-      },
-      body: JSON.stringify({
-        receiptHandle: payload.receiptHandle,
-      }),
-    }
-  );
-  if (!extendTimeoutRes.ok) {
-    console.error(
-      "Requesting timeout extension failed",
-      extendTimeoutRes.status,
-      await extendTimeoutRes.text()
-    );
-    return false;
-  }
-  console.log("Request timeout extension success", extendTimeoutRes.ok);
-  const response = await extendTimeoutRes.json();
-  const newFunctionsToken = response.functionsToken;
-  if (newFunctionsToken) {
-    backendFunctionsTokenManager.token = newFunctionsToken;
-    console.log("Backend functions token renewed");
-  }
-  return true;
-}

package/WebTemplate/shutdown.ts

@@ -1,64 +0,0 @@
-import { isJobRunMachine } from "./utils";
-import { promisify } from "util";
-import { AsyncRunEndpointController } from "./utils";
-import { Server } from "http";
-
-export class ShutdownController {
-  private static _instance?: ShutdownController;
-
-  static initialize(server: Server) {
-    if (this._instance) {
-      throw new Error("ShutdownController is already initialized");
-    }
-    this._instance = new ShutdownController(server);
-  }
-
-  public static get instance() {
-    if (!this._instance) {
-      throw new Error("ShutdownController is not initialized");
-    }
-    return this._instance;
-  }
-
-  private server: Server;
-  private timer: NodeJS.Timeout | null = null;
-  private constructor(server: Server) {
-    this.server = server;
-  }
-
-  async checkForShutdown() {
-    if (await this.serverShouldShutdown()) {
-      if (this.timer) clearTimeout(this.timer);
-      this.timer = setTimeout(async () => {
-        if (await this.serverShouldShutdown()) {
-          console.log("No active requests, shutting down the server.");
-          process.exit(0);
-        }
-        // TIME_TO_SHUTDOWN in seconds to match tired-proxy
-      }, this.calculateShutdownDelay());
-    }
-  }
-
-  private async getActiveRequests() {
-    const activeConnections = await promisify(
-      (...params: Parameters<typeof this.server.getConnections>) =>
-        this.server.getConnections(...params)
-    )();
-    return activeConnections + AsyncRunEndpointController.activeRequestsCount;
-  }
-
-  private async serverShouldShutdown() {
-    if (isJobRunMachine()) {
-      return false;
-    }
-    return (await this.getActiveRequests()) === 0;
-  }
-
-  private calculateShutdownDelay(): number | undefined {
-    const shutdownTime = +process.env.TIME_TO_SHUTDOWN;
-    if (isNaN(shutdownTime) || shutdownTime <= 0) {
-      return 60 * 1000;
-    }
-    return shutdownTime * 1000;
-  }
-}

package/WebTemplate/utils.ts

@@ -1,257 +0,0 @@
-import * as playwright from "@intuned/playwright-core";
-import { Handler, Response } from "@tinyhttp/app";
-import * as path from "path";
-import { getExecutionContext } from "@intuned/runtime";
-import { setTimeout } from "timers/promises";
-
-export class FunctionNotFoundError extends Error {
-  functionName: string;
-  path: string;
-  constructor(functionName: string, path: string) {
-    const message = `function ${functionName} not found in ${path}`;
-    super(message);
-    this.functionName = functionName;
-    this.path = path;
-    this.name = "FunctionNotFound";
-    Object.setPrototypeOf(this, FunctionNotFoundError.prototype);
-  }
-}
-
-export function getIsRetryableError(error: any) {
-  if (error?.message) {
-    return error.message.includes("ERR_NETWORK_CHANGED");
-  }
-  return false;
-}
-
-export function getErrorResponse(error: any): {
-  status: number;
-  body: any;
-} {
-  if (error instanceof FunctionNotFoundError) {
-    return {
-      status: 404,
-      body: {
-        message: error.message,
-        error: error.name,
-      },
-    };
-  }
-  if (error instanceof playwright.errors.TimeoutError) {
-    return {
-      status: 500,
-      body: { message: error.message, error: error.name },
-    };
-  }
-  /**
-   * here we use error.constructor.name instead of error instanceof RunError
-   * this is because the error is thrown by importing the runner from the api code on intuned app
-   * the definition of class RunError which is imported from here is different than the class RunError
-   * imported from the user.
-   */
-  if (error.constructor.name === "RunError") {
-    return {
-      status: 200,
-      body: {
-        status: error.options.status_code ?? 500,
-        message: error.message,
-        error: error.options.error_code ?? error.name,
-        intunedOptions: error.options,
-      },
-    };
-  }
-  return {
-    status: 500,
-    body: { error: error?.name ?? error, message: error?.message },
-  };
-}
-
-export function handlePlaywrightExecutionError(error: any, res: Response) {
-  const { status, body } = getErrorResponse(error);
-  return res.status(status).json(body);
-}
-
-type EventTraceEvent = {
-  type: "event";
-  time: number;
-  class: string;
-  method: string;
-  params: any;
-  pageId?: string;
-};
-
-type AppHandlerParams = Parameters<Handler>;
-
-export function errorRetryMiddleware(handler: Handler) {
-  return async (...args: AppHandlerParams) => {
-    let attempts = 1;
-    const [req, res, next] = args;
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      try {
-        await handler(req, res, next);
-        break;
-      } catch (error) {
-        console.log(error?.name, error?.message);
-        if (!getIsRetryableError(error) || attempts >= 3) {
-          return handlePlaywrightExecutionError(error, res);
-        }
-        attempts++;
-      }
-    }
-  };
-}
-
-export function getTraceFilePath(runId: string, attemptNumber?: string) {
-  const fileName = `${runId}${attemptNumber ? `_${attemptNumber}` : ""}`;
-  return path.join(process.env.TRACES_DIRECTORY ?? "", `${fileName}.zip`);
-}
-
-export interface ProxyConfig {
-  username: string;
-  server: string;
-  password: string;
-}
-
-export function proxyToUrl(proxy: ProxyConfig) {
-  const url = new URL(proxy.server);
-  url.username = proxy.username;
-  url.password = proxy.password;
-  return url.toString();
-}
-
-export function isJobRunMachine() {
-  return process.env.JOB_ID && process.env.JOB_RUN_ID;
-}
-
-export function isHeadless() {
-  return process.env.INTUNED_PLAYWRIGHT_HEADLESS !== "0";
-}
-
-export abstract class AsyncRunEndpointController {
-  private static activeRequests: Set<string> = new Set();
-
-  static isRunning(taskToken: string) {
-    return this.activeRequests.has(taskToken);
-  }
-
-  static addRequest(taskToken: string) {
-    this.activeRequests.add(taskToken);
-  }
-
-  static removeRequest(taskToken: string) {
-    this.activeRequests.delete(taskToken);
-  }
-
-  static get activeRequestsCount() {
-    return this.activeRequests.size;
-  }
-}
-
-export async function waitWithExtendableTimeout<T>({
-  initialTimeout,
-  promise,
-  abortController,
-}: {
-  initialTimeout: number;
-  promise: Promise<T>;
-  abortController?: AbortController;
-}): Promise<
-  | {
-      timedOut: true;
-    }
-  | {
-      timedOut: false;
-      result: T;
-    }
-> {
-  const context = getExecutionContext()!;
-  if (context.timeoutInfo) {
-    context.timeoutInfo.timeoutTimestamp = Date.now() + initialTimeout;
-  }
-  const timerSymbol = Symbol("timer");
-  if (!context) {
-    const result = await Promise.race([
-      promise,
-      setTimeout(initialTimeout, timerSymbol),
-    ]);
-    if (result === timerSymbol) {
-      abortController?.abort("Timed out");
-      return { timedOut: true };
-    }
-    return { timedOut: false, result };
-  }
-
-  let taskTimeout = initialTimeout;
-  while (true as boolean) {
-    const result = await Promise.race([
-      promise,
-      setTimeout(taskTimeout, timerSymbol),
-    ]);
-
-    if (result !== timerSymbol) {
-      break;
-    }
-
-    const remainingTime =
-      (context.timeoutInfo?.timeoutTimestamp ?? 0) - Date.now();
-    if (remainingTime < 0) {
-      abortController?.abort("Timed out");
-      return { timedOut: true };
-    }
-
-    taskTimeout = remainingTime;
-  }
-
-  return {
-    timedOut: false,
-    result: await promise,
-  };
-}
-
-export async function importModule(path: string) {
-  const [folderName, ...functionNameParts] = path.split("/");
-  const functionNameDepth = functionNameParts.length;
-
-  // string literals should be inline
-  // currently we support only 5 levels of depth
-  // rollup dynamic import does not support multiple levels of dynamic imports so we need to specify the possible paths explicitly
-  try {
-    let imported: any = undefined;
-    switch (functionNameDepth) {
-      case 1:
-        imported = await import(`./${folderName}/${functionNameParts[0]}.ts`);
-        break;
-      case 2:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}.ts`
-        );
-        break;
-      case 3:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}.ts`
-        );
-        break;
-      case 4:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}/${functionNameParts[3]}.ts`
-        );
-        break;
-      case 5:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}/${functionNameParts[3]}/${functionNameParts[4]}.ts`
-        );
-        break;
-      default:
-        throw new Error(
-          "intuned supports maximum 5 levels of depth in the api folder"
-        );
-    }
-    return imported;
-  } catch (error: any) {
-    if (error.message.includes("Unknown variable dynamic import")) {
-      throw new FunctionNotFoundError("", path);
-    }
-    throw error;
-  }
-}