@intuned/runtime-dev 0.0.1-split.0

package/WebTemplate/controllers/traces.ts

@@ -0,0 +1,151 @@
+import { getTraceFilePath } from "../utils";
+import { pathExists, remove, createReadStream, stat } from "fs-extra";
+import fetch from "node-fetch";
+import { Handler } from "@tinyhttp/app";
+import { ok, err } from "neverthrow";
+
+export async function uploadTrace({
+  runId,
+  attemptNumber,
+  signedUrl,
+}: {
+  runId: string;
+  attemptNumber?: string;
+  signedUrl: string;
+}) {
+  const traceFilePath = getTraceFilePath(
+    runId,
+    attemptNumber as string | undefined
+  );
+
+  if (!(await pathExists(traceFilePath))) {
+    return err({
+      message: "Trace file not found",
+      error: "Not found",
+      errorCode: "trace_file_not_found" as const,
+    });
+  }
+
+  const fileSize = (await stat(traceFilePath)).size;
+  const file = createReadStream(traceFilePath);
+
+  const response = await fetch(signedUrl, {
+    method: "PUT",
+    headers: {
+      "Content-Length": fileSize.toString(),
+    },
+    body: file,
+  });
+
+  if (!response.ok) {
+    return err({
+      message: `Upload trace failed with status code ${response.status}`,
+      error: "Upload trace failed",
+      errorCode: "upload_trace_failed" as const,
+    });
+  }
+
+  return ok({
+    traceName: `${runId}.zip`,
+  });
+}
+
+export async function deleteTrace({
+  runId,
+  attemptNumber,
+}: {
+  runId: string;
+  attemptNumber?: string;
+}) {
+  const traceFilePath = getTraceFilePath(
+    runId,
+    attemptNumber as string | undefined
+  );
+
+  if (!(await pathExists(traceFilePath))) {
+    return err({
+      message: "Trace file not found",
+      error: "Not found",
+      errorCode: "trace_file_not_found" as const,
+    });
+  }
+
+  try {
+    await remove(traceFilePath);
+    return ok(undefined);
+  } catch (error: any) {
+    return err({
+      message: `Delete trace file failed ${runId}`,
+      error: "Delete trace failed",
+      errorCode: "delete_trace_failed" as const,
+    });
+  }
+}
+
+export const uploadTraceController: Handler = async (req, res) => {
+  const { runId } = req.params;
+  const { attemptNumber } = req.query;
+
+  const signedUrl = req.body.signedUrl as string;
+  if (!runId) {
+    return res.status(400).json({
+      error: "runId not provided",
+      message: "Please add provide run id",
+    });
+  }
+
+  if (!signedUrl) {
+    return res.status(400).json({
+      error: "signedUrl not provided",
+      message: "Please add provide signedUrl",
+    });
+  }
+
+  const uploadTraceResult = await uploadTrace({
+    runId,
+    attemptNumber: attemptNumber as string | undefined,
+    signedUrl,
+  });
+
+  if (uploadTraceResult.isOk()) {
+    return res.status(200).json({
+      result: uploadTraceResult.value,
+    });
+  }
+
+  const error = uploadTraceResult.error;
+  if (error.errorCode === "trace_file_not_found") {
+    return res.status(404).json(error);
+  }
+
+  return res.status(500).json(error);
+};
+
+export const deleteTraceController: Handler = async (req, res) => {
+  const { runId } = req.params;
+  const { attemptNumber } = req.query;
+
+  if (!runId) {
+    return res.status(400).json({
+      error: "runId not provided",
+      message: "Please add provide run id",
+    });
+  }
+
+  const deleteTraceResult = await deleteTrace({
+    runId,
+    attemptNumber: attemptNumber as string | undefined,
+  });
+
+  if (deleteTraceResult.isOk()) {
+    return res.status(204).send("");
+  }
+
+  const error = deleteTraceResult.error;
+
+  if (error.errorCode === "trace_file_not_found") {
+    return res.status(404).json(error);
+  }
+
+  return res.status(500).json(error);
+};

package/WebTemplate/features.ts

@@ -0,0 +1,8 @@
+// sync with apps/web/lib/Services/IntegrationService/constants.ts
+export const FEATURES = [
+  "JOBS-V3",
+  "RUN-ASYNC-MODE",
+  "AUTH-SESSION-ASYNC-MODE",
+] as const;
+
+export type Feature = (typeof FEATURES)[number];

package/WebTemplate/headers.ts

@@ -0,0 +1,6 @@
+export const RUN_ID_HEADER = "x-run-id";
+export const JOB_ID_HEADER = "x-job-id";
+export const JOB_RUN_ID_HEADER = "x-job-run-id";
+export const QUEUE_ID_HEADER = "x-queue-id";
+export const ATTEMPT_NUMBER_HEADER = "x-attempt-number";
+export const SHOULD_SHUTDOWN_HEADER = "x-should-shutdown";

package/WebTemplate/index.playwright.ts

@@ -0,0 +1,47 @@
+import {
+  initializeAppInsights,
+  getTelemetryClient,
+} from "@intuned/runtime/dist/common/telemetry";
+import { getExecutionContext } from "@intuned/runtime";
+import { app } from "./app";
+import { RUN_ID_HEADER } from "./headers";
+import { ShutdownController } from "./shutdown";
+
+const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
+
+initializeAppInsights();
+
+app.use((req, res, next) => {
+  // Cleanup after the response is sent
+  res.on("finish", () => {
+    console.log("finished", req.headers[RUN_ID_HEADER]);
+    void ShutdownController.instance.checkForShutdown();
+  });
+
+  next();
+});
+
+export * from "./api";
+export * from "./jobs";
+
+const server = app.listen(port, () => {
+  // when deployed on flyio, the server will be turned on and
+  // will shutdown after the specified time in TIME_TO_SHUTDOWN env variable
+  ShutdownController.initialize(server);
+  void ShutdownController.instance.checkForShutdown();
+  console.log(`Server is running on port ${port}`);
+});
+
+process.on("unhandledRejection", (reason, promise) => {
+  const telemetryClient = getTelemetryClient();
+  console.error("Unhandled Rejection at:", promise, "reason:", reason);
+  const context = getExecutionContext();
+  telemetryClient?.trackEvent({
+    name: "UNHANDLED_REJECTION",
+    properties: {
+      runId: context?.runId,
+      promise,
+      reason,
+    },
+  });
+});

package/WebTemplate/index.vanilla.ts

@@ -0,0 +1,44 @@
+import { App } from "@tinyhttp/app";
+import { json as parser } from "milliparsec";
+import { accessKeyValidatorMiddleware } from "./accessKeyHelpers";
+import { callFunction } from "./utils";
+
+const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
+const app = new App();
+
+app.use("/", parser());
+
+app.post("/api/run/*", async (req, res, next) => {
+  // filter(Boolean) should remove empty strings in case of a trailing slash
+  const functionName = req.path.split("/").filter(Boolean).pop();
+
+  try {
+    const body = req.body;
+    const result = await callFunction("api", functionName as string, [body]);
+
+    res.status(200).json(result);
+  } catch (error) {
+    if (
+      error.message ===
+      `Unknown variable dynamic import: ./api/${functionName}.ts`
+    ) {
+      return res.status(404).json({});
+    }
+
+    next(error);
+  }
+});
+
+app.get("/api/health", async (req, res) => {
+  res.status(200).json({ status: "ok" });
+});
+
+app.use(accessKeyValidatorMiddleware);
+
+app.get("/protected/health", async (req, res) => {
+  res.status(200).json({ status: "ok" });
+});
+
+app.listen(port, () => {
+  console.log(`Server is running on port ${port}`);
+});

package/WebTemplate/jobs.ts

@@ -0,0 +1,356 @@
+import { runApi, RunBody } from "./controllers/runApi";
+import {
+  ATTEMPT_NUMBER_HEADER,
+  JOB_ID_HEADER,
+  JOB_RUN_ID_HEADER,
+  RUN_ID_HEADER,
+  SHOULD_SHUTDOWN_HEADER,
+} from "./headers";
+import {
+  getErrorResponse,
+  isHeadless,
+  isJobRunMachine,
+  ProxyConfig,
+  proxyToUrl,
+} from "./utils";
+import {
+  callBackendFunctionWithToken,
+  backendFunctionsTokenManager,
+} from "@intuned/runtime/dist/common/jwtTokenManager";
+import retry from "async-retry";
+import { runWithContext } from "@intuned/runtime";
+import { setTimeout } from "timers/promises";
+import { deleteTrace, uploadTrace } from "./controllers/traces";
+
+type JobPayload = {
+  headers: Record<string, string>;
+  originalPayload: any;
+  payload: RunBody;
+  startTime: number;
+  functionName: string;
+  receiptHandle: string;
+  traceSignedUrl?: string;
+};
+
+if (isJobRunMachine()) {
+  console.log("Running in job v3 mode");
+  void jobsV3();
+}
+
+async function runApiInContext(
+  jobPayload: JobPayload
+): Promise<Awaited<ReturnType<typeof runApi>>> {
+  const runId = jobPayload.headers[RUN_ID_HEADER] as string;
+  const jobId = process.env.JOB_ID as string;
+  const jobRunId = process.env.JOB_RUN_ID as string;
+  const proxy = jobPayload.payload.proxy
+    ? (proxyToUrl(jobPayload.payload.proxy as ProxyConfig) as string)
+    : undefined;
+  const contextData = {
+    runId: runId ?? "",
+    jobId,
+    jobRunId,
+    queueId: undefined,
+    proxy,
+    timeoutInfo: {
+      extendTimeoutCallback: async () => {
+        try {
+          await extendTimeout(jobPayload);
+        } catch (error: any) {
+          console.error("Error extending timeout", error);
+        }
+      },
+    },
+    ...(jobPayload.payload.executionContext ?? {}),
+  };
+
+  const backendFunctionsToken = jobPayload.payload.functionsToken;
+  if (backendFunctionsToken) {
+    backendFunctionsTokenManager.token = backendFunctionsToken;
+  }
+
+  try {
+    return await runWithContext(contextData, runApi, {
+      functionName: jobPayload.functionName,
+      ...jobPayload.payload,
+      runId: jobPayload.headers[RUN_ID_HEADER],
+      attemptNumber: jobPayload.headers[ATTEMPT_NUMBER_HEADER],
+      shouldSaveTrace: true,
+      headless: isHeadless(),
+    });
+  } catch (error: any) {
+    return getErrorResponse(error);
+  }
+}
+
+async function jobsV3() {
+  await reportReady();
+  const initialDelay = 1000;
+  let delay = initialDelay;
+  while (true as boolean) {
+    await setTimeout(delay);
+    try {
+      const { payload, shouldShutdown } = await requestPayload();
+      if (!payload) {
+        if (shouldShutdown) {
+          console.log("Received shutdown signal from job");
+          process.exit(0);
+        }
+        delay = Math.min(delay * 2, 60000);
+        continue;
+      }
+      delay = initialDelay;
+      console.log("Running payload", payload.functionName);
+      const { body, status } = await runApiInContext(payload);
+
+      await reportResultsAndUploadTraces(payload, status, body);
+
+      if (shouldShutdown) {
+        process.exit(0);
+      }
+    } catch (error: any) {
+      console.error("Error in payload", error);
+    }
+  }
+}
+
+async function reportReady() {
+  console.log("Ready");
+}
+
+async function requestPayload(): Promise<{
+  payload: JobPayload | null;
+  shouldShutdown: boolean;
+}> {
+  console.log("Requesting payload");
+  const result = await callBackendFunctionWithToken("jobs/requestPayload", {
+    headers: {
+      "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+      [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+      [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+    },
+  });
+  const shouldShutdown = result.headers.get(SHOULD_SHUTDOWN_HEADER) === "true";
+  if (result.status === 200) {
+    const payload = (await result.json()) as JobPayload;
+    console.log("Received payload", payload);
+    return {
+      payload: {
+        ...payload,
+        headers: {
+          ...payload.headers,
+          [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+          [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+        },
+      },
+      shouldShutdown,
+    };
+  }
+  if (result.status === 204) {
+    console.log("No payload available");
+    return {
+      payload: null,
+      shouldShutdown,
+    };
+  }
+  console.log("Failed to obtain payload", result.status, await result.text());
+  return {
+    payload: null,
+    shouldShutdown,
+  };
+}
+
+async function reportResultsAndUploadTraces(
+  payload: JobPayload,
+  statusCode: number,
+  body: any
+) {
+  await reportJobRunResult({
+    runId: payload.headers[RUN_ID_HEADER] as string,
+    result: {
+      statusCode,
+      body,
+      receiptHandle: payload.receiptHandle,
+    },
+    originalPayload: payload.originalPayload,
+    startTime: payload.startTime,
+    functionsToken: backendFunctionsTokenManager.token,
+  });
+  await uploadJobRunTrace(payload);
+}
+
+async function reportJobRunResult({
+  runId,
+  result,
+  originalPayload,
+  startTime,
+  functionsToken,
+}: {
+  runId: string;
+  result: {
+    statusCode: number;
+    body: any;
+    receiptHandle: string;
+  };
+  originalPayload: any;
+  startTime: number;
+  functionsToken?: string;
+}) {
+  console.log("Reporting payload result");
+  const reqResult = await retry(
+    async (bail) => {
+      const response = await callBackendFunctionWithToken("jobs/reportResult", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+          [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+          [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+        },
+        body: JSON.stringify({
+          runId,
+          result,
+          originalPayload,
+          startTime,
+        }),
+      });
+
+      if (!response.ok) {
+        if ([401, 403, 404, 413].includes(response.status)) {
+          bail(
+            new Error(
+              `Reporting result failed (non-retryable), status ${response.status}`
+            )
+          );
+        }
+        throw new Error(`Reporting result failed, status ${response.status}`);
+      }
+
+      return response;
+    },
+    {
+      retries: 5,
+      factor: 2,
+      maxTimeout: 1000 * 60, // 1 minute
+      minTimeout: 1000 * 5, // 5 seconds
+    }
+  );
+  console.log("Reported payload result", reqResult.ok, await reqResult.text());
+}
+
+async function uploadJobRunTrace(
+  payload: NonNullable<
+    NonNullable<Awaited<ReturnType<typeof requestPayload>>>["payload"]
+  >
+) {
+  if (!payload.traceSignedUrl) {
+    return;
+  }
+  const runId = payload.headers[RUN_ID_HEADER] as string;
+  const attemptNumber = payload.headers[ATTEMPT_NUMBER_HEADER] as
+    | string
+    | undefined;
+  console.log("Uploading trace", runId, attemptNumber);
+  const uploadTraceResult = await uploadTrace({
+    runId,
+    attemptNumber,
+    signedUrl: payload.traceSignedUrl,
+  });
+  if (uploadTraceResult.isErr()) {
+    console.error("Error uploading trace", uploadTraceResult.error, {
+      runId,
+      attemptNumber,
+    });
+  } else {
+    console.log(
+      "Trace uploaded successfully, reporting trace",
+      runId,
+      attemptNumber
+    );
+    await retry(
+      async (bail) => {
+        const response = await callBackendFunctionWithToken(
+          "jobs/reportTrace",
+          {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+              [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+              [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+            },
+            body: JSON.stringify({
+              runId,
+              attemptNumber,
+              traceSignedUrl: payload.traceSignedUrl,
+            }),
+          }
+        );
+
+        if (!response.ok) {
+          if ([401, 403, 404].includes(response.status)) {
+            bail(
+              new Error(
+                `Reporting trace failed (non-retryable), status ${response.status}`
+              )
+            );
+          }
+          throw new Error(`Reporting trace failed, status ${response.status}`);
+        }
+
+        console.log("Trace reported successfully", runId, attemptNumber);
+        return response;
+      },
+      {
+        retries: 2,
+        minTimeout: 1000 * 5,
+      }
+    ).catch(() => undefined);
+  }
+  console.log("Deleting trace", runId, attemptNumber);
+  const deleteTraceResult = await deleteTrace({
+    runId,
+    attemptNumber,
+  });
+  if (deleteTraceResult.isErr()) {
+    console.error("Error deleting trace", deleteTraceResult.error, {
+      runId,
+      attemptNumber,
+    });
+  }
+}
+
+async function extendTimeout(payload: JobPayload) {
+  console.log("Requesting timeout extension for", payload.functionName);
+  const extendTimeoutRes = await callBackendFunctionWithToken(
+    "jobs/extendTimeout",
+    {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+        [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+        [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+      },
+      body: JSON.stringify({
+        receiptHandle: payload.receiptHandle,
+      }),
+    }
+  );
+  if (!extendTimeoutRes.ok) {
+    console.error(
+      "Requesting timeout extension failed",
+      extendTimeoutRes.status,
+      await extendTimeoutRes.text()
+    );
+    return false;
+  }
+  console.log("Request timeout extension success", extendTimeoutRes.ok);
+  const response = await extendTimeoutRes.json();
+  const newFunctionsToken = response.functionsToken;
+  if (newFunctionsToken) {
+    backendFunctionsTokenManager.token = newFunctionsToken;
+    console.log("Backend functions token renewed");
+  }
+  return true;
+}

package/WebTemplate/shutdown.ts

@@ -0,0 +1,64 @@
+import { isJobRunMachine } from "./utils";
+import { promisify } from "util";
+import { AsyncRunEndpointController } from "./utils";
+import { Server } from "http";
+
+export class ShutdownController {
+  private static _instance?: ShutdownController;
+
+  static initialize(server: Server) {
+    if (this._instance) {
+      throw new Error("ShutdownController is already initialized");
+    }
+    this._instance = new ShutdownController(server);
+  }
+
+  public static get instance() {
+    if (!this._instance) {
+      throw new Error("ShutdownController is not initialized");
+    }
+    return this._instance;
+  }
+
+  private server: Server;
+  private timer: NodeJS.Timeout | null = null;
+  private constructor(server: Server) {
+    this.server = server;
+  }
+
+  async checkForShutdown() {
+    if (await this.serverShouldShutdown()) {
+      if (this.timer) clearTimeout(this.timer);
+      this.timer = setTimeout(async () => {
+        if (await this.serverShouldShutdown()) {
+          console.log("No active requests, shutting down the server.");
+          process.exit(0);
+        }
+        // TIME_TO_SHUTDOWN in seconds to match tired-proxy
+      }, this.calculateShutdownDelay());
+    }
+  }
+
+  private async getActiveRequests() {
+    const activeConnections = await promisify(
+      (...params: Parameters<typeof this.server.getConnections>) =>
+        this.server.getConnections(...params)
+    )();
+    return activeConnections + AsyncRunEndpointController.activeRequestsCount;
+  }
+
+  private async serverShouldShutdown() {
+    if (isJobRunMachine()) {
+      return false;
+    }
+    return (await this.getActiveRequests()) === 0;
+  }
+
+  private calculateShutdownDelay(): number | undefined {
+    const shutdownTime = +process.env.TIME_TO_SHUTDOWN;
+    if (isNaN(shutdownTime) || shutdownTime <= 0) {
+      return 60 * 1000;
+    }
+    return shutdownTime * 1000;
+  }
+}