@intuned/runtime-dev 0.1.0-test.26 → 0.1.0-test.28

package/WebTemplate/controllers/traces.js

@@ -0,0 +1,108 @@
+import { getTraceFilePath } from "../utils";
+import { pathExists, remove, createReadStream, stat } from "fs-extra";
+import fetch from "node-fetch";
+import { ok, err } from "neverthrow";
+export async function uploadTrace({ runId, attemptNumber, signedUrl, }) {
+    const traceFilePath = getTraceFilePath(runId, attemptNumber);
+    if (!(await pathExists(traceFilePath))) {
+        return err({
+            message: "Trace file not found",
+            error: "Not found",
+            errorCode: "trace_file_not_found",
+        });
+    }
+    const fileSize = (await stat(traceFilePath)).size;
+    const file = createReadStream(traceFilePath);
+    const response = await fetch(signedUrl, {
+        method: "PUT",
+        headers: {
+            "Content-Length": fileSize.toString(),
+        },
+        body: file,
+    });
+    if (!response.ok) {
+        return err({
+            message: `Upload trace failed with status code ${response.status}`,
+            error: "Upload trace failed",
+            errorCode: "upload_trace_failed",
+        });
+    }
+    return ok({
+        traceName: `${runId}.zip`,
+    });
+}
+export async function deleteTrace({ runId, attemptNumber, }) {
+    const traceFilePath = getTraceFilePath(runId, attemptNumber);
+    if (!(await pathExists(traceFilePath))) {
+        return err({
+            message: "Trace file not found",
+            error: "Not found",
+            errorCode: "trace_file_not_found",
+        });
+    }
+    try {
+        await remove(traceFilePath);
+        return ok(undefined);
+    }
+    catch (error) {
+        return err({
+            message: `Delete trace file failed ${runId}`,
+            error: "Delete trace failed",
+            errorCode: "delete_trace_failed",
+        });
+    }
+}
+export const uploadTraceController = async (req, res) => {
+    const { runId } = req.params;
+    const { attemptNumber } = req.query;
+    const signedUrl = req.body.signedUrl;
+    if (!runId) {
+        return res.status(400).json({
+            error: "runId not provided",
+            message: "Please add provide run id",
+        });
+    }
+    if (!signedUrl) {
+        return res.status(400).json({
+            error: "signedUrl not provided",
+            message: "Please add provide signedUrl",
+        });
+    }
+    const uploadTraceResult = await uploadTrace({
+        runId,
+        attemptNumber: attemptNumber,
+        signedUrl,
+    });
+    if (uploadTraceResult.isOk()) {
+        return res.status(200).json({
+            result: uploadTraceResult.value,
+        });
+    }
+    const error = uploadTraceResult.error;
+    if (error.errorCode === "trace_file_not_found") {
+        return res.status(404).json(error);
+    }
+    return res.status(500).json(error);
+};
+export const deleteTraceController = async (req, res) => {
+    const { runId } = req.params;
+    const { attemptNumber } = req.query;
+    if (!runId) {
+        return res.status(400).json({
+            error: "runId not provided",
+            message: "Please add provide run id",
+        });
+    }
+    const deleteTraceResult = await deleteTrace({
+        runId,
+        attemptNumber: attemptNumber,
+    });
+    if (deleteTraceResult.isOk()) {
+        return res.status(204).send("");
+    }
+    const error = deleteTraceResult.error;
+    if (error.errorCode === "trace_file_not_found") {
+        return res.status(404).json(error);
+    }
+    return res.status(500).json(error);
+};

package/WebTemplate/features.js

@@ -0,0 +1,6 @@
+// sync with apps/web/lib/Services/IntegrationService/constants.ts
+export const FEATURES = [
+    "JOBS-V3",
+    "RUN-ASYNC-MODE",
+    "AUTH-SESSION-ASYNC-MODE",
+];

package/WebTemplate/headers.js

@@ -0,0 +1,6 @@
+export const RUN_ID_HEADER = "x-run-id";
+export const JOB_ID_HEADER = "x-job-id";
+export const JOB_RUN_ID_HEADER = "x-job-run-id";
+export const QUEUE_ID_HEADER = "x-queue-id";
+export const ATTEMPT_NUMBER_HEADER = "x-attempt-number";
+export const SHOULD_SHUTDOWN_HEADER = "x-should-shutdown";

package/WebTemplate/index.playwright.js

@@ -0,0 +1,37 @@
+import { initializeAppInsights, getTelemetryClient, } from "@intuned/runtime/dist/common/telemetry";
+import { getExecutionContext } from "@intuned/runtime";
+import { app } from "./app";
+import { RUN_ID_HEADER } from "./headers";
+import { ShutdownController } from "./shutdown";
+const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
+initializeAppInsights();
+app.use((req, res, next) => {
+    // Cleanup after the response is sent
+    res.on("finish", () => {
+        console.log("finished", req.headers[RUN_ID_HEADER]);
+        void ShutdownController.instance.checkForShutdown();
+    });
+    next();
+});
+export * from "./api";
+export * from "./jobs";
+const server = app.listen(port, () => {
+    // when deployed on flyio, the server will be turned on and
+    // will shutdown after the specified time in TIME_TO_SHUTDOWN env variable
+    ShutdownController.initialize(server);
+    void ShutdownController.instance.checkForShutdown();
+    console.log(`Server is running on port ${port}`);
+});
+process.on("unhandledRejection", (reason, promise) => {
+    const telemetryClient = getTelemetryClient();
+    console.error("Unhandled Rejection at:", promise, "reason:", reason);
+    const context = getExecutionContext();
+    telemetryClient?.trackEvent({
+        name: "UNHANDLED_REJECTION",
+        properties: {
+            runId: context?.runId,
+            promise,
+            reason,
+        },
+    });
+});

package/WebTemplate/index.vanilla.js

@@ -0,0 +1,19 @@
+import { App } from "@tinyhttp/app";
+import { json as parser } from "milliparsec";
+import { accessKeyValidatorMiddleware } from "./accessKeyHelpers";
+const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
+const app = new App();
+app.use("/", parser());
+app.post("/api/run/*", async (req, res) => {
+    return res.status(502).json({ message: "Not implemented" });
+});
+app.get("/api/health", async (req, res) => {
+    res.status(200).json({ status: "ok" });
+});
+app.use(accessKeyValidatorMiddleware);
+app.get("/protected/health", async (req, res) => {
+    res.status(200).json({ status: "ok" });
+});
+app.listen(port, () => {
+    console.log(`Server is running on port ${port}`);
+});

package/WebTemplate/jobs.js

@@ -0,0 +1,259 @@
+import { runApi } from "./controllers/runApi";
+import { ATTEMPT_NUMBER_HEADER, JOB_ID_HEADER, JOB_RUN_ID_HEADER, RUN_ID_HEADER, SHOULD_SHUTDOWN_HEADER, } from "./headers";
+import { getErrorResponse, isHeadless, isJobRunMachine, proxyToUrl, } from "./utils";
+import { callBackendFunctionWithToken, backendFunctionsTokenManager, } from "@intuned/runtime/dist/common/jwtTokenManager";
+import retry from "async-retry";
+import { runWithContext } from "@intuned/runtime";
+import { setTimeout } from "timers/promises";
+import { deleteTrace, uploadTrace } from "./controllers/traces";
+if (isJobRunMachine()) {
+    console.log("Running in job v3 mode");
+    void jobsV3();
+}
+async function runApiInContext(jobPayload) {
+    const runId = jobPayload.headers[RUN_ID_HEADER];
+    const jobId = process.env.JOB_ID;
+    const jobRunId = process.env.JOB_RUN_ID;
+    const proxy = jobPayload.payload.proxy
+        ? proxyToUrl(jobPayload.payload.proxy)
+        : undefined;
+    const contextData = {
+        runId: runId ?? "",
+        jobId,
+        jobRunId,
+        queueId: undefined,
+        proxy,
+        timeoutInfo: {
+            extendTimeoutCallback: async () => {
+                try {
+                    await extendTimeout(jobPayload);
+                }
+                catch (error) {
+                    console.error("Error extending timeout", error);
+                }
+            },
+        },
+        ...(jobPayload.payload.executionContext ?? {}),
+    };
+    const backendFunctionsToken = jobPayload.payload.functionsToken;
+    if (backendFunctionsToken) {
+        backendFunctionsTokenManager.token = backendFunctionsToken;
+    }
+    try {
+        return await runWithContext(contextData, runApi, {
+            functionName: jobPayload.functionName,
+            ...jobPayload.payload,
+            runId: jobPayload.headers[RUN_ID_HEADER],
+            attemptNumber: jobPayload.headers[ATTEMPT_NUMBER_HEADER],
+            shouldSaveTrace: true,
+            headless: isHeadless(),
+        });
+    }
+    catch (error) {
+        return getErrorResponse(error);
+    }
+}
+async function jobsV3() {
+    await reportReady();
+    const initialDelay = 1000;
+    let delay = initialDelay;
+    while (true) {
+        await setTimeout(delay);
+        try {
+            const { payload, shouldShutdown } = await requestPayload();
+            if (!payload) {
+                if (shouldShutdown) {
+                    console.log("Received shutdown signal from job");
+                    process.exit(0);
+                }
+                delay = Math.min(delay * 2, 60000);
+                continue;
+            }
+            delay = initialDelay;
+            console.log("Running payload", payload.functionName);
+            const { body, status } = await runApiInContext(payload);
+            await reportResultsAndUploadTraces(payload, status, body);
+            if (shouldShutdown) {
+                process.exit(0);
+            }
+        }
+        catch (error) {
+            console.error("Error in payload", error);
+        }
+    }
+}
+async function reportReady() {
+    console.log("Ready");
+}
+async function requestPayload() {
+    console.log("Requesting payload");
+    const result = await callBackendFunctionWithToken("jobs/requestPayload", {
+        headers: {
+            "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+            [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+            [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+        },
+    });
+    const shouldShutdown = result.headers.get(SHOULD_SHUTDOWN_HEADER) === "true";
+    if (result.status === 200) {
+        const payload = (await result.json());
+        console.log("Received payload", payload);
+        return {
+            payload: {
+                ...payload,
+                headers: {
+                    ...payload.headers,
+                    [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+                    [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+                },
+            },
+            shouldShutdown,
+        };
+    }
+    if (result.status === 204) {
+        console.log("No payload available");
+        return {
+            payload: null,
+            shouldShutdown,
+        };
+    }
+    console.log("Failed to obtain payload", result.status, await result.text());
+    return {
+        payload: null,
+        shouldShutdown,
+    };
+}
+async function reportResultsAndUploadTraces(payload, statusCode, body) {
+    await reportJobRunResult({
+        runId: payload.headers[RUN_ID_HEADER],
+        result: {
+            statusCode,
+            body,
+            receiptHandle: payload.receiptHandle,
+        },
+        originalPayload: payload.originalPayload,
+        startTime: payload.startTime,
+        functionsToken: backendFunctionsTokenManager.token,
+    });
+    await uploadJobRunTrace(payload);
+}
+async function reportJobRunResult({ runId, result, originalPayload, startTime, functionsToken, }) {
+    console.log("Reporting payload result");
+    const reqResult = await retry(async (bail) => {
+        const response = await callBackendFunctionWithToken("jobs/reportResult", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+                [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+                [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+            },
+            body: JSON.stringify({
+                runId,
+                result,
+                originalPayload,
+                startTime,
+            }),
+        });
+        if (!response.ok) {
+            if ([401, 403, 404, 413].includes(response.status)) {
+                bail(new Error(`Reporting result failed (non-retryable), status ${response.status}`));
+            }
+            throw new Error(`Reporting result failed, status ${response.status}`);
+        }
+        return response;
+    }, {
+        retries: 5,
+        factor: 2,
+        maxTimeout: 1000 * 60, // 1 minute
+        minTimeout: 1000 * 5, // 5 seconds
+    });
+    console.log("Reported payload result", reqResult.ok, await reqResult.text());
+}
+async function uploadJobRunTrace(payload) {
+    if (!payload.traceSignedUrl) {
+        return;
+    }
+    const runId = payload.headers[RUN_ID_HEADER];
+    const attemptNumber = payload.headers[ATTEMPT_NUMBER_HEADER];
+    console.log("Uploading trace", runId, attemptNumber);
+    const uploadTraceResult = await uploadTrace({
+        runId,
+        attemptNumber,
+        signedUrl: payload.traceSignedUrl,
+    });
+    if (uploadTraceResult.isErr()) {
+        console.error("Error uploading trace", uploadTraceResult.error, {
+            runId,
+            attemptNumber,
+        });
+    }
+    else {
+        console.log("Trace uploaded successfully, reporting trace", runId, attemptNumber);
+        await retry(async (bail) => {
+            const response = await callBackendFunctionWithToken("jobs/reportTrace", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+                    [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+                    [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+                },
+                body: JSON.stringify({
+                    runId,
+                    attemptNumber,
+                    traceSignedUrl: payload.traceSignedUrl,
+                }),
+            });
+            if (!response.ok) {
+                if ([401, 403, 404].includes(response.status)) {
+                    bail(new Error(`Reporting trace failed (non-retryable), status ${response.status}`));
+                }
+                throw new Error(`Reporting trace failed, status ${response.status}`);
+            }
+            console.log("Trace reported successfully", runId, attemptNumber);
+            return response;
+        }, {
+            retries: 2,
+            minTimeout: 1000 * 5,
+        }).catch(() => undefined);
+    }
+    console.log("Deleting trace", runId, attemptNumber);
+    const deleteTraceResult = await deleteTrace({
+        runId,
+        attemptNumber,
+    });
+    if (deleteTraceResult.isErr()) {
+        console.error("Error deleting trace", deleteTraceResult.error, {
+            runId,
+            attemptNumber,
+        });
+    }
+}
+async function extendTimeout(payload) {
+    console.log("Requesting timeout extension for", payload.functionName);
+    const extendTimeoutRes = await callBackendFunctionWithToken("jobs/extendTimeout", {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+            [JOB_ID_HEADER]: process.env.JOB_ID ?? "",
+            [JOB_RUN_ID_HEADER]: process.env.JOB_RUN_ID ?? "",
+        },
+        body: JSON.stringify({
+            receiptHandle: payload.receiptHandle,
+        }),
+    });
+    if (!extendTimeoutRes.ok) {
+        console.error("Requesting timeout extension failed", extendTimeoutRes.status, await extendTimeoutRes.text());
+        return false;
+    }
+    console.log("Request timeout extension success", extendTimeoutRes.ok);
+    const response = await extendTimeoutRes.json();
+    const newFunctionsToken = response.functionsToken;
+    if (newFunctionsToken) {
+        backendFunctionsTokenManager.token = newFunctionsToken;
+        console.log("Backend functions token renewed");
+    }
+    return true;
+}

package/WebTemplate/shutdown.js

@@ -0,0 +1,51 @@
+import { isJobRunMachine } from "./utils";
+import { promisify } from "util";
+import { AsyncRunEndpointController } from "./utils";
+export class ShutdownController {
+    static initialize(server) {
+        if (this._instance) {
+            throw new Error("ShutdownController is already initialized");
+        }
+        this._instance = new ShutdownController(server);
+    }
+    static get instance() {
+        if (!this._instance) {
+            throw new Error("ShutdownController is not initialized");
+        }
+        return this._instance;
+    }
+    constructor(server) {
+        this.timer = null;
+        this.server = server;
+    }
+    async checkForShutdown() {
+        if (await this.serverShouldShutdown()) {
+            if (this.timer)
+                clearTimeout(this.timer);
+            this.timer = setTimeout(async () => {
+                if (await this.serverShouldShutdown()) {
+                    console.log("No active requests, shutting down the server.");
+                    process.exit(0);
+                }
+                // TIME_TO_SHUTDOWN in seconds to match tired-proxy
+            }, this.calculateShutdownDelay());
+        }
+    }
+    async getActiveRequests() {
+        const activeConnections = await promisify((...params) => this.server.getConnections(...params))();
+        return activeConnections + AsyncRunEndpointController.activeRequestsCount;
+    }
+    async serverShouldShutdown() {
+        if (isJobRunMachine()) {
+            return false;
+        }
+        return (await this.getActiveRequests()) === 0;
+    }
+    calculateShutdownDelay() {
+        const shutdownTime = +process.env.TIME_TO_SHUTDOWN;
+        if (isNaN(shutdownTime) || shutdownTime <= 0) {
+            return 60 * 1000;
+        }
+        return shutdownTime * 1000;
+    }
+}

package/WebTemplate/utils.js

@@ -0,0 +1,189 @@
+import * as playwright from "@intuned/playwright-core";
+import * as path from "path";
+import { getExecutionContext } from "@intuned/runtime";
+import { setTimeout } from "timers/promises";
+export class FunctionNotFoundError extends Error {
+    constructor(functionName, path) {
+        const message = `function ${functionName} not found in ${path}`;
+        super(message);
+        this.functionName = functionName;
+        this.path = path;
+        this.name = "FunctionNotFound";
+        Object.setPrototypeOf(this, FunctionNotFoundError.prototype);
+    }
+}
+export function getIsRetryableError(error) {
+    if (error?.message) {
+        return error.message.includes("ERR_NETWORK_CHANGED");
+    }
+    return false;
+}
+export function getErrorResponse(error) {
+    if (error instanceof FunctionNotFoundError) {
+        return {
+            status: 404,
+            body: {
+                message: error.message,
+                error: error.name,
+            },
+        };
+    }
+    if (error instanceof playwright.errors.TimeoutError) {
+        return {
+            status: 500,
+            body: { message: error.message, error: error.name },
+        };
+    }
+    /**
+     * here we use error.constructor.name instead of error instanceof RunError
+     * this is because the error is thrown by importing the runner from the api code on intuned app
+     * the definition of class RunError which is imported from here is different than the class RunError
+     * imported from the user.
+     */
+    if (error.constructor.name === "RunError") {
+        return {
+            status: 200,
+            body: {
+                status: error.options.status_code ?? 500,
+                message: error.message,
+                error: error.options.error_code ?? error.name,
+                intunedOptions: error.options,
+            },
+        };
+    }
+    return {
+        status: 500,
+        body: { error: error?.name ?? error, message: error?.message },
+    };
+}
+export function handlePlaywrightExecutionError(error, res) {
+    const { status, body } = getErrorResponse(error);
+    return res.status(status).json(body);
+}
+export function errorRetryMiddleware(handler) {
+    return async (...args) => {
+        let attempts = 1;
+        const [req, res, next] = args;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            try {
+                await handler(req, res, next);
+                break;
+            }
+            catch (error) {
+                console.log(error?.name, error?.message);
+                if (!getIsRetryableError(error) || attempts >= 3) {
+                    return handlePlaywrightExecutionError(error, res);
+                }
+                attempts++;
+            }
+        }
+    };
+}
+export function getTraceFilePath(runId, attemptNumber) {
+    const fileName = `${runId}${attemptNumber ? `_${attemptNumber}` : ""}`;
+    return path.join(process.env.TRACES_DIRECTORY ?? "", `${fileName}.zip`);
+}
+export function proxyToUrl(proxy) {
+    const url = new URL(proxy.server);
+    url.username = proxy.username;
+    url.password = proxy.password;
+    return url.toString();
+}
+export function isJobRunMachine() {
+    return process.env.JOB_ID && process.env.JOB_RUN_ID;
+}
+export function isHeadless() {
+    return process.env.INTUNED_PLAYWRIGHT_HEADLESS !== "0";
+}
+export class AsyncRunEndpointController {
+    static isRunning(taskToken) {
+        return this.activeRequests.has(taskToken);
+    }
+    static addRequest(taskToken) {
+        this.activeRequests.add(taskToken);
+    }
+    static removeRequest(taskToken) {
+        this.activeRequests.delete(taskToken);
+    }
+    static get activeRequestsCount() {
+        return this.activeRequests.size;
+    }
+}
+AsyncRunEndpointController.activeRequests = new Set();
+export async function waitWithExtendableTimeout({ initialTimeout, promise, abortController, }) {
+    const context = getExecutionContext();
+    if (context.timeoutInfo) {
+        context.timeoutInfo.timeoutTimestamp = Date.now() + initialTimeout;
+    }
+    const timerSymbol = Symbol("timer");
+    if (!context) {
+        const result = await Promise.race([
+            promise,
+            setTimeout(initialTimeout, timerSymbol),
+        ]);
+        if (result === timerSymbol) {
+            abortController?.abort("Timed out");
+            return { timedOut: true };
+        }
+        return { timedOut: false, result };
+    }
+    let taskTimeout = initialTimeout;
+    while (true) {
+        const result = await Promise.race([
+            promise,
+            setTimeout(taskTimeout, timerSymbol),
+        ]);
+        if (result !== timerSymbol) {
+            break;
+        }
+        const remainingTime = (context.timeoutInfo?.timeoutTimestamp ?? 0) - Date.now();
+        if (remainingTime < 0) {
+            abortController?.abort("Timed out");
+            return { timedOut: true };
+        }
+        taskTimeout = remainingTime;
+    }
+    return {
+        timedOut: false,
+        result: await promise,
+    };
+}
+export async function importModule(path) {
+    // cleanup environment variables before running the user code
+    // cleanEnvironmentVariables();
+    const [folderName, ...functionNameParts] = path.split("/");
+    const functionNameDepth = functionNameParts.length;
+    // string literals should be inline
+    // currently we support only 5 levels of depth
+    // rollup dynamic import does not support multiple levels of dynamic imports so we need to specify the possible paths explicitly
+    try {
+        let imported = undefined;
+        switch (functionNameDepth) {
+            case 1:
+                imported = await import(`./${folderName}/${functionNameParts[0]}.ts`);
+                break;
+            case 2:
+                imported = await import(`./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}.ts`);
+                break;
+            case 3:
+                imported = await import(`./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}.ts`);
+                break;
+            case 4:
+                imported = await import(`./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}/${functionNameParts[3]}.ts`);
+                break;
+            case 5:
+                imported = await import(`./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}/${functionNameParts[3]}/${functionNameParts[4]}.ts`);
+                break;
+            default:
+                throw new Error("intuned supports maximum 5 levels of depth in the api folder");
+        }
+        return imported;
+    }
+    catch (error) {
+        if (error.message.includes("Unknown variable dynamic import")) {
+            throw new FunctionNotFoundError("", path);
+        }
+        throw error;
+    }
+}

package/api/authed.js

@@ -0,0 +1,6 @@
+export default async function authed(params, page, context) {
+    await page.goto("https://setcookie.net");
+    return {
+        cookies: await page.locator("ul li code").allInnerTexts(),
+    };
+}

package/api/test.js

@@ -0,0 +1,3 @@
+export default async function whatever() {
+    console.log("whatever");
+}