@intuned/runtime-dev 0.0.1 → 0.1.0-test.0

package/WebTemplate/controllers/runApi/helpers.ts

@@ -1,27 +1,14 @@
-import {
-  callFunction,
-  getTraceFilePath,
-  checkAuthSessionWithRetries,
-  waitWithExtendableTimeout,
-} from "../../utils";
-import { IntunedSettings } from "@intuned/runtime/dist/common/settingsSchema";
-import { getDownloadDirectoryPath } from "@intuned/runtime";
-import { getProductionPlaywrightConstructs } from "@intuned/runtime";
+import { getTraceFilePath, waitWithExtendableTimeout } from "../../utils";
 import * as fs from "fs-extra";
-// @ts-ignore
-import { PayloadItem } from "../../../src/common/types";
-import { getTelemetryClient } from "@intuned/runtime/dist/common/telemetry";
 import { getExecutionContext } from "@intuned/runtime";
 import { backendFunctionsTokenManager } from "@intuned/runtime/dist/common/jwtTokenManager";
-import { setContextStorageState } from "@intuned/runtime/dist/common/contextStorageStateHelpers";
-import { remove } from "fs-extra";
 import { RunBody } from "./types";
+import { runApi as runApiInternal } from "@intuned/runtime/dist/common/runApi";
 
 export async function runApi({
   session,
   params,
   proxy,
-  executionContext,
   functionsToken,
   runId,
   attemptNumber,
@@ -38,9 +25,8 @@ export async function runApi({
   status: number;
   body: any;
 }> {
-  const telemetryClient = getTelemetryClient();
-  const integrationId = process.env.INTUNED_INTEGRATION_ID as string;
-  const traceFilePath = getTraceFilePath(runId, attemptNumber);
+  const isAuthSessionEnabled = (await fs.readJSON("./Intuned.json"))
+    .authSessions?.enabled;
 
   if (!runId) {
     return {
@@ -52,15 +38,7 @@ export async function runApi({
     };
   }
 
-  let traceStarted = false;
-
   backendFunctionsTokenManager.token = functionsToken;
-  const downloadsPath = getDownloadDirectoryPath();
-  const { page, context } = await getProductionPlaywrightConstructs({
-    headless,
-    proxy,
-    downloadsPath,
-  });
 
   // handle timeout
   const requestTimeout = +(process.env.REQUEST_TIMEOUT ?? 600) * 1000;
@@ -69,23 +47,8 @@ export async function runApi({
     timeoutDuration: requestTimeout,
   };
 
-  async function saveTraceIfNeeded(errorMessage: string) {
-    if (!shouldSaveTrace || !traceStarted) {
-      return;
-    }
-    try {
-      await context.tracing.stop({ path: traceFilePath });
-    } catch (error: any) {
-      console.log(errorMessage, error?.message);
-      await remove(traceFilePath);
-    }
-  }
-
   async function handleTimeout() {
     console.log("timeout triggered");
-    await saveTraceIfNeeded("failed to save script after timeout");
-
-    await context.close();
 
     return {
       status: 508,
@@ -96,113 +59,35 @@ export async function runApi({
     };
   }
 
-  if (shouldSaveTrace) {
-    await context.tracing.start({
-      screenshots: true,
-      snapshots: true,
-      sources: true,
-    });
-    traceStarted = true;
-  }
-
-  const isAuthSessionsEnabled = (
-    (await fs.readJSON("./Intuned.json", "utf-8")) as IntunedSettings
-  ).authSessions?.enabled;
-
   const abortController = new AbortController();
-  async function runFunction() {
-    try {
-      if (isAuthSessionsEnabled) {
-        if (!session) {
-          await context.close();
-          return {
-            status: 401,
-            message: "auth sessions is required",
-            error: "auth sessions is required",
-          };
-        }
-
-        await setContextStorageState(context, session);
-        const authSessionCheckResult = await checkAuthSessionWithRetries(
-          page,
-          context,
-          2
-        );
-
-        if (abortController.signal.aborted) {
-          return;
-        }
-
-        if (!authSessionCheckResult) {
-          try {
-            telemetryClient?.trackEvent({
-              name: "AUTH_SESSION_RECOVER_FAILED",
-              properties: {
-                runId,
-                functionName,
-                integrationId,
-                message: "auth session check failed",
-              },
-            });
-            await saveTraceIfNeeded("failed to save trace after check failed");
-          } catch (error: any) {
-            console.log(
-              "failed to save script after check failed",
-              error?.message
-            );
-          }
-
-          await context.close();
-          return {
-            status: 401,
-            message: "auth session check failed",
-            error: "AUTH_SESSION_CHECK_FAILED",
-          };
-        }
-      }
-
-      console.log("Running function", functionName);
-      const result = await callFunction("api", functionName as string, [
-        params ?? {},
-        page,
-        context,
-      ]);
-
-      await saveTraceIfNeeded("failed to save trace after run is done");
-      await context.close();
-
-      const response: {
-        result: any;
-        updatedSession?: object;
-        payloadToAppend: PayloadItem[] | null;
-        // TODO: remove
-        metadata?: {
-          startTime: number;
-          endTime: number;
-        };
-        status?: number;
-      } = {
-        result,
-        updatedSession: undefined,
-        payloadToAppend: getExecutionContext()!.extendedPayloads?.map((i) => ({
-          apiName: i.api,
-          parameters: i.parameters,
-        })),
-        status: 200,
-      };
-      return response;
-    } catch (error) {
-      console.log("run errored", error);
-      await saveTraceIfNeeded("failed to save trace after run errored");
-      await context.close();
-      throw error;
-    } finally {
-      await fs.remove(downloadsPath);
-    }
-  }
 
   const resultWithTimeout = await waitWithExtendableTimeout({
-    promise: runFunction(),
+    promise: runApiInternal({
+      automationFunction: {
+        name: `api/${functionName}`,
+        params: params ?? {},
+      },
+      auth: {
+        session: {
+          type: "state",
+          state: session,
+        },
+        runCheck: isAuthSessionEnabled,
+      },
+      runOptions: {
+        environment: "deployed",
+        headless,
+        proxy,
+      },
+      tracing: shouldSaveTrace
+        ? {
+            enabled: true,
+            filePath: getTraceFilePath(runId, attemptNumber),
+          }
+        : { enabled: false },
+      functionsToken,
+      abortSignal: abortController.signal,
+    }),
     initialTimeout: requestTimeout,
     abortController,
   });
@@ -213,8 +98,24 @@ export async function runApi({
 
   const { result } = resultWithTimeout;
 
+  if (result.isErr()) {
+    if (result.error.code === "AbortedError") {
+      return await handleTimeout();
+    }
+    return result.error.apiResponse;
+  }
+
   return {
     status: 200,
-    body: result,
+    body: {
+      status: 200,
+      result: result.value.result,
+      payloadToAppend: result.value.extendedPayloads?.map(
+        ({ api, parameters }) => ({
+          apiName: api,
+          parameters,
+        })
+      ),
+    },
   };
 }

package/WebTemplate/index.playwright.ts

@@ -6,42 +6,52 @@ import { getExecutionContext } from "@intuned/runtime";
 import { app } from "./app";
 import { RUN_ID_HEADER } from "./headers";
 import { ShutdownController } from "./shutdown";
+import { registerApiEndpoints } from "./api";
+import { runJobsLoop } from "./jobs";
+import { isJobRunMachine } from "./utils";
 
-const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
+void main();
 
-initializeAppInsights();
+async function main() {
+  const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
 
-app.use((req, res, next) => {
-  // Cleanup after the response is sent
-  res.on("finish", () => {
-    console.log("finished", req.headers[RUN_ID_HEADER]);
+  initializeAppInsights();
+
+  app.use((req, res, next) => {
+    // Cleanup after the response is sent
+    res.on("finish", () => {
+      console.log("finished", req.headers[RUN_ID_HEADER]);
+      void ShutdownController.instance.checkForShutdown();
+    });
+
+    next();
+  });
+
+  registerApiEndpoints();
+  if (isJobRunMachine()) {
+    console.log("Running in job v3 mode");
+    void runJobsLoop();
+  }
+
+  const server = app.listen(port, () => {
+    // when deployed on flyio, the server will be turned on and
+    // will shutdown after the specified time in TIME_TO_SHUTDOWN env variable
+    ShutdownController.initialize(server);
     void ShutdownController.instance.checkForShutdown();
+    console.log(`Server is running on port ${port}`);
   });
 
-  next();
-});
-
-export * from "./api";
-export * from "./jobs";
-
-const server = app.listen(port, () => {
-  // when deployed on flyio, the server will be turned on and
-  // will shutdown after the specified time in TIME_TO_SHUTDOWN env variable
-  ShutdownController.initialize(server);
-  void ShutdownController.instance.checkForShutdown();
-  console.log(`Server is running on port ${port}`);
-});
-
-process.on("unhandledRejection", (reason, promise) => {
-  const telemetryClient = getTelemetryClient();
-  console.error("Unhandled Rejection at:", promise, "reason:", reason);
-  const context = getExecutionContext();
-  telemetryClient?.trackEvent({
-    name: "UNHANDLED_REJECTION",
-    properties: {
-      runId: context?.runId,
-      promise,
-      reason,
-    },
+  process.on("unhandledRejection", (reason, promise) => {
+    const telemetryClient = getTelemetryClient();
+    console.error("Unhandled Rejection at:", promise, "reason:", reason);
+    const context = getExecutionContext();
+    telemetryClient?.trackEvent({
+      name: "UNHANDLED_REJECTION",
+      properties: {
+        runId: context?.runId,
+        promise,
+        reason,
+      },
+    });
   });
-});
+}

package/WebTemplate/index.vanilla.ts

@@ -1,32 +1,14 @@
 import { App } from "@tinyhttp/app";
 import { json as parser } from "milliparsec";
 import { accessKeyValidatorMiddleware } from "./accessKeyHelpers";
-import { callFunction } from "./utils";
 
 const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
 const app = new App();
 
 app.use("/", parser());
 
-app.post("/api/run/*", async (req, res, next) => {
-  // filter(Boolean) should remove empty strings in case of a trailing slash
-  const functionName = req.path.split("/").filter(Boolean).pop();
-
-  try {
-    const body = req.body;
-    const result = await callFunction("api", functionName as string, [body]);
-
-    res.status(200).json(result);
-  } catch (error) {
-    if (
-      error.message ===
-      `Unknown variable dynamic import: ./api/${functionName}.ts`
-    ) {
-      return res.status(404).json({});
-    }
-
-    next(error);
-  }
+app.post("/api/run/*", async (req, res) => {
+  return res.status(502).json({ message: "Not implemented" });
 });
 
 app.get("/api/health", async (req, res) => {

package/WebTemplate/jobs.ts

@@ -6,13 +6,7 @@ import {
   RUN_ID_HEADER,
   SHOULD_SHUTDOWN_HEADER,
 } from "./headers";
-import {
-  getErrorResponse,
-  isHeadless,
-  isJobRunMachine,
-  ProxyConfig,
-  proxyToUrl,
-} from "./utils";
+import { getErrorResponse, isHeadless, ProxyConfig, proxyToUrl } from "./utils";
 import {
   callBackendFunctionWithToken,
   backendFunctionsTokenManager,
@@ -32,11 +26,6 @@ type JobPayload = {
   traceSignedUrl?: string;
 };
 
-if (isJobRunMachine()) {
-  console.log("Running in job v3 mode");
-  void jobsV3();
-}
-
 async function runApiInContext(
   jobPayload: JobPayload
 ): Promise<Awaited<ReturnType<typeof runApi>>> {
@@ -83,7 +72,7 @@ async function runApiInContext(
   }
 }
 
-async function jobsV3() {
+export async function runJobsLoop() {
   await reportReady();
   const initialDelay = 1000;
   let delay = initialDelay;

package/WebTemplate/utils.ts

@@ -1,10 +1,7 @@
 import * as playwright from "@intuned/playwright-core";
 import { Handler, Response } from "@tinyhttp/app";
-import { cleanEnvironmentVariables } from "@intuned/runtime/dist/common/cleanEnvironmentVariables";
-import { AUTH_SESSIONS_FOLDER_NAME } from "@intuned/runtime/dist/common/constants";
-import { Page, BrowserContext } from "@intuned/playwright-core";
 import * as path from "path";
-import { getExecutionContext } from "@intuned/runtime";
+import { getExecutionContext } from "@intuned/runtime/dist/common/asyncLocalStorage";
 import { setTimeout } from "timers/promises";
 
 export class FunctionNotFoundError extends Error {
@@ -20,62 +17,6 @@ export class FunctionNotFoundError extends Error {
   }
 }
 
-export async function callFunction(
-  folderName: string,
-  functionName: string,
-  args: any[]
-) {
-  // cleanup environment variables before running the user code
-  cleanEnvironmentVariables();
-
-  const path = `./${folderName}/${functionName}.ts`;
-
-  const functionNameParts = functionName.split("/");
-  const functionNameDepth = functionNameParts.length;
-
-  // string literals should be inline
-  // currently we support only 5 levels of depth
-  // rollup dynamic import does not support multiple levels of dynamic imports so we need to specify the possible paths explicitly
-  try {
-    let imported: any = undefined;
-    switch (functionNameDepth) {
-      case 1:
-        imported = await import(`./${folderName}/${functionNameParts[0]}.ts`);
-        break;
-      case 2:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}.ts`
-        );
-        break;
-      case 3:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}.ts`
-        );
-        break;
-      case 4:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}/${functionNameParts[3]}.ts`
-        );
-        break;
-      case 5:
-        imported = await import(
-          `./${folderName}/${functionNameParts[0]}/${functionNameParts[1]}/${functionNameParts[2]}/${functionNameParts[3]}/${functionNameParts[4]}.ts`
-        );
-        break;
-      default:
-        throw new Error(
-          "intuned supports maximum 5 levels of depth in the api folder"
-        );
-    }
-    return await imported?.default(...args);
-  } catch (error: any) {
-    if (error.message.includes("Unknown variable dynamic import")) {
-      throw new FunctionNotFoundError(functionName, path);
-    }
-    throw error;
-  }
-}
-
 export function getIsRetryableError(error: any) {
   if (error?.message) {
     return error.message.includes("ERR_NETWORK_CHANGED");
@@ -161,31 +102,6 @@ export function errorRetryMiddleware(handler: Handler) {
   };
 }
 
-export async function checkAuthSessionWithRetries(
-  page: Page,
-  context: BrowserContext,
-  retries = 3
-): Promise<boolean> {
-  if (retries === 0) {
-    return false;
-  }
-
-  let tryNumber = 0;
-
-  while (retries > tryNumber) {
-    const result: boolean = await callFunction(
-      AUTH_SESSIONS_FOLDER_NAME,
-      "check" as string,
-      [page, context]
-    );
-    if (result) return true;
-
-    tryNumber++;
-  }
-
-  return false;
-}
-
 export function getTraceFilePath(runId: string, attemptNumber?: string) {
   const fileName = `${runId}${attemptNumber ? `_${attemptNumber}` : ""}`;
   return path.join(process.env.TRACES_DIRECTORY ?? "", `${fileName}.zip`);

package/api/authed.ts

@@ -0,0 +1,12 @@
+import { Page, BrowserContext } from "@intuned/playwright-core";
+
+export default async function authed(
+  params: any,
+  page: Page,
+  context: BrowserContext
+) {
+  await page.goto("https://setcookie.net");
+  return {
+    cookies: await page.locator("ul li code").allInnerTexts(),
+  };
+}

package/api/test.ts

@@ -0,0 +1,3 @@
+export default async function whatever() {
+  console.log("whatever");
+}

package/api/test2.ts

@@ -0,0 +1,25 @@
+import { Page, BrowserContext } from "@intuned/playwright-core";
+import { extendTimeout } from "../src/runtime/extendTimeout";
+import { requestOTP } from "../src/runtime/requestMoreInfo";
+
+export default async function* test2(
+  { n }: { n?: number },
+  page: Page,
+  context: BrowserContext
+) {
+  await page.goto("https://wikipedia.org/");
+  await page.waitForTimeout(1000);
+
+  yield requestOTP("Please enter the OTP");
+
+  const titles: string[] = [];
+
+  for (let i = 0; i < (n ?? 2); i++) {
+    await page.goto(`https://wikipedia.org/wiki/Special:Random`);
+    await page.waitForTimeout(1000);
+    titles.push(await page.locator("h1").innerText());
+    extendTimeout();
+  }
+
+  return { titles };
+}

package/auth-sessions/check.ts

@@ -0,0 +1,9 @@
+import { Page, BrowserContext } from "@intuned/playwright-core";
+
+export default async function check(
+  page: Page,
+  _context: BrowserContext
+): Promise<boolean> {
+  await page.goto("https://setcookie.net");
+  return (await page.locator("ul li code").all()).length > 0;
+}

package/auth-sessions/create.ts

@@ -0,0 +1,32 @@
+import { Page, BrowserContext } from "@intuned/playwright-core";
+import {
+  requestOTP,
+  requestMultipleChoice,
+} from "../src/runtime/requestMoreInfo";
+
+// eslint-disable-next-line require-yield
+export default async function* create(
+  params: any,
+  page: Page,
+  context: BrowserContext
+) {
+  await page.goto("https://setcookie.net/");
+  await page.locator("#name").fill("intuned");
+  await page.locator("#value").fill("password");
+  await page.locator("input[type=submit]").click();
+
+  console.log("Received", yield requestOTP("Enter useless otp"));
+  console.log(
+    "Received",
+    yield requestMultipleChoice("Choose the correct answer", [
+      "A",
+      "B",
+      "C",
+      "D",
+    ])
+  );
+
+  await page.goto("http://www.sharonminsuk.com/code/storage-test.html");
+  await page.locator("#local").fill("intuned");
+  await page.locator("#local + input[type=button]").click();
+}

package/authSessions

@@ -0,0 +1 @@
+{"cookies":[{"name":"intuned","value":"password","domain":"setcookie.net","path":"/","expires":-1,"httpOnly":true,"secure":false,"sameSite":"Lax"}],"origins":[{"origin":"http://www.sharonminsuk.com","localStorage":[{"name":"theirValue","value":"intuned"}]}],"sessionStorage":[{"origin":"http://www.sharonminsuk.com","sessionStorage":[]}]}

package/bin/intuned-api-run

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/api/run.js");

package/bin/intuned-auth-session-check

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/auth-sessions/run-check.js");

package/bin/intuned-auth-session-create

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/auth-sessions/run-create.js");

package/bin/intuned-auth-session-load

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/auth-sessions/load.js");

package/bin/intuned-auth-session-refresh

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/auth-sessions/run-refresh.js");

package/bin/intuned-browser-save-state

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/browser/save-state.js");

package/bin/intuned-browser-start

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/browser/start-browser.js");

package/bin/intuned-build

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/build.js");

package/bin/intuned-ts-check

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require("../dist/commands/ts-check.js");

package/dist/commands/api/run.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+export declare function executeCLI(apiName: string, mode: "vanilla" | "playwright" | "playwright-standalone" | "playwright-headless", inputData: object | null | undefined, options: {
+    cdpAddress: string;
+    authSessionPath: undefined | string;
+    outputFileId: string | undefined;
+}): Promise<void>;