@intuned/runtime-dev 0.0.1-split.7 → 0.0.1

package/.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+  "recommendations": ["zixuanchen.vitest-explorer"]
+}

package/.vscode/launch.json

@@ -0,0 +1,102 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Hiltion api",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": ["src/commands/api/run.ts", "hilton", "-j", "{}", "playwright"],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Books Extractor",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": [
+        "src/commands/api/run.ts",
+        "books-array",
+        "-j",
+        "{}",
+        "playwright"
+      ],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run execution-info",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": [
+        "src/commands/api/run.ts",
+        "execution-info",
+        "-j",
+        "{}",
+        "playwright"
+      ],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run progressive",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": [
+        "src/commands/api/run.ts",
+        "progressive",
+        "-j",
+        "{}",
+        "playwright"
+      ],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run progressive - standalone",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": ["src/commands/api/run.ts", "progressive", "-j", "{}"],
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run progressive extract - standalone",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": ["src/commands/api/run.ts", "progressive-extract", "-j", "{}"],
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    }
+  ]
+}

package/.vscode/settings.json

@@ -0,0 +1,12 @@
+{
+  "editor.formatOnSave": true,
+  "cSpell.words": ["backcompat", "ـget", "INFINITI", "neverthrow"],
+  "editor.detectIndentation": true,
+  "files.exclude": {
+    "dist": false,
+    "node_modules": false,
+    "output": false,
+    "intuned": false,
+    "htmlFiles": true
+  }
+}

package/WebTemplate/controllers/authSessions/create.ts

@@ -1,7 +1,7 @@
 import { authSessionsContextsStore } from "./store";
 import { AUTH_SESSIONS_FOLDER_NAME } from "@intuned/runtime/dist/common/constants";
 import { getDownloadDirectoryPath } from "@intuned/runtime";
-import { getProductionPlaywrightConstructs } from "@intuned/runtime";
+import { getProductionPlaywrightConstructs } from "../../../src/common/getPlaywrightConstructs";
 import { callFunction, getTraceFilePath, isHeadless } from "../../utils";
 import * as fs from "fs-extra";
 import { getContextStorageState } from "@intuned/runtime/dist/common/contextStorageStateHelpers";

package/WebTemplate/controllers/runApi/helpers.ts

@@ -8,6 +8,8 @@ import { IntunedSettings } from "@intuned/runtime/dist/common/settingsSchema";
 import { getDownloadDirectoryPath } from "@intuned/runtime";
 import { getProductionPlaywrightConstructs } from "@intuned/runtime";
 import * as fs from "fs-extra";
+// @ts-ignore
+import { PayloadItem } from "../../../src/common/types";
 import { getTelemetryClient } from "@intuned/runtime/dist/common/telemetry";
 import { getExecutionContext } from "@intuned/runtime";
 import { backendFunctionsTokenManager } from "@intuned/runtime/dist/common/jwtTokenManager";
@@ -172,7 +174,7 @@ export async function runApi({
       const response: {
         result: any;
         updatedSession?: object;
-        payloadToAppend: any[] | null;
+        payloadToAppend: PayloadItem[] | null;
         // TODO: remove
         metadata?: {
           startTime: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@intuned/runtime-dev",
-  "version": "0.0.1-split.7",
+  "version": "0.0.1",
   "description": "Intuned runtime",
   "exports": {
     ".": "./dist/index.js",
@@ -13,7 +13,6 @@
     "./dist/common/cleanEnvironmentVariables": "./dist/common/cleanEnvironmentVariables.js",
     "./dist/common/constants": "./dist/common/constants.js"
   },
-  "types": "./dist/index.d.ts",
   "author": "Intuned Team",
   "license": "MIT",
   "scripts": {

package/playwright.config.ts

@@ -0,0 +1,48 @@
+import { defineConfig, devices } from "@intuned/playwright-test";
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: "./e2e-tests",
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+    // headless: false,
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});

package/src/commands/api/run.ts

@@ -0,0 +1,225 @@
+#!/usr/bin/env node
+import { program, Argument } from "commander";
+import * as fs from "fs-extra";
+import { getDefaultExportFromFile } from "../common/getDefaultExportFromFile";
+import { getFullPathInProject } from "../common/utils/fileUtils";
+import { getSettings } from "../common/utils/settings";
+import getFirstLineNumber from "../common/getFirstLineNumber";
+import { sendBreakPointOnToClient } from "../common/sendMessageToClient";
+import dotenv from "dotenv";
+import {
+  getExecutionContext,
+  runWithContext,
+} from "../../common/asyncLocalStorage";
+import { RunEnvironment } from "../../runtime/enums";
+import { cleanEnvironmentVariables } from "../../common/cleanEnvironmentVariables";
+import { logger } from "../../common/Logger";
+import { nanoid } from "nanoid";
+import chalk from "chalk";
+import { getDownloadDirectoryPath } from "../../runtime";
+import { getPlaywrightConstructsForMode } from "../../common/getPlaywrightConstructs";
+
+dotenv.config({
+  path: `.env`,
+});
+
+export async function executeCLI(
+  apiName: string,
+  mode:
+    | "vanilla"
+    | "playwright"
+    | "playwright-standalone"
+    | "playwright-headless",
+  inputData: object | null | undefined,
+  options: {
+    cdpAddress: string;
+    authSessionPath: undefined | string;
+    outputFileId: string | undefined;
+  }
+) {
+  // cleanup environment variables before running the user code
+  cleanEnvironmentVariables();
+  // Path to the TypeScript file
+  const apiFilePath = getFullPathInProject("api", `${apiName}.ts`);
+
+  let authSessionPathToUse: string | null = null;
+
+  // check auth session
+  const settings = await getSettings();
+
+  if (settings.authSessions.enabled) {
+    // if auth session enabled, check if the auth session is provided
+    if (!options.authSessionPath) {
+      throw new Error("Auth session is enabled but no auth session provided");
+    }
+
+    authSessionPathToUse = options.authSessionPath as string;
+  } else {
+    // if auth session is not enabled, check if the auth session is provided
+    if (options.authSessionPath) {
+      throw new Error(
+        "Auth session is not enabled but auth session provided. To use auth session please enable it in Intuned.json"
+      );
+    }
+  }
+
+  // Dynamically import the TypeScript file
+  const defaultExport = await getDefaultExportFromFile(apiFilePath);
+  const isDebug = process.execArgv.some((arg) => arg.includes("--inspect"));
+  if (isDebug) {
+    const { sourceFileName, lineNumber } = await getFirstLineNumber(
+      apiFilePath
+    );
+    sendBreakPointOnToClient(sourceFileName, lineNumber);
+  }
+
+  if (mode === "vanilla") {
+    // Execute the function with the input data
+    const result = await defaultExport(inputData);
+    return result;
+  }
+
+  const { page, context } = await getPlaywrightConstructsForMode(
+    mode,
+    options.cdpAddress,
+    authSessionPathToUse ? authSessionPathToUse : undefined
+  );
+
+  // wait for the page to get stable, this is a workaround for a bug in playwright when you try to do something
+  // on a page after connecting to it immediately, the page needs some time to get stable.
+  let isPageStable = false;
+  let retry = 0;
+  // will wait maximum 2 seconds before failing.
+  while (!isPageStable && retry < 10) {
+    try {
+      await page.locator("html").all();
+      isPageStable = true;
+      break;
+    } catch (error) {
+      if (error?.message?.includes("Execution context was destroyed")) {
+        await new Promise((resolve) => setTimeout(resolve, 200));
+      } else {
+        // throwing to make sure we do not hide any other unexpected issues.
+        throw error;
+      }
+    }
+    retry++;
+  }
+
+  if (!isPageStable) {
+    throw new Error("The page is not stable, try to run again.");
+  }
+
+  const result = await defaultExport(inputData, page, context);
+  const downloadsPath = getDownloadDirectoryPath();
+  await fs.remove(downloadsPath);
+  if (mode == "playwright") {
+    await context.close();
+  }
+  const isResponseObject = typeof result === "object" && result !== null;
+
+  const hasMoreThank5Keys = isResponseObject && Object.keys(result).length > 5;
+  const hasNestedObjects =
+    isResponseObject &&
+    Object.values(result).some((value) => typeof value === "object");
+
+  const shouldWriteToFile =
+    isResponseObject && (hasMoreThank5Keys || hasNestedObjects);
+
+  const resultsDir = "/tmp/run-results";
+
+  if (options.outputFileId && shouldWriteToFile) {
+    logger.info(
+      chalk.underline.bgBlue.white(
+        `Click to Open: Results saved (Run: ${options.outputFileId})`
+      )
+    );
+
+    fs.ensureDirSync(resultsDir);
+    const path = `${resultsDir}/${options.outputFileId}.json`;
+    await fs.writeJson(path, result, {
+      spaces: 2,
+    });
+  } else {
+    console.log("result:", result);
+  }
+
+  const payloadToAppend = getExecutionContext()?.extendedPayloads;
+  const hasPayloadToAppend = payloadToAppend && payloadToAppend.length > 0;
+
+  if (hasPayloadToAppend && options.outputFileId) {
+    logger.info(
+      chalk.underline.bgBlue.white(
+        `Click to Open: payloads to append (Run: ${options.outputFileId})`
+      )
+    );
+
+    fs.ensureDirSync(resultsDir);
+
+    const path = `${resultsDir}/${options.outputFileId}-payloads-to-append.json`;
+    await fs.writeJson(path, payloadToAppend, {
+      spaces: 2,
+    });
+  } else if (hasPayloadToAppend) {
+    logger.info("payload to append:", payloadToAppend);
+    logger.info(
+      "This will only take an effect if this API run was part of a job."
+    );
+  }
+}
+
+program
+  .description("run the user function in the cli for testing purposes")
+  .option("-i, --input [file]", "input json file")
+  .option("-j, --json [json]", "input json string")
+  .option("--cdpAddress <cdpAddress>", "CDP address", "http://localhost:9222")
+  .option(
+    "--authSessionPath <authSession>",
+    "auth session to use when executing the api"
+  )
+  .option(
+    "--outputFileId <outputFileId>",
+    "the output file id to save the result in"
+  )
+  .option("--proxy <proxy>", "proxy to use")
+  .argument("[apiName]", "name of the api", "default")
+  .allowUnknownOption()
+  .addArgument(
+    new Argument("<mode>", "mode of execution")
+      .choices([
+        "vanilla",
+        "playwright",
+        "playwright-standalone",
+        "playwright-headless",
+      ])
+      .default("playwright-standalone")
+      .argOptional()
+  )
+  .action(async (apiName, mode, options) => {
+    let inputData: null | object = null;
+
+    if (options.input) {
+      inputData = await fs.readJSON(options.input);
+    } else if (options.json) {
+      inputData = JSON.parse(options.json);
+    } else {
+      inputData = {};
+    }
+
+    await runWithContext(
+      {
+        runEnvironment: RunEnvironment.IDE,
+        extendedPayloads: [],
+        runId: nanoid(),
+        proxy: options.proxy,
+      },
+      () => executeCLI(apiName, mode, inputData as object, options)
+    );
+
+    // playwright keeps an open handle to the connected browser and it stops the process from exiting
+    // there is no way to close the connection gracefully without destroying the context/browser
+    // https://github.com/microsoft/playwright/issues/4956
+    process.exit(0);
+  });
+
+program.parse(process.argv);

package/src/commands/auth-sessions/load.ts

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+import { program } from "commander";
+import { getRemotePlaywrightContext } from "../../common/getPlaywrightConstructs";
+import { getSettings } from "../common/utils/settings";
+
+import dotenv from "dotenv";
+import { loadAuthSessionToContext } from "../../common/getPlaywrightConstructs";
+
+dotenv.config({
+  path: `.env`,
+});
+
+program
+  .description("load auth session to browser")
+  .option("--cdpAddress <cdpAddress>", "CDP address", "http://localhost:9222")
+  .option(
+    "--authSessionPath <authSession>",
+    "auth session to use when executing the api"
+  )
+  .allowUnknownOption()
+  .action(
+    async ({
+      cdpAddress,
+      authSessionPath,
+    }: {
+      cdpAddress: string;
+      authSessionPath: string;
+    }) => {
+      const setting = await getSettings();
+
+      if (!setting.authSessions.enabled) {
+        throw new Error("Auth session is not enabled");
+      }
+
+      const { context } = await getRemotePlaywrightContext(cdpAddress);
+      await loadAuthSessionToContext(context, authSessionPath);
+
+      process.exit(0);
+    }
+  );
+
+program.parse(process.argv);

package/src/commands/auth-sessions/run-check.ts

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+import { Argument, program } from "commander";
+import { getDefaultExportFromFile } from "../common/getDefaultExportFromFile";
+import * as fs from "fs-extra";
+import { getFullPathInProject } from "../common/utils/fileUtils";
+import { getSettings } from "../common/utils/settings";
+import dotenv from "dotenv";
+import { AUTH_SESSIONS_FOLDER_NAME } from "../../common/constants";
+import { getPlaywrightConstructsForMode } from "../../common/getPlaywrightConstructs";
+
+dotenv.config({
+  path: `.env`,
+});
+
+program
+  .description("run auth session check")
+  .option("--cdpAddress <cdpAddress>", "CDP address", "http://localhost:9222")
+  .option(
+    "--authSessionPath <authSession>",
+    "auth session to use when executing the check"
+  )
+  .allowUnknownOption()
+  .addArgument(
+    new Argument("<mode>", "mode of execution")
+      .choices(["vanilla", "playwright", "playwright-standalone"])
+      .default("playwright-standalone")
+      .argOptional()
+  )
+  .action(
+    async (
+      mode: "vanilla" | "playwright" | "playwright-standalone",
+      {
+        cdpAddress,
+        authSessionPath,
+      }: { cdpAddress: string; authSessionPath?: string }
+    ) => {
+      const setting = await getSettings();
+      if (!setting.authSessions.enabled) {
+        throw new Error("auth session is not enabled");
+      }
+
+      const checkFilePath = getFullPathInProject(
+        AUTH_SESSIONS_FOLDER_NAME,
+        "check.ts"
+      );
+
+      if (!fs.exists(checkFilePath)) {
+        throw new Error("auth session check file not found");
+      }
+
+      // Dynamically import the TypeScript file
+      const defaultExport = await getDefaultExportFromFile(checkFilePath);
+      const { page, context } = await getPlaywrightConstructsForMode(
+        mode,
+        cdpAddress,
+        authSessionPath
+      );
+
+      const result: boolean = await defaultExport(page, context);
+      console.log("check result", result);
+
+      if (!result) {
+        throw new Error("auth session check failed");
+      }
+
+      process.exit(0);
+    }
+  );
+
+program.parse(process.argv);

package/src/commands/auth-sessions/run-create.ts

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+import { Argument, program } from "commander";
+import { getDefaultExportFromFile } from "../common/getDefaultExportFromFile";
+import * as fs from "fs-extra";
+import { saveSession } from "../common/browserUtils";
+import { prompt } from "promptly";
+import { getFullPathInProject } from "../common/utils/fileUtils";
+import { getSettings } from "../common/utils/settings";
+import dotenv from "dotenv";
+import { AUTH_SESSIONS_FOLDER_NAME } from "../../common/constants";
+import { getPlaywrightConstructsForMode } from "../../common/getPlaywrightConstructs";
+
+dotenv.config({
+  path: `.env`,
+});
+
+program
+  .description("run auth session create")
+  .option("--cdpAddress <cdpAddress>", "CDP address", "http://localhost:9222")
+  .option("-i, --input [file]", "input json file")
+  .option("-j, --json [json]", "input json string")
+  .option(
+    "--authSessionPath <authSession>",
+    "auth session to use when executing the api"
+  )
+  .option(
+    "--pathToSave <pathToSave>",
+    "path to save the auth session, if not provided, will discard the auth session"
+  )
+  .allowUnknownOption()
+  .addArgument(
+    new Argument("<mode>", "mode of execution")
+      .choices(["vanilla", "playwright", "playwright-standalone"])
+      .default("playwright-standalone")
+      .argOptional()
+  )
+  .action(
+    async (
+      mode: "vanilla" | "playwright" | "playwright-standalone",
+      {
+        cdpAddress,
+        pathToSave,
+        input,
+        json,
+      }: {
+        cdpAddress: string;
+        pathToSave: string | undefined;
+        input: string;
+        json: string;
+      }
+    ) => {
+      let inputData: null | object = null;
+      if (input) {
+        inputData = await fs.readJSON(input);
+      } else if (json) {
+        inputData = JSON.parse(json);
+      } else {
+        inputData = {};
+      }
+
+      const setting = await getSettings();
+
+      if (!setting.authSessions.enabled) {
+        throw new Error("auth sessions feature is not enabled");
+      }
+
+      const createFilePath = getFullPathInProject(
+        AUTH_SESSIONS_FOLDER_NAME,
+        "create.ts"
+      );
+
+      if (!fs.exists(createFilePath)) {
+        throw new Error("auth session create file not found");
+      }
+
+      // Dynamically import the TypeScript file
+      const defaultExport = await getDefaultExportFromFile(createFilePath);
+      const { page, context } = await getPlaywrightConstructsForMode(
+        mode,
+        cdpAddress
+      );
+
+      const generator = await defaultExport(inputData, page, context, false);
+      let nextGeneratorParam: string | undefined = undefined;
+
+      // eslint-disable-next-line no-constant-condition
+      while (true) {
+        const { value, done } = await generator.next(nextGeneratorParam);
+
+        if (done) {
+          if (pathToSave) {
+            await saveSession(pathToSave, context);
+          }
+          break;
+        }
+
+        if (
+          value.action === "request_more_info" &&
+          value.requestType == "multiple_choice"
+        ) {
+          nextGeneratorParam = await prompt(
+            value.messageToUser + `, choices: ${value.choices}`,
+            {
+              validator: (input) => {
+                if (!value.choices.includes(input)) {
+                  throw new Error("Please type on of the allowed choices");
+                }
+                return input;
+              },
+            }
+          );
+        } else if (
+          value.action === "request_more_info" &&
+          value.requestType == "otp"
+        ) {
+          nextGeneratorParam = await prompt(value.messageToUser, {});
+        }
+      }
+
+      process.exit(0);
+    }
+  );
+
+program.parse(process.argv);