@intuned/runtime-dev 0.0.1-testing.0 → 0.0.1

package/src/common/getPlaywrightConstructs.ts

@@ -0,0 +1,283 @@
+import * as playwright from "@intuned/playwright-core";
+import { existsSync, mkdir, mkdtemp, rm, writeFile, readFile } from "fs-extra";
+import {
+  IntunedStorageState,
+  setContextStorageState,
+} from "./contextStorageStateHelpers";
+import { join } from "path";
+import path from "path";
+import type { Browser, Page } from "@intuned/playwright-core";
+import * as fs from "fs-extra";
+import { getFullPathInProject } from "../commands/common/utils/fileUtils";
+import waitOn from "wait-on";
+import { getDownloadDirectoryPath } from "../runtime";
+
+interface Proxy {
+  server: string;
+  username: string;
+  password: string;
+}
+
+interface GetPlaywrightConstructsOptions {
+  proxy?: Proxy;
+  headless?: boolean;
+  storageState?: IntunedStorageState;
+  downloadsPath: string;
+  userDataDir?: string;
+}
+
+async function createUserDirWithPreferences() {
+  const playwrightTempDir = await mkdtemp("/tmp/pw-");
+  const userDir = join(playwrightTempDir, "userdir");
+  const defaultDir = join(userDir, "Default");
+  await mkdir(defaultDir, {
+    recursive: true,
+  });
+  const preferences = {
+    plugins: {
+      always_open_pdf_externally: true,
+    },
+  };
+  await writeFile(join(defaultDir, "Preferences"), JSON.stringify(preferences));
+
+  return userDir;
+}
+
+export async function getProductionPlaywrightConstructs({
+  proxy,
+  headless = true,
+  storageState,
+  downloadsPath,
+}: GetPlaywrightConstructsOptions): Promise<{
+  page: playwright.Page;
+  context: playwright.BrowserContext;
+}> {
+  const extraArgs = [
+    "--no-first-run",
+    "--disable-sync",
+    "--disable-translate",
+    "--disable-features=TranslateUI",
+    "--disable-features=NetworkService",
+    "--lang=en",
+  ];
+
+  if (headless) {
+    extraArgs.push("--headless=new");
+  }
+
+  const executablePath = playwright.chromium.executablePath();
+  // use chromium 127 instead of 125, this depends on the docker image playwright version.
+  const chromium127Path = executablePath.replace(
+    "chromium-1117",
+    "chromium-1124"
+  );
+
+  const isChrome127There = existsSync(chromium127Path);
+
+  const userAgent = `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${
+    isChrome127There ? 127 : 125
+  }.0.0.0 Safari/537.36`;
+
+  const userDataDir = await createUserDirWithPreferences();
+
+  const context = await playwright.chromium.launchPersistentContext(
+    userDataDir,
+    {
+      headless,
+      ignoreDefaultArgs: [...getChromiumLaunchArgsToIgnore(), "--headless"],
+      proxy,
+      executablePath: isChrome127There ? chromium127Path : executablePath,
+      args: extraArgs,
+      downloadsPath,
+      userAgent,
+    }
+  );
+
+  context.once("close", async () => {
+    try {
+      await rm(userDataDir, {
+        recursive: true,
+        force: true,
+        retryDelay: 1000,
+        maxRetries: 5,
+      });
+    } catch (error) {
+      console.error("Failed to remove user data dir", error);
+    }
+  });
+
+  if (storageState) {
+    await setContextStorageState(context, storageState);
+  }
+
+  const assetsFile = path.join(__dirname, "./assets/browser_scripts.js");
+  await context.addInitScript({
+    path: assetsFile,
+  });
+
+  let page = context.pages().at(0);
+
+  if (page) {
+    const scriptString = await readFile(assetsFile, "utf8");
+    await page.evaluate(scriptString);
+  } else {
+    page = await context.newPage();
+  }
+
+  return {
+    page,
+    context,
+  };
+}
+
+const getChromiumLaunchArgsToIgnore = () => [
+  "--disable-field-trial-config",
+  "--disable-background-networking",
+  "--enable-features=NetworkService,NetworkServiceInProcess",
+  "--disable-background-timer-throttling",
+  "--disable-backgrounding-occluded-windows",
+  "--disable-back-forward-cache",
+  "--disable-breakpad",
+  "--disable-client-side-phishing-detection",
+  "--disable-component-extensions-with-background-pages",
+  "--disable-component-update",
+  "--no-default-browser-check",
+  "--disable-default-apps",
+  "--disable-dev-shm-usage",
+  "--disable-extensions",
+  "--disable-features=ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate,TranslateUI",
+  "--allow-pre-commit-input",
+  "--disable-hang-monitor",
+  "--disable-ipc-flooding-protection",
+  "--disable-prompt-on-repost",
+  "--disable-renderer-backgrounding",
+  "--force-color-profile=srgb",
+  "--metrics-recording-only",
+  "--no-first-run",
+  "--enable-automation",
+  "--password-store=basic",
+  "--use-mock-keychain",
+  "--no-service-autorun",
+  "--export-tagged-pdf",
+  "--enable-use-zoom-for-dsf=false",
+];
+
+export async function getPlaywrightConstructsForMode(
+  mode:
+    | "vanilla"
+    | "playwright"
+    | "playwright-standalone"
+    | "playwright-headless",
+  cdpAddress: string | undefined,
+  authSessionPath?: string
+): Promise<{
+  page: playwright.Page;
+  context: playwright.BrowserContext;
+}> {
+  if (mode == "playwright-standalone") {
+    if (!cdpAddress) {
+      throw new Error("cdpAddress is required");
+    }
+
+    const { context } = await getRemotePlaywrightContext(cdpAddress);
+    if (!context) {
+      throw new Error("no context found");
+    }
+
+    const assetsFile = path.join(__dirname, "./assets/browser_scripts.js");
+    await context.addInitScript({
+      path: assetsFile,
+    });
+
+    const pages = await context.pages();
+    let page: Page | null = null;
+    if (pages.length > 0) {
+      page = pages[0];
+      const scriptString = await fs.readFile(assetsFile, "utf8");
+      await page.evaluate(scriptString);
+    } else {
+      page = await context.newPage();
+    }
+
+    if (authSessionPath) {
+      await loadAuthSessionToContext(context, authSessionPath);
+    }
+
+    return {
+      page,
+      context,
+    };
+  }
+
+  const downloadsPath = getDownloadDirectoryPath();
+  if (mode === "playwright" || mode === "playwright-headless") {
+    const productionConstructs = await getProductionPlaywrightConstructs({
+      headless: mode === "playwright-headless",
+      downloadsPath,
+    });
+
+    if (authSessionPath) {
+      await loadAuthSessionToContext(
+        productionConstructs.context,
+        authSessionPath
+      );
+    }
+
+    return {
+      ...productionConstructs,
+    };
+  }
+
+  throw "invalid mode";
+}
+
+export async function loadAuthSessionToContext(
+  context: playwright.BrowserContext,
+  authSessionPath: string
+) {
+  const fullPath = getFullPathInProject(authSessionPath);
+  const data = await fs.readJson(fullPath);
+
+  await setContextStorageState(context, data);
+}
+
+export async function getRemotePlaywrightContext(cdpAddress: string) {
+  const playwright = await import("@intuned/playwright-core");
+  let browser: Browser | null = null;
+
+  if (!cdpAddress) {
+    throw new Error("cdpAddress is required");
+  }
+
+  const cdpAddressWithoutProtocol = cdpAddress
+    .replace("http://", "")
+    .replace("https://", "")
+    .replace("localhost", "127.0.0.1");
+
+  try {
+    await waitOn({
+      resources: [`http-get://${cdpAddressWithoutProtocol}/json/version`],
+      delay: 100,
+      interval: 100,
+      timeout: 5000,
+      tcpTimeout: 1000,
+      window: 1000,
+    });
+  } catch (error) {
+    console.error("Failed to connect to the browser");
+    // 128 is the exit code for SIGINT https://tldp.org/LDP/abs/html/exitcodes.html
+    // 9 is the SIGKILL signal
+    process.exit(128 + 9);
+  }
+
+  try {
+    browser = await playwright.chromium.connectOverCDP(cdpAddress);
+  } catch (e) {
+    console.log(e);
+    throw new Error("failed to connect to the browser");
+  }
+
+  const context = browser.contexts()[0];
+
+  return { browser, context };
+}

package/src/common/jwtTokenManager.ts

@@ -0,0 +1,111 @@
+import fetch from "cross-fetch";
+import * as jwt from "jsonwebtoken";
+import { err, ResultAsync } from "neverthrow";
+
+// Manages JWT token with renewal
+class JwtTokenManager {
+  private _token: string | undefined;
+  private tokenRefreshTimeout: NodeJS.Timeout | undefined;
+  private refreshTokenPath: string;
+
+  constructor(refreshTokenPath: string) {
+    this.refreshTokenPath = refreshTokenPath;
+    this._token = undefined;
+  }
+
+  public get token() {
+    return this._token;
+  }
+
+  // When the token is set, the schedule for renewal is issued automatically
+  // This is currently being set it two places:
+  // 1. Whenever the runner starts, initializes it from the environment variable (set whenever the api is run from the authoring IDE)
+  // 2. Whenever a published api is called to run (/api/run/*), it is set to the token received in the run request.
+  public set token(newToken: string | undefined) {
+    if (this._token != newToken) {
+      this._token = newToken;
+      void this.scheduleTokenRefresh();
+    }
+  }
+
+  private get timeToRefresh() {
+    if (!this._token) return;
+
+    const payload = jwt.decode(this._token);
+
+    if (!payload || typeof payload == "string") return;
+
+    const expiry = payload.expiry;
+    if (!expiry || typeof expiry !== "number") return;
+
+    const timeWindow = 60 * 1000; // 1 minute
+
+    const timeToRefresh = expiry - Date.now() - timeWindow;
+    return Math.max(timeToRefresh, timeWindow);
+  }
+
+  private async scheduleTokenRefresh() {
+    if (process.env.RUN_ENVIRONMENT?.toLowerCase() !== "authoring") return;
+    const timeToRefresh = this.timeToRefresh;
+    if (timeToRefresh === undefined) return;
+
+    if (this.tokenRefreshTimeout) clearTimeout(this.tokenRefreshTimeout);
+
+    this.tokenRefreshTimeout = setTimeout(async () => {
+      const result = await this.refreshToken();
+      if (result && result.isErr()) {
+        console.error(`[Internal Error] ${result.error}`);
+        return;
+      }
+      await this.scheduleTokenRefresh();
+    }, timeToRefresh);
+  }
+
+  private async refreshToken() {
+    if (process.env.RUN_ENVIRONMENT?.toLowerCase() !== "authoring") return;
+    const res = await this.fetchWithToken(this.refreshTokenPath, {
+      method: "GET",
+    });
+
+    if (res.status === 401) {
+      return err("Unauthorized");
+    }
+
+    const jsonResult = await ResultAsync.fromPromise(
+      res.json(),
+      () => "not json"
+    );
+
+    if (jsonResult.isErr()) return;
+
+    const newToken = jsonResult.value.token;
+    if (newToken) this._token = newToken;
+  }
+
+  public fetchWithToken(...[input, init]: Parameters<typeof fetch>) {
+    const headers = new Headers(init?.headers);
+    headers.set("Authorization", `Bearer ${this.token}`);
+    return fetch(input, {
+      ...init,
+      headers,
+    });
+  }
+}
+
+const backendFunctionsPath = `${process.env.FUNCTIONS_DOMAIN}/api/${process.env.INTUNED_WORKSPACE_ID}/functions/${process.env.INTUNED_INTEGRATION_ID}`;
+
+export const backendFunctionsTokenManager = new JwtTokenManager(
+  `${backendFunctionsPath}/refreshBackendFunctionsToken`
+);
+backendFunctionsTokenManager.token =
+  process.env.INTUNED_AUTHORING_SESSION_BACKEND_FUNCTIONS_TOKEN;
+
+export function callBackendFunctionWithToken(
+  path: string,
+  init?: Parameters<typeof fetch>[1]
+) {
+  return backendFunctionsTokenManager.fetchWithToken(
+    `${backendFunctionsPath}/${path}`,
+    init
+  );
+}

package/src/common/settingsSchema.ts

@@ -0,0 +1,16 @@
+import * as z from "zod";
+
+const authSessionsSchema = z
+  .object({
+    enabled: z.boolean(),
+  })
+  .optional()
+  .default({
+    enabled: false,
+  });
+
+export const settingsSchema = z.object({
+  authSessions: authSessionsSchema,
+});
+
+export type IntunedSettings = z.infer<typeof settingsSchema>;

package/src/common/telemetry.ts

@@ -0,0 +1,49 @@
+import * as appInsights from "applicationinsights";
+
+function gracefulShutdown() {
+  console.log("Shutting down, so flushing app insights.");
+
+  appInsights.defaultClient.flush();
+
+  process.exit();
+}
+
+export function initializeAppInsights() {
+  const appInsightsConnectionString = process.env.APPINSIGHTS_CONNECTION_STRING;
+
+  if (appInsightsConnectionString) {
+    console.log("Initializing app insights.");
+
+    appInsights
+      .setup(appInsightsConnectionString)
+      .setAutoCollectConsole(true, true)
+      .setAutoCollectDependencies(true)
+      .setAutoCollectExceptions(true)
+      .setAutoCollectHeartbeat(true)
+      .setAutoCollectPerformance(true, true)
+      .setAutoCollectRequests(true)
+      .setAutoDependencyCorrelation(true)
+      .setDistributedTracingMode(appInsights.DistributedTracingModes.AI_AND_W3C)
+      .setSendLiveMetrics(true)
+      .setUseDiskRetryCaching(true);
+
+    appInsights.defaultClient.context.tags[
+      appInsights.defaultClient.context.keys.cloudRole
+    ] = "@intuned/sdk";
+
+    if (process.env.FLY_APP_NAME) {
+      appInsights.defaultClient.context.tags[
+        appInsights.defaultClient.context.keys.cloudRoleInstance
+      ] = process.env.FLY_APP_NAME;
+    }
+
+    process.on("SIGTERM", gracefulShutdown);
+    process.on("SIGINT", gracefulShutdown);
+
+    appInsights.start();
+  }
+}
+
+export function getTelemetryClient(): appInsights.TelemetryClient | undefined {
+  return appInsights.defaultClient;
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+export {
+  extendPayload,
+  extendTimeout,
+  runInfo,
+  RunError,
+  requestMultipleChoice,
+  requestOTP,
+} from "./runtime";
+export {
+  runWithContext,
+  getExecutionContext,
+} from "./common/asyncLocalStorage";
+export { getDownloadDirectoryPath } from "./runtime/downloadDirectory";
+export { getProductionPlaywrightConstructs } from "./common/getPlaywrightConstructs";

package/src/runtime/RunError.ts

@@ -0,0 +1,16 @@
+import { RunErrorOptions } from "./export";
+
+export class RunError extends Error {
+  options: RunErrorOptions;
+  constructor(message: string, options?: RunErrorOptions) {
+    super(message);
+    this.message = message;
+    this.name = "USER_GENERATED_ERROR";
+    this.options = options ?? {
+      retryable: false,
+    };
+    Object.setPrototypeOf(this, RunError.prototype);
+  }
+}
+
+new RunError("", {});

package/src/runtime/downloadDirectory.ts

@@ -0,0 +1,14 @@
+import { getExecutionContext } from "..";
+import { ensureDirSync } from "fs-extra";
+
+export function getDownloadDirectoryPath() {
+  const context = getExecutionContext();
+  if (!context) {
+    throw new Error("ExecutionContext not found");
+  }
+  const path = `/tmp/downloads/${context.runId}`;
+  ensureDirSync(path, {
+    mode: 0o2775,
+  });
+  return path;
+}

package/src/runtime/enums.ts

@@ -0,0 +1,11 @@
+export enum RunEnvironment {
+  IDE = "IDE",
+  DEPLOYED = "DEPLOYED",
+}
+
+export enum RunType {
+  SYNC = "SYNC",
+  ASYNC = "ASYNC",
+  JOB = "JOB",
+  QUEUE = "QUEUE",
+}

package/src/runtime/executionHelpers.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from "vitest";
+import {
+  getExecutionContext,
+  runWithContext,
+} from "../common/asyncLocalStorage";
+import { RunError, extendPayload, runInfo } from "./index";
+import { RunEnvironment, RunType } from "./enums";
+
+describe("Execution Helpers", () => {
+  it("should be able to get execution info", () => {
+    runWithContext(
+      {
+        runEnvironment: RunEnvironment.IDE,
+        // runType: RunType.SYNC,
+        runId: "test-run-id",
+        extendedPayloads: [],
+      },
+      () => {
+        expect(runInfo().runId).toEqual("test-run-id");
+      }
+    );
+  });
+
+  it("should be able to mutate extendedPayloads and get the accmulated value at the end ", () => {
+    const context = {
+      extendedPayloads: [],
+      runId: "test-run-id",
+    };
+
+    runWithContext(
+      {
+        runEnvironment: RunEnvironment.IDE,
+        // runType: RunType.SYNC,
+        runId: "test-run-id",
+        extendedPayloads: [],
+      },
+      () => {
+        expect(getExecutionContext()?.extendedPayloads).toEqual([]);
+        extendPayload({
+          api: "test-api",
+          parameters: {},
+        });
+
+        expect(getExecutionContext()?.extendedPayloads).toEqual([
+          {
+            api: "test-api",
+            parameters: {},
+          },
+        ]);
+
+        extendPayload({
+          api: "test-api",
+          parameters: {},
+        });
+        extendPayload({
+          api: "test-api",
+          parameters: {},
+        });
+        extendPayload({
+          api: "test-api",
+          parameters: {},
+        });
+
+        const context = getExecutionContext();
+
+        expect(context?.extendedPayloads).toHaveLength(4);
+      }
+    );
+  });
+});

package/src/runtime/extendPayload.ts

@@ -0,0 +1,22 @@
+import { getExecutionContext } from "../common/asyncLocalStorage";
+import { Payload } from "./export";
+import { extendTimeout } from "./extendTimeout";
+
+export function extendPayload(payload: Payload | Payload[]) {
+  const context = getExecutionContext();
+
+  if (!context) {
+    throw new Error("extendPayload failed due to an internal error.");
+  }
+
+  const items = Array.isArray(payload) ? payload : [payload];
+
+  if (!context.extendedPayloads) {
+    context.extendedPayloads = [...items];
+    return;
+  }
+
+  context?.extendedPayloads.push(...items);
+
+  extendTimeout();
+}

package/src/runtime/extendTimeout.ts

@@ -0,0 +1,32 @@
+import { getExecutionContext } from "../common/asyncLocalStorage";
+
+const _DEBOUNCE_TIME = 60_000;
+
+export function extendTimeout() {
+  const context = getExecutionContext();
+
+  if (!context) {
+    throw new Error("extendTimeout failed due to an internal error.");
+  }
+
+  const { timeoutInfo } = context;
+
+  if (
+    !timeoutInfo ||
+    timeoutInfo.timeoutTimestamp === undefined ||
+    timeoutInfo.timeoutDuration === undefined ||
+    timeoutInfo.extendTimeoutCallback === undefined
+  ) {
+    return;
+  }
+
+  const newTimeoutStamp = Date.now() + timeoutInfo.timeoutDuration;
+
+  if (newTimeoutStamp - timeoutInfo.timeoutTimestamp < _DEBOUNCE_TIME) {
+    return;
+  }
+
+  timeoutInfo.timeoutTimestamp = newTimeoutStamp;
+
+  void timeoutInfo.extendTimeoutCallback().catch(() => undefined);
+}

package/{dist/runtime/index.d.ts → src/runtime/index.ts}

@@ -2,5 +2,7 @@ export { extendPayload } from "./extendPayload";
 export { extendTimeout } from "./extendTimeout";
 export { runInfo } from "./runInfo";
 export { RunError } from "./RunError";
+
 export { requestMultipleChoice, requestOTP } from "./requestMoreInfo";
-export { getDownloadDirectoryPath } from "./downloadDirectory";
+
+export {getDownloadDirectoryPath} from "./downloadDirectory";

package/src/runtime/requestMoreInfo.ts

@@ -0,0 +1,40 @@
+interface RequestMoreInfoReturnTypeBase {
+  [REQUEST_MORE_INFO_KEY]: true;
+  action: "request_more_info";
+}
+
+interface RequestMultipleChoiceReturnType
+  extends RequestMoreInfoReturnTypeBase {
+  messageToUser: string;
+  choices: string[];
+  requestType: "multiple_choice";
+}
+
+interface RequestOtpReturnType extends RequestMoreInfoReturnTypeBase {
+  messageToUser: string;
+  requestType: "otp";
+}
+
+const REQUEST_MORE_INFO_KEY: unique symbol = Symbol("REQUEST_MORE_INFO");
+
+export function requestOTP(messageToUser: string): RequestOtpReturnType {
+  return {
+    [REQUEST_MORE_INFO_KEY]: true,
+    action: "request_more_info",
+    messageToUser,
+    requestType: "otp",
+  };
+}
+
+export function requestMultipleChoice(
+  messageToUser: string,
+  choices: string[]
+): RequestMultipleChoiceReturnType {
+  return {
+    [REQUEST_MORE_INFO_KEY]: true,
+    action: "request_more_info",
+    messageToUser,
+    requestType: "multiple_choice",
+    choices,
+  };
+}