@intuned/runtime-dev 0.0.1-split.0

package/src/common/asyncLocalStorage/index.ts

@@ -0,0 +1,29 @@
+import { AsyncLocalStorage } from "async_hooks";
+import { Payload, RunInfo } from "../../runtime/export";
+
+export const asyncLocalStorage = new AsyncLocalStorage<InternalRunInfo>();
+
+export function runWithContext<R, TArgs extends any[]>(
+  contextData: InternalRunInfo,
+  callback: (...args: TArgs) => R,
+  ...args: TArgs
+): R {
+  return asyncLocalStorage.run(contextData, callback, ...args);
+}
+
+interface TimeoutInfo {
+  extendTimeoutCallback?: () => Promise<void>;
+  timeoutDuration?: number;
+  timeoutTimestamp?: number;
+}
+
+interface InternalRunInfo extends RunInfo {
+  extendedPayloads: Payload[];
+  timeoutInfo?: TimeoutInfo;
+}
+
+export function getExecutionContext() {
+  const contextData = asyncLocalStorage.getStore();
+
+  return contextData as InternalRunInfo | undefined;
+}

package/src/common/cleanEnvironmentVariables.ts

@@ -0,0 +1,13 @@
+export function cleanEnvironmentVariables() {
+  Object.keys(process.env)
+    .filter((i) => {
+      if (i.toLocaleLowerCase().startsWith("npm")) {
+        return true;
+      }
+
+      if (i.toLocaleLowerCase().startsWith("fly") && i !== "FLY_ALLOC_ID") {
+        return true;
+      }
+    })
+    .forEach((i) => delete process.env[i]);
+}

package/src/common/constants.ts

@@ -0,0 +1 @@
+export const AUTH_SESSIONS_FOLDER_NAME = "auth-sessions";

package/src/common/contextStorageStateHelpers.ts

@@ -0,0 +1,71 @@
+import { BrowserContext } from "@intuned/playwright";
+import * as playwright from "@intuned/playwright-core";
+
+function sessionStorageToArray(
+  storage: Record<string, string>
+): { name: string; value: string }[] {
+  const result: { name: string; value: string }[] = [];
+
+  for (const key in storage) {
+    result.push({ name: key, value: storage[key] });
+  }
+
+  return result;
+}
+
+interface OriginSessionStorage {
+  origin: string;
+  sessionStorage: Array<{
+    name: string;
+    value: string;
+  }>;
+}
+
+export type IntunedStorageState = Exclude<
+  playwright.BrowserContextOptions["storageState"],
+  string
+> & {
+  sessionStorage?: OriginSessionStorage[];
+};
+
+export async function getContextStorageState(
+  context: BrowserContext
+): Promise<IntunedStorageState> {
+  const cookiesAndLocalStorage = await context.storageState();
+  const pages = await context.pages();
+
+  const pagesSessions: OriginSessionStorage[] = [];
+  for (const page of pages) {
+    const [origin, session] = await page.evaluate(() => [
+      location.origin,
+      sessionStorage as Record<string, string>,
+    ]);
+
+    const transformed = sessionStorageToArray(session);
+
+    pagesSessions.push({ origin, sessionStorage: transformed });
+  }
+
+  return {
+    ...cookiesAndLocalStorage,
+    sessionStorage: pagesSessions,
+  };
+}
+
+export async function setContextStorageState(
+  context: BrowserContext,
+  storage: IntunedStorageState
+) {
+  await (context as any).intunedSetStorageState(storage);
+
+  const sessionStorage = storage.sessionStorage;
+
+  await context.addInitScript((storage) => {
+    for (const { origin, sessionStorage } of storage) {
+      if (window.location.origin === origin) {
+        for (const item of sessionStorage)
+          window.sessionStorage.setItem(item.name, item.value);
+      }
+    }
+  }, sessionStorage);
+}

package/src/common/getPlaywrightConstructs.ts

@@ -0,0 +1,283 @@
+import * as playwright from "@intuned/playwright-core";
+import { existsSync, mkdir, mkdtemp, rm, writeFile, readFile } from "fs-extra";
+import {
+  IntunedStorageState,
+  setContextStorageState,
+} from "./contextStorageStateHelpers";
+import { join } from "path";
+import path from "path";
+import type { Browser, Page } from "@intuned/playwright-core";
+import * as fs from "fs-extra";
+import { getFullPathInProject } from "../commands/common/utils/fileUtils";
+import waitOn from "wait-on";
+import { getDownloadDirectoryPath } from "../runtime";
+
+interface Proxy {
+  server: string;
+  username: string;
+  password: string;
+}
+
+interface GetPlaywrightConstructsOptions {
+  proxy?: Proxy;
+  headless?: boolean;
+  storageState?: IntunedStorageState;
+  downloadsPath: string;
+  userDataDir?: string;
+}
+
+async function createUserDirWithPreferences() {
+  const playwrightTempDir = await mkdtemp("/tmp/pw-");
+  const userDir = join(playwrightTempDir, "userdir");
+  const defaultDir = join(userDir, "Default");
+  await mkdir(defaultDir, {
+    recursive: true,
+  });
+  const preferences = {
+    plugins: {
+      always_open_pdf_externally: true,
+    },
+  };
+  await writeFile(join(defaultDir, "Preferences"), JSON.stringify(preferences));
+
+  return userDir;
+}
+
+export async function getProductionPlaywrightConstructs({
+  proxy,
+  headless = true,
+  storageState,
+  downloadsPath,
+}: GetPlaywrightConstructsOptions): Promise<{
+  page: playwright.Page;
+  context: playwright.BrowserContext;
+}> {
+  const extraArgs = [
+    "--no-first-run",
+    "--disable-sync",
+    "--disable-translate",
+    "--disable-features=TranslateUI",
+    "--disable-features=NetworkService",
+    "--lang=en",
+  ];
+
+  if (headless) {
+    extraArgs.push("--headless=new");
+  }
+
+  const executablePath = playwright.chromium.executablePath();
+  // use chromium 127 instead of 125, this depends on the docker image playwright version.
+  const chromium127Path = executablePath.replace(
+    "chromium-1117",
+    "chromium-1124"
+  );
+
+  const isChrome127There = existsSync(chromium127Path);
+
+  const userAgent = `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${
+    isChrome127There ? 127 : 125
+  }.0.0.0 Safari/537.36`;
+
+  const userDataDir = await createUserDirWithPreferences();
+
+  const context = await playwright.chromium.launchPersistentContext(
+    userDataDir,
+    {
+      headless,
+      ignoreDefaultArgs: [...getChromiumLaunchArgsToIgnore(), "--headless"],
+      proxy,
+      executablePath: isChrome127There ? chromium127Path : executablePath,
+      args: extraArgs,
+      downloadsPath,
+      userAgent,
+    }
+  );
+
+  context.once("close", async () => {
+    try {
+      await rm(userDataDir, {
+        recursive: true,
+        force: true,
+        retryDelay: 1000,
+        maxRetries: 5,
+      });
+    } catch (error) {
+      console.error("Failed to remove user data dir", error);
+    }
+  });
+
+  if (storageState) {
+    await setContextStorageState(context, storageState);
+  }
+
+  const assetsFile = path.join(__dirname, "./assets/browser_scripts.js");
+  await context.addInitScript({
+    path: assetsFile,
+  });
+
+  let page = context.pages().at(0);
+
+  if (page) {
+    const scriptString = await readFile(assetsFile, "utf8");
+    await page.evaluate(scriptString);
+  } else {
+    page = await context.newPage();
+  }
+
+  return {
+    page,
+    context,
+  };
+}
+
+const getChromiumLaunchArgsToIgnore = () => [
+  "--disable-field-trial-config",
+  "--disable-background-networking",
+  "--enable-features=NetworkService,NetworkServiceInProcess",
+  "--disable-background-timer-throttling",
+  "--disable-backgrounding-occluded-windows",
+  "--disable-back-forward-cache",
+  "--disable-breakpad",
+  "--disable-client-side-phishing-detection",
+  "--disable-component-extensions-with-background-pages",
+  "--disable-component-update",
+  "--no-default-browser-check",
+  "--disable-default-apps",
+  "--disable-dev-shm-usage",
+  "--disable-extensions",
+  "--disable-features=ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate,TranslateUI",
+  "--allow-pre-commit-input",
+  "--disable-hang-monitor",
+  "--disable-ipc-flooding-protection",
+  "--disable-prompt-on-repost",
+  "--disable-renderer-backgrounding",
+  "--force-color-profile=srgb",
+  "--metrics-recording-only",
+  "--no-first-run",
+  "--enable-automation",
+  "--password-store=basic",
+  "--use-mock-keychain",
+  "--no-service-autorun",
+  "--export-tagged-pdf",
+  "--enable-use-zoom-for-dsf=false",
+];
+
+export async function getPlaywrightConstructsForMode(
+  mode:
+    | "vanilla"
+    | "playwright"
+    | "playwright-standalone"
+    | "playwright-headless",
+  cdpAddress: string | undefined,
+  authSessionPath?: string
+): Promise<{
+  page: playwright.Page;
+  context: playwright.BrowserContext;
+}> {
+  if (mode == "playwright-standalone") {
+    if (!cdpAddress) {
+      throw new Error("cdpAddress is required");
+    }
+
+    const { context } = await getRemotePlaywrightContext(cdpAddress);
+    if (!context) {
+      throw new Error("no context found");
+    }
+
+    const assetsFile = path.join(__dirname, "./assets/browser_scripts.js");
+    await context.addInitScript({
+      path: assetsFile,
+    });
+
+    const pages = await context.pages();
+    let page: Page | null = null;
+    if (pages.length > 0) {
+      page = pages[0];
+      const scriptString = await fs.readFile(assetsFile, "utf8");
+      await page.evaluate(scriptString);
+    } else {
+      page = await context.newPage();
+    }
+
+    if (authSessionPath) {
+      await loadAuthSessionToContext(context, authSessionPath);
+    }
+
+    return {
+      page,
+      context,
+    };
+  }
+
+  const downloadsPath = getDownloadDirectoryPath();
+  if (mode === "playwright" || mode === "playwright-headless") {
+    const productionConstructs = await getProductionPlaywrightConstructs({
+      headless: mode === "playwright-headless",
+      downloadsPath,
+    });
+
+    if (authSessionPath) {
+      await loadAuthSessionToContext(
+        productionConstructs.context,
+        authSessionPath
+      );
+    }
+
+    return {
+      ...productionConstructs,
+    };
+  }
+
+  throw "invalid mode";
+}
+
+export async function loadAuthSessionToContext(
+  context: playwright.BrowserContext,
+  authSessionPath: string
+) {
+  const fullPath = getFullPathInProject(authSessionPath);
+  const data = await fs.readJson(fullPath);
+
+  await setContextStorageState(context, data);
+}
+
+export async function getRemotePlaywrightContext(cdpAddress: string) {
+  const playwright = await import("@intuned/playwright-core");
+  let browser: Browser | null = null;
+
+  if (!cdpAddress) {
+    throw new Error("cdpAddress is required");
+  }
+
+  const cdpAddressWithoutProtocol = cdpAddress
+    .replace("http://", "")
+    .replace("https://", "")
+    .replace("localhost", "127.0.0.1");
+
+  try {
+    await waitOn({
+      resources: [`http-get://${cdpAddressWithoutProtocol}/json/version`],
+      delay: 100,
+      interval: 100,
+      timeout: 5000,
+      tcpTimeout: 1000,
+      window: 1000,
+    });
+  } catch (error) {
+    console.error("Failed to connect to the browser");
+    // 128 is the exit code for SIGINT https://tldp.org/LDP/abs/html/exitcodes.html
+    // 9 is the SIGKILL signal
+    process.exit(128 + 9);
+  }
+
+  try {
+    browser = await playwright.chromium.connectOverCDP(cdpAddress);
+  } catch (e) {
+    console.log(e);
+    throw new Error("failed to connect to the browser");
+  }
+
+  const context = browser.contexts()[0];
+
+  return { browser, context };
+}

package/src/common/jwtTokenManager.ts

@@ -0,0 +1,111 @@
+import fetch from "cross-fetch";
+import * as jwt from "jsonwebtoken";
+import { err, ResultAsync } from "neverthrow";
+
+// Manages JWT token with renewal
+class JwtTokenManager {
+  private _token: string | undefined;
+  private tokenRefreshTimeout: NodeJS.Timeout | undefined;
+  private refreshTokenPath: string;
+
+  constructor(refreshTokenPath: string) {
+    this.refreshTokenPath = refreshTokenPath;
+    this._token = undefined;
+  }
+
+  public get token() {
+    return this._token;
+  }
+
+  // When the token is set, the schedule for renewal is issued automatically
+  // This is currently being set it two places:
+  // 1. Whenever the runner starts, initializes it from the environment variable (set whenever the api is run from the authoring IDE)
+  // 2. Whenever a published api is called to run (/api/run/*), it is set to the token received in the run request.
+  public set token(newToken: string | undefined) {
+    if (this._token != newToken) {
+      this._token = newToken;
+      void this.scheduleTokenRefresh();
+    }
+  }
+
+  private get timeToRefresh() {
+    if (!this._token) return;
+
+    const payload = jwt.decode(this._token);
+
+    if (!payload || typeof payload == "string") return;
+
+    const expiry = payload.expiry;
+    if (!expiry || typeof expiry !== "number") return;
+
+    const timeWindow = 60 * 1000; // 1 minute
+
+    const timeToRefresh = expiry - Date.now() - timeWindow;
+    return Math.max(timeToRefresh, timeWindow);
+  }
+
+  private async scheduleTokenRefresh() {
+    if (process.env.RUN_ENVIRONMENT?.toLowerCase() !== "authoring") return;
+    const timeToRefresh = this.timeToRefresh;
+    if (timeToRefresh === undefined) return;
+
+    if (this.tokenRefreshTimeout) clearTimeout(this.tokenRefreshTimeout);
+
+    this.tokenRefreshTimeout = setTimeout(async () => {
+      const result = await this.refreshToken();
+      if (result && result.isErr()) {
+        console.error(`[Internal Error] ${result.error}`);
+        return;
+      }
+      await this.scheduleTokenRefresh();
+    }, timeToRefresh);
+  }
+
+  private async refreshToken() {
+    if (process.env.RUN_ENVIRONMENT?.toLowerCase() !== "authoring") return;
+    const res = await this.fetchWithToken(this.refreshTokenPath, {
+      method: "GET",
+    });
+
+    if (res.status === 401) {
+      return err("Unauthorized");
+    }
+
+    const jsonResult = await ResultAsync.fromPromise(
+      res.json(),
+      () => "not json"
+    );
+
+    if (jsonResult.isErr()) return;
+
+    const newToken = jsonResult.value.token;
+    if (newToken) this._token = newToken;
+  }
+
+  public fetchWithToken(...[input, init]: Parameters<typeof fetch>) {
+    const headers = new Headers(init?.headers);
+    headers.set("Authorization", `Bearer ${this.token}`);
+    return fetch(input, {
+      ...init,
+      headers,
+    });
+  }
+}
+
+const backendFunctionsPath = `${process.env.FUNCTIONS_DOMAIN}/api/${process.env.INTUNED_WORKSPACE_ID}/functions/${process.env.INTUNED_INTEGRATION_ID}`;
+
+export const backendFunctionsTokenManager = new JwtTokenManager(
+  `${backendFunctionsPath}/refreshBackendFunctionsToken`
+);
+backendFunctionsTokenManager.token =
+  process.env.INTUNED_AUTHORING_SESSION_BACKEND_FUNCTIONS_TOKEN;
+
+export function callBackendFunctionWithToken(
+  path: string,
+  init?: Parameters<typeof fetch>[1]
+) {
+  return backendFunctionsTokenManager.fetchWithToken(
+    `${backendFunctionsPath}/${path}`,
+    init
+  );
+}

package/src/common/settingsSchema.ts

@@ -0,0 +1,16 @@
+import * as z from "zod";
+
+const authSessionsSchema = z
+  .object({
+    enabled: z.boolean(),
+  })
+  .optional()
+  .default({
+    enabled: false,
+  });
+
+export const settingsSchema = z.object({
+  authSessions: authSessionsSchema,
+});
+
+export type IntunedSettings = z.infer<typeof settingsSchema>;

package/src/common/telemetry.ts

@@ -0,0 +1,49 @@
+import * as appInsights from "applicationinsights";
+
+function gracefulShutdown() {
+  console.log("Shutting down, so flushing app insights.");
+
+  appInsights.defaultClient.flush();
+
+  process.exit();
+}
+
+export function initializeAppInsights() {
+  const appInsightsConnectionString = process.env.APPINSIGHTS_CONNECTION_STRING;
+
+  if (appInsightsConnectionString) {
+    console.log("Initializing app insights.");
+
+    appInsights
+      .setup(appInsightsConnectionString)
+      .setAutoCollectConsole(true, true)
+      .setAutoCollectDependencies(true)
+      .setAutoCollectExceptions(true)
+      .setAutoCollectHeartbeat(true)
+      .setAutoCollectPerformance(true, true)
+      .setAutoCollectRequests(true)
+      .setAutoDependencyCorrelation(true)
+      .setDistributedTracingMode(appInsights.DistributedTracingModes.AI_AND_W3C)
+      .setSendLiveMetrics(true)
+      .setUseDiskRetryCaching(true);
+
+    appInsights.defaultClient.context.tags[
+      appInsights.defaultClient.context.keys.cloudRole
+    ] = "@intuned/sdk";
+
+    if (process.env.FLY_APP_NAME) {
+      appInsights.defaultClient.context.tags[
+        appInsights.defaultClient.context.keys.cloudRoleInstance
+      ] = process.env.FLY_APP_NAME;
+    }
+
+    process.on("SIGTERM", gracefulShutdown);
+    process.on("SIGINT", gracefulShutdown);
+
+    appInsights.start();
+  }
+}
+
+export function getTelemetryClient(): appInsights.TelemetryClient | undefined {
+  return appInsights.defaultClient;
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+export {
+  extendPayload,
+  extendTimeout,
+  runInfo,
+  RunError,
+  requestMultipleChoice,
+  requestOTP,
+} from "./runtime";
+export {
+  runWithContext,
+  getExecutionContext,
+} from "./common/asyncLocalStorage";
+export { getDownloadDirectoryPath } from "./runtime/downloadDirectory";
+export { getProductionPlaywrightConstructs } from "./common/getPlaywrightConstructs";

package/src/runtime/RunError.ts

@@ -0,0 +1,16 @@
+import { RunErrorOptions } from "./export";
+
+export class RunError extends Error {
+  options: RunErrorOptions;
+  constructor(message: string, options?: RunErrorOptions) {
+    super(message);
+    this.message = message;
+    this.name = "USER_GENERATED_ERROR";
+    this.options = options ?? {
+      retryable: false,
+    };
+    Object.setPrototypeOf(this, RunError.prototype);
+  }
+}
+
+new RunError("", {});

package/src/runtime/downloadDirectory.ts

@@ -0,0 +1,14 @@
+import { getExecutionContext } from "..";
+import { ensureDirSync } from "fs-extra";
+
+export function getDownloadDirectoryPath() {
+  const context = getExecutionContext();
+  if (!context) {
+    throw new Error("ExecutionContext not found");
+  }
+  const path = `/tmp/downloads/${context.runId}`;
+  ensureDirSync(path, {
+    mode: 0o2775,
+  });
+  return path;
+}

package/src/runtime/enums.d.ts

@@ -0,0 +1,11 @@
+export declare enum RunEnvironment {
+  IDE = "IDE",
+  DEPLOYED = "DEPLOYED",
+}
+
+export declare enum RunType {
+  SYNC = "SYNC",
+  ASYNC = "ASYNC",
+  JOB = "JOB",
+  QUEUE = "QUEUE",
+}

package/src/runtime/enums.ts

@@ -0,0 +1,11 @@
+export enum RunEnvironment {
+  IDE = "IDE",
+  DEPLOYED = "DEPLOYED",
+}
+
+export enum RunType {
+  SYNC = "SYNC",
+  ASYNC = "ASYNC",
+  JOB = "JOB",
+  QUEUE = "QUEUE",
+}