@intuned/runtime-dev 0.0.1-split.0

package/.babelrc

@@ -0,0 +1,21 @@
+{
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "chrome": 115,
+          "node": "16"
+        },
+        "modules": "commonjs"
+      }
+    ],
+    "@babel/preset-typescript"
+  ],
+  "plugins": [
+    "babel-plugin-macros",
+    "@babel/plugin-transform-export-namespace-from"
+  ],
+  "sourceMaps": false,
+  "comments": false
+}

package/.eslintignore

@@ -0,0 +1,10 @@
+node_modules
+dist
+.eslintrc.js
+*.test.ts
+*.js
+types-package
+vite.config.ts
+reports
+intuned
+**/*.d.ts

package/.eslintrc.js

@@ -0,0 +1,39 @@
+module.exports = {
+  root: true,
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    project: "./tsconfig.eslint.json",
+    tsconfigRootDir: __dirname,
+  },
+  plugins: ["@typescript-eslint", "deprecation", "prettier"],
+  ignorePatterns: ["src/common/browserScripts/rollup.config.mjs"],
+  extends: [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+  ],
+  rules: {
+    "@next/next/no-html-link-for-pages": 0,
+    "prettier/prettier": "error",
+
+    // recommended for safety
+    "@typescript-eslint/no-floating-promises": "error", // forgetting to await Activities and Workflow APIs is bad
+    "deprecation/deprecation": "warn",
+
+    // code style preference
+    "object-shorthand": ["warn", "always"],
+
+    // relaxed rules, for convenience
+    "@typescript-eslint/no-unused-vars": [
+      "warn",
+      {
+        argsIgnorePattern: "^_",
+        varsIgnorePattern: "^_",
+      },
+    ],
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-empty-interface": "off",
+    "@typescript-eslint/ban-ts-comment": "warn",
+    "no-empty": "warn",
+  },
+};

package/.vite/deps_temp_01af7156/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

package/.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+  "recommendations": ["zixuanchen.vitest-explorer"]
+}

package/.vscode/launch.json

@@ -0,0 +1,102 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Hiltion api",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": ["src/commands/api/run.ts", "hilton", "-j", "{}", "playwright"],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Books Extractor",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": [
+        "src/commands/api/run.ts",
+        "books-array",
+        "-j",
+        "{}",
+        "playwright"
+      ],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run execution-info",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": [
+        "src/commands/api/run.ts",
+        "execution-info",
+        "-j",
+        "{}",
+        "playwright"
+      ],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run progressive",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": [
+        "src/commands/api/run.ts",
+        "progressive",
+        "-j",
+        "{}",
+        "playwright"
+      ],
+
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run progressive - standalone",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": ["src/commands/api/run.ts", "progressive", "-j", "{}"],
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    },
+    {
+      "name": "Run progressive extract - standalone",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "node",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+
+      "args": ["src/commands/api/run.ts", "progressive-extract", "-j", "{}"],
+      "cwd": "${workspaceRoot}",
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": ["<node_internals>/**", "node_modules/**"]
+    }
+  ]
+}

package/.vscode/settings.json

@@ -0,0 +1,12 @@
+{
+  "editor.formatOnSave": true,
+  "cSpell.words": ["backcompat", "ـget", "INFINITI", "neverthrow"],
+  "editor.detectIndentation": true,
+  "files.exclude": {
+    "dist": false,
+    "node_modules": false,
+    "output": false,
+    "intuned": false,
+    "htmlFiles": true
+  }
+}

package/WebTemplate/accessKeyHelpers.ts

@@ -0,0 +1,28 @@
+import { createHash } from "crypto";
+
+function hashToken(token: string) {
+  // This has to match the logic from the workflow that saves adds the token to fly io App.
+  return createHash("sha256").update(token).digest("hex");
+}
+
+export const accessKeyValidatorMiddleware = (req: any, res: any, next: any) => {
+  const authHeader = req.headers.authorization;
+
+  if (!authHeader || !authHeader.startsWith("Bearer ")) {
+    return res.status(401).send("Unauthorized");
+  }
+
+  const token = authHeader.split(" ")[1];
+
+  // Create a hash of the provided token
+  const hashedToken = hashToken(token);
+
+  const hashedTokenFromEnv = process.env.HASHED_ACCESS_KEY;
+
+  if (hashedToken !== hashedTokenFromEnv) {
+    console.log("Token mismatch", hashedToken, hashedTokenFromEnv);
+    return res.status(401).send("Unauthorized");
+  }
+
+  next();
+};

package/WebTemplate/api.ts

@@ -0,0 +1,139 @@
+import {
+  errorRetryMiddleware,
+  proxyToUrl,
+  ProxyConfig,
+  isJobRunMachine,
+} from "./utils";
+import { accessKeyValidatorMiddleware } from "./accessKeyHelpers";
+import { runWithContext } from "@intuned/runtime";
+import {
+  createAuthSessionController,
+  createAuthSessionAsyncController,
+  resumeAuthSessionOperationController,
+  resumeAuthSessionOperationAsyncController,
+  killAuthSessionOperationController,
+  checkAuthSessionController,
+  checkAuthSessionAsyncController,
+} from "./controllers/authSessions";
+import { app } from "./app";
+import {
+  getActiveAsyncEndpointController,
+  runApiAsyncController,
+  runApiController,
+} from "./controllers/runApi";
+import {
+  uploadTraceController,
+  deleteTraceController,
+} from "./controllers/traces";
+import {
+  RUN_ID_HEADER,
+  JOB_ID_HEADER,
+  JOB_RUN_ID_HEADER,
+  QUEUE_ID_HEADER,
+} from "./headers";
+import { FEATURES } from "./features";
+
+if (!isJobRunMachine()) {
+  app.use((req, _, next) => {
+    const runId = req.headers[RUN_ID_HEADER] as string;
+    const jobId = req.headers[JOB_ID_HEADER] as string;
+    const jobRunId = req.headers[JOB_RUN_ID_HEADER] as string;
+    const queueId = req.headers[QUEUE_ID_HEADER] as string;
+    const proxy = req?.body?.proxy
+      ? (proxyToUrl(req.body.proxy as ProxyConfig) as string)
+      : undefined;
+    const contextData = {
+      runId: runId ?? "",
+      jobId,
+      jobRunId,
+      queueId,
+      proxy,
+      ...(req?.body?.executionContext ?? {}),
+    };
+    runWithContext(contextData, next);
+  });
+
+  app.get(
+    "/api/protected/health",
+    accessKeyValidatorMiddleware,
+    async (req, res) => {
+      res.status(200).json({ status: "ok" });
+    }
+  );
+
+  app.post(
+    "/api/auth-session/create",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(createAuthSessionController)
+  );
+
+  app.post(
+    "/api/auth-session-async/create",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(createAuthSessionAsyncController)
+  );
+
+  app.post(
+    "/api/auth-session/check",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(checkAuthSessionController)
+  );
+
+  app.post(
+    "/api/auth-session-async/check",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(checkAuthSessionAsyncController)
+  );
+
+  app.post(
+    "/api/auth-session/kill",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(killAuthSessionOperationController)
+  );
+
+  app.post(
+    "/api/auth-session/resume",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(resumeAuthSessionOperationController)
+  );
+
+  app.post(
+    "/api/auth-session-async/resume",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(resumeAuthSessionOperationAsyncController)
+  );
+
+  app.post(
+    "/api/run/*",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(runApiController)
+  );
+
+  app.post(
+    "/api/run-async/start/*",
+    accessKeyValidatorMiddleware,
+    errorRetryMiddleware(runApiAsyncController) // todo: this probably needs changing
+  );
+
+  app.get(
+    "/api/run-async/count",
+    accessKeyValidatorMiddleware,
+    getActiveAsyncEndpointController
+  );
+
+  app.post(
+    "/api/trace/upload/:runId",
+    accessKeyValidatorMiddleware,
+    uploadTraceController
+  );
+
+  app.post(
+    "/api/trace/delete/:runId",
+    accessKeyValidatorMiddleware,
+    deleteTraceController
+  );
+}
+
+app.get("/api/features", accessKeyValidatorMiddleware, async (_, res) => {
+  res.status(200).json({ features: FEATURES });
+});

package/WebTemplate/app.ts

@@ -0,0 +1,18 @@
+import { App } from "@tinyhttp/app";
+import { json as parser } from "milliparsec";
+
+export const port = process.env.PORT ? parseInt(process.env.PORT) : 4000;
+
+export const app = new App();
+
+app.use("/", parser());
+
+// add header with flyio instance id where the request is served from
+app.use((_, res, next) => {
+  res.setHeader("fly-instance-id", process.env.FLY_ALLOC_ID ?? "");
+  next();
+});
+
+app.get("/api/health", async (req, res) => {
+  res.status(200).json({ status: "ok" });
+});

package/WebTemplate/controllers/async.ts

@@ -0,0 +1,138 @@
+import { AsyncRunEndpointController, getErrorResponse } from "../utils";
+import {
+  backendFunctionsTokenManager,
+  callBackendFunctionWithToken,
+} from "@intuned/runtime/dist/common/jwtTokenManager";
+import retry from "async-retry";
+import { ShutdownController } from "../shutdown";
+import { Handler } from "@tinyhttp/app";
+import { RequestHandler } from "./controllers";
+
+export async function runEndpointAsync<_Parameters extends any[], _Result>(
+  handler: RequestHandler<_Parameters, _Result>,
+  parameters: _Parameters,
+  endpointParameters: {
+    functionsToken: string;
+    taskToken: string;
+    startTime: number;
+    // only used for logging
+    requestId?: string;
+  }
+): Promise<{
+  status: number;
+  body: _Result;
+}> {
+  const {
+    functionsToken,
+    taskToken,
+    startTime,
+    requestId = "endpoint",
+  } = endpointParameters;
+  AsyncRunEndpointController.addRequest(taskToken);
+
+  backendFunctionsTokenManager.token = functionsToken;
+
+  let status: number, body: _Result;
+  try {
+    console.log("Running", requestId, "in async mode");
+    ({ status, body } = await handler(...parameters));
+    console.log("Finished running", requestId, "in async mode");
+  } catch (e: any) {
+    console.error(`Error running ${requestId} in async mode:`, e);
+    ({ status, body } = getErrorResponse(e));
+  } finally {
+    AsyncRunEndpointController.removeRequest(taskToken);
+  }
+
+  // report result
+  console.log("Reporting result for", requestId);
+  try {
+    // `retry` will keep retrying any errors unless they are thrown through calling `bail`
+    await retry(
+      async (bail: (e: Error) => void) => {
+        const response = await callBackendFunctionWithToken(
+          "run/reportResult",
+          {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              "fly-instance-id": process.env.FLY_ALLOC_ID ?? "",
+            },
+            body: JSON.stringify({
+              status,
+              body,
+              startTime,
+              taskToken,
+            }),
+          }
+        );
+        if (!response.ok) {
+          // Do not retry unauthenticated, forbidden, not found or too large responses
+          if ([401, 403, 404, 413].includes(response.status)) {
+            bail(
+              new Error(
+                `Reporting result failed for ${requestId} (non-retryable), status ${response.status}`
+              )
+            );
+          }
+          // retry other errors (such as 502)
+          throw new Error(
+            `Reporting result failed for ${requestId}, status ${response.status}`
+          );
+        }
+
+        return response;
+      },
+      {
+        retries: 5,
+        factor: 2,
+        maxTimeout: 1000 * 60, // 1 minute
+        minTimeout: 1000 * 5, // 5 seconds
+      }
+    );
+    console.log("Reported result for", requestId);
+  } catch (e: any) {
+    console.log(e?.message ?? `Reporting result failed for ${requestId}`);
+  }
+
+  await ShutdownController.instance.checkForShutdown();
+  return { status, body };
+}
+
+export type AsyncEndpointBody<T = unknown> = T & {
+  functionsToken: string;
+  taskToken: string;
+  startTime: number;
+};
+
+export function makeAsyncEndpointController<_Parameters extends any[]>({
+  requestId,
+  handler,
+  parameters,
+}: {
+  requestId: string | ((req: any) => string);
+  handler: RequestHandler<_Parameters, any>;
+  parameters: _Parameters | ((req: any) => _Parameters);
+}): Handler {
+  return async (req, res) => {
+    const { taskToken, startTime, functionsToken } =
+      req.body as AsyncEndpointBody<_Parameters>;
+    if (AsyncRunEndpointController.isRunning(taskToken)) {
+      res.status(409).json({ error: "Already running" });
+      return;
+    }
+
+    void runEndpointAsync(
+      handler,
+      typeof parameters === "function" ? parameters(req) : parameters,
+      {
+        taskToken,
+        startTime,
+        functionsToken,
+        requestId: typeof requestId === "function" ? requestId(req) : requestId,
+      }
+    );
+
+    res.status(202).json({});
+  };
+}

package/WebTemplate/controllers/authSessions/check.ts

@@ -0,0 +1,68 @@
+import { getDownloadDirectoryPath } from "@intuned/runtime";
+import { getProductionPlaywrightConstructs } from "@intuned/runtime";
+import { checkAuthSessionWithRetries, isHeadless } from "../../utils";
+import * as fs from "fs-extra";
+import { readJSON } from "fs-extra";
+
+export async function checkAuthSession({
+  mode,
+  proxy,
+  session,
+}: {
+  mode: "async" | "sync";
+  session: any;
+  proxy?: {
+    server: string;
+    username: string;
+    password: string;
+  };
+}): Promise<{
+  status: number;
+  body: any;
+}> {
+  const isAuthSessionEnabled = (await readJSON("./Intuned.json")).authSessions
+    ?.enabled;
+
+  if (!isAuthSessionEnabled) {
+    return {
+      status: 400,
+      body: {
+        error: "Invalid Request",
+        message: "auth sessions are not enabled",
+      },
+    };
+  }
+
+  if (mode !== "async") {
+    return {
+      status: 400,
+      body: {
+        error: "Invalid Request",
+        message: "only async mode is supported",
+      },
+    };
+  }
+
+  const downloadsPath = getDownloadDirectoryPath();
+  const headless = isHeadless();
+  const { page, context } = await getProductionPlaywrightConstructs({
+    headless,
+    proxy,
+    downloadsPath,
+    storageState: session,
+  });
+
+  const authSessionCheckResult = await checkAuthSessionWithRetries(
+    page,
+    context,
+    2
+  );
+
+  await fs.remove(downloadsPath);
+  return {
+    status: 200,
+    body: {
+      result: authSessionCheckResult,
+    },
+  };
+}