@browserless.io/browserless 2.0.0-beta-1

package/build/routes/chromium/http/content-post.body.json

@@ -0,0 +1,514 @@
+{
+  "type": "object",
+  "properties": {
+    "addScriptTag": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/FrameAddScriptTagOptions"
+      }
+    },
+    "addStyleTag": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/FrameAddStyleTagOptions"
+      }
+    },
+    "authenticate": {
+      "$ref": "#/definitions/Credentials"
+    },
+    "bestAttempt": {
+      "description": "When bestAttempt is set to true, browserless attempt to proceed\nwhen \"awaited\" events fail or timeout. This includes things like\ngoto, waitForSelector, and more.",
+      "type": "boolean"
+    },
+    "cookies": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/Protocol.Network.CookieParam"
+      }
+    },
+    "emulateMediaType": {
+      "type": "string"
+    },
+    "gotoOptions": {
+      "$ref": "#/definitions/GoToOptions"
+    },
+    "html": {
+      "type": "string"
+    },
+    "rejectRequestPattern": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "rejectResourceTypes": {
+      "type": "array",
+      "items": {
+        "enum": [
+          "cspviolationreport",
+          "document",
+          "eventsource",
+          "fetch",
+          "font",
+          "image",
+          "manifest",
+          "media",
+          "other",
+          "ping",
+          "prefetch",
+          "preflight",
+          "script",
+          "signedexchange",
+          "stylesheet",
+          "texttrack",
+          "websocket",
+          "xhr"
+        ],
+        "type": "string"
+      }
+    },
+    "requestInterceptors": {
+      "type": "array",
+      "items": {
+        "description": "An array of patterns (using `req.url().match(r.pattern)` to match) and their\ncorresponding responses to use in order to fulfill those requests.",
+        "type": "object",
+        "properties": {
+          "pattern": {
+            "description": "An array of patterns (using `req.url().match(r.pattern)` to match) and their\ncorresponding responses to use in order to fulfill those requests.",
+            "type": "string"
+          },
+          "response": {
+            "$ref": "#/definitions/Partial<ResponseForRequest>"
+          }
+        },
+        "additionalProperties": false,
+        "required": [
+          "pattern",
+          "response"
+        ]
+      }
+    },
+    "setExtraHTTPHeaders": {
+      "$ref": "#/definitions/Record<string,string>"
+    },
+    "setJavaScriptEnabled": {
+      "description": "Whether or not to allow JavaScript to run on the page.",
+      "type": "boolean"
+    },
+    "url": {
+      "type": "string"
+    },
+    "userAgent": {
+      "type": "string"
+    },
+    "viewport": {
+      "$ref": "#/definitions/Viewport"
+    },
+    "waitForEvent": {
+      "type": "object",
+      "properties": {
+        "event": {
+          "type": "string"
+        },
+        "timeout": {
+          "type": "number"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "event"
+      ]
+    },
+    "waitForFunction": {
+      "type": "object",
+      "properties": {
+        "fn": {
+          "description": "The function, or statement, to be evaluated in browser context",
+          "type": "string"
+        },
+        "polling": {
+          "description": "An interval at which the pageFunction is executed, defaults to raf.\nIf polling is a number, then it is treated as an interval in milliseconds\nat which the function would be executed. If polling is a string,\nthen it can be one of the following values: \"raf\" or \"mutation\"",
+          "type": [
+            "string",
+            "number"
+          ]
+        },
+        "timeout": {
+          "description": "Maximum time to wait for in milliseconds. Defaults to 30000 (30 seconds).\nPass 0 to disable timeout.",
+          "type": "number"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "fn"
+      ]
+    },
+    "waitForSelector": {
+      "type": "object",
+      "properties": {
+        "hidden": {
+          "type": "boolean"
+        },
+        "selector": {
+          "type": "string"
+        },
+        "timeout": {
+          "type": "number"
+        },
+        "visible": {
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "selector"
+      ]
+    },
+    "waitForTimeout": {
+      "type": "number"
+    }
+  },
+  "additionalProperties": false,
+  "definitions": {
+    "FrameAddScriptTagOptions": {
+      "type": "object",
+      "properties": {
+        "url": {
+          "description": "URL of the script to be added.",
+          "type": "string"
+        },
+        "path": {
+          "description": "Path to a JavaScript file to be injected into the frame.",
+          "type": "string"
+        },
+        "content": {
+          "description": "JavaScript to be injected into the frame.",
+          "type": "string"
+        },
+        "type": {
+          "description": "Sets the `type` of the script. Use `module` in order to load an ES2015 module.",
+          "type": "string"
+        },
+        "id": {
+          "description": "Sets the `id` of the script.",
+          "type": "string"
+        }
+      },
+      "additionalProperties": false
+    },
+    "FrameAddStyleTagOptions": {
+      "type": "object",
+      "properties": {
+        "url": {
+          "description": "the URL of the CSS file to be added.",
+          "type": "string"
+        },
+        "path": {
+          "description": "The path to a CSS file to be injected into the frame.",
+          "type": "string"
+        },
+        "content": {
+          "description": "Raw CSS content to be injected into the frame.",
+          "type": "string"
+        }
+      },
+      "additionalProperties": false
+    },
+    "Credentials": {
+      "type": "object",
+      "properties": {
+        "username": {
+          "type": "string"
+        },
+        "password": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "password",
+        "username"
+      ]
+    },
+    "Protocol.Network.CookieParam": {
+      "description": "Cookie parameter object",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "Cookie name.",
+          "type": "string"
+        },
+        "value": {
+          "description": "Cookie value.",
+          "type": "string"
+        },
+        "url": {
+          "description": "The request-URI to associate with the setting of the cookie. This value can affect the\ndefault domain, path, source port, and source scheme values of the created cookie.",
+          "type": "string"
+        },
+        "domain": {
+          "description": "Cookie domain.",
+          "type": "string"
+        },
+        "path": {
+          "description": "Cookie path.",
+          "type": "string"
+        },
+        "secure": {
+          "description": "True if cookie is secure.",
+          "type": "boolean"
+        },
+        "httpOnly": {
+          "description": "True if cookie is http-only.",
+          "type": "boolean"
+        },
+        "sameSite": {
+          "description": "Cookie SameSite type.",
+          "enum": [
+            "Lax",
+            "None",
+            "Strict"
+          ],
+          "type": "string"
+        },
+        "expires": {
+          "description": "Cookie expiration date, session cookie if not set",
+          "type": "number"
+        },
+        "priority": {
+          "description": "Cookie Priority.",
+          "enum": [
+            "High",
+            "Low",
+            "Medium"
+          ],
+          "type": "string"
+        },
+        "sameParty": {
+          "description": "True if cookie is SameParty.",
+          "type": "boolean"
+        },
+        "sourceScheme": {
+          "description": "Cookie source scheme type.",
+          "enum": [
+            "NonSecure",
+            "Secure",
+            "Unset"
+          ],
+          "type": "string"
+        },
+        "sourcePort": {
+          "description": "Cookie source port. Valid values are {-1, [1, 65535]}, -1 indicates an unspecified port.\nAn unspecified port value allows protocol clients to emulate legacy cookie scope for the port.\nThis is a temporary ability and it will be removed in the future.",
+          "type": "number"
+        },
+        "partitionKey": {
+          "description": "Cookie partition key. The site of the top-level URL the browser was visiting at the start\nof the request to the endpoint that set the cookie.\nIf not set, the cookie will be set as not partitioned.",
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "name",
+        "value"
+      ]
+    },
+    "GoToOptions": {
+      "type": "object",
+      "properties": {
+        "referer": {
+          "description": "If provided, it will take preference over the referer header value set by\n{@link Page.setExtraHTTPHeaderspage.setExtraHTTPHeaders()}.",
+          "type": "string"
+        },
+        "referrerPolicy": {
+          "description": "If provided, it will take preference over the referer-policy header value\nset by {@link Page.setExtraHTTPHeaderspage.setExtraHTTPHeaders()}.",
+          "type": "string"
+        },
+        "timeout": {
+          "description": "Maximum wait time in milliseconds. Pass 0 to disable the timeout.\n\nThe default value can be changed by using the\n{@link Page.setDefaultTimeout} or {@link Page.setDefaultNavigationTimeout}\nmethods.",
+          "type": "number"
+        },
+        "waitUntil": {
+          "description": "When to consider waiting succeeds. Given an array of event strings, waiting\nis considered to be successful after all events have been fired.",
+          "anyOf": [
+            {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/PuppeteerLifeCycleEvent"
+              }
+            },
+            {
+              "enum": [
+                "domcontentloaded",
+                "load",
+                "networkidle0",
+                "networkidle2"
+              ],
+              "type": "string"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false
+    },
+    "PuppeteerLifeCycleEvent": {
+      "enum": [
+        "domcontentloaded",
+        "load",
+        "networkidle0",
+        "networkidle2"
+      ],
+      "type": "string"
+    },
+    "Partial<ResponseForRequest>": {
+      "type": "object",
+      "properties": {
+        "status": {
+          "type": "number"
+        },
+        "headers": {
+          "description": "Optional response headers. All values are converted to strings.",
+          "$ref": "#/definitions/Record<string,unknown>"
+        },
+        "contentType": {
+          "type": "string"
+        },
+        "body": {
+          "anyOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "patternProperties": {
+                "^[0-9]+$": {
+                  "type": "number"
+                }
+              },
+              "properties": {
+                "BYTES_PER_ELEMENT": {
+                  "type": "number"
+                },
+                "buffer": {
+                  "$ref": "#/definitions/ArrayBufferLike"
+                },
+                "byteLength": {
+                  "type": "number"
+                },
+                "byteOffset": {
+                  "type": "number"
+                },
+                "length": {
+                  "type": "number"
+                },
+                "__@toStringTag@10721": {
+                  "type": "string",
+                  "const": "Uint8Array"
+                }
+              },
+              "required": [
+                "BYTES_PER_ELEMENT",
+                "__@toStringTag@10721",
+                "buffer",
+                "byteLength",
+                "byteOffset",
+                "length"
+              ]
+            },
+            {
+              "type": "string"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false
+    },
+    "Record<string,unknown>": {
+      "type": "object",
+      "additionalProperties": false
+    },
+    "ArrayBufferLike": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/ArrayBuffer"
+        },
+        {
+          "$ref": "#/definitions/SharedArrayBuffer"
+        }
+      ]
+    },
+    "ArrayBuffer": {
+      "type": "object",
+      "properties": {
+        "byteLength": {
+          "type": "number"
+        },
+        "__@toStringTag@10721": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "__@toStringTag@10721",
+        "byteLength"
+      ]
+    },
+    "SharedArrayBuffer": {
+      "type": "object",
+      "properties": {
+        "byteLength": {
+          "type": "number"
+        },
+        "__@species@10822": {
+          "$ref": "#/definitions/SharedArrayBuffer"
+        },
+        "__@toStringTag@10721": {
+          "type": "string",
+          "const": "SharedArrayBuffer"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "__@species@10822",
+        "__@toStringTag@10721",
+        "byteLength"
+      ]
+    },
+    "Record<string,string>": {
+      "type": "object",
+      "additionalProperties": false
+    },
+    "Viewport": {
+      "type": "object",
+      "properties": {
+        "width": {
+          "description": "The page width in CSS pixels.",
+          "type": "number"
+        },
+        "height": {
+          "description": "The page height in CSS pixels.",
+          "type": "number"
+        },
+        "deviceScaleFactor": {
+          "description": "Specify device scale factor.\nSee {@link https://developer.mozilla.org/en-US/docs/Web/API/Window/devicePixelRatio devicePixelRatio} for more info.",
+          "type": "number"
+        },
+        "isMobile": {
+          "description": "Whether the `meta viewport` tag is taken into account.",
+          "type": "boolean"
+        },
+        "isLandscape": {
+          "description": "Specifies if the viewport is in landscape mode.",
+          "type": "boolean"
+        },
+        "hasTouch": {
+          "description": "Specify if the viewport supports touch events.",
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "height",
+        "width"
+      ]
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

package/build/routes/chromium/http/content-post.d.ts

@@ -0,0 +1,34 @@
+import { BrowserHTTPRoute, CDPLaunchOptions, SystemQueryParameters, WaitForEventOptions, WaitForFunctionOptions, WaitForSelectorOptions, bestAttempt, rejectRequestPattern, rejectResourceTypes, requestInterceptors, setJavaScriptEnabled } from '@browserless.io/browserless';
+import { Page } from 'puppeteer-core';
+export interface BodySchema {
+    addScriptTag?: Array<Parameters<Page['addScriptTag']>[0]>;
+    addStyleTag?: Array<Parameters<Page['addStyleTag']>[0]>;
+    authenticate?: Parameters<Page['authenticate']>[0];
+    bestAttempt?: bestAttempt;
+    cookies?: Array<Parameters<Page['setCookie']>[0]>;
+    emulateMediaType?: Parameters<Page['emulateMediaType']>[0];
+    gotoOptions?: Parameters<Page['goto']>[1];
+    html?: Parameters<Page['setContent']>[0];
+    rejectRequestPattern?: rejectRequestPattern[];
+    rejectResourceTypes?: rejectResourceTypes[];
+    requestInterceptors?: Array<requestInterceptors>;
+    setExtraHTTPHeaders?: Parameters<Page['setExtraHTTPHeaders']>[0];
+    setJavaScriptEnabled?: setJavaScriptEnabled;
+    url?: Parameters<Page['goto']>[0];
+    userAgent?: Parameters<Page['setUserAgent']>[0];
+    viewport?: Parameters<Page['setViewport']>[0];
+    waitForEvent?: WaitForEventOptions;
+    waitForFunction?: WaitForFunctionOptions;
+    waitForSelector?: WaitForSelectorOptions;
+    waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+}
+/**
+ * An HTML payload of the website or HTML after JavaScript
+ * parsing and execution.
+ */
+export type ResponseSchema = string;
+export type QuerySchema = SystemQueryParameters & {
+    launch?: CDPLaunchOptions | string;
+};
+declare const route: BrowserHTTPRoute;
+export default route;

package/build/routes/chromium/http/content-post.js

@@ -0,0 +1,109 @@
+import { APITags, BadRequest, CDPChromium, HTTPRoutes, Methods, bestAttemptCatch, contentTypes, noop, waitForEvent as waitForEvt, waitForFunction as waitForFn, writeResponse, } from '@browserless.io/browserless';
+const route = {
+    accepts: [contentTypes.json],
+    auth: true,
+    browser: CDPChromium,
+    concurrency: true,
+    contentTypes: [contentTypes.html],
+    description: `A JSON-based API. Given a "url" or "html" field, runs and returns HTML content after the page has loaded and JavaScript has parsed.`,
+    handler: async (req, res, browser) => {
+        const contentType = !req.headers.accept || req.headers.accept?.includes('*')
+            ? contentTypes.html
+            : req.headers.accept;
+        if (!req.body) {
+            throw new BadRequest(`Couldn't parse JSON body`);
+        }
+        res.setHeader('Content-Type', contentType);
+        const { bestAttempt = false, url, gotoOptions, html, authenticate, addScriptTag = [], addStyleTag = [], cookies = [], emulateMediaType, rejectRequestPattern = [], requestInterceptors = [], rejectResourceTypes = [], setExtraHTTPHeaders, setJavaScriptEnabled, userAgent, viewport, waitForTimeout, waitForFunction, waitForSelector, waitForEvent, } = req.body;
+        const content = url || html;
+        if (!content) {
+            throw new BadRequest(`One of "url" or "html" properties are required.`);
+        }
+        const page = (await browser.newPage());
+        const gotoCall = url ? page.goto.bind(page) : page.setContent.bind(page);
+        if (emulateMediaType) {
+            await page.emulateMediaType(emulateMediaType);
+        }
+        if (cookies.length) {
+            await page.setCookie(...cookies);
+        }
+        if (viewport) {
+            await page.setViewport(viewport);
+        }
+        if (userAgent) {
+            await page.setUserAgent(userAgent);
+        }
+        if (authenticate) {
+            await page.authenticate(authenticate);
+        }
+        if (setExtraHTTPHeaders) {
+            await page.setExtraHTTPHeaders(setExtraHTTPHeaders);
+        }
+        if (setJavaScriptEnabled) {
+            await page.setJavaScriptEnabled(setJavaScriptEnabled);
+        }
+        if (rejectRequestPattern.length ||
+            requestInterceptors.length ||
+            rejectResourceTypes.length) {
+            await page.setRequestInterception(true);
+            page.on('request', (req) => {
+                if (!!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
+                    rejectResourceTypes.includes(req.resourceType())) {
+                    return req.abort();
+                }
+                const interceptor = requestInterceptors.find((r) => req.url().match(r.pattern));
+                if (interceptor) {
+                    return req.respond(interceptor.response);
+                }
+                return req.continue();
+            });
+        }
+        if (addStyleTag.length) {
+            for (const tag in addStyleTag) {
+                await page.addStyleTag(addStyleTag[tag]);
+            }
+        }
+        if (addScriptTag.length) {
+            for (const tag in addScriptTag) {
+                await page.addScriptTag(addScriptTag[tag]);
+            }
+        }
+        const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
+        if (waitForTimeout) {
+            await page
+                .waitForTimeout(waitForTimeout)
+                .catch(bestAttemptCatch(bestAttempt));
+        }
+        if (waitForFunction) {
+            await waitForFn(page, waitForFunction).catch(bestAttemptCatch(bestAttempt));
+        }
+        if (waitForSelector) {
+            const { selector, hidden, timeout, visible } = waitForSelector;
+            await page
+                .waitForSelector(selector, { hidden, timeout, visible })
+                .catch(bestAttemptCatch(bestAttempt));
+        }
+        if (waitForEvent) {
+            await waitForEvt(page, waitForEvent).catch(bestAttemptCatch(bestAttempt));
+        }
+        const headers = {
+            'X-Response-Code': gotoResponse?.status(),
+            'X-Response-IP': gotoResponse?.remoteAddress().ip,
+            'X-Response-Port': gotoResponse?.remoteAddress().port,
+            'X-Response-Status': gotoResponse?.statusText(),
+            'X-Response-URL': gotoResponse?.url().substring(0, 1000),
+        };
+        for (const [key, value] of Object.entries(headers)) {
+            if (value !== undefined) {
+                res.setHeader(key, value);
+            }
+        }
+        const markup = await page.content();
+        page.close().catch(noop);
+        return writeResponse(res, 200, markup, contentTypes.html);
+    },
+    method: Methods.post,
+    path: HTTPRoutes.content,
+    tags: [APITags.browserAPI],
+};
+export default route;