@8ms/helpers 1.24.0 → 1.25.0

package/brightData/serpApi/buildGoogleSerpUrl.d.ts

@@ -0,0 +1,26 @@
+type BuildGoogleSerpUrl = {
+    country?: string;
+    device?: "mobile" | "android" | "androidTablet" | "iphone" | "ipad";
+    language?: string;
+    geoLocation?: string;
+    hotel?: {
+        dates?: string;
+        occupancy?: number;
+    };
+    pagination?: {
+        offset?: number;
+        perPage?: number;
+    };
+    searchType?: "images" | "jobs" | "news" | "shopping";
+    query: string;
+    googleDomain: string;
+};
+/**
+ * Construct the Google SERP url using Bright Data's API:
+ * https://brightdata.com/cp/serp_api/api/google/search?id=c_2cef083f
+ *
+ * For geolocation use value from:
+ * https://developers.google.com/google-ads/api/data/geotargets
+ */
+declare const buildGoogleSerpUrl: (props: BuildGoogleSerpUrl) => URL;
+export default buildGoogleSerpUrl;

package/brightData/serpApi/buildGoogleSerpUrl.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Construct the Google SERP url using Bright Data's API:
+ * https://brightdata.com/cp/serp_api/api/google/search?id=c_2cef083f
+ *
+ * For geolocation use value from:
+ * https://developers.google.com/google-ads/api/data/geotargets
+ */
+const buildGoogleSerpUrl = (props) => {
+    let url = new URL(`https://www${props.googleDomain}/`);
+    url.searchParams.append("q", props.query);
+    // Two-letter country code used to define the country of search
+    if (props?.country) {
+        url.searchParams.append("gl", props.country);
+    }
+    // Define what device type to be represented in user-agent - Default desktop
+    switch (props?.device) {
+        case "mobile":
+            url.searchParams.append("brd_mobile", "1");
+            break;
+        case "android":
+            url.searchParams.append("brd_mobile", "android");
+            break;
+        case "androidTablet":
+            url.searchParams.append("brd_mobile", "android_tablet");
+            break;
+        case "ipad":
+            url.searchParams.append("brd_mobile", "ipad");
+            break;
+        case "iphone":
+            url.searchParams.append("brd_mobile", "ios");
+            break;
+    }
+    // Stands for the encoded location you want to use for your search
+    if (props?.geoLocation) {
+        url.searchParams.append("uule", props.country);
+    }
+    if (props?.hotel) {
+        if (props.hotel?.dates) {
+            url.searchParams.append("hotel_dates", props.hotel.dates);
+        }
+        if (props.hotel?.occupancy) {
+            url.searchParams.append("hotel_occupancy", props.hotel.occupancy.toString());
+        }
+    }
+    // Two-letter language code used to define the page language
+    if (props?.language) {
+        url.searchParams.append("hl", props.language);
+    }
+    // Define search type. For regular search
+    switch (props?.searchType) {
+        case "images":
+            url.searchParams.append("tbm", "isch");
+            break;
+        case "jobs":
+            url.searchParams.append("ibp", "htl;jobs");
+            break;
+        case "news":
+            url.searchParams.append("tbm", "shop");
+            break;
+        case "shopping":
+            url.searchParams.append("tbm", "nws");
+            break;
+    }
+    if (props?.pagination) {
+        // Define the result offset
+        if (props.pagination?.offset) {
+            url.searchParams.append("start", props.pagination.offset.toString());
+        }
+        // Defines the number of results to return
+        if (props.pagination?.perPage) {
+            url.searchParams.append("num", props.pagination.perPage.toString());
+        }
+    }
+    // Always return the JSON response with HTML field
+    url.searchParams.append("brn_json", "html");
+    return url;
+};
+exports.default = buildGoogleSerpUrl;

package/brightData/serpApi/buildGoogleTrendsUrl.d.ts

@@ -0,0 +1,17 @@
+type BuildGoogleTrendsUrl = {
+    country?: string;
+    language?: string;
+    category?: string;
+    date?: string;
+    searchType?: "images" | "news" | "shopping" | "youtube";
+    query: string;
+};
+/**
+ * Construct the Google Trends url using Bright Data's API:
+ * https://brightdata.com/cp/serp_api/api/google/trends?id=c_2cef083f
+ *
+ * For category use value from:
+ * https://trends.google.com/trends/api/explore/pickers/category?lang=en-US&tz=240
+ */
+declare const buildGoogleTrendsUrl: (props: BuildGoogleTrendsUrl) => URL;
+export default buildGoogleTrendsUrl;

package/brightData/serpApi/buildGoogleTrendsUrl.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Construct the Google Trends url using Bright Data's API:
+ * https://brightdata.com/cp/serp_api/api/google/trends?id=c_2cef083f
+ *
+ * For category use value from:
+ * https://trends.google.com/trends/api/explore/pickers/category?lang=en-US&tz=240
+ */
+const buildGoogleTrendsUrl = (props) => {
+    let url = new URL(`https://trends.google.com/`);
+    url.searchParams.append("q", props.query);
+    // Location of interest, two-letter country code
+    if (props?.country) {
+        url.searchParams.append("geo", props.country);
+    }
+    // Preferred language, two-letter language code
+    if (props?.language) {
+        url.searchParams.append("hl", props.country);
+    }
+    if (props?.category) {
+        url.searchParams.append("cat", props.category);
+    }
+    if (props?.date) {
+        url.searchParams.append("date", props.date);
+    }
+    // Define search type. For regular search
+    switch (props?.searchType) {
+        case "images":
+            url.searchParams.append("gprop", "images");
+            break;
+        case "news":
+            url.searchParams.append("gprop", "news");
+            break;
+        case "shopping":
+            url.searchParams.append("gprop", "froogle");
+            break;
+        case "youtube":
+            url.searchParams.append("gprop", "youtube");
+            break;
+    }
+    return url;
+};
+exports.default = buildGoogleTrendsUrl;

package/brightData/serpApi/getRealtime.d.ts

@@ -0,0 +1,11 @@
+type GetRealtime = {
+    customerId: string;
+    host: string;
+    zonePassword: string;
+    url: string;
+};
+/**
+ * Make a request to the SERP API using the Bright Data proxy
+ */
+declare const getRealtime: (props: GetRealtime) => Promise<{}>;
+export default getRealtime;

package/brightData/serpApi/getRealtime.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const get_1 = __importDefault(require("../../axios/get"));
+/**
+ * Make a request to the SERP API using the Bright Data proxy
+ */
+const getRealtime = async (props) => {
+    let response = {};
+    const { HttpsProxyAgent } = require("https-proxy-agent");
+    // Prevent auth requests getting blocked
+    process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
+    // Construct the proxy URL
+    const proxyUrl = `http://${props.customerId}:${props.zonePassword}@${props.host}`;
+    // Replace <proxy_url> with your actual proxy URL
+    const httpsAgent = new HttpsProxyAgent(proxyUrl);
+    // Use fetch with the agent option to make an HTTP request through the proxy
+    // Replace <target_url> with the URL you want to request
+    const apiResponse = await (0, get_1.default)({
+        config: {
+            proxy: false,
+            httpsAgent,
+        },
+        url: props.url,
+    });
+    if (apiResponse.isSuccess()) {
+        response = apiResponse.getBody();
+    }
+    return response;
+};
+exports.default = getRealtime;

package/brightData/webScraperIde/getBatch.d.ts

@@ -0,0 +1,12 @@
+type GetBatch = {
+    apiKey: string;
+    data?: object;
+    scraperId: string;
+};
+/**
+ * Depends on the Scraper setup!
+ *
+ * Make the request to trigger the scraper and let it do its job.
+ */
+declare const getBatch: (props: GetBatch) => Promise<boolean>;
+export default getBatch;

package/brightData/webScraperIde/getBatch.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const post_1 = __importDefault(require("../../axios/post"));
+/**
+ * Depends on the Scraper setup!
+ *
+ * Make the request to trigger the scraper and let it do its job.
+ */
+const getBatch = async (props) => {
+    let response = false;
+    const requestUrl = `https://api.brightdata.com/dca/trigger?collector=${props.scraperId}&queue_next=1`;
+    const requestResponse = await (0, post_1.default)({
+        config: {
+            headers: {
+                Authorization: `Bearer ${props.apiKey}`
+            },
+        },
+        data: props?.data || {},
+        url: requestUrl,
+    });
+    if (requestResponse.isSuccess()) {
+        response = true;
+    }
+    return response;
+};
+exports.default = getBatch;

package/brightData/webScraperIde/getRealtime.d.ts

@@ -0,0 +1,12 @@
+type GetRealtime = {
+    apiKey: string;
+    data?: object;
+    scraperId: string;
+};
+/**
+ * Depends on the Scraper setup!
+ *
+ * Make the request and pool the response until the data is available.
+ */
+declare const getRealtime: (props: GetRealtime) => Promise<{}>;
+export default getRealtime;

package/brightData/webScraperIde/getRealtime.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const post_1 = __importDefault(require("../../axios/post"));
+const states_1 = __importDefault(require("../../api/states"));
+const get_1 = __importDefault(require("../../axios/get"));
+const sleep_1 = __importDefault(require("../../util/sleep"));
+/**
+ * Depends on the Scraper setup!
+ *
+ * Make the request and pool the response until the data is available.
+ */
+const getRealtime = async (props) => {
+    let response = {};
+    const requestUrl = `https://api.brightdata.com/dca/trigger_immediate?collector=${props.scraperId}`;
+    const requestResponse = await (0, post_1.default)({
+        config: {
+            headers: {
+                Authorization: `Bearer ${props.apiKey}`
+            },
+        },
+        data: props?.data || {},
+        url: requestUrl,
+    });
+    if (requestResponse.isSuccess()) {
+        const responseId = requestResponse.getBodyDefaultTo({
+            defaultValue: "",
+            keys: ["response_id"],
+        });
+        if ("" !== responseId) {
+            let state = states_1.default.PENDING;
+            while (states_1.default.PENDING === state) {
+                const resultUrl = `https://api.brightdata.com/dca/get_result?response_id=${responseId}`;
+                // Use fetch with the agent option to make an HTTP request through the proxy
+                // Replace <target_url> with the URL you want to request
+                const resultResponse = await (0, get_1.default)({
+                    config: {
+                        headers: {
+                            Authorization: `Bearer ${props.apiKey}`
+                        },
+                    },
+                    url: resultUrl,
+                });
+                if (resultResponse.isSuccess()) {
+                    if (undefined !== resultResponse.body.pending &&
+                        true === resultResponse.body.pending) {
+                        state = states_1.default.PENDING;
+                    }
+                    else {
+                        state = states_1.default.SUCCESS;
+                        response = resultResponse.getBody();
+                    }
+                }
+                await (0, sleep_1.default)({
+                    seconds: 1
+                });
+            }
+        }
+    }
+    return response;
+};
+exports.default = getRealtime;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name":            "@8ms/helpers",
 	"license":         "UNLICENSED",
-	"version":         "1.24.0",
+	"version":         "1.25.0",
 	"repository":      {
 		"type": "git",
 		"url":  "git+https://github.com/8millionstories-organisation/8ms-helpers-ts.git"
@@ -39,6 +39,7 @@
 		"@types/lodash":                 "4.17.7",
 		"@types/node":                   "20.14.12",
 		"babel-jest":                    "29.7.0",
+		"https-proxy-agent":             "7.0.5",
 		"jest":                          "29.7.0",
 		"node-fetch":                    "3.3.2",
 		"stream-chain":                  "2.2.5",