@8ms/helpers 1.27.0 → 1.29.0

package/brightData/getCustomerId.d.ts

@@ -0,0 +1,10 @@
+import { Param } from "./param";
+type GetCustomerId = {
+    auth: Param;
+    prefix?: string;
+};
+/**
+ * Return the customer Id based on the username.
+ */
+declare const getCustomerId: (props: GetCustomerId) => string;
+export default getCustomerId;

package/brightData/getCustomerId.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Return the customer Id based on the username.
+ */
+const getCustomerId = (props) => {
+    const segments = props.auth.username.split("-")[2];
+    const customerId = segments[2].replace("hl_", "");
+    return (props?.prefix || "") + customerId;
+};
+exports.default = getCustomerId;

package/brightData/getZone.d.ts

@@ -0,0 +1,9 @@
+import { Param } from "./param";
+type GetZone = {
+    auth: Param;
+};
+/**
+ * Return the Zone based on the username.
+ */
+declare const getZone: (props: GetZone) => string;
+export default getZone;

package/brightData/getZone.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Return the Zone based on the username.
+ */
+const getZone = (props) => {
+    const segments = props.auth.username.split("-");
+    return segments[segments.length - 1];
+};
+exports.default = getZone;

package/brightData/param.d.ts

@@ -0,0 +1,9 @@
+export type Param = {
+    apiKey: string;
+    host: string;
+    port: string;
+    username: string;
+    password: string;
+};
+declare const _default: {};
+export default _default;

package/brightData/param.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = {};

package/brightData/serpApi/buildGoogleSerpUrl.d.ts

@@ -11,9 +11,11 @@ type BuildGoogleSerpUrl = {
         offset?: number;
         perPage?: number;
     };
-    searchType?: "images" | "jobs" | "news" | "shopping";
+    searchType?: "images" | "jobs" | "news" | "shopping" | "video";
     query: string;
     googleDomain: string;
+    output?: "html" | "json";
+    returnType: "url" | "json";
 };
 /**
  * Construct the Google SERP url using Bright Data's API:
@@ -22,5 +24,9 @@ type BuildGoogleSerpUrl = {
  * For geolocation use value from:
  * https://developers.google.com/google-ads/api/data/geotargets
  */
-declare const buildGoogleSerpUrl: (props: BuildGoogleSerpUrl) => string;
+declare const buildGoogleSerpUrl: (props: BuildGoogleSerpUrl) => string | {
+    query: {
+        q: string;
+    };
+};
 export default buildGoogleSerpUrl;

package/brightData/serpApi/buildGoogleSerpUrl.js

@@ -8,73 +8,108 @@ Object.defineProperty(exports, "__esModule", { value: true });
  * https://developers.google.com/google-ads/api/data/geotargets
  */
 const buildGoogleSerpUrl = (props) => {
-    let url = new URL(`https://www.${props.googleDomain}/`);
+    let url = new URL(`https://www.${props.googleDomain}/search`);
+    let params = {
+        query: {
+            q: props.query,
+        }
+    };
     url.searchParams.append("q", props.query);
     // Two-letter country code used to define the country of search
     if (props?.country) {
+        params.query["gl"] = props.country;
         url.searchParams.append("gl", props.country);
     }
     // Define what device type to be represented in user-agent - Default desktop
     switch (props?.device) {
         case "mobile":
+            params.query["brd_mobile"] = "1";
             url.searchParams.append("brd_mobile", "1");
             break;
         case "android":
+            params.query["brd_mobile"] = "android";
             url.searchParams.append("brd_mobile", "android");
             break;
         case "androidTablet":
+            params.query["brd_mobile"] = "android_tablet";
             url.searchParams.append("brd_mobile", "android_tablet");
             break;
         case "ipad":
+            params.query["brd_mobile"] = "ipad";
             url.searchParams.append("brd_mobile", "ipad");
             break;
         case "iphone":
+            params.query["brd_mobile"] = "ios";
             url.searchParams.append("brd_mobile", "ios");
             break;
     }
     // Stands for the encoded location you want to use for your search
     if (props?.geoLocation) {
-        url.searchParams.append("uule", props.country);
+        params.query["uule"] = props.geoLocation;
+        url.searchParams.append("uule", props.geoLocation);
     }
     if (props?.hotel) {
         if (props.hotel?.dates) {
+            params.query["hotel_dates"] = props.hotel.dates.toString();
             url.searchParams.append("hotel_dates", props.hotel.dates);
         }
         if (props.hotel?.occupancy) {
+            params.query["hotel_occupancy"] = props.hotel.occupancy.toString();
             url.searchParams.append("hotel_occupancy", props.hotel.occupancy.toString());
         }
     }
     // Two-letter language code used to define the page language
     if (props?.language) {
+        params.query["uule"] = props.language;
         url.searchParams.append("hl", props.language);
     }
     // Define search type. For regular search
     switch (props?.searchType) {
         case "images":
+            params.query["tbm"] = "isch";
             url.searchParams.append("tbm", "isch");
             break;
         case "jobs":
+            params.query["ibp"] = "htl;jobs";
             url.searchParams.append("ibp", "htl;jobs");
             break;
         case "news":
-            url.searchParams.append("tbm", "shop");
+            params.query["tbm"] = "nws";
+            url.searchParams.append("tbm", "nws");
             break;
         case "shopping":
-            url.searchParams.append("tbm", "nws");
+            params.query["tbm"] = "shop";
+            url.searchParams.append("tbm", "shop");
+            break;
+        case "video":
+            params.query["tbm"] = "vid";
+            url.searchParams.append("tbm", "vid");
             break;
     }
     if (props?.pagination) {
         // Define the result offset
         if (props.pagination?.offset) {
+            params.query["start"] = props.pagination.offset.toString();
             url.searchParams.append("start", props.pagination.offset.toString());
         }
         // Defines the number of results to return
         if (props.pagination?.perPage) {
+            params.query["num"] = props.pagination.offset.toString();
             url.searchParams.append("num", props.pagination.perPage.toString());
         }
     }
-    // Always return the JSON response with HTML field
-    url.searchParams.append("brn_json", "html");
+    // Specify the output type
+    if ("json" === props.output) {
+        params["brd_json"] = "1";
+        url.searchParams.append("brn_json", "1");
+    }
+    else {
+        params["brd_json"] = "html";
+        url.searchParams.append("brn_json", "html");
+    }
+    if ("json" === props.returnType) {
+        return params;
+    }
     return url.toString();
 };
 exports.default = buildGoogleSerpUrl;

package/brightData/serpApi/buildGoogleTrendsUrl.d.ts

@@ -5,6 +5,7 @@ type BuildGoogleTrendsUrl = {
     date?: string;
     searchType?: "images" | "news" | "shopping" | "youtube";
     query: string;
+    returnType: "url" | "json";
 };
 /**
  * Construct the Google Trends url using Bright Data's API:
@@ -13,5 +14,9 @@ type BuildGoogleTrendsUrl = {
  * For category use value from:
  * https://trends.google.com/trends/api/explore/pickers/category?lang=en-US&tz=240
  */
-declare const buildGoogleTrendsUrl: (props: BuildGoogleTrendsUrl) => string;
+declare const buildGoogleTrendsUrl: (props: BuildGoogleTrendsUrl) => string | {
+    query: {
+        q: string;
+    };
+};
 export default buildGoogleTrendsUrl;

package/brightData/serpApi/buildGoogleTrendsUrl.js

@@ -10,35 +10,51 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const buildGoogleTrendsUrl = (props) => {
     let url = new URL(`http://trends.google.com/trends/explore`);
     url.searchParams.append("q", props.query);
+    let params = {
+        query: {
+            q: props.query,
+        }
+    };
     // Location of interest, two-letter country code
     if (props?.country) {
+        params.query['geo'] = props.country;
         url.searchParams.append("geo", props.country);
     }
     // Preferred language, two-letter language code
     if (props?.language) {
-        url.searchParams.append("hl", props.country);
+        params.query['hl'] = props.language;
+        url.searchParams.append("hl", props.language);
     }
     if (props?.category) {
+        params.query['cat'] = props.category;
         url.searchParams.append("cat", props.category);
     }
     if (props?.date) {
+        params.query['date'] = props.date;
         url.searchParams.append("date", props.date);
     }
     // Define search type. For regular search
     switch (props?.searchType) {
         case "images":
+            params.query['group'] = "images";
             url.searchParams.append("gprop", "images");
             break;
         case "news":
+            params.query['group'] = "news";
             url.searchParams.append("gprop", "news");
             break;
         case "shopping":
+            params.query['group'] = "froogle";
             url.searchParams.append("gprop", "froogle");
             break;
         case "youtube":
+            params.query['group'] = "youtube";
             url.searchParams.append("gprop", "youtube");
             break;
     }
+    if ("json" === props.returnType) {
+        return params;
+    }
     return url.toString();
 };
 exports.default = buildGoogleTrendsUrl;

package/brightData/serpApi/getAsyncRequestId.d.ts

@@ -0,0 +1,14 @@
+import { Param } from "../param";
+import { Type } from "./type";
+type GetAsyncRequestId = {
+    auth: Param;
+    data: object;
+    type: Type;
+};
+/**
+ * Make an async request to the SERP API using the Bright Data proxy.
+ * Returns a request ID used with getAsyncResults
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
+ */
+declare const getAsyncRequestId: (props: GetAsyncRequestId) => Promise<string>;
+export default getAsyncRequestId;

package/brightData/serpApi/getAsyncRequestId.js

@@ -0,0 +1,50 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const post_1 = __importDefault(require("../../axios/post"));
+const getCustomerId_1 = __importDefault(require("../getCustomerId"));
+const getZone_1 = __importDefault(require("../getZone"));
+const type_1 = require("./type");
+/**
+ * Make an async request to the SERP API using the Bright Data proxy.
+ * Returns a request ID used with getAsyncResults
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
+ */
+const getAsyncRequestId = async (props) => {
+    let url = "";
+    let response = "";
+    // Prevent auth requests getting blocked
+    process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
+    const customerId = (0, getCustomerId_1.default)({
+        auth: props.auth,
+    });
+    const zone = (0, getZone_1.default)({
+        auth: props.auth,
+    });
+    switch (props.type) {
+        case type_1.Type.GOOGLE_TRENDS:
+            url = `https://api.brightdata.com/serp/trends?customer=${customerId}&zone=${zone}`;
+            break;
+        default:
+            url = `https://api.brightdata.com/serp/req?customer=${customerId}&zone=${zone}`;
+    }
+    const requestResponse = await (0, post_1.default)({
+        config: {
+            headers: {
+                Authorization: `Bearer ${props.auth.apiKey}`
+            },
+        },
+        data: props?.data || {},
+        url,
+    });
+    if (requestResponse.isSuccess()) {
+        response = requestResponse.getBodyDefaultTo({
+            defaultValue: "",
+            keys: ["response_id"],
+        });
+    }
+    return response;
+};
+exports.default = getAsyncRequestId;

package/brightData/serpApi/getAsyncResults.d.ts

@@ -0,0 +1,11 @@
+import { Param } from "../param";
+type GetAsync = {
+    auth: Param;
+    requestId: string;
+};
+/**
+ * Using the request ID from getAsyncRequestId fetch to see if the data is available.
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
+ */
+declare const getAsync: (props: GetAsync) => Promise<{}>;
+export default getAsync;

package/brightData/serpApi/getAsyncResults.js

@@ -0,0 +1,36 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const get_1 = __importDefault(require("../../axios/get"));
+const getCustomerId_1 = __importDefault(require("../getCustomerId"));
+const getZone_1 = __importDefault(require("../getZone"));
+/**
+ * Using the request ID from getAsyncRequestId fetch to see if the data is available.
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
+ */
+const getAsync = async (props) => {
+    let response = {};
+    // Prevent auth requests getting blocked
+    process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
+    const customerId = (0, getCustomerId_1.default)({
+        auth: props.auth,
+    });
+    const zone = (0, getZone_1.default)({
+        auth: props.auth,
+    });
+    const requestResponse = await (0, get_1.default)({
+        config: {
+            headers: {
+                Authorization: `Bearer ${props.auth.apiKey}`
+            },
+        },
+        url: `https://api.brightdata.com/serp/get_result?customer=${customerId}&zone=${zone}&response_id=${props.requestId}`,
+    });
+    if (requestResponse.isSuccess()) {
+        response = requestResponse.getBody();
+    }
+    return response;
+};
+exports.default = getAsync;

package/brightData/serpApi/getRealtime.d.ts

@@ -1,10 +1,6 @@
+import { Param } from "../param";
 type GetRealtime = {
-    auth: {
-        host: string;
-        port: string;
-        username: string;
-        password: string;
-    };
+    auth: Param;
     url: string;
 };
 /**

package/brightData/serpApi/type.d.ts

@@ -0,0 +1,11 @@
+export declare enum Type {
+    GOOGLE_FLIGHTS = "GOOGLE_FLIGHTS",
+    GOOGLE_HOTELS = "GOOGLE_HOTELS",
+    GOOGLE_LENS = "GOOGLE_LENS",
+    GOOGLE_MAPS = "GOOGLE_MAPS",
+    GOOGLE_REVIEWS = "GOOGLE_REVIEWS",
+    GOOGLE_SEARCH = "GOOGLE_SEARCH",
+    GOOGLE_TRENDS = "GOOGLE_TRENDS"
+}
+declare const _default: {};
+export default _default;

package/brightData/serpApi/type.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Type = void 0;
+var Type;
+(function (Type) {
+    Type["GOOGLE_FLIGHTS"] = "GOOGLE_FLIGHTS";
+    Type["GOOGLE_HOTELS"] = "GOOGLE_HOTELS";
+    Type["GOOGLE_LENS"] = "GOOGLE_LENS";
+    Type["GOOGLE_MAPS"] = "GOOGLE_MAPS";
+    Type["GOOGLE_REVIEWS"] = "GOOGLE_REVIEWS";
+    Type["GOOGLE_SEARCH"] = "GOOGLE_SEARCH";
+    Type["GOOGLE_TRENDS"] = "GOOGLE_TRENDS";
+})(Type || (exports.Type = Type = {}));
+exports.default = {};

package/brightData/webScraperIde/getBatch.d.ts

@@ -1,5 +1,6 @@
+import { Param } from "../param";
 type GetBatch = {
-    apiKey: string;
+    auth: Param;
     data?: object;
     scraperId: string;
 };

package/brightData/webScraperIde/getBatch.js

@@ -15,7 +15,7 @@ const getBatch = async (props) => {
     const requestResponse = await (0, post_1.default)({
         config: {
             headers: {
-                Authorization: `Bearer ${props.apiKey}`
+                Authorization: `Bearer ${props.auth.apiKey}`
             },
         },
         data: props?.data || {},

package/brightData/webScraperIde/getRealtime.d.ts

@@ -1,5 +1,6 @@
+import { Param } from "../param";
 type GetRealtime = {
-    apiKey: string;
+    auth: Param;
     data?: object;
     scraperId: string;
 };

package/brightData/webScraperIde/getRealtime.js

@@ -18,7 +18,7 @@ const getRealtime = async (props) => {
     const requestResponse = await (0, post_1.default)({
         config: {
             headers: {
-                Authorization: `Bearer ${props.apiKey}`
+                Authorization: `Bearer ${props.auth.apiKey}`
             },
         },
         data: props?.data || {},
@@ -38,7 +38,7 @@ const getRealtime = async (props) => {
                 const resultResponse = await (0, get_1.default)({
                     config: {
                         headers: {
-                            Authorization: `Bearer ${props.apiKey}`
+                            Authorization: `Bearer ${props.auth.apiKey}`
                         },
                     },
                     url: resultUrl,

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name":            "@8ms/helpers",
 	"license":         "UNLICENSED",
-	"version":         "1.27.0",
+	"version":         "1.29.0",
 	"repository":      {
 		"type": "git",
 		"url":  "git+https://github.com/8millionstories-organisation/8ms-helpers-ts.git"