@8ms/helpers 1.28.0 → 1.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,10 @@
1
+ import { Param } from "./param";
2
+ type GetCustomerId = {
3
+ auth: Param;
4
+ prefix?: string;
5
+ };
6
+ /**
7
+ * Return the customer Id based on the username.
8
+ */
9
+ declare const getCustomerId: (props: GetCustomerId) => string;
10
+ export default getCustomerId;
@@ -0,0 +1,11 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ /**
4
+ * Return the customer Id based on the username.
5
+ */
6
+ const getCustomerId = (props) => {
7
+ const segments = props.auth.username.split("-")[2];
8
+ const customerId = segments[2].replace("hl_", "");
9
+ return (props?.prefix || "") + customerId;
10
+ };
11
+ exports.default = getCustomerId;
@@ -0,0 +1,9 @@
1
+ import { Param } from "./param";
2
+ type GetZone = {
3
+ auth: Param;
4
+ };
5
+ /**
6
+ * Return the Zone based on the username.
7
+ */
8
+ declare const getZone: (props: GetZone) => string;
9
+ export default getZone;
@@ -0,0 +1,10 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ /**
4
+ * Return the Zone based on the username.
5
+ */
6
+ const getZone = (props) => {
7
+ const segments = props.auth.username.split("-");
8
+ return segments[segments.length - 1];
9
+ };
10
+ exports.default = getZone;
@@ -0,0 +1,9 @@
1
+ export type Param = {
2
+ apiKey: string;
3
+ host: string;
4
+ port: string;
5
+ username: string;
6
+ password: string;
7
+ };
8
+ declare const _default: {};
9
+ export default _default;
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.default = {};
@@ -11,10 +11,11 @@ type BuildGoogleSerpUrl = {
11
11
  offset?: number;
12
12
  perPage?: number;
13
13
  };
14
- searchType?: "images" | "jobs" | "news" | "shopping";
14
+ searchType?: "images" | "jobs" | "news" | "shopping" | "video";
15
15
  query: string;
16
16
  googleDomain: string;
17
17
  output?: "html" | "json";
18
+ returnType: "url" | "json";
18
19
  };
19
20
  /**
20
21
  * Construct the Google SERP url using Bright Data's API:
@@ -23,5 +24,9 @@ type BuildGoogleSerpUrl = {
23
24
  * For geolocation use value from:
24
25
  * https://developers.google.com/google-ads/api/data/geotargets
25
26
  */
26
- declare const buildGoogleSerpUrl: (props: BuildGoogleSerpUrl) => string;
27
+ declare const buildGoogleSerpUrl: (props: BuildGoogleSerpUrl) => string | {
28
+ query: {
29
+ q: string;
30
+ };
31
+ };
27
32
  export default buildGoogleSerpUrl;
@@ -9,77 +9,107 @@ Object.defineProperty(exports, "__esModule", { value: true });
9
9
  */
10
10
  const buildGoogleSerpUrl = (props) => {
11
11
  let url = new URL(`https://www.${props.googleDomain}/search`);
12
+ let params = {
13
+ query: {
14
+ q: props.query,
15
+ }
16
+ };
12
17
  url.searchParams.append("q", props.query);
13
18
  // Two-letter country code used to define the country of search
14
19
  if (props?.country) {
20
+ params.query["gl"] = props.country;
15
21
  url.searchParams.append("gl", props.country);
16
22
  }
17
23
  // Define what device type to be represented in user-agent - Default desktop
18
24
  switch (props?.device) {
19
25
  case "mobile":
26
+ params.query["brd_mobile"] = "1";
20
27
  url.searchParams.append("brd_mobile", "1");
21
28
  break;
22
29
  case "android":
30
+ params.query["brd_mobile"] = "android";
23
31
  url.searchParams.append("brd_mobile", "android");
24
32
  break;
25
33
  case "androidTablet":
34
+ params.query["brd_mobile"] = "android_tablet";
26
35
  url.searchParams.append("brd_mobile", "android_tablet");
27
36
  break;
28
37
  case "ipad":
38
+ params.query["brd_mobile"] = "ipad";
29
39
  url.searchParams.append("brd_mobile", "ipad");
30
40
  break;
31
41
  case "iphone":
42
+ params.query["brd_mobile"] = "ios";
32
43
  url.searchParams.append("brd_mobile", "ios");
33
44
  break;
34
45
  }
35
46
  // Stands for the encoded location you want to use for your search
36
47
  if (props?.geoLocation) {
37
- url.searchParams.append("uule", props.country);
48
+ params.query["uule"] = props.geoLocation;
49
+ url.searchParams.append("uule", props.geoLocation);
38
50
  }
39
51
  if (props?.hotel) {
40
52
  if (props.hotel?.dates) {
53
+ params.query["hotel_dates"] = props.hotel.dates.toString();
41
54
  url.searchParams.append("hotel_dates", props.hotel.dates);
42
55
  }
43
56
  if (props.hotel?.occupancy) {
57
+ params.query["hotel_occupancy"] = props.hotel.occupancy.toString();
44
58
  url.searchParams.append("hotel_occupancy", props.hotel.occupancy.toString());
45
59
  }
46
60
  }
47
61
  // Two-letter language code used to define the page language
48
62
  if (props?.language) {
63
+ params.query["uule"] = props.language;
49
64
  url.searchParams.append("hl", props.language);
50
65
  }
51
66
  // Define search type. For regular search
52
67
  switch (props?.searchType) {
53
68
  case "images":
69
+ params.query["tbm"] = "isch";
54
70
  url.searchParams.append("tbm", "isch");
55
71
  break;
56
72
  case "jobs":
73
+ params.query["ibp"] = "htl;jobs";
57
74
  url.searchParams.append("ibp", "htl;jobs");
58
75
  break;
59
76
  case "news":
60
- url.searchParams.append("tbm", "shop");
77
+ params.query["tbm"] = "nws";
78
+ url.searchParams.append("tbm", "nws");
61
79
  break;
62
80
  case "shopping":
63
- url.searchParams.append("tbm", "nws");
81
+ params.query["tbm"] = "shop";
82
+ url.searchParams.append("tbm", "shop");
83
+ break;
84
+ case "video":
85
+ params.query["tbm"] = "vid";
86
+ url.searchParams.append("tbm", "vid");
64
87
  break;
65
88
  }
66
89
  if (props?.pagination) {
67
90
  // Define the result offset
68
91
  if (props.pagination?.offset) {
92
+ params.query["start"] = props.pagination.offset.toString();
69
93
  url.searchParams.append("start", props.pagination.offset.toString());
70
94
  }
71
95
  // Defines the number of results to return
72
96
  if (props.pagination?.perPage) {
97
+ params.query["num"] = props.pagination.offset.toString();
73
98
  url.searchParams.append("num", props.pagination.perPage.toString());
74
99
  }
75
100
  }
76
101
  // Specify the output type
77
102
  if ("json" === props.output) {
103
+ params["brd_json"] = "1";
78
104
  url.searchParams.append("brn_json", "1");
79
105
  }
80
106
  else {
107
+ params["brd_json"] = "html";
81
108
  url.searchParams.append("brn_json", "html");
82
109
  }
110
+ if ("json" === props.returnType) {
111
+ return params;
112
+ }
83
113
  return url.toString();
84
114
  };
85
115
  exports.default = buildGoogleSerpUrl;
@@ -5,6 +5,7 @@ type BuildGoogleTrendsUrl = {
5
5
  date?: string;
6
6
  searchType?: "images" | "news" | "shopping" | "youtube";
7
7
  query: string;
8
+ returnType: "url" | "json";
8
9
  };
9
10
  /**
10
11
  * Construct the Google Trends url using Bright Data's API:
@@ -13,5 +14,9 @@ type BuildGoogleTrendsUrl = {
13
14
  * For category use value from:
14
15
  * https://trends.google.com/trends/api/explore/pickers/category?lang=en-US&tz=240
15
16
  */
16
- declare const buildGoogleTrendsUrl: (props: BuildGoogleTrendsUrl) => string;
17
+ declare const buildGoogleTrendsUrl: (props: BuildGoogleTrendsUrl) => string | {
18
+ query: {
19
+ q: string;
20
+ };
21
+ };
17
22
  export default buildGoogleTrendsUrl;
@@ -10,35 +10,51 @@ Object.defineProperty(exports, "__esModule", { value: true });
10
10
  const buildGoogleTrendsUrl = (props) => {
11
11
  let url = new URL(`http://trends.google.com/trends/explore`);
12
12
  url.searchParams.append("q", props.query);
13
+ let params = {
14
+ query: {
15
+ q: props.query,
16
+ }
17
+ };
13
18
  // Location of interest, two-letter country code
14
19
  if (props?.country) {
20
+ params.query['geo'] = props.country;
15
21
  url.searchParams.append("geo", props.country);
16
22
  }
17
23
  // Preferred language, two-letter language code
18
24
  if (props?.language) {
19
- url.searchParams.append("hl", props.country);
25
+ params.query['hl'] = props.language;
26
+ url.searchParams.append("hl", props.language);
20
27
  }
21
28
  if (props?.category) {
29
+ params.query['cat'] = props.category;
22
30
  url.searchParams.append("cat", props.category);
23
31
  }
24
32
  if (props?.date) {
33
+ params.query['date'] = props.date;
25
34
  url.searchParams.append("date", props.date);
26
35
  }
27
36
  // Define search type. For regular search
28
37
  switch (props?.searchType) {
29
38
  case "images":
39
+ params.query['group'] = "images";
30
40
  url.searchParams.append("gprop", "images");
31
41
  break;
32
42
  case "news":
43
+ params.query['group'] = "news";
33
44
  url.searchParams.append("gprop", "news");
34
45
  break;
35
46
  case "shopping":
47
+ params.query['group'] = "froogle";
36
48
  url.searchParams.append("gprop", "froogle");
37
49
  break;
38
50
  case "youtube":
51
+ params.query['group'] = "youtube";
39
52
  url.searchParams.append("gprop", "youtube");
40
53
  break;
41
54
  }
55
+ if ("json" === props.returnType) {
56
+ return params;
57
+ }
42
58
  return url.toString();
43
59
  };
44
60
  exports.default = buildGoogleTrendsUrl;
@@ -0,0 +1,14 @@
1
+ import { Param } from "../param";
2
+ import { Type } from "./type";
3
+ type GetAsyncRequestId = {
4
+ auth: Param;
5
+ data: object;
6
+ type: Type;
7
+ };
8
+ /**
9
+ * Make an async request to the SERP API using the Bright Data proxy.
10
+ * Returns a request ID used with getAsyncResults
11
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
12
+ */
13
+ declare const getAsyncRequestId: (props: GetAsyncRequestId) => Promise<string>;
14
+ export default getAsyncRequestId;
@@ -0,0 +1,50 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const post_1 = __importDefault(require("../../axios/post"));
7
+ const getCustomerId_1 = __importDefault(require("../getCustomerId"));
8
+ const getZone_1 = __importDefault(require("../getZone"));
9
+ const type_1 = require("./type");
10
+ /**
11
+ * Make an async request to the SERP API using the Bright Data proxy.
12
+ * Returns a request ID used with getAsyncResults
13
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
14
+ */
15
+ const getAsyncRequestId = async (props) => {
16
+ let url = "";
17
+ let response = "";
18
+ // Prevent auth requests getting blocked
19
+ process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
20
+ const customerId = (0, getCustomerId_1.default)({
21
+ auth: props.auth,
22
+ });
23
+ const zone = (0, getZone_1.default)({
24
+ auth: props.auth,
25
+ });
26
+ switch (props.type) {
27
+ case type_1.Type.GOOGLE_TRENDS:
28
+ url = `https://api.brightdata.com/serp/trends?customer=${customerId}&zone=${zone}`;
29
+ break;
30
+ default:
31
+ url = `https://api.brightdata.com/serp/req?customer=${customerId}&zone=${zone}`;
32
+ }
33
+ const requestResponse = await (0, post_1.default)({
34
+ config: {
35
+ headers: {
36
+ Authorization: `Bearer ${props.auth.apiKey}`
37
+ },
38
+ },
39
+ data: props?.data || {},
40
+ url,
41
+ });
42
+ if (requestResponse.isSuccess()) {
43
+ response = requestResponse.getBodyDefaultTo({
44
+ defaultValue: "",
45
+ keys: ["response_id"],
46
+ });
47
+ }
48
+ return response;
49
+ };
50
+ exports.default = getAsyncRequestId;
@@ -0,0 +1,11 @@
1
+ import { Param } from "../param";
2
+ type GetAsync = {
3
+ auth: Param;
4
+ requestId: string;
5
+ };
6
+ /**
7
+ * Using the request ID from getAsyncRequestId fetch to see if the data is available.
8
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
9
+ */
10
+ declare const getAsync: (props: GetAsync) => Promise<{}>;
11
+ export default getAsync;
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const get_1 = __importDefault(require("../../axios/get"));
7
+ const getCustomerId_1 = __importDefault(require("../getCustomerId"));
8
+ const getZone_1 = __importDefault(require("../getZone"));
9
+ /**
10
+ * Using the request ID from getAsyncRequestId fetch to see if the data is available.
11
+ * https://docs.brightdata.com/scraping-automation/serp-api/asynchronous-requests
12
+ */
13
+ const getAsync = async (props) => {
14
+ let response = {};
15
+ // Prevent auth requests getting blocked
16
+ process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
17
+ const customerId = (0, getCustomerId_1.default)({
18
+ auth: props.auth,
19
+ });
20
+ const zone = (0, getZone_1.default)({
21
+ auth: props.auth,
22
+ });
23
+ const requestResponse = await (0, get_1.default)({
24
+ config: {
25
+ headers: {
26
+ Authorization: `Bearer ${props.auth.apiKey}`
27
+ },
28
+ },
29
+ url: `https://api.brightdata.com/serp/get_result?customer=${customerId}&zone=${zone}&response_id=${props.requestId}`,
30
+ });
31
+ if (requestResponse.isSuccess()) {
32
+ response = requestResponse.getBody();
33
+ }
34
+ return response;
35
+ };
36
+ exports.default = getAsync;
@@ -1,10 +1,6 @@
1
+ import { Param } from "../param";
1
2
  type GetRealtime = {
2
- auth: {
3
- host: string;
4
- port: string;
5
- username: string;
6
- password: string;
7
- };
3
+ auth: Param;
8
4
  url: string;
9
5
  };
10
6
  /**
@@ -0,0 +1,11 @@
1
+ export declare enum Type {
2
+ GOOGLE_FLIGHTS = "GOOGLE_FLIGHTS",
3
+ GOOGLE_HOTELS = "GOOGLE_HOTELS",
4
+ GOOGLE_LENS = "GOOGLE_LENS",
5
+ GOOGLE_MAPS = "GOOGLE_MAPS",
6
+ GOOGLE_REVIEWS = "GOOGLE_REVIEWS",
7
+ GOOGLE_SEARCH = "GOOGLE_SEARCH",
8
+ GOOGLE_TRENDS = "GOOGLE_TRENDS"
9
+ }
10
+ declare const _default: {};
11
+ export default _default;
@@ -0,0 +1,14 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Type = void 0;
4
+ var Type;
5
+ (function (Type) {
6
+ Type["GOOGLE_FLIGHTS"] = "GOOGLE_FLIGHTS";
7
+ Type["GOOGLE_HOTELS"] = "GOOGLE_HOTELS";
8
+ Type["GOOGLE_LENS"] = "GOOGLE_LENS";
9
+ Type["GOOGLE_MAPS"] = "GOOGLE_MAPS";
10
+ Type["GOOGLE_REVIEWS"] = "GOOGLE_REVIEWS";
11
+ Type["GOOGLE_SEARCH"] = "GOOGLE_SEARCH";
12
+ Type["GOOGLE_TRENDS"] = "GOOGLE_TRENDS";
13
+ })(Type || (exports.Type = Type = {}));
14
+ exports.default = {};
@@ -1,5 +1,6 @@
1
+ import { Param } from "../param";
1
2
  type GetBatch = {
2
- apiKey: string;
3
+ auth: Param;
3
4
  data?: object;
4
5
  scraperId: string;
5
6
  };
@@ -15,7 +15,7 @@ const getBatch = async (props) => {
15
15
  const requestResponse = await (0, post_1.default)({
16
16
  config: {
17
17
  headers: {
18
- Authorization: `Bearer ${props.apiKey}`
18
+ Authorization: `Bearer ${props.auth.apiKey}`
19
19
  },
20
20
  },
21
21
  data: props?.data || {},
@@ -1,5 +1,6 @@
1
+ import { Param } from "../param";
1
2
  type GetRealtime = {
2
- apiKey: string;
3
+ auth: Param;
3
4
  data?: object;
4
5
  scraperId: string;
5
6
  };
@@ -18,7 +18,7 @@ const getRealtime = async (props) => {
18
18
  const requestResponse = await (0, post_1.default)({
19
19
  config: {
20
20
  headers: {
21
- Authorization: `Bearer ${props.apiKey}`
21
+ Authorization: `Bearer ${props.auth.apiKey}`
22
22
  },
23
23
  },
24
24
  data: props?.data || {},
@@ -38,7 +38,7 @@ const getRealtime = async (props) => {
38
38
  const resultResponse = await (0, get_1.default)({
39
39
  config: {
40
40
  headers: {
41
- Authorization: `Bearer ${props.apiKey}`
41
+ Authorization: `Bearer ${props.auth.apiKey}`
42
42
  },
43
43
  },
44
44
  url: resultUrl,
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@8ms/helpers",
3
3
  "license": "UNLICENSED",
4
- "version": "1.28.0",
4
+ "version": "1.29.0",
5
5
  "repository": {
6
6
  "type": "git",
7
7
  "url": "git+https://github.com/8millionstories-organisation/8ms-helpers-ts.git"