@8ms/helpers 1.23.0 → 1.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,5 +5,5 @@ type IsFileExists = {
5
5
  /**
6
6
  * Check to see if a file exists but don't get the contents.
7
7
  */
8
- declare const isFileExists: ({ bucket, key }: IsFileExists) => Promise<boolean>;
8
+ declare const isFileExists: (props: IsFileExists) => Promise<boolean>;
9
9
  export default isFileExists;
@@ -7,14 +7,14 @@ const isResponse200_1 = __importDefault(require("../isResponse200"));
7
7
  /**
8
8
  * Check to see if a file exists but don't get the contents.
9
9
  */
10
- const isFileExists = async ({ bucket, key }) => {
10
+ const isFileExists = async (props) => {
11
11
  let response = false;
12
12
  try {
13
- const { HeadObjectCommand } = require('@aws-sdk/client-s3');
13
+ const { HeadObjectCommand } = require("@aws-sdk/client-s3");
14
14
  // Fetch from S3
15
15
  const apiResponse = await global.awsS3Client.send(new HeadObjectCommand({
16
- Bucket: bucket,
17
- Key: key,
16
+ Bucket: props.bucket,
17
+ Key: props.key,
18
18
  }));
19
19
  // If the status is 200 then it does exist
20
20
  response = (0, isResponse200_1.default)({ apiResponse });
@@ -0,0 +1,26 @@
1
+ type BuildGoogleSerpUrl = {
2
+ country?: string;
3
+ device?: "mobile" | "android" | "androidTablet" | "iphone" | "ipad";
4
+ language?: string;
5
+ geoLocation?: string;
6
+ hotel?: {
7
+ dates?: string;
8
+ occupancy?: number;
9
+ };
10
+ pagination?: {
11
+ offset?: number;
12
+ perPage?: number;
13
+ };
14
+ searchType?: "images" | "jobs" | "news" | "shopping";
15
+ query: string;
16
+ googleDomain: string;
17
+ };
18
+ /**
19
+ * Construct the Google SERP url using Bright Data's API:
20
+ * https://brightdata.com/cp/serp_api/api/google/search?id=c_2cef083f
21
+ *
22
+ * For geolocation use value from:
23
+ * https://developers.google.com/google-ads/api/data/geotargets
24
+ */
25
+ declare const buildGoogleSerpUrl: (props: BuildGoogleSerpUrl) => URL;
26
+ export default buildGoogleSerpUrl;
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ /**
4
+ * Construct the Google SERP url using Bright Data's API:
5
+ * https://brightdata.com/cp/serp_api/api/google/search?id=c_2cef083f
6
+ *
7
+ * For geolocation use value from:
8
+ * https://developers.google.com/google-ads/api/data/geotargets
9
+ */
10
+ const buildGoogleSerpUrl = (props) => {
11
+ let url = new URL(`https://www${props.googleDomain}/`);
12
+ url.searchParams.append("q", props.query);
13
+ // Two-letter country code used to define the country of search
14
+ if (props?.country) {
15
+ url.searchParams.append("gl", props.country);
16
+ }
17
+ // Define what device type to be represented in user-agent - Default desktop
18
+ switch (props?.device) {
19
+ case "mobile":
20
+ url.searchParams.append("brd_mobile", "1");
21
+ break;
22
+ case "android":
23
+ url.searchParams.append("brd_mobile", "android");
24
+ break;
25
+ case "androidTablet":
26
+ url.searchParams.append("brd_mobile", "android_tablet");
27
+ break;
28
+ case "ipad":
29
+ url.searchParams.append("brd_mobile", "ipad");
30
+ break;
31
+ case "iphone":
32
+ url.searchParams.append("brd_mobile", "ios");
33
+ break;
34
+ }
35
+ // Stands for the encoded location you want to use for your search
36
+ if (props?.geoLocation) {
37
+ url.searchParams.append("uule", props.country);
38
+ }
39
+ if (props?.hotel) {
40
+ if (props.hotel?.dates) {
41
+ url.searchParams.append("hotel_dates", props.hotel.dates);
42
+ }
43
+ if (props.hotel?.occupancy) {
44
+ url.searchParams.append("hotel_occupancy", props.hotel.occupancy.toString());
45
+ }
46
+ }
47
+ // Two-letter language code used to define the page language
48
+ if (props?.language) {
49
+ url.searchParams.append("hl", props.language);
50
+ }
51
+ // Define search type. For regular search
52
+ switch (props?.searchType) {
53
+ case "images":
54
+ url.searchParams.append("tbm", "isch");
55
+ break;
56
+ case "jobs":
57
+ url.searchParams.append("ibp", "htl;jobs");
58
+ break;
59
+ case "news":
60
+ url.searchParams.append("tbm", "shop");
61
+ break;
62
+ case "shopping":
63
+ url.searchParams.append("tbm", "nws");
64
+ break;
65
+ }
66
+ if (props?.pagination) {
67
+ // Define the result offset
68
+ if (props.pagination?.offset) {
69
+ url.searchParams.append("start", props.pagination.offset.toString());
70
+ }
71
+ // Defines the number of results to return
72
+ if (props.pagination?.perPage) {
73
+ url.searchParams.append("num", props.pagination.perPage.toString());
74
+ }
75
+ }
76
+ // Always return the JSON response with HTML field
77
+ url.searchParams.append("brn_json", "html");
78
+ return url;
79
+ };
80
+ exports.default = buildGoogleSerpUrl;
@@ -0,0 +1,17 @@
1
+ type BuildGoogleTrendsUrl = {
2
+ country?: string;
3
+ language?: string;
4
+ category?: string;
5
+ date?: string;
6
+ searchType?: "images" | "news" | "shopping" | "youtube";
7
+ query: string;
8
+ };
9
+ /**
10
+ * Construct the Google Trends url using Bright Data's API:
11
+ * https://brightdata.com/cp/serp_api/api/google/trends?id=c_2cef083f
12
+ *
13
+ * For category use value from:
14
+ * https://trends.google.com/trends/api/explore/pickers/category?lang=en-US&tz=240
15
+ */
16
+ declare const buildGoogleTrendsUrl: (props: BuildGoogleTrendsUrl) => URL;
17
+ export default buildGoogleTrendsUrl;
@@ -0,0 +1,44 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ /**
4
+ * Construct the Google Trends url using Bright Data's API:
5
+ * https://brightdata.com/cp/serp_api/api/google/trends?id=c_2cef083f
6
+ *
7
+ * For category use value from:
8
+ * https://trends.google.com/trends/api/explore/pickers/category?lang=en-US&tz=240
9
+ */
10
+ const buildGoogleTrendsUrl = (props) => {
11
+ let url = new URL(`https://trends.google.com/`);
12
+ url.searchParams.append("q", props.query);
13
+ // Location of interest, two-letter country code
14
+ if (props?.country) {
15
+ url.searchParams.append("geo", props.country);
16
+ }
17
+ // Preferred language, two-letter language code
18
+ if (props?.language) {
19
+ url.searchParams.append("hl", props.country);
20
+ }
21
+ if (props?.category) {
22
+ url.searchParams.append("cat", props.category);
23
+ }
24
+ if (props?.date) {
25
+ url.searchParams.append("date", props.date);
26
+ }
27
+ // Define search type. For regular search
28
+ switch (props?.searchType) {
29
+ case "images":
30
+ url.searchParams.append("gprop", "images");
31
+ break;
32
+ case "news":
33
+ url.searchParams.append("gprop", "news");
34
+ break;
35
+ case "shopping":
36
+ url.searchParams.append("gprop", "froogle");
37
+ break;
38
+ case "youtube":
39
+ url.searchParams.append("gprop", "youtube");
40
+ break;
41
+ }
42
+ return url;
43
+ };
44
+ exports.default = buildGoogleTrendsUrl;
@@ -0,0 +1,11 @@
1
+ type GetRealtime = {
2
+ customerId: string;
3
+ host: string;
4
+ zonePassword: string;
5
+ url: string;
6
+ };
7
+ /**
8
+ * Make a request to the SERP API using the Bright Data proxy
9
+ */
10
+ declare const getRealtime: (props: GetRealtime) => Promise<{}>;
11
+ export default getRealtime;
@@ -0,0 +1,33 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const get_1 = __importDefault(require("../../axios/get"));
7
+ /**
8
+ * Make a request to the SERP API using the Bright Data proxy
9
+ */
10
+ const getRealtime = async (props) => {
11
+ let response = {};
12
+ const { HttpsProxyAgent } = require("https-proxy-agent");
13
+ // Prevent auth requests getting blocked
14
+ process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
15
+ // Construct the proxy URL
16
+ const proxyUrl = `http://${props.customerId}:${props.zonePassword}@${props.host}`;
17
+ // Replace <proxy_url> with your actual proxy URL
18
+ const httpsAgent = new HttpsProxyAgent(proxyUrl);
19
+ // Use fetch with the agent option to make an HTTP request through the proxy
20
+ // Replace <target_url> with the URL you want to request
21
+ const apiResponse = await (0, get_1.default)({
22
+ config: {
23
+ proxy: false,
24
+ httpsAgent,
25
+ },
26
+ url: props.url,
27
+ });
28
+ if (apiResponse.isSuccess()) {
29
+ response = apiResponse.getBody();
30
+ }
31
+ return response;
32
+ };
33
+ exports.default = getRealtime;
@@ -0,0 +1,12 @@
1
+ type GetBatch = {
2
+ apiKey: string;
3
+ data?: object;
4
+ scraperId: string;
5
+ };
6
+ /**
7
+ * Depends on the Scraper setup!
8
+ *
9
+ * Make the request to trigger the scraper and let it do its job.
10
+ */
11
+ declare const getBatch: (props: GetBatch) => Promise<boolean>;
12
+ export default getBatch;
@@ -0,0 +1,29 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const post_1 = __importDefault(require("../../axios/post"));
7
+ /**
8
+ * Depends on the Scraper setup!
9
+ *
10
+ * Make the request to trigger the scraper and let it do its job.
11
+ */
12
+ const getBatch = async (props) => {
13
+ let response = false;
14
+ const requestUrl = `https://api.brightdata.com/dca/trigger?collector=${props.scraperId}&queue_next=1`;
15
+ const requestResponse = await (0, post_1.default)({
16
+ config: {
17
+ headers: {
18
+ Authorization: `Bearer ${props.apiKey}`
19
+ },
20
+ },
21
+ data: props?.data || {},
22
+ url: requestUrl,
23
+ });
24
+ if (requestResponse.isSuccess()) {
25
+ response = true;
26
+ }
27
+ return response;
28
+ };
29
+ exports.default = getBatch;
@@ -0,0 +1,12 @@
1
+ type GetRealtime = {
2
+ apiKey: string;
3
+ data?: object;
4
+ scraperId: string;
5
+ };
6
+ /**
7
+ * Depends on the Scraper setup!
8
+ *
9
+ * Make the request and pool the response until the data is available.
10
+ */
11
+ declare const getRealtime: (props: GetRealtime) => Promise<{}>;
12
+ export default getRealtime;
@@ -0,0 +1,64 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const post_1 = __importDefault(require("../../axios/post"));
7
+ const states_1 = __importDefault(require("../../api/states"));
8
+ const get_1 = __importDefault(require("../../axios/get"));
9
+ const sleep_1 = __importDefault(require("../../util/sleep"));
10
+ /**
11
+ * Depends on the Scraper setup!
12
+ *
13
+ * Make the request and pool the response until the data is available.
14
+ */
15
+ const getRealtime = async (props) => {
16
+ let response = {};
17
+ const requestUrl = `https://api.brightdata.com/dca/trigger_immediate?collector=${props.scraperId}`;
18
+ const requestResponse = await (0, post_1.default)({
19
+ config: {
20
+ headers: {
21
+ Authorization: `Bearer ${props.apiKey}`
22
+ },
23
+ },
24
+ data: props?.data || {},
25
+ url: requestUrl,
26
+ });
27
+ if (requestResponse.isSuccess()) {
28
+ const responseId = requestResponse.getBodyDefaultTo({
29
+ defaultValue: "",
30
+ keys: ["response_id"],
31
+ });
32
+ if ("" !== responseId) {
33
+ let state = states_1.default.PENDING;
34
+ while (states_1.default.PENDING === state) {
35
+ const resultUrl = `https://api.brightdata.com/dca/get_result?response_id=${responseId}`;
36
+ // Use fetch with the agent option to make an HTTP request through the proxy
37
+ // Replace <target_url> with the URL you want to request
38
+ const resultResponse = await (0, get_1.default)({
39
+ config: {
40
+ headers: {
41
+ Authorization: `Bearer ${props.apiKey}`
42
+ },
43
+ },
44
+ url: resultUrl,
45
+ });
46
+ if (resultResponse.isSuccess()) {
47
+ if (undefined !== resultResponse.body.pending &&
48
+ true === resultResponse.body.pending) {
49
+ state = states_1.default.PENDING;
50
+ }
51
+ else {
52
+ state = states_1.default.SUCCESS;
53
+ response = resultResponse.getBody();
54
+ }
55
+ }
56
+ await (0, sleep_1.default)({
57
+ seconds: 1
58
+ });
59
+ }
60
+ }
61
+ }
62
+ return response;
63
+ };
64
+ exports.default = getRealtime;
@@ -0,0 +1,10 @@
1
+ type IsFileExists = {
2
+ bucket: string;
3
+ key: string;
4
+ projectId?: string;
5
+ };
6
+ /**
7
+ * Check to see if a file exists but don't get the contents.
8
+ */
9
+ declare const isFileExists: (props: IsFileExists) => Promise<boolean>;
10
+ export default isFileExists;
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const getClient_1 = __importDefault(require("./getClient"));
7
+ /**
8
+ * Check to see if a file exists but don't get the contents.
9
+ */
10
+ const isFileExists = async (props) => {
11
+ const { bucket, key, projectId, } = props;
12
+ const googleFile = await (0, getClient_1.default)({ projectId })
13
+ .bucket(bucket)
14
+ .file(key)
15
+ .exists();
16
+ const response = googleFile?.[0];
17
+ return response;
18
+ };
19
+ exports.default = isFileExists;
@@ -2,11 +2,10 @@ type WriteFile = {
2
2
  bucket: string;
3
3
  data: any;
4
4
  key: string;
5
- projectId?: string;
6
5
  [options: string]: any;
7
6
  };
8
7
  /**
9
8
  * Write a file to Google Cloud Storage.
10
9
  */
11
- export declare const writeFile: ({ bucket, contentType, data, key, projectId, ...options }: WriteFile) => Promise<any>;
10
+ export declare const writeFile: (props: WriteFile) => Promise<any>;
12
11
  export default writeFile;
@@ -8,8 +8,9 @@ const getClient_1 = __importDefault(require("./getClient"));
8
8
  /**
9
9
  * Write a file to Google Cloud Storage.
10
10
  */
11
- const writeFile = async ({ bucket, contentType, data, key, projectId, ...options }) => {
12
- const stream = require('stream');
11
+ const writeFile = async (props) => {
12
+ const { bucket, contentType, data, key, projectId, ...options } = props;
13
+ const stream = require("stream");
13
14
  const dataStream = new stream.PassThrough();
14
15
  const googleFile = (0, getClient_1.default)({ projectId })
15
16
  .bucket(bucket)
@@ -18,16 +19,16 @@ const writeFile = async ({ bucket, contentType, data, key, projectId, ...options
18
19
  dataStream.push(null);
19
20
  return await new Promise((resolve, reject) => {
20
21
  dataStream.pipe(googleFile.createWriteStream({
21
- contentType: 'auto',
22
+ contentType: "auto",
22
23
  resumable: false,
23
24
  validation: false,
24
25
  metadata: {},
25
26
  ...options,
26
27
  }))
27
- .on('error', (error) => {
28
+ .on("error", (error) => {
28
29
  reject(error);
29
30
  })
30
- .on('finish', () => {
31
+ .on("finish", () => {
31
32
  resolve(true);
32
33
  });
33
34
  });
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@8ms/helpers",
3
3
  "license": "UNLICENSED",
4
- "version": "1.23.0",
4
+ "version": "1.25.0",
5
5
  "repository": {
6
6
  "type": "git",
7
7
  "url": "git+https://github.com/8millionstories-organisation/8ms-helpers-ts.git"
@@ -39,6 +39,7 @@
39
39
  "@types/lodash": "4.17.7",
40
40
  "@types/node": "20.14.12",
41
41
  "babel-jest": "29.7.0",
42
+ "https-proxy-agent": "7.0.5",
42
43
  "jest": "29.7.0",
43
44
  "node-fetch": "3.3.2",
44
45
  "stream-chain": "2.2.5",