google-img-scrap 1.1.6 → 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +2 -6
- package/dist/core/limitResultSize.d.ts +1 -1
- package/dist/core/limitResultSize.js.map +1 -1
- package/dist/core/parseGoogleImages.d.ts +1 -2
- package/dist/core/parseGoogleImages.js +19 -6
- package/dist/core/parseGoogleImages.js.map +1 -1
- package/dist/types/config.d.ts +1 -2
- package/package.json +5 -6
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -27,7 +27,7 @@ import { GOOGLE_IMG_SCRAP, GOOGLE_QUERY } from 'google-img-scrap';
|
|
|
27
27
|
## Options definition
|
|
28
28
|
|
|
29
29
|
- "search" `string` what you want to search
|
|
30
|
-
- "proxy" `
|
|
30
|
+
- "proxy" `string` configure a proxy url
|
|
31
31
|
- "excludeWords" `string[]` exclude some words from the search
|
|
32
32
|
- "domains" `string[]` filter by domains
|
|
33
33
|
- "excludeDomains" `string[]` exclude some domains
|
|
@@ -116,11 +116,7 @@ See axios documentation to setup the proxy
|
|
|
116
116
|
```js
|
|
117
117
|
const test = await GOOGLE_IMG_SCRAP({
|
|
118
118
|
search: 'cats',
|
|
119
|
-
proxy:
|
|
120
|
-
protocol: 'https',
|
|
121
|
-
host: 'example.com',
|
|
122
|
-
port: 8080
|
|
123
|
-
}
|
|
119
|
+
proxy: 'http://user:pass@proxy:80'
|
|
124
120
|
});
|
|
125
121
|
|
|
126
122
|
console.log(test);
|
|
@@ -5,5 +5,5 @@ import ImageResultItem from '../types/imageResultItem';
|
|
|
5
5
|
* @param imagesItems
|
|
6
6
|
* @returns
|
|
7
7
|
*/
|
|
8
|
-
declare function limitResultSize(limit: number, imagesItems: ImageResultItem[]): ImageResultItem[];
|
|
8
|
+
declare function limitResultSize(limit: number | undefined, imagesItems: ImageResultItem[]): ImageResultItem[];
|
|
9
9
|
export default limitResultSize;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"limitResultSize.js","sourceRoot":"","sources":["../../src/core/limitResultSize.ts"],"names":[],"mappings":";;AAEA;;;;;GAKG;AACH,SAAS,eAAe,
|
|
1
|
+
{"version":3,"file":"limitResultSize.js","sourceRoot":"","sources":["../../src/core/limitResultSize.ts"],"names":[],"mappings":";;AAEA;;;;;GAKG;AACH,SAAS,eAAe,CACtB,KAAyB,EACzB,WAA8B;IAE9B,IAAI,YAAY,GAAsB,EAAE,CAAC;IAEzC,IAAI,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK,EAAE;QACpD,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;KAC5C;IAED,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC;AAC9D,CAAC;AAED,kBAAe,eAAe,CAAC"}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import ImageResultItem from '../types/imageResultItem';
|
|
2
|
-
import { HttpsProxyAgent } from 'https-proxy-agent';
|
|
3
2
|
/**
|
|
4
3
|
* Parse the html from google image to get the images links
|
|
5
4
|
* @param url
|
|
6
5
|
* @returns
|
|
7
6
|
*/
|
|
8
|
-
declare function parseGoogleImages(url: string, proxy?:
|
|
7
|
+
declare function parseGoogleImages(url: string, proxy?: string): Promise<ImageResultItem[]>;
|
|
9
8
|
export default parseGoogleImages;
|
|
@@ -14,7 +14,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
const utils_1 = require("../utils/utils");
|
|
16
16
|
const GOOGLE_CONSTANT_1 = __importDefault(require("../constant/GOOGLE_CONSTANT"));
|
|
17
|
-
const
|
|
17
|
+
const impit_1 = require("impit");
|
|
18
18
|
/**
|
|
19
19
|
* Scrap google images scripts tag
|
|
20
20
|
* @param url
|
|
@@ -22,11 +22,19 @@ const axios_1 = __importDefault(require("axios"));
|
|
|
22
22
|
*/
|
|
23
23
|
function scrapGoogleImages(url, proxy) {
|
|
24
24
|
return __awaiter(this, void 0, void 0, function* () {
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
const impit = new impit_1.Impit({
|
|
26
|
+
browser: 'firefox',
|
|
27
|
+
proxyUrl: proxy,
|
|
28
|
+
ignoreTlsErrors: true,
|
|
29
|
+
followRedirects: true,
|
|
30
|
+
timeout: 30000,
|
|
31
|
+
maxRedirects: 5,
|
|
32
|
+
headers: GOOGLE_CONSTANT_1.default.headers
|
|
28
33
|
});
|
|
29
|
-
|
|
34
|
+
const req = yield impit.fetch(url, {
|
|
35
|
+
headers: GOOGLE_CONSTANT_1.default.headers
|
|
36
|
+
});
|
|
37
|
+
return req.text();
|
|
30
38
|
});
|
|
31
39
|
}
|
|
32
40
|
/**
|
|
@@ -52,6 +60,7 @@ function getGoogleImageObject(informationsMatch, otherInformationsMatch) {
|
|
|
52
60
|
*/
|
|
53
61
|
function parseGoogleImages(url, proxy) {
|
|
54
62
|
return __awaiter(this, void 0, void 0, function* () {
|
|
63
|
+
const processedIds = new Set();
|
|
55
64
|
const result = [];
|
|
56
65
|
const body = yield scrapGoogleImages(url, proxy);
|
|
57
66
|
//getting image url, height, width, color average
|
|
@@ -70,7 +79,11 @@ function parseGoogleImages(url, proxy) {
|
|
|
70
79
|
if (informationsMatch[1].match(/http/gi).length > 2 ||
|
|
71
80
|
otherInformationsMatch[2].match(/http/gi).length > 2)
|
|
72
81
|
continue;
|
|
73
|
-
|
|
82
|
+
const obj = getGoogleImageObject(informationsMatch, otherInformationsMatch);
|
|
83
|
+
if (processedIds.has(obj.id))
|
|
84
|
+
continue;
|
|
85
|
+
processedIds.add(obj.id);
|
|
86
|
+
result.push(obj);
|
|
74
87
|
}
|
|
75
88
|
return result;
|
|
76
89
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parseGoogleImages.js","sourceRoot":"","sources":["../../src/core/parseGoogleImages.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0CAA+C;AAC/C,kFAA0D;
|
|
1
|
+
{"version":3,"file":"parseGoogleImages.js","sourceRoot":"","sources":["../../src/core/parseGoogleImages.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0CAA+C;AAC/C,kFAA0D;AAE1D,iCAA8B;AAE9B;;;;GAIG;AACH,SAAe,iBAAiB,CAAC,GAAW,EAAE,KAAc;;QAC1D,MAAM,KAAK,GAAG,IAAI,aAAK,CAAC;YACtB,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE,KAAK;YACf,eAAe,EAAE,IAAI;YACrB,eAAe,EAAE,IAAI;YACrB,OAAO,EAAE,KAAM;YACf,YAAY,EAAE,CAAC;YACf,OAAO,EAAE,yBAAe,CAAC,OAAO;SACjC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE,yBAAe,CAAC,OAAO;SACjC,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;CAAA;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,iBAAkC,EAClC,sBAAuC;IAEvC,OAAO;QACL,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC;QAC7B,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC;QAChC,GAAG,EAAE,IAAA,qBAAa,EAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;QACxC,WAAW,EAAE,sBAAsB,CAAC,CAAC,CAAC;QACtC,MAAM,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1C,KAAK,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAe,iBAAiB,CAAC,GAAW,EAAE,KAAc;;QAC1D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,MAAM,MAAM,GAAsB,EAAE,CAAC;QAErC,MAAM,IAAI,GAAW,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAEzD,iDAAiD;QACjD,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;QAC7D,gCAAgC;QAChC,MAAM,sBAAsB,GAAG,mDAAmD,CAAC;QAEnF,IAAI,iBAAyC,CAAC;QAE9C,OAAO,CAAC,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;YAClE,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,oCAAoC,CAAC;gBAAE,SAAS;YAEpF,MAAM,sBAAsB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjE,IAAI,sBAAsB,KAAK,IAAI;gBAAE,OAAO,MAAM,CAAC;YAEnD,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,IAAI,sBAAsB,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAChF,IACE,iBAAiB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,CAAC;gBAChD,sBAAsB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,CAAC;gBAErD,SAAS;YAEX,MAAM,GAAG,GAAG,oBAAoB,CAAC,iBAAiB,EAAE,sBAAsB,CAAC,CAAC;YAC5E,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YACvC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAClB;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CAAA;AAED,kBAAe,iBAAiB,CAAC"}
|
package/dist/types/config.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "google-img-scrap",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.8",
|
|
4
4
|
"description": "Scrap images from google images with customs pre filled google dork options",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -58,12 +58,8 @@
|
|
|
58
58
|
"url": "https://github.com/yoannchb-pro/google-img-scrap/issues"
|
|
59
59
|
},
|
|
60
60
|
"homepage": "https://github.com/yoannchb-pro/google-img-scrap#readme",
|
|
61
|
-
"dependencies": {
|
|
62
|
-
"axios": "^1.2.3",
|
|
63
|
-
"https-proxy-agent": "^7.0.6"
|
|
64
|
-
},
|
|
65
61
|
"devDependencies": {
|
|
66
|
-
"@types/jest": "^29.
|
|
62
|
+
"@types/jest": "^29.5.14",
|
|
67
63
|
"@types/node": "^18.11.18",
|
|
68
64
|
"@typescript-eslint/eslint-plugin": "^8.13.0",
|
|
69
65
|
"@typescript-eslint/parser": "^8.13.0",
|
|
@@ -74,5 +70,8 @@
|
|
|
74
70
|
"ts-jest": "^29.0.5",
|
|
75
71
|
"ts-node": "^10.9.1",
|
|
76
72
|
"typescript": "^4.7.4"
|
|
73
|
+
},
|
|
74
|
+
"dependencies": {
|
|
75
|
+
"impit": "^0.9.2"
|
|
77
76
|
}
|
|
78
77
|
}
|