google-img-scrap 1.1.8 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +2 -10
- package/dist/constant/GOOGLE_CONSTANT.d.ts +12 -11
- package/dist/constant/GOOGLE_CONSTANT.js +8 -6
- package/dist/constant/GOOGLE_CONSTANT.js.map +1 -1
- package/dist/core/buildGoogleDork.js +6 -20
- package/dist/core/buildGoogleDork.js.map +1 -1
- package/dist/core/constructGoogleUrl.js +3 -4
- package/dist/core/constructGoogleUrl.js.map +1 -1
- package/dist/core/cookies.d.ts +10 -0
- package/dist/core/cookies.js +115 -0
- package/dist/core/cookies.js.map +1 -0
- package/dist/core/parseGoogleImages.d.ts +1 -1
- package/dist/core/parseGoogleImages.js +11 -6
- package/dist/core/parseGoogleImages.js.map +1 -1
- package/dist/types/config.d.ts +2 -2
- package/package.json +9 -7
- package/tsconfig.json +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
### 1.2.1
|
|
4
|
+
|
|
5
|
+
- Fixed some bugs
|
|
6
|
+
- New structure for `filterByTitles` and `urlMatch`
|
|
7
|
+
|
|
8
|
+
### 1.2.0
|
|
9
|
+
|
|
10
|
+
- Since the last update of google we need to get the cookies from google. This is why we use `puppeteer-real-browser` in this new version to retrieve the cookie and save them in cache for faster requests after this.
|
|
11
|
+
|
|
3
12
|
### 1.1.8
|
|
4
13
|
|
|
5
14
|
- Removed duplicates result
|
package/README.md
CHANGED
|
@@ -186,16 +186,8 @@ console.log(test);
|
|
|
186
186
|
```js
|
|
187
187
|
const test = await GOOGLE_IMG_SCRAP({
|
|
188
188
|
search: 'cats',
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
['draw', 'white'],
|
|
192
|
-
['albino', 'white']
|
|
193
|
-
],
|
|
194
|
-
//will build something like this "(cdn and wikipedia) or (cdn istockphoto)"
|
|
195
|
-
urlMatch: [
|
|
196
|
-
['cdn', 'wikipedia'],
|
|
197
|
-
['cdn', 'istockphoto']
|
|
198
|
-
]
|
|
189
|
+
filterByTitles: ['draw', 'albino'],
|
|
190
|
+
urlMatch: ['wikipedia', 'istockphoto']
|
|
199
191
|
});
|
|
200
192
|
|
|
201
193
|
console.log(test);
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
declare const GOOGLE_CONSTANT: {
|
|
2
|
-
url:
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
readonly url: "https://www.google.com/search";
|
|
3
|
+
readonly queryParam: "tbs";
|
|
4
|
+
readonly forceGoogleImage: {
|
|
5
|
+
readonly tbm: "isch";
|
|
6
|
+
readonly udm: "2";
|
|
7
7
|
};
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
'
|
|
8
|
+
readonly headers: {
|
|
9
|
+
readonly accept: "text/html";
|
|
10
|
+
readonly 'accept-encoding': "gzip, deflate";
|
|
11
|
+
readonly 'accept-language': "en-US,en";
|
|
12
|
+
readonly referer: "https://www.google.com/";
|
|
13
|
+
readonly 'upgrade-insecure-requests': "1";
|
|
14
|
+
readonly 'User-Agent': "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
|
|
14
15
|
};
|
|
15
16
|
};
|
|
16
17
|
export default GOOGLE_CONSTANT;
|
|
@@ -2,16 +2,18 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const GOOGLE_CONSTANT = {
|
|
4
4
|
url: 'https://www.google.com/search',
|
|
5
|
-
inverse: {
|
|
6
|
-
base: 'https://lens.google.com/',
|
|
7
|
-
upload: 'https://lens.google.com/v3/upload',
|
|
8
|
-
url: 'https://lens.google.com/uploadbyurl?url='
|
|
9
|
-
},
|
|
10
5
|
queryParam: 'tbs',
|
|
6
|
+
//needed to search on google image instead of google
|
|
11
7
|
forceGoogleImage: {
|
|
12
|
-
tbm: 'isch'
|
|
8
|
+
tbm: 'isch',
|
|
9
|
+
udm: '2'
|
|
13
10
|
},
|
|
14
11
|
headers: {
|
|
12
|
+
accept: 'text/html',
|
|
13
|
+
'accept-encoding': 'gzip, deflate',
|
|
14
|
+
'accept-language': 'en-US,en',
|
|
15
|
+
referer: 'https://www.google.com/',
|
|
16
|
+
'upgrade-insecure-requests': '1',
|
|
15
17
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'
|
|
16
18
|
}
|
|
17
19
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GOOGLE_CONSTANT.js","sourceRoot":"","sources":["../../src/constant/GOOGLE_CONSTANT.ts"],"names":[],"mappings":";;AAAA,MAAM,eAAe,GAAG;IACtB,GAAG,EAAE,+BAA+B;IACpC,
|
|
1
|
+
{"version":3,"file":"GOOGLE_CONSTANT.js","sourceRoot":"","sources":["../../src/constant/GOOGLE_CONSTANT.ts"],"names":[],"mappings":";;AAAA,MAAM,eAAe,GAAG;IACtB,GAAG,EAAE,+BAA+B;IACpC,UAAU,EAAE,KAAK;IACjB,oDAAoD;IACpD,gBAAgB,EAAE;QAChB,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,GAAG;KACT;IACD,OAAO,EAAE;QACP,MAAM,EAAE,WAAW;QACnB,iBAAiB,EAAE,eAAe;QAClC,iBAAiB,EAAE,UAAU;QAC7B,OAAO,EAAE,yBAAyB;QAClC,2BAA2B,EAAE,GAAG;QAChC,YAAY,EACV,uGAAuG;KAC1G;CACO,CAAC;AAEX,kBAAe,eAAe,CAAC"}
|
|
@@ -6,16 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
* @returns
|
|
7
7
|
*/
|
|
8
8
|
function filterByTitlesBuilder(config) {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const value = titleFilter.map(title => {
|
|
13
|
-
return `intitle:"${title}"`;
|
|
14
|
-
});
|
|
15
|
-
FILTER_TITLE.push(`(${value.join(' AND ')})`);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
return FILTER_TITLE.join(' ');
|
|
9
|
+
return config.filterByTitles && config.filterByTitles.length > 0
|
|
10
|
+
? '(' + config.filterByTitles.map(title => `intitle:"${title}"`).join(' OR ') + ')'
|
|
11
|
+
: '';
|
|
19
12
|
}
|
|
20
13
|
/**
|
|
21
14
|
* Show only images without some specific words
|
|
@@ -65,16 +58,9 @@ function excludeDomainsBuilder(config) {
|
|
|
65
58
|
* @returns
|
|
66
59
|
*/
|
|
67
60
|
function urlMatchBuilder(config) {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const value = urlMatch.map(content => {
|
|
72
|
-
return `inurl:${content}`;
|
|
73
|
-
});
|
|
74
|
-
URL_MATCH.push(`(${value.join(' AND ')})`);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return URL_MATCH.join(' OR ');
|
|
61
|
+
return config.urlMatch && config.urlMatch.length > 0
|
|
62
|
+
? '(' + config.urlMatch.map(match => `inurl:"${match}"`).join(' OR ') + ')'
|
|
63
|
+
: '';
|
|
78
64
|
}
|
|
79
65
|
/**
|
|
80
66
|
* Build google dork string based on the config query
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"buildGoogleDork.js","sourceRoot":"","sources":["../../src/core/buildGoogleDork.ts"],"names":[],"mappings":";;AAEA;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,MAAc;IAC3C,MAAM,
|
|
1
|
+
{"version":3,"file":"buildGoogleDork.js","sourceRoot":"","sources":["../../src/core/buildGoogleDork.ts"],"names":[],"mappings":";;AAEA;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,MAAc;IAC3C,OAAO,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;QAC9D,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG;QACnF,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,MAAc;IACzC,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,MAAM,CAAC,YAAY,EAAE;QACvB,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,YAAY,EAAE;YAC7C,aAAa,CAAC,IAAI,CAAC,KAAK,WAAW,GAAG,CAAC,CAAC;SACzC;KACF;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAc;IACxC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE;YACnC,OAAO,CAAC,IAAI,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC;SAClC;KACF;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC9B,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,MAAc;IAC3C,MAAM,eAAe,GAAG,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,cAAc,EAAE;QACzB,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,cAAc,EAAE;YACjD,eAAe,CAAC,IAAI,CAAC,UAAU,aAAa,GAAG,CAAC,CAAC;SAClD;KACF;IACD,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnC,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;QAClD,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG;QAC3E,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,OAAO;QACL,MAAM,CAAC,MAAM;QACb,eAAe,CAAC,MAAM,CAAC;QACvB,mBAAmB,CAAC,MAAM,CAAC;QAC3B,qBAAqB,CAAC,MAAM,CAAC;QAC7B,kBAAkB,CAAC,MAAM,CAAC;QAC1B,qBAAqB,CAAC,MAAM,CAAC;KAC9B;SACE,IAAI,CAAC,GAAG,CAAC;SACT,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,kBAAe,gBAAgB,CAAC"}
|
|
@@ -15,10 +15,9 @@ function constructGoogleUrl(config) {
|
|
|
15
15
|
const GOOGLE_DORK = (0, buildGoogleDork_1.default)(config);
|
|
16
16
|
const CUSTOM_PARAM = config.custom ? `&${config.custom}` : '';
|
|
17
17
|
const SAFE_SEARCH = config.safeSearch ? `&safe=active` : '';
|
|
18
|
-
const QUERY = Object.assign(GOOGLE_CONSTANT_1.default.forceGoogleImage, {
|
|
19
|
-
[GOOGLE_CONSTANT_1.default.queryParam]: Object.values(config.query
|
|
20
|
-
|
|
21
|
-
});
|
|
18
|
+
const QUERY = Object.assign(GOOGLE_CONSTANT_1.default.forceGoogleImage, Object.assign(Object.assign({}, (config.query && {
|
|
19
|
+
[GOOGLE_CONSTANT_1.default.queryParam]: Object.values(config.query).join(',')
|
|
20
|
+
})), { q: GOOGLE_DORK }));
|
|
22
21
|
return GOOGLE_CONSTANT_1.default.url + (0, utils_1.buildQuery)(QUERY) + CUSTOM_PARAM + SAFE_SEARCH;
|
|
23
22
|
}
|
|
24
23
|
exports.default = constructGoogleUrl;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constructGoogleUrl.js","sourceRoot":"","sources":["../../src/core/constructGoogleUrl.ts"],"names":[],"mappings":";;;;;AACA,kFAA0D;AAC1D,0CAA4C;AAC5C,wEAAiD;AAEjD;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAc;IACxC,MAAM,WAAW,GAAG,IAAA,yBAAgB,EAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,yBAAe,CAAC,gBAAgB,
|
|
1
|
+
{"version":3,"file":"constructGoogleUrl.js","sourceRoot":"","sources":["../../src/core/constructGoogleUrl.ts"],"names":[],"mappings":";;;;;AACA,kFAA0D;AAC1D,0CAA4C;AAC5C,wEAAiD;AAEjD;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAc;IACxC,MAAM,WAAW,GAAG,IAAA,yBAAgB,EAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,yBAAe,CAAC,gBAAgB,kCACvD,CAAC,MAAM,CAAC,KAAK,IAAI;QAClB,CAAC,yBAAe,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;KACpE,CAAC,KACF,CAAC,EAAE,WAAW,IACd,CAAC;IAEH,OAAO,yBAAe,CAAC,GAAG,GAAG,IAAA,kBAAU,EAAC,KAAK,CAAC,GAAG,YAAY,GAAG,WAAW,CAAC;AAC9E,CAAC;AAED,kBAAe,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.getCookies = exports.clearCacheCookies = void 0;
|
|
16
|
+
const fs_1 = __importDefault(require("fs"));
|
|
17
|
+
const path_1 = __importDefault(require("path"));
|
|
18
|
+
const puppeteer_real_browser_1 = require("puppeteer-real-browser");
|
|
19
|
+
const GOOGLE_CONSTANT_1 = __importDefault(require("../constant/GOOGLE_CONSTANT"));
|
|
20
|
+
const CACHE_FILE = path_1.default.join(__dirname, 'cookies-cache.json');
|
|
21
|
+
// Cache lifetime (1 hour)
|
|
22
|
+
const CACHE_DURATION = 60 * 60 * 1000;
|
|
23
|
+
/**
|
|
24
|
+
* Check if the cache is still valid.
|
|
25
|
+
* The cache is only valid if:
|
|
26
|
+
* - The proxy is the same
|
|
27
|
+
* - The cache has not expired
|
|
28
|
+
*/
|
|
29
|
+
function isCacheValid(cache, proxy) {
|
|
30
|
+
const now = Date.now();
|
|
31
|
+
return cache.proxy === proxy && now - cache.timestamp < CACHE_DURATION;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Load cookies from cache.
|
|
35
|
+
* Returns null if:
|
|
36
|
+
* - Cache does not exist
|
|
37
|
+
* - Proxy changed
|
|
38
|
+
* - Cache expired
|
|
39
|
+
*/
|
|
40
|
+
function loadCache(proxy) {
|
|
41
|
+
if (!fs_1.default.existsSync(CACHE_FILE)) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
const cache = JSON.parse(fs_1.default.readFileSync(CACHE_FILE, 'utf-8'));
|
|
45
|
+
if (!isCacheValid(cache, proxy)) {
|
|
46
|
+
fs_1.default.unlinkSync(CACHE_FILE);
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
return cache.cookies;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Save cookies with the current proxy.
|
|
53
|
+
*/
|
|
54
|
+
function saveCache(cookies, proxy) {
|
|
55
|
+
const cache = {
|
|
56
|
+
cookies,
|
|
57
|
+
proxy,
|
|
58
|
+
timestamp: Date.now()
|
|
59
|
+
};
|
|
60
|
+
fs_1.default.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2));
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Clear the cookies cache file.
|
|
64
|
+
*/
|
|
65
|
+
function clearCacheCookies() {
|
|
66
|
+
if (fs_1.default.existsSync(CACHE_FILE)) {
|
|
67
|
+
fs_1.default.unlinkSync(CACHE_FILE);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
exports.clearCacheCookies = clearCacheCookies;
|
|
71
|
+
/**
|
|
72
|
+
* Convert Puppeteer cookies into a Cookie header string.
|
|
73
|
+
*/
|
|
74
|
+
function cookiesToString(cookies) {
|
|
75
|
+
return cookies.map(cookie => `${cookie.name}=${cookie.value}`).join('; ');
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Open browser and retrieve new cookies.
|
|
79
|
+
*/
|
|
80
|
+
function fetchNewCookies(proxy) {
|
|
81
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
82
|
+
const { browser, page } = yield (0, puppeteer_real_browser_1.connect)({
|
|
83
|
+
headless: true,
|
|
84
|
+
args: [`--proxy-server=${proxy}`]
|
|
85
|
+
});
|
|
86
|
+
yield page.setUserAgent(GOOGLE_CONSTANT_1.default.headers['User-Agent']);
|
|
87
|
+
yield page.setExtraHTTPHeaders({
|
|
88
|
+
'Accept-Language': 'en-US,en;q=0.9'
|
|
89
|
+
});
|
|
90
|
+
yield page.goto('https://www.google.com/search?udm=2&q=cats', {
|
|
91
|
+
waitUntil: 'networkidle2'
|
|
92
|
+
});
|
|
93
|
+
const cookies = yield page.browserContext().cookies();
|
|
94
|
+
const cookieString = cookiesToString(cookies);
|
|
95
|
+
yield browser.close();
|
|
96
|
+
saveCache(cookieString, proxy);
|
|
97
|
+
return cookieString;
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Main function.
|
|
102
|
+
*
|
|
103
|
+
* Returns cookies matching the current proxy.
|
|
104
|
+
*/
|
|
105
|
+
function getCookies(proxy) {
|
|
106
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
107
|
+
const cachedCookies = loadCache(proxy);
|
|
108
|
+
if (cachedCookies) {
|
|
109
|
+
return cachedCookies;
|
|
110
|
+
}
|
|
111
|
+
return yield fetchNewCookies(proxy);
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
exports.getCookies = getCookies;
|
|
115
|
+
//# sourceMappingURL=cookies.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookies.js","sourceRoot":"","sources":["../../src/core/cookies.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,4CAAoB;AACpB,gDAAwB;AACxB,mEAAiD;AACjD,kFAA0D;AAE1D,MAAM,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;AAE9D,0BAA0B;AAC1B,MAAM,cAAc,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAQtC;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAAkB,EAAE,KAAa;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,OAAO,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,GAAG,cAAc,CAAC;AACzE,CAAC;AAED;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,KAAa;IAC9B,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC9B,OAAO,IAAI,CAAC;KACb;IAED,MAAM,KAAK,GAAgB,IAAI,CAAC,KAAK,CAAC,YAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAE5E,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE;QAC/B,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC;KACb;IAED,OAAO,KAAK,CAAC,OAAO,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,OAAe,EAAE,KAAa;IAC/C,MAAM,KAAK,GAAgB;QACzB,OAAO;QACP,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,YAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB;IAC/B,IAAI,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC7B,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;KAC3B;AACH,CAAC;AAJD,8CAIC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,OAAc;IACrC,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,SAAe,eAAe,CAAC,KAAa;;QAC1C,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,IAAA,gCAAO,EAAC;YACtC,QAAQ,EAAE,IAAI;YACd,IAAI,EAAE,CAAC,kBAAkB,KAAK,EAAE,CAAC;SAClC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,YAAY,CAAC,yBAAe,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAE/D,MAAM,IAAI,CAAC,mBAAmB,CAAC;YAC7B,iBAAiB,EAAE,gBAAgB;SACpC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,IAAI,CAAC,4CAA4C,EAAE;YAC5D,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,OAAO,EAAE,CAAC;QAEtD,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QAE9C,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEtB,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAE/B,OAAO,YAAY,CAAC;IACtB,CAAC;CAAA;AAED;;;;GAIG;AACH,SAAsB,UAAU,CAAC,KAAa;;QAC5C,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAEvC,IAAI,aAAa,EAAE;YACjB,OAAO,aAAa,CAAC;SACtB;QAED,OAAO,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;CAAA;AARD,gCAQC"}
|
|
@@ -4,5 +4,5 @@ import ImageResultItem from '../types/imageResultItem';
|
|
|
4
4
|
* @param url
|
|
5
5
|
* @returns
|
|
6
6
|
*/
|
|
7
|
-
declare function parseGoogleImages(url: string, proxy?: string): Promise<ImageResultItem[]>;
|
|
7
|
+
declare function parseGoogleImages(url: string, proxy?: string, resetCookies?: boolean): Promise<ImageResultItem[]>;
|
|
8
8
|
export default parseGoogleImages;
|
|
@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
15
15
|
const utils_1 = require("../utils/utils");
|
|
16
16
|
const GOOGLE_CONSTANT_1 = __importDefault(require("../constant/GOOGLE_CONSTANT"));
|
|
17
17
|
const impit_1 = require("impit");
|
|
18
|
+
const cookies_1 = require("./cookies");
|
|
18
19
|
/**
|
|
19
20
|
* Scrap google images scripts tag
|
|
20
21
|
* @param url
|
|
@@ -22,18 +23,17 @@ const impit_1 = require("impit");
|
|
|
22
23
|
*/
|
|
23
24
|
function scrapGoogleImages(url, proxy) {
|
|
24
25
|
return __awaiter(this, void 0, void 0, function* () {
|
|
26
|
+
const cookies = yield (0, cookies_1.getCookies)(proxy || '');
|
|
25
27
|
const impit = new impit_1.Impit({
|
|
26
|
-
browser: '
|
|
28
|
+
browser: 'chrome',
|
|
27
29
|
proxyUrl: proxy,
|
|
28
30
|
ignoreTlsErrors: true,
|
|
29
31
|
followRedirects: true,
|
|
30
32
|
timeout: 30000,
|
|
31
33
|
maxRedirects: 5,
|
|
32
|
-
headers: GOOGLE_CONSTANT_1.default.headers
|
|
33
|
-
});
|
|
34
|
-
const req = yield impit.fetch(url, {
|
|
35
|
-
headers: GOOGLE_CONSTANT_1.default.headers
|
|
34
|
+
headers: Object.assign(Object.assign({}, GOOGLE_CONSTANT_1.default.headers), { cookie: cookies })
|
|
36
35
|
});
|
|
36
|
+
const req = yield impit.fetch(url);
|
|
37
37
|
return req.text();
|
|
38
38
|
});
|
|
39
39
|
}
|
|
@@ -58,7 +58,7 @@ function getGoogleImageObject(informationsMatch, otherInformationsMatch) {
|
|
|
58
58
|
* @param url
|
|
59
59
|
* @returns
|
|
60
60
|
*/
|
|
61
|
-
function parseGoogleImages(url, proxy) {
|
|
61
|
+
function parseGoogleImages(url, proxy, resetCookies = true) {
|
|
62
62
|
return __awaiter(this, void 0, void 0, function* () {
|
|
63
63
|
const processedIds = new Set();
|
|
64
64
|
const result = [];
|
|
@@ -85,6 +85,11 @@ function parseGoogleImages(url, proxy) {
|
|
|
85
85
|
processedIds.add(obj.id);
|
|
86
86
|
result.push(obj);
|
|
87
87
|
}
|
|
88
|
+
// If we don't get any results, it might be due to invalid cookies. Clear the cache and try again.
|
|
89
|
+
if (result.length === 0 && resetCookies) {
|
|
90
|
+
(0, cookies_1.clearCacheCookies)();
|
|
91
|
+
return yield parseGoogleImages(url, proxy, false);
|
|
92
|
+
}
|
|
88
93
|
return result;
|
|
89
94
|
});
|
|
90
95
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parseGoogleImages.js","sourceRoot":"","sources":["../../src/core/parseGoogleImages.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0CAA+C;AAC/C,kFAA0D;AAE1D,iCAA8B;
|
|
1
|
+
{"version":3,"file":"parseGoogleImages.js","sourceRoot":"","sources":["../../src/core/parseGoogleImages.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0CAA+C;AAC/C,kFAA0D;AAE1D,iCAA8B;AAC9B,uCAA0D;AAE1D;;;;GAIG;AACH,SAAe,iBAAiB,CAAC,GAAW,EAAE,KAAc;;QAC1D,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAU,EAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,IAAI,aAAK,CAAC;YACtB,OAAO,EAAE,QAAQ;YACjB,QAAQ,EAAE,KAAK;YACf,eAAe,EAAE,IAAI;YACrB,eAAe,EAAE,IAAI;YACrB,OAAO,EAAE,KAAM;YACf,YAAY,EAAE,CAAC;YACf,OAAO,kCACF,yBAAe,CAAC,OAAO,KAC1B,MAAM,EAAE,OAAO,GAChB;SACF,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEnC,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;CAAA;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,iBAAkC,EAClC,sBAAuC;IAEvC,OAAO;QACL,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC;QAC7B,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC;QAChC,GAAG,EAAE,IAAA,qBAAa,EAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;QACxC,WAAW,EAAE,sBAAsB,CAAC,CAAC,CAAC;QACtC,MAAM,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1C,KAAK,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAe,iBAAiB,CAC9B,GAAW,EACX,KAAc,EACd,YAAY,GAAG,IAAI;;QAEnB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,MAAM,MAAM,GAAsB,EAAE,CAAC;QAErC,MAAM,IAAI,GAAW,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAEzD,iDAAiD;QACjD,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;QAC7D,gCAAgC;QAChC,MAAM,sBAAsB,GAAG,mDAAmD,CAAC;QAEnF,IAAI,iBAAyC,CAAC;QAE9C,OAAO,CAAC,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;YAClE,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,oCAAoC,CAAC;gBAAE,SAAS;YAEpF,MAAM,sBAAsB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjE,IAAI,sBAAsB,KAAK,IAAI;gBAAE,OAAO,MAAM,CAAC;YAEnD,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,IAAI,sBAAsB,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAChF,IACE,iBAAiB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,CAAC;gBAChD,sBAAsB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,CAAC;gBAErD,SAAS;YAEX,MAAM,GAAG,GAAG,oBAAoB,CAAC,iBAAiB,EAAE,sBAAsB,CAAC,CAAC;YAC5E,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YACvC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAClB;QAED,kGAAkG;QAClG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,EAAE;YACvC,IAAA,2BAAiB,GAAE,CAAC;YACpB,OAAO,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SACnD;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CAAA;AAED,kBAAe,iBAAiB,CAAC"}
|
package/dist/types/config.d.ts
CHANGED
|
@@ -10,12 +10,12 @@ type Config = {
|
|
|
10
10
|
LICENCE?: string;
|
|
11
11
|
EXTENSION?: string;
|
|
12
12
|
};
|
|
13
|
-
urlMatch?: string[]
|
|
13
|
+
urlMatch?: string[];
|
|
14
14
|
domains?: string[];
|
|
15
15
|
excludeWords?: string[];
|
|
16
16
|
custom?: string;
|
|
17
17
|
safeSearch?: boolean;
|
|
18
18
|
excludeDomains?: string[];
|
|
19
|
-
filterByTitles?: string[]
|
|
19
|
+
filterByTitles?: string[];
|
|
20
20
|
};
|
|
21
21
|
export default Config;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "google-img-scrap",
|
|
3
|
-
"version": "1.1
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"description": "Scrap images from google images with customs pre filled google dork options",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -10,10 +10,11 @@
|
|
|
10
10
|
"fastBuild": "tsc",
|
|
11
11
|
"lint": "eslint . --ext .ts",
|
|
12
12
|
"prettier": "prettier --write .",
|
|
13
|
-
"test": "
|
|
13
|
+
"test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js"
|
|
14
14
|
},
|
|
15
15
|
"files": [
|
|
16
16
|
"dist",
|
|
17
|
+
"!dist/core/cookies-cache.json",
|
|
17
18
|
"CHANGELOG.md",
|
|
18
19
|
"README.md",
|
|
19
20
|
"tsconfig.json",
|
|
@@ -61,17 +62,18 @@
|
|
|
61
62
|
"devDependencies": {
|
|
62
63
|
"@types/jest": "^29.5.14",
|
|
63
64
|
"@types/node": "^18.11.18",
|
|
64
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
65
|
-
"@typescript-eslint/parser": "^8.
|
|
65
|
+
"@typescript-eslint/eslint-plugin": "^8.60.1",
|
|
66
|
+
"@typescript-eslint/parser": "^8.60.1",
|
|
66
67
|
"eslint": "^8.57.0",
|
|
67
68
|
"eslint-config-prettier": "^9.1.0",
|
|
68
69
|
"jest": "^29.0.0",
|
|
69
|
-
"prettier": "^3.
|
|
70
|
-
"ts-jest": "^29.
|
|
70
|
+
"prettier": "^3.8.3",
|
|
71
|
+
"ts-jest": "^29.4.11",
|
|
71
72
|
"ts-node": "^10.9.1",
|
|
72
73
|
"typescript": "^4.7.4"
|
|
73
74
|
},
|
|
74
75
|
"dependencies": {
|
|
75
|
-
"impit": "^0.
|
|
76
|
+
"impit": "^0.14.1",
|
|
77
|
+
"puppeteer-real-browser": "^1.4.4"
|
|
76
78
|
}
|
|
77
79
|
}
|