google-img-scrap 1.1.7 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +1 -1
- package/dist/constant/GOOGLE_CONSTANT.d.ts +12 -11
- package/dist/constant/GOOGLE_CONSTANT.js +8 -6
- package/dist/constant/GOOGLE_CONSTANT.js.map +1 -1
- package/dist/core/buildGoogleDork.js +3 -9
- package/dist/core/buildGoogleDork.js.map +1 -1
- package/dist/core/constructGoogleUrl.js +3 -4
- package/dist/core/constructGoogleUrl.js.map +1 -1
- package/dist/core/cookies-cache.json +5 -0
- package/dist/core/cookies.d.ts +10 -0
- package/dist/core/cookies.js +115 -0
- package/dist/core/cookies.js.map +1 -0
- package/dist/core/parseGoogleImages.d.ts +1 -1
- package/dist/core/parseGoogleImages.js +17 -7
- package/dist/core/parseGoogleImages.js.map +1 -1
- package/package.json +8 -7
- package/tsconfig.json +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
### 1.2.0
|
|
4
|
+
|
|
5
|
+
- Since the last update of google we need to get the cookies from google. This is why we use `puppeteer-real-browser` in this new version to retrieve the cookie and save them in cache for faster requests after this.
|
|
6
|
+
|
|
7
|
+
### 1.1.8
|
|
8
|
+
|
|
9
|
+
- Removed duplicates result
|
|
10
|
+
|
|
3
11
|
### 1.1.7
|
|
4
12
|
|
|
5
13
|
- Moved from axios to impit
|
package/README.md
CHANGED
|
@@ -191,7 +191,7 @@ const test = await GOOGLE_IMG_SCRAP({
|
|
|
191
191
|
['draw', 'white'],
|
|
192
192
|
['albino', 'white']
|
|
193
193
|
],
|
|
194
|
-
//will build something like this "(cdn and wikipedia) or (cdn istockphoto)"
|
|
194
|
+
//will build something like this "(cdn and wikipedia) or (cdn and istockphoto)"
|
|
195
195
|
urlMatch: [
|
|
196
196
|
['cdn', 'wikipedia'],
|
|
197
197
|
['cdn', 'istockphoto']
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
declare const GOOGLE_CONSTANT: {
|
|
2
|
-
url:
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
readonly url: "https://www.google.com/search";
|
|
3
|
+
readonly queryParam: "tbs";
|
|
4
|
+
readonly forceGoogleImage: {
|
|
5
|
+
readonly tbm: "isch";
|
|
6
|
+
readonly udm: "2";
|
|
7
7
|
};
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
'
|
|
8
|
+
readonly headers: {
|
|
9
|
+
readonly accept: "text/html";
|
|
10
|
+
readonly 'accept-encoding': "gzip, deflate";
|
|
11
|
+
readonly 'accept-language': "en-US,en";
|
|
12
|
+
readonly referer: "https://www.google.com/";
|
|
13
|
+
readonly 'upgrade-insecure-requests': "1";
|
|
14
|
+
readonly 'User-Agent': "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
|
|
14
15
|
};
|
|
15
16
|
};
|
|
16
17
|
export default GOOGLE_CONSTANT;
|
|
@@ -2,16 +2,18 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const GOOGLE_CONSTANT = {
|
|
4
4
|
url: 'https://www.google.com/search',
|
|
5
|
-
inverse: {
|
|
6
|
-
base: 'https://lens.google.com/',
|
|
7
|
-
upload: 'https://lens.google.com/v3/upload',
|
|
8
|
-
url: 'https://lens.google.com/uploadbyurl?url='
|
|
9
|
-
},
|
|
10
5
|
queryParam: 'tbs',
|
|
6
|
+
//needed to search on google image instead of google
|
|
11
7
|
forceGoogleImage: {
|
|
12
|
-
tbm: 'isch'
|
|
8
|
+
tbm: 'isch',
|
|
9
|
+
udm: '2'
|
|
13
10
|
},
|
|
14
11
|
headers: {
|
|
12
|
+
accept: 'text/html',
|
|
13
|
+
'accept-encoding': 'gzip, deflate',
|
|
14
|
+
'accept-language': 'en-US,en',
|
|
15
|
+
referer: 'https://www.google.com/',
|
|
16
|
+
'upgrade-insecure-requests': '1',
|
|
15
17
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'
|
|
16
18
|
}
|
|
17
19
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GOOGLE_CONSTANT.js","sourceRoot":"","sources":["../../src/constant/GOOGLE_CONSTANT.ts"],"names":[],"mappings":";;AAAA,MAAM,eAAe,GAAG;IACtB,GAAG,EAAE,+BAA+B;IACpC,
|
|
1
|
+
{"version":3,"file":"GOOGLE_CONSTANT.js","sourceRoot":"","sources":["../../src/constant/GOOGLE_CONSTANT.ts"],"names":[],"mappings":";;AAAA,MAAM,eAAe,GAAG;IACtB,GAAG,EAAE,+BAA+B;IACpC,UAAU,EAAE,KAAK;IACjB,oDAAoD;IACpD,gBAAgB,EAAE;QAChB,GAAG,EAAE,MAAM;QACX,GAAG,EAAE,GAAG;KACT;IACD,OAAO,EAAE;QACP,MAAM,EAAE,WAAW;QACnB,iBAAiB,EAAE,eAAe;QAClC,iBAAiB,EAAE,UAAU;QAC7B,OAAO,EAAE,yBAAyB;QAClC,2BAA2B,EAAE,GAAG;QAChC,YAAY,EACV,uGAAuG;KAC1G;CACO,CAAC;AAEX,kBAAe,eAAe,CAAC"}
|
|
@@ -9,13 +9,10 @@ function filterByTitlesBuilder(config) {
|
|
|
9
9
|
const FILTER_TITLE = [];
|
|
10
10
|
if (config.filterByTitles) {
|
|
11
11
|
for (const titleFilter of config.filterByTitles) {
|
|
12
|
-
|
|
13
|
-
return `intitle:"${title}"`;
|
|
14
|
-
});
|
|
15
|
-
FILTER_TITLE.push(`(${value.join(' AND ')})`);
|
|
12
|
+
FILTER_TITLE.push(`(allintitle:"${titleFilter.join(' ')}")`);
|
|
16
13
|
}
|
|
17
14
|
}
|
|
18
|
-
return FILTER_TITLE.join(' ');
|
|
15
|
+
return FILTER_TITLE.join(' OR ');
|
|
19
16
|
}
|
|
20
17
|
/**
|
|
21
18
|
* Show only images without some specific words
|
|
@@ -68,10 +65,7 @@ function urlMatchBuilder(config) {
|
|
|
68
65
|
const URL_MATCH = [];
|
|
69
66
|
if (config.urlMatch) {
|
|
70
67
|
for (const urlMatch of config.urlMatch) {
|
|
71
|
-
|
|
72
|
-
return `inurl:${content}`;
|
|
73
|
-
});
|
|
74
|
-
URL_MATCH.push(`(${value.join(' AND ')})`);
|
|
68
|
+
URL_MATCH.push(`(allinurl:"${urlMatch.join(' ')}")`);
|
|
75
69
|
}
|
|
76
70
|
}
|
|
77
71
|
return URL_MATCH.join(' OR ');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"buildGoogleDork.js","sourceRoot":"","sources":["../../src/core/buildGoogleDork.ts"],"names":[],"mappings":";;AAEA;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,MAAc;IAC3C,MAAM,YAAY,
|
|
1
|
+
{"version":3,"file":"buildGoogleDork.js","sourceRoot":"","sources":["../../src/core/buildGoogleDork.ts"],"names":[],"mappings":";;AAEA;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,MAAc;IAC3C,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,MAAM,CAAC,cAAc,EAAE;QACzB,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,cAAc,EAAE;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;SAC9D;KACF;IACD,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,MAAc;IACzC,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,MAAM,CAAC,YAAY,EAAE;QACvB,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,YAAY,EAAE;YAC7C,aAAa,CAAC,IAAI,CAAC,KAAK,WAAW,GAAG,CAAC,CAAC;SACzC;KACF;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAc;IACxC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE;YACnC,OAAO,CAAC,IAAI,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC;SAClC;KACF;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC9B,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,MAAc;IAC3C,MAAM,eAAe,GAAG,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,cAAc,EAAE;QACzB,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,cAAc,EAAE;YACjD,eAAe,CAAC,IAAI,CAAC,UAAU,aAAa,GAAG,CAAC,CAAC;SAClD;KACF;IACD,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnC,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE;YACtC,SAAS,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;SACtD;KACF;IACD,OAAO,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,OAAO;QACL,MAAM,CAAC,MAAM;QACb,eAAe,CAAC,MAAM,CAAC;QACvB,mBAAmB,CAAC,MAAM,CAAC;QAC3B,qBAAqB,CAAC,MAAM,CAAC;QAC7B,kBAAkB,CAAC,MAAM,CAAC;QAC1B,qBAAqB,CAAC,MAAM,CAAC;KAC9B;SACE,IAAI,CAAC,GAAG,CAAC;SACT,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,kBAAe,gBAAgB,CAAC"}
|
|
@@ -15,10 +15,9 @@ function constructGoogleUrl(config) {
|
|
|
15
15
|
const GOOGLE_DORK = (0, buildGoogleDork_1.default)(config);
|
|
16
16
|
const CUSTOM_PARAM = config.custom ? `&${config.custom}` : '';
|
|
17
17
|
const SAFE_SEARCH = config.safeSearch ? `&safe=active` : '';
|
|
18
|
-
const QUERY = Object.assign(GOOGLE_CONSTANT_1.default.forceGoogleImage, {
|
|
19
|
-
[GOOGLE_CONSTANT_1.default.queryParam]: Object.values(config.query
|
|
20
|
-
|
|
21
|
-
});
|
|
18
|
+
const QUERY = Object.assign(GOOGLE_CONSTANT_1.default.forceGoogleImage, Object.assign(Object.assign({}, (config.query && {
|
|
19
|
+
[GOOGLE_CONSTANT_1.default.queryParam]: Object.values(config.query).join(',')
|
|
20
|
+
})), { q: GOOGLE_DORK }));
|
|
22
21
|
return GOOGLE_CONSTANT_1.default.url + (0, utils_1.buildQuery)(QUERY) + CUSTOM_PARAM + SAFE_SEARCH;
|
|
23
22
|
}
|
|
24
23
|
exports.default = constructGoogleUrl;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constructGoogleUrl.js","sourceRoot":"","sources":["../../src/core/constructGoogleUrl.ts"],"names":[],"mappings":";;;;;AACA,kFAA0D;AAC1D,0CAA4C;AAC5C,wEAAiD;AAEjD;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAc;IACxC,MAAM,WAAW,GAAG,IAAA,yBAAgB,EAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,yBAAe,CAAC,gBAAgB,
|
|
1
|
+
{"version":3,"file":"constructGoogleUrl.js","sourceRoot":"","sources":["../../src/core/constructGoogleUrl.ts"],"names":[],"mappings":";;;;;AACA,kFAA0D;AAC1D,0CAA4C;AAC5C,wEAAiD;AAEjD;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAc;IACxC,MAAM,WAAW,GAAG,IAAA,yBAAgB,EAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,yBAAe,CAAC,gBAAgB,kCACvD,CAAC,MAAM,CAAC,KAAK,IAAI;QAClB,CAAC,yBAAe,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;KACpE,CAAC,KACF,CAAC,EAAE,WAAW,IACd,CAAC;IAEH,OAAO,yBAAe,CAAC,GAAG,GAAG,IAAA,kBAAU,EAAC,KAAK,CAAC,GAAG,YAAY,GAAG,WAAW,CAAC;AAC9E,CAAC;AAED,kBAAe,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cookies": "__Secure-BUCKET=COIB; __Secure-ENID=34.SE=lYXtPhxPp9zFj0NPDOqxmjpXKMiGZUdwhDaYwseXPrwivHg4stTYwUViyNXeatqx_Dj8j2J7ZoI8Bmup2LfKwc0mrLe4S_6NuATYwRbANzUKNw52D64sbU9jFfnR-b4RTDtwNCpEh5WTcLetlGx9oW2z9DF9OfLgRbXRYAGD9jQ5emh8nnYitAkX7zR2g8ASWyVkiK2pU84S4nlHElNAL0oD7CekAhJDikxtwxptlOV0S5G3twX2yshephkeYAiqFimd2UyMG6giwKrxA5eESjaMY4AJ2DhgOv22NLIKwVU",
|
|
3
|
+
"proxy": "",
|
|
4
|
+
"timestamp": 1781613406823
|
|
5
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.getCookies = exports.clearCacheCookies = void 0;
|
|
16
|
+
const fs_1 = __importDefault(require("fs"));
|
|
17
|
+
const path_1 = __importDefault(require("path"));
|
|
18
|
+
const puppeteer_real_browser_1 = require("puppeteer-real-browser");
|
|
19
|
+
const GOOGLE_CONSTANT_1 = __importDefault(require("../constant/GOOGLE_CONSTANT"));
|
|
20
|
+
const CACHE_FILE = path_1.default.join(__dirname, 'cookies-cache.json');
|
|
21
|
+
// Cache lifetime (1 hour)
|
|
22
|
+
const CACHE_DURATION = 60 * 60 * 1000;
|
|
23
|
+
/**
|
|
24
|
+
* Check if the cache is still valid.
|
|
25
|
+
* The cache is only valid if:
|
|
26
|
+
* - The proxy is the same
|
|
27
|
+
* - The cache has not expired
|
|
28
|
+
*/
|
|
29
|
+
function isCacheValid(cache, proxy) {
|
|
30
|
+
const now = Date.now();
|
|
31
|
+
return cache.proxy === proxy && now - cache.timestamp < CACHE_DURATION;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Load cookies from cache.
|
|
35
|
+
* Returns null if:
|
|
36
|
+
* - Cache does not exist
|
|
37
|
+
* - Proxy changed
|
|
38
|
+
* - Cache expired
|
|
39
|
+
*/
|
|
40
|
+
function loadCache(proxy) {
|
|
41
|
+
if (!fs_1.default.existsSync(CACHE_FILE)) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
const cache = JSON.parse(fs_1.default.readFileSync(CACHE_FILE, 'utf-8'));
|
|
45
|
+
if (!isCacheValid(cache, proxy)) {
|
|
46
|
+
fs_1.default.unlinkSync(CACHE_FILE);
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
return cache.cookies;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Save cookies with the current proxy.
|
|
53
|
+
*/
|
|
54
|
+
function saveCache(cookies, proxy) {
|
|
55
|
+
const cache = {
|
|
56
|
+
cookies,
|
|
57
|
+
proxy,
|
|
58
|
+
timestamp: Date.now()
|
|
59
|
+
};
|
|
60
|
+
fs_1.default.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2));
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Clear the cookies cache file.
|
|
64
|
+
*/
|
|
65
|
+
function clearCacheCookies() {
|
|
66
|
+
if (fs_1.default.existsSync(CACHE_FILE)) {
|
|
67
|
+
fs_1.default.unlinkSync(CACHE_FILE);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
exports.clearCacheCookies = clearCacheCookies;
|
|
71
|
+
/**
|
|
72
|
+
* Convert Puppeteer cookies into a Cookie header string.
|
|
73
|
+
*/
|
|
74
|
+
function cookiesToString(cookies) {
|
|
75
|
+
return cookies.map(cookie => `${cookie.name}=${cookie.value}`).join('; ');
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Open browser and retrieve new cookies.
|
|
79
|
+
*/
|
|
80
|
+
function fetchNewCookies(proxy) {
|
|
81
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
82
|
+
const { browser, page } = yield (0, puppeteer_real_browser_1.connect)({
|
|
83
|
+
headless: true,
|
|
84
|
+
args: [`--proxy-server=${proxy}`]
|
|
85
|
+
});
|
|
86
|
+
yield page.setUserAgent(GOOGLE_CONSTANT_1.default.headers['User-Agent']);
|
|
87
|
+
yield page.setExtraHTTPHeaders({
|
|
88
|
+
'Accept-Language': 'en-US,en;q=0.9'
|
|
89
|
+
});
|
|
90
|
+
yield page.goto('https://www.google.com/search?udm=2&q=cats', {
|
|
91
|
+
waitUntil: 'networkidle2'
|
|
92
|
+
});
|
|
93
|
+
const cookies = yield page.browserContext().cookies();
|
|
94
|
+
const cookieString = cookiesToString(cookies);
|
|
95
|
+
yield browser.close();
|
|
96
|
+
saveCache(cookieString, proxy);
|
|
97
|
+
return cookieString;
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Main function.
|
|
102
|
+
*
|
|
103
|
+
* Returns cookies matching the current proxy.
|
|
104
|
+
*/
|
|
105
|
+
function getCookies(proxy) {
|
|
106
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
107
|
+
const cachedCookies = loadCache(proxy);
|
|
108
|
+
if (cachedCookies) {
|
|
109
|
+
return cachedCookies;
|
|
110
|
+
}
|
|
111
|
+
return yield fetchNewCookies(proxy);
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
exports.getCookies = getCookies;
|
|
115
|
+
//# sourceMappingURL=cookies.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookies.js","sourceRoot":"","sources":["../../src/core/cookies.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,4CAAoB;AACpB,gDAAwB;AACxB,mEAAiD;AACjD,kFAA0D;AAE1D,MAAM,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;AAE9D,0BAA0B;AAC1B,MAAM,cAAc,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAQtC;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAAkB,EAAE,KAAa;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,OAAO,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,GAAG,cAAc,CAAC;AACzE,CAAC;AAED;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,KAAa;IAC9B,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC9B,OAAO,IAAI,CAAC;KACb;IAED,MAAM,KAAK,GAAgB,IAAI,CAAC,KAAK,CAAC,YAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAE5E,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE;QAC/B,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC;KACb;IAED,OAAO,KAAK,CAAC,OAAO,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,OAAe,EAAE,KAAa;IAC/C,MAAM,KAAK,GAAgB;QACzB,OAAO;QACP,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,YAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB;IAC/B,IAAI,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC7B,YAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;KAC3B;AACH,CAAC;AAJD,8CAIC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,OAAc;IACrC,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,SAAe,eAAe,CAAC,KAAa;;QAC1C,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,IAAA,gCAAO,EAAC;YACtC,QAAQ,EAAE,IAAI;YACd,IAAI,EAAE,CAAC,kBAAkB,KAAK,EAAE,CAAC;SAClC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,YAAY,CAAC,yBAAe,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAE/D,MAAM,IAAI,CAAC,mBAAmB,CAAC;YAC7B,iBAAiB,EAAE,gBAAgB;SACpC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,IAAI,CAAC,4CAA4C,EAAE;YAC5D,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,OAAO,EAAE,CAAC;QAEtD,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QAE9C,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEtB,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAE/B,OAAO,YAAY,CAAC;IACtB,CAAC;CAAA;AAED;;;;GAIG;AACH,SAAsB,UAAU,CAAC,KAAa;;QAC5C,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAEvC,IAAI,aAAa,EAAE;YACjB,OAAO,aAAa,CAAC;SACtB;QAED,OAAO,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;CAAA;AARD,gCAQC"}
|
|
@@ -4,5 +4,5 @@ import ImageResultItem from '../types/imageResultItem';
|
|
|
4
4
|
* @param url
|
|
5
5
|
* @returns
|
|
6
6
|
*/
|
|
7
|
-
declare function parseGoogleImages(url: string, proxy?: string): Promise<ImageResultItem[]>;
|
|
7
|
+
declare function parseGoogleImages(url: string, proxy?: string, resetCookies?: boolean): Promise<ImageResultItem[]>;
|
|
8
8
|
export default parseGoogleImages;
|
|
@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
15
15
|
const utils_1 = require("../utils/utils");
|
|
16
16
|
const GOOGLE_CONSTANT_1 = __importDefault(require("../constant/GOOGLE_CONSTANT"));
|
|
17
17
|
const impit_1 = require("impit");
|
|
18
|
+
const cookies_1 = require("./cookies");
|
|
18
19
|
/**
|
|
19
20
|
* Scrap google images scripts tag
|
|
20
21
|
* @param url
|
|
@@ -22,18 +23,17 @@ const impit_1 = require("impit");
|
|
|
22
23
|
*/
|
|
23
24
|
function scrapGoogleImages(url, proxy) {
|
|
24
25
|
return __awaiter(this, void 0, void 0, function* () {
|
|
26
|
+
const cookies = yield (0, cookies_1.getCookies)(proxy || '');
|
|
25
27
|
const impit = new impit_1.Impit({
|
|
26
|
-
browser: '
|
|
28
|
+
browser: 'chrome',
|
|
27
29
|
proxyUrl: proxy,
|
|
28
30
|
ignoreTlsErrors: true,
|
|
29
31
|
followRedirects: true,
|
|
30
32
|
timeout: 30000,
|
|
31
33
|
maxRedirects: 5,
|
|
32
|
-
headers: GOOGLE_CONSTANT_1.default.headers
|
|
33
|
-
});
|
|
34
|
-
const req = yield impit.fetch(url, {
|
|
35
|
-
headers: GOOGLE_CONSTANT_1.default.headers
|
|
34
|
+
headers: Object.assign(Object.assign({}, GOOGLE_CONSTANT_1.default.headers), { cookie: cookies })
|
|
36
35
|
});
|
|
36
|
+
const req = yield impit.fetch(url);
|
|
37
37
|
return req.text();
|
|
38
38
|
});
|
|
39
39
|
}
|
|
@@ -58,8 +58,9 @@ function getGoogleImageObject(informationsMatch, otherInformationsMatch) {
|
|
|
58
58
|
* @param url
|
|
59
59
|
* @returns
|
|
60
60
|
*/
|
|
61
|
-
function parseGoogleImages(url, proxy) {
|
|
61
|
+
function parseGoogleImages(url, proxy, resetCookies = true) {
|
|
62
62
|
return __awaiter(this, void 0, void 0, function* () {
|
|
63
|
+
const processedIds = new Set();
|
|
63
64
|
const result = [];
|
|
64
65
|
const body = yield scrapGoogleImages(url, proxy);
|
|
65
66
|
//getting image url, height, width, color average
|
|
@@ -78,7 +79,16 @@ function parseGoogleImages(url, proxy) {
|
|
|
78
79
|
if (informationsMatch[1].match(/http/gi).length > 2 ||
|
|
79
80
|
otherInformationsMatch[2].match(/http/gi).length > 2)
|
|
80
81
|
continue;
|
|
81
|
-
|
|
82
|
+
const obj = getGoogleImageObject(informationsMatch, otherInformationsMatch);
|
|
83
|
+
if (processedIds.has(obj.id))
|
|
84
|
+
continue;
|
|
85
|
+
processedIds.add(obj.id);
|
|
86
|
+
result.push(obj);
|
|
87
|
+
}
|
|
88
|
+
// If we don't get any results, it might be due to invalid cookies. Clear the cache and try again.
|
|
89
|
+
if (result.length === 0 && resetCookies) {
|
|
90
|
+
(0, cookies_1.clearCacheCookies)();
|
|
91
|
+
return yield parseGoogleImages(url, proxy, false);
|
|
82
92
|
}
|
|
83
93
|
return result;
|
|
84
94
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parseGoogleImages.js","sourceRoot":"","sources":["../../src/core/parseGoogleImages.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0CAA+C;AAC/C,kFAA0D;AAE1D,iCAA8B;
|
|
1
|
+
{"version":3,"file":"parseGoogleImages.js","sourceRoot":"","sources":["../../src/core/parseGoogleImages.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0CAA+C;AAC/C,kFAA0D;AAE1D,iCAA8B;AAC9B,uCAA0D;AAE1D;;;;GAIG;AACH,SAAe,iBAAiB,CAAC,GAAW,EAAE,KAAc;;QAC1D,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAU,EAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,IAAI,aAAK,CAAC;YACtB,OAAO,EAAE,QAAQ;YACjB,QAAQ,EAAE,KAAK;YACf,eAAe,EAAE,IAAI;YACrB,eAAe,EAAE,IAAI;YACrB,OAAO,EAAE,KAAM;YACf,YAAY,EAAE,CAAC;YACf,OAAO,kCACF,yBAAe,CAAC,OAAO,KAC1B,MAAM,EAAE,OAAO,GAChB;SACF,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEnC,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;CAAA;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,iBAAkC,EAClC,sBAAuC;IAEvC,OAAO;QACL,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC;QAC7B,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC;QAChC,GAAG,EAAE,IAAA,qBAAa,EAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;QACxC,WAAW,EAAE,sBAAsB,CAAC,CAAC,CAAC;QACtC,MAAM,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1C,KAAK,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAe,iBAAiB,CAC9B,GAAW,EACX,KAAc,EACd,YAAY,GAAG,IAAI;;QAEnB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,MAAM,MAAM,GAAsB,EAAE,CAAC;QAErC,MAAM,IAAI,GAAW,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAEzD,iDAAiD;QACjD,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;QAC7D,gCAAgC;QAChC,MAAM,sBAAsB,GAAG,mDAAmD,CAAC;QAEnF,IAAI,iBAAyC,CAAC;QAE9C,OAAO,CAAC,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;YAClE,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,oCAAoC,CAAC;gBAAE,SAAS;YAEpF,MAAM,sBAAsB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjE,IAAI,sBAAsB,KAAK,IAAI;gBAAE,OAAO,MAAM,CAAC;YAEnD,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,IAAI,sBAAsB,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAChF,IACE,iBAAiB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,CAAC;gBAChD,sBAAsB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,CAAC;gBAErD,SAAS;YAEX,MAAM,GAAG,GAAG,oBAAoB,CAAC,iBAAiB,EAAE,sBAAsB,CAAC,CAAC;YAC5E,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YACvC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAClB;QAED,kGAAkG;QAClG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,EAAE;YACvC,IAAA,2BAAiB,GAAE,CAAC;YACpB,OAAO,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SACnD;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CAAA;AAED,kBAAe,iBAAiB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "google-img-scrap",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "Scrap images from google images with customs pre filled google dork options",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"fastBuild": "tsc",
|
|
11
11
|
"lint": "eslint . --ext .ts",
|
|
12
12
|
"prettier": "prettier --write .",
|
|
13
|
-
"test": "
|
|
13
|
+
"test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js"
|
|
14
14
|
},
|
|
15
15
|
"files": [
|
|
16
16
|
"dist",
|
|
@@ -61,17 +61,18 @@
|
|
|
61
61
|
"devDependencies": {
|
|
62
62
|
"@types/jest": "^29.5.14",
|
|
63
63
|
"@types/node": "^18.11.18",
|
|
64
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
65
|
-
"@typescript-eslint/parser": "^8.
|
|
64
|
+
"@typescript-eslint/eslint-plugin": "^8.60.1",
|
|
65
|
+
"@typescript-eslint/parser": "^8.60.1",
|
|
66
66
|
"eslint": "^8.57.0",
|
|
67
67
|
"eslint-config-prettier": "^9.1.0",
|
|
68
68
|
"jest": "^29.0.0",
|
|
69
|
-
"prettier": "^3.
|
|
70
|
-
"ts-jest": "^29.
|
|
69
|
+
"prettier": "^3.8.3",
|
|
70
|
+
"ts-jest": "^29.4.11",
|
|
71
71
|
"ts-node": "^10.9.1",
|
|
72
72
|
"typescript": "^4.7.4"
|
|
73
73
|
},
|
|
74
74
|
"dependencies": {
|
|
75
|
-
"impit": "^0.
|
|
75
|
+
"impit": "^0.14.1",
|
|
76
|
+
"puppeteer-real-browser": "^1.4.4"
|
|
76
77
|
}
|
|
77
78
|
}
|