google-img-scrap 1.0.0 → 1.0.4

package/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+### v1.0.4
+
+- New option ```urlMatch```. You know get image when an url match a string (example: "cdn")
+- New option ```filterByTitles```. Filter images by titles
+
+### v1.0.3
+- New option ```execute```. allow you to execute a function to remove "gstatic.com" domains for example
+
+### v1.0.2
+- Cannot set 'domains' and 'excludeDomains' as same time
+- Fixed some bugs
+- New option ```excludeWords```
+
+### v1.0.1
+- Added the missing dependencie

package/README.md

@@ -1,5 +1,10 @@
-# Google-img-scrap v1.0.0
-Scrap images from google image with lot of options
+# Google-img-scrap v1.0.4
+
+Scrap images from google image with lot of tools and options.
+
+## Update
+
+- See [changelog](CHANGELOG.md)
 
 ## Found a bug ?
 
@@ -13,21 +18,56 @@ npm i google-img-scrap
 
 ## Import
 
+- NPM
+
+```js
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('google-img-scrap');
+```
+
+- From GITHUB
+
 ```js
-const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('./src/google-img-scrap');
 ```
 
+## Params
+
+- "search" (String) what you want to search
+- "execute" (Function) allow you to execute a function to remove "gstatic.com" domains for example
+- "excludeWords" (Array of String) exclude some words from the search
+- "domains" (Array of String) filter by domains
+- "excludeDomains" (Array of String) exclude some domains
+- "safeSearch" (Boolean) active safe search or not for nsfw for example
+- "custom" (String) add extra query
+- "urlMatch" (Array of Array) get image when an url match a string (example: "cdn") | ```example below```
+- "filterByTitles" (Array of Array) filter images by titles | ```example below```
+- "query" (Object) set a query (can be [TYPE, DATE, COLOR, SIZE, LICENCE, EXTENSION]) (use GOOGLE_QUERY items | ```example below```
+
 ## Result
 
 ```js
-[
+}
+{
+  url: 'https://images.google.com/search?tbm=isch&tbs=itp:clipart,qdr:y,ic:gray,isz:l,il:ol,ift:jpg&q=cats',
+  result: [
+    {
+      url: 'https://media.istockphoto.com/vectors/black-cats-set-vector-id599123506',
+      height: '806',
+      width: '1024'
+    },
     {
-        url: "...",
-        width: 1920,
-        height: 1080
+      url: 'https://media.istockphoto.com/vectors/cats-vector-id455327075',
+      height: '860',
+      width: '1024'
+    },
+    {
+      url: 'https://media.istockphoto.com/vectors/purring-cats-vector-silhouette-vector-id165749810?s=2048x2048',
+      height: '1895',
+      width: '2048'
     },
     ...
-]
+  ]
+}
 ```
 
 ## How to use ?
@@ -35,7 +75,9 @@ const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
 - For the query parameter you need to set the name in upper case !
 
 ```js
-const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('google-img-scrap');
+
+console.log(GOOGLE_QUERY);
 
 (async function(){
     const test = await GOOGLE_IMG_SCRAP({
@@ -48,10 +90,11 @@ const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
             LICENCE: GOOGLE_QUERY.LICENCE.COMMERCIAL_AND_OTHER,
             EXTENSION: GOOGLE_QUERY.EXTENSION.JPG
         },
-        domains: [],
+        domains: ["alamy.com", "istockphoto.com", "vecteezy.com"],
+        excludeWords: ["black", "white"], //If you don't like black and white cats
         custom: "name=content&name2=content2",
         safeSearch: false,
-        excludeDomains: []
+        // excludeDomains: ["istockphoto.com", "alamy.com"]
     });
 
     console.log(test, test.result.length);
@@ -61,7 +104,7 @@ const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
 OR ALSO
 
 ```js
-const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('google-img-scrap');
 
 (async function(){
     const test = await GOOGLE_IMG_SCRAP({
@@ -71,14 +114,52 @@ const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
     console.log(test, test.result.length);
 })();
 ```
-## Params
 
-- "search" what you want to search
-- "domains" filter by domains
-- "excludeDomains" exclude some domains
-- "safeSearch" active safe search or not
-- "custom" add extra query
-- "query" set a query (can be [TYPE, DATE, COLOR, SIZE, LICENCE, EXTENSION])
+## Removing gstatic.com
+
+```js
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('google-img-scrap');
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "demon slayer background hd",
+        query: {
+            SIZE: GOOGLE_QUERY.SIZE.LARGE,
+        },
+        domains: ["alphacoders.com"],
+        safeSearch: false,
+        execute: function(element){
+            if(!element.url.match('gstatic.com')) return element;
+        }
+    });
+
+    console.log(test, test.result[test.result.length-1].url, test.result.length);
+})();
+```
+
+## How urlMatch and filterByTitles work ?
+
+- urlMatch work like filterByTiles
+
+```js
+const { GOOGLE_IMG_SCRAP } = require('google-img-scrap');
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "cats",
+        //will build something like this "(draw and white) or (albino and white)"
+        filterByTitles: [
+            ["draw", "white"],
+            ["albino", "white"]
+        ],
+        execute: function(element){
+            if(!element.url.match('gstatic.com')) return element;
+        }
+    });
+
+    console.log(test, test.result.length);
+})();
+```
 
 ## Google query
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "google-img-scrap",
-  "version": "1.0.0",
+  "version": "1.0.4",
   "description": "Scrap images from google images with a lot of options",
   "main": "./src/google-img-scrap.js",
   "directories": {
@@ -35,6 +35,7 @@
   },
   "homepage": "https://github.com/yoannchb-pro/google-img-scrap#readme",
   "dependencies": {
+    "fast-html-dom-parser": "^1.0.5",
     "got": "^11.0.0"
   }
 }

package/src/google-img-scrap.js

@@ -10,6 +10,8 @@ const { buildQuery, unicodeToChar } = require('./utils/UTILS');
 
 //verify good configuration
 function verify(config){
+    if(config.excludeDomains && config.domains) throw "Can not set 'excludeDomains' and 'domains' as same times";
+
     if(!config.search || config.search.trim() == "") throw "'search' can not be empty";
 
     if(config.query){
@@ -77,16 +79,52 @@ async function GOOGLE_IMG_SCRAP(config = {}){
 
     //exclude domains
     const EXCLUDE_DOMAINS = [];
-    if(config.excludeDomains) config.excludeDomains.forEach((domain) => EXCLUDE_DOMAINS.push(`-site:${domain}`));
+    if(config.excludeDomains) config.excludeDomains.forEach((domain) => EXCLUDE_DOMAINS.push(`-site:"${domain}"`));
 
     //domains
     const DOMAINS = [];
     if(config.domains) config.domains.forEach((domain) => DOMAINS.push(`site:"${domain}"`));
 
+    //exclude words
+    const EXCLUDE_WORDS = [];
+    if(config.excludeWords) config.excludeWords.forEach((word) => EXCLUDE_WORDS.push(`-"${word}"`));
+
+    //filter by titles
+    const FILTER_TITLE = [];
+    if(config.filterByTitles) config.filterByTitles.forEach((titleFilter) => {
+
+        const value = titleFilter.map((title) => {
+            return `intitle:"${title}"`;
+        });
+
+        FILTER_TITLE.push(`(${value.join(' AND ')})`);
+
+    });
+
+    //url match words
+    const URL_MATCH = [];
+    if(config.urlMatch) config.urlMatch.forEach((urlMatch) => {
+
+        const value = urlMatch.map((content) => {
+            return `inurl:${content}`;
+        });
+
+        URL_MATCH.push(`(${value.join(' AND ')})`);
+
+    });
+
     //building url
+    const SEARCH_TERM = config.search + 
+    " " + URL_MATCH.join(" OR ") +
+    " " + FILTER_TITLE.join(" OR ") +
+    " " + EXCLUDE_WORDS.join(" ") + 
+    " " + EXCLUDE_DOMAINS.join(" ") + 
+    " " + DOMAINS.join(" OR ");
+
+    const SEARCH = encodeURIComponent(SEARCH_TERM.trim())
     const QUERY = Object.assign(GOOGLE_CONSTANT.forceGoogleImage, {
         [GOOGLE_CONSTANT.queryParam]: Object.values(config.query || {}).join(','),
-        q: encodeURIComponent(config.search + " " + EXCLUDE_DOMAINS.join(" ") + " " + DOMAINS.join(' OR ')),
+        q: SEARCH,
     });
 
     const CUSTOM_PARAM = config.custom ? `&${config.custom}` : "";
@@ -97,8 +135,16 @@ async function GOOGLE_IMG_SCRAP(config = {}){
     //parsing
     const result = await parse(URL);
 
+    //excute function
+    let finalResult = [];
+    if(config.execute) result.forEach((element) => {
+        const value = config.execute(element);
+        if(value) finalResult.push(value);
+    });
+    else finalResult = result;
+
     //result
-    return {url: URL, result: result};
+    return {url: URL, result: finalResult};
 };
 
 module.exports = { GOOGLE_IMG_SCRAP , GOOGLE_QUERY };

package/test/test-filter-titles.js

@@ -0,0 +1,16 @@
+const { GOOGLE_IMG_SCRAP } = require('../src/google-img-scrap');
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "cats",
+        filterByTitles: [
+            ["draw", "white"],
+            ["albino", "white"]
+        ],
+        execute: function(element){
+            if(!element.url.match('gstatic.com')) return element;
+        }
+    });
+
+    console.log(test, test.result.length);
+})();

package/test/test-url-match.js

@@ -0,0 +1,16 @@
+const { GOOGLE_IMG_SCRAP } = require('../src/google-img-scrap');
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "cats",
+        urlMatch: [
+            ["cdn"],
+            ["istockphoto"]
+        ],
+        execute: function(element){
+            if(!element.url.match('gstatic.com')) return element;
+        }
+    });
+
+    console.log(test, test.result.length);
+})();

package/test/test-wallpaper.js

@@ -0,0 +1,17 @@
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "demon slayer background hd",
+        query: {
+            SIZE: GOOGLE_QUERY.SIZE.LARGE,
+        },
+        domains: ["alphacoders.com"],
+        safeSearch: false,
+        execute: function(element){
+            if(!element.url.match('gstatic.com')) return element;
+        }
+    });
+
+    console.log(test, test.result[test.result.length-1].url, test.result.length);
+})();

package/test/test.js

@@ -1,6 +1,6 @@
 const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
 
-console.log(GOOGLE_QUERY);
+// console.log(GOOGLE_QUERY);
 
 (async function(){
     const test = await GOOGLE_IMG_SCRAP({
@@ -13,10 +13,14 @@ console.log(GOOGLE_QUERY);
             LICENCE: GOOGLE_QUERY.LICENCE.COMMERCIAL_AND_OTHER,
             EXTENSION: GOOGLE_QUERY.EXTENSION.JPG
         },
-        domains: [],
+        domains: ["alamy.com", "istockphoto.com", "vecteezy.com"],
+        excludeWords: ["black", "white"], //If you don't like black and white cats
         custom: "name=content&name2=content2",
         safeSearch: false,
-        excludeDomains: []
+        execute: function(element){
+            if(!element.url.match('gstatic.com')) return element;
+        }
+        // excludeDomains: ["istockphoto.com", "alamy.com"]
     });
 
     console.log(test, test.result.length);