google-img-scrap 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 yoannchb-pro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,134 @@
+# Google-img-scrap v1.0.0
+Scrap images from google image with lot of options
+
+## Found a bug ?
+
+- Tell it in my github issues dont be afraid :)
+
+## Installation
+
+```
+npm i google-img-scrap
+```
+
+## Import
+
+```js
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+```
+
+## Result
+
+```js
+[
+    {
+        url: "...",
+        width: 1920,
+        height: 1080
+    },
+    ...
+]
+```
+
+## How to use ?
+
+- For the query parameter you need to set the name in upper case !
+
+```js
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "cats",
+        query: {
+            TYPE: GOOGLE_QUERY.TYPE.CLIPART,
+            DATE: GOOGLE_QUERY.DATE.YEAR,
+            COLOR: GOOGLE_QUERY.COLOR.BLACK_AND_WHITE,
+            SIZE: GOOGLE_QUERY.SIZE.LARGE,
+            LICENCE: GOOGLE_QUERY.LICENCE.COMMERCIAL_AND_OTHER,
+            EXTENSION: GOOGLE_QUERY.EXTENSION.JPG
+        },
+        domains: [],
+        custom: "name=content&name2=content2",
+        safeSearch: false,
+        excludeDomains: []
+    });
+
+    console.log(test, test.result.length);
+})();
+```
+
+OR ALSO
+
+```js
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "cats",
+    });
+
+    console.log(test, test.result.length);
+})();
+```
+## Params
+
+- "search" what you want to search
+- "domains" filter by domains
+- "excludeDomains" exclude some domains
+- "safeSearch" active safe search or not
+- "custom" add extra query
+- "query" set a query (can be [TYPE, DATE, COLOR, SIZE, LICENCE, EXTENSION])
+
+## Google query
+
+```js
+{
+  SIZE: { 
+    LARGE, 
+    MEDIUM, 
+    ICON 
+  },
+  COLOR: {
+    BLACK_AND_WHITE,
+    TRANSPARENT,
+    RED,
+    BLUE,
+    PURPLE,
+    ORANGE,
+    YELLOW,
+    GREEN,
+    TEAL,
+    PINK,
+    WHITE,
+    GRAY,
+    BLACK,
+    BROWN
+  },
+  TYPE: { 
+    CLIPART, 
+    DRAW, 
+    GIF 
+  },
+  EXTENSION: {
+    JPG,
+    GIF,
+    BMP,
+    PNG,
+    SVG,
+    WEBP,
+    ICO,
+    RAW
+  },
+  DATE: { 
+    DAY, 
+    WEEK, 
+    MONTH, 
+    YEAR 
+  },
+  LICENCE: { 
+    CREATIVE_COMMONS, 
+    COMMERCIAL_AND_OTHER 
+  }
+}
+```

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "google-img-scrap",
+  "version": "1.0.0",
+  "description": "Scrap images from google images with a lot of options",
+  "main": "./src/google-img-scrap.js",
+  "directories": {
+    "test": "test"
+  },
+  "scripts": {
+    "test": "node ./test/test.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yoannchb-pro/google-img-scrap.git"
+  },
+  "keywords": [
+    "google",
+    "image",
+    "scrap",
+    "options",
+    "query",
+    "powerfull",
+    "easy",
+    "type",
+    "color",
+    "extension",
+    "filter",
+    "date",
+    "licence"
+  ],
+  "author": "yoannchb",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/yoannchb-pro/google-img-scrap/issues"
+  },
+  "homepage": "https://github.com/yoannchb-pro/google-img-scrap#readme",
+  "dependencies": {
+    "got": "^11.0.0"
+  }
+}

package/src/constant/GOOGLE_CONSTANT.js

@@ -0,0 +1,13 @@
+const GOOGLE_CONSTANT = {
+    url: "https://images.google.com/search",
+    queryParam: "tbs",
+    forceGoogleImage: {
+        tbm: "isch" //needed to search on google image instead of google
+    },
+    headers: {
+        'User-Agent':
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
+    }
+};
+
+module.exports = { GOOGLE_CONSTANT };

package/src/constant/extensions/IMAGES_EXTENSIONS.json

@@ -0,0 +1 @@
+["jpg", "gif", "bmp", "png", "svg", "webp", "ico", "raw"]

package/src/constant/query/GOOGLE_COLORS.js

@@ -0,0 +1,14 @@
+const COLORS = ["red",
+                "blue", 
+                "purple", 
+                "orange", 
+                "yellow", 
+                "green", 
+                "teal", 
+                "pink", 
+                "white", 
+                "gray", 
+                "black", 
+                "brown"];
+
+module.exports = { COLORS };

package/src/constant/query/GOOGLE_PARAMS.js

@@ -0,0 +1,9 @@
+const SIZE_PARAM = "isz",
+      COLOR_PARAM = "ic",
+      SPECIFIC_COLOR_PARAM = "Cisc",
+      TYPE_PARAM = "itp",
+      DATE_PARAM = "qdr",
+      LICENCE_PARAM = "il",
+      IMAGE_EXTENSION_PARAM = "ift";
+
+module.exports = { SIZE_PARAM, COLOR_PARAM, SPECIFIC_COLOR_PARAM, TYPE_PARAM, DATE_PARAM, LICENCE_PARAM, IMAGE_EXTENSION_PARAM };

package/src/constant/query/GOOGLE_QUERY.js

@@ -0,0 +1,50 @@
+const { SIZE_PARAM, COLOR_PARAM, SPECIFIC_COLOR_PARAM, TYPE_PARAM, DATE_PARAM, LICENCE_PARAM, IMAGE_EXTENSION_PARAM } = require('./GOOGLE_PARAMS');
+const { COLORS } = require('./GOOGLE_COLORS');
+const EXTENSIONS = require("../extensions/IMAGES_EXTENSIONS.json");
+
+const GOOGLE_QUERY = {
+    SIZE: {
+        LARGE: SIZE_PARAM + ":l",
+        MEDIUM: SIZE_PARAM + ":m",
+        ICON: SIZE_PARAM + ":i",
+    },
+
+    COLOR: {
+        BLACK_AND_WHITE: COLOR_PARAM + ":gray",
+        TRANSPARENT: COLOR_PARAM + ":trans",
+    },
+
+    TYPE: {
+        CLIPART: TYPE_PARAM + ":clipart",
+        DRAW: TYPE_PARAM + ":lineart",
+        GIF: TYPE_PARAM + ":animated",
+    },
+
+    EXTENSION: {},
+
+    DATE: {
+        DAY: DATE_PARAM + ":d",
+        WEEK: DATE_PARAM + ":w",
+        MONTH: DATE_PARAM + ":m",
+        YEAR: DATE_PARAM + ":y",
+    },
+
+    LICENCE: {
+        CREATIVE_COMMONS: LICENCE_PARAM + ":cl",
+        COMMERCIAL_AND_OTHER: LICENCE_PARAM + ":ol",
+    },
+};
+
+//build extension
+EXTENSIONS.forEach((EXTENSION) => {
+    const queryName = EXTENSION.toUpperCase();
+    GOOGLE_QUERY.EXTENSION[queryName] = IMAGE_EXTENSION_PARAM + ":" + EXTENSION;
+});
+
+//build colors
+COLORS.forEach((COLOR) => {
+    const queryName = COLOR.toUpperCase();
+    GOOGLE_QUERY.COLOR[queryName] = COLOR_PARAM + ":specific," + SPECIFIC_COLOR_PARAM + ":" + COLOR;
+});
+
+module.exports = { GOOGLE_QUERY };

package/src/constant/translator/TRANSLATOR.js

@@ -0,0 +1,3 @@
+const TRANSLATOR = {};
+
+module.exports = { TRANSLATOR };

package/src/google-img-scrap.js

@@ -0,0 +1,104 @@
+const got = require('got');
+const { FastHTMLParser } = require('fast-html-dom-parser');
+
+const { GOOGLE_CONSTANT } = require('./constant/GOOGLE_CONSTANT');
+const { GOOGLE_QUERY } = require('./constant/query/GOOGLE_QUERY');
+const { TRANSLATOR } = require("./constant/translator/TRANSLATOR");
+const EXTENSIONS = require('./constant/extensions/IMAGES_EXTENSIONS.json');
+
+const { buildQuery, unicodeToChar } = require('./utils/UTILS');
+
+//verify good configuration
+function verify(config){
+    if(!config.search || config.search.trim() == "") throw "'search' can not be empty";
+
+    if(config.query){
+        const queryToVerify = Object.keys(GOOGLE_QUERY);
+    
+        for(const key of Object.keys(config.query)){
+            if(!queryToVerify.includes(key)) throw `Invalide query name '${key}'`;
+
+            const VALUES = Object.values(GOOGLE_QUERY[key]);
+            const ACTUAL_VALUE = config.query[key];
+            if(!VALUES.includes(ACTUAL_VALUE)) throw `'${ACTUAL_VALUE}' is not a valide argument for the query : '${key}'`;
+        };
+    }
+};
+
+//verify imag extension
+function containImage(content = ""){
+    for(const EXTENSION of EXTENSIONS){
+        if(content.includes(EXTENSION)) return true;
+    }
+
+    return false;
+}
+
+//parse HTML
+async function parse(url){
+    const result = [];
+
+    const response = await got(url, {
+        headers: GOOGLE_CONSTANT.headers
+    });
+    const parser = new FastHTMLParser(response.body);
+
+    const scripts = parser.getElementsByTagName('script');
+
+    if(!scripts) return result;
+
+    for(const script of scripts){
+        const body = script.innerHTML;
+
+        const valide = containImage(body);
+
+        if(valide){
+            const regex = /\["(http.+?)",(\d+),(\d+)\]/gi;
+
+            let res = null;
+
+            while((res = regex.exec(body)) != null){
+                if(res.length >= 4 && res[1].match(/http/gi).length < 2) result.push({
+                    url: unicodeToChar(res[1]),
+                    height: res[2],
+                    width: res[3]
+                });
+            }
+        }
+    }
+
+    return result;
+};
+
+//main
+async function GOOGLE_IMG_SCRAP(config = {}){
+    //verify config
+    verify(config);
+
+    //exclude domains
+    const EXCLUDE_DOMAINS = [];
+    if(config.excludeDomains) config.excludeDomains.forEach((domain) => EXCLUDE_DOMAINS.push(`-site:${domain}`));
+
+    //domains
+    const DOMAINS = [];
+    if(config.domains) config.domains.forEach((domain) => DOMAINS.push(`site:"${domain}"`));
+
+    //building url
+    const QUERY = Object.assign(GOOGLE_CONSTANT.forceGoogleImage, {
+        [GOOGLE_CONSTANT.queryParam]: Object.values(config.query || {}).join(','),
+        q: encodeURIComponent(config.search + " " + EXCLUDE_DOMAINS.join(" ") + " " + DOMAINS.join(' OR ')),
+    });
+
+    const CUSTOM_PARAM = config.custom ? `&${config.custom}` : "";
+    const SAFE_SEARCH = config.safeSearch ? `&safe=active` : "";
+
+    const URL = GOOGLE_CONSTANT.url + buildQuery(QUERY, TRANSLATOR) + CUSTOM_PARAM + SAFE_SEARCH;
+
+    //parsing
+    const result = await parse(URL);
+
+    //result
+    return {url: URL, result: result};
+};
+
+module.exports = { GOOGLE_IMG_SCRAP , GOOGLE_QUERY };

package/src/utils/UTILS.js

@@ -0,0 +1,24 @@
+function buildQuery(query, translator){
+    let result = [];
+
+    let params = Object.keys(query);
+    let toTranslate = Object.keys(translator);
+
+    for(const param of params){
+        let queryName = param;
+        if(toTranslate.includes(param)) queryName = toTranslate[param];
+
+        result.push(`${queryName}=${query[param]}`);
+    }
+
+    return "?" + result.join('&');
+}
+
+function unicodeToChar(text) {
+    return text.replace(/\\u[\dA-F]{4}/gi, 
+           function (match) {
+                return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));
+           });
+ }
+
+module.exports = { buildQuery, unicodeToChar };

package/test/test.js

@@ -0,0 +1,23 @@
+const { GOOGLE_IMG_SCRAP , GOOGLE_QUERY } = require('../src/google-img-scrap');
+
+console.log(GOOGLE_QUERY);
+
+(async function(){
+    const test = await GOOGLE_IMG_SCRAP({
+        search: "cats",
+        query: {
+            TYPE: GOOGLE_QUERY.TYPE.CLIPART,
+            DATE: GOOGLE_QUERY.DATE.YEAR,
+            COLOR: GOOGLE_QUERY.COLOR.BLACK_AND_WHITE,
+            SIZE: GOOGLE_QUERY.SIZE.LARGE,
+            LICENCE: GOOGLE_QUERY.LICENCE.COMMERCIAL_AND_OTHER,
+            EXTENSION: GOOGLE_QUERY.EXTENSION.JPG
+        },
+        domains: [],
+        custom: "name=content&name2=content2",
+        safeSearch: false,
+        excludeDomains: []
+    });
+
+    console.log(test, test.result.length);
+})();