ing-web-es 1.0.0

package/dump-scripts.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+import requests
+from bs4 import BeautifulSoup as bs
+import os
+import hashlib
+import argparse
+
+
+"""
+Simple python utility to dump all scripts from a given page.
+By: @SamuelAnttila
+License: MIT
+"""
+
+def download_script(url,downloads_dir_path,headers={},prettify=False):
+    """Download script into given directory. Note: Does nothing to avoid name collisions"""
+
+    # /asdf/file.js?123=123 -> file.js
+    url_path = requests.compat.urlparse(url).path
+    local_filename = os.path.basename(url_path)
+
+    if prettify:
+        # we can't stream if we want to prettify since we need the full file contents
+        res = requests.get(url, headers=headers)
+        code = jsbeautifier.beautify(res.text)
+        with open(os.path.join(downloads_dir_path,local_filename), 'w+') as f:
+            f.write(code)
+    else:
+        # streaming file download because putting everything in memory at once is silly if we don't have to
+        with requests.get(url, stream=True, headers=headers) as r:
+            r.raise_for_status()
+            with open(os.path.join(downloads_dir_path,local_filename), 'wb+') as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    # If you have chunk encoded response uncomment if
+                    # and set chunk_size parameter to None.
+                    #if chunk:
+                    f.write(chunk)
+    return local_filename
+
+
+def ensure_dir(file_path):
+    """Ensure directory exists by creating it if not present"""
+    directory = os.path.dirname(file_path)
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+if __name__ == "__main__":
+    downloads_dirname = "scripts/"
+    curr_path = os.path.dirname(os.path.realpath(__file__))
+    downloads_path = os.path.join(curr_path,downloads_dirname)
+    ensure_dir(downloads_path)
+
+    parser = argparse.ArgumentParser(description='Download all scripts from a website into a scripts/ folder underneath this script')
+    parser.add_argument('url', metavar='URL', type=str, help='The url (including schema) from which to dump scripts')
+    parser.add_argument('--useragent', dest='useragent', type=str, default="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36", help='User agent to use when making requests')
+    parser.add_argument('--prettify', dest='prettify', action="store_true", default=False, help='If enabled uses jsbeautify to prettify/deobfuscate all downloaded javascript. Will fail if "jsbeautifier" not installed with pip. Recommended if you\'re dealing with minified files.')
+    args = parser.parse_args()
+
+    if args.prettify:
+        import jsbeautifier # only necessary if we actually want to use this functionality
+
+    res = requests.get(args.url,headers={"User-Agent":args.useragent})
+    soup = bs(res.text,features="html.parser")
+    for script in soup.find_all("script"):
+        if "src" in script.attrs:
+            #externally loaded script
+            download_url = requests.compat.urljoin(args.url, script.attrs["src"])
+            print(f'Downloaded {download_script(download_url,downloads_path,headers={"User-Agent":args.useragent},prettify=args.prettify)}')
+        else:
+            #inline script
+            print(script.text)
+            m = hashlib.sha256()
+            m.update(script.text.encode("utf32"))
+            local_filename = m.hexdigest() # To give all inline scripts a unique name we take the hash of its contents. Only identical scripts should collide.
+            with open(os.path.join(downloads_path,local_filename)+".js", 'w+') as f:
+                if not args.prettify:
+                    f.write(script.text)
+                else:
+                    f.write(jsbeautifier.beautify(script.text))
+    print("Done downloading scripts. They should be under the 'scripts/' folder.")
+
+
+

package/ing.com.txt

@@ -0,0 +1,365 @@
+b--ing.com
+www.b--ing.com
+www.ing.com
+3ds-acc.ing.com
+www.3ds-acc.ing.com
+dcr.3ds-acc.ing.com
+wpr.3ds-acc.ing.com
+201.7.96.56.ing.com
+201.7.96.57.ing.com
+201.7.96.58.ing.com
+201.7.96.59.ing.com
+201.7.96.60.ing.com
+201.7.96.61.ing.com
+201.7.96.63.ing.com
+Conf1.ing.com
+Conf2.ing.com
+Lyncdiscover.ing.com
+Research.ing.com
+Sip1.ing.com
+www.Sip1.ing.com
+Sip2.ing.com
+www.Sip2.ing.com
+acc-ingworld.ing.com
+accept-art.ing.com
+acceptance.ing.com
+www.acceptance.ing.com
+cardsft.acceptance.ing.com
+ftp.service.acceptance.ing.com
+www.ftp.service.acceptance.ing.com
+acceptance-contr.ing.com
+www.acceptance-contr.ing.com
+annualreview.ing.com
+www.annualreview.ing.com
+annualreview2013.ing.com
+www.annualreview2013.ing.com
+portal.ap.ing.com
+sip.ap.ing.com
+vpn.ap.ing.com
+wc.ap.ing.com
+api.ing.com
+www.api.ing.com
+accp.api.ing.com
+www.accp.api.ing.com
+dcr.accp.api.ing.com
+wpr.accp.api.ing.com
+dcr.api.ing.com
+sandbox.api.ing.com
+test.api.ing.com
+www.test.api.ing.com
+wpr.api.ing.com
+art.ing.com
+IRS.asia.ing.com
+www.IRS.asia.ing.com
+Mx3.asia.ing.com
+Workdesk.asia.ing.com
+www.Workdesk.asia.ing.com
+irs.asia.ing.com
+mh2.asia.ing.com
+mx1.asia.ing.com
+mx2.asia.ing.com
+mx3.asia.ing.com
+mx4.asia.ing.com
+www.mx4.asia.ing.com
+portal1.asia.ing.com
+www.portal1.asia.ing.com
+portal2.asia.ing.com
+workdesk.asia.ing.com
+careers.ate.ing.com
+autodiscover.ing.com
+av1.ing.com
+av2.ing.com
+av3.ing.com
+av4.ing.com
+fts.b2b01.ing.com
+www.fts.b2b01.ing.com
+fts.b2b02.ing.com
+www.fts.b2b02.ing.com
+isabel.services.be.ing.com
+www.isabel.services.be.ing.com
+accp.isabel.services.be.ing.com
+www.accp.isabel.services.be.ing.com
+test.isabel.services.be.ing.com
+www.test.isabel.services.be.ing.com
+brand.ing.com
+www.brand.ing.com
+brandportal.ing.com
+www.brandportal.ing.com
+acceptation.brandportal.ing.com
+www.acceptation.brandportal.ing.com
+email.brandportal.ing.com
+production.brandportal.ing.com
+home.fr-eda-email.bulkmail.ing.com
+www.home.fr-eda-email.bulkmail.ing.com
+tracking.fr-eda-email.bulkmail.ing.com
+www.tracking.fr-eda-email.bulkmail.ing.com
+home.fr-eda-event.bulkmail.ing.com
+www.home.fr-eda-event.bulkmail.ing.com
+tracking.fr-eda-event.bulkmail.ing.com
+www.tracking.fr-eda-event.bulkmail.ing.com
+lm.ibicontactlab.bulkmail.ing.com
+t.ibicontactlab.bulkmail.ing.com
+cardsft.ing.com
+cbbrandportal.ing.com
+acceptation.cbbrandportal.ing.com
+cdn.ing.com
+www.cdn.ing.com
+staging.cdn.ing.com
+www.staging.cdn.ing.com
+colouring.ing.com
+conf1.ing.com
+conf2.ing.com
+conf3.ing.com
+conf4.ing.com
+contribution.ing.com
+www.contribution.ing.com
+coo.ing.com
+creditview.ing.com
+www.creditview.ing.com
+acc.creditview.ing.com
+www.acc.creditview.ing.com
+developer.ing.com
+www.developer.ing.com
+api.developer.ing.com
+www.api.developer.ing.com
+linus.digitalassistant.ing.com
+www.linus.digitalassistant.ing.com
+acc.linus.digitalassistant.ing.com
+www.acc.linus.digitalassistant.ing.com
+dev.linus.digitalassistant.ing.com
+www.dev.linus.digitalassistant.ing.com
+test.linus.digitalassistant.ing.com
+www.test.linus.digitalassistant.ing.com
+marie.digitalassistant.ing.com
+www.marie.digitalassistant.ing.com
+acc.marie.digitalassistant.ing.com
+www.acc.marie.digitalassistant.ing.com
+dev.marie.digitalassistant.ing.com
+www.dev.marie.digitalassistant.ing.com
+test.marie.digitalassistant.ing.com
+www.test.marie.digitalassistant.ing.com
+donate.ing.com
+www.donate.ing.com
+api.accp.nrc.esaeat.ing.com
+www.api.accp.nrc.esaeat.ing.com
+api.nrc.esaeat.ing.com
+www.api.nrc.esaeat.ing.com
+api.accp.tax.esaeat.ing.com
+www.api.accp.tax.esaeat.ing.com
+api.tax.esaeat.ing.com
+www.api.tax.esaeat.ing.com
+extnotify.ing.com
+www.extnotify.ing.com
+accp.extnotify.ing.com
+www.accp.extnotify.ing.com
+feedback.ing.com
+www.feedback.ing.com
+accp.feedback.ing.com
+www.accp.feedback.ing.com
+api.accp.feedback.ing.com
+www.api.accp.feedback.ing.com
+api.feedback.ing.com
+www.api.feedback.ing.com
+test.feedback.ing.com
+www.test.feedback.ing.com
+api.test.feedback.ing.com
+www.api.test.feedback.ing.com
+dcr.api.test.feedback.ing.com
+wpr.api.test.feedback.ing.com
+dcr.test.feedback.ing.com
+wpr.test.feedback.ing.com
+3ds.fr.ing.com
+www.3ds.fr.ing.com
+dcr.3ds.fr.ing.com
+wpr.3ds.fr.ing.com
+3dsecure.fr.ing.com
+www.3dsecure.fr.ing.com
+www.giftshop.ing.com
+eu.globalping.ing.com
+www.eu.globalping.ing.com
+accp.eu.globalping.ing.com
+www.accp.eu.globalping.ing.com
+api.accp.eu.globalping.ing.com
+www.api.accp.eu.globalping.ing.com
+api.eu.globalping.ing.com
+www.api.eu.globalping.ing.com
+group2.ing.com
+o2.halito.ing.com
+acceptation.im.ing.com
+in.ing.com
+ingworld.ing.com
+www.ingworld.ing.com
+acc.ingworld.ing.com
+www.acc.ingworld.ing.com
+images.ingworld.ing.com
+acc.ingworld-acc.ing.com
+insidebusiness.ing.com
+www.insidebusiness.ing.com
+internalsurvey.ing.com
+internalvideo.ing.com
+www.internalvideo.ing.com
+jaaroverzicht.ing.com
+www.jaaroverzicht.ing.com
+jaaroverzicht2013.ing.com
+www.jaaroverzicht2013.ing.com
+test.joep.ing.com
+www.test.joep.ing.com
+lyncdiscover.ing.com
+m.ing.com
+mail3.ing.com
+mail4.ing.com
+mail5.ing.com
+mailing.ing.com
+www.mailing.ing.com
+marking.ing.com
+microfinance-seminar.ing.com
+mobile.ing.com
+api.accp.mobile.ing.com
+www.api.accp.mobile.ing.com
+api.accp2.mobile.ing.com
+www.api.accp2.mobile.ing.com
+api.mobile.ing.com
+www.api.mobile.ing.com
+api.test.mobile.ing.com
+www.api.test.mobile.ing.com
+dcr.api.test.mobile.ing.com
+wpr.api.test.mobile.ing.com
+mobileaccess.ing.com
+www.mobileaccess.ing.com
+accp.mobileaccess.ing.com
+www.accp.mobileaccess.ing.com
+api.accp.mobileaccess.ing.com
+www.api.accp.mobileaccess.ing.com
+pt-accp.mobileaccess.ing.com
+www.pt-accp.mobileaccess.ing.com
+mobileapp.ing.com
+mobileapp-acc.ing.com
+mx.ing.com
+myaccount.ing.com
+www.myaccount.ing.com
+accp.myaccount.ing.com
+www.accp.myaccount.ing.com
+api.accp.myaccount.ing.com
+www.api.accp.myaccount.ing.com
+dcr.accp.myaccount.ing.com
+wpr.accp.myaccount.ing.com
+api.myaccount.ing.com
+www.api.myaccount.ing.com
+dcr.myaccount.ing.com
+test.myaccount.ing.com
+api.test.myaccount.ing.com
+tst.myaccount.ing.com
+www.tst.myaccount.ing.com
+wpr.myaccount.ing.com
+mybank.ing.com
+www.mybank.ing.com
+accp.mybank.ing.com
+www.accp.mybank.ing.com
+api.accp.mybank.ing.com
+www.api.accp.mybank.ing.com
+api.mybank.ing.com
+www.api.mybank.ing.com
+lab.mybank.ing.com
+www.lab.mybank.ing.com
+api.lab.mybank.ing.com
+www.api.lab.mybank.ing.com
+mobile.mybank.ing.com
+www.mobile.mybank.ing.com
+accp.mobile.mybank.ing.com
+www.accp.mobile.mybank.ing.com
+api.accp.mobile.mybank.ing.com
+www.api.accp.mobile.mybank.ing.com
+api.mobile.mybank.ing.com
+www.api.mobile.mybank.ing.com
+outbound.ing.com
+www.outbound.ing.com
+accp.outbound.ing.com
+www.accp.outbound.ing.com
+banktrade.pcm.ing.com
+pki.ing.com
+www.pki.ing.com
+3ds.pl.ing.com
+www.3ds.pl.ing.com
+dcr.3ds.pl.ing.com
+wpr.3ds.pl.ing.com
+3dsecure.pl.ing.com
+www.3dsecure.pl.ing.com
+psrv.ing.com
+www.psrv.ing.com
+accp.psrv.ing.com
+www.accp.psrv.ing.com
+psrv-ws.ing.com
+www.psrv-ws.ing.com
+accp.psrv-ws.ing.com
+www.accp.psrv-ws.ing.com
+research.ing.com
+www.research.ing.com
+api.sandbox.ing.com
+www.api.sandbox.ing.com
+wpr.api.sandbox.ing.com
+myaccount.sandbox.ing.com
+www.myaccount.sandbox.ing.com
+api.myaccount.sandbox.ing.com
+www.api.myaccount.sandbox.ing.com
+search.ing.com
+www.search.ing.com
+search-acc.ing.com
+www.search-acc.ing.com
+secure.ing.com
+ftp.service.ing.com
+www.ftp.service.ing.com
+test.ftp.service.ing.com
+ftp-dr.service.ing.com
+connective.services.ing.com
+accp.connective.services.ing.com
+sip1.ing.com
+www.sip1.ing.com
+sip2.ing.com
+www.sip2.ing.com
+sip3.ing.com
+www.sip3.ing.com
+sip4.ing.com
+www.sip4.ing.com
+sk7p1100v830.ing.com
+www.sk7p1100v830.ing.com
+sk7p1100v831.ing.com
+www.sk7p1100v831.ing.com
+stockprice.ing.com
+stockquotes.ing.com
+streaming.ing.com
+www.streaming.ing.com
+www.structuredproducts.ing.com
+survey.ing.com
+www.survey.ing.com
+survey2.ing.com
+www.survey2.ing.com
+surveys.ing.com
+www.surveys.ing.com
+insidebusiness.tft.test.ing.com
+insidebusiness.tft.ing.com
+think.ing.com
+www.think.ing.com
+timeorange.ing.com
+secure.trade.ing.com
+uk.ing.com
+uk-ssl-proxy.ing.com
+uk-ssl-proxy2.ing.com
+b2b.us.ing.com
+bld9527050.us.ing.com
+bld9527051.us.ing.com
+jaxmail.us.ing.com
+jaxmail1.us.ing.com
+mail.us.ing.com
+mspmail.us.ing.com
+mspmail1.us.ing.com
+video.ing.com
+www.video.ing.com
+still.video.ing.com
+wmv.video.ing.com
+webcast.ing.com
+www.webcast.ing.com
+origin.www.ing.com
+m--ing.com
+www.m--ing.com
+trad--ing.com
+www.trad--ing.com

package/ing.com_200List.txt

@@ -0,0 +1,30 @@
+b--ing.com
+www.b--ing.com
+www.ing.com
+art.ing.com
+brand.ing.com
+brandportal.ing.com
+acceptation.brandportal.ing.com
+email.brandportal.ing.com
+production.brandportal.ing.com
+lm.ibicontactlab.bulkmail.ing.com
+t.ibicontactlab.bulkmail.ing.com
+developer.ing.com
+feedback.ing.com
+eu.globalping.ing.com
+insidebusiness.ing.com
+www.insidebusiness.ing.com
+m.ing.com
+mailing.ing.com
+mobile.ing.com
+myaccount.ing.com
+pki.ing.com
+www.pki.ing.com
+myaccount.sandbox.ing.com
+surveys.ing.com
+think.ing.com
+webcast.ing.com
+m--ing.com
+www.m--ing.com
+trad--ing.com
+www.trad--ing.com