wmap 2.4.8 → 2.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/googleBot +1 -1
- data/lib/wmap/host_tracker.rb +1 -1
- data/lib/wmap/site_tracker.rb +11 -8
- data/logs/wmap.log +6 -0
- data/settings/google_keywords.txt +13 -2
- data/version.txt +2 -2
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7df3c5698974ccb73832bafefc264aebf82addba523ce2b5a2804684977bc558
|
|
4
|
+
data.tar.gz: 16546e8b0daf8b16dd0bb4a89815569a3dbf058bd9ec6cddc3341a1c33640e91
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f089db7404c7bb058aedf90ae195cf37768482ba1d08f099a1b3cc85c7e0d0510552adce9546c597053aa6440f378fecc561e034a352914780d04408be5eb04b
|
|
7
|
+
data.tar.gz: a7a74b5527d09296de0c33864dc680a10b8454d7b7545b3f1f803b3245c321cc708e384197cc469dabf5d4007fea3cb0bdfd11274ab13188853b66f053817a97
|
data/bin/googleBot
CHANGED
|
@@ -14,7 +14,7 @@ Wmap.wlog("Execute the command: googleBot","googleBot",Log_dir+"wmap.log")
|
|
|
14
14
|
abort "Incorrect program argument - no argument needed! Proper Usage: googleBot " unless ARGV.length==0
|
|
15
15
|
|
|
16
16
|
puts "Start the Google search scraper. This may take a while, please be patient ..."
|
|
17
|
-
dis=Wmap::GoogleSearchScraper.new
|
|
17
|
+
dis=Wmap::GoogleSearchScraper.new(:verbose=>false)
|
|
18
18
|
dis.workers
|
|
19
19
|
sites=dis.discovered_sites_from_scraper.keys
|
|
20
20
|
puts "Update Google search results into the data repository:"
|
data/lib/wmap/host_tracker.rb
CHANGED
|
@@ -116,7 +116,7 @@ class Wmap::HostTracker
|
|
|
116
116
|
puts "Domain root: #{root}" if @verbose
|
|
117
117
|
domain_tracker=Wmap::DomainTracker.instance
|
|
118
118
|
domain_tracker.data_dir=@data_dir
|
|
119
|
-
if domain_tracker.
|
|
119
|
+
if domain_tracker.domain_known?(root)
|
|
120
120
|
domain_tracker=nil
|
|
121
121
|
record[host]=ip
|
|
122
122
|
record[ip]=host
|
data/lib/wmap/site_tracker.rb
CHANGED
|
@@ -103,11 +103,14 @@ class Wmap::SiteTracker
|
|
|
103
103
|
|
|
104
104
|
# Setter to add site entry to the cache one at a time
|
|
105
105
|
def add(site)
|
|
106
|
-
|
|
107
|
-
|
|
106
|
+
#begin
|
|
107
|
+
puts "Add entry to the site store: #{site}"
|
|
108
108
|
# Preliminary sanity check
|
|
109
109
|
site=site.strip.downcase unless site.nil?
|
|
110
|
-
|
|
110
|
+
if site_known?(site)
|
|
111
|
+
puts "Site is already exist. Skip #{site}"
|
|
112
|
+
return nil
|
|
113
|
+
end
|
|
111
114
|
site=normalize_url(site) if is_url?(site)
|
|
112
115
|
site=url_2_site(site) if is_url?(site)
|
|
113
116
|
puts "Site in standard format: #{site}" if @verbose
|
|
@@ -233,11 +236,11 @@ class Wmap::SiteTracker
|
|
|
233
236
|
my_tracker=nil
|
|
234
237
|
return nil
|
|
235
238
|
end
|
|
236
|
-
rescue => ee
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
end
|
|
239
|
+
#rescue => ee
|
|
240
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
|
241
|
+
# deact=nil
|
|
242
|
+
# return nil
|
|
243
|
+
#end
|
|
241
244
|
end
|
|
242
245
|
|
|
243
246
|
# Setter to add site entry to the cache table in batch (from a file)
|
data/logs/wmap.log
CHANGED
|
@@ -1544,3 +1544,9 @@
|
|
|
1544
1544
|
2019-02-17 22:09:16 -0500: wmap: Execute the command: wmap penguinrandomhouse.com
|
|
1545
1545
|
2019-02-17 22:11:57 -0500: wmap: Execute the command: wmap /tmp/test
|
|
1546
1546
|
2019-02-17 22:50:46 -0500: wmap: Execute the command: wmap /tmp/test1
|
|
1547
|
+
2019-02-20 20:41:31 -0500: googleBot: Execute the command: googleBot
|
|
1548
|
+
2019-02-20 20:56:04 -0500: googleBot: Execute the command: googleBot
|
|
1549
|
+
2019-02-20 21:05:21 -0500: googleBot: Execute the command: googleBot
|
|
1550
|
+
2019-02-20 21:10:59 -0500: googleBot: Execute the command: googleBot
|
|
1551
|
+
2019-02-20 21:17:21 -0500: googleBot: Execute the command: googleBot
|
|
1552
|
+
2019-02-20 21:26:13 -0500: googleBot: Execute the command: googleBot
|
|
@@ -2,8 +2,19 @@
|
|
|
2
2
|
# Used by the Wmap::GoogleSearchScraper class
|
|
3
3
|
|
|
4
4
|
# Official whole name
|
|
5
|
-
|
|
5
|
+
Penguin Random House
|
|
6
6
|
|
|
7
7
|
# 2014 purchase
|
|
8
|
-
|
|
8
|
+
Random House
|
|
9
9
|
|
|
10
|
+
# Parent company
|
|
11
|
+
Bertelsmann
|
|
12
|
+
|
|
13
|
+
# Division
|
|
14
|
+
Crown Books
|
|
15
|
+
|
|
16
|
+
# Division
|
|
17
|
+
Dorling Kindersley
|
|
18
|
+
|
|
19
|
+
# Division
|
|
20
|
+
Penguin Random House Canada
|
data/version.txt
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
###############################################################################
|
|
4
4
|
package = wmap
|
|
5
5
|
# wmap version 2.0 == web_discovery version 1.5.3
|
|
6
|
-
version = 2.4.
|
|
7
|
-
date = 2019-02-
|
|
6
|
+
version = 2.4.9
|
|
7
|
+
date = 2019-02-20
|
|
8
8
|
|
|
9
9
|
author = Sam (Yang) Li
|
|
10
10
|
email = yang.li@owasp.org
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wmap
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.4.
|
|
4
|
+
version: 2.4.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sam (Yang) Li
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-02-
|
|
11
|
+
date: 2019-02-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: dnsruby
|