wmap 2.7.7 → 2.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 559cac84dd38902d968cc9e7327e77115ae3f946020caa21b7743ceb5777a96f
4
- data.tar.gz: a4dcc0eafc10d7497c47c1f9955774b880ae5cdafa5ed0c6904ef5362087bd98
3
+ metadata.gz: 723c7f71bbe80edf1e1c2ed5cad74fe045a8d3e6494b6394921cdd4546d562e7
4
+ data.tar.gz: 072757ff8e19fee784c3387e362ce9feef86bb7869998df86f8c55437dc9b199
5
5
  SHA512:
6
- metadata.gz: 0b430ed1da47cefd8cb8a7bedddd75ed2e7b1dafbfe94cdb2185ad2eb7e26d69a1429e79a9ece6b3cc68d6d964161f092e1143a00c0c64e241e930304d9e5a65
7
- data.tar.gz: fe4d50b292849e51c202f47083c0837228d11934eb0d71c5a6262da24467480ce26994f5db91bdae5cf3b52dc0e6ef4fef698396575e820779c7420c731d0d87
6
+ metadata.gz: f783d31ad51063734b4ba54cc683399196b838d482cff57d9a27643bcb81451653475953aa2ae4ece6e277ec3756e3a3db05e250c2a6be9595fd7df993088088
7
+ data.tar.gz: 6502c7c0f5b989440b0dd4f849a1665ddf37cb0cb2d90a673ae31e9928148a1bff164111f46696578b0833ef5ca97a376bfc46793d032bc4596986fd57c1c74b
data/dicts/tlds.txt CHANGED
@@ -1,5 +1,5 @@
1
1
  # http://data.iana.org/TLD/tlds-alpha-by-domain.txt
2
- # Version 2018110500, Last Updated Mon Nov 5 07:07:02 2018 UTC
2
+ # Version 2020033000, Last Updated Mon Mar 30 07:07:01 2020 UTC
3
3
  AAA
4
4
  AARP
5
5
  ABARTH
@@ -16,7 +16,6 @@ ACCENTURE
16
16
  ACCOUNTANT
17
17
  ACCOUNTANTS
18
18
  ACO
19
- ACTIVE
20
19
  ACTOR
21
20
  AD
22
21
  ADAC
@@ -146,7 +145,6 @@ BIZ
146
145
  BJ
147
146
  BLACK
148
147
  BLACKFRIDAY
149
- BLANCO
150
148
  BLOCKBUSTER
151
149
  BLOG
152
150
  BLOOMBERG
@@ -155,7 +153,6 @@ BM
155
153
  BMS
156
154
  BMW
157
155
  BN
158
- BNL
159
156
  BNPPARIBAS
160
157
  BO
161
158
  BOATS
@@ -214,7 +211,6 @@ CARE
214
211
  CAREER
215
212
  CAREERS
216
213
  CARS
217
- CARTIER
218
214
  CASA
219
215
  CASE
220
216
  CASEIH
@@ -247,7 +243,6 @@ CHEAP
247
243
  CHINTAI
248
244
  CHRISTMAS
249
245
  CHROME
250
- CHRYSLER
251
246
  CHURCH
252
247
  CI
253
248
  CIPRIANI
@@ -299,6 +294,7 @@ COUNTRY
299
294
  COUPON
300
295
  COUPONS
301
296
  COURSES
297
+ CPA
302
298
  CR
303
299
  CREDIT
304
300
  CREDITCARD
@@ -360,9 +356,7 @@ DNP
360
356
  DO
361
357
  DOCS
362
358
  DOCTOR
363
- DODGE
364
359
  DOG
365
- DOHA
366
360
  DOMAINS
367
361
  DOT
368
362
  DOWNLOAD
@@ -371,7 +365,6 @@ DTV
371
365
  DUBAI
372
366
  DUCK
373
367
  DUNLOP
374
- DUNS
375
368
  DUPONT
376
369
  DURBAN
377
370
  DVAG
@@ -392,7 +385,6 @@ ENERGY
392
385
  ENGINEER
393
386
  ENGINEERING
394
387
  ENTERPRISES
395
- EPOST
396
388
  EPSON
397
389
  EQUIPMENT
398
390
  ER
@@ -408,7 +400,6 @@ EU
408
400
  EUROVISION
409
401
  EUS
410
402
  EVENTS
411
- EVERBANK
412
403
  EXCHANGE
413
404
  EXPERT
414
405
  EXPOSED
@@ -488,6 +479,7 @@ GAME
488
479
  GAMES
489
480
  GAP
490
481
  GARDEN
482
+ GAY
491
483
  GB
492
484
  GBIZ
493
485
  GD
@@ -580,7 +572,6 @@ HOMEGOODS
580
572
  HOMES
581
573
  HOMESENSE
582
574
  HONDA
583
- HONEYWELL
584
575
  HORSE
585
576
  HOSPITAL
586
577
  HOST
@@ -634,7 +625,6 @@ IQ
634
625
  IR
635
626
  IRISH
636
627
  IS
637
- ISELECT
638
628
  ISMAILI
639
629
  IST
640
630
  ISTANBUL
@@ -699,12 +689,10 @@ KYOTO
699
689
  KZ
700
690
  LA
701
691
  LACAIXA
702
- LADBROKES
703
692
  LAMBORGHINI
704
693
  LAMER
705
694
  LANCASTER
706
695
  LANCIA
707
- LANCOME
708
696
  LAND
709
697
  LANDROVER
710
698
  LANXESS
@@ -725,7 +713,6 @@ LEGO
725
713
  LEXUS
726
714
  LGBT
727
715
  LI
728
- LIAISON
729
716
  LIDL
730
717
  LIFE
731
718
  LIFEINSURANCE
@@ -744,6 +731,7 @@ LIVING
744
731
  LIXIL
745
732
  LK
746
733
  LLC
734
+ LLP
747
735
  LOAN
748
736
  LOANS
749
737
  LOCKER
@@ -819,7 +807,6 @@ MN
819
807
  MO
820
808
  MOBI
821
809
  MOBILE
822
- MOBILY
823
810
  MODA
824
811
  MOE
825
812
  MOI
@@ -827,7 +814,6 @@ MOM
827
814
  MONASH
828
815
  MONEY
829
816
  MONSTER
830
- MOPAR
831
817
  MORMON
832
818
  MORTGAGE
833
819
  MOSCOW
@@ -835,7 +821,6 @@ MOTO
835
821
  MOTORCYCLES
836
822
  MOV
837
823
  MOVIE
838
- MOVISTAR
839
824
  MP
840
825
  MQ
841
826
  MR
@@ -854,7 +839,6 @@ MY
854
839
  MZ
855
840
  NA
856
841
  NAB
857
- NADEX
858
842
  NAGOYA
859
843
  NAME
860
844
  NATIONWIDE
@@ -955,7 +939,6 @@ PHOTO
955
939
  PHOTOGRAPHY
956
940
  PHOTOS
957
941
  PHYSIO
958
- PIAGET
959
942
  PICS
960
943
  PICTET
961
944
  PICTURES
@@ -1152,18 +1135,16 @@ SONG
1152
1135
  SONY
1153
1136
  SOY
1154
1137
  SPACE
1155
- SPIEGEL
1156
1138
  SPORT
1157
1139
  SPOT
1158
1140
  SPREADBETTING
1159
1141
  SR
1160
1142
  SRL
1161
- SRT
1143
+ SS
1162
1144
  ST
1163
1145
  STADA
1164
1146
  STAPLES
1165
1147
  STAR
1166
- STARHUB
1167
1148
  STATEBANK
1168
1149
  STATEFARM
1169
1150
  STC
@@ -1211,7 +1192,6 @@ TEAM
1211
1192
  TECH
1212
1193
  TECHNOLOGY
1213
1194
  TEL
1214
- TELEFONICA
1215
1195
  TEMASEK
1216
1196
  TENNIS
1217
1197
  TEVA
@@ -1271,7 +1251,6 @@ TZ
1271
1251
  UA
1272
1252
  UBANK
1273
1253
  UBS
1274
- UCONNECT
1275
1254
  UG
1276
1255
  UK
1277
1256
  UNICOM
@@ -1305,7 +1284,6 @@ VIP
1305
1284
  VIRGIN
1306
1285
  VISA
1307
1286
  VISION
1308
- VISTAPRINT
1309
1287
  VIVA
1310
1288
  VIVO
1311
1289
  VLAANDEREN
@@ -1324,7 +1302,6 @@ WALMART
1324
1302
  WALTER
1325
1303
  WANG
1326
1304
  WANGGOU
1327
- WARMAN
1328
1305
  WATCH
1329
1306
  WATCHES
1330
1307
  WEATHER
@@ -1452,13 +1429,14 @@ XN--MGBA7C0BBN0A
1452
1429
  XN--MGBAAKC7DVF
1453
1430
  XN--MGBAAM7A8H
1454
1431
  XN--MGBAB2BD
1432
+ XN--MGBAH1A3HJKRD
1455
1433
  XN--MGBAI9AZGQP6J
1456
1434
  XN--MGBAYH7GPA
1457
- XN--MGBB9FBPOB
1458
1435
  XN--MGBBH1A
1459
1436
  XN--MGBBH1A71E
1460
1437
  XN--MGBC0A9AZCG
1461
1438
  XN--MGBCA7DZDO
1439
+ XN--MGBCPQ6GPA1A
1462
1440
  XN--MGBERP4A5D4AR
1463
1441
  XN--MGBGU82A
1464
1442
  XN--MGBI4ECEXP
@@ -1484,8 +1462,10 @@ XN--P1AI
1484
1462
  XN--PBT977C
1485
1463
  XN--PGBS0DH
1486
1464
  XN--PSSY2U
1465
+ XN--Q7CE6A
1487
1466
  XN--Q9JYB4C
1488
1467
  XN--QCKA1PMC
1468
+ XN--QXA6A
1489
1469
  XN--QXAM
1490
1470
  XN--RHQV96G
1491
1471
  XN--ROVU88B
@@ -1530,7 +1510,6 @@ ZAPPOS
1530
1510
  ZARA
1531
1511
  ZERO
1532
1512
  ZIP
1533
- ZIPPO
1534
1513
  ZM
1535
1514
  ZONE
1536
1515
  ZUERICH
@@ -99,8 +99,8 @@ class Wmap::CidrTracker
99
99
  #@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
100
100
  @known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
101
101
  @known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
102
- #rescue => ee
103
- # puts "Exception on method #{__method__}: #{ee}" # if @verbose
102
+ rescue => ee
103
+ puts "Exception on method #{__method__}: #{ee}" # if @verbose
104
104
  end
105
105
 
106
106
  # 'setter' to remove an entry to CIDR store @known_cidr_blks
@@ -167,6 +167,7 @@ class Wmap::CidrTracker
167
167
  known = cidr4.contains?(ip+'/32')
168
168
  break if known
169
169
  end
170
+ return known
170
171
  rescue => ee
171
172
  puts "Exception on method #{__method__}: #{ee}" if @verbose
172
173
  return false
@@ -192,7 +192,7 @@ module Wmap
192
192
  # Function to print instance variable - General top level domain list
193
193
  def print_gtld
194
194
  puts @gtld
195
- return @gtld
195
+ return @gtld
196
196
  end
197
197
 
198
198
  # Function to print instance variable - Country code top-level domain list
@@ -159,20 +159,18 @@ module Wmap
159
159
  # Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host.
160
160
  def is_fqdn? (host)
161
161
  puts "Validate the host-name format is valid: #{host}" if @verbose
162
- begin
163
- return false if is_ip?(host) or is_url?(host)
164
- domain=get_domain_root(host)
165
- if domain.nil?
166
- return false
167
- elsif is_domain_root?(domain)
168
- return true
169
- else
170
- return false
171
- end
172
- rescue => ee
173
- puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
162
+ return false if is_ip?(host) or is_url?(host)
163
+ domain=get_domain_root(host)
164
+ if domain.nil?
165
+ return false
166
+ elsif is_domain_root?(domain)
167
+ return true
168
+ else
174
169
  return false
175
170
  end
171
+ # rescue => ee
172
+ # puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
173
+ # return false
176
174
  end
177
175
  alias_method :is_host?, :is_fqdn?
178
176
 
@@ -239,7 +239,11 @@ module Wmap
239
239
  if tag.to_s.include?(pattern)
240
240
  puts tag.to_s if @verbose
241
241
  k=nil
242
- return tag.to_s.scan(/[\d+\.]+\d+/).first
242
+ if tag.to_s.scan(/[\d+\.]+\d+/).first =~ /\d+\./
243
+ return tag.to_s.scan(/[\d+\.]+\d+/).first
244
+ else
245
+ return nil
246
+ end
243
247
  end
244
248
  end
245
249
  end
@@ -0,0 +1,36 @@
1
+ #--
2
+ # Wmap
3
+ #
4
+ # A pure Ruby library for the Internet web application discovery and tracking.
5
+ #
6
+ # Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
7
+ #++
8
+ # Unit Test File for Wmap::DomainTracker.instance class
9
+
10
+ require "minitest/autorun"
11
+ require "Wmap"
12
+
13
+ class CidrTrackerTest < MiniTest::Unit::TestCase
14
+ include Wmap::Utils
15
+
16
+ def test_cidr_add
17
+ w = Wmap::CidrTracker.new
18
+ w.add("192.168.1.0/24")
19
+ assert_equal true, w.known_cidr_blks.key?("192.168.1.0/24")
20
+ end
21
+
22
+ def test_cidr_delete
23
+ w = Wmap::CidrTracker.new
24
+ w.add("10.0.0.0/8")
25
+ w.delete("10.0.0.0/8")
26
+ assert_equal false, w.known_cidr_blks.key?("10.0.0.0/8")
27
+ end
28
+
29
+ def test_ip_trusted?
30
+ w = Wmap::CidrTracker.new
31
+ w.add("192.168.1.0/24")
32
+ assert_equal true, w.ip_trusted?("192.168.1.1")
33
+ assert_equal true, w.ip_trusted?("192.168.1.255")
34
+ end
35
+
36
+ end
data/test/utils_test.rb CHANGED
@@ -12,7 +12,7 @@ require "Wmap"
12
12
 
13
13
  class UtilsTest < MiniTest::Unit::TestCase
14
14
  include Wmap::Utils
15
-
15
+
16
16
  def test_sld_domain_conversion
17
17
  assert_equal "yahoo.com", get_domain_root("yahoo.com")
18
18
  end
@@ -28,75 +28,75 @@ class UtilsTest < MiniTest::Unit::TestCase
28
28
  def test_is_domain_root_case_1?
29
29
  assert_equal false, is_domain_root?("www.yahoo.co.uk")
30
30
  end
31
-
31
+
32
32
  def test_is_domain_root_case_2?
33
33
  assert_equal true, is_domain_root?("yahoo.co.uk")
34
34
  end
35
-
35
+
36
36
  def test_get_sub_domain
37
37
  assert_equal "mail.yahoo.co.uk", get_sub_domain("www.mail.yahoo.co.uk")
38
38
  end
39
39
 
40
40
  def test_is_url_case_1?
41
41
  assert_equal true, is_url?("http://www.mail.yahoo.co.uk/")
42
- end
42
+ end
43
43
 
44
44
  def test_is_url_case_2?
45
45
  assert_equal true, is_url?("https://www.mail.yahoo.co.uk/")
46
- end
46
+ end
47
47
 
48
48
  def test_is_url_case_3?
49
49
  assert_equal false, is_url?("http://www.mail.yahoo.uii/")
50
- end
50
+ end
51
51
 
52
52
  def test_is_url_case_4?
53
53
  assert_equal false, is_url?("http:\\www.mail.yahoo.co.uk")
54
- end
55
-
54
+ end
55
+
56
56
  def test_is_ssl?
57
57
  assert_equal false, is_ssl?("http://www.mail.yahoo.co.uk/")
58
- end
59
-
58
+ end
59
+
60
60
  def test_is_site?
61
61
  assert_equal false, is_site?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
62
- end
63
-
62
+ end
63
+
64
64
  def test_url_2_host
65
65
  assert_equal "login.yahoo.com", url_2_host("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
66
- end
67
-
66
+ end
67
+
68
68
  def test_url_2_site_case_1
69
69
  assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
70
- end
70
+ end
71
71
 
72
72
  def test_url_2_site_case_2
73
73
  assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
74
- end
74
+ end
75
75
 
76
76
  def test_url_2_site_case_3
77
77
  assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com#.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
78
- end
79
-
78
+ end
79
+
80
80
  def test_url_2_path
81
81
  assert_equal "/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", url_2_path("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
82
- end
82
+ end
83
83
 
84
84
  def test_urls_on_same_domain?
85
85
  assert_equal true, urls_on_same_domain?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", "https://us-mg4.mail.yahoo.com/neo/launch?.rand=8hjd08hc6t1lq")
86
- end
86
+ end
87
87
 
88
88
  def test_host_2_url_case_1
89
89
  assert_equal "https://mail.yahoo.com/", host_2_url("mail.yahoo.com",443)
90
- end
90
+ end
91
91
 
92
92
  def test_host_2_url_case_2
93
93
  assert_equal "http://mail.yahoo.com/", host_2_url("mail.yahoo.com")
94
- end
95
-
94
+ end
95
+
96
96
  def test_make_absolute
97
97
  assert_equal "http://games.yahoo.com/game/the-magic-snowman-flash.html", make_absolute("http://games.yahoo.com/","game/the-magic-snowman-flash.html")
98
98
  end
99
-
99
+
100
100
  def test_create_absolute_url_from_base
101
101
  assert_equal "http://images.search.yahoo.com/search/images?p=raiders", create_absolute_url_from_base("http://images.search.yahoo.com/images","/search/images?p=raiders")
102
102
  end
@@ -108,7 +108,7 @@ class UtilsTest < MiniTest::Unit::TestCase
108
108
  def test_normalize_url_case_1
109
109
  assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/./images/search/images?p=raiders")
110
110
  end
111
-
111
+
112
112
  def test_normalize_url_case_2
113
113
  assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/../images/../search/images?p=raiders")
114
114
  end
@@ -116,53 +116,58 @@ class UtilsTest < MiniTest::Unit::TestCase
116
116
  def test_normalize_url_case_3
117
117
  assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com./../images/../search/images?p=raiders")
118
118
  end
119
-
119
+
120
120
  def test_is_ip_case_1?
121
121
  assert_equal false, is_ip?("256.2.3.1")
122
- end
122
+ end
123
123
 
124
124
  def test_is_ip_case_2?
125
125
  assert_equal false, is_ip?("25.2.3.1.22")
126
- end
126
+ end
127
127
 
128
128
  def test_is_ip_case_3?
129
129
  assert_equal true, is_ip?("196.168.230.1")
130
- end
130
+ end
131
131
 
132
132
  def test_is_fqdn_case_1?
133
133
  assert_equal true, is_fqdn?("images.search.yahoo.com")
134
- end
134
+ end
135
135
 
136
136
  def test_is_fqdn_case_2?
137
137
  assert_equal true, is_fqdn?("yahoo.com")
138
- end
139
-
138
+ end
139
+
140
140
  def test_is_fqdn_case_3?
141
- assert_equal false, is_fqdn?("images.search.yahoo")
142
- end
143
-
141
+ # according to latest tlds list - http://data.iana.org/TLD/tlds-alpha-by-domain.txt
142
+ assert_equal true, is_fqdn?("images.search.yahoo")
143
+ end
144
+
144
145
  def test_is_fqdn_case_4?
145
146
  assert_equal false, is_fqdn?("images")
146
- end
147
-
147
+ end
148
+
149
+ def test_is_fqdn_case_5?
150
+ assert_equal false, is_fqdn?("images.search.gargle")
151
+ end
152
+
148
153
  def test_is_cidr_case_1?
149
154
  assert_equal false, is_cidr?("196.168.230.1")
150
- end
155
+ end
151
156
 
152
157
  def test_is_cidr_case_2?
153
158
  assert_equal false, is_cidr?("196.168.2.257/12")
154
- end
155
-
159
+ end
160
+
156
161
  def test_is_cidr_case_3?
157
162
  assert_equal true, is_cidr?("196.168.2.25/12")
158
- end
159
-
163
+ end
164
+
160
165
  def test_cidr_2_ips
161
166
  assert_equal ["192.168.1.1"], cidr_2_ips("192.168.1.1/32")
162
- end
163
-
167
+ end
168
+
164
169
  def test_sort_ips
165
170
  assert_equal ["192.168.1.1", "192.168.1.2", "192.168.2.1"], sort_ips(["192.168.1.2", "192.168.2.1","192.168.1.1"])
166
- end
167
-
171
+ end
172
+
168
173
  end
data/version.txt CHANGED
@@ -3,8 +3,8 @@
3
3
  ###############################################################################
4
4
  package = wmap
5
5
  # wmap version 2.0 == web_discovery version 1.5.3
6
- version = 2.7.7
7
- date = 2020-03-24
6
+ version = 2.7.9
7
+ date = 2020-03-30
8
8
 
9
9
  author = Sam (Yang) Li
10
10
  email = yang.li@owasp.org
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wmap
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.7.7
4
+ version: 2.7.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam (Yang) Li
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-24 00:00:00.000000000 Z
11
+ date: 2020-03-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: dnsruby
@@ -309,6 +309,7 @@ files:
309
309
  - settings/google_keywords.txt
310
310
  - settings/google_locator.txt
311
311
  - settings/tag_signatures
312
+ - test/cidr_tracker_test.rb
312
313
  - test/domain_tracker_test.rb
313
314
  - test/utils_test.rb
314
315
  - version.txt