wmap 2.4.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (141) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +141 -0
  3. data/LICENSE.txt +15 -0
  4. data/README.rdoc +98 -0
  5. data/TODO +13 -0
  6. data/bin/deprime +21 -0
  7. data/bin/distrust +38 -0
  8. data/bin/googleBot +23 -0
  9. data/bin/prime +21 -0
  10. data/bin/refresh +26 -0
  11. data/bin/run_tests +16 -0
  12. data/bin/spiderBot +26 -0
  13. data/bin/trust +38 -0
  14. data/bin/updateAll +57 -0
  15. data/bin/wadd +25 -0
  16. data/bin/wadds +26 -0
  17. data/bin/wcheck +28 -0
  18. data/bin/wdel +25 -0
  19. data/bin/wdump +21 -0
  20. data/bin/wmap +151 -0
  21. data/bin/wscan +32 -0
  22. data/data/cidrs +2 -0
  23. data/data/deactivated_sites +1 -0
  24. data/data/domains +2 -0
  25. data/data/hosts +1 -0
  26. data/data/prime_hosts +1 -0
  27. data/data/sites +2 -0
  28. data/data/sub_domains +2 -0
  29. data/demos/bruter.rb +27 -0
  30. data/demos/dns_brutes.rb +28 -0
  31. data/demos/filter_cidr.rb +18 -0
  32. data/demos/filter_crawls.rb +5 -0
  33. data/demos/filter_domain.rb +25 -0
  34. data/demos/filter_geoip.rb +26 -0
  35. data/demos/filter_known_services.rb +59 -0
  36. data/demos/filter_netinfo.rb +23 -0
  37. data/demos/filter_prime.rb +25 -0
  38. data/demos/filter_profiler.rb +3 -0
  39. data/demos/filter_redirection.rb +19 -0
  40. data/demos/filter_site.rb +40 -0
  41. data/demos/filter_siteip.rb +31 -0
  42. data/demos/filter_status.rb +17 -0
  43. data/demos/filter_timestamp.rb +23 -0
  44. data/demos/filter_url.rb +19 -0
  45. data/demos/new_fnd.rb +66 -0
  46. data/demos/nmap_parser.pl +138 -0
  47. data/demos/site_format.rb +18 -0
  48. data/demos/whois_domain.rb +78 -0
  49. data/dicts/GeoIP.dat +0 -0
  50. data/dicts/GeoIPASNum.dat +0 -0
  51. data/dicts/GeoLiteCity.dat +0 -0
  52. data/dicts/ccsld.txt +2646 -0
  53. data/dicts/cctld.txt +243 -0
  54. data/dicts/gtld.txt +25 -0
  55. data/dicts/hostnames-dict.big +1402 -0
  56. data/dicts/hostnames-dict.txt +101 -0
  57. data/lib/wmap/cidr_tracker.rb +327 -0
  58. data/lib/wmap/dns_bruter.rb +308 -0
  59. data/lib/wmap/domain_tracker/sub_domain.rb +142 -0
  60. data/lib/wmap/domain_tracker.rb +342 -0
  61. data/lib/wmap/geoip_tracker.rb +72 -0
  62. data/lib/wmap/google_search_scraper.rb +177 -0
  63. data/lib/wmap/host_tracker/primary_host.rb +130 -0
  64. data/lib/wmap/host_tracker.rb +550 -0
  65. data/lib/wmap/network_profiler.rb +144 -0
  66. data/lib/wmap/port_scanner.rb +208 -0
  67. data/lib/wmap/site_tracker/deactivated_site.rb +85 -0
  68. data/lib/wmap/site_tracker.rb +937 -0
  69. data/lib/wmap/url_checker.rb +314 -0
  70. data/lib/wmap/url_crawler.rb +381 -0
  71. data/lib/wmap/utils/domain_root.rb +184 -0
  72. data/lib/wmap/utils/logger.rb +53 -0
  73. data/lib/wmap/utils/url_magic.rb +343 -0
  74. data/lib/wmap/utils/utils.rb +333 -0
  75. data/lib/wmap/whois.rb +76 -0
  76. data/lib/wmap.rb +227 -0
  77. data/logs/wmap.log +17 -0
  78. data/ruby_whois_patches/base_cocca2.rb +149 -0
  79. data/ruby_whois_patches/kero.yachay.pe.rb +120 -0
  80. data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +124 -0
  81. data/ruby_whois_patches/whois.above.com.rb +61 -0
  82. data/ruby_whois_patches/whois.adamsnames.tc.rb +107 -0
  83. data/ruby_whois_patches/whois.aeda.net.ae.rb +105 -0
  84. data/ruby_whois_patches/whois.ai.rb +112 -0
  85. data/ruby_whois_patches/whois.arnes.si.rb +121 -0
  86. data/ruby_whois_patches/whois.ascio.com.rb +91 -0
  87. data/ruby_whois_patches/whois.cnnic.cn.rb +123 -0
  88. data/ruby_whois_patches/whois.corporatedomains.com.rb +67 -0
  89. data/ruby_whois_patches/whois.crsnic.net.rb +108 -0
  90. data/ruby_whois_patches/whois.denic.de.rb +174 -0
  91. data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +120 -0
  92. data/ruby_whois_patches/whois.dns.be.rb +134 -0
  93. data/ruby_whois_patches/whois.dns.lu.rb +129 -0
  94. data/ruby_whois_patches/whois.dns.pl.rb +150 -0
  95. data/ruby_whois_patches/whois.dns.pt.rb +119 -0
  96. data/ruby_whois_patches/whois.domain.kg.rb +126 -0
  97. data/ruby_whois_patches/whois.domainregistry.my.rb +123 -0
  98. data/ruby_whois_patches/whois.domreg.lt.rb +110 -0
  99. data/ruby_whois_patches/whois.dot.tk.rb +140 -0
  100. data/ruby_whois_patches/whois.hkirc.hk.rb +121 -0
  101. data/ruby_whois_patches/whois.isnic.is.rb +130 -0
  102. data/ruby_whois_patches/whois.je.rb +119 -0
  103. data/ruby_whois_patches/whois.jprs.jp.rb +137 -0
  104. data/ruby_whois_patches/whois.kenic.or.ke.rb +140 -0
  105. data/ruby_whois_patches/whois.markmonitor.com.rb +118 -0
  106. data/ruby_whois_patches/whois.melbourneit.com.rb +58 -0
  107. data/ruby_whois_patches/whois.nic.as.rb +96 -0
  108. data/ruby_whois_patches/whois.nic.at.rb +109 -0
  109. data/ruby_whois_patches/whois.nic.ch.rb +141 -0
  110. data/ruby_whois_patches/whois.nic.cl.rb +117 -0
  111. data/ruby_whois_patches/whois.nic.ec.rb +157 -0
  112. data/ruby_whois_patches/whois.nic.im.rb +120 -0
  113. data/ruby_whois_patches/whois.nic.it.rb +170 -0
  114. data/ruby_whois_patches/whois.nic.lv.rb +116 -0
  115. data/ruby_whois_patches/whois.nic.ly.rb +127 -0
  116. data/ruby_whois_patches/whois.nic.mu.rb +27 -0
  117. data/ruby_whois_patches/whois.nic.mx.rb +123 -0
  118. data/ruby_whois_patches/whois.nic.net.sa.rb +111 -0
  119. data/ruby_whois_patches/whois.nic.or.kr.rb +101 -0
  120. data/ruby_whois_patches/whois.nic.tel.rb +129 -0
  121. data/ruby_whois_patches/whois.nic.tr.rb +133 -0
  122. data/ruby_whois_patches/whois.nic.us.rb +129 -0
  123. data/ruby_whois_patches/whois.nic.ve.rb +135 -0
  124. data/ruby_whois_patches/whois.norid.no.rb +127 -0
  125. data/ruby_whois_patches/whois.pandi.or.id.rb +118 -0
  126. data/ruby_whois_patches/whois.psi-usa.info.rb +63 -0
  127. data/ruby_whois_patches/whois.registro.br.rb +109 -0
  128. data/ruby_whois_patches/whois.registrygate.com.rb +55 -0
  129. data/ruby_whois_patches/whois.rrpproxy.net.rb +61 -0
  130. data/ruby_whois_patches/whois.sgnic.sg.rb +130 -0
  131. data/ruby_whois_patches/whois.srs.net.nz.rb +166 -0
  132. data/ruby_whois_patches/whois.tucows.com.rb +70 -0
  133. data/ruby_whois_patches/whois.twnic.net.tw.rb +133 -0
  134. data/settings/discovery_ports +24 -0
  135. data/settings/google_keywords.txt +9 -0
  136. data/settings/google_locator.txt +23 -0
  137. data/test/domain_tracker_test.rb +31 -0
  138. data/test/utils_test.rb +168 -0
  139. data/version.txt +13 -0
  140. data/wmap.gemspec +49 -0
  141. metadata +202 -0
data/dicts/cctld.txt ADDED
@@ -0,0 +1,243 @@
1
+ # Country code top-level domain list
2
+ # Refer to: http://en.wikipedia.org/wiki/Country_code_top-level_domain
3
+ ac
4
+ ad
5
+ ae
6
+ af
7
+ ag
8
+ ai
9
+ al
10
+ am
11
+ an
12
+ ao
13
+ aq
14
+ ar
15
+ as
16
+ at
17
+ au
18
+ aw
19
+ ax
20
+ az
21
+ ba
22
+ bb
23
+ bd
24
+ be
25
+ bf
26
+ bg
27
+ bh
28
+ bi
29
+ bj
30
+ bm
31
+ bn
32
+ bo
33
+ br
34
+ bs
35
+ bt
36
+ bw
37
+ by
38
+ bz
39
+ ca
40
+ cc
41
+ cd
42
+ cf
43
+ cg
44
+ ch
45
+ ci
46
+ ck
47
+ cl
48
+ cm
49
+ cn
50
+ co
51
+ cr
52
+ cu
53
+ cv
54
+ cx
55
+ cy
56
+ cz
57
+ de
58
+ dj
59
+ dk
60
+ dm
61
+ do
62
+ dz
63
+ ec
64
+ ee
65
+ eg
66
+ er
67
+ es
68
+ et
69
+ eu
70
+ fi
71
+ fj
72
+ fk
73
+ fm
74
+ fo
75
+ fr
76
+ ga
77
+ gd
78
+ ge
79
+ gf
80
+ gg
81
+ gh
82
+ gi
83
+ gl
84
+ gm
85
+ gn
86
+ gp
87
+ gq
88
+ gr
89
+ gs
90
+ gt
91
+ gu
92
+ gw
93
+ gy
94
+ hk
95
+ hm
96
+ hn
97
+ hr
98
+ ht
99
+ hu
100
+ id
101
+ ie
102
+ il
103
+ im
104
+ in
105
+ io
106
+ iq
107
+ ir
108
+ is
109
+ it
110
+ je
111
+ jm
112
+ jo
113
+ jp
114
+ ke
115
+ kg
116
+ kh
117
+ ki
118
+ km
119
+ kn
120
+ kp
121
+ kr
122
+ kw
123
+ ky
124
+ kz
125
+ la
126
+ lb
127
+ lc
128
+ li
129
+ lk
130
+ lr
131
+ ls
132
+ lt
133
+ lu
134
+ lv
135
+ ly
136
+ ma
137
+ mc
138
+ md
139
+ me
140
+ mg
141
+ mh
142
+ mk
143
+ ml
144
+ mm
145
+ mn
146
+ mo
147
+ mp
148
+ mq
149
+ mr
150
+ ms
151
+ mt
152
+ mu
153
+ mv
154
+ mw
155
+ mx
156
+ my
157
+ mz
158
+ na
159
+ nc
160
+ ne
161
+ nf
162
+ ng
163
+ ni
164
+ nl
165
+ no
166
+ np
167
+ nr
168
+ nu
169
+ nz
170
+ om
171
+ pa
172
+ pe
173
+ pf
174
+ pg
175
+ ph
176
+ pk
177
+ pl
178
+ pn
179
+ pr
180
+ ps
181
+ pt
182
+ pw
183
+ py
184
+ qa
185
+ re
186
+ ro
187
+ rs
188
+ ru
189
+ rw
190
+ sa
191
+ sb
192
+ sc
193
+ sd
194
+ se
195
+ sg
196
+ sh
197
+ si
198
+ sk
199
+ sl
200
+ sm
201
+ sn
202
+ so
203
+ sr
204
+ st
205
+ su
206
+ sv
207
+ sy
208
+ sz
209
+ tc
210
+ td
211
+ tf
212
+ tg
213
+ th
214
+ tj
215
+ tk
216
+ tl
217
+ tm
218
+ tn
219
+ to
220
+ tr
221
+ tt
222
+ tv
223
+ tw
224
+ tz
225
+ ua
226
+ ug
227
+ uk
228
+ us
229
+ uy
230
+ uz
231
+ va
232
+ vc
233
+ ve
234
+ vg
235
+ vi
236
+ vn
237
+ vu
238
+ wf
239
+ ws
240
+ ye
241
+ za
242
+ zm
243
+ zw
data/dicts/gtld.txt ADDED
@@ -0,0 +1,25 @@
1
+ # Generic Top Level Domain List
2
+ # Refer to: http://en.wikipedia.org/wiki/Generic_top-level_domain
3
+ aero
4
+ asia
5
+ biz
6
+ cat
7
+ com
8
+ coop
9
+ edu
10
+ gov
11
+ info
12
+ int
13
+ jobs
14
+ mil
15
+ mobi
16
+ museum
17
+ name
18
+ net
19
+ org
20
+ post
21
+ pro
22
+ tel
23
+ travel
24
+ tm
25
+ xxx