apache-log-geo 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +109 -0
- data/apache-log-geo +72 -0
- data/lib.rb +275 -0
- data/mmdb-lookup +63 -0
- metadata +66 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 88358db0f03c32a5141201af91a3fa617f123b00c0fad7a8e063a4e51a1fe75e
|
|
4
|
+
data.tar.gz: 37906a57323328d07d516d3b7b6bb5cc71ffb200949158206f952708e1df0cb0
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: edba9d9da0d8e342b84403df8d6459f449aac6c65cb5d69b59f696a69562aea26c4a3a7febae2928d2852721301bb0b9564d38a38fa21fca6c1fe062e5c39234
|
|
7
|
+
data.tar.gz: dde46e90f32f32fc69b7292e12db7687f2319d3b6a308f088996f9fab24b91a30d4c355a75e38448e3eeff5fd7b5c35954a3acf7423b34230b55c3ced543698b
|
data/README.md
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# apache-log-geo
|
|
2
|
+
|
|
3
|
+
An offline GeoIP CLI filter for Apache (common, combined) logs. It's
|
|
4
|
+
like grep but with a knowledge about what data an ip holds. Supa
|
|
5
|
+
handy!
|
|
6
|
+
|
|
7
|
+
Reqs:
|
|
8
|
+
|
|
9
|
+
* ruby
|
|
10
|
+
* `dnf install libmaxminddb-devel geolite2-city`
|
|
11
|
+
|
|
12
|
+
If there's no geolite2-city pkg (that contains `GeoLite2-City.mmdb`
|
|
13
|
+
file) for your system, register on MaxMind's website, get a license
|
|
14
|
+
key & install geoipupdate to fetch the db file.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
gem install apache-log-geo
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
The pkg contains 2 CLI utils only. There's no reusable library code.
|
|
23
|
+
|
|
24
|
+
### apache-log-geo
|
|
25
|
+
|
|
26
|
+
This is a simple grep-like filter:
|
|
27
|
+
|
|
28
|
+
~~~
|
|
29
|
+
$ ./apache-log-geo -h
|
|
30
|
+
Usage: apache-log-geo [-d GeoLite2-City.mmdb] [-v] [--key val ...]
|
|
31
|
+
-d path maxmind db file
|
|
32
|
+
-v invert match
|
|
33
|
+
--city regexp
|
|
34
|
+
--country regexp
|
|
35
|
+
--cc str 2 letter country code
|
|
36
|
+
--continent regexp
|
|
37
|
+
--postcode regexp
|
|
38
|
+
--sub regexp subdivisions
|
|
39
|
+
~~~
|
|
40
|
+
|
|
41
|
+
It tries to guess the location of the .mmdb file, thus specifying `-d`
|
|
42
|
+
opt is often unnecessary.
|
|
43
|
+
|
|
44
|
+
Options that begin with `--` constitute test conditions for a
|
|
45
|
+
filter. Conditions are anded. Unlike grep, specifying no conditions is
|
|
46
|
+
not an error--the util will act as a pass through for each valid log
|
|
47
|
+
line that starts with an ip address that is known to the GeoLite2 db.
|
|
48
|
+
|
|
49
|
+
`--cc` opt is special: it doesn't take a regexp, but a 2-letter codes
|
|
50
|
+
separated with `|`.
|
|
51
|
+
|
|
52
|
+
#### Examples
|
|
53
|
+
|
|
54
|
+
A pass through:
|
|
55
|
+
|
|
56
|
+
~~~
|
|
57
|
+
$ head -2 test/access.log | ./apache-log-geo
|
|
58
|
+
52.18.122.238 - - [06/Mar/2020:00:02:00 -0500] "GET /~alex/doc/bunz%2Cmercedes__school-will-never-end/ HTTP/1.1" 200 17133 "-" "Apache-HttpClient/4.3.6 (java 1.5)"
|
|
59
|
+
54.174.110.177 - - [06/Mar/2020:00:03:10 -0500] "GET /~alex/doc/bunz%2Cmercedes__school-will-never-end/ HTTP/1.1" 200 17133 "-" "Ruby"
|
|
60
|
+
~~~
|
|
61
|
+
|
|
62
|
+
Filter by a country code:
|
|
63
|
+
|
|
64
|
+
~~~
|
|
65
|
+
$ head -2 test/access.log | ./apache-log-geo --cc ie
|
|
66
|
+
52.18.122.238 - - [06/Mar/2020:00:02:00 -0500] "GET /~alex/doc/bunz%2Cmercedes__school-will-never-end/ HTTP/1.1" 200 17133 "-" "Apache-HttpClient/4.3.6 (java 1.5)"
|
|
67
|
+
|
|
68
|
+
$ cat test/access.log | ./apache-log-geo --cc 'ie|de' | wc -l
|
|
69
|
+
11
|
|
70
|
+
~~~
|
|
71
|
+
|
|
72
|
+
### mmdb-lookup
|
|
73
|
+
|
|
74
|
+
Renders the data about ip addresses in json (default) or a suitable
|
|
75
|
+
for a shell script formats:
|
|
76
|
+
|
|
77
|
+
~~~
|
|
78
|
+
$ ./mmdb-lookup -h
|
|
79
|
+
Usage: mmdb-lookup [-d GeoLite2-City.mmdb] [-f fmt] ip...
|
|
80
|
+
-d path maxmind db file
|
|
81
|
+
-f fmt output format: json, shell
|
|
82
|
+
~~~
|
|
83
|
+
|
|
84
|
+
IPs can come either from the command line or from the stdin. Again,
|
|
85
|
+
`-d` is optional.
|
|
86
|
+
|
|
87
|
+
#### Examples
|
|
88
|
+
|
|
89
|
+
Evaluate a printed shell code:
|
|
90
|
+
|
|
91
|
+
~~~
|
|
92
|
+
$ (eval `./mmdb-lookup 5.1.0.0 -f shell`; echo $subdivisions)
|
|
93
|
+
Kyiv City
|
|
94
|
+
~~~
|
|
95
|
+
|
|
96
|
+
Replicate `apache-log-geo` util--print only the requests from the Irish
|
|
97
|
+
(the example requires `npm -g json`):
|
|
98
|
+
|
|
99
|
+
$ awk '{print $1}' test/access.log | ./mmdb-lookup | json -c 'this.country_code == "IE"' -a ip | grep -h -f - test/access.log
|
|
100
|
+
|
|
101
|
+
## Exit status
|
|
102
|
+
|
|
103
|
+
* 0 -- some lines were matched
|
|
104
|
+
* 1 -- nothing was matched
|
|
105
|
+
* 2 -- an error occurred
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
MIT.
|
data/apache-log-geo
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'optparse'
|
|
4
|
+
require 'mmdb'
|
|
5
|
+
require_relative 'lib'
|
|
6
|
+
include ApacheLogGeo
|
|
7
|
+
|
|
8
|
+
# exact match: country_code, postcode
|
|
9
|
+
# regexp: continent, country, subdivisions, city
|
|
10
|
+
#
|
|
11
|
+
# conditions are anded. no conditions == true
|
|
12
|
+
def match_by_geo info, query
|
|
13
|
+
query.each do |k,v|
|
|
14
|
+
if k == :subdivisions # it's an array, one match of its elements is enough
|
|
15
|
+
return false unless (info[k] || []).any? {|val| val.to_s =~ /#{v}/i }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
return false unless info[k].to_s =~ /#{v}/i
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
true
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
opt = {
|
|
25
|
+
db: geo_db_location,
|
|
26
|
+
v: false,
|
|
27
|
+
query: {}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
OptionParser.new do |o|
|
|
31
|
+
o.banner = "Usage: #{File.basename $0} [-d GeoLite2-City.mmdb] [-v] [--key val ...]"
|
|
32
|
+
o.on("-d path", "maxmind db file") { |v| opt[:db] = v }
|
|
33
|
+
o.on("-v", "invert match") { |v| opt[:v] = v }
|
|
34
|
+
o.on("--city regexp") { |v| opt[:query][:city] = v }
|
|
35
|
+
o.on("--country regexp") { |v| opt[:query][:country] = v }
|
|
36
|
+
o.on("--cc str", "2 letter country code") { |v|
|
|
37
|
+
v.split('|').each do |cc|
|
|
38
|
+
errx 2, "invalid country code: #{cc}" unless COUNTRY_CODES.key? cc.upcase
|
|
39
|
+
end
|
|
40
|
+
opt[:query][:country_code] = v
|
|
41
|
+
}
|
|
42
|
+
o.on("--continent regexp") { |v| opt[:query][:continent] = v }
|
|
43
|
+
o.on("--postcode regexp") { |v| opt[:query][:postcode] = v }
|
|
44
|
+
o.on("--sub regexp", "subdivisions") { |v| opt[:query][:subdivisions] = v }
|
|
45
|
+
end.parse!
|
|
46
|
+
|
|
47
|
+
begin
|
|
48
|
+
geo = MaxMindDB.new opt[:db]
|
|
49
|
+
rescue
|
|
50
|
+
errx 2, "failed to open #{opt[:db]}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
found_anything = false
|
|
54
|
+
$stdin.each_line do |line|
|
|
55
|
+
next if line =~ /^\s*$/
|
|
56
|
+
ip = line.split[0]
|
|
57
|
+
begin
|
|
58
|
+
info = geo.lookup ip
|
|
59
|
+
rescue
|
|
60
|
+
warnx "no data, filtering the entry out: `${ip[0..15]}`"
|
|
61
|
+
next
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
match = match_by_geo info, opt[:query]
|
|
65
|
+
match = !match if opt[:v]
|
|
66
|
+
if match
|
|
67
|
+
found_anything = true
|
|
68
|
+
puts line
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
exit found_anything ? 0 : 1
|
data/lib.rb
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
module ApacheLogGeo
|
|
3
|
+
# save user some typing if we can guess the OS
|
|
4
|
+
def geo_db_location
|
|
5
|
+
datadir = case RUBY_PLATFORM
|
|
6
|
+
when /linux/ then '/usr/share'
|
|
7
|
+
when /cygwin|msys/ then '/usr/share'
|
|
8
|
+
when /darwin/ then '/usr/local/var' # untested
|
|
9
|
+
when /freebsd/ then '/usr/local/share'
|
|
10
|
+
else nil
|
|
11
|
+
end
|
|
12
|
+
return 'GeoLite2-City.mmdb' unless datadir
|
|
13
|
+
File.join datadir, 'GeoIP/GeoLite2-City.mmdb'
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def errx exit_code, msg
|
|
17
|
+
$stderr.puts "#{File.basename $0} error: #{msg}"
|
|
18
|
+
exit exit_code
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def warnx msg; $stderr.puts "#{File.basename $0} warning: #{msg}"; end
|
|
22
|
+
|
|
23
|
+
# https://datahub.io/core/country-list
|
|
24
|
+
COUNTRY_CODES = {
|
|
25
|
+
"AF" => "Afghanistan",
|
|
26
|
+
"AX" => "Åland Islands",
|
|
27
|
+
"AL" => "Albania",
|
|
28
|
+
"DZ" => "Algeria",
|
|
29
|
+
"AS" => "American Samoa",
|
|
30
|
+
"AD" => "Andorra",
|
|
31
|
+
"AO" => "Angola",
|
|
32
|
+
"AI" => "Anguilla",
|
|
33
|
+
"AQ" => "Antarctica",
|
|
34
|
+
"AG" => "Antigua and Barbuda",
|
|
35
|
+
"AR" => "Argentina",
|
|
36
|
+
"AM" => "Armenia",
|
|
37
|
+
"AW" => "Aruba",
|
|
38
|
+
"AU" => "Australia",
|
|
39
|
+
"AT" => "Austria",
|
|
40
|
+
"AZ" => "Azerbaijan",
|
|
41
|
+
"BS" => "Bahamas",
|
|
42
|
+
"BH" => "Bahrain",
|
|
43
|
+
"BD" => "Bangladesh",
|
|
44
|
+
"BB" => "Barbados",
|
|
45
|
+
"BY" => "Belarus",
|
|
46
|
+
"BE" => "Belgium",
|
|
47
|
+
"BZ" => "Belize",
|
|
48
|
+
"BJ" => "Benin",
|
|
49
|
+
"BM" => "Bermuda",
|
|
50
|
+
"BT" => "Bhutan",
|
|
51
|
+
"BO" => "Bolivia, Plurinational State of",
|
|
52
|
+
"BQ" => "Bonaire, Sint Eustatius and Saba",
|
|
53
|
+
"BA" => "Bosnia and Herzegovina",
|
|
54
|
+
"BW" => "Botswana",
|
|
55
|
+
"BV" => "Bouvet Island",
|
|
56
|
+
"BR" => "Brazil",
|
|
57
|
+
"IO" => "British Indian Ocean Territory",
|
|
58
|
+
"BN" => "Brunei Darussalam",
|
|
59
|
+
"BG" => "Bulgaria",
|
|
60
|
+
"BF" => "Burkina Faso",
|
|
61
|
+
"BI" => "Burundi",
|
|
62
|
+
"KH" => "Cambodia",
|
|
63
|
+
"CM" => "Cameroon",
|
|
64
|
+
"CA" => "Canada",
|
|
65
|
+
"CV" => "Cape Verde",
|
|
66
|
+
"KY" => "Cayman Islands",
|
|
67
|
+
"CF" => "Central African Republic",
|
|
68
|
+
"TD" => "Chad",
|
|
69
|
+
"CL" => "Chile",
|
|
70
|
+
"CN" => "China",
|
|
71
|
+
"CX" => "Christmas Island",
|
|
72
|
+
"CC" => "Cocos (Keeling) Islands",
|
|
73
|
+
"CO" => "Colombia",
|
|
74
|
+
"KM" => "Comoros",
|
|
75
|
+
"CG" => "Congo",
|
|
76
|
+
"CD" => "Congo, the Democratic Republic of the",
|
|
77
|
+
"CK" => "Cook Islands",
|
|
78
|
+
"CR" => "Costa Rica",
|
|
79
|
+
"CI" => "Côte d'Ivoire",
|
|
80
|
+
"HR" => "Croatia",
|
|
81
|
+
"CU" => "Cuba",
|
|
82
|
+
"CW" => "Curaçao",
|
|
83
|
+
"CY" => "Cyprus",
|
|
84
|
+
"CZ" => "Czech Republic",
|
|
85
|
+
"DK" => "Denmark",
|
|
86
|
+
"DJ" => "Djibouti",
|
|
87
|
+
"DM" => "Dominica",
|
|
88
|
+
"DO" => "Dominican Republic",
|
|
89
|
+
"EC" => "Ecuador",
|
|
90
|
+
"EG" => "Egypt",
|
|
91
|
+
"SV" => "El Salvador",
|
|
92
|
+
"GQ" => "Equatorial Guinea",
|
|
93
|
+
"ER" => "Eritrea",
|
|
94
|
+
"EE" => "Estonia",
|
|
95
|
+
"ET" => "Ethiopia",
|
|
96
|
+
"FK" => "Falkland Islands (Malvinas)",
|
|
97
|
+
"FO" => "Faroe Islands",
|
|
98
|
+
"FJ" => "Fiji",
|
|
99
|
+
"FI" => "Finland",
|
|
100
|
+
"FR" => "France",
|
|
101
|
+
"GF" => "French Guiana",
|
|
102
|
+
"PF" => "French Polynesia",
|
|
103
|
+
"TF" => "French Southern Territories",
|
|
104
|
+
"GA" => "Gabon",
|
|
105
|
+
"GM" => "Gambia",
|
|
106
|
+
"GE" => "Georgia",
|
|
107
|
+
"DE" => "Germany",
|
|
108
|
+
"GH" => "Ghana",
|
|
109
|
+
"GI" => "Gibraltar",
|
|
110
|
+
"GR" => "Greece",
|
|
111
|
+
"GL" => "Greenland",
|
|
112
|
+
"GD" => "Grenada",
|
|
113
|
+
"GP" => "Guadeloupe",
|
|
114
|
+
"GU" => "Guam",
|
|
115
|
+
"GT" => "Guatemala",
|
|
116
|
+
"GG" => "Guernsey",
|
|
117
|
+
"GN" => "Guinea",
|
|
118
|
+
"GW" => "Guinea-Bissau",
|
|
119
|
+
"GY" => "Guyana",
|
|
120
|
+
"HT" => "Haiti",
|
|
121
|
+
"HM" => "Heard Island and McDonald Islands",
|
|
122
|
+
"VA" => "Holy See (Vatican City State)",
|
|
123
|
+
"HN" => "Honduras",
|
|
124
|
+
"HK" => "Hong Kong",
|
|
125
|
+
"HU" => "Hungary",
|
|
126
|
+
"IS" => "Iceland",
|
|
127
|
+
"IN" => "India",
|
|
128
|
+
"ID" => "Indonesia",
|
|
129
|
+
"IR" => "Iran, Islamic Republic of",
|
|
130
|
+
"IQ" => "Iraq",
|
|
131
|
+
"IE" => "Ireland",
|
|
132
|
+
"IM" => "Isle of Man",
|
|
133
|
+
"IL" => "Israel",
|
|
134
|
+
"IT" => "Italy",
|
|
135
|
+
"JM" => "Jamaica",
|
|
136
|
+
"JP" => "Japan",
|
|
137
|
+
"JE" => "Jersey",
|
|
138
|
+
"JO" => "Jordan",
|
|
139
|
+
"KZ" => "Kazakhstan",
|
|
140
|
+
"KE" => "Kenya",
|
|
141
|
+
"KI" => "Kiribati",
|
|
142
|
+
"KP" => "Korea, Democratic People's Republic of",
|
|
143
|
+
"KR" => "Korea, Republic of",
|
|
144
|
+
"KW" => "Kuwait",
|
|
145
|
+
"KG" => "Kyrgyzstan",
|
|
146
|
+
"LA" => "Lao People's Democratic Republic",
|
|
147
|
+
"LV" => "Latvia",
|
|
148
|
+
"LB" => "Lebanon",
|
|
149
|
+
"LS" => "Lesotho",
|
|
150
|
+
"LR" => "Liberia",
|
|
151
|
+
"LY" => "Libya",
|
|
152
|
+
"LI" => "Liechtenstein",
|
|
153
|
+
"LT" => "Lithuania",
|
|
154
|
+
"LU" => "Luxembourg",
|
|
155
|
+
"MO" => "Macao",
|
|
156
|
+
"MK" => "Macedonia, the Former Yugoslav Republic of",
|
|
157
|
+
"MG" => "Madagascar",
|
|
158
|
+
"MW" => "Malawi",
|
|
159
|
+
"MY" => "Malaysia",
|
|
160
|
+
"MV" => "Maldives",
|
|
161
|
+
"ML" => "Mali",
|
|
162
|
+
"MT" => "Malta",
|
|
163
|
+
"MH" => "Marshall Islands",
|
|
164
|
+
"MQ" => "Martinique",
|
|
165
|
+
"MR" => "Mauritania",
|
|
166
|
+
"MU" => "Mauritius",
|
|
167
|
+
"YT" => "Mayotte",
|
|
168
|
+
"MX" => "Mexico",
|
|
169
|
+
"FM" => "Micronesia, Federated States of",
|
|
170
|
+
"MD" => "Moldova, Republic of",
|
|
171
|
+
"MC" => "Monaco",
|
|
172
|
+
"MN" => "Mongolia",
|
|
173
|
+
"ME" => "Montenegro",
|
|
174
|
+
"MS" => "Montserrat",
|
|
175
|
+
"MA" => "Morocco",
|
|
176
|
+
"MZ" => "Mozambique",
|
|
177
|
+
"MM" => "Myanmar",
|
|
178
|
+
"NA" => "Namibia",
|
|
179
|
+
"NR" => "Nauru",
|
|
180
|
+
"NP" => "Nepal",
|
|
181
|
+
"NL" => "Netherlands",
|
|
182
|
+
"NC" => "New Caledonia",
|
|
183
|
+
"NZ" => "New Zealand",
|
|
184
|
+
"NI" => "Nicaragua",
|
|
185
|
+
"NE" => "Niger",
|
|
186
|
+
"NG" => "Nigeria",
|
|
187
|
+
"NU" => "Niue",
|
|
188
|
+
"NF" => "Norfolk Island",
|
|
189
|
+
"MP" => "Northern Mariana Islands",
|
|
190
|
+
"NO" => "Norway",
|
|
191
|
+
"OM" => "Oman",
|
|
192
|
+
"PK" => "Pakistan",
|
|
193
|
+
"PW" => "Palau",
|
|
194
|
+
"PS" => "Palestine, State of",
|
|
195
|
+
"PA" => "Panama",
|
|
196
|
+
"PG" => "Papua New Guinea",
|
|
197
|
+
"PY" => "Paraguay",
|
|
198
|
+
"PE" => "Peru",
|
|
199
|
+
"PH" => "Philippines",
|
|
200
|
+
"PN" => "Pitcairn",
|
|
201
|
+
"PL" => "Poland",
|
|
202
|
+
"PT" => "Portugal",
|
|
203
|
+
"PR" => "Puerto Rico",
|
|
204
|
+
"QA" => "Qatar",
|
|
205
|
+
"RE" => "Réunion",
|
|
206
|
+
"RO" => "Romania",
|
|
207
|
+
"RU" => "Russian Federation",
|
|
208
|
+
"RW" => "Rwanda",
|
|
209
|
+
"BL" => "Saint Barthélemy",
|
|
210
|
+
"SH" => "Saint Helena, Ascension and Tristan da Cunha",
|
|
211
|
+
"KN" => "Saint Kitts and Nevis",
|
|
212
|
+
"LC" => "Saint Lucia",
|
|
213
|
+
"MF" => "Saint Martin (French part)",
|
|
214
|
+
"PM" => "Saint Pierre and Miquelon",
|
|
215
|
+
"VC" => "Saint Vincent and the Grenadines",
|
|
216
|
+
"WS" => "Samoa",
|
|
217
|
+
"SM" => "San Marino",
|
|
218
|
+
"ST" => "Sao Tome and Principe",
|
|
219
|
+
"SA" => "Saudi Arabia",
|
|
220
|
+
"SN" => "Senegal",
|
|
221
|
+
"RS" => "Serbia",
|
|
222
|
+
"SC" => "Seychelles",
|
|
223
|
+
"SL" => "Sierra Leone",
|
|
224
|
+
"SG" => "Singapore",
|
|
225
|
+
"SX" => "Sint Maarten (Dutch part)",
|
|
226
|
+
"SK" => "Slovakia",
|
|
227
|
+
"SI" => "Slovenia",
|
|
228
|
+
"SB" => "Solomon Islands",
|
|
229
|
+
"SO" => "Somalia",
|
|
230
|
+
"ZA" => "South Africa",
|
|
231
|
+
"GS" => "South Georgia and the South Sandwich Islands",
|
|
232
|
+
"SS" => "South Sudan",
|
|
233
|
+
"ES" => "Spain",
|
|
234
|
+
"LK" => "Sri Lanka",
|
|
235
|
+
"SD" => "Sudan",
|
|
236
|
+
"SR" => "Suriname",
|
|
237
|
+
"SJ" => "Svalbard and Jan Mayen",
|
|
238
|
+
"SZ" => "Swaziland",
|
|
239
|
+
"SE" => "Sweden",
|
|
240
|
+
"CH" => "Switzerland",
|
|
241
|
+
"SY" => "Syrian Arab Republic",
|
|
242
|
+
"TW" => "Taiwan, Province of China",
|
|
243
|
+
"TJ" => "Tajikistan",
|
|
244
|
+
"TZ" => "Tanzania, United Republic of",
|
|
245
|
+
"TH" => "Thailand",
|
|
246
|
+
"TL" => "Timor-Leste",
|
|
247
|
+
"TG" => "Togo",
|
|
248
|
+
"TK" => "Tokelau",
|
|
249
|
+
"TO" => "Tonga",
|
|
250
|
+
"TT" => "Trinidad and Tobago",
|
|
251
|
+
"TN" => "Tunisia",
|
|
252
|
+
"TR" => "Turkey",
|
|
253
|
+
"TM" => "Turkmenistan",
|
|
254
|
+
"TC" => "Turks and Caicos Islands",
|
|
255
|
+
"TV" => "Tuvalu",
|
|
256
|
+
"UG" => "Uganda",
|
|
257
|
+
"UA" => "Ukraine",
|
|
258
|
+
"AE" => "United Arab Emirates",
|
|
259
|
+
"GB" => "United Kingdom",
|
|
260
|
+
"US" => "United States",
|
|
261
|
+
"UM" => "United States Minor Outlying Islands",
|
|
262
|
+
"UY" => "Uruguay",
|
|
263
|
+
"UZ" => "Uzbekistan",
|
|
264
|
+
"VU" => "Vanuatu",
|
|
265
|
+
"VE" => "Venezuela, Bolivarian Republic of",
|
|
266
|
+
"VN" => "Viet Nam",
|
|
267
|
+
"VG" => "Virgin Islands, British",
|
|
268
|
+
"VI" => "Virgin Islands, U.S.",
|
|
269
|
+
"WF" => "Wallis and Futuna",
|
|
270
|
+
"EH" => "Western Sahara",
|
|
271
|
+
"YE" => "Yemen",
|
|
272
|
+
"ZM" => "Zambia",
|
|
273
|
+
"ZW" => "Zimbabwe"
|
|
274
|
+
}
|
|
275
|
+
end
|
data/mmdb-lookup
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'optparse'
|
|
4
|
+
require 'mmdb'
|
|
5
|
+
require_relative 'lib'
|
|
6
|
+
include ApacheLogGeo
|
|
7
|
+
|
|
8
|
+
class FMT
|
|
9
|
+
def initialize input, geo
|
|
10
|
+
@input = input; @geo = geo
|
|
11
|
+
@errors = 0
|
|
12
|
+
end
|
|
13
|
+
attr_accessor :errors
|
|
14
|
+
|
|
15
|
+
def shell # print only the 1st entry
|
|
16
|
+
require 'shellwords'
|
|
17
|
+
ip = @input.to_enum.next.split.first
|
|
18
|
+
r = ["ip=#{ip.shellescape}"]
|
|
19
|
+
|
|
20
|
+
info = @geo.lookup ip rescue nil
|
|
21
|
+
if !info
|
|
22
|
+
r << "error=#{@errors += 1}"
|
|
23
|
+
else
|
|
24
|
+
r += info.map do |k,v|
|
|
25
|
+
"#{k}=#{(v.is_a?(Array) ? v.join(',') : v.to_s).shellescape}"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
r.join "\n"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def json
|
|
32
|
+
require 'json'
|
|
33
|
+
@input.map do |list|
|
|
34
|
+
list.split.map do |ip|
|
|
35
|
+
r = {ip: ip}
|
|
36
|
+
info = @geo.lookup ip rescue nil
|
|
37
|
+
info ? r.merge(info) : r.merge({error: @errors += 1})
|
|
38
|
+
end
|
|
39
|
+
end.flatten.to_json
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
opt = {
|
|
45
|
+
db: geo_db_location,
|
|
46
|
+
fmt: 'json'
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
OptionParser.new do |o|
|
|
50
|
+
o.banner = "Usage: #{File.basename $0} [-d GeoLite2-City.mmdb] [-f fmt] ip..."
|
|
51
|
+
o.on("-d path", "maxmind db file") { |v| opt[:db] = v }
|
|
52
|
+
o.on("-f fmt", "output format: json, shell") { |v| opt[:fmt] = v }
|
|
53
|
+
end.parse!
|
|
54
|
+
|
|
55
|
+
begin
|
|
56
|
+
geo = MaxMindDB.new opt[:db]
|
|
57
|
+
rescue
|
|
58
|
+
errx 2, "failed to open #{opt[:db]}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
fmt = FMT.new (ARGV.size > 0 ? ARGV : $stdin), geo
|
|
62
|
+
puts fmt.send opt[:fmt]
|
|
63
|
+
exit fmt.errors > 0 ? 1 : 0
|
metadata
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: apache-log-geo
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.0.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Alexander Gromnitsky
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: "."
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2020-03-08 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: mmdb
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: 0.3.2
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: 0.3.2
|
|
27
|
+
description: |
|
|
28
|
+
It's like grep but with a knowledge about what data an ip
|
|
29
|
+
holds. Requires MaxMind's GeoLite2 DB (GeoLite2-City.mmdb) installed.
|
|
30
|
+
email: alexander.gromnitsky@gmail.com
|
|
31
|
+
executables:
|
|
32
|
+
- apache-log-geo
|
|
33
|
+
- mmdb-lookup
|
|
34
|
+
extensions: []
|
|
35
|
+
extra_rdoc_files: []
|
|
36
|
+
files:
|
|
37
|
+
- "./apache-log-geo"
|
|
38
|
+
- "./mmdb-lookup"
|
|
39
|
+
- README.md
|
|
40
|
+
- apache-log-geo
|
|
41
|
+
- lib.rb
|
|
42
|
+
- mmdb-lookup
|
|
43
|
+
homepage: https://github.com/gromnitsky/apache-log-geo
|
|
44
|
+
licenses:
|
|
45
|
+
- MIT
|
|
46
|
+
metadata: {}
|
|
47
|
+
post_install_message:
|
|
48
|
+
rdoc_options: []
|
|
49
|
+
require_paths:
|
|
50
|
+
- lib
|
|
51
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
52
|
+
requirements:
|
|
53
|
+
- - ">="
|
|
54
|
+
- !ruby/object:Gem::Version
|
|
55
|
+
version: 2.3.0
|
|
56
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - ">="
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '0'
|
|
61
|
+
requirements: []
|
|
62
|
+
rubygems_version: 3.1.2
|
|
63
|
+
signing_key:
|
|
64
|
+
specification_version: 4
|
|
65
|
+
summary: An offline GeoIP CLI filter for Apache (common, combined) logs.
|
|
66
|
+
test_files: []
|