apache-log-geo 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88358db0f03c32a5141201af91a3fa617f123b00c0fad7a8e063a4e51a1fe75e
-  data.tar.gz: 37906a57323328d07d516d3b7b6bb5cc71ffb200949158206f952708e1df0cb0
+  metadata.gz: d5f696d02b3eea816384b389250916a180e3b219ad52ecc01ad8fcc42ce664de
+  data.tar.gz: 3b2c584d7cde234fd15918eca2826d4d674806d43bf8ea244b22022916b55dab
 SHA512:
-  metadata.gz: edba9d9da0d8e342b84403df8d6459f449aac6c65cb5d69b59f696a69562aea26c4a3a7febae2928d2852721301bb0b9564d38a38fa21fca6c1fe062e5c39234
-  data.tar.gz: dde46e90f32f32fc69b7292e12db7687f2319d3b6a308f088996f9fab24b91a30d4c355a75e38448e3eeff5fd7b5c35954a3acf7423b34230b55c3ced543698b
+  metadata.gz: 57bb8e647db969849aa7e341def0459ec28c0ccc7806e6cca6a5d6c63beb976bdd097621f6d4f66a8a65b7b780ab5a76653cf190fd83ceb99b3a1be2f96e81e5
+  data.tar.gz: d3341602fb3df5c81af39f1dfcbdbf83576472bdc4bcc1208538db93497e6153f18c2afe50b73787524aa524d428e0a3c2e1310dc1b248e28b48fed5fd3109ff

data/README.md

@@ -17,6 +17,12 @@ key & install geoipupdate to fetch the db file.
 
     gem install apache-log-geo
 
+It uses a C extension [geoip2_c][], instead of the official maxmind-db
+gem, for in my local tests, the latter (in conjunction with the Ruby
+IO) is 9 (nine) times slower!
+
+[geoip2_c]: https://github.com/fluent-plugins-nursery/geoip2_c
+
 ## Usage
 
 The pkg contains 2 CLI utils only. There's no reusable library code.
@@ -33,6 +39,7 @@ Usage: apache-log-geo [-d GeoLite2-City.mmdb] [-v] [--key val ...]
         --city regexp
         --country regexp
         --cc str                     2 letter country code
+        --eu                         is an EU member?
         --continent regexp
         --postcode regexp
         --sub regexp                 subdivisions
@@ -96,7 +103,7 @@ Kyiv City
 Replicate `apache-log-geo` util--print only the requests from the Irish
 (the example requires `npm -g json`):
 
-    $ awk '{print $1}' test/access.log | ./mmdb-lookup | json -c 'this.country_code == "IE"' -a ip | grep -h -f - test/access.log
+    $ cat test/access.log | ./mmdb-lookup | json -c 'this.country_code == "IE"' -a ip | grep -h -f - test/access.log
 
 ## Exit status
 

data/apache-log-geo

@@ -1,16 +1,16 @@
 #!/usr/bin/env ruby
 
 require 'optparse'
-require 'mmdb'
 require_relative 'lib'
 include ApacheLogGeo
 
-# exact match: country_code, postcode
-# regexp: continent, country, subdivisions, city
-#
 # conditions are anded. no conditions == true
 def match_by_geo info, query
   query.each do |k,v|
+    if k == :eu                 # --eu is a boolean
+      return false unless info[k]
+    end
+
     if k == :subdivisions # it's an array, one match of its elements is enough
       return false unless (info[k] || []).any? {|val| val.to_s =~ /#{v}/i }
     end
@@ -39,25 +39,25 @@ OptionParser.new do |o|
     end
     opt[:query][:country_code] = v
   }
+  o.on("--eu", "is an EU member?") { opt[:query][:eu] = true }
   o.on("--continent regexp") { |v| opt[:query][:continent] = v }
   o.on("--postcode regexp") { |v| opt[:query][:postcode] = v }
   o.on("--sub regexp", "subdivisions") { |v| opt[:query][:subdivisions] = v }
 end.parse!
 
 begin
-  geo = MaxMindDB.new opt[:db]
+  geo = GeoDB.new opt[:db]
 rescue
-  errx 2, "failed to open #{opt[:db]}"
+  errx 2, $!
 end
 
 found_anything = false
 $stdin.each_line do |line|
   next if line =~ /^\s*$/
   ip = line.split[0]
-  begin
-    info = geo.lookup ip
-  rescue
-    warnx "no data, filtering the entry out: `${ip[0..15]}`"
+  info = geo.get ip
+  unless info
+    warnx "line #{$stdin.lineno}: no data, filtering out: #{ip[0..15]}"
     next
   end
 

data/lib.rb

@@ -273,3 +273,28 @@ module ApacheLogGeo
     "ZW" => "Zimbabwe"
   }
 end
+
+require 'geoip2'
+
+class ApacheLogGeo::GeoDB
+  def initialize path
+    @db = GeoIP2::Database.new path
+  end
+
+  def get ip
+    r = @db.lookup ip rescue nil
+    return nil unless r
+
+    {
+      city:         r.dig('city', 'names', 'en'),
+      continent:    r.dig('continent', 'names', 'en'),
+      country:      r.dig('country', 'names', 'en'),
+      country_code: r.dig('country', 'iso_code'),
+      eu:           r.dig('country', 'is_in_european_union'),
+      latitude:     r.dig('location', 'latitude'),
+      longitude:    r.dig('location', 'longitude'),
+      postcode:     r.dig('postal', 'code'),
+      subdivisions: r.dig('subdivisions')&.map {|v| v&.dig('names', 'en') }&.compact
+    }
+  end
+end

data/mmdb-lookup

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'optparse'
-require 'mmdb'
 require_relative 'lib'
 include ApacheLogGeo
 
@@ -17,7 +16,7 @@ class FMT
     ip = @input.to_enum.next.split.first
     r = ["ip=#{ip.shellescape}"]
 
-    info = @geo.lookup ip rescue nil
+    info = @geo.get ip
     if !info
       r << "error=#{@errors += 1}"
     else
@@ -30,13 +29,19 @@ class FMT
 
   def json
     require 'json'
-    @input.map do |list|
-      list.split.map do |ip|
-        r = {ip: ip}
-        info = @geo.lookup ip rescue nil
-        info ? r.merge(info) : r.merge({error: @errors += 1})
-      end
-     end.flatten.to_json
+    trans = -> (ip) do
+      r = {ip: ip}
+      info = @geo.get ip
+      info ? r.merge(info) : r.merge({error: @errors += 1})
+    end
+
+    if @input.is_a?(IO)      # stdin, in which we expect an apache log
+      @input.map { |line| trans.call line.split[0] }.to_json
+    else                     # argv
+      @input.map do |list|
+        list.split.map { |ip| trans.call ip }
+      end.flatten.to_json
+    end
   end
 end
 
@@ -53,9 +58,9 @@ OptionParser.new do |o|
 end.parse!
 
 begin
-  geo = MaxMindDB.new opt[:db]
+  geo = GeoDB.new opt[:db]
 rescue
-  errx 2, "failed to open #{opt[:db]}"
+  errx 2, $!
 end
 
 fmt = FMT.new (ARGV.size > 0 ? ARGV : $stdin), geo

metadata

@@ -1,30 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: apache-log-geo
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Alexander Gromnitsky
 autorequire: 
 bindir: "."
 cert_chain: []
-date: 2020-03-08 00:00:00.000000000 Z
+date: 2020-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: mmdb
+  name: geoip2_c
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.3
 description: |
+  An offline GeoIP CLI filter for Apache (common, combined) logs.
   It's like grep but with a knowledge about what data an ip
   holds. Requires MaxMind's GeoLite2 DB (GeoLite2-City.mmdb) installed.
 email: alexander.gromnitsky@gmail.com