logstash-codec-nmap 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d458ee42b5d24fefa55f73c60bd0fe7a00a420db
-  data.tar.gz: 30af8aa70c481d419e41dd86f1392fd5bae475c5
+  metadata.gz: 143adb46a1b2feaab1d6044904306db059bf9926
+  data.tar.gz: d8702c4591629eb9132e5240dce2ad2d5384f00d
 SHA512:
-  metadata.gz: a8a4cd414003eca34f0509ac006da1cc079cc4cd610334d99d31fb8506c82e4740c21ef5c41d8046f0ebc39c46ef57a9fb553d19dd2687109d1256ded115539d
-  data.tar.gz: deddff052858c75fccc3a94a3b121b05344e117983e9bfbc554d6f809b00dde62c0b539fd5446fff63c52eb353eeca0033db05dd4edd9cb9c88d5eabe5cf4995
+  metadata.gz: 6d8218521bb1492e7a218b1112b17f309a579f21379b04a3bcfc8a4eec678a9e46ecb18ece60c56e25d2fe6b3692850f7e5fe3a4cf2185e804d154d6087ae268
+  data.tar.gz: ebf1d0a5f9ba96cec647c10210be09ffff9ab665e54af0b0884f08930bec505c74356cd1c3b00473a74458632b3da2ed63eb68e028919776f74a85a118c421e1

data/lib/logstash/codecs/nmap.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/codecs/base"
 require "nmap/xml"
+require 'securerandom'
 
 # This codec may be used to decode (via inputs) only.
 # It decodes nmap generated XML and outputs each host as its own event
@@ -8,6 +9,13 @@ require "nmap/xml"
 class LogStash::Codecs::Nmap < LogStash::Codecs::Base
   config_name "nmap"
 
+  # Emit all host data as a nested document (including ports + traceroutes) with the type 'nmap_fullscan'
+  config :emit_hosts, :validate => :boolean, :default => true
+  # Emit each port as a separate document with type 'nmap_port'
+  config :emit_ports, :validate => :boolean, :default => true
+  # Emit each hop_tuple of the traceroute with type 'nmap_traceroute_link'
+  config :emit_traceroute_links, :validate => :boolean, :default => true
+
   public
   def register
   end
@@ -15,13 +23,57 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
   public
   def decode(data)
     xml = Nmap::XML.parse(data)
-    xml.each_host do |host|
-      event = host_to_event(host, xml)
+    scan_id = SecureRandom.uuid
+
+    xml.hosts.each_with_index do |host,idx|
+      # Convert the host to a 'base' host event
+      # This will be used for the later port/hop types
+      base = hashify_host(host, xml)
+
+      # Add some scanner-wide attributes
+      base['arguments'] = xml.scanner.arguments
+      base['version'] = xml.scanner.version
+      base['scan_id'] = scan_id
+
+      # Pull out the detail
+      ports = host.ports.map {|p| hashify_port(p)}
+      traceroute = hashify_traceroute(host.traceroute)
+
+      scan_host_id = scan_id + "-h#{idx}"
+
+      if @emit_ports && ports
+        ports.each.with_index do |port,idx|
+          yield LogStash::Event.new(base.merge(
+            'type' => 'nmap_port',
+            'port' => port,
+            'scan_host_id' => scan_host_id,
+            'id' => scan_host_id+"-p#{idx}"
+          ))
+        end
+      end
 
-      event['arguments'.freeze] = xml.scanner.arguments
-      event['version'.freeze] = xml.scanner.version
+      if @emit_traceroute_links && traceroute && (hops = traceroute['hops'])
+        hops.each_with_index do |hop,idx|
+          next_hop = hops[idx+1]
+          yield LogStash::Event.new(base.merge(
+            'type' =>'nmap_traceroute_link',
+            'from' => hop,
+            'to' => next_hop,
+            'rtt_diff' => (next_hop ? next_hop['rtt'] - hop['rtt'] : nil),
+            'scan_host_id' => scan_host_id,
+            'id' => scan_host_id+"-tl#{idx}"
+          ))
+        end
+      end
 
-      yield event
+      if @emit_hosts
+        yield LogStash::Event.new(base.merge(
+          'type' => 'nmap_host',
+          'ports' => ports,
+          'traceroute' => traceroute,
+          'id' => scan_host_id
+        ))
+      end
     end
   rescue StandardError => e
     @logger.warn("An unexpected error occurred parsing nmap XML",
@@ -31,38 +83,36 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
                  :backtrace => e.backtrace)
   end
 
-  def host_to_event(host, xml)
+  def hashify_host(host, xml)
     scan_start = timeify(xml.scanner.start_time)
 
-    event = LogStash::Event.new()
-    event['start_time'.freeze] = timeify(host.start_time, scan_start)
-    event['end_time'.freeze] = timeify(host.end_time, scan_start)
+    h = {}
+    h['start_time'] = timeify(host.start_time, scan_start)
+    h['end_time'] = timeify(host.end_time, scan_start)
 
     # These two are actually different.
     # Address may contain a MAC, addresses will not AFAICT
-    event['addresses'.freeze] = hashify_structs(host.addresses)
-    event['address'.freeze] = host.address # str
-
-    event['ip'.freeze] = host.ip # str
-    event['ipv4'.freeze] = host.ipv4 # str
-    event['ipv6'.freeze] = host.ipv6 # str
-    event['ports'.freeze] = host.ports.map {|p| hashify_port(p)}
-    event['mac'.freeze] = host.mac # str
-    event['status'.freeze] = hashify_status(host.status)
-    event['hostname'.freeze] = hashify_hostname(host.hostname)
-    event['uptime'.freeze] = hashify_uptime(host.uptime)
-    event['os'.freeze] = hashify_os(host.os)
-    event['traceroute'.freeze] = hashify_traceroute(host.traceroute)
-
-    event
+    h['addresses'] = hashify_structs(host.addresses)
+    h['address'] = host.address # str
+
+    h['ip'] = host.ip # str
+    h['ipv4'] = host.ipv4 # str
+    h['ipv6'] = host.ipv6 # str
+    h['mac'] = host.mac # str
+    h['status'] = hashify_status(host.status)
+    h['hostname'] = hashify_hostname(host.hostname)
+    h['uptime'] = hashify_uptime(host.uptime)
+    h['os'] = hashify_os(host.os)
+
+    h
   end
 
   def hashify_status(status)
     return unless status
 
     {
-      'state'.freeze => status.state.to_s, # str
-      'reason'.freeze => status.reason # str
+      'state' => status.state.to_s, # str
+      'reason' => status.reason # str
     }
   end
 
@@ -70,8 +120,8 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     return unless hostname
 
     {
-      'name'.freeze => hostname.name, # str
-      'type'.freeze => hostname.type, # str
+      'name' => hostname.name, # str
+      'type' => hostname.type, # str
     }
   end
 
@@ -79,10 +129,10 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     return unless os
 
     {
-      'ports_used'.freeze => os.ports_used,
-      'fingerprint'.freeze => os.fingerprint,
-      'classes'.freeze => hashify_os_matches(os.classes),
-      'matches'.freeze => hashify_structs(os_matches)
+      'ports_used' => os.ports_used,
+      'fingerprint' => os.fingerprint,
+      'classes' => hashify_os_matches(os.classes),
+      'matches' => hashify_structs(os_matches)
     }
   end
 
@@ -91,11 +141,11 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
 
     classes.each do |klass|
       {
-        'type'.freeze => klass.type.to_s, # returned as sym originally
-        'vendor'.freeze => klass.vendor.to_s,
-        'family'.freeze => klass.family.to_s,
-        'gen'.freeze => klass.gen.to_s,
-        'accuracy'.freeze => klass.accuracy # int
+        'type' => klass.type.to_s, # returned as sym originally
+        'vendor' => klass.vendor.to_s,
+        'family' => klass.family.to_s,
+        'gen' => klass.gen.to_s,
+        'accuracy' => klass.accuracy # int
       }
     end
   end
@@ -104,8 +154,8 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     return unless uptime
 
     {
-      'seconds'.freeze => uptime.seconds,
-      'last_boot'.freeze => timeify(uptime.last_boot)
+      'seconds' => uptime.seconds,
+      'last_boot' => timeify(uptime.last_boot)
     }
   end
 
@@ -113,15 +163,15 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     return unless service
 
     {
-      'name'.freeze => service.name,
-      'ssl'.freeze => service.ssl?,
-      'protocol'.freeze => service.protocol,
-      'product'.freeze => service.product,
-      'hostname'.freeze => service.hostname, # This is just a string
-      'device_type'.freeze => service.device_type,
-      'fingerprint_method'.freeze => service.fingerprint_method.to_s,
-      'fingerprint'.freeze => service.fingerprint,
-      'confidence'.freeze => service.confidence
+      'name' => service.name,
+      'ssl' => service.ssl?,
+      'protocol' => service.protocol,
+      'product' => service.product,
+      'hostname' => service.hostname, # This is just a string
+      'device_type' => service.device_type,
+      'fingerprint_method' => service.fingerprint_method.to_s,
+      'fingerprint' => service.fingerprint,
+      'confidence' => service.confidence
     }
   end
 
@@ -129,11 +179,11 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     return unless port
 
     {
-      'number'.freeze => port.number,
-      'reason'.freeze => port.reason,
-      'protocol'.freeze => port.protocol.to_s,
-      'service'.freeze => hashify_service(port.service),
-      'state'.freeze => port.state.to_s
+      'number' => port.number,
+      'reason' => port.reason,
+      'protocol' => port.protocol.to_s,
+      'service' => hashify_service(port.service),
+      'state' => port.state.to_s
     }
   end
 
@@ -141,13 +191,14 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     return unless traceroute
 
     {
-      'port'.freeze => traceroute.port, # int
-      'protocol'.freeze => traceroute.protocol.to_s,
+      'port' => traceroute.port, # int
+      'protocol' => traceroute.protocol.to_s,
       'hops' => traceroute.map.with_index do |hop, idx|
         {
-          'address'.freeze => hop.addr, # str
-          'hostname'.freeze => hop.host, # str
-          'ttl'.freeze => hop.ttl.to_i, # int
+          'address' => hop.addr, # str
+          'hostname' => hop.host, # str
+          'ttl' => hop.ttl.to_i, # int
+          'rtt' => hop.rtt.to_i, # int
           'index' => idx # int (for searching by distance)
         }
       end
@@ -166,7 +217,6 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     value.is_a?(Symbol) ? value.to_s : value
   end
 
-
   EPOCH = LogStash::Timestamp.new(Time.at(0))
   def timeify(time, default=nil)
     timestamp = time ? LogStash::Timestamp.new(time) : nil
@@ -178,4 +228,10 @@ class LogStash::Codecs::Nmap < LogStash::Codecs::Base
     end
   end
 
+  # Some strings have quoted values, we may want to remove leading/trailing quotes
+  def dequote(str)
+    return nil unless str
+    str.gsub(/\A"|"\Z/, '')
+  end
+
 end

data/logstash-codec-nmap.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-nmap'
-  s.version         = '0.0.5'
+  s.version         = '0.0.6'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This codec may be used to decode Nmap XML"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/scanme_traceroute.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE nmaprun>
+<?xml-stylesheet href="file:///usr/local/bin/../share/nmap/nmap.xsl" type="text/xsl"?>
+<!-- Nmap 7.01 scan initiated Sun Jan 17 11:19:16 2016 as: nmap -tr -oX - scanme.netmap.org -->
+<nmaprun scanner="nmap" args="nmap -tr -oX - scanme.netmap.org" start="1453051156" startstr="Sun Jan 17 11:19:16 2016" version="7.01" xmloutputversion="1.04">
+<scaninfo type="syn" protocol="tcp" numservices="1000" services="1,3-4,6-7,9,13,17,19-26,30,32-33,37,42-43,49,53,70,79-85,88-90,99-100,106,109-111,113,119,125,135,139,143-144,146,161,163,179,199,211-212,222,254-256,259,264,280,301,306,311,340,366,389,406-407,416-417,425,427,443-445,458,464-465,481,497,500,512-515,524,541,543-545,548,554-555,563,587,593,616-617,625,631,636,646,648,666-668,683,687,691,700,705,711,714,720,722,726,749,765,777,783,787,800-801,808,843,873,880,888,898,900-903,911-912,981,987,990,992-993,995,999-1002,1007,1009-1011,1021-1100,1102,1104-1108,1110-1114,1117,1119,1121-1124,1126,1130-1132,1137-1138,1141,1145,1147-1149,1151-1152,1154,1163-1166,1169,1174-1175,1183,1185-1187,1192,1198-1199,1201,1213,1216-1218,1233-1234,1236,1244,1247-1248,1259,1271-1272,1277,1287,1296,1300-1301,1309-1311,1322,1328,1334,1352,1417,1433-1434,1443,1455,1461,1494,1500-1501,1503,1521,1524,1533,1556,1580,1583,1594,1600,1641,1658,1666,1687-1688,1700,1717-1721,1723,1755,1761,1782-1783,1801,1805,1812,1839-1840,1862-1864,1875,1900,1914,1935,1947,1971-1972,1974,1984,1998-2010,2013,2020-2022,2030,2033-2035,2038,2040-2043,2045-2049,2065,2068,2099-2100,2103,2105-2107,2111,2119,2121,2126,2135,2144,2160-2161,2170,2179,2190-2191,2196,2200,2222,2251,2260,2288,2301,2323,2366,2381-2383,2393-2394,2399,2401,2492,2500,2522,2525,2557,2601-2602,2604-2605,2607-2608,2638,2701-2702,2710,2717-2718,2725,2800,2809,2811,2869,2875,2909-2910,2920,2967-2968,2998,3000-3001,3003,3005-3007,3011,3013,3017,3030-3031,3052,3071,3077,3128,3168,3211,3221,3260-3261,3268-3269,3283,3300-3301,3306,3322-3325,3333,3351,3367,3369-3372,3389-3390,3404,3476,3493,3517,3527,3546,3551,3580,3659,3689-3690,3703,3737,3766,3784,3800-3801,3809,3814,3826-3828,3851,3869,3871,3878,3880,3889,3905,3914,3918,3920,3945,3971,3986,3995,3998,4000-4006,4045,4111,4125-4126,4129,4224,4242,4279,4321,4343,4443-4446,4449,4550,4567,4662,4848,4899-4900,4998,5000-5004,5009,5030,5033,5050-5051,5054,5060-5061,5080,5087,5100-5102,5120,5190,5200,5214,5221-5222,5225-5226,5269,5280,5298,5357,5405,5414,5431-5432,5440,5500,5510,5544,5550,5555,5560,5566,5631,5633,5666,5678-5679,5718,5730,5800-5802,5810-5811,5815,5822,5825,5850,5859,5862,5877,5900-5904,5906-5907,5910-5911,5915,5922,5925,5950,5952,5959-5963,5987-5989,5998-6007,6009,6025,6059,6100-6101,6106,6112,6123,6129,6156,6346,6389,6502,6510,6543,6547,6565-6567,6580,6646,6666-6669,6689,6692,6699,6779,6788-6789,6792,6839,6881,6901,6969,7000-7002,7004,7007,7019,7025,7070,7100,7103,7106,7200-7201,7402,7435,7443,7496,7512,7625,7627,7676,7741,7777-7778,7800,7911,7920-7921,7937-7938,7999-8002,8007-8011,8021-8022,8031,8042,8045,8080-8090,8093,8099-8100,8180-8181,8192-8194,8200,8222,8254,8290-8292,8300,8333,8383,8400,8402,8443,8500,8600,8649,8651-8652,8654,8701,8800,8873,8888,8899,8994,9000-9003,9009-9011,9040,9050,9071,9080-9081,9090-9091,9099-9103,9110-9111,9200,9207,9220,9290,9415,9418,9485,9500,9502-9503,9535,9575,9593-9595,9618,9666,9876-9878,9898,9900,9917,9929,9943-9944,9968,9998-10004,10009-10010,10012,10024-10025,10082,10180,10215,10243,10566,10616-10617,10621,10626,10628-10629,10778,11110-11111,11967,12000,12174,12265,12345,13456,13722,13782-13783,14000,14238,14441-14442,15000,15002-15004,15660,15742,16000-16001,16012,16016,16018,16080,16113,16992-16993,17877,17988,18040,18101,18988,19101,19283,19315,19350,19780,19801,19842,20000,20005,20031,20221-20222,20828,21571,22939,23502,24444,24800,25734-25735,26214,27000,27352-27353,27355-27356,27715,28201,30000,30718,30951,31038,31337,32768-32785,33354,33899,34571-34573,35500,38292,40193,40911,41511,42510,44176,44442-44443,44501,45100,48080,49152-49161,49163,49165,49167,49175-49176,49400,49999-50003,50006,50300,50389,50500,50636,50800,51103,51493,52673,52822,52848,52869,54045,54328,55055-55056,55555,55600,56737-56738,57294,57797,58080,60020,60443,61532,61900,62078,63331,64623,64680,65000,65129,65389"/>
+<verbose level="0"/>
+<debugging level="0"/>
+<taskprogress task="SYN Stealth Scan" time="1453051160" percent="74.30" remaining="2" etc="1453051161"/>
+<taskprogress task="SYN Stealth Scan" time="1453051161" percent="85.85" remaining="1" etc="1453051162"/>
+<taskprogress task="SYN Stealth Scan" time="1453051161" percent="97.60" remaining="1" etc="1453051161"/>
+<taskprogress task="Traceroute" time="1453051163" percent="70.97" remaining="1" etc="1453051164"/>
+<host starttime="1453051156" endtime="1453051161"><status state="up" reason="echo-reply" reason_ttl="49"/>
+<address addr="207.148.248.143" addrtype="ipv4"/>
+<hostnames>
+<hostname name="scanme.netmap.org" type="user"/>
+</hostnames>
+<ports><extraports state="filtered" count="997">
+<extrareasons reason="no-responses" count="997"/>
+</extraports>
+<port protocol="tcp" portid="43"><state state="closed" reason="reset" reason_ttl="49"/><service name="whois" method="table" conf="3"/></port>
+<port protocol="tcp" portid="80"><state state="open" reason="syn-ack" reason_ttl="49"/><service name="http" method="table" conf="3"/></port>
+<port protocol="tcp" portid="443"><state state="closed" reason="reset" reason_ttl="49"/><service name="https" method="table" conf="3"/></port>
+</ports>
+<trace port="443" proto="tcp">
+<hop ttl="1" ipaddr="192.168.1.1" rtt="2.69" host="router.asus.com"/>
+<hop ttl="2" ipaddr="96.120.48.165" rtt="15.63"/>
+<hop ttl="3" ipaddr="68.85.165.253" rtt="26.58" host="te-0-3-0-20-sur01.swmpls.mn.minn.comcast.net"/>
+<hop ttl="4" ipaddr="69.139.219.214" rtt="26.59" host="te-0-0-0-0-sur02.swmpls.mn.minn.comcast.net"/>
+<hop ttl="5" ipaddr="69.139.219.225" rtt="26.61" host="te-0-8-0-11-ar01.roseville.mn.minn.comcast.net"/>
+<hop ttl="6" ipaddr="68.86.94.81" rtt="26.62" host="be-13367-cr02.350ecermak.il.ibone.comcast.net"/>
+<hop ttl="7" ipaddr="68.86.83.50" rtt="26.62" host="hu-0-10-0-0-pe04.350ecermak.il.ibone.comcast.net"/>
+<hop ttl="8" ipaddr="23.30.206.134" rtt="26.64"/>
+<hop ttl="9" ipaddr="154.54.46.177" rtt="26.65" host="be2766.ccr42.ord01.atlas.cogentco.com"/>
+<hop ttl="10" ipaddr="154.54.7.130" rtt="28.16" host="be2718.ccr22.cle04.atlas.cogentco.com"/>
+<hop ttl="11" ipaddr="154.54.43.185" rtt="47.97" host="be2189.ccr22.alb02.atlas.cogentco.com"/>
+<hop ttl="12" ipaddr="154.54.43.13" rtt="56.51" host="be2302.ccr22.bos01.atlas.cogentco.com"/>
+<hop ttl="13" ipaddr="154.54.47.254" rtt="51.16" host="te0-4-1-7.agr21.bos01.atlas.cogentco.com"/>
+<hop ttl="14" ipaddr="38.104.186.138" rtt="50.43"/>
+<hop ttl="16" ipaddr="207.148.248.143" rtt="50.38"/>
+</trace>
+<times srtt="50527" rttvar="6451" to="100000"/>
+</host>
+<runstats><finished time="1453051164" timestr="Sun Jan 17 11:19:24 2016" elapsed="8.53" summary="Nmap done at Sun Jan 17 11:19:24 2016; 1 IP address (1 host up) scanned in 8.53 seconds" exit="success"/><hosts up="1" down="0" total="1"/>
+</runstats>
+</nmaprun>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-nmap
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Elastic
@@ -76,6 +76,7 @@ files:
 - spec/fixtures/ipv6_all.xml
 - spec/fixtures/localscan.xml
 - spec/fixtures/pingsweep.xml
+- spec/fixtures/scanme_traceroute.xml
 - spec/fixtures/traceroutes.xml
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -108,4 +109,5 @@ test_files:
 - spec/fixtures/ipv6_all.xml
 - spec/fixtures/localscan.xml
 - spec/fixtures/pingsweep.xml
+- spec/fixtures/scanme_traceroute.xml
 - spec/fixtures/traceroutes.xml