sns_utils 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6a569b79f304d0060cd3d6c21898000777995042
+  data.tar.gz: b15db537d09f79751fdca09d94474a90e185fb76
+SHA512:
+  metadata.gz: a6f4bb613fd181471772a0a60886d5d09332724a3a1b29c014a0fea132b6273f68456a3233945fb26bffbe836737682e34592ed7784ce01fbae8ed35e4ab9da9
+  data.tar.gz: ca7c31915d5669691554b0d46cd6ae448ffef1dc528274a078dc03a8dd09c89d40b805d962fad3e52e1c3d327faf70da91ac968498255ffb40ff966b9ad82219

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+spec/fixtures/xlarge.log
+mac_addrs.txt
+ip_addrs.txt
+SOLUTION.txt
+.idea
+*.iml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sns_utils.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 sahglie
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,60 @@
+# SnsUtils
+
+Find and extract IP and MAC addresses in FILE based on repeated occurrences.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sns_utils'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sns_utils
+
+## Usage
+    $ gem install gem-man (if you haven't already)
+    $ gem man sns_utils
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+## Performance Notes
+
+### Grep Baseline Perf
+
+xlarge.log is 5GB with 50 million lines
+
+    time grep -c  -E '(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' spec/fixtures/xlarge.log
+    => real 12m0.658s
+       user 11m59.672s
+       sys  0m0.991s
+
+
+xlarge.log is 514MB with 5 million lines
+
+    time grep -c  -E '(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' spec/fixtures/xlarge.log
+    => real 1m12.063s
+       user 1m11.956s
+       sys  0m0.108s
+
+## ip_extract
+
+xlarge.log is 514MB with 5 million lines
+
+    ip_extract spec/fixtures/xlarge.log
+    => real 2m18.938s
+       user 2m18.356s
+       sys  0m0.578s
+
+

data/Rakefile

@@ -0,0 +1,34 @@
+require "bundler/gem_tasks"
+require 'ronn'
+
+
+namespace :man do
+  directory "man"
+
+  Dir["man/*.ronn"].each do |ronn|
+    basename = File.basename(ronn, ".ronn")
+    roff = "man/#{basename}"
+
+    file roff => ["man", ronn] do
+      sh "#{Gem.ruby} -S ronn --roff --pipe #{ronn} > #{roff}"
+    end
+
+    file "#{roff}.txt" => roff do
+      sh "groff -Wall -mtty-char -mandoc -Tascii #{roff} | col -b > #{roff}.txt"
+    end
+
+    task :build_all_pages => "#{roff}.txt"
+  end
+
+  desc "Build the man pages"
+  task :build => "man:build_all_pages"
+
+  desc "Clean up from the built man pages"
+  task :clean do
+    rm "man/ipex.1"
+    rm "man/ipex.1.txt"
+  end
+end
+
+task :build => ["man:clean", "man:build"]
+task :release => ["man:clean", "man:build"]

data/bin/ipex

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+# Exit cleanly from an early interrupt
+Signal.trap("INT") { exit 1 }
+
+require_relative "../lib/sns_utils"
+
+begin
+  SnsUtils::IpExtractor.new(ARGV).run
+rescue Errno::ENOENT => err
+  abort "ipex: #{err.message}"
+rescue OptionParser::InvalidOption, OptionParser::InvalidArgument => err
+  abort "usage: ipex FILE [-m MAC_THRESHOLD] [-i IP_THRESHOLD] [-d OUTPUT_DIR]"
+end

data/lib/sns_utils/ip_extractor.rb

@@ -0,0 +1,88 @@
+module SnsUtils
+  class IpExtractor
+    attr_accessor :file, :options
+    attr_accessor :ip_addrs, :ip_addrs_log, :mac_addrs, :mac_addrs_log
+
+    IPV4_REGEX = ::SnsUtils::IPv4::REGEX
+    IPV6_REGEX = ::SnsUtils::IPv6::REGEX
+    MAC_REGEX = ::SnsUtils::MAC::REGEX
+    IP_REGEX = / ((?: ^|\s) (?: #{IPV6_REGEX} | #{IPV4_REGEX} | #{MAC_REGEX}) (?: \s|$)) /xi
+
+    def initialize(argv)
+      @file, @options = parse_options(argv)
+      self.ip_addrs = {}
+      self.mac_addrs = {}
+    end
+
+    def run
+      extract_addresses
+      log_ip_addrs
+      log_mac_addrs
+      self
+    end
+
+    private
+
+    def extract_addresses
+      File.open(file, 'r').each do |line|
+        line.scan(IP_REGEX).each { |md| log_addr(md[0].to_s.strip!) }
+      end
+    end
+
+    def log_ip_addrs
+      self.ip_addrs_log  = ip_addrs.select { |_, count| count >= options.ip_threshold }.keys
+      write_file(::SnsUtils.ip_out_file, ip_addrs_log)
+    end
+
+    def log_mac_addrs
+      self.mac_addrs_log = mac_addrs.select { |_, count| count >= options.mac_threshold }.keys
+      write_file(::SnsUtils.mac_out_file, mac_addrs_log)
+    end
+
+    def write_file(file, entries)
+      file_path = output_dir(options.output_dir, file)
+      File.open(file_path, "w+") do |f|
+        entries.each { |e| f.puts(e) }
+      end
+    end
+
+    def output_dir(dir, file)
+      dir ? File.expand_path(File.join(dir, file)) : file
+    end
+
+    def log_addr(ip)
+      key = IPAddr.new(ip).to_string
+      ip_addrs[key] ||= 0
+      ip_addrs[key] += 1
+    rescue IPAddr::InvalidAddressError => e
+      mac_addrs[ip] ||= 0
+      mac_addrs[ip] += 1
+    end
+
+    def parse_options(argv)
+      options = OpenStruct.new
+      options.mac_threshold = 8
+      options.ip_threshold = 10
+      options.output_dir = nil
+
+      parser = OptionParser.new do |opts|
+        opts.on("-m N", Integer, "MAC address threshold, logs entries with N <= occurrences") do |n|
+          options.mac_threshold = Integer(n).abs
+        end
+
+        opts.on("-i N", Integer, "IP address threshold, logs entries with N =< occurrences") do |n|
+          options.ip_threshold = Integer(n).abs
+        end
+
+        opts.on("-d OUTPUT_DIR", String, "Dir used to write output files") do |dir|
+          options.output_dir = dir
+        end
+      end
+
+      parser.parse(argv)
+
+      file = argv[0] || raise(OptionParser::InvalidOption.new)
+      [file, options]
+    end
+  end
+end

data/lib/sns_utils/ipv4.rb

@@ -0,0 +1,12 @@
+module SnsUtils
+  # NOTE: the regexes are from Resolv::IPv4 in ruby's stdlib.  They were copied here because we
+  # need to match IPs in the context of a large string and the Resolv regex has string start
+  # and string end anchors which don't allow this.
+  module IPv4
+    REGEX_256 = /0
+               |1(?:[0-9][0-9]?)?
+               |2(?:[0-4][0-9]?|5[0-5]?|[6-9])?
+               |[3-9][0-9]?/x
+    REGEX = /#{REGEX_256}\.#{REGEX_256}\.#{REGEX_256}\.#{REGEX_256}/x
+  end
+end

data/lib/sns_utils/ipv6.rb

@@ -0,0 +1,44 @@
+module SnsUtils
+  # NOTE: the regexes are from Resolv::IPv4 in ruby's stdlib.  They were copied here because we
+  # need to match IPs in the context of a large string and the Resolv regex has string start
+  # and string end anchors which don't allow this.
+  module IPv6
+    ##
+    # IPv6 address format a:b:c:d:e:f:g:h
+    REGEX_8HEX = /
+      (?:[0-9A-Fa-f]{1,4}:){7}
+         [0-9A-Fa-f]{1,4}
+    /x
+
+    ##
+    # Compressed IPv6 address format a::b
+    REGEX_COMPRESSEDHEX = /
+      (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)? ::
+      (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?
+    /x
+
+    ##
+    # IPv4 mapped IPv6 address format a:b:c:d:e:f:w.x.y.z
+    REGEX_6HEX4DEC = /
+      (?:[0-9A-Fa-f]{1,4}:){6,6}
+      \d+\.\d+\.\d+\.\d+
+    /x
+
+    ##
+    # Compressed IPv4 mapped IPv6 address format a::b:w.x.y.z
+    REGEX_COMPRESSEDHEX4DEC = /
+      (?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)? ::
+      (?:[0-9A-Fa-f]{1,4}:)*
+      \d+\.\d+\.\d+\.\d+
+    /x
+
+    ##
+    # A composite IPv6 address Regexp.
+    REGEX = /
+      (?:#{REGEX_8HEX}) |
+      (?:#{REGEX_COMPRESSEDHEX}) |
+      (?:#{REGEX_6HEX4DEC}) |
+      (?:#{REGEX_COMPRESSEDHEX4DEC})
+    /x
+  end
+end

data/lib/sns_utils/mac.rb

@@ -0,0 +1,5 @@
+module SnsUtils
+  module MAC
+    REGEX = / (?: [0-9a-fA-F]{2}[:-]){5} [0-9a-fA-F]{2} /xi
+  end
+end

data/lib/sns_utils/version.rb

@@ -0,0 +1,3 @@
+module SnsUtils
+  VERSION = "0.0.1"
+end

data/lib/sns_utils.rb

@@ -0,0 +1,46 @@
+require 'pp'
+require "strscan"
+require 'ipaddr'
+require "optparse"
+require 'ostruct'
+
+require_relative 'sns_utils/ipv4'
+require_relative 'sns_utils/ipv6'
+require_relative 'sns_utils/mac'
+require_relative "sns_utils/ip_extractor"
+require_relative "sns_utils/version"
+
+
+module SnsUtils
+  def self.root
+    @root ||= File.expand_path(File.join(File.dirname(__FILE__), ".."))
+  end
+
+  def self.ip_out_file
+    "ip_addrs.txt"
+  end
+
+  def self.mac_out_file
+    "mac_addrs.txt"
+  end
+
+  def self.create_test_log(line, rep=1_000)
+    File.open("#{::SnsUtils.root}/spec/fixtures/xlarge.log", "w+") do |f|
+      rep.times do
+        f.puts("2013-08-20 16:05:06,975 [http-8082-3] INFO  -   Rendered text template (0.0ms)")
+        f.puts("2013-08-20 16:05:06,975 [http-8082-3] INFO  - Completed 200 OK in 5295ms (Views: 1.0ms | ActiveRecord: 0.0ms)")
+        f.puts(line)
+        f.puts("2013-08-20 16:08:39,281 [http-8082-3] INFO  - Redirected to https://apsearch-qa.berkeley.edu/login")
+        f.puts("169.229.216.83 - - [25/Aug/2013:07:02:29 -0700] \"GET / HTTP/1.0\" 302 303 \"-\" \"check_http/v2053 (nagios-plugins 1.4.13)")
+      end
+    end
+  end
+end
+
+
+
+if $PROGRAM_NAME == __FILE__
+  line = "9.9.9 blah blah blah VALID 0.0.0.0 xxxx<<> 2001:0000:1234:0000:0000:C1C0:ABCD:0876 VALID >>> blah blah blah VALID 00:A0:C9:14:C8:29"
+  ::SnsUtils.create_test_log(line, 1_000_000)
+end
+

data/man/ipex.1

@@ -0,0 +1,56 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "IPEX" "1" "August 2013" "" ""
+.
+.SH "NAME"
+\fBipex\fR \- IP/Mac Address Extractor
+.
+.SH "SYNOPSIS"
+\fBipex FILE [\-m THRESHOLD] [\-i THRESHOLD] [\-d OUTPUT_DIR]\fR
+.
+.SH "DESCRIPTION"
+Find and extract IP and MAC addresses in FILE\. Addresses with repeat occurrences greater than or equal to a given threshold are logged to ip_addrs\.txt and mac_addrs\.txt respectively\. By default, MAC addresses that occur at least 8 times are logged and IP addresses that occur at least 10 times are logged\. All matches are performed as case insensitive\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-m\fR
+Set the MAC address threshold, defaults to 8
+.
+.TP
+\fB\-i\fR
+Set the IP address threshold, defaults to 10
+.
+.TP
+\fB\-d\fR
+Specify directory where MAC and IP output files are written to, defaults to current directory
+.
+.SH "FORMATS"
+Matches the following address formats:
+.
+.IP "\(bu" 4
+IPv4: standard format a\.b\.c\.d
+.
+.IP "\(bu" 4
+IPv6: canonical format a:b:c:d:e:f:g:h
+.
+.IP "\(bu" 4
+IPv6: compressed format a::b
+.
+.IP "\(bu" 4
+IPv6: IPv4 mapped IPv6 format a:b:c:d:e:f:w\.x\.y\.z
+.
+.IP "\(bu" 4
+IPv6: compressed IPv4 mapped IPv6 format a::b:w\.x\.y\.z
+.
+.IP "\(bu" 4
+MAC: Separated by \'\-\' aa\-bb\-cc\-dd\-ee\-ff
+.
+.IP "\(bu" 4
+MAC: Separated by \':\' aa:bb:cc:dd:ee:ff
+.
+.IP "" 0
+.
+.SH "NOTE ON LARGE FILES"
+Depending on how large the file is, this utility can take a long time to complete: for a 512MB file with (5 million) entries it took just over 2 min \-\- YMMV depending on your hardware

data/man/ipex.1.ronn

@@ -0,0 +1,43 @@
+ipex(1) -- IP/Mac Address Extractor
+=======================================
+
+## SYNOPSIS
+
+`ipex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]`
+
+## DESCRIPTION
+
+Find and extract IP and MAC addresses in FILE.  Addresses with repeat occurrences
+greater than or equal to a given threshold are logged to ip_addrs.txt and mac_addrs.txt
+respectively.  By default, MAC addresses that occur at least 8 times are logged
+and IP addresses that occur at least 10 times are logged. All matches are performed
+as case insensitive.
+
+## OPTIONS
+
+* `-m`:
+  Set the MAC address threshold, defaults to 8
+
+* `-i`:
+  Set the IP address threshold, defaults to 10
+
+* `-d`:
+  Specify directory where MAC and IP output files are written to, defaults to
+  current directory
+
+## FORMATS
+Matches the following address formats:
+
+* IPv4: standard format a.b.c.d
+* IPv6: canonical format a:b:c:d:e:f:g:h
+* IPv6: compressed format a::b
+* IPv6: IPv4 mapped IPv6 format a:b:c:d:e:f:w.x.y.z
+* IPv6: compressed IPv4 mapped IPv6 format a::b:w.x.y.z
+* MAC: Separated by '-' aa-bb-cc-dd-ee-ff
+* MAC: Separated by ':' aa:bb:cc:dd:ee:ff
+
+## NOTE ON LARGE FILES
+Depending on how large the file is, this utility can take a long time to
+complete: for a 512MB file with (5 million) entries it took just over 2 min
+-- YMMV depending on your hardware
+

data/man/ipex.1.txt

@@ -0,0 +1,53 @@
+IPEX(1) 							       IPEX(1)
+
+
+
+NAME
+       ipex - IP/Mac Address Extractor
+
+SYNOPSIS
+       ipex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]
+
+DESCRIPTION
+       Find  and  extract  IP and MAC addresses in FILE. Addresses with repeat
+       occurrences greater than or equal to a given threshold  are  logged  to
+       ip_addrs.txt  and mac_addrs.txt respectively. By default, MAC addresses
+       that occur at least 8 times are logged and IP addresses that  occur  at
+       least  10  times are logged. All matches are performed as case insensi-
+       tive.
+
+OPTIONS
+       -m     Set the MAC address threshold, defaults to 8
+
+       -i     Set the IP address threshold, defaults to 10
+
+       -d     Specify directory where MAC and IP output files are written  to,
+	      defaults to current directory
+
+FORMATS
+       Matches the following address formats:
+
+       o   IPv4: standard format a.b.c.d
+
+       o   IPv6: canonical format a:b:c:d:e:f:g:h
+
+       o   IPv6: compressed format a::b
+
+       o   IPv6: IPv4 mapped IPv6 format a:b:c:d:e:f:w.x.y.z
+
+       o   IPv6: compressed IPv4 mapped IPv6 format a::b:w.x.y.z
+
+       o   MAC: Separated by '-' aa-bb-cc-dd-ee-ff
+
+       o   MAC: Separated by ':' aa:bb:cc:dd:ee:ff
+
+
+
+NOTE ON LARGE FILES
+       Depending  on  how large the file is, this utility can take a long time
+       to complete: for a 512MB file with (5 million)  entries	it  took  just
+       over 2 min -- YMMV depending on your hardware
+
+
+
+				  August 2013			       IPEX(1)

data/sns_utils.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sns_utils/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sns_utils"
+  spec.version       = SnsUtils::VERSION
+  spec.authors       = ["sahglie"]
+  spec.email         = ["sahglie@gmail.com"]
+  spec.description   = %q{Find and extract IP and MAC addresses in FILE}
+  spec.summary       = %q{Find and extract IP and MAC addresses in FILE}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.files      += Dir.glob('man/*')
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "ronn"
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rspec", "~> 2.7"
+  spec.add_development_dependency "rake"
+end

data/spec/fixtures/ipv4_ipv6_mac.log

@@ -0,0 +1,4 @@
+# blah blah blah VALID 0.0.0.0 xxxx<<> 2001:0000:1234:0000:0000:C1C0:ABCD:0876 VALID >>>
+# 3ffe:0b00:0000:0000:0001:0000:0000:000a VALID <<<<<<<<<< INVALID 1.1.1.
+# blah blah blah VALID 99.1.1.1 xxxx<<> VALID FF02:0000:0000:0000:0000:0000:0000:0001 >>>
+# blah blah blah INVALID a5::91::7 xxxx<<> VALID 00:A0:C9:14:C8:29 >>>

data/spec/fixtures/ipv4_simple.log

@@ -0,0 +1,12 @@
+#
+# valid 0.0.0.0
+# valid 1.1.1.1
+# valid 111.111.111.111
+# valid 111.111.111.1
+# valid 99.1.1.1
+
+# invalid 1.1.1
+# invalid 1
+# invalid 1.1
+# invalid 192.168.1.500
+# invalid 999.1.1.1

data/spec/fixtures/ipv6_simple.log

@@ -0,0 +1,172 @@
+# NOTE, some of the following IPs are duplicates, just in
+# a different format so the 81 valid ips map to 61 valid
+# ips.
+
+# valid 2001:0000:1234:0000:0000:C1C0:ABCD:0876
+# valid 3ffe:0b00:0000:0000:0001:0000:0000:000a
+# valid FF02:0000:0000:0000:0000:0000:0000:0001
+# valid 0000:0000:0000:0000:0000:0000:0000:0001
+# valid 0000:0000:0000:0000:0000:0000:0000:0000
+# valid ::ffff:192.168.1.26
+# valid 2::10
+# valid ff02::1
+# valid fe80::
+# valid 2002::
+# valid 2001:db8::
+# valid 2001:0db8:1234::
+# valid ::ffff:0:0
+# valid ::1
+# valid ::ffff:192.168.1.1
+# valid 1:2:3:4:5:6:7:8
+# valid 1:2:3:4:5:6::8
+# valid 1:2:3:4:5::8
+# valid 1:2:3:4::8
+# valid 1:2:3::8
+# valid 1:2::8
+# valid 1::8
+# valid 1::2:3:4:5:6:7
+# valid 1::2:3:4:5:6
+# valid 1::2:3:4:5
+# valid 1::2:3:4
+# valid 1::2:3
+# valid 1::8
+# valid ::2:3:4:5:6:7:8
+# valid ::2:3:4:5:6:7
+# valid ::2:3:4:5:6
+# valid ::2:3:4:5
+# valid ::2:3:4
+# valid ::2:3
+# valid ::8
+# valid 1:2:3:4:5:6::
+# valid 1:2:3:4:5::
+# valid 1:2:3:4::
+# valid 1:2:3::
+# valid 1:2::
+# valid 1::
+# valid 1:2:3:4:5::7:8
+# valid 1:2:3:4::7:8
+# valid 1:2:3::7:8
+# valid 1:2::7:8
+# valid 1::7:8
+# valid 1:2:3:4:5:6:1.2.3.4
+# valid 1:2:3:4:5::1.2.3.4
+# valid 1:2:3:4::1.2.3.4
+# valid 1:2:3::1.2.3.4
+# valid 1:2::1.2.3.4
+# valid 1::1.2.3.4
+# valid 1:2:3:4::5:1.2.3.4
+# valid 1:2:3::5:1.2.3.4
+# valid 1:2::5:1.2.3.4
+# valid 1::5:1.2.3.4
+# valid 1::5:11.22.33.44
+# valid  2001:0000:1234:0000:0000:C1C0:ABCD:0876
+# valid  2001:0000:1234:0000:0000:C1C0:ABCD:0876  
+# valid fe80::217:f2ff:254.7.237.98
+# valid fe80::217:f2ff:fe07:ed62
+# valid 2001:DB8:0:0:8:800:200C:417A # unicast, full
+# valid FF01:0:0:0:0:0:0:101 # multicast, full
+# valid 0:0:0:0:0:0:0:1 # loopback, full
+# valid 0:0:0:0:0:0:0:0 # unspecified, full
+# valid 2001:DB8::8:800:200C:417A # unicast, compressed
+# valid FF01::101 # multicast, compressed
+# valid ::1 # loopback, compressed, non-routable
+# valid :: # unspecified, compressed, non-routable
+# valid 0:0:0:0:0:0:13.1.68.3 # IPv4-compatible IPv6 address, full, deprecated
+# valid 0:0:0:0:0:FFFF:129.144.52.38 # IPv4-mapped IPv6 address, full
+# valid ::13.1.68.3 # IPv4-compatible IPv6 address, compressed, deprecated
+# valid ::FFFF:129.144.52.38 # IPv4-mapped IPv6 address, compressed
+# valid fe80:0000:0000:0000:0204:61ff:fe9d:f156
+# valid fe80:0:0:0:204:61ff:fe9d:f156
+# valid fe80::204:61ff:fe9d:f156
+# valid fe80:0:0:0:204:61ff:254.157.241.86
+# valid fe80::204:61ff:254.157.241.86
+# valid ::1
+# valid fe80::
+# valid fe80::1
+
+# not valid 02001:0000:1234:0000:0000:C1C0:ABCD:0876      # extra 0 not allowed!
+# not valid 2001:0000:1234:0000:00001:C1C0:ABCD:0876      # extra 0 not allowed!
+# not valid  2001:0000:1234:0000:0000:C1C0:ABCD:0876  0
+# not valid 2001:0000:1234: 0000:0000:C1C0:ABCD:0876
+# not valid 3ffe:0b00:0000:0001:0000:0000:000a
+# not valid FF02:0000:0000:0000:0000:0000:0000:0000:0001
+# not valid 3ffe:b00::1::a
+# not valid ::1111:2222:3333:4444:5555:6666::
+# not valid 1:2:3::4:5::7:8
+# not valid 12345::6:7:8
+# not valid 1::5:400.2.3.4
+# not valid 1::5:260.2.3.4
+# not valid 1::5:256.2.3.4
+# not valid 1::5:1.256.3.4
+# not valid 1::5:1.2.256.4
+# not valid 1::5:1.2.3.256
+# not valid 1::5:300.2.3.4
+# not valid 1::5:1.300.3.4
+# not valid 1::5:1.2.300.4
+# not valid 1::5:1.2.3.300
+# not valid 1::5:900.2.3.4
+# not valid 1::5:1.900.3.4
+# not valid 1::5:1.2.900.4
+# not valid 1::5:1.2.3.900
+# not valid 1::5:300.300.300.300
+# not valid 1::5:3000.30.30.30
+# not valid 1::400.2.3.4
+# not valid 1::260.2.3.4
+# not valid 1::256.2.3.4
+# not valid 1::1.256.3.4
+# not valid 1::1.2.256.4
+# not valid 1::1.2.3.256
+# not valid 1::300.2.3.4
+# not valid 1::1.300.3.4
+# not valid 1::1.2.300.4
+# not valid 1::1.2.3.300
+# not valid 1::900.2.3.4
+# not valid 1::1.900.3.4
+# not valid 1::1.2.900.4
+# not valid 1::1.2.3.900
+# not valid 1::300.300.300.300
+# not valid 1::3000.30.30.30
+# not valid ::400.2.3.4
+# not valid ::260.2.3.4
+# not valid ::256.2.3.4
+# not valid ::1.256.3.4
+# not valid ::1.2.256.4
+# not valid ::1.2.3.256
+# not valid ::300.2.3.4
+# not valid ::1.300.3.4
+# not valid ::1.2.300.4
+# not valid ::1.2.3.300
+# not valid ::900.2.3.4
+# not valid ::1.900.3.4
+# not valid ::1.2.900.4
+# not valid ::1.2.3.900
+# not valid ::300.300.300.300
+# not valid ::3000.30.30.30
+# not valid 2001:DB8:0:0:8:800:200C:417A:221 # unicast, full
+# not valid FF01::101::2 # multicast, compressed
+# not valid  # nothing
+# Not sure about this one; apparently some systems treat the leading "0" in ".086" as the start of an octal number
+# not valid fe80:0000:0000:0000:0204:61ff:254.157.241.086
+# not valid :
+# not valid 1111:2222:3333:4444::5555:
+# not valid 1111:2222:3333::5555:
+# not valid 1111:2222::5555:
+# not valid 1111::5555:
+# not valid ::5555:
+# not valid :::
+# not valid 1111:
+# not valid :
+# not valid :1111:2222:3333:4444::5555
+# not valid :1111:2222:3333::5555
+# not valid :1111:2222::5555
+# not valid :1111::5555
+# not valid :::5555
+# not valid :::
+# not valid 1.2.3.4:1111:2222:3333:4444::5555
+# not valid 1.2.3.4:1111:2222:3333::5555
+# not valid 1.2.3.4:1111:2222::5555
+# not valid 1.2.3.4:1111::5555
+# not valid 1.2.3.4::5555
+# not valid 1.2.3.4::
+# not valid 2001:1:1:1:1:1:255Z255X255Y255
+# not valid ::ffff:192x168.1.26

data/spec/fixtures/mac_simple.log

@@ -0,0 +1,6 @@
+# valid 00:A0:C9:14:C8:29
+# valid 00:00:00:00:00:00
+
+# invalid 00:00:00:00:00
+# invalid A0:C9:14:C8:29
+# invalid 00:A:C9:14:C8:29

data/spec/fixtures/thresholds.log

@@ -0,0 +1,35 @@
+# IPv4 occurs 10 times
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+# valid 1.1.1.1
+
+# MAC occurs 8 times
+# valid 00:A0:C9:14:C8:29
+# valid 00:A0:C9:14:C8:29
+# valid 00:A0:C9:14:C8:29
+# valid 00:A0:C9:14:C8:29
+# valid 00:A0:C9:14:C8:29
+# valid 00:A0:C9:14:C8:29
+# valid 00:A0:C9:14:C8:29
+# valid 00:A0:C9:14:C8:29
+
+# IPv4 occurs 5 times
+# 0.0.0.0
+# 0.0.0.0
+# 0.0.0.0
+# 0.0.0.0
+# 0.0.0.0
+
+# IPv6 occurs 5 times
+# valid 00:00:00:00:00:00
+# valid 00:00:00:00:00:00
+# valid 00:00:00:00:00:00
+# valid 00:00:00:00:00:00
+# valid 00:00:00:00:00:00

data/spec/sns_utils/ip_extractor_spec.rb

@@ -0,0 +1,103 @@
+require_relative "../spec_helper"
+
+describe SnsUtils::IpExtractor do
+  context "extracting IPv4 ips" do
+    let(:file) { fixture_path("ipv4_simple.log") }
+
+    it "finds valid entries" do
+      extractor = SnsUtils::IpExtractor.new([file]).run
+      extractor.ip_addrs.should have(5).entries
+    end
+  end
+
+  context "extracting IPv6 ips" do
+    let(:file) { fixture_path("ipv6_simple.log") }
+
+    it "finds valid entries" do
+      extractor = SnsUtils::IpExtractor.new([file]).run
+      extractor.ip_addrs.should have(61).entries
+    end
+  end
+
+  context "extracting MAC addresses" do
+    let(:file) { fixture_path("mac_simple.log") }
+
+    it "finds valid entries" do
+      extractor = SnsUtils::IpExtractor.new([file]).run
+      extractor.mac_addrs.should have(2).entries
+    end
+  end
+
+  context "extracting IPv4, IPv6, and MACs" do
+    let(:file) { fixture_path("ipv4_ipv6_mac.log") }
+
+    it "finds valid entries" do
+      extractor = SnsUtils::IpExtractor.new([file]).run
+      extractor.ip_addrs.should have(5).entries
+      extractor.mac_addrs.should have(1).entry
+    end
+  end
+
+  context "Occurrence threshold options" do
+    let(:file) { fixture_path("thresholds.log") }
+
+    it "raises error for invalid thresholds" do
+      expect { SnsUtils::IpExtractor.new([file, "-i", "x"]) }.to \
+          raise_error(OptionParser::InvalidArgument)
+
+      expect { SnsUtils::IpExtractor.new([file, "-m", "x"]) }.to \
+          raise_error(OptionParser::InvalidArgument)
+    end
+
+    it "has default IP and MAC threshold" do
+      extractor = SnsUtils::IpExtractor.new([file])
+      extractor.options.mac_threshold.should eql(8)
+      extractor.options.ip_threshold.should eql(10)
+    end
+
+    it "uses default IP and MAC threshold" do
+      extractor = SnsUtils::IpExtractor.new([file])
+      extractor.run
+
+      extractor.ip_addrs_log.should have(1).entry
+      extractor.mac_addrs_log.should have(1).entry
+    end
+
+    it "adjusts default IP and MAC threshold" do
+      extractor = SnsUtils::IpExtractor.new([file, "-i", "5", "-m", "4"])
+      extractor.options.mac_threshold.should eql(4)
+      extractor.options.ip_threshold.should eql(5)
+    end
+
+    it "uses adjusted IP and MAC threshold" do
+      extractor = SnsUtils::IpExtractor.new([file, "-i", "5", "-m", "5"]).run
+      extractor.ip_addrs_log.should have(2).entries
+      extractor.mac_addrs_log.should have(2).entries
+
+      extractor = SnsUtils::IpExtractor.new([file, "-i", "11", "-m", "11"]).run
+      extractor.ip_addrs_log.should be_empty
+      extractor.mac_addrs_log.should be_empty
+    end
+  end
+
+  context "Output dir option" do
+    let(:file) { fixture_path("ipv4_simple.log") }
+    let(:tmp_dir) { "#{::SnsUtils.root}/tmp" }
+
+    after(:all) do
+      File.delete("#{tmp_dir}/#{::SnsUtils.ip_out_file}")
+      File.delete("#{tmp_dir}/#{::SnsUtils.mac_out_file}")
+    end
+
+    it "configures custom output directory" do
+      options = SnsUtils::IpExtractor.new([file, "-d", tmp_dir]).options
+      options.output_dir.should eql(tmp_dir)
+    end
+
+    it "writes to custom output directory" do
+      SnsUtils::IpExtractor.new([file, "-d", tmp_dir]).run
+      File.exists?("#{tmp_dir}/#{::SnsUtils.ip_out_file}").should be_true
+      File.exists?("#{tmp_dir}/#{::SnsUtils.mac_out_file}").should be_true
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require_relative '../lib/sns_utils'
+
+RSpec.configure do |config|
+end
+
+def fixture_path(file)
+  File.expand_path(File.join(SnsUtils.root, "spec", "fixtures", file))
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: sns_utils
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- sahglie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ronn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Find and extract IP and MAC addresses in FILE
+email:
+- sahglie@gmail.com
+executables:
+- ipex
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/ipex
+- lib/sns_utils.rb
+- lib/sns_utils/ip_extractor.rb
+- lib/sns_utils/ipv4.rb
+- lib/sns_utils/ipv6.rb
+- lib/sns_utils/mac.rb
+- lib/sns_utils/version.rb
+- man/ipex.1
+- man/ipex.1.ronn
+- man/ipex.1.txt
+- sns_utils.gemspec
+- spec/fixtures/ipv4_ipv6_mac.log
+- spec/fixtures/ipv4_simple.log
+- spec/fixtures/ipv6_simple.log
+- spec/fixtures/mac_simple.log
+- spec/fixtures/thresholds.log
+- spec/sns_utils/ip_extractor_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Find and extract IP and MAC addresses in FILE
+test_files:
+- spec/fixtures/ipv4_ipv6_mac.log
+- spec/fixtures/ipv4_simple.log
+- spec/fixtures/ipv6_simple.log
+- spec/fixtures/mac_simple.log
+- spec/fixtures/thresholds.log
+- spec/sns_utils/ip_extractor_spec.rb
+- spec/spec_helper.rb
+has_rdoc: