sns_utils 0.0.3 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 620632d0cf549c2f51495c8d91a64c89d6dfeb0f
-  data.tar.gz: 674efcd5281dc6202312988c95c74142978a00c0
+  metadata.gz: 912c980791a92d1b4e63a686bcc44da8328f3d64
+  data.tar.gz: 2be2ca81465a58d402112520482760622ce07413
 SHA512:
-  metadata.gz: 8b5f6de14432d3ec5e97f753eee6cdf4f27e7b29fc89ac5fca1bef70018a76f875da696542bc263a5f2fb77f06a2e1c675c533e62ba8541fb68afcebbdaba366
-  data.tar.gz: bb79f4bc4ede80e832832acdf40447fdbc7c9fb28b8625cf268ce0b347dd87719fd70b5fa14a771cfb8f337cebc8e8420cf4ee8059b05f314fe67eb45c0c93ab
+  metadata.gz: 13a6635b712fc95e9f15b7b723aa13c53b8033afd31032a297976bc35e270c45167862f969fcb243e5406c4b898b355083ddea67691a7c18ad0b954b5098cfc8
+  data.tar.gz: 91238caf75f3a901c9db6fb79b6324f01233991937dd08571f128f6ea15e77f89b8a62ccfa7a63f1e29e372c71c778537878a9e26e239b2c5bbc3c5be1ba81b4

data/README.md

@@ -33,28 +33,28 @@ Or install it yourself as:
 
 ### Grep Baseline Perf
 
+The following grep commands try to match only IPv6 and IPv4 addresses using the
+same regex as the addrex utility
+
 xlarge.log is 5GB with 50 million lines
 
-    time grep -c  -E '(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' spec/fixtures/xlarge.log
-    => real 12m0.658s
-       user 11m59.672s
-       sys  0m0.991s
+    time grep -c  -E '\b(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?|(?:[0-9A-Fa-f]{1,4}:){6,6}\d+\.\d+\.\d+\.\d+|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}:)*\d+\.\d+\.\d+\.\d+)\b' spec/fixtures/xlarge.log
+    => Exceeded 15 min
 
 
 xlarge.log is 514MB with 5 million lines
 
-    time grep -c  -E '(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' spec/fixtures/xlarge.log
-    => real 1m12.063s
-       user 1m11.956s
+    time grep -c  -E '\b(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?|(?:[0-9A-Fa-f]{1,4}:){6,6}\d+\.\d+\.\d+\.\d+|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}:)*\d+\.\d+\.\d+\.\d+)\b' spec/fixtures/xlarge.log
+    => real 4m12.063s
+       user 4m10.956s
        sys  0m0.108s
 
 ## ip_extract
 
 xlarge.log is 514MB with 5 million lines
 
-    ip_extract spec/fixtures/xlarge.log
-    => real 2m18.938s
-       user 2m18.356s
+    addrex spec/fixtures/xlarge.log
+    => real 3m7.938s
+       user 3m6.356s
        sys  0m0.578s
 
-

data/Rakefile

@@ -25,8 +25,8 @@ namespace :man do
 
   desc "Clean up from the built man pages"
   task :clean do
-    rm "man/ipex.1"
-    rm "man/ipex.1.txt"
+    rm "man/addrex.1"
+    rm "man/addrex.1.txt"
   end
 end
 

data/bin/{ipex → addrex}

@@ -6,9 +6,9 @@ Signal.trap("INT") { exit 1 }
 require_relative "../lib/sns_utils"
 
 begin
-  SnsUtils::IpExtractor.new(ARGV).run
+  SnsUtils::AddressExtractor.new(ARGV).run
 rescue Errno::ENOENT => err
-  abort "ipex: #{err.message}"
-rescue OptionParser::InvalidOption, OptionParser::InvalidArgument => err
-  abort "usage: ipex FILE [-m MAC_THRESHOLD] [-i IP_THRESHOLD] [-d OUTPUT_DIR]"
+  abort "addrex: #{err.message}"
+rescue OptionParser::InvalidOption, OptionParser::InvalidArgument
+  abort "usage: addrex FILE [-m MAC_THRESHOLD] [-i IP_THRESHOLD] [-d OUTPUT_DIR]"
 end

data/lib/sns_utils.rb

@@ -7,7 +7,7 @@ require 'ostruct'
 require_relative 'sns_utils/ipv4'
 require_relative 'sns_utils/ipv6'
 require_relative 'sns_utils/mac'
-require_relative "sns_utils/ip_extractor"
+require_relative "sns_utils/address_extractor"
 require_relative "sns_utils/version"
 
 

data/lib/sns_utils/{ip_extractor.rb → address_extractor.rb}

@@ -1,12 +1,16 @@
 module SnsUtils
-  class IpExtractor
+  class AddressExtractor
     attr_accessor :file, :options
     attr_accessor :ip_addrs, :ip_addrs_log, :mac_addrs, :mac_addrs_log
 
     IP_REGEX = /
-        #{::SnsUtils::IPv4::REGEX} |
-        #{::SnsUtils::IPv6::REGEX} |
-        #{::SnsUtils::MAC::REGEX}
+        \b
+        (
+          #{::SnsUtils::IPv4::REGEX} |
+          #{::SnsUtils::IPv6::REGEX} |
+          #{::SnsUtils::MAC::REGEX}
+        )
+        \b
     /xi
 
     def initialize(argv)
@@ -26,8 +30,8 @@ module SnsUtils
 
     def extract_addresses
       File.open(file, 'r').each do |line|
-        line.scan(IP_REGEX).each do |ip|
-          log_addr(ip)
+        line.scan(IP_REGEX).each do |md|
+          log_addr(md[0].to_s.strip)
         end
       end
     end

data/lib/sns_utils/version.rb

@@ -1,3 +1,3 @@
 module SnsUtils
-  VERSION = "0.0.3"
+  VERSION = "0.0.6"
 end

data/man/{ipex.1 → addrex.1}

@@ -1,13 +1,13 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "IPEX" "1" "August 2013" "" ""
+.TH "ADDREX" "1" "August 2013" "" ""
 .
 .SH "NAME"
-\fBipex\fR \- IP/Mac Address Extractor
+\fBaddrex\fR \- IP/Mac Address Extractor
 .
 .SH "SYNOPSIS"
-\fBipex FILE [\-m THRESHOLD] [\-i THRESHOLD] [\-d OUTPUT_DIR]\fR
+\fBaddrex FILE [\-m THRESHOLD] [\-i THRESHOLD] [\-d OUTPUT_DIR]\fR
 .
 .SH "DESCRIPTION"
 Find and extract IP and MAC addresses in FILE\. Addresses with repeat occurrences greater than or equal to a given threshold are logged to ip_addrs\.txt and mac_addrs\.txt respectively\. By default, MAC addresses that occur at least 8 times are logged and IP addresses that occur at least 10 times are logged\. All matches are performed as case insensitive\.
@@ -53,4 +53,4 @@ MAC: Separated by \':\' aa:bb:cc:dd:ee:ff
 .IP "" 0
 .
 .SH "NOTE ON LARGE FILES"
-Depending on how large the file is, this utility can take a long time to complete: for a 512MB file with (5 million) entries it took just over 2 min \-\- YMMV depending on your hardware
+Depending on how large the file is, this utility can take a long time to complete: for a 512MB file with (5 million) entries it took just over 3 min \-\- YMMV depending on your hardware

data/man/{ipex.1.ronn → addrex.1.ronn}

@@ -1,9 +1,9 @@
-ipex(1) -- IP/Mac Address Extractor
+addrex(1) -- IP/Mac Address Extractor
 =======================================
 
 ## SYNOPSIS
 
-`ipex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]`
+`addrex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]`
 
 ## DESCRIPTION
 
@@ -38,6 +38,6 @@ Matches the following address formats:
 
 ## NOTE ON LARGE FILES
 Depending on how large the file is, this utility can take a long time to
-complete: for a 512MB file with (5 million) entries it took just over 2 min
+complete: for a 512MB file with (5 million) entries it took just over 3 min
 -- YMMV depending on your hardware
 

data/man/{ipex.1.txt → addrex.1.txt}

@@ -1,12 +1,12 @@
-IPEX(1) 							       IPEX(1)
+ADDREX(1)							     ADDREX(1)
 
 
 
 NAME
-       ipex - IP/Mac Address Extractor
+       addrex - IP/Mac Address Extractor
 
 SYNOPSIS
-       ipex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]
+       addrex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]
 
 DESCRIPTION
        Find  and  extract  IP and MAC addresses in FILE. Addresses with repeat
@@ -46,8 +46,8 @@ FORMATS
 NOTE ON LARGE FILES
        Depending  on  how large the file is, this utility can take a long time
        to complete: for a 512MB file with (5 million)  entries	it  took  just
-       over 2 min -- YMMV depending on your hardware
+       over 3 min -- YMMV depending on your hardware
 
 
 
-				  August 2013			       IPEX(1)
+				  August 2013			     ADDREX(1)

data/sns_utils.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "ronn"
+  spec.add_development_dependency "ronn"
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rspec", "~> 2.7"
   spec.add_development_dependency "rake"

data/spec/fixtures/edge_cases.log

@@ -1,8 +1,9 @@
-# should match one ip 123.12.12.126-16-2013
-# should match one ip 6-7-201123.12.12.127
-# should match one ip 6-7-201123.12.12.128-16-2013
-# invalid but should match by truncating last '0' 192.168.1.500
-# invalid but should match by truncating last '5' 192.168.1.555:80
-# ipv4 with port number 192.168.1.51:8080
-# ipv8 with port number # valid 2001:0000:1234:0000:0000:C1C0:ABCD:0876:8080
-# multi-line xyz:1 ip 123.12.12.129-16-2013 2013-1xxxy&& 2001:0000:1234:0000:0000:C1C0:ABCD:0877:8080 9%%%#+ 192.168.1.565:80
+123.12.12.125-16-2013 # will match
+# will match 123.12.12.126-16-2013
+# will not match 6-7-201123.12.12.127
+# will not match 6-7-201123.12.12.128-16-2013
+# will not match by truncating last '0' 192.168.1.500
+# will not match by truncating last '5' 192.168.1.555:80
+# will match, ipv4 with port number 192.168.1.51:8080
+# will match, ipv8 with port number # valid 2001:0000:1234:0000:0000:C1C0:ABCD:0876:8080
+# will match all, multi-line xyz:1 ip 123.12.12.129-16-2013 2013-1xxxy&& 2001:0000:1234:0000:0000:C1C0:ABCD:0877:8080 9%%%#+ 192.168.1.52:80

data/spec/sns_utils/{ip_extractor_spec.rb → address_extractor_spec.rb}

@@ -1,12 +1,12 @@
 require_relative "../spec_helper"
 
-describe SnsUtils::IpExtractor do
+describe SnsUtils::AddressExtractor do
   context "extracting IPv4 ips" do
     let(:file) { fixture_path("ipv4_simple.log") }
 
     it "finds valid entries" do
-      ipex = SnsUtils::IpExtractor.new([file]).run
-      ipex.ip_addrs.should have(5).entries
+      addrex = SnsUtils::AddressExtractor.new([file]).run
+      addrex.ip_addrs.should have(5).entries
     end
   end
 
@@ -14,8 +14,8 @@ describe SnsUtils::IpExtractor do
     let(:file) { fixture_path("ipv6_simple.log") }
 
     it "finds valid entries" do
-      ipex = SnsUtils::IpExtractor.new([file]).run
-      ipex.ip_addrs.should have(96).entries
+      addrex = SnsUtils::AddressExtractor.new([file]).run
+      addrex.ip_addrs.should have(72).entries
     end
   end
 
@@ -23,8 +23,8 @@ describe SnsUtils::IpExtractor do
     let(:file) { fixture_path("mac_simple.log") }
 
     it "finds valid entries" do
-      ipex = SnsUtils::IpExtractor.new([file]).run
-      ipex.mac_addrs.should have(2).entries
+      addrex = SnsUtils::AddressExtractor.new([file]).run
+      addrex.mac_addrs.should have(2).entries
     end
   end
 
@@ -32,10 +32,10 @@ describe SnsUtils::IpExtractor do
     let(:file) { fixture_path("ipv4_ipv6_mac.log") }
 
     it "finds valid entries" do
-      ipex = SnsUtils::IpExtractor.new([file]).run
-      pp ipex.ip_addrs
-      ipex.ip_addrs.should have(5).entries
-      ipex.mac_addrs.should have(1).entry
+      addrex = SnsUtils::AddressExtractor.new([file]).run
+      pp addrex.ip_addrs
+      addrex.ip_addrs.should have(5).entries
+      addrex.mac_addrs.should have(1).entry
     end
   end
 
@@ -43,41 +43,41 @@ describe SnsUtils::IpExtractor do
     let(:file) { fixture_path("thresholds.log") }
 
     it "raises error for invalid thresholds" do
-      expect { SnsUtils::IpExtractor.new([file, "-i", "x"]) }.to \
+      expect { SnsUtils::AddressExtractor.new([file, "-i", "x"]) }.to \
           raise_error(OptionParser::InvalidArgument)
 
-      expect { SnsUtils::IpExtractor.new([file, "-m", "x"]) }.to \
+      expect { SnsUtils::AddressExtractor.new([file, "-m", "x"]) }.to \
           raise_error(OptionParser::InvalidArgument)
     end
 
     it "has default IP and MAC threshold" do
-      ipex = SnsUtils::IpExtractor.new([file])
-      ipex.options.mac_threshold.should eql(8)
-      ipex.options.ip_threshold.should eql(10)
+      addrex = SnsUtils::AddressExtractor.new([file])
+      addrex.options.mac_threshold.should eql(8)
+      addrex.options.ip_threshold.should eql(10)
     end
 
     it "uses default IP and MAC threshold" do
-      ipex = SnsUtils::IpExtractor.new([file])
-      ipex.run
+      addrex = SnsUtils::AddressExtractor.new([file])
+      addrex.run
 
-      ipex.ip_addrs_log.should have(1).entry
-      ipex.mac_addrs_log.should have(1).entry
+      addrex.ip_addrs_log.should have(1).entry
+      addrex.mac_addrs_log.should have(1).entry
     end
 
     it "adjusts default IP and MAC threshold" do
-      ipex = SnsUtils::IpExtractor.new([file, "-i", "5", "-m", "4"])
-      ipex.options.mac_threshold.should eql(4)
-      ipex.options.ip_threshold.should eql(5)
+      addrex = SnsUtils::AddressExtractor.new([file, "-i", "5", "-m", "4"])
+      addrex.options.mac_threshold.should eql(4)
+      addrex.options.ip_threshold.should eql(5)
     end
 
     it "uses adjusted IP and MAC threshold" do
-      ipex = SnsUtils::IpExtractor.new([file, "-i", "5", "-m", "5"]).run
-      ipex.ip_addrs_log.should have(2).entries
-      ipex.mac_addrs_log.should have(2).entries
+      addrex = SnsUtils::AddressExtractor.new([file, "-i", "5", "-m", "5"]).run
+      addrex.ip_addrs_log.should have(2).entries
+      addrex.mac_addrs_log.should have(2).entries
 
-      ipex = SnsUtils::IpExtractor.new([file, "-i", "11", "-m", "11"]).run
-      ipex.ip_addrs_log.should be_empty
-      ipex.mac_addrs_log.should be_empty
+      addrex = SnsUtils::AddressExtractor.new([file, "-i", "11", "-m", "11"]).run
+      addrex.ip_addrs_log.should be_empty
+      addrex.mac_addrs_log.should be_empty
     end
   end
 
@@ -91,12 +91,12 @@ describe SnsUtils::IpExtractor do
     end
 
     it "configures custom output directory" do
-      options = SnsUtils::IpExtractor.new([file, "-d", tmp_dir]).options
+      options = SnsUtils::AddressExtractor.new([file, "-d", tmp_dir]).options
       options.output_dir.should eql(tmp_dir)
     end
 
     it "writes to custom output directory" do
-      SnsUtils::IpExtractor.new([file, "-d", tmp_dir]).run
+      SnsUtils::AddressExtractor.new([file, "-d", tmp_dir]).run
       File.exists?("#{tmp_dir}/#{::SnsUtils.ip_out_file}").should be_true
       File.exists?("#{tmp_dir}/#{::SnsUtils.mac_out_file}").should be_true
     end
@@ -109,10 +109,10 @@ describe "SnsUtils::IpExtractor" do
     let(:file) { fixture_path("edge_cases.log") }
 
     it "finds valid entries" do
-      ipex = SnsUtils::IpExtractor.new([file]).run
+      addrex = SnsUtils::AddressExtractor.new([file]).run
 
-      pp ipex.ip_addrs
-      ipex.ip_addrs.should have(10).entries
+      pp addrex.ip_addrs
+      addrex.ip_addrs.should have(7).entries
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sns_utils
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.6
 platform: ruby
 authors:
 - sahglie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-29 00:00:00.000000000 Z
+date: 2013-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ronn
@@ -17,7 +17,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -70,7 +70,7 @@ description: Find and extract IP and MAC addresses in FILE
 email:
 - sahglie@gmail.com
 executables:
-- ipex
+- addrex
 extensions: []
 extra_rdoc_files: []
 files:
@@ -79,16 +79,16 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/ipex
+- bin/addrex
 - lib/sns_utils.rb
-- lib/sns_utils/ip_extractor.rb
+- lib/sns_utils/address_extractor.rb
 - lib/sns_utils/ipv4.rb
 - lib/sns_utils/ipv6.rb
 - lib/sns_utils/mac.rb
 - lib/sns_utils/version.rb
-- man/ipex.1
-- man/ipex.1.ronn
-- man/ipex.1.txt
+- man/addrex.1
+- man/addrex.1.ronn
+- man/addrex.1.txt
 - sns_utils.gemspec
 - spec/fixtures/edge_cases.log
 - spec/fixtures/ipv4_ipv6_mac.log
@@ -96,7 +96,7 @@ files:
 - spec/fixtures/ipv6_simple.log
 - spec/fixtures/mac_simple.log
 - spec/fixtures/thresholds.log
-- spec/sns_utils/ip_extractor_spec.rb
+- spec/sns_utils/address_extractor_spec.rb
 - spec/spec_helper.rb
 homepage: ''
 licenses:
@@ -129,6 +129,5 @@ test_files:
 - spec/fixtures/ipv6_simple.log
 - spec/fixtures/mac_simple.log
 - spec/fixtures/thresholds.log
-- spec/sns_utils/ip_extractor_spec.rb
+- spec/sns_utils/address_extractor_spec.rb
 - spec/spec_helper.rb
-has_rdoc: