sns_utils 0.0.3 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -11
- data/Rakefile +2 -2
- data/bin/{ipex → addrex} +4 -4
- data/lib/sns_utils.rb +1 -1
- data/lib/sns_utils/{ip_extractor.rb → address_extractor.rb} +10 -6
- data/lib/sns_utils/version.rb +1 -1
- data/man/{ipex.1 → addrex.1} +4 -4
- data/man/{ipex.1.ronn → addrex.1.ronn} +3 -3
- data/man/{ipex.1.txt → addrex.1.txt} +5 -5
- data/sns_utils.gemspec +1 -1
- data/spec/fixtures/edge_cases.log +9 -8
- data/spec/sns_utils/{ip_extractor_spec.rb → address_extractor_spec.rb} +34 -34
- metadata +11 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 912c980791a92d1b4e63a686bcc44da8328f3d64
|
4
|
+
data.tar.gz: 2be2ca81465a58d402112520482760622ce07413
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 13a6635b712fc95e9f15b7b723aa13c53b8033afd31032a297976bc35e270c45167862f969fcb243e5406c4b898b355083ddea67691a7c18ad0b954b5098cfc8
|
7
|
+
data.tar.gz: 91238caf75f3a901c9db6fb79b6324f01233991937dd08571f128f6ea15e77f89b8a62ccfa7a63f1e29e372c71c778537878a9e26e239b2c5bbc3c5be1ba81b4
|
data/README.md
CHANGED
@@ -33,28 +33,28 @@ Or install it yourself as:
|
|
33
33
|
|
34
34
|
### Grep Baseline Perf
|
35
35
|
|
36
|
+
The following grep commands try to match only IPv6 and IPv4 addresses using the
|
37
|
+
same regex as the addrex utility
|
38
|
+
|
36
39
|
xlarge.log is 5GB with 50 million lines
|
37
40
|
|
38
|
-
time grep -c -E '(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' spec/fixtures/xlarge.log
|
39
|
-
=>
|
40
|
-
user 11m59.672s
|
41
|
-
sys 0m0.991s
|
41
|
+
time grep -c -E '\b(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?|(?:[0-9A-Fa-f]{1,4}:){6,6}\d+\.\d+\.\d+\.\d+|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}:)*\d+\.\d+\.\d+\.\d+)\b' spec/fixtures/xlarge.log
|
42
|
+
=> Exceeded 15 min
|
42
43
|
|
43
44
|
|
44
45
|
xlarge.log is 514MB with 5 million lines
|
45
46
|
|
46
|
-
time grep -c -E '(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' spec/fixtures/xlarge.log
|
47
|
-
=> real
|
48
|
-
user
|
47
|
+
time grep -c -E '\b(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?|(?:[0-9A-Fa-f]{1,4}:){6,6}\d+\.\d+\.\d+\.\d+|(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?::(?:[0-9A-Fa-f]{1,4}:)*\d+\.\d+\.\d+\.\d+)\b' spec/fixtures/xlarge.log
|
48
|
+
=> real 4m12.063s
|
49
|
+
user 4m10.956s
|
49
50
|
sys 0m0.108s
|
50
51
|
|
51
52
|
## ip_extract
|
52
53
|
|
53
54
|
xlarge.log is 514MB with 5 million lines
|
54
55
|
|
55
|
-
|
56
|
-
=> real
|
57
|
-
user
|
56
|
+
addrex spec/fixtures/xlarge.log
|
57
|
+
=> real 3m7.938s
|
58
|
+
user 3m6.356s
|
58
59
|
sys 0m0.578s
|
59
60
|
|
60
|
-
|
data/Rakefile
CHANGED
data/bin/{ipex → addrex}
RENAMED
@@ -6,9 +6,9 @@ Signal.trap("INT") { exit 1 }
|
|
6
6
|
require_relative "../lib/sns_utils"
|
7
7
|
|
8
8
|
begin
|
9
|
-
SnsUtils::
|
9
|
+
SnsUtils::AddressExtractor.new(ARGV).run
|
10
10
|
rescue Errno::ENOENT => err
|
11
|
-
abort "
|
12
|
-
rescue OptionParser::InvalidOption, OptionParser::InvalidArgument
|
13
|
-
abort "usage:
|
11
|
+
abort "addrex: #{err.message}"
|
12
|
+
rescue OptionParser::InvalidOption, OptionParser::InvalidArgument
|
13
|
+
abort "usage: addrex FILE [-m MAC_THRESHOLD] [-i IP_THRESHOLD] [-d OUTPUT_DIR]"
|
14
14
|
end
|
data/lib/sns_utils.rb
CHANGED
@@ -7,7 +7,7 @@ require 'ostruct'
|
|
7
7
|
require_relative 'sns_utils/ipv4'
|
8
8
|
require_relative 'sns_utils/ipv6'
|
9
9
|
require_relative 'sns_utils/mac'
|
10
|
-
require_relative "sns_utils/
|
10
|
+
require_relative "sns_utils/address_extractor"
|
11
11
|
require_relative "sns_utils/version"
|
12
12
|
|
13
13
|
|
@@ -1,12 +1,16 @@
|
|
1
1
|
module SnsUtils
|
2
|
-
class
|
2
|
+
class AddressExtractor
|
3
3
|
attr_accessor :file, :options
|
4
4
|
attr_accessor :ip_addrs, :ip_addrs_log, :mac_addrs, :mac_addrs_log
|
5
5
|
|
6
6
|
IP_REGEX = /
|
7
|
-
|
8
|
-
|
9
|
-
|
7
|
+
\b
|
8
|
+
(
|
9
|
+
#{::SnsUtils::IPv4::REGEX} |
|
10
|
+
#{::SnsUtils::IPv6::REGEX} |
|
11
|
+
#{::SnsUtils::MAC::REGEX}
|
12
|
+
)
|
13
|
+
\b
|
10
14
|
/xi
|
11
15
|
|
12
16
|
def initialize(argv)
|
@@ -26,8 +30,8 @@ module SnsUtils
|
|
26
30
|
|
27
31
|
def extract_addresses
|
28
32
|
File.open(file, 'r').each do |line|
|
29
|
-
line.scan(IP_REGEX).each do |
|
30
|
-
log_addr(
|
33
|
+
line.scan(IP_REGEX).each do |md|
|
34
|
+
log_addr(md[0].to_s.strip)
|
31
35
|
end
|
32
36
|
end
|
33
37
|
end
|
data/lib/sns_utils/version.rb
CHANGED
data/man/{ipex.1 → addrex.1}
RENAMED
@@ -1,13 +1,13 @@
|
|
1
1
|
.\" generated with Ronn/v0.7.3
|
2
2
|
.\" http://github.com/rtomayko/ronn/tree/0.7.3
|
3
3
|
.
|
4
|
-
.TH "
|
4
|
+
.TH "ADDREX" "1" "August 2013" "" ""
|
5
5
|
.
|
6
6
|
.SH "NAME"
|
7
|
-
\
|
7
|
+
\fBaddrex\fR \- IP/Mac Address Extractor
|
8
8
|
.
|
9
9
|
.SH "SYNOPSIS"
|
10
|
-
\
|
10
|
+
\fBaddrex FILE [\-m THRESHOLD] [\-i THRESHOLD] [\-d OUTPUT_DIR]\fR
|
11
11
|
.
|
12
12
|
.SH "DESCRIPTION"
|
13
13
|
Find and extract IP and MAC addresses in FILE\. Addresses with repeat occurrences greater than or equal to a given threshold are logged to ip_addrs\.txt and mac_addrs\.txt respectively\. By default, MAC addresses that occur at least 8 times are logged and IP addresses that occur at least 10 times are logged\. All matches are performed as case insensitive\.
|
@@ -53,4 +53,4 @@ MAC: Separated by \':\' aa:bb:cc:dd:ee:ff
|
|
53
53
|
.IP "" 0
|
54
54
|
.
|
55
55
|
.SH "NOTE ON LARGE FILES"
|
56
|
-
Depending on how large the file is, this utility can take a long time to complete: for a 512MB file with (5 million) entries it took just over
|
56
|
+
Depending on how large the file is, this utility can take a long time to complete: for a 512MB file with (5 million) entries it took just over 3 min \-\- YMMV depending on your hardware
|
@@ -1,9 +1,9 @@
|
|
1
|
-
|
1
|
+
addrex(1) -- IP/Mac Address Extractor
|
2
2
|
=======================================
|
3
3
|
|
4
4
|
## SYNOPSIS
|
5
5
|
|
6
|
-
`
|
6
|
+
`addrex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]`
|
7
7
|
|
8
8
|
## DESCRIPTION
|
9
9
|
|
@@ -38,6 +38,6 @@ Matches the following address formats:
|
|
38
38
|
|
39
39
|
## NOTE ON LARGE FILES
|
40
40
|
Depending on how large the file is, this utility can take a long time to
|
41
|
-
complete: for a 512MB file with (5 million) entries it took just over
|
41
|
+
complete: for a 512MB file with (5 million) entries it took just over 3 min
|
42
42
|
-- YMMV depending on your hardware
|
43
43
|
|
@@ -1,12 +1,12 @@
|
|
1
|
-
|
1
|
+
ADDREX(1) ADDREX(1)
|
2
2
|
|
3
3
|
|
4
4
|
|
5
5
|
NAME
|
6
|
-
|
6
|
+
addrex - IP/Mac Address Extractor
|
7
7
|
|
8
8
|
SYNOPSIS
|
9
|
-
|
9
|
+
addrex FILE [-m THRESHOLD] [-i THRESHOLD] [-d OUTPUT_DIR]
|
10
10
|
|
11
11
|
DESCRIPTION
|
12
12
|
Find and extract IP and MAC addresses in FILE. Addresses with repeat
|
@@ -46,8 +46,8 @@ FORMATS
|
|
46
46
|
NOTE ON LARGE FILES
|
47
47
|
Depending on how large the file is, this utility can take a long time
|
48
48
|
to complete: for a 512MB file with (5 million) entries it took just
|
49
|
-
over
|
49
|
+
over 3 min -- YMMV depending on your hardware
|
50
50
|
|
51
51
|
|
52
52
|
|
53
|
-
August 2013
|
53
|
+
August 2013 ADDREX(1)
|
data/sns_utils.gemspec
CHANGED
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
20
20
|
spec.require_paths = ["lib"]
|
21
21
|
|
22
|
-
spec.
|
22
|
+
spec.add_development_dependency "ronn"
|
23
23
|
spec.add_development_dependency "bundler", "~> 1.3"
|
24
24
|
spec.add_development_dependency "rspec", "~> 2.7"
|
25
25
|
spec.add_development_dependency "rake"
|
@@ -1,8 +1,9 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
6
|
-
#
|
7
|
-
#
|
8
|
-
#
|
1
|
+
123.12.12.125-16-2013 # will match
|
2
|
+
# will match 123.12.12.126-16-2013
|
3
|
+
# will not match 6-7-201123.12.12.127
|
4
|
+
# will not match 6-7-201123.12.12.128-16-2013
|
5
|
+
# will not match by truncating last '0' 192.168.1.500
|
6
|
+
# will not match by truncating last '5' 192.168.1.555:80
|
7
|
+
# will match, ipv4 with port number 192.168.1.51:8080
|
8
|
+
# will match, ipv8 with port number # valid 2001:0000:1234:0000:0000:C1C0:ABCD:0876:8080
|
9
|
+
# will match all, multi-line xyz:1 ip 123.12.12.129-16-2013 2013-1xxxy&& 2001:0000:1234:0000:0000:C1C0:ABCD:0877:8080 9%%%#+ 192.168.1.52:80
|
@@ -1,12 +1,12 @@
|
|
1
1
|
require_relative "../spec_helper"
|
2
2
|
|
3
|
-
describe SnsUtils::
|
3
|
+
describe SnsUtils::AddressExtractor do
|
4
4
|
context "extracting IPv4 ips" do
|
5
5
|
let(:file) { fixture_path("ipv4_simple.log") }
|
6
6
|
|
7
7
|
it "finds valid entries" do
|
8
|
-
|
9
|
-
|
8
|
+
addrex = SnsUtils::AddressExtractor.new([file]).run
|
9
|
+
addrex.ip_addrs.should have(5).entries
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
@@ -14,8 +14,8 @@ describe SnsUtils::IpExtractor do
|
|
14
14
|
let(:file) { fixture_path("ipv6_simple.log") }
|
15
15
|
|
16
16
|
it "finds valid entries" do
|
17
|
-
|
18
|
-
|
17
|
+
addrex = SnsUtils::AddressExtractor.new([file]).run
|
18
|
+
addrex.ip_addrs.should have(72).entries
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
@@ -23,8 +23,8 @@ describe SnsUtils::IpExtractor do
|
|
23
23
|
let(:file) { fixture_path("mac_simple.log") }
|
24
24
|
|
25
25
|
it "finds valid entries" do
|
26
|
-
|
27
|
-
|
26
|
+
addrex = SnsUtils::AddressExtractor.new([file]).run
|
27
|
+
addrex.mac_addrs.should have(2).entries
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
@@ -32,10 +32,10 @@ describe SnsUtils::IpExtractor do
|
|
32
32
|
let(:file) { fixture_path("ipv4_ipv6_mac.log") }
|
33
33
|
|
34
34
|
it "finds valid entries" do
|
35
|
-
|
36
|
-
pp
|
37
|
-
|
38
|
-
|
35
|
+
addrex = SnsUtils::AddressExtractor.new([file]).run
|
36
|
+
pp addrex.ip_addrs
|
37
|
+
addrex.ip_addrs.should have(5).entries
|
38
|
+
addrex.mac_addrs.should have(1).entry
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
@@ -43,41 +43,41 @@ describe SnsUtils::IpExtractor do
|
|
43
43
|
let(:file) { fixture_path("thresholds.log") }
|
44
44
|
|
45
45
|
it "raises error for invalid thresholds" do
|
46
|
-
expect { SnsUtils::
|
46
|
+
expect { SnsUtils::AddressExtractor.new([file, "-i", "x"]) }.to \
|
47
47
|
raise_error(OptionParser::InvalidArgument)
|
48
48
|
|
49
|
-
expect { SnsUtils::
|
49
|
+
expect { SnsUtils::AddressExtractor.new([file, "-m", "x"]) }.to \
|
50
50
|
raise_error(OptionParser::InvalidArgument)
|
51
51
|
end
|
52
52
|
|
53
53
|
it "has default IP and MAC threshold" do
|
54
|
-
|
55
|
-
|
56
|
-
|
54
|
+
addrex = SnsUtils::AddressExtractor.new([file])
|
55
|
+
addrex.options.mac_threshold.should eql(8)
|
56
|
+
addrex.options.ip_threshold.should eql(10)
|
57
57
|
end
|
58
58
|
|
59
59
|
it "uses default IP and MAC threshold" do
|
60
|
-
|
61
|
-
|
60
|
+
addrex = SnsUtils::AddressExtractor.new([file])
|
61
|
+
addrex.run
|
62
62
|
|
63
|
-
|
64
|
-
|
63
|
+
addrex.ip_addrs_log.should have(1).entry
|
64
|
+
addrex.mac_addrs_log.should have(1).entry
|
65
65
|
end
|
66
66
|
|
67
67
|
it "adjusts default IP and MAC threshold" do
|
68
|
-
|
69
|
-
|
70
|
-
|
68
|
+
addrex = SnsUtils::AddressExtractor.new([file, "-i", "5", "-m", "4"])
|
69
|
+
addrex.options.mac_threshold.should eql(4)
|
70
|
+
addrex.options.ip_threshold.should eql(5)
|
71
71
|
end
|
72
72
|
|
73
73
|
it "uses adjusted IP and MAC threshold" do
|
74
|
-
|
75
|
-
|
76
|
-
|
74
|
+
addrex = SnsUtils::AddressExtractor.new([file, "-i", "5", "-m", "5"]).run
|
75
|
+
addrex.ip_addrs_log.should have(2).entries
|
76
|
+
addrex.mac_addrs_log.should have(2).entries
|
77
77
|
|
78
|
-
|
79
|
-
|
80
|
-
|
78
|
+
addrex = SnsUtils::AddressExtractor.new([file, "-i", "11", "-m", "11"]).run
|
79
|
+
addrex.ip_addrs_log.should be_empty
|
80
|
+
addrex.mac_addrs_log.should be_empty
|
81
81
|
end
|
82
82
|
end
|
83
83
|
|
@@ -91,12 +91,12 @@ describe SnsUtils::IpExtractor do
|
|
91
91
|
end
|
92
92
|
|
93
93
|
it "configures custom output directory" do
|
94
|
-
options = SnsUtils::
|
94
|
+
options = SnsUtils::AddressExtractor.new([file, "-d", tmp_dir]).options
|
95
95
|
options.output_dir.should eql(tmp_dir)
|
96
96
|
end
|
97
97
|
|
98
98
|
it "writes to custom output directory" do
|
99
|
-
SnsUtils::
|
99
|
+
SnsUtils::AddressExtractor.new([file, "-d", tmp_dir]).run
|
100
100
|
File.exists?("#{tmp_dir}/#{::SnsUtils.ip_out_file}").should be_true
|
101
101
|
File.exists?("#{tmp_dir}/#{::SnsUtils.mac_out_file}").should be_true
|
102
102
|
end
|
@@ -109,10 +109,10 @@ describe "SnsUtils::IpExtractor" do
|
|
109
109
|
let(:file) { fixture_path("edge_cases.log") }
|
110
110
|
|
111
111
|
it "finds valid entries" do
|
112
|
-
|
112
|
+
addrex = SnsUtils::AddressExtractor.new([file]).run
|
113
113
|
|
114
|
-
pp
|
115
|
-
|
114
|
+
pp addrex.ip_addrs
|
115
|
+
addrex.ip_addrs.should have(7).entries
|
116
116
|
end
|
117
117
|
end
|
118
118
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sns_utils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sahglie
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-08-
|
11
|
+
date: 2013-08-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ronn
|
@@ -17,7 +17,7 @@ dependencies:
|
|
17
17
|
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
|
-
type: :
|
20
|
+
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
@@ -70,7 +70,7 @@ description: Find and extract IP and MAC addresses in FILE
|
|
70
70
|
email:
|
71
71
|
- sahglie@gmail.com
|
72
72
|
executables:
|
73
|
-
-
|
73
|
+
- addrex
|
74
74
|
extensions: []
|
75
75
|
extra_rdoc_files: []
|
76
76
|
files:
|
@@ -79,16 +79,16 @@ files:
|
|
79
79
|
- LICENSE.txt
|
80
80
|
- README.md
|
81
81
|
- Rakefile
|
82
|
-
- bin/
|
82
|
+
- bin/addrex
|
83
83
|
- lib/sns_utils.rb
|
84
|
-
- lib/sns_utils/
|
84
|
+
- lib/sns_utils/address_extractor.rb
|
85
85
|
- lib/sns_utils/ipv4.rb
|
86
86
|
- lib/sns_utils/ipv6.rb
|
87
87
|
- lib/sns_utils/mac.rb
|
88
88
|
- lib/sns_utils/version.rb
|
89
|
-
- man/
|
90
|
-
- man/
|
91
|
-
- man/
|
89
|
+
- man/addrex.1
|
90
|
+
- man/addrex.1.ronn
|
91
|
+
- man/addrex.1.txt
|
92
92
|
- sns_utils.gemspec
|
93
93
|
- spec/fixtures/edge_cases.log
|
94
94
|
- spec/fixtures/ipv4_ipv6_mac.log
|
@@ -96,7 +96,7 @@ files:
|
|
96
96
|
- spec/fixtures/ipv6_simple.log
|
97
97
|
- spec/fixtures/mac_simple.log
|
98
98
|
- spec/fixtures/thresholds.log
|
99
|
-
- spec/sns_utils/
|
99
|
+
- spec/sns_utils/address_extractor_spec.rb
|
100
100
|
- spec/spec_helper.rb
|
101
101
|
homepage: ''
|
102
102
|
licenses:
|
@@ -129,6 +129,5 @@ test_files:
|
|
129
129
|
- spec/fixtures/ipv6_simple.log
|
130
130
|
- spec/fixtures/mac_simple.log
|
131
131
|
- spec/fixtures/thresholds.log
|
132
|
-
- spec/sns_utils/
|
132
|
+
- spec/sns_utils/address_extractor_spec.rb
|
133
133
|
- spec/spec_helper.rb
|
134
|
-
has_rdoc:
|